Bausager/Flux
1
0
Fork 0
Code Issues 36 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Binomial_CrossEnthophy

Browse Source 
fixed rowwise/colswise mean/sum and implemented binomial_corssentrhopy. Next up is regression.
This commit is contained in:
Michelle
2026-05-22 10:11:43 +02:00
parent eb0a49591e
commit cb65174cf4
21 changed files with 894 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/modules/neural_networks/activation_functions/Activation_Sigmoid.h
+46
View File
@@ -0,0 +1,46 @@
#pragma once
#include "core/omp_config.h"
#include "utils/vector.h"
#include "utils/matrix.h"
#include "numerics/neg.h"
#include "numerics/exp.h"
#include "numerics/add.h"
#include "numerics/div.h"
#include "numerics/sub.h"
#include "numerics/mul.h"
namespace neural_networks{
template <typename T>
struct Activation_Sigmoid{
utils::Matrix<T> _inputs;
utils::Matrix<T> outputs;
utils::Matrix<T> dinputs;
void forward(const utils::Matrix<T>& inputs){
_inputs = inputs;
outputs = numerics::neg(inputs);
outputs = numerics::exp(outputs);
outputs = numerics::add(outputs, T{1});
outputs = numerics::div(T{1}, outputs);
}
void backward(const utils::Matrix<T>& dvalues){
dinputs = numerics::sub(T{1}, outputs);
dinputs = numerics::mul(dvalues, dinputs);
dinputs = numerics::mul(dinputs, outputs);
}
};
} // end namespace neural_networks
include/modules/neural_networks/activation_functions/Activation_Softmax.h
+1 -1
View File
@@ -29,7 +29,7 @@ namespace neural_networks{
utils::Matrix<T> exp_values = numerics::exp(numerics::sub_colwise(inputs, numerics::max_rowwise(inputs)));
// Normalize them for each sample
utils::Matrix<T> probabilities = numerics::div_colwise(exp_values, numerics::sum_colwise(exp_values));
utils::Matrix<T> probabilities = numerics::div_colwise(exp_values, numerics::sum_rowwise(exp_values));
outputs = probabilities;
}
include/modules/neural_networks/layers/Dense_Layer.h
+1 -1
View File
@@ -63,7 +63,7 @@ namespace neural_networks{
void backward(const utils::Matrix<T>& dvalues){
// Gradients on parameters
dweights = numerics::matmul(numerics::transpose(_inputs), dvalues);
dbiases = numerics::sum_rowwise(dvalues);
dbiases = numerics::sum_colwise(dvalues);
// Gradients on regularization
include/modules/neural_networks/loss/Loss_BinaryCrossentropy.h
+81
View File
@@ -0,0 +1,81 @@
#pragma once
#include "core/omp_config.h"
#include "utils/vector.h"
#include "utils/matrix.h"
#include "utils/matcast.h"
#include "numerics/clip.h"
#include "numerics/log.h"
#include "numerics/sub.h"
#include "Loss.h"
namespace neural_networks{
template <typename Td, typename Ti>
struct Loss_BinaryCrossentropy : Loss<Td, Ti> {
utils::Matrix<Td> dinputs;
utils::Matrix<Td> y_true;
utils::Vector<Td> forward(const utils::Matrix<Td>& y_pred, const utils::Matrix<Ti>& y_true) override{
this->y_true = utils::matcast<Td, Ti>(y_true);
// Clip daa to prevent division by 0
// Clip both sides not to drag mean towards any value
utils::Matrix<Td> y_pred_clipped = numerics::clip(y_pred, Td{1e-7}, Td{1.0} - Td{1e-7});
// Calculate sample-wise loss
utils::Matrix<Td> sample_losses_temp = numerics::log(numerics::sub(Td{1}, y_pred_clipped));
sample_losses_temp = numerics::mul(sample_losses_temp, numerics::sub(Td{1}, this->y_true));
sample_losses_temp = numerics::add(sample_losses_temp, numerics::mul(this->y_true, numerics::log(y_pred_clipped)));
sample_losses_temp = numerics::neg(sample_losses_temp);
utils::Vector<Td> sample_losses = numerics::mean_rowwise(sample_losses_temp);
// Return losses
return sample_losses;
}
void backward(const utils::Matrix<Td>& dvalues, const utils::Matrix<Ti>& y_true) override{
/*std::cout << "BCE backward y_true: "
<< y_true.rows() << " x " << y_true.cols()
<< std::endl;*/
// Number of samples
const Td samples = static_cast<Td> (this->y_true.rows());
// Number of outputs in every sample
const Td outputs = static_cast<Td> (dvalues.cols());
// Clip data to prevent division by 0
// Clip both sides to not drag mean towards any value
utils::Matrix<Td> clipped_dvalues = numerics::clip(dvalues, Td{1e-7}, Td{1.0} - Td{1e-7});
// Calculate gradient
dinputs = numerics::div(numerics::neg(numerics::sub(numerics::div(this->y_true, clipped_dvalues), numerics::div(numerics::sub(Td{1}, this->y_true), numerics::sub(Td{1}, clipped_dvalues)))), outputs);
// Normalize gradients
dinputs = numerics::div(dinputs, samples);
/*
std::cout << "BCE backward dinputs: "
<< dinputs.rows() << " x " << dinputs.cols()
<< std::endl;*/
}
};
} // end namespace neural_networks
include/modules/neural_networks/neural_networks.h
+2
View File
@@ -12,10 +12,12 @@
#include "activation_functions/Activation_ReLU.h"
#include "activation_functions/Activation_Softmax.h"
#include "activation_functions/Activation_Softmax_Loss_CategoricalCrossentropy.h"
#include "activation_functions/Activation_Sigmoid.h"
#include "loss/Loss.h" // Base
#include "loss/Loss_CategoricalCrossentrophy.h"
#include "loss/Loss_BinaryCrossentropy.h"
#include "optimizers/Optimizer_SGD.h"
include/numerics/binary_threshold.h
+51
View File
@@ -0,0 +1,51 @@
#pragma once
#include "core/omp_config.h"
#include "detail/binary_threshold_serial.h"
namespace numerics{
// ---------------- Elementwise ----------------
template <typename T>
inline void inplace_greater_than(T& a, const T c) {
detail::inplace_greater_than_serial(a, c);
}
template <typename T>
inline T greater_than(const T a, const T c) {
T out = a;
inplace_greater_than(out, c);
return out;
}
template <typename T>
inline void inplace_greater_than(utils::Vector<T>& v, const T c) {
detail::inplace_greater_than_serial(v, c);
}
template <typename T>
inline utils::Vector<T> greater_than(const utils::Vector<T>& v, const T c) {
utils::Vector<T> out = v;
inplace_greater_than(out, c);
return out;
}
template <typename T>
inline void inplace_greater_than(utils::Matrix<T>& A, const T c) {
detail::inplace_greater_than_serial(A, c);
}
template <typename T>
inline utils::Matrix<T> greater_than(const utils::Matrix<T>& A, const T c) {
utils::Matrix<T> out = A;
inplace_greater_than(out, c);
return out;
}
}
include/numerics/detail/binary_threshold_serial.h
+44
View File
@@ -0,0 +1,44 @@
#pragma once
#include <cstdint> //uint64_t
#include <cmath> // std::abs
#include "utils/vector.h"
#include "utils/matrix.h"
namespace numerics::detail{
// ---------------- Elemenwise ----------------
template <typename T>
void inplace_greater_than_serial(T& a, const T c) {
if (a > c){
a = T{1};
}
else{
a = T{0};
}
}
template <typename T>
void inplace_greater_than_serial(utils::Vector<T>& v, const T c) {
for (uint64_t i = 0; i < v.size(); ++i){
inplace_greater_than_serial(v[i], c);
}
}
template <typename T>
void inplace_greater_than_serial(utils::Matrix<T>& A, const T c) {
const uint64_t rows = A.rows();
const uint64_t cols = A.cols();
for (uint64_t i = 0; i < rows; ++i){
for (uint64_t j = 0; j < cols; ++j){
inplace_greater_than_serial(A(i,j), c);
}
}
}
} // namespace numerics
include/numerics/detail/div_serial.h
+19
View File
@@ -20,12 +20,31 @@ namespace numerics::detail{
}
}
template <typename T>
void inplace_div_scalar_serial(const T c, utils::Matrix<T>& A) {
const uint64_t rows = A.rows();
const uint64_t cols = A.cols();
for (uint64_t i = 0; i < rows; ++i){
for (uint64_t j = 0; j < cols; ++j){
A(i,j) = c / A(i,j);
}
}
}
template <typename T>
void inplace_div_scalar_serial(utils::Vector<T>& v, const T c) {
for (uint64_t i = 0; i < v.size(); ++i){
v[i] /= c;
}
}
template <typename T>
void inplace_div_scalar_serial(const T c, utils::Vector<T>& v) {
for (uint64_t i = 0; i < v.size(); ++i){
v[i] = c / v[i];
}
}
// ---------------- Elemenwise ----------------
template <typename T>
void inplace_div_elementwise_serial(utils::Matrix<T>& A, const utils::Matrix<T>& B) {
include/numerics/detail/sub_serial.h
+21
View File
@@ -20,12 +20,33 @@ namespace numerics::detail{
}
}
template <typename T>
void inplace_sub_scalar_serial(const T c, utils::Matrix<T>& A) {
const uint64_t rows = A.rows();
const uint64_t cols = A.cols();
for (uint64_t i = 0; i < rows; ++i){
for (uint64_t j = 0; j < cols; ++j){
A(i,j) = c - A(i,j);
}
}
}
template <typename T>
void inplace_sub_scalar_serial(utils::Vector<T>& v, const T c) {
for (uint64_t i = 0; i < v.size(); ++i){
v[i] -= c;
}
}
template <typename T>
void inplace_sub_scalar_serial(const T c, utils::Vector<T>& v) {
for (uint64_t i = 0; i < v.size(); ++i){
v[i] = c - v[i];
}
}
// ---------------- Elemenwise ----------------
template <typename T>
void inplace_sub_elementwise_serial(utils::Matrix<T>& A, const utils::Matrix<T>& B) {
include/numerics/detail/sum_serial.h
+4 -4
View File
@@ -38,10 +38,10 @@ namespace numerics::detail{
utils::Vector<T> sum_rowwise_serial(const utils::Matrix<T>& A) {
const uint64_t rows = A.rows();
const uint64_t cols = A.cols();
utils::Vector<T> sum(cols, T{0});
utils::Vector<T> sum(rows, T{0});
for (uint64_t i = 0; i < rows; ++i){
for (uint64_t j = 0; j < cols; ++j){
sum[j] += A(i,j);
sum[i] += A(i,j);
}
}
return sum;
@@ -51,10 +51,10 @@ namespace numerics::detail{
utils::Vector<T> sum_colwise_serial(const utils::Matrix<T>& A) {
const uint64_t rows = A.rows();
const uint64_t cols = A.cols();
utils::Vector<T> sum(rows, T{0});
utils::Vector<T> sum(cols, T{0});
for (uint64_t i = 0; i < rows; ++i){
for (uint64_t j = 0; j < cols; ++j){
sum[i] += A(i,j);
sum[j] += A(i,j);
}
}
return sum;
include/numerics/div.h
+25 -2
View File
@@ -11,7 +11,6 @@ namespace numerics{
inline void inplace_div(utils::Matrix<T>& A, const T b) {
detail::inplace_div_scalar_serial(A,b);
}
template <typename T>
inline utils::Matrix<T> div(const utils::Matrix<T>& A, const T b) {
utils::Matrix<T> out = A;
@@ -19,17 +18,41 @@ namespace numerics{
return out;
}
template <typename T>
inline void inplace_div(const T b, utils::Matrix<T>& A) {
detail::inplace_div_scalar_serial(b, A);
}
template <typename T>
inline utils::Matrix<T> div(const T b, const utils::Matrix<T>& A) {
utils::Matrix<T> out = A;
inplace_div(b, out);
return out;
}
template <typename T>
inline void inplace_div(utils::Vector<T>& v, const T b) {
detail::inplace_div_scalar_serial(v,b);
}
template <typename T>
inline utils::Vector<T> div(const utils::Vector<T>& v, const T b) {
utils::Vector<T> out = v;
inplace_div(out, b);
return out;
}
template <typename T>
inline void inplace_div(const T b, utils::Vector<T>& v) {
detail::inplace_div_scalar_serial(b, v);
}
template <typename T>
inline utils::Vector<T> div(const T b, const utils::Vector<T>& v) {
utils::Vector<T> out = v;
inplace_div(b, out);
return out;
}
// ---------------- Elementwise ----------------
template <typename T>
inline void inplace_div(utils::Matrix<T>& A, const utils::Matrix<T>& B) {
include/numerics/numerics.h
+1
View File
@@ -4,6 +4,7 @@
#include "./numerics/add.h"
#include "./numerics/argmax.h"
#include "./numerics/argmin.h"
#include "./numerics/binary_threshold.h"
#include "./numerics/clip.h"
#include "./numerics/div.h"
#include "./numerics/dot.h"
include/numerics/sub.h
+24
View File
@@ -11,6 +11,11 @@ namespace numerics{
inline void inplace_sub(utils::Matrix<T>& A, const T b) {
detail::inplace_sub_scalar_serial(A,b);
}
template <typename T>
inline void inplace_sub(const T b, utils::Matrix<T>& A) {
detail::inplace_sub_scalar_serial(b,A);
}
template <typename T>
inline utils::Matrix<T> sub(const utils::Matrix<T>& A, const T b) {
@@ -18,11 +23,22 @@ namespace numerics{
inplace_sub(out, b);
return out;
}
template <typename T>
inline utils::Matrix<T> sub(const T b, const utils::Matrix<T>& A) {
utils::Matrix<T> out = A;
inplace_sub(b, out);
return out;
}
template <typename T>
inline void inplace_sub(utils::Vector<T>& v, const T b) {
detail::inplace_sub_scalar_serial(v,b);
}
template <typename T>
inline void inplace_sub(const T b, utils::Vector<T>& v) {
detail::inplace_sub_scalar_serial(b,v);
}
template <typename T>
inline utils::Vector<T> sub(const utils::Vector<T>& v, const T b) {
@@ -30,6 +46,14 @@ namespace numerics{
inplace_sub(out, b);
return out;
}
template <typename T>
inline utils::Vector<T> sub(const T b, const utils::Vector<T>& v) {
utils::Vector<T> out = v;
inplace_sub(b, out);
return out;
}
// ---------------- Elementwise ----------------
template <typename T>
inline void inplace_sub(utils::Matrix<T>& A, const utils::Matrix<T>& B) {
include/utils/matcast.h
+3
View File
@@ -31,5 +31,8 @@ namespace utils{
return B;
}
// utils::matcast<float, uint64_t> (A);
} // end namespace utils