From 9a69d64d79b0b54540ba94a3c7f237b258eaee36 Mon Sep 17 00:00:00 2001 From: Gitea CI Date: Thu, 9 Oct 2025 08:44:15 +0000 Subject: [PATCH] Sync public subset from Flux --- .../activation_functions/Activation_ReLU.h | 19 ++++++- .../neural_networks/layers/Dense_Layer.h | 19 ++++++- include/numerics/matrandom.h | 55 +++++++++++++++++++ include/numerics/numerics.h | 2 + include/numerics/vecrandom.h | 48 ++++++++++++++++ include/utils/matrix.h | 1 - 6 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 include/numerics/matrandom.h create mode 100644 include/numerics/vecrandom.h diff --git a/include/modules/neural_networks/activation_functions/Activation_ReLU.h b/include/modules/neural_networks/activation_functions/Activation_ReLU.h index add7906..2427503 100644 --- a/include/modules/neural_networks/activation_functions/Activation_ReLU.h +++ b/include/modules/neural_networks/activation_functions/Activation_ReLU.h @@ -11,12 +11,29 @@ namespace neural_networks{ template struct Activation_ReLU{ - //utils::Matrix inputs; + utils::Matrix _inputs; utils::Matrix outputs; + utils::Matrix dinputs; + void forward(const utils::Matrix& inputs){ + _inputs = inputs; outputs = numerics::matclip_low(inputs, T{0}); } + void backward(const utils::Matrix& dvalues){ + // Since we need to modify the original variable, + // let's make a copy of the values first + dinputs = dvalues; + + // Zero gradients where input values were negative + for (uint64_t i = 0; i < dinputs.rows(); ++i){ + for (uint64_t j = 0; j < dinputs.cols(); ++j){ + if (_inputs(i,j) <= T{0}){ + dinputs(i,j) = T{0}; + } + } + } + } }; diff --git a/include/modules/neural_networks/layers/Dense_Layer.h b/include/modules/neural_networks/layers/Dense_Layer.h index 4acc229..ecf9faa 100644 --- a/include/modules/neural_networks/layers/Dense_Layer.h +++ b/include/modules/neural_networks/layers/Dense_Layer.h @@ -12,11 +12,15 @@ namespace neural_networks{ template struct Dense_Layer{ - //utils::Matrix _inputs; + utils::Matrix _inputs; utils::Matrix weights; utils::Vector biases; utils::Matrix outputs; + utils::Matrix dweights; + utils::Vector dbiases; + utils::Matrix dinputs; + // Default Constructor Dense_Layer() = default; @@ -29,9 +33,20 @@ namespace neural_networks{ } - void forward(utils::Matrix& inputs){ + void forward(const utils::Matrix& inputs){ + _inputs = inputs; outputs = numerics::matadd(numerics::matmul_auto(inputs, weights), biases, "row"); } + + void backward(const utils::Matrix& dvalues){ + // Gradients on parameters + dweights = numerics::matmul(numerics::transpose(_inputs), dvalues); + dbiases = numerics::matsum(dvalues, "row"); + //Gradient on values + dinputs = numerics::matmul(dvalues, numerics::transpose(weights)); + + } + }; diff --git a/include/numerics/matrandom.h b/include/numerics/matrandom.h new file mode 100644 index 0000000..b64093d --- /dev/null +++ b/include/numerics/matrandom.h @@ -0,0 +1,55 @@ +#ifndef _matrandom_n_ +#define _matrandom_n_ + +#include "./utils/vector.h" +#include "./utils/matrix.h" +#include "./core/omp_config.h" + +namespace numerics{ + + template + void inplace_matrandom_add(utils::Matrix& A, const T lower, const T higher) { + + const uint64_t rows = A.rows(); + const uint64_t cols = A.cols(); + + utils::Matrix B; + B.random(rows,cols, lower, higher); + + numerics::inplace_matadd_mat(A, B); + } + + template + void inplace_matrandom_mul(utils::Matrix& A, const T lower, const T higher) { + + const uint64_t rows = A.rows(); + const uint64_t cols = A.cols(); + + utils::Matrix B; + B.random(rows,cols, lower, higher); + + for (uint64_t i = 0; i < rows; ++i){ + for (uint64_t j = 0; j < cols; ++j){ + A(i,j) *= B(i,j); + } + } + } + + + template + utils::Matrix matrandom_add(const utils::Matrix& A, const T lower, const T higher) { + + const uint64_t rows = A.rows(); + const uint64_t cols = A.cols(); + + utils::Matrix B = A; + + numerics::inplace_matadd_mat(B, lower, higher); + + return B; + } + + +} // namespace numerics + +#endif // _matrandom_n_ \ No newline at end of file diff --git a/include/numerics/numerics.h b/include/numerics/numerics.h index 27f2a10..c3135e4 100644 --- a/include/numerics/numerics.h +++ b/include/numerics/numerics.h @@ -8,6 +8,7 @@ #include "./numerics/vecmax.h" #include "./numerics/veclog.h" #include "./numerics/vecargmax.h" +#include "./numerics/vecrandom.h" #include "./numerics/initializers/eye.h" #include "./numerics/matequal.h" #include "./numerics/transpose.h" @@ -18,6 +19,7 @@ #include "./numerics/matdiv.h" #include "./numerics/matvec.h" #include "./numerics/matadd.h" +#include "./numerics/matrandom.h" #include "./numerics/matsubtract.h" #include "./numerics/matsum.h" #include "./numerics/matclip.h" diff --git a/include/numerics/vecrandom.h b/include/numerics/vecrandom.h new file mode 100644 index 0000000..75e60b5 --- /dev/null +++ b/include/numerics/vecrandom.h @@ -0,0 +1,48 @@ +#ifndef _vecrandom_n_ +#define _vecrandom_n_ + +#include "./utils/vector.h" +#include "./utils/matrix.h" +#include "./core/omp_config.h" + +namespace numerics{ + + template + void inplace_vecrandom_add(utils::Vector& a, const T lower, const T higher) { + + const uint64_t N = a.size(); + + utils::Vector b; + b.random(N, lower, higher); + + a.inplace_add(b); + } + + template + void inplace_vecrandom_mul(utils::Vector& a, const T lower, const T higher) { + + const uint64_t N = a.size(); + + utils::Vector b; + b.random(N, lower, higher); + + a.inplace_multiply(b); + + } + + + template + utils::Vector vecrandom_add(const utils::Vector& a, const T lower, const T higher) { + + const uint64_t N = a.size(); + + utils::Vector b; + inplace_vecrandom_add(b, lower, higher); + + return b; + } + + +} // namespace numerics + +#endif // _vecrandom_n_ \ No newline at end of file diff --git a/include/utils/matrix.h b/include/utils/matrix.h index 8929e72..7a41f76 100644 --- a/include/utils/matrix.h +++ b/include/utils/matrix.h @@ -23,7 +23,6 @@ namespace utils{ class Matrix{ public: Matrix() : rows_(0), cols_(0), data_() {} // Default constructor -#include // Constructor to initialize matrix with rows × cols and a fill value Matrix(uint64_t rows, uint64_t cols, const T& value = T()) : rows_(rows), cols_(cols), data_(rows * cols, value) {}