Bausager/Flux
1
0
Fork 0
Code Issues 36 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Backpass on reg

Browse Source 
Made backpass on regulariaztion and made it easy to see in main.cpp
This commit is contained in:
Michelle
2026-05-16 21:34:34 +02:00
parent d2fe8aa65c
commit eb2374eaf5
5 changed files with 180 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/modules/neural_networks/layers/Dense_Layer.h
+49 -7
View File
@@ -12,11 +12,11 @@ namespace neural_networks{
template <typename T>
struct Dense_Layer{
T weight_regularizer_l1 = {1e-4};
T weight_regularizer_l2 = {1e-4};
T weight_regularizer_l1 = {0};
T weight_regularizer_l2 = {0};
T bias_regularizer_l1 = {1e-4};
T bias_regularizer_l2 = {1e-4};
T bias_regularizer_l1 = {0};
T bias_regularizer_l2 = {0};
utils::Matrix<T> _inputs;
utils::Matrix<T> weights;
@@ -37,8 +37,17 @@ namespace neural_networks{
Dense_Layer() = default;
// Constructor
Dense_Layer(const uint64_t n_inputs, const uint64_t n_neurons){
Dense_Layer(const uint64_t n_inputs, const uint64_t n_neurons,
const T weight_regularizer_l1=T{0},
const T weight_regularizer_l2=T{0},
const T bias_regularizer_l1=T{0},
const T bias_regularizer_l2=T{0}){
this->weight_regularizer_l1 = weight_regularizer_l1;
this->weight_regularizer_l2 = weight_regularizer_l2;
this->bias_regularizer_l1 = bias_regularizer_l1;
this->bias_regularizer_l2 = bias_regularizer_l2;
weights.random(n_inputs, n_neurons, -1, 1);
//weights = numerics::random_matrix(n_inputs, n_neurons, -1, 1);
biases.resize(n_neurons, T{0});
@@ -55,8 +64,41 @@ namespace neural_networks{
// Gradients on parameters
dweights = numerics::matmul(numerics::transpose(_inputs), dvalues);
dbiases = numerics::sum_rowwise(dvalues);
// Gradients on regularization
// L1 on weights
if(weight_regularizer_l1){
utils::Matrix<T> dL1(weights.rows(), weights.cols(), T{1});
for (uint64_t i = 0; i < weights.rows(); ++i){
for (uint64_t j = 0; j < weights.cols(); ++j){
if (weights(i,j) < 0){
dL1(i,j) = -1;
}
}
}
dweights = numerics::add(dweights, numerics::mul(dL1, weight_regularizer_l1));
}
// L2 on weights
if(weight_regularizer_l2){
dweights = numerics::add(dweights, numerics::mul(numerics::mul(weights, weight_regularizer_l2),T{2}));
}
// L1 on biases
if(bias_regularizer_l1){
utils::Vector<T> dL1(biases.size(), T{1});
for (uint64_t i = 0; i < dL1.size(); ++i){
if (biases[i] < 0){
dL1[i] = -1;
}
}
dbiases = numerics::add(dbiases, numerics::mul(dL1, bias_regularizer_l1));
}
// L2 on biases
if (bias_regularizer_l2){
dbiases = numerics::add(dbiases, numerics::mul(numerics::mul(biases, bias_regularizer_l2),T{2}));
}
//Gradient on values
dinputs = numerics::matmul(dvalues, numerics::transpose(weights));
}