Bausager/Flux
1
0
Fork 0
Code Issues 36 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Regulaization
Sync public mirror / sync (push) Failing after 27s
Details

Browse Source 
Started on regulaization in  Loss.h. I need to refactor the matsum.h since I need a total sum over the matrix. Also matmul needs a elementwise matmul function, which is the next this in the ragulaization
This commit is contained in:
Michelle
2026-01-03 22:10:50 +01:00
parent 32ba0518fa
commit 48f329feef
17 changed files with 881 additions and 510 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/modules/neural_networks/loss/Loss.h
+48 -13
View File
@@ -5,30 +5,65 @@
#include "./utils/vector.h"
#include "./utils/matrix.h"
#include "./numerics/vecmean.h"
#include "numerics/vecmean.h"
#include "numerics/matabs.h"
#include "numerics/matmean.h"
namespace neural_networks{
template <typename Td, typename Ti>
struct Loss{
utils::Vector<Td> sample_losses;
utils::Matrix<Td> dinputs;
Td data_loss;
utils::Vector<Td> sample_losses;
utils::Matrix<Td> dinputs;
Td data_loss;
Td regularization_losss;
virtual utils::Vector<Td> forward(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y) = 0;
virtual void backward(const utils::Matrix<Td>& dvalues, const utils::Matrix<Ti>& y) = 0;
virtual utils::Vector<Td> forward(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y) = 0;
virtual void backward(const utils::Matrix<Td>& dvalues, const utils::Matrix<Ti>& y) = 0;
Td calculate(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y){
// Calculate sample losses
sample_losses = forward(output, y);
Td calculate(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y){
// Calculate sample losses
sample_losses = forward(output, y);
// Calculate mean loss
data_loss = numerics::vecmean(sample_losses);
return data_loss;
// Calculate mean loss
data_loss = numerics::vecmean(sample_losses);
return data_loss;
}
template <typename Layer>
Td regularization_loss(const Layer& layer){
// 0 by default
regularization_losss = 0;
// L1 regularization - weights
// calculate only when factor greater than 0
if (layer.weight_regularizer_l1){
regularization_losss += layer.weight_regularizer_l1 * numerics::matsum_coeff(numerics::matabs(layer.weights));
}
// L2 regularization - weights
if (layer.weight_regularizer_l2){
regularization_losss += layer.weight_regularizer_l2 * numerics::matsum_coeff(numerics::matmul(layer.weights,layer.weights)); // elementwise!
}
// L1 regularization - biases
// calculate only when factor greater than 0
if (layer.bias_regularizer_l1){
regularization_losss += layer.bias_regularizer_l1 * layer.biases.abs().sum();
}
// L2 regularization - biases
if (layer.bias_regularizer_l2){
regularization_losss += layer.bias_regularizer_l2 * layer.biases.multiply(layer.biases).sum();
}
return regularization_losss;
}
};
} // end namespace neural_networks