Bausager/Flux
1
0
Fork 0
Code Issues 36 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Regulaization
Sync public mirror / sync (push) Failing after 27s
Details

Browse Source 
Started on regulaization in  Loss.h. I need to refactor the matsum.h since I need a total sum over the matrix. Also matmul needs a elementwise matmul function, which is the next this in the ragulaization
This commit is contained in:
Michelle
2026-01-03 22:10:50 +01:00
parent 32ba0518fa
commit 48f329feef
17 changed files with 881 additions and 510 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/modules/neural_networks/layers/Dense_Layer.h
+39 -33
View File
@@ -12,45 +12,51 @@ namespace neural_networks{
template <typename T>
struct Dense_Layer{
utils::Matrix<T> _inputs;
utils::Matrix<T> weights;
utils::Vector<T> biases;
utils::Matrix<T> outputs;
T weight_regularizer_l1 = {1e-4};
T weight_regularizer_l2 = {1e-4};
utils::Matrix<T> dweights;
utils::Vector<T> dbiases;
utils::Matrix<T> dinputs;
T bias_regularizer_l1 = {1e-4};
T bias_regularizer_l2 = {1e-4};
// Variables for optimizers
utils::Matrix<T> weight_momentums;
utils::Vector<T> bias_momentums;
utils::Matrix<T> weight_cache;
utils::Vector<T> bias_cache;
// Default Constructor
Dense_Layer() = default;
utils::Matrix<T> _inputs;
utils::Matrix<T> weights;
utils::Vector<T> biases;
utils::Matrix<T> outputs;
// Constructor
Dense_Layer(const uint64_t n_inputs, const uint64_t n_neurons){
weights.random(n_inputs, n_neurons, -1, 1);
biases.resize(n_neurons, T{0});
}
utils::Matrix<T> dweights;
utils::Vector<T> dbiases;
utils::Matrix<T> dinputs;
void forward(const utils::Matrix<T>& inputs){
_inputs = inputs;
outputs = numerics::matadd(numerics::matmul_auto(inputs, weights), biases, "row");
}
// Variables for optimizers
utils::Matrix<T> weight_momentums;
utils::Vector<T> bias_momentums;
utils::Matrix<T> weight_cache;
utils::Vector<T> bias_cache;
// Default Constructor
Dense_Layer() = default;
void backward(const utils::Matrix<T>& dvalues){
// Gradients on parameters
dweights = numerics::matmul(numerics::transpose(_inputs), dvalues);
dbiases = numerics::matsum(dvalues, "row");
//Gradient on values
dinputs = numerics::matmul(dvalues, numerics::transpose(weights));
// Constructor
Dense_Layer(const uint64_t n_inputs, const uint64_t n_neurons){
weights.random(n_inputs, n_neurons, -1, 1);
biases.resize(n_neurons, T{0});
}
}
void forward(const utils::Matrix<T>& inputs){
_inputs = inputs;
outputs = numerics::matadd(numerics::matmul_auto(inputs, weights), biases, "row");
}
void backward(const utils::Matrix<T>& dvalues){
// Gradients on parameters
dweights = numerics::matmul(numerics::transpose(_inputs), dvalues);
dbiases = numerics::matsum(dvalues, "row");
//Gradient on values
dinputs = numerics::matmul(dvalues, numerics::transpose(weights));
}
};
include/modules/neural_networks/loss/Loss.h
+48 -13
View File
@@ -5,30 +5,65 @@
#include "./utils/vector.h"
#include "./utils/matrix.h"
#include "./numerics/vecmean.h"
#include "numerics/vecmean.h"
#include "numerics/matabs.h"
#include "numerics/matmean.h"
namespace neural_networks{
template <typename Td, typename Ti>
struct Loss{
utils::Vector<Td> sample_losses;
utils::Matrix<Td> dinputs;
Td data_loss;
utils::Vector<Td> sample_losses;
utils::Matrix<Td> dinputs;
Td data_loss;
Td regularization_losss;
virtual utils::Vector<Td> forward(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y) = 0;
virtual void backward(const utils::Matrix<Td>& dvalues, const utils::Matrix<Ti>& y) = 0;
virtual utils::Vector<Td> forward(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y) = 0;
virtual void backward(const utils::Matrix<Td>& dvalues, const utils::Matrix<Ti>& y) = 0;
Td calculate(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y){
// Calculate sample losses
sample_losses = forward(output, y);
Td calculate(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y){
// Calculate sample losses
sample_losses = forward(output, y);
// Calculate mean loss
data_loss = numerics::vecmean(sample_losses);
return data_loss;
// Calculate mean loss
data_loss = numerics::vecmean(sample_losses);
return data_loss;
}
template <typename Layer>
Td regularization_loss(const Layer& layer){
// 0 by default
regularization_losss = 0;
// L1 regularization - weights
// calculate only when factor greater than 0
if (layer.weight_regularizer_l1){
regularization_losss += layer.weight_regularizer_l1 * numerics::matsum_coeff(numerics::matabs(layer.weights));
}
// L2 regularization - weights
if (layer.weight_regularizer_l2){
regularization_losss += layer.weight_regularizer_l2 * numerics::matsum_coeff(numerics::matmul(layer.weights,layer.weights)); // elementwise!
}
// L1 regularization - biases
// calculate only when factor greater than 0
if (layer.bias_regularizer_l1){
regularization_losss += layer.bias_regularizer_l1 * layer.biases.abs().sum();
}
// L2 regularization - biases
if (layer.bias_regularizer_l2){
regularization_losss += layer.bias_regularizer_l2 * layer.biases.multiply(layer.biases).sum();
}
return regularization_losss;
}
};
} // end namespace neural_networks
include/modules/neural_networks/neural_networks.h
+2
View File
@@ -19,3 +19,5 @@
#include "optimizers/Optimizer_SGD.h"
#include "optimizers/Optimizer_Adagrad.h"
#include "optimizers/Optimizer_RMSprop.h"
#include "optimizers/Optimizer_Adam.h"
include/modules/neural_networks/optimizers/Optimizer_Adam.h
+134
View File
@@ -0,0 +1,134 @@
#pragma once
#include "./core/omp_config.h"
#include "./utils/vector.h"
#include "./utils/matrix.h"
#include "./numerics/matmul.h"
#include <math.h>
namespace neural_networks{
template <typename T>
struct Optimizer_Adam{
T learning_rate = T{1};
T current_learning_rate = learning_rate;
T decay = T{0};
T epsilon = T{1e-7};
T beta_1 = T{0.9};
T beta_2 = T{0.999};
uint64_t iterations = 0;
utils::Matrix<T> weight_momentums_corrected;
utils::Vector<T> bias_momentums_corrected;
utils::Matrix<T> weight_cache_corrected;
utils::Vector<T> bias_cache_corrected;
// Default Constructor
Optimizer_Adam() = default;
// Constructor
explicit Optimizer_Adam(const T lr, const T lr_decay, const T epsilons, const T beta1, const T beta2):
learning_rate(lr),
current_learning_rate{lr},
decay(lr_decay),
epsilon(epsilons),
beta_1(beta1),
beta_2(beta2) {}
void pre_update_params(){
if(decay){
current_learning_rate = learning_rate * (T{1}/(T{1}+(decay*iterations)));
//std::cout << current_learning_rate << std::endl;
}
}
template <typename Layer>
void update_params(Layer& layer){
// if layer does not contain cache arrays, create them filled with zeros.
if ((layer.weight_cache.rows() != layer.weights.rows()) || (layer.weight_cache.cols() != layer.weights.cols())){
layer.weight_momentums.resize(layer.weights.rows(), layer.weights.cols(), T{0});
layer.weight_cache.resize(layer.weights.rows(), layer.weights.cols(), T{0});
}
if (layer.bias_cache.size() != layer.biases.size()){
layer.bias_momentums.resize(layer.biases.size(), T{0});
layer.bias_cache.resize(layer.biases.size(), T{0});
}
// Update momentum with current gradients
for (uint64_t i = 0; i < layer.weights.rows(); ++i){
for (uint64_t j = 0; j < layer.weights.cols(); ++j){
layer.weight_momentums(i,j) = (beta_1 * layer.weight_momentums(i,j)) + ((T{1} - beta_1) * layer.dweights(i,j));
}
}
for (uint64_t i = 0; i < layer.biases.size(); ++i){
layer.bias_momentums[i] = (beta_1 * layer.bias_momentums[i]) + ((T{1} - beta_1) * layer.dbiases[i]);
}
// Get corrected momentum
// interation is 0 at first pass
// and we need to start with 1 here
weight_momentums_corrected.resize(layer.weights.rows(),layer.weights.cols()); // can be optimized out later
for (uint64_t i = 0; i < layer.weights.rows(); ++i){
for (uint64_t j = 0; j < layer.weights.cols(); ++j){
weight_momentums_corrected(i,j) = layer.weight_momentums(i,j) / (T{1} - std::pow(beta_1, iterations+1));
}
}
bias_momentums_corrected.resize(layer.biases.size()); // can be optimized out later
for (uint64_t i = 0; i < layer.biases.size(); ++i){
bias_momentums_corrected[i] = layer.bias_momentums[i] / (T{1} - std::pow(beta_1, iterations+1));
}
// Update cache with squared current gradients
for (uint64_t i = 0; i < layer.weights.rows(); ++i){
for (uint64_t j = 0; j < layer.weights.cols(); ++j){
layer.weight_cache(i,j) = (beta_2*layer.weight_cache(i,j)) + ((T{1}-beta_2) * (layer.dweights(i,j)*layer.dweights(i,j)));
}
}
for (uint64_t i = 0; i < layer.biases.size(); ++i){ // can maybe be included when updating weights (saves time)
layer.bias_cache[i] = (beta_2*layer.bias_cache[i]) + ((T{1}-beta_2) * (layer.dbiases[i]*layer.dbiases[i]));
}
// Get corrected cache
// interation is 0 at first pass
// and we need to start with 1 here
weight_cache_corrected.resize(layer.weights.rows(),layer.weights.cols()); // can be optimized out later
for (uint64_t i = 0; i < layer.weights.rows(); ++i){
for (uint64_t j = 0; j < layer.weights.cols(); ++j){
weight_cache_corrected(i,j) = layer.weight_cache(i,j) / (T{1} - std::pow(beta_2, iterations+1));
}
}
bias_cache_corrected.resize(layer.biases.size()); // can be optimized out later
for (uint64_t i = 0; i < layer.biases.size(); ++i){
bias_cache_corrected[i] = layer.bias_cache[i] / (T{1} - std::pow(beta_2, iterations+1));
}
// Vanilla SGD parameter update + normalization with squared rooted cache
for (uint64_t i = 0; i < layer.weights.rows(); ++i){
for (uint64_t j = 0; j < layer.weights.cols(); ++j){
layer.weights(i,j) -= (current_learning_rate*weight_momentums_corrected(i,j)) / (std::sqrt(weight_cache_corrected(i,j)) + epsilon);
}
}
for (uint64_t i = 0; i < layer.biases.size(); ++i){
layer.biases[i] -= (current_learning_rate*bias_momentums_corrected[i]) / (std::sqrt(bias_cache_corrected[i]) + epsilon);
}
}
void post_update_params(){
iterations++;
}
};
} // end namespace neural_networks
include/modules/neural_networks/optimizers/Optimizer_RMSprop.h
+81
View File
@@ -0,0 +1,81 @@
#pragma once
#include "./core/omp_config.h"
#include "./utils/vector.h"
#include "./utils/matrix.h"
#include "./numerics/matmul.h"
#include <math.h>
namespace neural_networks{
template <typename T>
struct Optimizer_RMSprop{
T learning_rate = T{1};
T current_learning_rate = learning_rate;
T decay = T{0};
T epsilon = T{1e-7};
T rho = T{0.9};
uint64_t iterations = 0;
// Default Constructor
Optimizer_RMSprop() = default;
// Constructor
explicit Optimizer_RMSprop(const T lr, const T lr_decay, const T epsilons, const T rhos): learning_rate(lr), current_learning_rate{lr}, decay(lr_decay), epsilon(epsilons), rho(rhos) {}
void pre_update_params(){
if(decay){
current_learning_rate = learning_rate * (T{1}/(T{1}+(decay*iterations)));
//std::cout << current_learning_rate << std::endl;
}
}
template <typename Layer>
void update_params(Layer& layer){
// if layer does not contain cache arrays, create them filled with zeros.
if ((layer.weight_cache.rows() != layer.weights.rows()) || (layer.weight_cache.cols() != layer.weights.cols())){
layer.weight_cache.resize(layer.weights.rows(), layer.weights.cols(), T{0});
}
if (layer.bias_cache.size() != layer.biases.size()){
layer.bias_cache.resize(layer.biases.size(), T{0});
}
// Update cache with squared current gradients
for (uint64_t i = 0; i < layer.weights.rows(); ++i){
for (uint64_t j = 0; j < layer.weights.cols(); ++j){
layer.weight_cache(i,j) = (rho*layer.weight_cache(i,j)) + ((T{1}-rho) * (layer.dweights(i,j)*layer.dweights(i,j)));
}
}
for (uint64_t i = 0; i < layer.biases.size(); ++i){ // can maybe be included when updating weights (saves time)
layer.bias_cache[i] = (rho*layer.bias_cache[i]) + ((T{1}-rho) * (layer.dbiases[i]*layer.dbiases[i]));
}
// Vanilla SGD parameter update + normalization with squared rooted cache
for (uint64_t i = 0; i < layer.weights.rows(); ++i){
for (uint64_t j = 0; j < layer.weights.cols(); ++j){
layer.weights(i,j) -= (current_learning_rate*layer.dweights(i,j)) / (std::sqrt(layer.weight_cache(i,j)) + epsilon);
}
}
for (uint64_t i = 0; i < layer.biases.size(); ++i){
layer.biases[i] -= (current_learning_rate*layer.dbiases[i]) / (std::sqrt(layer.bias_cache[i]) + epsilon);
}
}
void post_update_params(){
iterations++;
}
};
} // end namespace neural_networks