Bausager/Flux
1
0
Fork 0
Code Issues 36 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Starting on the model.h, but need to make layer structs and structs for loss and optimizers

Browse Source
This commit is contained in:
Michelle
2026-05-30 09:13:52 +02:00
parent cb65174cf4
commit edad247227
30 changed files with 1879 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/modules/neural_networks/activation_functions/Activation_Linear.h
+33
View File
@@ -0,0 +1,33 @@
#pragma once
#include "core/omp_config.h"
#include "utils/vector.h"
#include "utils/matrix.h"
#include "modules/neural_networks/layers/Layer.h"
namespace neural_networks{
template <typename T>
struct Activation_Linear : Layer<T>{
utils::Matrix<T> _inputs;
utils::Matrix<T> outputs;
utils::Matrix<T> dinputs;
void forward(const utils::Matrix<T>& inputs){
_inputs = inputs;
outputs = inputs;
}
void backward(const utils::Matrix<T>& dvalues){
dinputs = dvalues;
}
};
} // end namespace neural_networks
include/modules/neural_networks/activation_functions/Activation_ReLU.h
+2 -2
View File
@@ -4,12 +4,12 @@
#include "utils/vector.h"
#include "utils/matrix.h"
#include "modules/neural_networks/layers/Layer.h"
namespace neural_networks{
template <typename T>
struct Activation_ReLU{
struct Activation_ReLU : Layer<T>{
utils::Matrix<T> _inputs;
utils::Matrix<T> outputs;
include/modules/neural_networks/activation_functions/Activation_Sigmoid.h
+2 -1
View File
@@ -4,6 +4,7 @@
#include "utils/vector.h"
#include "utils/matrix.h"
#include "modules/neural_networks/layers/Layer.h"
#include "numerics/neg.h"
#include "numerics/exp.h"
@@ -15,7 +16,7 @@
namespace neural_networks{
template <typename T>
struct Activation_Sigmoid{
struct Activation_Sigmoid : Layer<T>{
utils::Matrix<T> _inputs;
utils::Matrix<T> outputs;
include/modules/neural_networks/activation_functions/Activation_Softmax.h
+2 -1
View File
@@ -4,6 +4,7 @@
#include "utils/vector.h"
#include "utils/matrix.h"
#include "modules/neural_networks/layers/Layer.h"
#include "numerics/max.h"
#include "numerics/sub.h"
@@ -15,7 +16,7 @@
namespace neural_networks{
template <typename T>
struct Activation_Softmax{
struct Activation_Softmax : Layer<T>{
//utils::Matrix<T> exp_values;
//utils::Matrix<T> probabilities;
include/modules/neural_networks/activation_functions/Activation_Softmax_Loss_CategoricalCrossentropy.h
+2 -1
View File
@@ -4,6 +4,7 @@
#include "utils/vector.h"
#include "utils/matrix.h"
#include "modules/neural_networks/layers/Layer.h"
#include "numerics/max.h"
#include "numerics/sub.h"
@@ -17,7 +18,7 @@
namespace neural_networks{
template <typename Td, typename Ti>
struct Activation_Softmax_Loss_CategoricalCrossentropy{
struct Activation_Softmax_Loss_CategoricalCrossentropy : Layer<Td>{
neural_networks::Activation_Softmax<Td> activation;
neural_networks::Loss_CategoricalCrossentrophy<Td, Ti> loss;
include/modules/neural_networks/datasets/sine.h
+37
View File
@@ -0,0 +1,37 @@
#pragma once
#include "./core/omp_config.h"
#include "./utils/matrix.h"
#include "./utils/vector.h"
#include "utils/generators/linspace.h"
#include <math.h> /* sin */
//#include "./utils/random.h"
//#include <math.h>
namespace neural_networks{
template <typename T>
void create_sine_data(const uint64_t samples, const T length, utils::Matrix<T>& X, utils::Matrix<T>& y) {
X.resize(samples, 1);
X.set_col(0, utils::linspace(
T{0}, // start
length, // stop
samples, // N
true // endpoint
));
y.resize(samples, 1);
for (uint64_t i = 0; i < samples; ++i){
y(i,0) = std::sin(X(i,0));
}
}
} // end namesoace NN
include/modules/neural_networks/layers/Dense_Layer.h
+6 -2
View File
@@ -4,13 +4,17 @@
#include "utils/vector.h"
#include "utils/matrix.h"
#include "Layer.h"
#include "utils/random.h"
#include "random/random.h"
namespace neural_networks{
template <typename T>
struct Dense_Layer{
struct Dense_Layer : Layer<T>{
T weight_regularizer_l1 = {0};
T weight_regularizer_l2 = {0};
@@ -48,7 +52,7 @@ namespace neural_networks{
this->bias_regularizer_l1 = bias_regularizer_l1;
this->bias_regularizer_l2 = bias_regularizer_l2;
weights.random(n_inputs, n_neurons, -1, 1);
weights = numerics::mul( rng::normal(n_inputs, n_neurons, 0.0f, 1.0f), 0.1f);
//weights = numerics::random_matrix(n_inputs, n_neurons, -1, 1);
biases.resize(n_neurons, T{0});
include/modules/neural_networks/layers/Dropout_Layer.h
+3 -1
View File
@@ -4,13 +4,15 @@
#include "utils/vector.h"
#include "utils/matrix.h"
#include "modules/neural_networks/layers/Layer.h"
#include "random/random.h"
namespace neural_networks{
template <typename T>
struct Dropout_Layer{
struct Dropout_Layer : Layer<T>{
// Store rate, we invert it as for example for dropout
// of 0.1 we need a success rate of 0.9
include/modules/neural_networks/layers/Layer.h
+21
View File
@@ -0,0 +1,21 @@
#pragma once
#include "core/omp_config.h"
#include "utils/vector.h"
#include "utils/matrix.h"
namespace neural_networks{
template <typename T>
struct Layer {
utils::Matrix<T> outputs;
utils::Matrix<T> dinputs;
virtual void forward(const utils::Matrix<T>& inputs) = 0;
virtual void backward(const utils::Matrix<T>& dvalues) = 0;
virtual ~Layer() = default;
};
}
include/modules/neural_networks/loss/Loss.h
+1 -1
View File
@@ -10,7 +10,7 @@
namespace neural_networks{
template <typename Td, typename Ti>
template <typename Td, typename Ti = Td>
struct Loss{
utils::Vector<Td> sample_losses;
include/modules/neural_networks/loss/Loss_MeanAbsoluteError.h
+58
View File
@@ -0,0 +1,58 @@
#pragma once
#include "core/omp_config.h"
#include "utils/vector.h"
#include "utils/matrix.h"
#include "utils/matcast.h"
#include "numerics/clip.h"
#include "numerics/log.h"
#include "numerics/sub.h"
#include "Loss.h"
namespace neural_networks{
template <typename T>
struct Loss_MeanAbsoluteError : Loss<T> {
utils::Matrix<T> dinputs;
utils::Matrix<T> y_true;
utils::Vector<T> sample_losses;
utils::Vector<T> forward(const utils::Matrix<T>& y_pred, const utils::Matrix<T>& y_true) override{
// Calculate loss
sample_losses = numerics::mean_rowwise(numerics::abs(numerics::sub(y_true, y_pred)));
// Return losses
return sample_losses;
}
void backward(const utils::Matrix<T>& dvalues, const utils::Matrix<T>& y_true) override{
// Number of samples
const T samples = static_cast<T> (dvalues.rows());
// Number of outputs in every sample
const T outputs = static_cast<T> (dvalues.cols());
// Gradient values
dinputs = numerics::div(numerics::sign(numerics::sub(y_true, dvalues)), outputs);
// Normalise gradient
dinputs = numerics::div(dinputs, samples);
}
};
} // end namespace neural_networks
include/modules/neural_networks/loss/Loss_MeanSquaredError.h
+58
View File
@@ -0,0 +1,58 @@
#pragma once
#include "core/omp_config.h"
#include "utils/vector.h"
#include "utils/matrix.h"
#include "utils/matcast.h"
#include "numerics/clip.h"
#include "numerics/log.h"
#include "numerics/sub.h"
#include "Loss.h"
namespace neural_networks{
template <typename T>
struct Loss_MeanSquaredError : Loss<T> {
utils::Matrix<T> dinputs;
utils::Matrix<T> y_true;
utils::Vector<T> sample_losses;
utils::Vector<T> forward(const utils::Matrix<T>& y_pred, const utils::Matrix<T>& y_true) override{
// Calculate loss
sample_losses = numerics::mean_rowwise(numerics::pow(numerics::sub(y_true, y_pred), T{2}));
// Return losses
return sample_losses;
}
void backward(const utils::Matrix<T>& dvalues, const utils::Matrix<T>& y_true) override{
// Number of samples
const T samples = static_cast<T> (dvalues.rows());
// Number of outputs in every sample
const T outputs = static_cast<T> (dvalues.cols());
// Gradient values
dinputs = numerics::mul(numerics::div(numerics::sub(y_true, dvalues), outputs), T{-2});
// Normalise gradient
dinputs = numerics::div(dinputs, samples);
}
};
} // end namespace neural_networks
include/modules/neural_networks/model/model.h
+30
View File
@@ -0,0 +1,30 @@
#pragma once
#include "core/omp_config.h"
#include "utils/vector.h"
#include "utils/matrix.h"
#include "modules/neural_networks/layers/Layer.h"
namespace neural_networks {
template <typename T>
struct Model {
std::vector<Layer<T>*> layers;
Model() = default;
void add(Layer<T>& layer) {
layers.push_back(&layer);
}
};
} // end namespace neural_networks
include/modules/neural_networks/neural_networks.h
+10 -2
View File
@@ -3,8 +3,9 @@
#include "datasets/spiral.h"
#include "datasets/vertical.h"
#include "datasets/sine.h"
#include "layers/Layer.h"
#include "layers/Dense_Layer.h"
#include "layers/Dropout_Layer.h"
@@ -13,14 +14,21 @@
#include "activation_functions/Activation_Softmax.h"
#include "activation_functions/Activation_Softmax_Loss_CategoricalCrossentropy.h"
#include "activation_functions/Activation_Sigmoid.h"
#include "activation_functions/Activation_Linear.h"
#include "loss/Loss.h" // Base
#include "loss/Loss_CategoricalCrossentrophy.h"
#include "loss/Loss_BinaryCrossentropy.h"
#include "loss/Loss_MeanSquaredError.h"
#include "optimizers/Optimizer_SGD.h"
#include "optimizers/Optimizer_Adagrad.h"
#include "optimizers/Optimizer_RMSprop.h"
#include "optimizers/Optimizer_Adam.h"
#include "optimizers/Optimizer_Adam.h"
#include "model/model.h"