Bausager/Flux
1
0
Fork 0
Code Issues 36 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Backwards - ReLU & Dense Lasyer
Sync public mirror / sync (push) Successful in 27s
Details

Browse Source 
Made the backwards for dense layers and ReLU, but I think I soon need to refactor some code to make it all steamlined.
This commit is contained in:
Michelle
2025-10-09 10:43:45 +02:00
parent c164d790db
commit 7fe6be9ba7
10 changed files with 171 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/abc_lab
BIN
View File
Binary file not shown.
include/modules/neural_networks/activation_functions/Activation_ReLU.h
+18 -1
View File
@@ -11,12 +11,29 @@ namespace neural_networks{
template <typename T>
struct Activation_ReLU{
//utils::Matrix<T> inputs;
utils::Matrix<T> _inputs;
utils::Matrix<T> outputs;
utils::Matrix<T> dinputs;
void forward(const utils::Matrix<T>& inputs){
_inputs = inputs;
outputs = numerics::matclip_low(inputs, T{0});
}
void backward(const utils::Matrix<T>& dvalues){
// Since we need to modify the original variable,
// let's make a copy of the values first
dinputs = dvalues;
// Zero gradients where input values were negative
for (uint64_t i = 0; i < dinputs.rows(); ++i){
for (uint64_t j = 0; j < dinputs.cols(); ++j){
if (_inputs(i,j) <= T{0}){
dinputs(i,j) = T{0};
}
}
}
}
};
include/modules/neural_networks/layers/Dense_Layer.h
+17 -2
View File
@@ -12,11 +12,15 @@ namespace neural_networks{
template <typename T>
struct Dense_Layer{
//utils::Matrix<T> _inputs;
utils::Matrix<T> _inputs;
utils::Matrix<T> weights;
utils::Vector<T> biases;
utils::Matrix<T> outputs;
utils::Matrix<T> dweights;
utils::Vector<T> dbiases;
utils::Matrix<T> dinputs;
// Default Constructor
Dense_Layer() = default;
@@ -29,9 +33,20 @@ namespace neural_networks{
}
void forward(utils::Matrix<T>& inputs){
void forward(const utils::Matrix<T>& inputs){
_inputs = inputs;
outputs = numerics::matadd(numerics::matmul_auto(inputs, weights), biases, "row");
}
void backward(const utils::Matrix<T>& dvalues){
// Gradients on parameters
dweights = numerics::matmul(numerics::transpose(_inputs), dvalues);
dbiases = numerics::matsum(dvalues, "row");
//Gradient on values
dinputs = numerics::matmul(dvalues, numerics::transpose(weights));
}
};
include/numerics/matrandom.h
+55
View File
@@ -0,0 +1,55 @@
#ifndef _matrandom_n_
#define _matrandom_n_
#include "./utils/vector.h"
#include "./utils/matrix.h"
#include "./core/omp_config.h"
namespace numerics{
template <typename T>
void inplace_matrandom_add(utils::Matrix<T>& A, const T lower, const T higher) {
const uint64_t rows = A.rows();
const uint64_t cols = A.cols();
utils::Matrix<T> B;
B.random(rows,cols, lower, higher);
numerics::inplace_matadd_mat(A, B);
}
template <typename T>
void inplace_matrandom_mul(utils::Matrix<T>& A, const T lower, const T higher) {
const uint64_t rows = A.rows();
const uint64_t cols = A.cols();
utils::Matrix<T> B;
B.random(rows,cols, lower, higher);
for (uint64_t i = 0; i < rows; ++i){
for (uint64_t j = 0; j < cols; ++j){
A(i,j) *= B(i,j);
}
}
}
template <typename T>
utils::Matrix<T> matrandom_add(const utils::Matrix<T>& A, const T lower, const T higher) {
const uint64_t rows = A.rows();
const uint64_t cols = A.cols();
utils::Matrix<T> B = A;
numerics::inplace_matadd_mat(B, lower, higher);
return B;
}
} // namespace numerics
#endif // _matrandom_n_
include/numerics/numerics.h
+2
View File
@@ -8,6 +8,7 @@
#include "./numerics/vecmax.h"
#include "./numerics/veclog.h"
#include "./numerics/vecargmax.h"
#include "./numerics/vecrandom.h"
#include "./numerics/initializers/eye.h"
#include "./numerics/matequal.h"
#include "./numerics/transpose.h"
@@ -18,6 +19,7 @@
#include "./numerics/matdiv.h"
#include "./numerics/matvec.h"
#include "./numerics/matadd.h"
#include "./numerics/matrandom.h"
#include "./numerics/matsubtract.h"
#include "./numerics/matsum.h"
#include "./numerics/matclip.h"
include/numerics/vecrandom.h
+48
View File
@@ -0,0 +1,48 @@
#ifndef _vecrandom_n_
#define _vecrandom_n_
#include "./utils/vector.h"
#include "./utils/matrix.h"
#include "./core/omp_config.h"
namespace numerics{
template <typename T>
void inplace_vecrandom_add(utils::Vector<T>& a, const T lower, const T higher) {
const uint64_t N = a.size();
utils::Vector<T> b;
b.random(N, lower, higher);
a.inplace_add(b);
}
template <typename T>
void inplace_vecrandom_mul(utils::Vector<T>& a, const T lower, const T higher) {
const uint64_t N = a.size();
utils::Vector<T> b;
b.random(N, lower, higher);
a.inplace_multiply(b);
}
template <typename T>
utils::Vector<T> vecrandom_add(const utils::Vector<T>& a, const T lower, const T higher) {
const uint64_t N = a.size();
utils::Vector<T> b;
inplace_vecrandom_add(b, lower, higher);
return b;
}
} // namespace numerics
#endif // _vecrandom_n_
include/utils/matrix.h
-1
View File
@@ -23,7 +23,6 @@ namespace utils{
class Matrix{
public:
Matrix() : rows_(0), cols_(0), data_() {} // Default constructor
#include <random>
// Constructor to initialize matrix with rows × cols and a fill value
Matrix(uint64_t rows, uint64_t cols, const T& value = T())
: rows_(rows), cols_(cols), data_(rows * cols, value) {}
obj/main.d
+11 -8
View File
@@ -7,19 +7,20 @@ obj/main.o: src/main.cpp include/./core/omp_config.h \
include/./numerics/exp.h include/./numerics/log.h \
include/./numerics/vecclip.h include/./numerics/vecexp.h \
include/./numerics/vecmax.h include/./numerics/veclog.h \
include/./numerics/vecargmax.h include/./numerics/initializers/eye.h \
include/./numerics/matequal.h include/./numerics/abs.h \
include/./numerics/transpose.h include/./numerics/inverse.h \
include/./numerics/vecargmax.h include/./numerics/vecrandom.h \
include/./numerics/initializers/eye.h include/./numerics/matequal.h \
include/./numerics/abs.h include/./numerics/transpose.h \
include/./numerics/inverse.h \
include/./numerics/inverse/inverse_gauss_jordan.h \
include/./numerics/inverse/inverse_lu.h include/./decomp/lu.h \
include/./numerics/matmul.h include/./numerics/matscalar.h \
include/./numerics/matmax.h include/./numerics/matdiv.h \
include/./numerics/matvec.h include/./numerics/matadd.h \
include/./numerics/matsubtract.h include/./numerics/matsum.h \
include/./numerics/matclip.h include/./numerics/matexp.h \
include/./numerics/matlog.h include/./numerics/matdot.h \
include/./numerics/matargmax.h include/./numerics/min.h \
include/./numerics/interpolation1d.h \
include/./numerics/matrandom.h include/./numerics/matsubtract.h \
include/./numerics/matsum.h include/./numerics/matclip.h \
include/./numerics/matexp.h include/./numerics/matlog.h \
include/./numerics/matdot.h include/./numerics/matargmax.h \
include/./numerics/min.h include/./numerics/interpolation1d.h \
include/./numerics/interpolation1d/interpolation1d_barycentric.h \
include/./numerics/interpolation1d/interpolation1d_base.h \
include/./numerics/interpolation1d/interpolation1d_cubic_spline.h \
@@ -57,6 +58,7 @@ include/./numerics/vecexp.h:
include/./numerics/vecmax.h:
include/./numerics/veclog.h:
include/./numerics/vecargmax.h:
include/./numerics/vecrandom.h:
include/./numerics/initializers/eye.h:
include/./numerics/matequal.h:
include/./numerics/abs.h:
@@ -71,6 +73,7 @@ include/./numerics/matmax.h:
include/./numerics/matdiv.h:
include/./numerics/matvec.h:
include/./numerics/matadd.h:
include/./numerics/matrandom.h:
include/./numerics/matsubtract.h:
include/./numerics/matsum.h:
include/./numerics/matclip.h:
obj/main.o
BIN
View File
Binary file not shown.
src/main.cpp
+20 -32
View File
@@ -44,7 +44,7 @@ int main(int argc, char const *argv[])
utils::Vf best_dense_1_biases = dense1.biases;
utils::Mf best_dense_2_weights = dense2.weights;
utils::Vf best_dense_2_biases = dense2.biases;
utils::Mf matrRND;
utils::Vf vectRND;
utils::Vector<uint64_t> predections;
@@ -53,20 +53,15 @@ int main(int argc, char const *argv[])
for (uint64_t i = 0; i < 10000; ++i){
for (uint64_t i = 0; i < 1000; ++i){
// Generate a new set of weights for iteration
matrRND.random(dense1.weights.rows(), dense1.weights.cols(), -1, 1);
numerics::inplace_matscalar(matrRND,0.05f);
numerics::inplace_matadd(dense1.weights, matrRND);
numerics::inplace_matrandom_mul(dense1.weights,0.98f, 1.02f);
numerics::inplace_vecrandom_mul(dense1.biases,0.98f, 1.02f);
dense1.biases.random(dense1.biases.size(), -1, 1);
matrRND.random(dense2.weights.rows(), dense2.weights.cols(), -1, 1);
numerics::inplace_matscalar(matrRND,0.05f);
dense2.weights.random(dense2.weights.rows(), dense2.weights.cols(), -1, 1);
dense2.biases.random(dense2.biases.size(), -1, 1);
numerics::inplace_matrandom_mul(dense2.weights,0.98f, 1.02f);
numerics::inplace_vecrandom_mul(dense2.biases,0.98f, 1.02f);
// Perform a forward pass of the training data through this layer
dense1.forward(X);
@@ -95,36 +90,29 @@ int main(int argc, char const *argv[])
best_dense_2_weights = dense2.weights;
best_dense_2_biases = dense2.biases;
lowest_loss = loss;
} else{
//std::cout << "HERE" << std::endl;
dense1.weights = best_dense_1_weights;
dense1.biases = best_dense_1_biases;
dense2.weights = best_dense_2_weights;
dense2.biases = best_dense_2_biases;
}
}
for (int i = 0; i < 5; ++i){
std::cout << activation2.outputs.get_row(i) << std::endl;
}
std::cout << loss << std::endl;
//predections.print();
//predections.print();
//class_targets.print();
std::cout << accuracy << std::endl;
utils::Mf dvalues{{1, 1, 1},
{2, 2, 2},
{3, 3, 3}};
utils::Vf biases{2,3,0.5};
utils::Vf dbiases = numerics::matsum(dvalues, "row");
dbiases.print();