Sync public subset from Flux

include/modules/neural_networks/activation_functions/Activation_ReLU.h

@@ -11,12 +11,29 @@ namespace neural_networks{
 	template <typename T>
 	struct Activation_ReLU{
 
-		//utils::Matrix<T> inputs;
+		utils::Matrix<T> _inputs;
 		utils::Matrix<T> outputs;
 
+		utils::Matrix<T> dinputs;
+
         void forward(const utils::Matrix<T>& inputs){
+        	_inputs = inputs;
 		    outputs = numerics::matclip_low(inputs, T{0});
         }
+        void backward(const utils::Matrix<T>& dvalues){
+        	// Since we need to modify the original variable,
+        	// let's make a copy of the values first
+        	dinputs = dvalues;
+
+        	// Zero gradients where input values were negative
+        	for (uint64_t i = 0; i < dinputs.rows(); ++i){
+        		for (uint64_t j = 0; j < dinputs.cols(); ++j){
+        			if (_inputs(i,j) <= T{0}){
+        				dinputs(i,j) = T{0};
+        			}
+        		}
+        	}
+        }
 	};
 
 

include/modules/neural_networks/layers/Dense_Layer.h

@@ -12,11 +12,15 @@ namespace neural_networks{
 	template <typename T>
 	struct Dense_Layer{
 
-			//utils::Matrix<T> _inputs;
+			utils::Matrix<T> _inputs;
 			utils::Matrix<T> weights;
 			utils::Vector<T> biases;
 			utils::Matrix<T> outputs;
 
+			utils::Matrix<T> dweights;
+			utils::Vector<T> dbiases;
+			utils::Matrix<T> dinputs;
+
 			
 			// Default Constructor
 			Dense_Layer() = default;
@@ -29,9 +33,20 @@ namespace neural_networks{
 	        	
 	        }
 
-	        void forward(utils::Matrix<T>& inputs){
+	        void forward(const utils::Matrix<T>& inputs){
+	        	_inputs = inputs;
 			    outputs = numerics::matadd(numerics::matmul_auto(inputs, weights), biases, "row");
 	        }
+
+	        void backward(const utils::Matrix<T>& dvalues){
+	        	// Gradients on parameters
+	        	dweights  = numerics::matmul(numerics::transpose(_inputs), dvalues);
+	        	dbiases = numerics::matsum(dvalues, "row");
+	        	//Gradient on values
+	        	dinputs = numerics::matmul(dvalues, numerics::transpose(weights));
+
+	        }
+
 	};
 
 

include/numerics/matrandom.h

@@ -0,0 +1,55 @@
+#ifndef _matrandom_n_
+#define _matrandom_n_
+
+#include "./utils/vector.h"
+#include "./utils/matrix.h"
+#include "./core/omp_config.h"
+
+namespace numerics{
+	
+	template <typename T>
+	void inplace_matrandom_add(utils::Matrix<T>& A, const T lower, const T higher) {
+	    
+	    const uint64_t rows = A.rows();
+	    const uint64_t cols = A.cols();
+
+	    utils::Matrix<T> B;
+	    B.random(rows,cols, lower, higher);
+
+	    numerics::inplace_matadd_mat(A, B);
+	}
+
+	template <typename T>
+	void inplace_matrandom_mul(utils::Matrix<T>& A, const T lower, const T higher) {
+	    
+	    const uint64_t rows = A.rows();
+	    const uint64_t cols = A.cols();
+
+	    utils::Matrix<T> B;
+	    B.random(rows,cols, lower, higher);
+
+	    for (uint64_t i = 0; i < rows; ++i){
+	    	for (uint64_t j = 0; j < cols; ++j){
+	    		A(i,j) *= B(i,j);
+	    	}
+	    }
+	}
+
+
+	template <typename T>
+	utils::Matrix<T> matrandom_add(const utils::Matrix<T>& A, const T lower, const T higher) {
+	    
+	    const uint64_t rows = A.rows();
+	    const uint64_t cols = A.cols();
+
+	    utils::Matrix<T> B = A;
+
+	    numerics::inplace_matadd_mat(B, lower, higher);
+
+	    return B;
+	}
+
+
+} // namespace numerics
+
+#endif // _matrandom_n_

include/numerics/numerics.h

@@ -8,6 +8,7 @@
 #include "./numerics/vecmax.h"
 #include "./numerics/veclog.h"
 #include "./numerics/vecargmax.h"
+#include "./numerics/vecrandom.h"
 #include "./numerics/initializers/eye.h"
 #include "./numerics/matequal.h"
 #include "./numerics/transpose.h"
@@ -18,6 +19,7 @@
 #include "./numerics/matdiv.h"
 #include "./numerics/matvec.h"
 #include "./numerics/matadd.h"
+#include "./numerics/matrandom.h"
 #include "./numerics/matsubtract.h"
 #include "./numerics/matsum.h"
 #include "./numerics/matclip.h"

include/numerics/vecrandom.h

@@ -0,0 +1,48 @@
+#ifndef _vecrandom_n_
+#define _vecrandom_n_
+
+#include "./utils/vector.h"
+#include "./utils/matrix.h"
+#include "./core/omp_config.h"
+
+namespace numerics{
+	
+	template <typename T>
+	void inplace_vecrandom_add(utils::Vector<T>& a, const T lower, const T higher) {
+	    
+	    const uint64_t N = a.size();
+
+	    utils::Vector<T> b;
+	    b.random(N, lower, higher);
+
+	    a.inplace_add(b);
+	}
+
+	template <typename T>
+	void inplace_vecrandom_mul(utils::Vector<T>& a, const T lower, const T higher) {
+	    
+	    const uint64_t N = a.size();
+
+	    utils::Vector<T> b;
+	    b.random(N, lower, higher);
+
+	    a.inplace_multiply(b);
+
+	}
+
+
+	template <typename T>
+	utils::Vector<T> vecrandom_add(const utils::Vector<T>& a, const T lower, const T higher) {
+	    
+	    const uint64_t N = a.size();
+
+	    utils::Vector<T> b;
+	    inplace_vecrandom_add(b, lower, higher);
+
+	    return b;
+	}
+
+
+} // namespace numerics
+
+#endif // _vecrandom_n_

include/utils/matrix.h

@@ -23,7 +23,6 @@ namespace utils{
 	class Matrix{
 public:
 	    Matrix() : rows_(0), cols_(0), data_() {} // Default constructor
-#include <random>
 	   // Constructor to initialize matrix with rows × cols and a fill value
 	    Matrix(uint64_t rows, uint64_t cols, const T& value = T()) 
 	    : rows_(rows), cols_(cols), data_(rows * cols, value) {}