Regulaization

Started on regulaization in Loss.h. I need to refactor the matsum.h since I need a total sum over the matrix. Also matmul needs a elementwise matmul function, which is the next this in the ragulaization
2026-01-03 22:10:50 +01:00
parent 32ba0518fa
commit 48f329feef
17 changed files with 881 additions and 510 deletions
@@ -12,45 +12,51 @@ namespace neural_networks{
 	template <typename T>
 	struct Dense_Layer{

-			utils::Matrix<T> _inputs;
-			utils::Matrix<T> weights;
-			utils::Vector<T> biases;
-			utils::Matrix<T> outputs;
+		T weight_regularizer_l1 = {1e-4};
+		T weight_regularizer_l2 = {1e-4};

-			utils::Matrix<T> dweights;
-			utils::Vector<T> dbiases;
-			utils::Matrix<T> dinputs;
+		T bias_regularizer_l1 = {1e-4};
+		T bias_regularizer_l2 = {1e-4};

-			// Variables for optimizers
-		    utils::Matrix<T> weight_momentums;
-		    utils::Vector<T> bias_momentums;
-		    utils::Matrix<T> weight_cache;
-		    utils::Vector<T> bias_cache;
-			
-			// Default Constructor
-			Dense_Layer() = default;
+		utils::Matrix<T> _inputs;
+		utils::Matrix<T> weights;
+		utils::Vector<T> biases;
+		utils::Matrix<T> outputs;

-			// Constructor
-	    	Dense_Layer(const uint64_t n_inputs, const uint64_t n_neurons){
-	        	
-	        	weights.random(n_inputs, n_neurons, -1, 1);
-	        	biases.resize(n_neurons, T{0});
-	        	
-	        }
+		utils::Matrix<T> dweights;
+		utils::Vector<T> dbiases;
+		utils::Matrix<T> dinputs;

-	        void forward(const utils::Matrix<T>& inputs){
-	        	_inputs = inputs;
-			    outputs = numerics::matadd(numerics::matmul_auto(inputs, weights), biases, "row");
-	        }
+		// Variables for optimizers
+	    utils::Matrix<T> weight_momentums;
+	    utils::Vector<T> bias_momentums;
+	    utils::Matrix<T> weight_cache;
+	    utils::Vector<T> bias_cache;
+		
+		// Default Constructor
+		Dense_Layer() = default;

-	        void backward(const utils::Matrix<T>& dvalues){
-	        	// Gradients on parameters
-	        	dweights  = numerics::matmul(numerics::transpose(_inputs), dvalues);
-	        	dbiases = numerics::matsum(dvalues, "row");
-	        	//Gradient on values
-	        	dinputs = numerics::matmul(dvalues, numerics::transpose(weights));
+		// Constructor
+    	Dense_Layer(const uint64_t n_inputs, const uint64_t n_neurons){
+        	
+        	weights.random(n_inputs, n_neurons, -1, 1);
+        	biases.resize(n_neurons, T{0});
+        	
+        }

-	        }
+        void forward(const utils::Matrix<T>& inputs){
+        	_inputs = inputs;
+		    outputs = numerics::matadd(numerics::matmul_auto(inputs, weights), biases, "row");
+        }
+
+        void backward(const utils::Matrix<T>& dvalues){
+        	// Gradients on parameters
+        	dweights  = numerics::matmul(numerics::transpose(_inputs), dvalues);
+        	dbiases = numerics::matsum(dvalues, "row");
+        	//Gradient on values
+        	dinputs = numerics::matmul(dvalues, numerics::transpose(weights));
+
+        }

 	};