Backwards - ReLU & Dense Lasyer

Made the backwards for dense layers and ReLU, but I think I soon need to refactor some code to make it all steamlined.

include/modules/neural_networks/activation_functions/Activation_ReLU.h

@@ -11,12 +11,29 @@ namespace neural_networks{
 	template <typename T>
 	struct Activation_ReLU{
 
-		//utils::Matrix<T> inputs;
+		utils::Matrix<T> _inputs;
 		utils::Matrix<T> outputs;
 
+		utils::Matrix<T> dinputs;
+
         void forward(const utils::Matrix<T>& inputs){
+        	_inputs = inputs;
 		    outputs = numerics::matclip_low(inputs, T{0});
         }
+        void backward(const utils::Matrix<T>& dvalues){
+        	// Since we need to modify the original variable,
+        	// let's make a copy of the values first
+        	dinputs = dvalues;
+
+        	// Zero gradients where input values were negative
+        	for (uint64_t i = 0; i < dinputs.rows(); ++i){
+        		for (uint64_t j = 0; j < dinputs.cols(); ++j){
+        			if (_inputs(i,j) <= T{0}){
+        				dinputs(i,j) = T{0};
+        			}
+        		}
+        	}
+        }
 	};
 
 

include/modules/neural_networks/layers/Dense_Layer.h

@@ -12,11 +12,15 @@ namespace neural_networks{
 	template <typename T>
 	struct Dense_Layer{
 
-			//utils::Matrix<T> _inputs;
+			utils::Matrix<T> _inputs;
 			utils::Matrix<T> weights;
 			utils::Vector<T> biases;
 			utils::Matrix<T> outputs;
 
+			utils::Matrix<T> dweights;
+			utils::Vector<T> dbiases;
+			utils::Matrix<T> dinputs;
+
 			
 			// Default Constructor
 			Dense_Layer() = default;
@@ -29,9 +33,20 @@ namespace neural_networks{
 	        	
 	        }
 
-	        void forward(utils::Matrix<T>& inputs){
+	        void forward(const utils::Matrix<T>& inputs){
+	        	_inputs = inputs;
 			    outputs = numerics::matadd(numerics::matmul_auto(inputs, weights), biases, "row");
 	        }
+
+	        void backward(const utils::Matrix<T>& dvalues){
+	        	// Gradients on parameters
+	        	dweights  = numerics::matmul(numerics::transpose(_inputs), dvalues);
+	        	dbiases = numerics::matsum(dvalues, "row");
+	        	//Gradient on values
+	        	dinputs = numerics::matmul(dvalues, numerics::transpose(weights));
+
+	        }
+
 	};