Binomial_CrossEnthophy

fixed rowwise/colswise mean/sum and implemented binomial_corssentrhopy. Next up is regression.

include/modules/neural_networks/activation_functions/Activation_Sigmoid.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "core/omp_config.h"
+
+#include "utils/vector.h"
+#include "utils/matrix.h"
+
+#include "numerics/neg.h"
+#include "numerics/exp.h"
+#include "numerics/add.h"
+#include "numerics/div.h"
+#include "numerics/sub.h"
+#include "numerics/mul.h"
+
+namespace neural_networks{
+
+	template <typename T>
+	struct Activation_Sigmoid{
+
+		utils::Matrix<T> _inputs;
+		utils::Matrix<T> outputs;
+
+		utils::Matrix<T> dinputs;
+
+        void forward(const utils::Matrix<T>& inputs){
+        	_inputs = inputs;
+
+        	outputs = numerics::neg(inputs);
+        	outputs = numerics::exp(outputs);
+        	outputs = numerics::add(outputs, T{1});
+        	outputs = numerics::div(T{1}, outputs);
+        }
+        void backward(const utils::Matrix<T>& dvalues){
+        	
+        	dinputs = numerics::sub(T{1}, outputs);
+        	dinputs = numerics::mul(dvalues, dinputs);
+        	dinputs = numerics::mul(dinputs, outputs);
+
+
+
+        }
+	};
+
+
+
+} // end namespace neural_networks

include/modules/neural_networks/activation_functions/Activation_Softmax.h

@@ -29,7 +29,7 @@ namespace neural_networks{
 			    utils::Matrix<T> exp_values = numerics::exp(numerics::sub_colwise(inputs, numerics::max_rowwise(inputs)));
 
 			    // Normalize them for each sample
-			    utils::Matrix<T> probabilities = numerics::div_colwise(exp_values, numerics::sum_colwise(exp_values));
+			    utils::Matrix<T> probabilities = numerics::div_colwise(exp_values, numerics::sum_rowwise(exp_values));
 			    outputs = probabilities;
 	        }
 

include/modules/neural_networks/layers/Dense_Layer.h

@@ -63,7 +63,7 @@ namespace neural_networks{
         void backward(const utils::Matrix<T>& dvalues){
         	// Gradients on parameters
         	dweights  = numerics::matmul(numerics::transpose(_inputs), dvalues);
-        	dbiases = numerics::sum_rowwise(dvalues);
+        	dbiases = numerics::sum_colwise(dvalues);
 
 
         	// Gradients on regularization 

include/modules/neural_networks/loss/Loss_BinaryCrossentropy.h

@@ -0,0 +1,81 @@
+#pragma once
+
+#include "core/omp_config.h"
+
+#include "utils/vector.h"
+#include "utils/matrix.h"
+#include "utils/matcast.h"
+
+#include "numerics/clip.h"
+#include "numerics/log.h"
+#include "numerics/sub.h"
+
+#include "Loss.h"
+
+
+namespace neural_networks{
+
+	template <typename Td, typename Ti>
+	struct Loss_BinaryCrossentropy : Loss<Td, Ti>  {
+
+		utils::Matrix<Td> dinputs;
+		utils::Matrix<Td> y_true;
+
+	
+		utils::Vector<Td> forward(const utils::Matrix<Td>& y_pred, const utils::Matrix<Ti>& y_true) override{
+			
+			this->y_true = utils::matcast<Td, Ti>(y_true);
+
+			// Clip daa to prevent division by 0
+			// Clip both sides not to drag mean towards any value
+			utils::Matrix<Td> y_pred_clipped = numerics::clip(y_pred, Td{1e-7}, Td{1.0} - Td{1e-7});
+
+			// Calculate sample-wise loss
+			utils::Matrix<Td> sample_losses_temp = numerics::log(numerics::sub(Td{1}, y_pred_clipped));
+			sample_losses_temp = numerics::mul(sample_losses_temp, numerics::sub(Td{1}, this->y_true));
+			sample_losses_temp = numerics::add(sample_losses_temp, numerics::mul(this->y_true, numerics::log(y_pred_clipped)));
+			sample_losses_temp = numerics::neg(sample_losses_temp);
+
+
+
+
+
+			utils::Vector<Td> sample_losses = numerics::mean_rowwise(sample_losses_temp);
+
+			// Return losses
+			return sample_losses;
+		}
+		
+		void backward(const utils::Matrix<Td>& dvalues, const utils::Matrix<Ti>& y_true) override{
+
+				/*std::cout << "BCE backward y_true: "
+				          << y_true.rows() << " x " << y_true.cols()
+				          << std::endl;*/
+
+
+			// Number of samples
+			const Td samples = static_cast<Td> (this->y_true.rows());
+			// Number of outputs in every sample
+			const Td outputs = static_cast<Td> (dvalues.cols());
+
+			// Clip data to prevent division by 0
+			// Clip both sides to not drag mean towards any value
+			utils::Matrix<Td> clipped_dvalues = numerics::clip(dvalues, Td{1e-7}, Td{1.0} - Td{1e-7});
+
+
+			// Calculate gradient
+			dinputs = numerics::div(numerics::neg(numerics::sub(numerics::div(this->y_true, clipped_dvalues), numerics::div(numerics::sub(Td{1}, this->y_true), numerics::sub(Td{1}, clipped_dvalues)))), outputs);
+			// Normalize gradients
+			dinputs = numerics::div(dinputs, samples);
+
+			/*
+			std::cout << "BCE backward dinputs: "
+          << dinputs.rows() << " x " << dinputs.cols()
+          << std::endl;*/
+
+		}
+	};
+
+
+
+} // end namespace neural_networks

include/modules/neural_networks/neural_networks.h

@@ -12,10 +12,12 @@
 #include "activation_functions/Activation_ReLU.h"
 #include "activation_functions/Activation_Softmax.h"
 #include "activation_functions/Activation_Softmax_Loss_CategoricalCrossentropy.h"
+#include "activation_functions/Activation_Sigmoid.h"
 
 
 #include "loss/Loss.h" // Base
 #include "loss/Loss_CategoricalCrossentrophy.h"
+#include "loss/Loss_BinaryCrossentropy.h"
 
 
 #include "optimizers/Optimizer_SGD.h"