Started Loss, done softmax, up to p.125

I've implemented alot of support functions that needs to be refactored, optimised and tested; mean.h, exponential.h, matdiv.h matsum.h matsubtract.h. Maybe we need to have a look at if matdiv/matmul should be in the same. Same with matadd/matsubtract and if some of it should be in matvec.h.

include/modules/neural_networks/activation_functions/Softmax.h

@@ -4,7 +4,11 @@
 
 #include "./utils/vector.h"
 #include "./utils/matrix.h"
-#include "./utils/random.h"
+
+#include "./numerics/max.h"
+#include "./numerics/matsubtract.h"
+#include "./numerics/exponential.h"
+#include "./numerics/matdiv.h"
 
 
 namespace neural_networks{
@@ -12,15 +16,20 @@ namespace neural_networks{
 	template <typename T>
 	struct activation_softmax{
 
+			utils::Matrix<T> exp_values;
+			utils::Matrix<T> probabilities;
 			utils::Matrix<T> outputs;
 
-	        void forward(utils::Matrix<T> inputs){
-			    //outputs = numerics::max(inputs, T{0});
-			    //outputs.print();
+	        void forward(const utils::Matrix<T> inputs){
+	
+
+			    exp_values = numerics::exponential(numerics::matsubtract(inputs, numerics::max(inputs, "rows"), "col"));
+			    probabilities = numerics::matdiv(exp_values, numerics::matsum(exp_values, "col"), "col");
+
+			    outputs = probabilities;
+
 	        }
 
-
-
 	};
 
 

include/modules/neural_networks/datasets/spiral.h

@@ -10,11 +10,11 @@
 
 namespace neural_networks{
 
-		template <typename T>
-		void create_spital_data(const uint64_t samples, const uint64_t classes, utils::Matrix<T>& X, utils::Vector<T>& y) {
+		template <typename TX, typename Ty>
+		void create_spital_data(const uint64_t samples, const uint64_t classes, utils::Matrix<TX>& X, utils::Vector<Ty>& y) {
 
 			const uint64_t rows = samples*classes;
-			T r, t;
+			TX r, t;
 			uint64_t row_idx;
 
 
@@ -27,34 +27,15 @@ namespace neural_networks{
 
 			for (uint64_t i = 0; i < classes; ++i){
 				for (uint64_t j = 0; j < samples; ++j){
-					r = static_cast<T>(j)/static_cast<T>(samples);
-					t =  static_cast<T>(i)*4.0 + (4.0+r);
+					r = static_cast<TX>(j)/static_cast<TX>(samples);
+					t =  static_cast<TX>(i)*4.0 + (4.0+r);
 					row_idx = (i*samples) + j;
 
-					X(row_idx, 0) = r*std::cos(t*2.5) + utils::random(T{-0.15}, T{0.15});
-					X(row_idx, 1) = r*std::sin(t*2.5) + utils::random(T{-0.15}, T{0.15});
-					y[row_idx] = i;
+					X(row_idx, 0) = r*std::cos(t*2.5) + utils::random(TX{-0.15}, TX{0.15});
+					X(row_idx, 1) = r*std::sin(t*2.5) + utils::random(TX{-0.15}, TX{0.15});
+					y[row_idx] = static_cast<Ty>(i);
 				}
 			}
-
-			
-
-			/*
-
-			utils::Matrix<T> X(static_cast<uint64_t>(samples*classes), 3, T{0});
-		    
-		    const uint64_t rows = A.rows();
-		    const uint64_t cols = A.cols();
-
-		    if (rows != x.size()) {
-		        throw std::runtime_error("inplace_matadd_colvec: dimension mismatch");
-		    }
-
-		    for (uint64_t i = 0; i < cols; ++i) {
-		        for (uint64_t j = 0; j < rows; ++j) {
-		            A(j, i) +=  x[j];
-		        }
-		    }*/
 		}
 
 

include/modules/neural_networks/layers/dense_layer.h

@@ -23,7 +23,7 @@ namespace neural_networks{
 	        	
 	        	weights.random(n_inputs, n_neurons, -1, 1);
 	        	biases.resize(n_neurons, T{0});
-	        	weights.print();
+	        	//weights.print();
 	        	//outputs.resize()
 	        	
 	        }

include/modules/neural_networks/loss/Loss _CategoricalCrossentrophy.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "./core/omp_config.h"
+
+#include "./utils/vector.h"
+#include "./utils/matrix.h"
+
+
+namespace neural_networks{
+
+	template <typename Td, typename Ti>
+	struct Loss{
+
+			utils::Matrix<Td> sample_losses;
+			Td data_losses;
+
+			virtual utils::Vector<Td> forward(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y) = 0;
+
+			Td calculate(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y){
+				// Calculate sample losses
+				sample_losses = forward(output, y);
+
+				// Calculate mean loss
+				data_losses = numerics::mean(sample_losses);
+				return data_losses;
+
+			}
+
+
+	};
+
+
+
+} // end namespace neural_networks

include/modules/neural_networks/loss/loss.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "./core/omp_config.h"
+
+#include "./utils/vector.h"
+#include "./utils/matrix.h"
+
+
+namespace neural_networks{
+
+	template <typename Td, typename Ti>
+	struct Loss{
+
+			utils::Matrix<Td> sample_losses;
+			Td data_losses;
+
+			virtual utils::Vector<Td> forward(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y) = 0;
+
+			Td calculate(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y){
+				// Calculate sample losses
+				sample_losses = forward(output, y);
+
+				// Calculate mean loss
+				data_losses = numerics::mean(sample_losses);
+				return data_losses;
+
+			}
+
+
+	};
+
+
+
+} // end namespace neural_networks

include/modules/neural_networks/neural_networks.h

@@ -6,4 +6,7 @@
 #include "layers/dense_layer.h"
 
 
-#include "activation_functions/ReLU.h"
+#include "activation_functions/ReLU.h"
+#include "activation_functions/Softmax.h"
+
+#include "loss/loss.h"