Sync public subset from Flux (private)

include/modules/neural_networks/activation_functions/ReLU.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "./core/omp_config.h"
+
+#include "./utils/vector.h"
+#include "./utils/matrix.h"
+#include "./utils/random.h"
+
+
+namespace neural_networks{
+
+	template <typename T>
+	struct activation_ReLU{
+
+			utils::Matrix<T> outputs;
+
+	        void forward(utils::Matrix<T> inputs){
+			    outputs = numerics::max(inputs, T{0});
+			    //outputs.print();
+	        }
+
+
+
+	};
+
+
+
+} // end namespace neural_networks

include/modules/neural_networks/activation_functions/Softmax.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "./core/omp_config.h"
+
+#include "./utils/vector.h"
+#include "./utils/matrix.h"
+
+#include "./numerics/max.h"
+#include "./numerics/matsubtract.h"
+#include "./numerics/exponential.h"
+#include "./numerics/matdiv.h"
+
+
+namespace neural_networks{
+
+	template <typename T>
+	struct activation_softmax{
+
+			utils::Matrix<T> exp_values;
+			utils::Matrix<T> probabilities;
+			utils::Matrix<T> outputs;
+
+	        void forward(const utils::Matrix<T> inputs){
+	
+
+			    exp_values = numerics::exponential(numerics::matsubtract(inputs, numerics::max(inputs, "rows"), "col"));
+			    probabilities = numerics::matdiv(exp_values, numerics::matsum(exp_values, "col"), "col");
+
+			    outputs = probabilities;
+
+	        }
+
+	};
+
+
+
+} // end namespace neural_networks