Dropout Layer

Implemented rng::uniform and rng::binomial for single values, vectors and matrices. implemeted dropout layers and tested it. Also fixed the validation code. Before it used y one place, now it uses y_test as it should.

include/modules/neural_networks/layers/Dropout_Layer.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include "core/omp_config.h"
+
+#include "utils/vector.h"
+#include "utils/matrix.h"
+#include "random/random.h"
+
+
+namespace neural_networks{
+
+	template <typename T>
+	struct Dropout_Layer{
+
+		// Store rate, we invert it as for example for dropout
+		// of 0.1 we need a success rate of 0.9
+		T rate = T{0};
+		utils::Matrix<T> binary_mask;
+		
+		utils::Matrix<T> _inputs;
+		utils::Matrix<T> outputs;
+
+		utils::Matrix<T> dinputs;
+
+		
+		
+		// Default Constructor
+		Dropout_Layer() = default;
+
+		// Constructor
+    	Dropout_Layer(const T rate){
+    		this->rate = T{1} - rate;
+        	
+        }
+
+        void forward(const utils::Matrix<T>& inputs){
+        	// Save input values
+        	_inputs = inputs;
+
+        	// Generate binary_mask
+        	binary_mask = rng::binomial<T>(inputs.rows(), inputs.cols(), 1, rate);
+        	// Scale binary_mask
+        	binary_mask = numerics::div(binary_mask, rate);
+
+        	// Apply binary mask to output values
+        	outputs = numerics::mul(binary_mask, inputs);
+
+        }
+
+        void backward(const utils::Matrix<T>& dvalues){
+        	dinputs = numerics::mul(dvalues, binary_mask);
+        }
+
+	};
+
+
+
+} // end namespace neural_networks

include/modules/neural_networks/neural_networks.h

@@ -6,6 +6,7 @@
 
 
 #include "layers/Dense_Layer.h"
+#include "layers/Dropout_Layer.h"
 
 
 #include "activation_functions/Activation_ReLU.h"

include/random/binomial.h

@@ -0,0 +1,42 @@
+#pragma once
+
+
+#include "detail/binomial_serial.h"
+
+namespace rng {
+
+    template <typename T>
+    T binomial(const uint64_t trials, const double probability) {
+        return rng::detail::binomial_serial<T>(trials, probability);
+    }
+
+    template <typename T>
+    utils::Vector<T> binomial(const uint64_t size, const uint64_t trials, const double probability) {
+        return rng::detail::binomial_serial<T>(size, trials, probability);
+    }
+
+    template <typename T>
+    utils::Matrix<T> binomial(const uint64_t rows, const uint64_t cols, const uint64_t trials, const double probability) {
+        return rng::detail::binomial_serial<T>(rows, cols, trials, probability);
+    }
+
+}
+
+
+/*
+    std::cout << rng::uniform(-1.0f, 1.0f) << std::endl;
+
+    utils::Vector<float> random1 = rng::uniform(3, -1.0f, 1.0f);
+    random1.print();
+
+    utils::Matrix<float> random2 = rng::uniform(3, 3, -1.0f, 1.0f);
+    random2.print();
+
+    std::cout << rng::binomial<uint16_t>(1, 0.5) << std::endl;
+    
+    utils::Vector<uint16_t> random3 = rng::binomial<uint16_t>(3, 1, 0.5);
+    random3.print();
+
+    utils::Matrix<uint16_t> random4 = rng::binomial<uint16_t>(3, 3, 1, 0.5);
+    random4.print();
+    */

include/random/detail/binomial_serial.h

@@ -0,0 +1,45 @@
+#pragma once
+
+#include "random/engine.h"
+
+#include <random>
+//#include <type_traits>
+
+#include "utils/matrix.h"
+#include "utils/vector.h"
+
+namespace rng::detail {
+
+
+    template <typename T>
+    inline T binomial_serial(const uint64_t trials, const double probability) {
+        std::binomial_distribution<int> dist(trials, probability);
+
+        return static_cast<T>(dist(rng::engine()));
+    }
+
+    template <typename T>
+    utils::Vector<T> binomial_serial(const uint64_t size, const uint64_t trials, const double probability){
+    	utils::Vector<T> v(size);
+    	for (uint64_t i = 0; i < size; ++i){
+    		v[i] = binomial_serial<T>(trials, probability);
+    	}
+    	return v;
+    }
+
+    template <typename T>
+    utils::Matrix<T> binomial_serial(const uint64_t rows, const uint64_t cols, const uint64_t trials, const double probability){
+    	utils::Matrix<T> A(rows, cols);
+    	for (uint64_t i = 0; i < rows; ++i){
+    		for (uint64_t j = 0; j < cols; ++j){
+    			A(i,j) = binomial_serial<T>(trials, probability);
+    		}
+    	}
+    	return A;
+    }
+
+
+
+
+
+}

include/random/detail/uniform_serial.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#include "random/engine.h"
+
+#include <random>
+//#include <type_traits>
+
+#include "utils/matrix.h"
+#include "utils/vector.h"
+
+namespace rng::detail {
+
+	//
+	// Base functions int
+	//
+    template <typename T>
+    typename std::enable_if<std::is_integral<T>::value, T>::type
+    uniform_serial(const T low, const T high) {
+        std::uniform_int_distribution<T> dist(low, high);
+        return dist(rng::engine());
+    }
+
+	//
+	// Base functions float
+	//
+    template <typename T>
+    typename std::enable_if<std::is_floating_point<T>::value, T>::type
+    uniform_serial(const T low, const T high) {
+        std::uniform_real_distribution<T> dist(low, high);
+        return dist(rng::engine());
+    }
+
+    template <typename T>
+    utils::Vector<T> uniform_serial(const uint64_t size, const T low, const T high){
+    	utils::Vector<T> v(size);
+    	for (uint64_t i = 0; i < size; ++i){
+    		v[i] = uniform_serial(low, high);
+    	}
+    	return v;
+    }
+
+    template <typename T>
+    utils::Matrix<T> uniform_serial(const uint64_t rows, const uint64_t cols, const T low, const T high){
+    	utils::Matrix<T> A(rows, cols);
+    	for (uint64_t i = 0; i < rows; ++i){
+    		for (uint64_t j = 0; j < cols; ++j){
+    			A(i,j) = uniform_serial(low, high);
+    		}
+    	}
+    	return A;
+    }
+
+
+
+
+
+}

include/random/engine.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <random>
+#include <cstdint>
+
+namespace rng {
+
+    using seed_type = std::uint64_t;
+    using engine_type = std::mt19937_64;
+
+    inline engine_type& engine() {
+        static thread_local engine_type eng{std::random_device{}()};
+        return eng;
+    }
+
+    inline void seed(seed_type value) {
+        engine().seed(value);
+    }
+
+}

include/random/random.h

@@ -0,0 +1,11 @@
+// "./utils/utils.h"
+#pragma once
+
+
+
+#include "random/engine.h"
+
+#include "uniform.h"
+#include "binomial.h"
+
+

include/random/uniform.h

@@ -0,0 +1,28 @@
+#pragma once
+
+
+#include "detail/uniform_serial.h"
+
+namespace rng {
+
+    template <typename T>
+    T uniform(const T low, const T high) {
+        return rng::detail::uniform_serial(low, high);
+    }
+
+    template <typename T>
+    utils::Vector<T> uniform(const uint64_t size, const T low, const T high) {
+        return rng::detail::uniform_serial(size, low, high);
+    }
+
+    template <typename T>
+    utils::Matrix<T> uniform(const uint64_t rows, const uint64_t cols, const T low, const T high) {
+        return rng::detail::uniform_serial(rows, cols, low, high);
+    }
+
+
+
+
+}
+
+