Next up is dropout layers

include/neural_network/layer/layer_dense.hpp

@@ -71,7 +71,12 @@ struct layer_dense : trainable_layer{
      * @param neurons Amount of neurons in the layer
      * 
      */
-	layer_dense(panic::types::uint_t input_size, panic::types::uint_t neurons);
+	layer_dense(panic::types::uint_t input_size, 
+                panic::types::uint_t neurons,
+                panic::types::real_t weight_regularizer_l1 = 0,
+                panic::types::real_t weight_regularizer_l2 = 0,
+                panic::types::real_t bias_regularizer_l1 = 0,
+                panic::types::real_t bias_regularizer_l2 = 0);
 
     /**
      * @brief Default de-constructor

include/neural_network/layer/trainable_layer.hpp

@@ -63,6 +63,12 @@ struct trainable_layer:layer{
     panic::tensor::real_matrix dweights;
     panic::tensor::real_vector dbiases;
 
+    panic::types::real_t weight_regularizer_l1;
+    panic::types::real_t weight_regularizer_l2;
+
+    panic::types::real_t bias_regularizer_l1;
+    panic::types::real_t bias_regularizer_l2;
+
     /**
      * @brief Previous parameter updates used by momentum SGD.
      *
@@ -82,6 +88,9 @@ struct trainable_layer:layer{
     panic::tensor::real_vector bias_cache;
 
 
+
+
+
 	virtual ~trainable_layer() = default;
 
 

include/neural_network/loss/loss.hpp

@@ -39,6 +39,7 @@
 #include <tensor/matrix.hpp> // panic::tensor::real_matrix (uint_matrix, int_matrix)
 #include <tensor/vector.hpp>
 
+#include <neural_network/layer/trainable_layer.hpp>
 
 namespace panic{
 	namespace neural_network{
@@ -73,6 +74,11 @@ struct loss{
      */
     panic::types::real_t data_loss = 0;
 
+    /**
+     * @brief Regularization loss over a layer.
+     */
+    panic::types::real_t regularization_loss_value = 0;
+
     /**
      * @brief Gradient with respect to the loss input.
      *
@@ -172,6 +178,15 @@ struct loss{
         const panic::tensor::real_matrix& y_pred, 
         const panic::tensor::real_matrix& y_true);
 
+
+    /**
+     * @brief Caclculates the  regularization loss of a trainable layer
+     * 
+     */
+    bool regularization_loss(const trainable_layer& layer);
+
+
+
 };
 
 

include/neural_network/model/model.hpp

@@ -205,10 +205,12 @@ struct model{
      *
      * @note This function is convenient, but it allocates a new layer.
      */
-    bool add_layer_dense(
-        panic::types::uint_t input_size,
-        panic::types::uint_t neuron_count
-    );
+    bool add_layer_dense(panic::types::uint_t input_size, 
+                panic::types::uint_t neurons,
+                panic::types::real_t weight_regularizer_l1 = 0,
+                panic::types::real_t weight_regularizer_l2 = 0,
+                panic::types::real_t bias_regularizer_l1 = 0,
+                panic::types::real_t bias_regularizer_l2 = 0);
 
     /**
      * @brief Adds a activation ReLU layer to the model.
@@ -412,6 +414,18 @@ struct model{
      */
     bool optimize();
 
+    /**
+     * @brief  Calculates regulaization for parameters on trainable layers
+     *
+     * Computes:
+     * @code
+     * model.calculate_regularization_loss()
+     * @endcode
+     *
+     * @return true if optimization is done correctly.
+     *
+     */
+    bool calculate_regularization_loss(panic::types::real_t& regularization_loss);
 
 
     /**