Next up is dropout layers

include/math/abs.hpp

@@ -0,0 +1,163 @@
+/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ *                                              PANIC
+ *                              Portable Algorithms and Numerics In C++
+ *
+ *                       Scientific computing from scratch, with feeling.
+ *
+ * Copyright (c) 2026 Michelle Bausager
+ *
+ * This file is part of PANIC.
+ *
+ * PANIC is free software licensed under the GNU General Public License v3.0 or later.
+ * You may redistribute and/or modify it under the terms of the GPL.
+ *
+ * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the LICENSE file for the full license text.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ * Project Name:      PANIC
+ * Module Name:       math
+ * File Name:         abs.cpp
+ * Revision:          0.1.0
+ * Date:              07-08-2026
+ * Author:            Michelle Bausager
+ *
+ * Description:
+ * Functions to calculate the abs of value
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
+#pragma once
+
+#include <tensor/vector.hpp> // for panic::vector
+#include <tensor/matrix.hpp> // for panic::matrix
+
+namespace panic{
+namespace math{
+
+
+
+/**
+ * @brief calculates the abs of a value.
+ *
+ * Computes:
+ * @code
+ * abs(x, y)
+ * @endcode
+ *
+ * @tparam T Numeric element type.
+ * @param x Value to take the abs of.
+ * @param y Result.
+ *
+ *
+ * @note This function is omp-friendly.
+ */
+template <typename T>
+bool abs(const T x, T& y);
+
+
+/**
+ * @brief calculates the abs of a value.
+ *
+ * Computes:
+ * @code
+ * result = abs(k)
+ * @endcode
+ *
+ * @tparam T Numeric element type.
+ * @param x Value to take the abs of.
+ *
+ * @return The calculated value
+ *
+ * @note This function is omp-friendly.
+ */
+template <typename T>
+T abs(const T x);
+
+
+/**
+ * @brief Calculates the abs elementwise in a vector
+ *
+ * Computes:
+ * @code
+ * c[i] = abs(a[i])
+ * @endcode
+ *
+ * @tparam T Numeric element type.
+ * @param a Input vector.
+ * @param c Output vector. Resized to match @p a.
+ *
+ * @return true if @p c was resized and filled successfully.
+ * @return false if resizing @p c failed.
+ *
+ * @note This overload writes the result into an existing vector to avoid
+ *       unnecessary temporary allocations.
+ */
+template <typename T>
+bool abs(const panic::tensor::vector<T>& a, panic::tensor::vector<T>& c);
+
+/**
+ * @brief Calculates the abs elementwise in a vector
+ *
+ * Computes:
+ * @code
+ * result[i] = abs(a[i])
+ * @endcode
+ *
+ * @tparam T Numeric element type.
+ * @param a Input vector.
+ *
+ * @return A new vector containing the result.
+ * @return An empty vector if the operation fails.
+ *
+ * @note This overload is convenient, but may allocate a new vector.
+ */
+template <typename T>
+panic::tensor::vector<T> abs(const panic::tensor::vector<T>& a);
+
+/**
+ * @brief Calculates the abs elementwise of a matrix
+ *
+ * Computes:
+ * @code
+ * C(i,j) = abs(A(i,j))
+ * @endcode
+ *
+ * @tparam T Numeric element type.
+ * @param A Input matrix.
+ * @param C Output Matrix. Resized to match @p A.
+ *
+ * @return true if @p C was resized and filled successfully.
+ * @return false if resizing @p C failed.
+ *
+ * @note This overload writes the result into an existing vector to avoid
+ *       unnecessary temporary allocations.
+ */
+template <typename T>
+bool abs(const panic::tensor::matrix<T>& A, panic::tensor::matrix<T>& C);
+
+/**
+ * @brief Returns the calculated abs elementwise of the matrix
+ *
+ * Computes:
+ * @code
+ * result(i,j) = abs(A(i,j))
+ * @endcode
+ *
+ * @tparam T Numeric element type.
+ * @param A Input matrix.
+ *
+ * @return A new matrix containing the result.
+ * @return An empty matrix if the operation fails.
+ *
+ * @note This overload is convenient, but may allocate a new vector.
+ */
+template <typename T>
+panic::tensor::matrix<T> abs(const panic::tensor::matrix<T>& A);
+
+} // namespace math
+} // namespace panic

include/neural_network/layer/layer_dense.hpp

@@ -71,7 +71,12 @@ struct layer_dense : trainable_layer{
      * @param neurons Amount of neurons in the layer
      * 
      */
-	layer_dense(panic::types::uint_t input_size, panic::types::uint_t neurons);
+	layer_dense(panic::types::uint_t input_size, 
+                panic::types::uint_t neurons,
+                panic::types::real_t weight_regularizer_l1 = 0,
+                panic::types::real_t weight_regularizer_l2 = 0,
+                panic::types::real_t bias_regularizer_l1 = 0,
+                panic::types::real_t bias_regularizer_l2 = 0);
 
     /**
      * @brief Default de-constructor

include/neural_network/layer/trainable_layer.hpp

@@ -63,6 +63,12 @@ struct trainable_layer:layer{
     panic::tensor::real_matrix dweights;
     panic::tensor::real_vector dbiases;
 
+    panic::types::real_t weight_regularizer_l1;
+    panic::types::real_t weight_regularizer_l2;
+
+    panic::types::real_t bias_regularizer_l1;
+    panic::types::real_t bias_regularizer_l2;
+
     /**
      * @brief Previous parameter updates used by momentum SGD.
      *
@@ -82,6 +88,9 @@ struct trainable_layer:layer{
     panic::tensor::real_vector bias_cache;
 
 
+
+
+
 	virtual ~trainable_layer() = default;
 
 

include/neural_network/loss/loss.hpp

@@ -39,6 +39,7 @@
 #include <tensor/matrix.hpp> // panic::tensor::real_matrix (uint_matrix, int_matrix)
 #include <tensor/vector.hpp>
 
+#include <neural_network/layer/trainable_layer.hpp>
 
 namespace panic{
 	namespace neural_network{
@@ -73,6 +74,11 @@ struct loss{
      */
     panic::types::real_t data_loss = 0;
 
+    /**
+     * @brief Regularization loss over a layer.
+     */
+    panic::types::real_t regularization_loss_value = 0;
+
     /**
      * @brief Gradient with respect to the loss input.
      *
@@ -172,6 +178,15 @@ struct loss{
         const panic::tensor::real_matrix& y_pred, 
         const panic::tensor::real_matrix& y_true);
 
+
+    /**
+     * @brief Caclculates the  regularization loss of a trainable layer
+     * 
+     */
+    bool regularization_loss(const trainable_layer& layer);
+
+
+
 };
 
 

include/neural_network/model/model.hpp

@@ -205,10 +205,12 @@ struct model{
      *
      * @note This function is convenient, but it allocates a new layer.
      */
-    bool add_layer_dense(
-        panic::types::uint_t input_size,
-        panic::types::uint_t neuron_count
-    );
+    bool add_layer_dense(panic::types::uint_t input_size, 
+                panic::types::uint_t neurons,
+                panic::types::real_t weight_regularizer_l1 = 0,
+                panic::types::real_t weight_regularizer_l2 = 0,
+                panic::types::real_t bias_regularizer_l1 = 0,
+                panic::types::real_t bias_regularizer_l2 = 0);
 
     /**
      * @brief Adds a activation ReLU layer to the model.
@@ -412,6 +414,18 @@ struct model{
      */
     bool optimize();
 
+    /**
+     * @brief  Calculates regulaization for parameters on trainable layers
+     *
+     * Computes:
+     * @code
+     * model.calculate_regularization_loss()
+     * @endcode
+     *
+     * @return true if optimization is done correctly.
+     *
+     */
+    bool calculate_regularization_loss(panic::types::real_t& regularization_loss);
 
 
     /**