Save

I've created the optimizer class

include/neural_network/activation/activation_relu.hpp

@@ -75,7 +75,6 @@ struct activation_relu : public layer{
      *
      * @param inputs Data input for forward pass.
      * 
-     * @Note Calculates -> outputs = inputs * weights + biases
      */
 	bool forward(const panic::tensor::real_matrix& input_data);
 

include/neural_network/activation/activation_softmax.hpp

@@ -75,7 +75,6 @@ struct activation_softmax : public layer{
      *
      * @param inputs Data input for forward pass.
      * 
-     * @Note Calculates -> outputs = inputs * weights + biases
      */
 	bool forward(const panic::tensor::real_matrix& input_data);
 

include/neural_network/activation_loss/activation_softmax_loss_categorical_crossentropy.hpp

@@ -0,0 +1,216 @@
+/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ *                                              PANIC
+ *                              Portable Algorithms and Numerics In C++
+ *
+ *                       Scientific computing from scratch, with feeling.
+ *
+ * Copyright (c) 2026 Michelle Bausager
+ *
+ * This file is part of PANIC.
+ *
+ * PANIC is free software licensed under the GNU General Public License v3.0 or later.
+ * You may redistribute and/or modify it under the terms of the GPL.
+ *
+ * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the LICENSE file for the full license text.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ * Project Name:      PANIC
+ * Module Name:       neural_network
+ * File Name:         activation_softmax_loss_categorical_crossentropy.Hpp
+ * Revision:          0.1.0
+ * Date:              28-07-2026
+ * Author:            Michelle Bausager
+ *
+ * Description:
+ * Defines the combined activation function softmax and 
+ * categorical crossentropy loss used in neural network
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
+#pragma once
+
+//---------------------------------------------------------------------------------------------------------------------------
+// INCLUDE DESCRIPTION
+//---------------------------------------------------------------------------------------------------------------------------
+#include <config/types.hpp> // panic::uint_t, panic::int_t, and panic::real_t
+#include <tensor/vector.hpp>
+#include <tensor/matrix.hpp>
+
+#include <neural_network/activation/activation_softmax.hpp>
+#include <neural_network/loss/loss_categorical_crossentropy.hpp>
+
+
+namespace panic{
+	namespace neural_network{
+
+/**
+ * @brief struct for activation_softmax_loss_categorical_crossentropy object used in neural networks
+ * 
+ *
+ * The struct is used in PANIC nural_network library.
+ */
+struct activation_softmax_loss_categorical_crossentropy: loss{
+
+     activation_softmax activation;
+     loss_categorical_crossentropy loss;
+
+     /**
+     * @brief Emphty matrix to store input data for bacward pass
+     *
+     */
+    panic::tensor::real_matrix dinputs;
+
+     /**
+     * @brief Emphty matrix to store output data
+     *
+     */
+    panic::tensor::real_matrix outputs;
+
+
+
+    /**
+     * @brief Empthy constructor
+     *
+     */
+	activation_softmax_loss_categorical_crossentropy();
+
+    /**
+     * @brief Default de-constructor
+     *
+     */
+	~activation_softmax_loss_categorical_crossentropy() = default;
+
+    /**
+     * @brief forward function to calculate losses
+     *
+     * @param y_pred Matrix of model predection.  
+     * @param y_true Vector of true label of data.
+     * 
+     */
+     bool forward(const panic::tensor::real_matrix& y_pred, const panic::tensor::uint_vector& y_true);
+
+    /**
+     * @brief forward function to calculate losses
+     *
+     * @param y_pred Matrix of model predection.  
+     * @param y_true Vector of true label of data.
+     * 
+     * @Note Overloaded if one-shot endcoded
+     *         is used.
+     */
+    bool forward(const panic::tensor::real_matrix& y_pred, const panic::tensor::real_matrix& y_true);
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+    /**
+     * @brief forward function to calculate losses
+     *
+     * @param y_pred Matrix of model predection.  
+     * @param y_true Vector of true label of data.
+     * 
+     */
+     bool calculate(const panic::tensor::real_matrix& y_pred, const panic::tensor::uint_vector& y_true);
+
+    /**
+     * @brief forward function to calculate losses
+     *
+     * @param y_pred Matrix of model predection.  
+     * @param y_true Vector of true label of data.
+     * 
+     * @Note Overloaded if one-shot endcoded
+     *         is used.
+     */
+    bool calculate(const panic::tensor::real_matrix& y_pred, const panic::tensor::real_matrix& y_true);
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+    /**
+     * @brief backward function to calculate from losses
+     *
+     * @param y_pred Matrix of model predection.  
+     * @param y_true Vector of true label of data.
+     * 
+     */
+    bool backward(const panic::tensor::real_matrix& dvalues, const panic::tensor::uint_vector& y_true);
+
+    /**
+     * @brief backward function to calculate from losses
+     *
+     * @param y_pred Matrix of model predection.  
+     * @param y_true Vector of true label of data.
+     * 
+     * @Note Overloaded if one-shot endcoded
+     *         is used.
+     */
+    bool backward(const panic::tensor::real_matrix& dvalues, const panic::tensor::real_matrix& y_true);
+
+
+
+
+};
+
+
+	} // namespace tensor
+} // namespace panic
+
+//---------------------------------------------------------------------------------------------------------------------------
+// VARIABLE DESCRIPTION
+//---------------------------------------------------------------------------------------------------------------------------
+
+//---------------------------------------------------------------------------------------------------------------------------
+// FUNCTION PROTOTYPE
+//---------------------------------------------------------------------------------------------------------------------------
+
+

include/neural_network/layer/layer_dense.hpp

@@ -37,7 +37,7 @@
 // INCLUDE DESCRIPTION
 //---------------------------------------------------------------------------------------------------------------------------
 #include <config/types.hpp> // panic::uint_t, panic::int_t, and panic::real_t
-#include <neural_network/layer/layer.hpp> // for base layer struct
+#include <neural_network/layer/trainable_layer.hpp> // for base trainable_layer struct
 #include <tensor/vector.hpp>
 #include <tensor/matrix.hpp>
 
@@ -56,7 +56,7 @@ namespace panic{
  *
  * The struct is used in PANIC nural_network library.
  */
-struct layer_dense : public layer{
+struct layer_dense : trainable_layer{
 
      /**
      * @brief Emphty matrix to store input data

include/neural_network/layer/trainable_layer.hpp

@@ -0,0 +1,78 @@
+/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ *                                              PANIC
+ *                              Portable Algorithms and Numerics In C++
+ *
+ *                       Scientific computing from scratch, with feeling.
+ *
+ * Copyright (c) 2026 Michelle Bausager
+ *
+ * This file is part of PANIC.
+ *
+ * PANIC is free software licensed under the GNU General Public License v3.0 or later.
+ * You may redistribute and/or modify it under the terms of the GPL.
+ *
+ * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the LICENSE file for the full license text.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ * Project Name:      PANIC
+ * Module Name:       neural_network
+ * File Name:         trainable_layer.hpp
+ * Revision:          0.1.0
+ * Date:              23-06-2026
+ * Author:            Michelle Bausager
+ *
+ * Description:
+ * Defines the base trainable_layer struct used in other layers in neural network
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
+#pragma once
+//---------------------------------------------------------------------------------------------------------------------------
+// INCLUDE DESCRIPTION
+//---------------------------------------------------------------------------------------------------------------------------
+#include <config/types.hpp> // panic::uint_t, panic::int_t, and panic::real_t
+#include <neural_network/layer/layer.hpp>
+#include <tensor/matrix.hpp> // panic::tensor::real_matrix (uint_matrix, int_matrix)
+
+
+namespace panic{
+	namespace neural_network{
+
+
+
+/**
+ * @brief Base trainable_layer for the rest of the neural network library to use
+ *
+ *    	This base trainable_layer should be used in all layers/activations that have trainable variables
+ * 		This is done so it's easy to make a list of layers in the model to loop over.
+ * 		The virtual means it should use derived object's version when called with a pointer.
+ * 		The =0 means the derivative object NEEDS to have these functions to work.
+ * 
+ * The struct is used for PANIC neural_network library.
+ */
+struct trainable_layer:layer{
+
+    panic::tensor::real_matrix weights;
+    panic::tensor::real_vector biases;
+
+    panic::tensor::real_matrix dweights;
+    panic::tensor::real_vector dbiases;
+
+
+	virtual ~trainable_layer() = default;
+
+
+};
+
+
+
+	} // namespace tensor
+} // namespace panic
+
+
+

include/neural_network/loss/loss.hpp

@@ -114,7 +114,7 @@ struct loss{
      * @param y_true Vector of true label of data.
      * 
      */
-	bool calculate(
+	virtual bool calculate(
         const panic::tensor::real_matrix& y_pred, 
         const panic::tensor::uint_vector& y_true);
 
@@ -125,7 +125,7 @@ struct loss{
      * @param y_true Matrix of true label of data.
      * 
      */
-    bool calculate(
+    virtual bool calculate(
         const panic::tensor::real_matrix& y_pred, 
         const panic::tensor::real_matrix& y_true);
 

include/neural_network/loss/loss_categorical_crossentropy.hpp

@@ -83,7 +83,6 @@ struct loss_categorical_crossentropy: loss{
         const panic::tensor::real_matrix& y_true);
 
 
-
     /**
      * @brief backward function to calculate from losses
      *

include/neural_network/model/model.hpp

@@ -40,8 +40,14 @@
 #include <tensor/matrix.hpp>                    // panic::tensor::real_matrix
 
 #include <neural_network/layer/layer.hpp>       // Base layer struct
+#include <neural_network/layer/trainable_layer.hpp>
 #include <neural_network/layer/layer_dense.hpp> // fully connected dense layer
 
+#include <neural_network/loss/loss.hpp>
+
+#include <neural_network/optimizers/optimizer.hpp>
+#include <neural_network/optimizers/optimizer_sgd.hpp>
+
 //---------------------------------------------------------------------------------------------------------------------------
 // TYPE DESCRIPTION
 //---------------------------------------------------------------------------------------------------------------------------
@@ -77,6 +83,22 @@ struct model{
      */
     layer** layers;
 
+
+    trainable_layer** trainable_layers;
+    panic::types::uint_t trainable_layer_count;
+
+    optimizer* optimizer_function;
+
+
+    /**
+     * @brief a pointer the loss function
+     *
+     * 
+     * @note The model owns these layers and deletes them in clear().
+     * 
+     */
+    loss* loss_function;
+
     // Number of layers currently stored in the model.
     /**
      * @brief Stores the number of layers
@@ -86,6 +108,8 @@ struct model{
 
     // model output (may be deleted and also used for debug)
     panic::tensor::real_matrix outputs;
+    // model dinputs (may be deleted and also used for debug)
+    panic::tensor::real_matrix dinputs;
 
     /**
      * @brief Empthy constructor
@@ -116,7 +140,21 @@ struct model{
      *  @note it adds an already-inplemented layer to the model.
      *
      */
-    bool add(layer* new_layer);
+    bool add_layer(layer* new_layer);
+
+    /**
+     * @brief Helper function for adding layers
+     * 
+     * Computes:
+     * @code
+     * layer_dense* new_layer = new layer_dense(3, 4);
+     * add(new_layer)
+     * @endcode
+     * 
+     *  @note it adds an already-inplemented layer to the model.
+     *
+     */
+    bool add_trainable_layer(trainable_layer* new_layer);
 
     /**
      * @brief Adds a dense layer to the model.
@@ -187,8 +225,99 @@ struct model{
      */
     bool forward(const panic::tensor::real_matrix& inputs);
 
-    // Delete all layers and reset the model.
 
+    /**
+     * @brief Loops over all layers backward function
+     *
+     * Computes:
+     * @code
+     * model.bacward(dvalues_data_matrix)
+     * @endcode
+     *
+     * @param dvalues diput data.
+     *
+     * @return true looped over every layer.
+     * 
+     *
+     */
+    bool backward(const panic::tensor::real_matrix& dvalues);
+
+
+    /**
+     * @brief Add loss for categorical crossentropy.
+     *
+     * Computes:
+     * @code
+     * model.add_loss_categorical_crossentropy();
+     * @endcode
+     *
+     *
+     * @return true if loss is added
+     *
+     * @note This function is convenient, but it allocates a new layer.
+     */
+    bool add_loss_categorical_crossentropy();
+
+
+
+    /**
+     * @brief Adds activation softmax AND loss for categorical crossentropy.
+     *
+     * Computes:
+     * @code
+     * model.activation_softmax_loss_categorical_crossentropy();
+     * @endcode
+     *
+     *
+     * @return true if activation and loss is added
+     *
+     * @note This function is convenient, but it allocates a new layer.
+     */
+    bool activation_softmax_loss_categorical_crossentropy();
+
+
+
+    /**
+     * @brief Adds optimizer_sgd to the model
+     *
+     * Computes:
+     * @code
+     * model.optimizer_sgd(1e-4)
+     * @endcode
+     *
+     * @param learning_rate Learning rate for update_param (default 1e-3).
+     *
+     * @return true If looped and optimized every trainable layer.
+     *  
+     *
+     */
+    bool add_optimizer_sgd(const panic::types::real_t learning_rate = static_cast<panic::types::real_t>(1e-3));
+
+
+
+
+
+    /**
+     * @brief Trains the model with input data
+     *
+     * Computes:
+     * @code
+     * model.backward(input_data_matrix)
+     * @endcode
+     *
+     * @param X_train Input X data for training.
+     * @param epochs Number of training iterations.
+     * @param print_every Prints every n iteration.
+     * 
+     *
+     * @return true if training is done correctly.
+     * 
+     *
+     */
+    bool train(const panic::tensor::real_matrix& X_train, 
+                    const panic::tensor::uint_vector& y_train,
+                    const panic::types::uint_t epochs, 
+                    const panic::types::uint_t print_every);
     /**
      * @brief Clears and deletes all layers and resets the model
      *

include/neural_network/optimizers/optimizer.hpp

@@ -0,0 +1,84 @@
+/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ *                                              PANIC
+ *                              Portable Algorithms and Numerics In C++
+ *
+ *                       Scientific computing from scratch, with feeling.
+ *
+ * Copyright (c) 2026 Michelle Bausager
+ *
+ * This file is part of PANIC.
+ *
+ * PANIC is free software licensed under the GNU General Public License v3.0 or later.
+ * You may redistribute and/or modify it under the terms of the GPL.
+ *
+ * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the LICENSE file for the full license text.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ * Project Name:      PANIC
+ * Module Name:       neural_network
+ * File Name:         optimizer.hpp
+ * Revision:          0.1.0
+ * Date:              23-06-2026
+ * Author:            Michelle Bausager
+ *
+ * Description:
+ * Defines the base optimizer struct used in neural network
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
+#pragma once
+//---------------------------------------------------------------------------------------------------------------------------
+// INCLUDE DESCRIPTION
+//---------------------------------------------------------------------------------------------------------------------------
+#include <config/types.hpp> // panic::uint_t, panic::int_t, and panic::real_t
+#include <neural_network/layer/trainable_layer.hpp>
+
+
+namespace panic{
+     namespace neural_network{
+
+
+
+/**
+ * @brief Base optimizer for the rest of the neural network library to use
+ *
+ *        This base optimizer should be used in neural networks
+ *        This is done so it's easy to optimize the trainable layers in the model to loop over.
+ *        The virtual means it should use derived object's version when called with a pointer.
+ *        The =0 means the derivative object NEEDS to have these functions to work.
+ * 
+ * The struct is used for PANIC neural_network library.
+ */
+struct optimizer{
+
+    /**
+     * @brief Default de-constructor
+     *
+     */
+     virtual ~optimizer() = default;
+
+
+    /**
+     * @brief Virtual forward function for derivative layers
+     *
+     * @param inputs Data matrix input for forward function.
+     * 
+     * @Note It's equal to 0 because it make the derivative 
+     *         object NEEDS to have these function to work.
+     */
+     virtual bool update_params(trainable_layer& layer) = 0;
+
+};
+
+
+
+     } // namespace tensor
+} // namespace panic
+
+
+

include/neural_network/optimizers/optimizer_sgd.hpp

@@ -0,0 +1,88 @@
+/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ *                                              PANIC
+ *                              Portable Algorithms and Numerics In C++
+ *
+ *                       Scientific computing from scratch, with feeling.
+ *
+ * Copyright (c) 2026 Michelle Bausager
+ *
+ * This file is part of PANIC.
+ *
+ * PANIC is free software licensed under the GNU General Public License v3.0 or later.
+ * You may redistribute and/or modify it under the terms of the GPL.
+ *
+ * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the LICENSE file for the full license text.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ * Project Name:      PANIC
+ * Module Name:       neural_network
+ * File Name:         optimizer_sgd.hpp
+ * Revision:          0.1.0
+ * Date:              04-08-2026
+ * Author:            Michelle Bausager
+ *
+ * Description:
+ * Defines the optimizer_sgd struct used in neural network
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
+#pragma once
+//---------------------------------------------------------------------------------------------------------------------------
+// INCLUDE DESCRIPTION
+//---------------------------------------------------------------------------------------------------------------------------
+#include <config/types.hpp>
+#include <neural_network/optimizers/optimizer.hpp>
+
+
+namespace panic{
+     namespace neural_network{
+
+
+
+/**
+ * @brief optimizer_sgd for the rest of the neural network library to use
+ *
+ */
+struct optimizer_sgd: optimizer{
+
+     panic::types::real_t learning_rate;
+
+
+    /**
+     * @brief Constructor
+     *
+     * @param learning_rate The learning rate for the optimization.
+     * 
+     */
+     optimizer_sgd(const panic::types::real_t learning_rate = static_cast<panic::types::real_t>(1e-3));
+
+
+    /**
+     * @brief Default de-constructor
+     *
+     */
+     ~optimizer_sgd() = default;
+
+
+    /**
+     * @brief Updates weights and biases in trainable  layers
+     *
+     * @param layer Trianable layer to update.
+     * 
+     */
+     bool update_params(trainable_layer& layer) override;
+
+};
+
+
+
+     } // namespace tensor
+} // namespace panic
+
+
+