Save

I've created the optimizer class

include/neural_network/activation/activation_relu.hpp

@@ -75,7 +75,6 @@ struct activation_relu : public layer{
      *
      * @param inputs Data input for forward pass.
      * 
-     * @Note Calculates -> outputs = inputs * weights + biases
      */
 	bool forward(const panic::tensor::real_matrix& input_data);
 

include/neural_network/activation/activation_softmax.hpp

@@ -75,7 +75,6 @@ struct activation_softmax : public layer{
      *
      * @param inputs Data input for forward pass.
      * 
-     * @Note Calculates -> outputs = inputs * weights + biases
      */
 	bool forward(const panic::tensor::real_matrix& input_data);
 

include/neural_network/activation_loss/activation_softmax_loss_categorical_crossentropy.hpp

@@ -0,0 +1,216 @@
+/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ *                                              PANIC
+ *                              Portable Algorithms and Numerics In C++
+ *
+ *                       Scientific computing from scratch, with feeling.
+ *
+ * Copyright (c) 2026 Michelle Bausager
+ *
+ * This file is part of PANIC.
+ *
+ * PANIC is free software licensed under the GNU General Public License v3.0 or later.
+ * You may redistribute and/or modify it under the terms of the GPL.
+ *
+ * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the LICENSE file for the full license text.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ * Project Name:      PANIC
+ * Module Name:       neural_network
+ * File Name:         activation_softmax_loss_categorical_crossentropy.Hpp
+ * Revision:          0.1.0
+ * Date:              28-07-2026
+ * Author:            Michelle Bausager
+ *
+ * Description:
+ * Defines the combined activation function softmax and 
+ * categorical crossentropy loss used in neural network
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
+#pragma once
+
+//---------------------------------------------------------------------------------------------------------------------------
+// INCLUDE DESCRIPTION
+//---------------------------------------------------------------------------------------------------------------------------
+#include <config/types.hpp> // panic::uint_t, panic::int_t, and panic::real_t
+#include <tensor/vector.hpp>
+#include <tensor/matrix.hpp>
+
+#include <neural_network/activation/activation_softmax.hpp>
+#include <neural_network/loss/loss_categorical_crossentropy.hpp>
+
+
+namespace panic{
+	namespace neural_network{
+
+/**
+ * @brief struct for activation_softmax_loss_categorical_crossentropy object used in neural networks
+ * 
+ *
+ * The struct is used in PANIC nural_network library.
+ */
+struct activation_softmax_loss_categorical_crossentropy: loss{
+
+     activation_softmax activation;
+     loss_categorical_crossentropy loss;
+
+     /**
+     * @brief Emphty matrix to store input data for bacward pass
+     *
+     */
+    panic::tensor::real_matrix dinputs;
+
+     /**
+     * @brief Emphty matrix to store output data
+     *
+     */
+    panic::tensor::real_matrix outputs;
+
+
+
+    /**
+     * @brief Empthy constructor
+     *
+     */
+	activation_softmax_loss_categorical_crossentropy();
+
+    /**
+     * @brief Default de-constructor
+     *
+     */
+	~activation_softmax_loss_categorical_crossentropy() = default;
+
+    /**
+     * @brief forward function to calculate losses
+     *
+     * @param y_pred Matrix of model predection.  
+     * @param y_true Vector of true label of data.
+     * 
+     */
+     bool forward(const panic::tensor::real_matrix& y_pred, const panic::tensor::uint_vector& y_true);
+
+    /**
+     * @brief forward function to calculate losses
+     *
+     * @param y_pred Matrix of model predection.  
+     * @param y_true Vector of true label of data.
+     * 
+     * @Note Overloaded if one-shot endcoded
+     *         is used.
+     */
+    bool forward(const panic::tensor::real_matrix& y_pred, const panic::tensor::real_matrix& y_true);
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+    /**
+     * @brief forward function to calculate losses
+     *
+     * @param y_pred Matrix of model predection.  
+     * @param y_true Vector of true label of data.
+     * 
+     */
+     bool calculate(const panic::tensor::real_matrix& y_pred, const panic::tensor::uint_vector& y_true);
+
+    /**
+     * @brief forward function to calculate losses
+     *
+     * @param y_pred Matrix of model predection.  
+     * @param y_true Vector of true label of data.
+     * 
+     * @Note Overloaded if one-shot endcoded
+     *         is used.
+     */
+    bool calculate(const panic::tensor::real_matrix& y_pred, const panic::tensor::real_matrix& y_true);
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+    /**
+     * @brief backward function to calculate from losses
+     *
+     * @param y_pred Matrix of model predection.  
+     * @param y_true Vector of true label of data.
+     * 
+     */
+    bool backward(const panic::tensor::real_matrix& dvalues, const panic::tensor::uint_vector& y_true);
+
+    /**
+     * @brief backward function to calculate from losses
+     *
+     * @param y_pred Matrix of model predection.  
+     * @param y_true Vector of true label of data.
+     * 
+     * @Note Overloaded if one-shot endcoded
+     *         is used.
+     */
+    bool backward(const panic::tensor::real_matrix& dvalues, const panic::tensor::real_matrix& y_true);
+
+
+
+
+};
+
+
+	} // namespace tensor
+} // namespace panic
+
+//---------------------------------------------------------------------------------------------------------------------------
+// VARIABLE DESCRIPTION
+//---------------------------------------------------------------------------------------------------------------------------
+
+//---------------------------------------------------------------------------------------------------------------------------
+// FUNCTION PROTOTYPE
+//---------------------------------------------------------------------------------------------------------------------------
+
+

include/neural_network/layer/layer_dense.hpp

@@ -37,7 +37,7 @@
 // INCLUDE DESCRIPTION
 //---------------------------------------------------------------------------------------------------------------------------
 #include <config/types.hpp> // panic::uint_t, panic::int_t, and panic::real_t
-#include <neural_network/layer/layer.hpp> // for base layer struct
+#include <neural_network/layer/trainable_layer.hpp> // for base trainable_layer struct
 #include <tensor/vector.hpp>
 #include <tensor/matrix.hpp>
 
@@ -56,7 +56,7 @@ namespace panic{
  *
  * The struct is used in PANIC nural_network library.
  */
-struct layer_dense : public layer{
+struct layer_dense : trainable_layer{
 
      /**
      * @brief Emphty matrix to store input data

include/neural_network/layer/trainable_layer.hpp

@@ -0,0 +1,78 @@
+/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ *                                              PANIC
+ *                              Portable Algorithms and Numerics In C++
+ *
+ *                       Scientific computing from scratch, with feeling.
+ *
+ * Copyright (c) 2026 Michelle Bausager
+ *
+ * This file is part of PANIC.
+ *
+ * PANIC is free software licensed under the GNU General Public License v3.0 or later.
+ * You may redistribute and/or modify it under the terms of the GPL.
+ *
+ * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the LICENSE file for the full license text.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ * Project Name:      PANIC
+ * Module Name:       neural_network
+ * File Name:         trainable_layer.hpp
+ * Revision:          0.1.0
+ * Date:              23-06-2026
+ * Author:            Michelle Bausager
+ *
+ * Description:
+ * Defines the base trainable_layer struct used in other layers in neural network
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
+#pragma once
+//---------------------------------------------------------------------------------------------------------------------------
+// INCLUDE DESCRIPTION
+//---------------------------------------------------------------------------------------------------------------------------
+#include <config/types.hpp> // panic::uint_t, panic::int_t, and panic::real_t
+#include <neural_network/layer/layer.hpp>
+#include <tensor/matrix.hpp> // panic::tensor::real_matrix (uint_matrix, int_matrix)
+
+
+namespace panic{
+	namespace neural_network{
+
+
+
+/**
+ * @brief Base trainable_layer for the rest of the neural network library to use
+ *
+ *    	This base trainable_layer should be used in all layers/activations that have trainable variables
+ * 		This is done so it's easy to make a list of layers in the model to loop over.
+ * 		The virtual means it should use derived object's version when called with a pointer.
+ * 		The =0 means the derivative object NEEDS to have these functions to work.
+ * 
+ * The struct is used for PANIC neural_network library.
+ */
+struct trainable_layer:layer{
+
+    panic::tensor::real_matrix weights;
+    panic::tensor::real_vector biases;
+
+    panic::tensor::real_matrix dweights;
+    panic::tensor::real_vector dbiases;
+
+
+	virtual ~trainable_layer() = default;
+
+
+};
+
+
+
+	} // namespace tensor
+} // namespace panic
+
+
+

include/neural_network/loss/loss.hpp

@@ -114,7 +114,7 @@ struct loss{
      * @param y_true Vector of true label of data.
      * 
      */
-	bool calculate(
+	virtual bool calculate(
         const panic::tensor::real_matrix& y_pred, 
         const panic::tensor::uint_vector& y_true);
 
@@ -125,7 +125,7 @@ struct loss{
      * @param y_true Matrix of true label of data.
      * 
      */
-    bool calculate(
+    virtual bool calculate(
         const panic::tensor::real_matrix& y_pred, 
         const panic::tensor::real_matrix& y_true);
 

include/neural_network/loss/loss_categorical_crossentropy.hpp

@@ -83,7 +83,6 @@ struct loss_categorical_crossentropy: loss{
         const panic::tensor::real_matrix& y_true);
 
 
-
     /**
      * @brief backward function to calculate from losses
      *

include/neural_network/model/model.hpp

@@ -40,8 +40,14 @@
 #include <tensor/matrix.hpp>                    // panic::tensor::real_matrix
 
 #include <neural_network/layer/layer.hpp>       // Base layer struct
+#include <neural_network/layer/trainable_layer.hpp>
 #include <neural_network/layer/layer_dense.hpp> // fully connected dense layer
 
+#include <neural_network/loss/loss.hpp>
+
+#include <neural_network/optimizers/optimizer.hpp>
+#include <neural_network/optimizers/optimizer_sgd.hpp>
+
 //---------------------------------------------------------------------------------------------------------------------------
 // TYPE DESCRIPTION
 //---------------------------------------------------------------------------------------------------------------------------
@@ -77,6 +83,22 @@ struct model{
      */
     layer** layers;
 
+
+    trainable_layer** trainable_layers;
+    panic::types::uint_t trainable_layer_count;
+
+    optimizer* optimizer_function;
+
+
+    /**
+     * @brief a pointer the loss function
+     *
+     * 
+     * @note The model owns these layers and deletes them in clear().
+     * 
+     */
+    loss* loss_function;
+
     // Number of layers currently stored in the model.
     /**
      * @brief Stores the number of layers
@@ -86,6 +108,8 @@ struct model{
 
     // model output (may be deleted and also used for debug)
     panic::tensor::real_matrix outputs;
+    // model dinputs (may be deleted and also used for debug)
+    panic::tensor::real_matrix dinputs;
 
     /**
      * @brief Empthy constructor
@@ -116,7 +140,21 @@ struct model{
      *  @note it adds an already-inplemented layer to the model.
      *
      */
-    bool add(layer* new_layer);
+    bool add_layer(layer* new_layer);
+
+    /**
+     * @brief Helper function for adding layers
+     * 
+     * Computes:
+     * @code
+     * layer_dense* new_layer = new layer_dense(3, 4);
+     * add(new_layer)
+     * @endcode
+     * 
+     *  @note it adds an already-inplemented layer to the model.
+     *
+     */
+    bool add_trainable_layer(trainable_layer* new_layer);
 
     /**
      * @brief Adds a dense layer to the model.
@@ -187,8 +225,99 @@ struct model{
      */
     bool forward(const panic::tensor::real_matrix& inputs);
 
-    // Delete all layers and reset the model.
 
+    /**
+     * @brief Loops over all layers backward function
+     *
+     * Computes:
+     * @code
+     * model.bacward(dvalues_data_matrix)
+     * @endcode
+     *
+     * @param dvalues diput data.
+     *
+     * @return true looped over every layer.
+     * 
+     *
+     */
+    bool backward(const panic::tensor::real_matrix& dvalues);
+
+
+    /**
+     * @brief Add loss for categorical crossentropy.
+     *
+     * Computes:
+     * @code
+     * model.add_loss_categorical_crossentropy();
+     * @endcode
+     *
+     *
+     * @return true if loss is added
+     *
+     * @note This function is convenient, but it allocates a new layer.
+     */
+    bool add_loss_categorical_crossentropy();
+
+
+
+    /**
+     * @brief Adds activation softmax AND loss for categorical crossentropy.
+     *
+     * Computes:
+     * @code
+     * model.activation_softmax_loss_categorical_crossentropy();
+     * @endcode
+     *
+     *
+     * @return true if activation and loss is added
+     *
+     * @note This function is convenient, but it allocates a new layer.
+     */
+    bool activation_softmax_loss_categorical_crossentropy();
+
+
+
+    /**
+     * @brief Adds optimizer_sgd to the model
+     *
+     * Computes:
+     * @code
+     * model.optimizer_sgd(1e-4)
+     * @endcode
+     *
+     * @param learning_rate Learning rate for update_param (default 1e-3).
+     *
+     * @return true If looped and optimized every trainable layer.
+     *  
+     *
+     */
+    bool add_optimizer_sgd(const panic::types::real_t learning_rate = static_cast<panic::types::real_t>(1e-3));
+
+
+
+
+
+    /**
+     * @brief Trains the model with input data
+     *
+     * Computes:
+     * @code
+     * model.backward(input_data_matrix)
+     * @endcode
+     *
+     * @param X_train Input X data for training.
+     * @param epochs Number of training iterations.
+     * @param print_every Prints every n iteration.
+     * 
+     *
+     * @return true if training is done correctly.
+     * 
+     *
+     */
+    bool train(const panic::tensor::real_matrix& X_train, 
+                    const panic::tensor::uint_vector& y_train,
+                    const panic::types::uint_t epochs, 
+                    const panic::types::uint_t print_every);
     /**
      * @brief Clears and deletes all layers and resets the model
      *

include/neural_network/optimizers/optimizer.hpp

@@ -0,0 +1,84 @@
+/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ *                                              PANIC
+ *                              Portable Algorithms and Numerics In C++
+ *
+ *                       Scientific computing from scratch, with feeling.
+ *
+ * Copyright (c) 2026 Michelle Bausager
+ *
+ * This file is part of PANIC.
+ *
+ * PANIC is free software licensed under the GNU General Public License v3.0 or later.
+ * You may redistribute and/or modify it under the terms of the GPL.
+ *
+ * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the LICENSE file for the full license text.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ * Project Name:      PANIC
+ * Module Name:       neural_network
+ * File Name:         optimizer.hpp
+ * Revision:          0.1.0
+ * Date:              23-06-2026
+ * Author:            Michelle Bausager
+ *
+ * Description:
+ * Defines the base optimizer struct used in neural network
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
+#pragma once
+//---------------------------------------------------------------------------------------------------------------------------
+// INCLUDE DESCRIPTION
+//---------------------------------------------------------------------------------------------------------------------------
+#include <config/types.hpp> // panic::uint_t, panic::int_t, and panic::real_t
+#include <neural_network/layer/trainable_layer.hpp>
+
+
+namespace panic{
+     namespace neural_network{
+
+
+
+/**
+ * @brief Base optimizer for the rest of the neural network library to use
+ *
+ *        This base optimizer should be used in neural networks
+ *        This is done so it's easy to optimize the trainable layers in the model to loop over.
+ *        The virtual means it should use derived object's version when called with a pointer.
+ *        The =0 means the derivative object NEEDS to have these functions to work.
+ * 
+ * The struct is used for PANIC neural_network library.
+ */
+struct optimizer{
+
+    /**
+     * @brief Default de-constructor
+     *
+     */
+     virtual ~optimizer() = default;
+
+
+    /**
+     * @brief Virtual forward function for derivative layers
+     *
+     * @param inputs Data matrix input for forward function.
+     * 
+     * @Note It's equal to 0 because it make the derivative 
+     *         object NEEDS to have these function to work.
+     */
+     virtual bool update_params(trainable_layer& layer) = 0;
+
+};
+
+
+
+     } // namespace tensor
+} // namespace panic
+
+
+

include/neural_network/optimizers/optimizer_sgd.hpp

@@ -0,0 +1,88 @@
+/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ *                                              PANIC
+ *                              Portable Algorithms and Numerics In C++
+ *
+ *                       Scientific computing from scratch, with feeling.
+ *
+ * Copyright (c) 2026 Michelle Bausager
+ *
+ * This file is part of PANIC.
+ *
+ * PANIC is free software licensed under the GNU General Public License v3.0 or later.
+ * You may redistribute and/or modify it under the terms of the GPL.
+ *
+ * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the LICENSE file for the full license text.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ * Project Name:      PANIC
+ * Module Name:       neural_network
+ * File Name:         optimizer_sgd.hpp
+ * Revision:          0.1.0
+ * Date:              04-08-2026
+ * Author:            Michelle Bausager
+ *
+ * Description:
+ * Defines the optimizer_sgd struct used in neural network
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
+#pragma once
+//---------------------------------------------------------------------------------------------------------------------------
+// INCLUDE DESCRIPTION
+//---------------------------------------------------------------------------------------------------------------------------
+#include <config/types.hpp>
+#include <neural_network/optimizers/optimizer.hpp>
+
+
+namespace panic{
+     namespace neural_network{
+
+
+
+/**
+ * @brief optimizer_sgd for the rest of the neural network library to use
+ *
+ */
+struct optimizer_sgd: optimizer{
+
+     panic::types::real_t learning_rate;
+
+
+    /**
+     * @brief Constructor
+     *
+     * @param learning_rate The learning rate for the optimization.
+     * 
+     */
+     optimizer_sgd(const panic::types::real_t learning_rate = static_cast<panic::types::real_t>(1e-3));
+
+
+    /**
+     * @brief Default de-constructor
+     *
+     */
+     ~optimizer_sgd() = default;
+
+
+    /**
+     * @brief Updates weights and biases in trainable  layers
+     *
+     * @param layer Trianable layer to update.
+     * 
+     */
+     bool update_params(trainable_layer& layer) override;
+
+};
+
+
+
+     } // namespace tensor
+} // namespace panic
+
+
+

main.cpp

@@ -63,6 +63,7 @@
 #include <math/argmax.hpp>
 #include <math/equal.hpp>
 #include <math/mean.hpp>
+#include <neural_network/activation_loss/activation_softmax_loss_categorical_crossentropy.hpp>
 
 #include <math.h>
 
@@ -735,32 +736,15 @@ int main(void) {
     // Create activation softmax layer
     mymodel.add_activation_softmax();
 
-    // create loss function
-    panic::neural_network::loss_categorical_crossentropy loss_function;
-
-    mymodel.forward(X);
-
-    loss_function.calculate(mymodel.outputs, y);
-
-    panic::tensor::uint_vector prediction;
-    prediction = panic::math::argmax_rowwise(mymodel.outputs);
-
-    panic::types::real_t accuracy;
-    panic::tensor::uint_vector comparisons;
-
-    comparisons = panic::math::equal(prediction, y);
-
-    accuracy = panic::math::mean(comparisons);
-
-    std::cout << "loss: " << loss_function.data_loss << std::endl;
-
-    std::cout << "acc: " << accuracy << std::endl;
-
-
+    mymodel.add_loss_categorical_crossentropy();
+    //mymodel.activation_softmax_loss_categorical_crossentropy();
 
+    mymodel.add_optimizer_sgd();
     
+    panic::types::uint_t epochs = 10;
+    panic::types::uint_t print_every = 1;
 
-    
+    mymodel.train(X, y, epochs, print_every);
 
    
 

src/neural_network/activation_loss/activation_softmax_loss_categorical_crossentropy.cpp

@@ -0,0 +1,267 @@
+/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ *                                              PANIC
+ *                              Portable Algorithms and Numerics In C++
+ *
+ *                       Scientific computing from scratch, with feeling.
+ *
+ * Copyright (c) 2026 Michelle Bausager
+ *
+ * This file is part of PANIC.
+ *
+ * PANIC is free software licensed under the GNU General Public License v3.0 or later.
+ * You may redistribute and/or modify it under the terms of the GPL.
+ *
+ * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the LICENSE file for the full license text.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ * Project Name:      PANIC
+ * Module Name:       neural_network
+ * File Name:         activation_softmax_loss_categorical_crossentropy.cpp
+ * Revision:          0.1.0
+ * Date:              28-07-2026
+ * Author:            Michelle Bausager
+ *
+ * Description:
+ * Defines the combined activation function softmax and 
+ * categorical crossentropy loss used in neural network
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
+
+//---------------------------------------------------------------------------------------------------------------------------
+// INCLUDE DESCRIPTION
+//---------------------------------------------------------------------------------------------------------------------------
+#include <neural_network/activation_loss/activation_softmax_loss_categorical_crossentropy.hpp>
+#include <config/omp.hpp>
+
+#include <math/mul.hpp>
+#include <math/argmax.hpp>
+
+
+//---------------------------------------------------------------------------------------------------------------------------
+// PRIVATE CONSTANTS
+//---------------------------------------------------------------------------------------------------------------------------
+/**
+ * @brief Minimum number of element operations before using the OpenMP-enabled loop.
+ *
+ * Small vectors and matrices are kept serial because the overhead of starting
+ * worker threads can be larger than the work itself.
+ */
+static const panic::types::uint_t activation_softmax_loss_categorical_crossentropy_omp_min_size = 500;
+//---------------------------------------------------------------------------------------------------------------------------
+// INPLEMENTATION
+//---------------------------------------------------------------------------------------------------------------------------
+
+namespace panic{
+	namespace neural_network{
+
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Constructor Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy
+//
+// Description:
+//		Creates an empty layer.
+//--------------------------------------------------------------------------------------------------------------------------
+activation_softmax_loss_categorical_crossentropy::activation_softmax_loss_categorical_crossentropy() {
+}
+
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Function Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy.forward
+//
+// Description:
+//		Calculated the forward pass
+//--------------------------------------------------------------------------------------------------------------------------
+bool activation_softmax_loss_categorical_crossentropy::forward(const panic::tensor::real_matrix& y_pred, const panic::tensor::uint_vector& y_true){
+
+	// Output layers activation function
+	if (!activation.forward(y_pred)){
+	 	return false;
+	 } 
+	// Set the output
+	outputs = activation.outputs;
+
+	// calculate the loss value.
+	if (!loss.calculate(outputs, y_true)){
+		return false;
+	}
+	
+	return true;
+}
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Function Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy.forward
+//
+// Description:
+//		Calculated the forward pass
+//--------------------------------------------------------------------------------------------------------------------------
+bool activation_softmax_loss_categorical_crossentropy::forward(const panic::tensor::real_matrix& y_pred, const panic::tensor::real_matrix& y_true){
+
+	// Output layers activation function
+	if (!activation.forward(y_pred)){
+	 	return false;
+	 } 
+	// Set the output
+	outputs = activation.outputs;
+
+	// calculate the loss value.
+	if (!loss.calculate(outputs, y_true)){
+		return false;
+	}
+	
+	return true;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Function Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy.calculate
+//
+// Description:
+//		Calculated the calculate pass
+//--------------------------------------------------------------------------------------------------------------------------
+bool activation_softmax_loss_categorical_crossentropy::calculate(const panic::tensor::real_matrix& y_pred, const panic::tensor::uint_vector& y_true){
+
+	// Output layers activation function
+	if (!activation.forward(y_pred)){
+	 	return false;
+	 } 
+
+	// Set the output
+	outputs = activation.outputs;
+
+	// calculate the loss value.
+	if (!loss.calculate(outputs, y_true)){
+		return false;
+	}
+	
+	data_loss = loss.data_loss;
+
+	return true;
+}
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Function Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy.calculate
+//
+// Description:
+//		Calculated the calculate pass
+//--------------------------------------------------------------------------------------------------------------------------
+bool activation_softmax_loss_categorical_crossentropy::calculate(const panic::tensor::real_matrix& y_pred, const panic::tensor::real_matrix& y_true){
+
+	// Output layers activation function
+	if (!activation.forward(y_pred)){
+	 	return false;
+	 } 
+	// Set the output
+	outputs = activation.outputs;
+
+	// calculate the loss value.
+	if (!loss.calculate(outputs, y_true)){
+		return false;
+	}
+
+	data_loss = loss.data_loss;
+	
+	return true;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Function Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy.backward
+//
+// Description:
+//      Default implementation for catecorical labels.
+//--------------------------------------------------------------------------------------------------------------------------
+bool activation_softmax_loss_categorical_crossentropy::backward(const panic::tensor::real_matrix& dvalues, const panic::tensor::uint_vector& y_true){
+
+    const panic::types::uint_t samples = dvalues.rows();
+    const panic::types::uint_t classes = dvalues.cols();
+
+    if (samples == 0 || classes == 0 || y_true.size() != samples){
+        return false;
+    }
+
+    // Copy Softmax output
+    dinputs = dvalues;
+
+    // Subtract 1 from the correct class of every sample
+    PANIC_OMP_PARALLEL_FOR_IF(samples > activation_softmax_loss_categorical_crossentropy_omp_min_size)
+    for (panic::types::uint_t i = 0; i < samples; ++i){
+        dinputs(i, y_true[i]) -= static_cast<panic::types::real_t>(1);
+    }
+
+    // Scale to normalize gradients
+    const panic::types::real_t scale = static_cast<panic::types::real_t>(1) / static_cast<panic::types::real_t>(samples);
+
+	// Normalize gradients
+    if (!panic::math::mul(dinputs, scale, dinputs)){
+        return false;
+    }
+
+    return true;
+}
+
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Function Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy.backward
+//
+// Description:
+//      Default one-hot encoded labels implementation.
+//--------------------------------------------------------------------------------------------------------------------------
+bool activation_softmax_loss_categorical_crossentropy::backward(const panic::tensor::real_matrix& dvalues, const panic::tensor::real_matrix& y_true){
+
+    if (y_true.rows() != dvalues.rows() || y_true.cols() != dvalues.cols()){
+        return false;
+    }
+
+    panic::tensor::uint_vector categorical_labels;
+
+    if (!panic::math::argmax_rowwise(y_true, categorical_labels)){
+        return false;
+    }
+
+    return backward(dvalues, categorical_labels);
+}
+
+
+	} // namespace tensor
+} // namespace panic

src/neural_network/loss/loss_categorical_crossentropy.cpp

@@ -224,17 +224,6 @@ bool loss_categorical_crossentropy::backward(
 }
 
 
-
-
-
-
-
-
-
-
-
-
-
 	} // namespace tensor
 } // namespace panic
 

src/neural_network/model/model.cpp

@@ -45,6 +45,20 @@
 #include <neural_network/activation/activation_relu.hpp>
 #include <neural_network/activation/activation_softmax.hpp>
 
+#include <neural_network/loss/loss_categorical_crossentropy.hpp>
+
+#include <neural_network/activation_loss/activation_softmax_loss_categorical_crossentropy.hpp>
+
+#include <neural_network/optimizers/optimizer_sgd.hpp>    
+
+#include <math/argmax.hpp>
+#include <math/equal.hpp>
+#include <math/mean.hpp>
+
+
+// Remember ti disable
+#include <iostream> // for std::cout, std::endl
+
 //---------------------------------------------------------------------------------------------------------------------------
 // IMPLEMENTATION
 //---------------------------------------------------------------------------------------------------------------------------
@@ -63,9 +77,18 @@ model::model(){
     // No layers yet.
     layers = 0;
 
+    // No loss function yet.
+    loss_function = 0;
+
     // Number of layers is zero.
     layer_count = 0;
 
+
+    trainable_layers = 0;
+    trainable_layer_count = 0;
+
+    optimizer_function = 0;
+
 }
 
 
@@ -81,7 +104,7 @@ model::~model(){
 
 
 //--------------------------------------------------------------------------------------------------------------------------
-// Function Name : panic::neural_network::model::add
+// Function Name : panic::neural_network::model::add_layer
 //
 // Description:
 //      Adds a layer to the model.
@@ -89,7 +112,7 @@ model::~model(){
 //      The layer pointer array
 //      is resized every time a new layer is added.
 //--------------------------------------------------------------------------------------------------------------------------
-bool model::add(layer* new_layer){
+bool model::add_layer(layer* new_layer){
 
     // Do not add a null layer.
     if (new_layer == 0){
@@ -141,6 +164,74 @@ bool model::add(layer* new_layer){
 }
 
 
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Function Name : panic::neural_network::model::add_layer
+//
+// Description:
+//      Adds a layer to the model.
+//
+//      The layer pointer array
+//      is resized every time a new layer is added.
+//--------------------------------------------------------------------------------------------------------------------------
+bool model::add_trainable_layer(trainable_layer* new_layer){
+
+    // Do not add a null layer.
+    if (new_layer == 0){
+        return false;
+    }
+
+    if (!add_layer(new_layer)){
+        return false;
+    }
+
+    // The new array needs room for all old layers plus the new one.
+    const panic::types::uint_t new_trainable_layer_count = trainable_layer_count + 1;
+
+    // Allocate a new array of layer pointers.
+    //
+    // layer*  means "pointer to one layer"
+    // layer** means "pointer to many layer pointers"
+    //
+    // So this creates:
+    //
+    //      [ layer* ][ layer* ][ layer* ] ...
+    //
+    trainable_layer** new_trainable_layers = new trainable_layer*[new_trainable_layer_count];
+
+    // Copy the old layer pointers into the new array.
+    //
+    // Important:
+    //      This does not copy the layers themselves.
+    //      It only copies the addresses of the layers.
+
+    for (panic::types::uint_t i = 0; i < trainable_layer_count; ++i ){
+        new_trainable_layers[i] = trainable_layers[i];
+    }
+
+    // Put the new layer at the end.
+    new_trainable_layers[trainable_layer_count] = new_layer;
+
+    // Delete the old array of pointers.
+    //
+    // Important:
+    //      Do NOT delete layers[i] here.
+    //      The actual layer objects are still used in new_layers.
+    //
+    // This only deletes the old pointer array.
+    delete[] trainable_layers;
+
+
+    // Make the model use the new bigger array.
+    trainable_layers = new_trainable_layers;
+
+    // Update the layer count.
+    trainable_layer_count = new_trainable_layer_count;
+
+    return true;
+}
+
+
 //--------------------------------------------------------------------------------------------------------------------------
 // Function Name : panic::neural_network::model::add_layer_dense
 //
@@ -164,7 +255,7 @@ bool model::add_layer_dense(
     // Add it to the model.
     //
     // If add() fails, delete the layer so we do not leak memory.
-    if (!add(new_layer)){
+    if (!add_trainable_layer(new_layer)){
         delete new_layer;
         return false;
     }
@@ -192,7 +283,7 @@ bool model::add_activation_relu(){
     // Add it to the model.
     //
     // If add() fails, delete the layer so we do not leak memory.
-    if (!add(new_layer)){
+    if (!add_layer(new_layer)){
         delete new_layer;
         return false;
     }
@@ -221,7 +312,7 @@ bool model::add_activation_softmax(){
     // Add it to the model.
     //
     // If add() fails, delete the layer so we do not leak memory.
-    if (!add(new_layer)){
+    if (!add_layer(new_layer)){
         delete new_layer;
         return false;
     }
@@ -229,6 +320,9 @@ bool model::add_activation_softmax(){
     return true;
 }
 
+
+
+
 //--------------------------------------------------------------------------------------------------------------------------
 // Function Name : panic::neural_network::model::forward
 //
@@ -264,6 +358,126 @@ bool model::forward(const panic::tensor::real_matrix& inputs){
 }
 
 
+//--------------------------------------------------------------------------------------------------------------------------
+// Function Name : panic::neural_network::model::bacward
+//
+// Description:
+//      Runs the dinputs through every layer in reverse order.
+//--------------------------------------------------------------------------------------------------------------------------
+bool model::backward(const panic::tensor::real_matrix& dvalues){
+
+    return true;
+}
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Function Name : panic::neural_network::model::add_loss_categorical_crossentropy
+//
+// Description:
+//      Sets the loss function.
+//--------------------------------------------------------------------------------------------------------------------------
+bool model::add_loss_categorical_crossentropy(){
+
+    delete loss_function;
+
+    loss_function = new panic::neural_network::loss_categorical_crossentropy();
+
+    return true;
+}
+
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Function Name : panic::neural_network::model::activation_softmax_loss_categorical_crossentropy
+//
+// Description:
+//      Sets the loss function.
+//--------------------------------------------------------------------------------------------------------------------------
+bool model::activation_softmax_loss_categorical_crossentropy(){
+
+    delete loss_function;
+
+    loss_function = new panic::neural_network::activation_softmax_loss_categorical_crossentropy();
+
+    return true;
+}
+
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Function Name : panic::neural_network::model::add_optimizer_sgd
+//
+// Description:
+//      Adds Stocastient Gradient Decent as an optimizer to the model.
+//
+// Example:
+//      model.add_optimizer_sgd(1e-4);
+//--------------------------------------------------------------------------------------------------------------------------
+bool model::add_optimizer_sgd(const panic::types::real_t learning_rate){
+
+    optimizer_sgd* new_optimizer = new optimizer_sgd(learning_rate);
+
+    if (new_optimizer == 0){
+        return false;
+    }
+
+    delete optimizer_function;
+    optimizer_function = new_optimizer;
+
+    return true;
+}
+
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Function Name : panic::neural_network::model::train
+//
+// Description:
+//      Trains the model with input data.
+//--------------------------------------------------------------------------------------------------------------------------
+bool model::train(const panic::tensor::real_matrix& X_train, 
+                    const panic::tensor::uint_vector& y_train,
+                    const panic::types::uint_t epochs, 
+                    const panic::types::uint_t print_every){
+
+    panic::tensor::uint_vector prediction;
+    panic::types::real_t accuracy;
+    panic::tensor::uint_vector comparisons;
+
+    for (panic::types::uint_t epoch = 0; epoch < epochs; ++epoch){
+
+
+        forward(X_train);
+
+        if (!loss_function->calculate(outputs, y_train)){
+            return false;
+        }
+
+        
+        prediction = panic::math::argmax_rowwise(outputs);
+
+        comparisons = panic::math::equal(prediction, y_train);
+
+        accuracy = panic::math::mean(comparisons);
+
+        if (epoch % print_every == static_cast<panic::types::uint_t>(0)){
+            std::cout << "Epoch: " << epoch;
+            std::cout << "      loss: " << loss_function->data_loss;
+            std::cout << "      acc: " << accuracy << std::endl;
+        }
+
+
+        //backward();
+
+        //optimize();
+
+    }
+    return true;
+}
+
+
+
+
+
+
+
+
 //--------------------------------------------------------------------------------------------------------------------------
 // Function Name : panic::neural_network::model::clear
 //
@@ -284,10 +498,18 @@ void model::clear(){
         delete[] layers;
     }
 
+
     // Reset to empty state.
     layers = 0;
+    loss_function = 0;
     layer_count = 0;
 
+    delete[] trainable_layers;
+    trainable_layers = 0;
+    trainable_layer_count = 0;
+
+    delete optimizer_function;
+
     outputs.resize(0, 0);
 }
 

src/neural_network/optimizers/optimizer_sgd.cpp

@@ -0,0 +1,105 @@
+/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ *                                              PANIC
+ *                              Portable Algorithms and Numerics In C++
+ *
+ *                       Scientific computing from scratch, with feeling.
+ *
+ * Copyright (c) 2026 Michelle Bausager
+ *
+ * This file is part of PANIC.
+ *
+ * PANIC is free software licensed under the GNU General Public License v3.0 or later.
+ * You may redistribute and/or modify it under the terms of the GPL.
+ *
+ * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the LICENSE file for the full license text.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ *
+ * Project Name:      PANIC
+ * Module Name:       neural_network
+ * File Name:         optimizer_sgd.cpp
+ * Revision:          0.1.0
+ * Date:              04-08-2026
+ * Author:            Michelle Bausager
+ *
+ * Description:
+ * Defines the optimizer_sgd used in neural network
+ *
+ *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
+
+//---------------------------------------------------------------------------------------------------------------------------
+// INCLUDE DESCRIPTION
+//---------------------------------------------------------------------------------------------------------------------------
+#include <neural_network/optimizers/optimizer_sgd.hpp>
+#include <config/omp.hpp>
+
+#include <math/mul.hpp>
+#include <math/add.hpp>
+
+//---------------------------------------------------------------------------------------------------------------------------
+// PRIVATE CONSTANTS
+//---------------------------------------------------------------------------------------------------------------------------
+/**
+ * @brief Minimum number of element operations before using the OpenMP-enabled loop.
+ *
+ * Small vectors and matrices are kept serial because the overhead of starting
+ * worker threads can be larger than the work itself.
+ */
+static const panic::types::uint_t optimizer_sgd_omp_min_size = 500;
+//---------------------------------------------------------------------------------------------------------------------------
+// INPLEMENTATION
+//---------------------------------------------------------------------------------------------------------------------------
+
+namespace panic{
+	namespace neural_network{
+
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Constructor Name : panic::neural_network::optimizer_sgd
+//
+// Description:
+//		Constructor for optimizer_sgd.
+//--------------------------------------------------------------------------------------------------------------------------
+optimizer_sgd::optimizer_sgd(const panic::types::real_t learning_rate) {
+	this->learning_rate = learning_rate;
+}
+
+//--------------------------------------------------------------------------------------------------------------------------
+// Constructor Name : panic::neural_network::update_params
+//
+// Description:
+//		Updates weights and biases in layer.
+//--------------------------------------------------------------------------------------------------------------------------
+bool optimizer_sgd::update_params(trainable_layer& layer) {
+
+    panic::tensor::real_matrix weight_updates;
+    panic::tensor::real_vector bias_updates;
+
+    if (!panic::math::mul(layer.weights, -learning_rate, weight_updates)){
+    	return false;
+    }
+
+    if (!panic::math::add(layer.weights, weight_updates, layer.weights)){
+    	return false;
+    }
+
+    if (!panic::math::mul(layer.biases, -learning_rate, bias_updates)){
+    	return false;
+    }
+
+    if (!panic::math::add(layer.biases, bias_updates, layer.biases)){
+    	return false;
+    }
+
+    return true;
+	
+}
+
+
+	} // namespace tensor
+} // namespace panic