diff --git a/include/neural_network/activation/activation_relu.hpp b/include/neural_network/activation/activation_relu.hpp index 15330d6..2172932 100644 --- a/include/neural_network/activation/activation_relu.hpp +++ b/include/neural_network/activation/activation_relu.hpp @@ -75,7 +75,6 @@ struct activation_relu : public layer{ * * @param inputs Data input for forward pass. * - * @Note Calculates -> outputs = inputs * weights + biases */ bool forward(const panic::tensor::real_matrix& input_data); diff --git a/include/neural_network/activation/activation_softmax.hpp b/include/neural_network/activation/activation_softmax.hpp index 491df45..1c9fbec 100644 --- a/include/neural_network/activation/activation_softmax.hpp +++ b/include/neural_network/activation/activation_softmax.hpp @@ -75,7 +75,6 @@ struct activation_softmax : public layer{ * * @param inputs Data input for forward pass. * - * @Note Calculates -> outputs = inputs * weights + biases */ bool forward(const panic::tensor::real_matrix& input_data); diff --git a/include/neural_network/activation_loss/activation_softmax_loss_categorical_crossentropy.hpp b/include/neural_network/activation_loss/activation_softmax_loss_categorical_crossentropy.hpp new file mode 100644 index 0000000..6bec961 --- /dev/null +++ b/include/neural_network/activation_loss/activation_softmax_loss_categorical_crossentropy.hpp @@ -0,0 +1,216 @@ +/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + * + * PANIC + * Portable Algorithms and Numerics In C++ + * + * Scientific computing from scratch, with feeling. + * + * Copyright (c) 2026 Michelle Bausager + * + * This file is part of PANIC. + * + * PANIC is free software licensed under the GNU General Public License v3.0 or later. + * You may redistribute and/or modify it under the terms of the GPL. + * + * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the LICENSE file for the full license text. + * + * SPDX-License-Identifier: GPL-3.0-or-later + * + *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + * + * Project Name: PANIC + * Module Name: neural_network + * File Name: activation_softmax_loss_categorical_crossentropy.Hpp + * Revision: 0.1.0 + * Date: 28-07-2026 + * Author: Michelle Bausager + * + * Description: + * Defines the combined activation function softmax and + * categorical crossentropy loss used in neural network + * + *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ +#pragma once + +//--------------------------------------------------------------------------------------------------------------------------- +// INCLUDE DESCRIPTION +//--------------------------------------------------------------------------------------------------------------------------- +#include // panic::uint_t, panic::int_t, and panic::real_t +#include +#include + +#include +#include + + +namespace panic{ + namespace neural_network{ + +/** + * @brief struct for activation_softmax_loss_categorical_crossentropy object used in neural networks + * + * + * The struct is used in PANIC nural_network library. + */ +struct activation_softmax_loss_categorical_crossentropy: loss{ + + activation_softmax activation; + loss_categorical_crossentropy loss; + + /** + * @brief Emphty matrix to store input data for bacward pass + * + */ + panic::tensor::real_matrix dinputs; + + /** + * @brief Emphty matrix to store output data + * + */ + panic::tensor::real_matrix outputs; + + + + /** + * @brief Empthy constructor + * + */ + activation_softmax_loss_categorical_crossentropy(); + + /** + * @brief Default de-constructor + * + */ + ~activation_softmax_loss_categorical_crossentropy() = default; + + /** + * @brief forward function to calculate losses + * + * @param y_pred Matrix of model predection. + * @param y_true Vector of true label of data. + * + */ + bool forward(const panic::tensor::real_matrix& y_pred, const panic::tensor::uint_vector& y_true); + + /** + * @brief forward function to calculate losses + * + * @param y_pred Matrix of model predection. + * @param y_true Vector of true label of data. + * + * @Note Overloaded if one-shot endcoded + * is used. + */ + bool forward(const panic::tensor::real_matrix& y_pred, const panic::tensor::real_matrix& y_true); + + + + + + + + + + + + + + + + + + + /** + * @brief forward function to calculate losses + * + * @param y_pred Matrix of model predection. + * @param y_true Vector of true label of data. + * + */ + bool calculate(const panic::tensor::real_matrix& y_pred, const panic::tensor::uint_vector& y_true); + + /** + * @brief forward function to calculate losses + * + * @param y_pred Matrix of model predection. + * @param y_true Vector of true label of data. + * + * @Note Overloaded if one-shot endcoded + * is used. + */ + bool calculate(const panic::tensor::real_matrix& y_pred, const panic::tensor::real_matrix& y_true); + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /** + * @brief backward function to calculate from losses + * + * @param y_pred Matrix of model predection. + * @param y_true Vector of true label of data. + * + */ + bool backward(const panic::tensor::real_matrix& dvalues, const panic::tensor::uint_vector& y_true); + + /** + * @brief backward function to calculate from losses + * + * @param y_pred Matrix of model predection. + * @param y_true Vector of true label of data. + * + * @Note Overloaded if one-shot endcoded + * is used. + */ + bool backward(const panic::tensor::real_matrix& dvalues, const panic::tensor::real_matrix& y_true); + + + + +}; + + + } // namespace tensor +} // namespace panic + +//--------------------------------------------------------------------------------------------------------------------------- +// VARIABLE DESCRIPTION +//--------------------------------------------------------------------------------------------------------------------------- + +//--------------------------------------------------------------------------------------------------------------------------- +// FUNCTION PROTOTYPE +//--------------------------------------------------------------------------------------------------------------------------- + + diff --git a/include/neural_network/layer/layer_dense.hpp b/include/neural_network/layer/layer_dense.hpp index 0f39612..a4297fe 100644 --- a/include/neural_network/layer/layer_dense.hpp +++ b/include/neural_network/layer/layer_dense.hpp @@ -37,7 +37,7 @@ // INCLUDE DESCRIPTION //--------------------------------------------------------------------------------------------------------------------------- #include // panic::uint_t, panic::int_t, and panic::real_t -#include // for base layer struct +#include // for base trainable_layer struct #include #include @@ -56,7 +56,7 @@ namespace panic{ * * The struct is used in PANIC nural_network library. */ -struct layer_dense : public layer{ +struct layer_dense : trainable_layer{ /** * @brief Emphty matrix to store input data diff --git a/include/neural_network/layer/trainable_layer.hpp b/include/neural_network/layer/trainable_layer.hpp new file mode 100644 index 0000000..1566b16 --- /dev/null +++ b/include/neural_network/layer/trainable_layer.hpp @@ -0,0 +1,78 @@ +/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + * + * PANIC + * Portable Algorithms and Numerics In C++ + * + * Scientific computing from scratch, with feeling. + * + * Copyright (c) 2026 Michelle Bausager + * + * This file is part of PANIC. + * + * PANIC is free software licensed under the GNU General Public License v3.0 or later. + * You may redistribute and/or modify it under the terms of the GPL. + * + * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the LICENSE file for the full license text. + * + * SPDX-License-Identifier: GPL-3.0-or-later + * + *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + * + * Project Name: PANIC + * Module Name: neural_network + * File Name: trainable_layer.hpp + * Revision: 0.1.0 + * Date: 23-06-2026 + * Author: Michelle Bausager + * + * Description: + * Defines the base trainable_layer struct used in other layers in neural network + * + *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ +#pragma once +//--------------------------------------------------------------------------------------------------------------------------- +// INCLUDE DESCRIPTION +//--------------------------------------------------------------------------------------------------------------------------- +#include // panic::uint_t, panic::int_t, and panic::real_t +#include +#include // panic::tensor::real_matrix (uint_matrix, int_matrix) + + +namespace panic{ + namespace neural_network{ + + + +/** + * @brief Base trainable_layer for the rest of the neural network library to use + * + * This base trainable_layer should be used in all layers/activations that have trainable variables + * This is done so it's easy to make a list of layers in the model to loop over. + * The virtual means it should use derived object's version when called with a pointer. + * The =0 means the derivative object NEEDS to have these functions to work. + * + * The struct is used for PANIC neural_network library. + */ +struct trainable_layer:layer{ + + panic::tensor::real_matrix weights; + panic::tensor::real_vector biases; + + panic::tensor::real_matrix dweights; + panic::tensor::real_vector dbiases; + + + virtual ~trainable_layer() = default; + + +}; + + + + } // namespace tensor +} // namespace panic + + + diff --git a/include/neural_network/loss/loss.hpp b/include/neural_network/loss/loss.hpp index 03f322a..ff8c573 100644 --- a/include/neural_network/loss/loss.hpp +++ b/include/neural_network/loss/loss.hpp @@ -114,7 +114,7 @@ struct loss{ * @param y_true Vector of true label of data. * */ - bool calculate( + virtual bool calculate( const panic::tensor::real_matrix& y_pred, const panic::tensor::uint_vector& y_true); @@ -125,7 +125,7 @@ struct loss{ * @param y_true Matrix of true label of data. * */ - bool calculate( + virtual bool calculate( const panic::tensor::real_matrix& y_pred, const panic::tensor::real_matrix& y_true); diff --git a/include/neural_network/loss/loss_categorical_crossentropy.hpp b/include/neural_network/loss/loss_categorical_crossentropy.hpp index ce9f9d7..15710af 100644 --- a/include/neural_network/loss/loss_categorical_crossentropy.hpp +++ b/include/neural_network/loss/loss_categorical_crossentropy.hpp @@ -83,7 +83,6 @@ struct loss_categorical_crossentropy: loss{ const panic::tensor::real_matrix& y_true); - /** * @brief backward function to calculate from losses * diff --git a/include/neural_network/model/model.hpp b/include/neural_network/model/model.hpp index 8740c73..4af8f3f 100644 --- a/include/neural_network/model/model.hpp +++ b/include/neural_network/model/model.hpp @@ -40,8 +40,14 @@ #include // panic::tensor::real_matrix #include // Base layer struct +#include #include // fully connected dense layer +#include + +#include +#include + //--------------------------------------------------------------------------------------------------------------------------- // TYPE DESCRIPTION //--------------------------------------------------------------------------------------------------------------------------- @@ -77,6 +83,22 @@ struct model{ */ layer** layers; + + trainable_layer** trainable_layers; + panic::types::uint_t trainable_layer_count; + + optimizer* optimizer_function; + + + /** + * @brief a pointer the loss function + * + * + * @note The model owns these layers and deletes them in clear(). + * + */ + loss* loss_function; + // Number of layers currently stored in the model. /** * @brief Stores the number of layers @@ -86,6 +108,8 @@ struct model{ // model output (may be deleted and also used for debug) panic::tensor::real_matrix outputs; + // model dinputs (may be deleted and also used for debug) + panic::tensor::real_matrix dinputs; /** * @brief Empthy constructor @@ -116,7 +140,21 @@ struct model{ * @note it adds an already-inplemented layer to the model. * */ - bool add(layer* new_layer); + bool add_layer(layer* new_layer); + + /** + * @brief Helper function for adding layers + * + * Computes: + * @code + * layer_dense* new_layer = new layer_dense(3, 4); + * add(new_layer) + * @endcode + * + * @note it adds an already-inplemented layer to the model. + * + */ + bool add_trainable_layer(trainable_layer* new_layer); /** * @brief Adds a dense layer to the model. @@ -187,8 +225,99 @@ struct model{ */ bool forward(const panic::tensor::real_matrix& inputs); - // Delete all layers and reset the model. + /** + * @brief Loops over all layers backward function + * + * Computes: + * @code + * model.bacward(dvalues_data_matrix) + * @endcode + * + * @param dvalues diput data. + * + * @return true looped over every layer. + * + * + */ + bool backward(const panic::tensor::real_matrix& dvalues); + + + /** + * @brief Add loss for categorical crossentropy. + * + * Computes: + * @code + * model.add_loss_categorical_crossentropy(); + * @endcode + * + * + * @return true if loss is added + * + * @note This function is convenient, but it allocates a new layer. + */ + bool add_loss_categorical_crossentropy(); + + + + /** + * @brief Adds activation softmax AND loss for categorical crossentropy. + * + * Computes: + * @code + * model.activation_softmax_loss_categorical_crossentropy(); + * @endcode + * + * + * @return true if activation and loss is added + * + * @note This function is convenient, but it allocates a new layer. + */ + bool activation_softmax_loss_categorical_crossentropy(); + + + + /** + * @brief Adds optimizer_sgd to the model + * + * Computes: + * @code + * model.optimizer_sgd(1e-4) + * @endcode + * + * @param learning_rate Learning rate for update_param (default 1e-3). + * + * @return true If looped and optimized every trainable layer. + * + * + */ + bool add_optimizer_sgd(const panic::types::real_t learning_rate = static_cast(1e-3)); + + + + + + /** + * @brief Trains the model with input data + * + * Computes: + * @code + * model.backward(input_data_matrix) + * @endcode + * + * @param X_train Input X data for training. + * @param epochs Number of training iterations. + * @param print_every Prints every n iteration. + * + * + * @return true if training is done correctly. + * + * + */ + bool train(const panic::tensor::real_matrix& X_train, + const panic::tensor::uint_vector& y_train, + const panic::types::uint_t epochs, + const panic::types::uint_t print_every); /** * @brief Clears and deletes all layers and resets the model * diff --git a/include/neural_network/optimizers/optimizer.hpp b/include/neural_network/optimizers/optimizer.hpp new file mode 100644 index 0000000..99b0b28 --- /dev/null +++ b/include/neural_network/optimizers/optimizer.hpp @@ -0,0 +1,84 @@ +/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + * + * PANIC + * Portable Algorithms and Numerics In C++ + * + * Scientific computing from scratch, with feeling. + * + * Copyright (c) 2026 Michelle Bausager + * + * This file is part of PANIC. + * + * PANIC is free software licensed under the GNU General Public License v3.0 or later. + * You may redistribute and/or modify it under the terms of the GPL. + * + * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the LICENSE file for the full license text. + * + * SPDX-License-Identifier: GPL-3.0-or-later + * + *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + * + * Project Name: PANIC + * Module Name: neural_network + * File Name: optimizer.hpp + * Revision: 0.1.0 + * Date: 23-06-2026 + * Author: Michelle Bausager + * + * Description: + * Defines the base optimizer struct used in neural network + * + *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ +#pragma once +//--------------------------------------------------------------------------------------------------------------------------- +// INCLUDE DESCRIPTION +//--------------------------------------------------------------------------------------------------------------------------- +#include // panic::uint_t, panic::int_t, and panic::real_t +#include + + +namespace panic{ + namespace neural_network{ + + + +/** + * @brief Base optimizer for the rest of the neural network library to use + * + * This base optimizer should be used in neural networks + * This is done so it's easy to optimize the trainable layers in the model to loop over. + * The virtual means it should use derived object's version when called with a pointer. + * The =0 means the derivative object NEEDS to have these functions to work. + * + * The struct is used for PANIC neural_network library. + */ +struct optimizer{ + + /** + * @brief Default de-constructor + * + */ + virtual ~optimizer() = default; + + + /** + * @brief Virtual forward function for derivative layers + * + * @param inputs Data matrix input for forward function. + * + * @Note It's equal to 0 because it make the derivative + * object NEEDS to have these function to work. + */ + virtual bool update_params(trainable_layer& layer) = 0; + +}; + + + + } // namespace tensor +} // namespace panic + + + diff --git a/include/neural_network/optimizers/optimizer_sgd.hpp b/include/neural_network/optimizers/optimizer_sgd.hpp new file mode 100644 index 0000000..b2c9c6e --- /dev/null +++ b/include/neural_network/optimizers/optimizer_sgd.hpp @@ -0,0 +1,88 @@ +/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + * + * PANIC + * Portable Algorithms and Numerics In C++ + * + * Scientific computing from scratch, with feeling. + * + * Copyright (c) 2026 Michelle Bausager + * + * This file is part of PANIC. + * + * PANIC is free software licensed under the GNU General Public License v3.0 or later. + * You may redistribute and/or modify it under the terms of the GPL. + * + * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the LICENSE file for the full license text. + * + * SPDX-License-Identifier: GPL-3.0-or-later + * + *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + * + * Project Name: PANIC + * Module Name: neural_network + * File Name: optimizer_sgd.hpp + * Revision: 0.1.0 + * Date: 04-08-2026 + * Author: Michelle Bausager + * + * Description: + * Defines the optimizer_sgd struct used in neural network + * + *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ +#pragma once +//--------------------------------------------------------------------------------------------------------------------------- +// INCLUDE DESCRIPTION +//--------------------------------------------------------------------------------------------------------------------------- +#include +#include + + +namespace panic{ + namespace neural_network{ + + + +/** + * @brief optimizer_sgd for the rest of the neural network library to use + * + */ +struct optimizer_sgd: optimizer{ + + panic::types::real_t learning_rate; + + + /** + * @brief Constructor + * + * @param learning_rate The learning rate for the optimization. + * + */ + optimizer_sgd(const panic::types::real_t learning_rate = static_cast(1e-3)); + + + /** + * @brief Default de-constructor + * + */ + ~optimizer_sgd() = default; + + + /** + * @brief Updates weights and biases in trainable layers + * + * @param layer Trianable layer to update. + * + */ + bool update_params(trainable_layer& layer) override; + +}; + + + + } // namespace tensor +} // namespace panic + + + diff --git a/main.cpp b/main.cpp index 068b058..b4601a3 100644 --- a/main.cpp +++ b/main.cpp @@ -63,6 +63,7 @@ #include #include #include +#include #include @@ -735,32 +736,15 @@ int main(void) { // Create activation softmax layer mymodel.add_activation_softmax(); - // create loss function - panic::neural_network::loss_categorical_crossentropy loss_function; - - mymodel.forward(X); - - loss_function.calculate(mymodel.outputs, y); - - panic::tensor::uint_vector prediction; - prediction = panic::math::argmax_rowwise(mymodel.outputs); - - panic::types::real_t accuracy; - panic::tensor::uint_vector comparisons; - - comparisons = panic::math::equal(prediction, y); - - accuracy = panic::math::mean(comparisons); - - std::cout << "loss: " << loss_function.data_loss << std::endl; - - std::cout << "acc: " << accuracy << std::endl; - - + mymodel.add_loss_categorical_crossentropy(); + //mymodel.activation_softmax_loss_categorical_crossentropy(); + mymodel.add_optimizer_sgd(); + panic::types::uint_t epochs = 10; + panic::types::uint_t print_every = 1; - + mymodel.train(X, y, epochs, print_every); diff --git a/src/neural_network/activation_loss/activation_softmax_loss_categorical_crossentropy.cpp b/src/neural_network/activation_loss/activation_softmax_loss_categorical_crossentropy.cpp new file mode 100644 index 0000000..bf7391a --- /dev/null +++ b/src/neural_network/activation_loss/activation_softmax_loss_categorical_crossentropy.cpp @@ -0,0 +1,267 @@ +/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + * + * PANIC + * Portable Algorithms and Numerics In C++ + * + * Scientific computing from scratch, with feeling. + * + * Copyright (c) 2026 Michelle Bausager + * + * This file is part of PANIC. + * + * PANIC is free software licensed under the GNU General Public License v3.0 or later. + * You may redistribute and/or modify it under the terms of the GPL. + * + * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the LICENSE file for the full license text. + * + * SPDX-License-Identifier: GPL-3.0-or-later + * + *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + * + * Project Name: PANIC + * Module Name: neural_network + * File Name: activation_softmax_loss_categorical_crossentropy.cpp + * Revision: 0.1.0 + * Date: 28-07-2026 + * Author: Michelle Bausager + * + * Description: + * Defines the combined activation function softmax and + * categorical crossentropy loss used in neural network + * + *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ + +//--------------------------------------------------------------------------------------------------------------------------- +// INCLUDE DESCRIPTION +//--------------------------------------------------------------------------------------------------------------------------- +#include +#include + +#include +#include + + +//--------------------------------------------------------------------------------------------------------------------------- +// PRIVATE CONSTANTS +//--------------------------------------------------------------------------------------------------------------------------- +/** + * @brief Minimum number of element operations before using the OpenMP-enabled loop. + * + * Small vectors and matrices are kept serial because the overhead of starting + * worker threads can be larger than the work itself. + */ +static const panic::types::uint_t activation_softmax_loss_categorical_crossentropy_omp_min_size = 500; +//--------------------------------------------------------------------------------------------------------------------------- +// INPLEMENTATION +//--------------------------------------------------------------------------------------------------------------------------- + +namespace panic{ + namespace neural_network{ + + +//-------------------------------------------------------------------------------------------------------------------------- +// Constructor Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy +// +// Description: +// Creates an empty layer. +//-------------------------------------------------------------------------------------------------------------------------- +activation_softmax_loss_categorical_crossentropy::activation_softmax_loss_categorical_crossentropy() { +} + + +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy.forward +// +// Description: +// Calculated the forward pass +//-------------------------------------------------------------------------------------------------------------------------- +bool activation_softmax_loss_categorical_crossentropy::forward(const panic::tensor::real_matrix& y_pred, const panic::tensor::uint_vector& y_true){ + + // Output layers activation function + if (!activation.forward(y_pred)){ + return false; + } + // Set the output + outputs = activation.outputs; + + // calculate the loss value. + if (!loss.calculate(outputs, y_true)){ + return false; + } + + return true; +} + +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy.forward +// +// Description: +// Calculated the forward pass +//-------------------------------------------------------------------------------------------------------------------------- +bool activation_softmax_loss_categorical_crossentropy::forward(const panic::tensor::real_matrix& y_pred, const panic::tensor::real_matrix& y_true){ + + // Output layers activation function + if (!activation.forward(y_pred)){ + return false; + } + // Set the output + outputs = activation.outputs; + + // calculate the loss value. + if (!loss.calculate(outputs, y_true)){ + return false; + } + + return true; +} + + + + + + + + + + + + + +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy.calculate +// +// Description: +// Calculated the calculate pass +//-------------------------------------------------------------------------------------------------------------------------- +bool activation_softmax_loss_categorical_crossentropy::calculate(const panic::tensor::real_matrix& y_pred, const panic::tensor::uint_vector& y_true){ + + // Output layers activation function + if (!activation.forward(y_pred)){ + return false; + } + + // Set the output + outputs = activation.outputs; + + // calculate the loss value. + if (!loss.calculate(outputs, y_true)){ + return false; + } + + data_loss = loss.data_loss; + + return true; +} + +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy.calculate +// +// Description: +// Calculated the calculate pass +//-------------------------------------------------------------------------------------------------------------------------- +bool activation_softmax_loss_categorical_crossentropy::calculate(const panic::tensor::real_matrix& y_pred, const panic::tensor::real_matrix& y_true){ + + // Output layers activation function + if (!activation.forward(y_pred)){ + return false; + } + // Set the output + outputs = activation.outputs; + + // calculate the loss value. + if (!loss.calculate(outputs, y_true)){ + return false; + } + + data_loss = loss.data_loss; + + return true; +} + + + + + + + + + + + + + + + + + + + + + + + + + + +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy.backward +// +// Description: +// Default implementation for catecorical labels. +//-------------------------------------------------------------------------------------------------------------------------- +bool activation_softmax_loss_categorical_crossentropy::backward(const panic::tensor::real_matrix& dvalues, const panic::tensor::uint_vector& y_true){ + + const panic::types::uint_t samples = dvalues.rows(); + const panic::types::uint_t classes = dvalues.cols(); + + if (samples == 0 || classes == 0 || y_true.size() != samples){ + return false; + } + + // Copy Softmax output + dinputs = dvalues; + + // Subtract 1 from the correct class of every sample + PANIC_OMP_PARALLEL_FOR_IF(samples > activation_softmax_loss_categorical_crossentropy_omp_min_size) + for (panic::types::uint_t i = 0; i < samples; ++i){ + dinputs(i, y_true[i]) -= static_cast(1); + } + + // Scale to normalize gradients + const panic::types::real_t scale = static_cast(1) / static_cast(samples); + + // Normalize gradients + if (!panic::math::mul(dinputs, scale, dinputs)){ + return false; + } + + return true; +} + + +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::activation_softmax_loss_categorical_crossentropy.backward +// +// Description: +// Default one-hot encoded labels implementation. +//-------------------------------------------------------------------------------------------------------------------------- +bool activation_softmax_loss_categorical_crossentropy::backward(const panic::tensor::real_matrix& dvalues, const panic::tensor::real_matrix& y_true){ + + if (y_true.rows() != dvalues.rows() || y_true.cols() != dvalues.cols()){ + return false; + } + + panic::tensor::uint_vector categorical_labels; + + if (!panic::math::argmax_rowwise(y_true, categorical_labels)){ + return false; + } + + return backward(dvalues, categorical_labels); +} + + + } // namespace tensor +} // namespace panic diff --git a/src/neural_network/loss/loss_categorical_crossentropy.cpp b/src/neural_network/loss/loss_categorical_crossentropy.cpp index c57443a..cccd83f 100644 --- a/src/neural_network/loss/loss_categorical_crossentropy.cpp +++ b/src/neural_network/loss/loss_categorical_crossentropy.cpp @@ -224,17 +224,6 @@ bool loss_categorical_crossentropy::backward( } - - - - - - - - - - - } // namespace tensor } // namespace panic diff --git a/src/neural_network/model/model.cpp b/src/neural_network/model/model.cpp index ae403f1..db12228 100644 --- a/src/neural_network/model/model.cpp +++ b/src/neural_network/model/model.cpp @@ -45,6 +45,20 @@ #include #include +#include + +#include + +#include + +#include +#include +#include + + +// Remember ti disable +#include // for std::cout, std::endl + //--------------------------------------------------------------------------------------------------------------------------- // IMPLEMENTATION //--------------------------------------------------------------------------------------------------------------------------- @@ -63,9 +77,18 @@ model::model(){ // No layers yet. layers = 0; + // No loss function yet. + loss_function = 0; + // Number of layers is zero. layer_count = 0; + + trainable_layers = 0; + trainable_layer_count = 0; + + optimizer_function = 0; + } @@ -81,7 +104,7 @@ model::~model(){ //-------------------------------------------------------------------------------------------------------------------------- -// Function Name : panic::neural_network::model::add +// Function Name : panic::neural_network::model::add_layer // // Description: // Adds a layer to the model. @@ -89,7 +112,7 @@ model::~model(){ // The layer pointer array // is resized every time a new layer is added. //-------------------------------------------------------------------------------------------------------------------------- -bool model::add(layer* new_layer){ +bool model::add_layer(layer* new_layer){ // Do not add a null layer. if (new_layer == 0){ @@ -141,6 +164,74 @@ bool model::add(layer* new_layer){ } + +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::model::add_layer +// +// Description: +// Adds a layer to the model. +// +// The layer pointer array +// is resized every time a new layer is added. +//-------------------------------------------------------------------------------------------------------------------------- +bool model::add_trainable_layer(trainable_layer* new_layer){ + + // Do not add a null layer. + if (new_layer == 0){ + return false; + } + + if (!add_layer(new_layer)){ + return false; + } + + // The new array needs room for all old layers plus the new one. + const panic::types::uint_t new_trainable_layer_count = trainable_layer_count + 1; + + // Allocate a new array of layer pointers. + // + // layer* means "pointer to one layer" + // layer** means "pointer to many layer pointers" + // + // So this creates: + // + // [ layer* ][ layer* ][ layer* ] ... + // + trainable_layer** new_trainable_layers = new trainable_layer*[new_trainable_layer_count]; + + // Copy the old layer pointers into the new array. + // + // Important: + // This does not copy the layers themselves. + // It only copies the addresses of the layers. + + for (panic::types::uint_t i = 0; i < trainable_layer_count; ++i ){ + new_trainable_layers[i] = trainable_layers[i]; + } + + // Put the new layer at the end. + new_trainable_layers[trainable_layer_count] = new_layer; + + // Delete the old array of pointers. + // + // Important: + // Do NOT delete layers[i] here. + // The actual layer objects are still used in new_layers. + // + // This only deletes the old pointer array. + delete[] trainable_layers; + + + // Make the model use the new bigger array. + trainable_layers = new_trainable_layers; + + // Update the layer count. + trainable_layer_count = new_trainable_layer_count; + + return true; +} + + //-------------------------------------------------------------------------------------------------------------------------- // Function Name : panic::neural_network::model::add_layer_dense // @@ -164,7 +255,7 @@ bool model::add_layer_dense( // Add it to the model. // // If add() fails, delete the layer so we do not leak memory. - if (!add(new_layer)){ + if (!add_trainable_layer(new_layer)){ delete new_layer; return false; } @@ -192,7 +283,7 @@ bool model::add_activation_relu(){ // Add it to the model. // // If add() fails, delete the layer so we do not leak memory. - if (!add(new_layer)){ + if (!add_layer(new_layer)){ delete new_layer; return false; } @@ -221,7 +312,7 @@ bool model::add_activation_softmax(){ // Add it to the model. // // If add() fails, delete the layer so we do not leak memory. - if (!add(new_layer)){ + if (!add_layer(new_layer)){ delete new_layer; return false; } @@ -229,6 +320,9 @@ bool model::add_activation_softmax(){ return true; } + + + //-------------------------------------------------------------------------------------------------------------------------- // Function Name : panic::neural_network::model::forward // @@ -264,6 +358,126 @@ bool model::forward(const panic::tensor::real_matrix& inputs){ } +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::model::bacward +// +// Description: +// Runs the dinputs through every layer in reverse order. +//-------------------------------------------------------------------------------------------------------------------------- +bool model::backward(const panic::tensor::real_matrix& dvalues){ + + return true; +} + +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::model::add_loss_categorical_crossentropy +// +// Description: +// Sets the loss function. +//-------------------------------------------------------------------------------------------------------------------------- +bool model::add_loss_categorical_crossentropy(){ + + delete loss_function; + + loss_function = new panic::neural_network::loss_categorical_crossentropy(); + + return true; +} + + +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::model::activation_softmax_loss_categorical_crossentropy +// +// Description: +// Sets the loss function. +//-------------------------------------------------------------------------------------------------------------------------- +bool model::activation_softmax_loss_categorical_crossentropy(){ + + delete loss_function; + + loss_function = new panic::neural_network::activation_softmax_loss_categorical_crossentropy(); + + return true; +} + + +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::model::add_optimizer_sgd +// +// Description: +// Adds Stocastient Gradient Decent as an optimizer to the model. +// +// Example: +// model.add_optimizer_sgd(1e-4); +//-------------------------------------------------------------------------------------------------------------------------- +bool model::add_optimizer_sgd(const panic::types::real_t learning_rate){ + + optimizer_sgd* new_optimizer = new optimizer_sgd(learning_rate); + + if (new_optimizer == 0){ + return false; + } + + delete optimizer_function; + optimizer_function = new_optimizer; + + return true; +} + + +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::model::train +// +// Description: +// Trains the model with input data. +//-------------------------------------------------------------------------------------------------------------------------- +bool model::train(const panic::tensor::real_matrix& X_train, + const panic::tensor::uint_vector& y_train, + const panic::types::uint_t epochs, + const panic::types::uint_t print_every){ + + panic::tensor::uint_vector prediction; + panic::types::real_t accuracy; + panic::tensor::uint_vector comparisons; + + for (panic::types::uint_t epoch = 0; epoch < epochs; ++epoch){ + + + forward(X_train); + + if (!loss_function->calculate(outputs, y_train)){ + return false; + } + + + prediction = panic::math::argmax_rowwise(outputs); + + comparisons = panic::math::equal(prediction, y_train); + + accuracy = panic::math::mean(comparisons); + + if (epoch % print_every == static_cast(0)){ + std::cout << "Epoch: " << epoch; + std::cout << " loss: " << loss_function->data_loss; + std::cout << " acc: " << accuracy << std::endl; + } + + + //backward(); + + //optimize(); + + } + return true; +} + + + + + + + + //-------------------------------------------------------------------------------------------------------------------------- // Function Name : panic::neural_network::model::clear // @@ -284,10 +498,18 @@ void model::clear(){ delete[] layers; } + // Reset to empty state. layers = 0; + loss_function = 0; layer_count = 0; + delete[] trainable_layers; + trainable_layers = 0; + trainable_layer_count = 0; + + delete optimizer_function; + outputs.resize(0, 0); } diff --git a/src/neural_network/optimizers/optimizer_sgd.cpp b/src/neural_network/optimizers/optimizer_sgd.cpp new file mode 100644 index 0000000..f811ff2 --- /dev/null +++ b/src/neural_network/optimizers/optimizer_sgd.cpp @@ -0,0 +1,105 @@ +/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + * + * PANIC + * Portable Algorithms and Numerics In C++ + * + * Scientific computing from scratch, with feeling. + * + * Copyright (c) 2026 Michelle Bausager + * + * This file is part of PANIC. + * + * PANIC is free software licensed under the GNU General Public License v3.0 or later. + * You may redistribute and/or modify it under the terms of the GPL. + * + * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the LICENSE file for the full license text. + * + * SPDX-License-Identifier: GPL-3.0-or-later + * + *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + * + * Project Name: PANIC + * Module Name: neural_network + * File Name: optimizer_sgd.cpp + * Revision: 0.1.0 + * Date: 04-08-2026 + * Author: Michelle Bausager + * + * Description: + * Defines the optimizer_sgd used in neural network + * + *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ + +//--------------------------------------------------------------------------------------------------------------------------- +// INCLUDE DESCRIPTION +//--------------------------------------------------------------------------------------------------------------------------- +#include +#include + +#include +#include + +//--------------------------------------------------------------------------------------------------------------------------- +// PRIVATE CONSTANTS +//--------------------------------------------------------------------------------------------------------------------------- +/** + * @brief Minimum number of element operations before using the OpenMP-enabled loop. + * + * Small vectors and matrices are kept serial because the overhead of starting + * worker threads can be larger than the work itself. + */ +static const panic::types::uint_t optimizer_sgd_omp_min_size = 500; +//--------------------------------------------------------------------------------------------------------------------------- +// INPLEMENTATION +//--------------------------------------------------------------------------------------------------------------------------- + +namespace panic{ + namespace neural_network{ + + +//-------------------------------------------------------------------------------------------------------------------------- +// Constructor Name : panic::neural_network::optimizer_sgd +// +// Description: +// Constructor for optimizer_sgd. +//-------------------------------------------------------------------------------------------------------------------------- +optimizer_sgd::optimizer_sgd(const panic::types::real_t learning_rate) { + this->learning_rate = learning_rate; +} + +//-------------------------------------------------------------------------------------------------------------------------- +// Constructor Name : panic::neural_network::update_params +// +// Description: +// Updates weights and biases in layer. +//-------------------------------------------------------------------------------------------------------------------------- +bool optimizer_sgd::update_params(trainable_layer& layer) { + + panic::tensor::real_matrix weight_updates; + panic::tensor::real_vector bias_updates; + + if (!panic::math::mul(layer.weights, -learning_rate, weight_updates)){ + return false; + } + + if (!panic::math::add(layer.weights, weight_updates, layer.weights)){ + return false; + } + + if (!panic::math::mul(layer.biases, -learning_rate, bias_updates)){ + return false; + } + + if (!panic::math::add(layer.biases, bias_updates, layer.biases)){ + return false; + } + + return true; + +} + + + } // namespace tensor +} // namespace panic