From 7f92e6884d430c922b0c4c7b8c9984f4e9d8f551 Mon Sep 17 00:00:00 2001 From: Michelle Date: Thu, 6 Aug 2026 10:22:52 +0200 Subject: [PATCH] Backup Save --- include/neural_network/model/model.hpp | 18 +++++++++- .../neural_network/optimizers/optimizer.hpp | 2 +- .../optimizers/optimizer_sgd.hpp | 1 - main.cpp | 16 +++++---- src/neural_network/model/model.cpp | 36 ++++++++++++++++++- .../optimizers/optimizer_sgd.cpp | 21 +++++++++-- 6 files changed, 81 insertions(+), 13 deletions(-) diff --git a/include/neural_network/model/model.hpp b/include/neural_network/model/model.hpp index f1ef1dd..fefa82a 100644 --- a/include/neural_network/model/model.hpp +++ b/include/neural_network/model/model.hpp @@ -323,7 +323,7 @@ struct model{ * * */ - bool add_optimizer_sgd(const panic::types::real_t learning_rate = static_cast(1e-3)); + bool add_optimizer_sgd(const panic::types::real_t learning_rate = static_cast(1)); @@ -337,6 +337,22 @@ struct model{ */ bool finalize(); + + /** + * @brief Optimizes the model with parameters on trainable layers + * + * Computes: + * @code + * model.optimize() + * @endcode + * + * @return true if optimization is done correctly. + * + */ + bool optimize(); + + + /** * @brief Trains the model with input data * diff --git a/include/neural_network/optimizers/optimizer.hpp b/include/neural_network/optimizers/optimizer.hpp index 99b0b28..ff08e61 100644 --- a/include/neural_network/optimizers/optimizer.hpp +++ b/include/neural_network/optimizers/optimizer.hpp @@ -24,7 +24,7 @@ * Module Name: neural_network * File Name: optimizer.hpp * Revision: 0.1.0 - * Date: 23-06-2026 + * Date: 06-08-2026 * Author: Michelle Bausager * * Description: diff --git a/include/neural_network/optimizers/optimizer_sgd.hpp b/include/neural_network/optimizers/optimizer_sgd.hpp index b2c9c6e..3459195 100644 --- a/include/neural_network/optimizers/optimizer_sgd.hpp +++ b/include/neural_network/optimizers/optimizer_sgd.hpp @@ -52,7 +52,6 @@ struct optimizer_sgd: optimizer{ panic::types::real_t learning_rate; - /** * @brief Constructor * diff --git a/main.cpp b/main.cpp index 3b04e5f..b6de104 100644 --- a/main.cpp +++ b/main.cpp @@ -705,7 +705,7 @@ int main(void) { panic::tensor::real_matrix X; panic::tensor::uint_vector y; - panic::types::uint_t samples = 10; + panic::types::uint_t samples = 100; panic::types::uint_t classes = 3; @@ -716,7 +716,7 @@ int main(void) { panic::neural_network::model mymodel; // Create Dense layer with 2 input features and 3 output values - if (!mymodel.add_layer_dense(2,3)){ + if (!mymodel.add_layer_dense(2,64)){ return false; } @@ -726,7 +726,7 @@ int main(void) { } // Create a second dense layer with 3 inputs and 3 outputs - if (!mymodel.add_layer_dense(3, 3)){ + if (!mymodel.add_layer_dense(64, 3)){ return false; } @@ -740,15 +740,19 @@ int main(void) { if (!mymodel.add_loss_categorical_crossentropy()){ return false; } + + if (! mymodel.add_optimizer_sgd()){ + return false; + } if (!mymodel.finalize()){ return false; } - //mymodel.add_optimizer_sgd(); - panic::types::uint_t epochs = 10; - panic::types::uint_t print_every = 1; + + panic::types::uint_t epochs = 10000; + panic::types::uint_t print_every = 100; if (!mymodel.train(X, y, epochs, print_every)){ std::cout << "Training failed" << std::endl; diff --git a/src/neural_network/model/model.cpp b/src/neural_network/model/model.cpp index 4cede4e..c57cc33 100644 --- a/src/neural_network/model/model.cpp +++ b/src/neural_network/model/model.cpp @@ -606,6 +606,38 @@ bool model::finalize(){ } +//-------------------------------------------------------------------------------------------------------------------------- +// Function Name : panic::neural_network::model::optimize +// +// Description: +// Optimizes the model with parameters on trainable layers +//-------------------------------------------------------------------------------------------------------------------------- +bool model::optimize(){ + + if (optimizer_function == 0){ + return false; + } + + + + for(panic::types::uint_t i = 0; i < trainable_layer_count; ++i){ + + // Validate the stored layer pointers before using them. + if (trainable_layers[i] == 0){ + return false; + } + + if (! optimizer_function->update_params(*trainable_layers[i]) ){ + return false; + } + + } + + + return true; +} + + //-------------------------------------------------------------------------------------------------------------------------- // Function Name : panic::neural_network::model::train @@ -668,7 +700,9 @@ bool model::train(const panic::tensor::real_matrix& X_train, } - //optimize(); + if (! optimize()){ + return false; + } } diff --git a/src/neural_network/optimizers/optimizer_sgd.cpp b/src/neural_network/optimizers/optimizer_sgd.cpp index f811ff2..4a14981 100644 --- a/src/neural_network/optimizers/optimizer_sgd.cpp +++ b/src/neural_network/optimizers/optimizer_sgd.cpp @@ -50,7 +50,7 @@ * Small vectors and matrices are kept serial because the overhead of starting * worker threads can be larger than the work itself. */ -static const panic::types::uint_t optimizer_sgd_omp_min_size = 500; +static const panic::types::uint_t optimizer_sgd_omp_min_size = 250; //--------------------------------------------------------------------------------------------------------------------------- // INPLEMENTATION //--------------------------------------------------------------------------------------------------------------------------- @@ -77,21 +77,36 @@ optimizer_sgd::optimizer_sgd(const panic::types::real_t learning_rate) { //-------------------------------------------------------------------------------------------------------------------------- bool optimizer_sgd::update_params(trainable_layer& layer) { + + // Gradients must match their corresponding parameters. + if (layer.weights.rows() != layer.dweights.rows() || layer.weights.cols() != layer.dweights.cols() || layer.biases.size() != layer.dbiases.size()){ + return false; + } + + if (layer.weights.rows() == static_cast(0) || layer.weights.cols() == static_cast(0) || layer.biases.size() == static_cast(0) ){ + return false; + } + + panic::tensor::real_matrix weight_updates; panic::tensor::real_vector bias_updates; - if (!panic::math::mul(layer.weights, -learning_rate, weight_updates)){ + // weight_updates = -learning_rate * dweights + if (!panic::math::mul(layer.dweights, -learning_rate, weight_updates)){ return false; } + // weights += weight_updates if (!panic::math::add(layer.weights, weight_updates, layer.weights)){ return false; } - if (!panic::math::mul(layer.biases, -learning_rate, bias_updates)){ + // bias_updates = -learning_rate * dbiases + if (!panic::math::mul(layer.dbiases, -learning_rate, bias_updates)){ return false; } + // biases += bias_updates if (!panic::math::add(layer.biases, bias_updates, layer.biases)){ return false; }