Bausager/panic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimazation is done

Browse Source
This commit is contained in:
Bausager
2026-08-06 18:56:36 +02:00
parent 7f92e6884d
commit fb59d61ad5
19 changed files with 2179 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/neural_network/layer/trainable_layer.hpp
+18
View File
@@ -63,6 +63,24 @@ struct trainable_layer:layer{
panic::tensor::real_matrix dweights;
panic::tensor::real_vector dbiases;
/**
* @brief Previous parameter updates used by momentum SGD.
*
* These remain empty unless an optimizer using momentum
* initializes them.
*/
panic::tensor::real_matrix weight_momentums;
panic::tensor::real_vector bias_momentums;
/**
* @brief Previous parameter updates used by AdaGrad.
*
* These remain empty unless an optimizer using cache
* initializes them.
*/
panic::tensor::real_matrix weight_cache;
panic::tensor::real_vector bias_cache;
virtual ~trainable_layer() = default;
include/neural_network/model/model.hpp
+62 -1
View File
@@ -323,9 +323,70 @@ struct model{
*
*
*/
bool add_optimizer_sgd(const panic::types::real_t learning_rate = static_cast<panic::types::real_t>(1));
bool add_optimizer_sgd(const panic::types::real_t learning_rate = 1,
const panic::types::real_t decay = 0,
const panic::types::real_t momentum = 0);
/**
* @brief Adds optimizer_adagrad to the model
*
* Computes:
* @code
* model.optimizer_adagrad(1e-4)
* @endcode
*
* @param learning_rate Learning rate for update_param (default 1e-3).
*
* @return true If looped and optimized every trainable layer.
*
*
*/
bool add_optimizer_adagrad(const panic::types::real_t learning_rate = 1,
const panic::types::real_t decay = 0,
const panic::types::real_t epsilon = 1e-7);
/**
* @brief Adds optimizer_rmsprop to the model
*
* Computes:
* @code
* model.optimizer_adagrad(1e-4)
* @endcode
*
* @param learning_rate Learning rate for update_param (default 1e-3).
*
* @return true If looped and optimized every trainable layer.
*
*
*/
bool add_optimizer_rmsprop(const panic::types::real_t learning_rate = 0.001,
const panic::types::real_t decay = 0,
const panic::types::real_t epsilon = 1e-7,
const panic::types::real_t rho = 0.9);
/**
* @brief Adds optimizer_adam to the model
*
* Computes:
* @code
* model.optimizer_adam(1e-4)
* @endcode
*
* @param learning_rate Learning rate for update_param (default 1e-3).
*
* @return true If looped and optimized every trainable layer.
*
*
*/
bool add_optimizer_adam(const panic::types::real_t learning_rate = 0.001,
const panic::types::real_t decay = 0,
const panic::types::real_t epsilon = 1e-7,
const panic::types::real_t beta_1 = 0.9,
const panic::types::real_t beta_2 = 0.999);
/**
* @brief Finalizes the model configuration.
include/neural_network/optimizers/optimizer.hpp
+22 -2
View File
@@ -56,6 +56,8 @@ namespace panic{
*/
struct optimizer{
panic::types::real_t current_learning_rate;
/**
* @brief Default de-constructor
*
@@ -64,15 +66,33 @@ struct optimizer{
/**
* @brief Virtual forward function for derivative layers
* @brief Virtual update parameters function for derivative optimizers
*
* @param inputs Data matrix input for forward function.
* @param layer trainable layer to have their parameters updated
*
* @Note It's equal to 0 because it make the derivative
* object NEEDS to have these function to work.
*/
virtual bool update_params(trainable_layer& layer) = 0;
/**
* @brief Virtual function to update internal parameters before update_params()
*
*
*/
virtual bool pre_update_params(){
return true;
}
/**
* @brief Virtual function to update internal parameters after update_params()
*
*
*/
virtual bool post_update_params(){
return true;
}
};
include/neural_network/optimizers/optimizer_adagrad.hpp
+112
View File
@@ -0,0 +1,112 @@
/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
*
* PANIC
* Portable Algorithms and Numerics In C++
*
* Scientific computing from scratch, with feeling.
*
* Copyright (c) 2026 Michelle Bausager
*
* This file is part of PANIC.
*
* PANIC is free software licensed under the GNU General Public License v3.0 or later.
* You may redistribute and/or modify it under the terms of the GPL.
*
* PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the LICENSE file for the full license text.
*
* SPDX-License-Identifier: GPL-3.0-or-later
*
*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
*
* Project Name: PANIC
* Module Name: neural_network
* File Name: optimizer_adagrad.hpp
* Revision: 0.1.0
* Date: 04-08-2026
* Author: Michelle Bausager
*
* Description:
* Defines the optimizer_adagrad struct used in neural network
*
*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
#pragma once
//---------------------------------------------------------------------------------------------------------------------------
// INCLUDE DESCRIPTION
//---------------------------------------------------------------------------------------------------------------------------
#include <config/types.hpp>
#include <neural_network/optimizers/optimizer.hpp>
namespace panic{
namespace neural_network{
/**
* @brief optimizer_adagrad for the rest of the neural network library to use
*
*/
struct optimizer_adagrad: optimizer{
panic::types::real_t learning_rate;
panic::types::real_t decay;
panic::types::real_t epsilon;
panic::types::uint_t iterations;
/**
* @brief Constructor
*
* @param learning_rate The learning rate for the optimization.
* @param decay The decay for the learning rate over the interations.
*
*/
optimizer_adagrad(const panic::types::real_t learning_rate = static_cast<panic::types::real_t>(1),
const panic::types::real_t decay = static_cast<panic::types::real_t>(0),
const panic::types::real_t epsilon = static_cast<panic::types::real_t>(1e-7));
/**
* @brief Default de-constructor
*
*/
~optimizer_adagrad() = default;
/**
* @brief Updates weights and biases in trainable layers
*
* @param layer Trianable layer to update.
*
*/
bool update_params(trainable_layer& layer) override;
/**
* @brief function to update internal parameters before update_params()
*
* @param layer Trianable layer to update.
*
*/
bool pre_update_params() override;
/**
* @brief function to update internal parameters after update_params()
*
*/
bool post_update_params() override;
};
} // namespace tensor
} // namespace panic
include/neural_network/optimizers/optimizer_adam.hpp
+120
View File
@@ -0,0 +1,120 @@
/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
*
* PANIC
* Portable Algorithms and Numerics In C++
*
* Scientific computing from scratch, with feeling.
*
* Copyright (c) 2026 Michelle Bausager
*
* This file is part of PANIC.
*
* PANIC is free software licensed under the GNU General Public License v3.0 or later.
* You may redistribute and/or modify it under the terms of the GPL.
*
* PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the LICENSE file for the full license text.
*
* SPDX-License-Identifier: GPL-3.0-or-later
*
*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
*
* Project Name: PANIC
* Module Name: neural_network
* File Name: optimizer_rmsprop.hpp
* Revision: 0.1.0
* Date: 04-08-2026
* Author: Michelle Bausager
*
* Description:
* Defines the optimizer_adam struct used in neural network
*
*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
#pragma once
//---------------------------------------------------------------------------------------------------------------------------
// INCLUDE DESCRIPTION
//---------------------------------------------------------------------------------------------------------------------------
#include <config/types.hpp>
#include <neural_network/optimizers/optimizer.hpp>
namespace panic{
namespace neural_network{
/**
* @brief optimizer_adam for the rest of the neural network library to use
*
*/
struct optimizer_adam: optimizer{
panic::types::real_t learning_rate;
panic::types::real_t decay;
panic::types::real_t epsilon;
panic::types::real_t beta_1;
panic::types::real_t beta_2;
panic::types::real_t beta_1_power;
panic::types::real_t beta_2_power;
panic::types::uint_t iterations;
/**
* @brief Constructor
*
* @param learning_rate The learning rate for the optimization.
* @param decay The decay for the learning rate over the interations.
*
*/
optimizer_adam(const panic::types::real_t learning_rate = static_cast<panic::types::real_t>(0.001),
const panic::types::real_t decay = static_cast<panic::types::real_t>(0),
const panic::types::real_t epsilon = static_cast<panic::types::real_t>(1e-7),
const panic::types::real_t beta_1 = static_cast<panic::types::real_t>(0.9),
const panic::types::real_t beta_2 = static_cast<panic::types::real_t>(0.999));
/**
* @brief Default de-constructor
*
*/
~optimizer_adam() = default;
/**
* @brief Updates weights and biases in trainable layers
*
* @param layer Trianable layer to update.
*
*/
bool update_params(trainable_layer& layer) override;
/**
* @brief function to update internal parameters before update_params()
*
* @param layer Trianable layer to update.
*
*/
bool pre_update_params() override;
/**
* @brief function to update internal parameters after update_params()
*
*/
bool post_update_params() override;
};
} // namespace tensor
} // namespace panic
include/neural_network/optimizers/optimizer_rmsprop.hpp
+115
View File
@@ -0,0 +1,115 @@
/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
*
* PANIC
* Portable Algorithms and Numerics In C++
*
* Scientific computing from scratch, with feeling.
*
* Copyright (c) 2026 Michelle Bausager
*
* This file is part of PANIC.
*
* PANIC is free software licensed under the GNU General Public License v3.0 or later.
* You may redistribute and/or modify it under the terms of the GPL.
*
* PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
* See the LICENSE file for the full license text.
*
* SPDX-License-Identifier: GPL-3.0-or-later
*
*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
*
* Project Name: PANIC
* Module Name: neural_network
* File Name: optimizer_rmsprop.hpp
* Revision: 0.1.0
* Date: 04-08-2026
* Author: Michelle Bausager
*
* Description:
* Defines the optimizer_rmsprop struct used in neural network
*
*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
#pragma once
//---------------------------------------------------------------------------------------------------------------------------
// INCLUDE DESCRIPTION
//---------------------------------------------------------------------------------------------------------------------------
#include <config/types.hpp>
#include <neural_network/optimizers/optimizer.hpp>
namespace panic{
namespace neural_network{
/**
* @brief optimizer_rmsprop for the rest of the neural network library to use
*
*/
struct optimizer_rmsprop: optimizer{
panic::types::real_t learning_rate;
panic::types::real_t decay;
panic::types::real_t epsilon;
panic::types::real_t rho;
panic::types::uint_t iterations;
/**
* @brief Constructor
*
* @param learning_rate The learning rate for the optimization.
* @param decay The decay for the learning rate over the interations.
*
*/
optimizer_rmsprop(const panic::types::real_t learning_rate = static_cast<panic::types::real_t>(0.001),
const panic::types::real_t decay = static_cast<panic::types::real_t>(0),
const panic::types::real_t epsilon = static_cast<panic::types::real_t>(1e-7),
const panic::types::real_t rho = static_cast<panic::types::real_t>(0.9));
/**
* @brief Default de-constructor
*
*/
~optimizer_rmsprop() = default;
/**
* @brief Updates weights and biases in trainable layers
*
* @param layer Trianable layer to update.
*
*/
bool update_params(trainable_layer& layer) override;
/**
* @brief function to update internal parameters before update_params()
*
* @param layer Trianable layer to update.
*
*/
bool pre_update_params() override;
/**
* @brief function to update internal parameters after update_params()
*
*/
bool post_update_params() override;
};
} // namespace tensor
} // namespace panic
include/neural_network/optimizers/optimizer_sgd.hpp
+28 -3
View File
@@ -52,13 +52,24 @@ struct optimizer_sgd: optimizer{
panic::types::real_t learning_rate;
panic::types::real_t decay;
panic::types::real_t momentum;
panic::types::uint_t iterations;
/**
* @brief Constructor
*
* @param learning_rate The learning rate for the optimization.
* @param decay The decay for the learning rate over the interations.
*
*/
optimizer_sgd(const panic::types::real_t learning_rate = static_cast<panic::types::real_t>(1e-3));
optimizer_sgd(const panic::types::real_t learning_rate = static_cast<panic::types::real_t>(1),
const panic::types::real_t decay = static_cast<panic::types::real_t>(0),
const panic::types::real_t momentum = static_cast<panic::types::real_t>(0));
/**
@@ -76,10 +87,24 @@ struct optimizer_sgd: optimizer{
*/
bool update_params(trainable_layer& layer) override;
/**
* @brief function to update internal parameters before update_params()
*
* @param layer Trianable layer to update.
*
*/
bool pre_update_params() override;
/**
* @brief function to update internal parameters after update_params()
*
*/
bool post_update_params() override;
};
} // namespace tensor
} // namespace panic