/**++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ * * PANIC * Portable Algorithms and Numerics In C++ * * Scientific computing from scratch, with feeling. * * Copyright (c) 2026 Michelle Bausager * * This file is part of PANIC. * * PANIC is free software licensed under the GNU General Public License v3.0 or later. * You may redistribute and/or modify it under the terms of the GPL. * * PANIC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the LICENSE file for the full license text. * * SPDX-License-Identifier: GPL-3.0-or-later * *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ * * Project Name: PANIC * Module Name: neural_network * File Name: optimizer_rmsprop.cpp * Revision: 0.1.0 * Date: 04-08-2026 * Author: Michelle Bausager * * Description: * Defines the optimizer_rmsprop used in neural network * *++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ //--------------------------------------------------------------------------------------------------------------------------- // INCLUDE DESCRIPTION //--------------------------------------------------------------------------------------------------------------------------- #include #include #include #include #include #include #include //--------------------------------------------------------------------------------------------------------------------------- // PRIVATE CONSTANTS //--------------------------------------------------------------------------------------------------------------------------- /** * @brief Minimum number of element operations before using the OpenMP-enabled loop. * * Small vectors and matrices are kept serial because the overhead of starting * worker threads can be larger than the work itself. */ static const panic::types::uint_t optimizer_rmsprop_omp_min_size = 250; static const panic::types::real_t real_t_1 = static_cast(1); //--------------------------------------------------------------------------------------------------------------------------- // INPLEMENTATION //--------------------------------------------------------------------------------------------------------------------------- namespace panic{ namespace neural_network{ //-------------------------------------------------------------------------------------------------------------------------- // Constructor Name : panic::neural_network::optimizer_rmsprop // // Description: // Constructor for optimizer_rmsprop. //-------------------------------------------------------------------------------------------------------------------------- optimizer_rmsprop::optimizer_rmsprop(const panic::types::real_t learning_rate, const panic::types::real_t decay, const panic::types::real_t epsilon, const panic::types::real_t rho){ this->learning_rate = learning_rate; this->decay = decay; this->epsilon = epsilon; this->rho = rho; iterations = static_cast(0); current_learning_rate = learning_rate; } //-------------------------------------------------------------------------------------------------------------------------- // Constructor Name : panic::neural_network::update_params // // Description: // Updates weights and biases in layer. //-------------------------------------------------------------------------------------------------------------------------- bool optimizer_rmsprop::update_params(trainable_layer& layer) { // Gradients must match their corresponding parameters. if (layer.weights.rows() != layer.dweights.rows() || layer.weights.cols() != layer.dweights.cols() || layer.biases.size() != layer.dbiases.size()){ return false; } if (layer.weights.rows() == static_cast(0) || layer.weights.cols() == static_cast(0) || layer.biases.size() == static_cast(0) ){ return false; } // If layer doesn't contain cache arrays, create them // and fill them with zeros if(layer.weights.rows() != layer.weight_cache.rows() || layer.weights.cols() != layer.weight_cache.cols() || layer.biases.size() != layer.bias_cache.size()){ if (! layer.weight_cache.resize(layer.weights.rows(), layer.weights.cols())){ return false; } layer.weight_cache.fill(0); if (! layer.bias_cache.resize(layer.biases.size())){ return false; } layer.bias_cache.fill(0); } //-------------------------------------------------------------------------------------------------------------------------- // Update caches with squared current gradients //-------------------------------------------------------------------------------------------------------------------------- const panic::types::real_t one_minus_rho = static_cast(1) - rho; //-------------------------------------------------------------------------------------------------------------------------- // Weight cache //-------------------------------------------------------------------------------------------------------------------------- panic::tensor::real_matrix retained_weight_cache; panic::tensor::real_matrix squared_dweights; panic::tensor::real_matrix weighted_dweights; if (!panic::math::mul( layer.weight_cache, rho, retained_weight_cache )){ return false; } if (!panic::math::mul( layer.dweights, layer.dweights, squared_dweights )){ return false; } if (!panic::math::mul( squared_dweights, one_minus_rho, weighted_dweights )){ return false; } if (!panic::math::add( retained_weight_cache, weighted_dweights, layer.weight_cache )){ return false; } //-------------------------------------------------------------------------------------------------------------------------- // Bias cache //-------------------------------------------------------------------------------------------------------------------------- panic::tensor::real_vector retained_bias_cache; panic::tensor::real_vector squared_dbiases; panic::tensor::real_vector weighted_dbiases; if (!panic::math::mul( layer.bias_cache, rho, retained_bias_cache )){ return false; } if (!panic::math::mul( layer.dbiases, layer.dbiases, squared_dbiases )){ return false; } if (!panic::math::mul( squared_dbiases, one_minus_rho, weighted_dbiases )){ return false; } if (!panic::math::add( retained_bias_cache, weighted_dbiases, layer.bias_cache )){ return false; } //-------------------------------------------------------------------------------------------------------------------------- // Calculate weight updates //-------------------------------------------------------------------------------------------------------------------------- panic::tensor::real_matrix weight_updates; if (!panic::math::mul( layer.dweights, -current_learning_rate, weight_updates )){ return false; } panic::tensor::real_matrix sqrt_weight_cache; panic::tensor::real_matrix weight_denominator; if (!panic::math::sqrt( layer.weight_cache, sqrt_weight_cache )){ return false; } if (!panic::math::add( sqrt_weight_cache, epsilon, weight_denominator )){ return false; } if (!panic::math::div( weight_updates, weight_denominator, weight_updates )){ return false; } if (!panic::math::add( layer.weights, weight_updates, layer.weights )){ return false; } //-------------------------------------------------------------------------------------------------------------------------- // Calculate bias updates //-------------------------------------------------------------------------------------------------------------------------- panic::tensor::real_vector bias_updates; if (!panic::math::mul( layer.dbiases, -current_learning_rate, bias_updates )){ return false; } panic::tensor::real_vector sqrt_bias_cache; panic::tensor::real_vector bias_denominator; if (!panic::math::sqrt( layer.bias_cache, sqrt_bias_cache )){ return false; } if (!panic::math::add( sqrt_bias_cache, epsilon, bias_denominator )){ return false; } if (!panic::math::div( bias_updates, bias_denominator, bias_updates )){ return false; } if (!panic::math::add( layer.biases, bias_updates, layer.biases )){ return false; } return true; } //-------------------------------------------------------------------------------------------------------------------------- // Constructor Name : panic::neural_network::pre_update_params // // Description: // function to update internal parameters before update_params() //-------------------------------------------------------------------------------------------------------------------------- bool optimizer_rmsprop::pre_update_params(){ if (decay){ current_learning_rate = learning_rate * (real_t_1 / (real_t_1 + (decay * iterations))); } return true; } //-------------------------------------------------------------------------------------------------------------------------- // Constructor Name : panic::neural_network::post_update_params // // Description: // function to update internal parameters after update_params() //-------------------------------------------------------------------------------------------------------------------------- bool optimizer_rmsprop::post_update_params(){ iterations += static_cast(1); return true; } } // namespace tensor } // namespace panic