Bausager
48f329feef
Sync public mirror / sync (push) Failing after 27s
Started on regulaization in Loss.h. I need to refactor the matsum.h since I need a total sum over the matrix. Also matmul needs a elementwise matmul function, which is the next this in the ragulaization
65 lines
1.5 KiB
C++
65 lines
1.5 KiB
C++
#pragma once
|
|
|
|
#include "./core/omp_config.h"
|
|
|
|
#include "./utils/vector.h"
|
|
#include "./utils/matrix.h"
|
|
#include "./utils/random.h"
|
|
|
|
|
|
namespace neural_networks{
|
|
|
|
template <typename T>
|
|
struct Dense_Layer{
|
|
|
|
T weight_regularizer_l1 = {1e-4};
|
|
T weight_regularizer_l2 = {1e-4};
|
|
|
|
T bias_regularizer_l1 = {1e-4};
|
|
T bias_regularizer_l2 = {1e-4};
|
|
|
|
utils::Matrix<T> _inputs;
|
|
utils::Matrix<T> weights;
|
|
utils::Vector<T> biases;
|
|
utils::Matrix<T> outputs;
|
|
|
|
utils::Matrix<T> dweights;
|
|
utils::Vector<T> dbiases;
|
|
utils::Matrix<T> dinputs;
|
|
|
|
// Variables for optimizers
|
|
utils::Matrix<T> weight_momentums;
|
|
utils::Vector<T> bias_momentums;
|
|
utils::Matrix<T> weight_cache;
|
|
utils::Vector<T> bias_cache;
|
|
|
|
// Default Constructor
|
|
Dense_Layer() = default;
|
|
|
|
// Constructor
|
|
Dense_Layer(const uint64_t n_inputs, const uint64_t n_neurons){
|
|
|
|
weights.random(n_inputs, n_neurons, -1, 1);
|
|
biases.resize(n_neurons, T{0});
|
|
|
|
}
|
|
|
|
void forward(const utils::Matrix<T>& inputs){
|
|
_inputs = inputs;
|
|
outputs = numerics::matadd(numerics::matmul_auto(inputs, weights), biases, "row");
|
|
}
|
|
|
|
void backward(const utils::Matrix<T>& dvalues){
|
|
// Gradients on parameters
|
|
dweights = numerics::matmul(numerics::transpose(_inputs), dvalues);
|
|
dbiases = numerics::matsum(dvalues, "row");
|
|
//Gradient on values
|
|
dinputs = numerics::matmul(dvalues, numerics::transpose(weights));
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
} // end namespace neural_networks
|