diff --git a/bin/abc_lab b/bin/abc_lab index fc17bdc..1a58c21 100755 Binary files a/bin/abc_lab and b/bin/abc_lab differ diff --git a/include/modules/neural_networks/datasets/vertical.h b/include/modules/neural_networks/datasets/vertical.h new file mode 100644 index 0000000..a5c1dc8 --- /dev/null +++ b/include/modules/neural_networks/datasets/vertical.h @@ -0,0 +1,40 @@ +#pragma once + +#include "./core/omp_config.h" +#include "./utils/matrix.h" +#include "./utils/vector.h" +#include "./utils/random.h" + +//#include + + +namespace neural_networks{ + + template + void create_vertical_data(const uint64_t samples, const uint64_t classes, utils::Matrix& X, utils::Matrix& y) { + + const uint64_t rows = samples*classes; + uint64_t row_idx; + + + if ((rows != X.rows()) || (X.cols() != 2)){ + X.resize(samples*classes, 2); + } + if (rows != y.rows()){ + y.resize(rows, 1); + } + + for (uint64_t i = 0; i < classes; ++i){ + for (uint64_t j = 0; j < samples; ++j){ + + row_idx = (i*samples) + j; + + X(row_idx, 0) = static_cast(i)/static_cast(classes) + utils::random(TX{-0.1}, TX{0.1}); + X(row_idx, 1) = TX{0.5} + utils::random(TX{-0.5}, TX{0.5}); + y(row_idx, 0) = static_cast(i); + } + } + } + + +} // end namesoace NN \ No newline at end of file diff --git a/include/modules/neural_networks/neural_networks.h b/include/modules/neural_networks/neural_networks.h index 3db2b2b..85523ed 100644 --- a/include/modules/neural_networks/neural_networks.h +++ b/include/modules/neural_networks/neural_networks.h @@ -2,6 +2,7 @@ #pragma once #include "datasets/spiral.h" +#include "datasets/vertical.h" #include "layers/Dense_Layer.h" diff --git a/include/numerics/matadd.h b/include/numerics/matadd.h index d3890c8..624e3c3 100644 --- a/include/numerics/matadd.h +++ b/include/numerics/matadd.h @@ -7,6 +7,39 @@ namespace numerics{ + template + void inplace_matadd_mat(utils::Matrix& A, const utils::Matrix& B) { + + const uint64_t rows = A.rows(); + const uint64_t cols = A.cols(); + + if (rows != B.rows() || cols != B.cols()) { + throw std::runtime_error("inplace_matadd: dimension mismatch"); + } + + for (uint64_t i = 0; i < cols; ++i) { + for (uint64_t j = 0; j < rows; ++j) { + A(j, i) += B(j, i); + } + } + } + + template + utils::Matrix matadd_mat(const utils::Matrix& A, const utils::Matrix& B) { + + const uint64_t rows = A.rows(); + const uint64_t cols = A.cols(); + + utils::Matrix C = A; + + if (rows != B.rows() || cols != B.cols()) { + throw std::runtime_error("inplace_matadd: dimension mismatch"); + } + + inplace_matadd_mat(C, B); + return C; + } + template void inplace_matadd_colvec(utils::Matrix& A, const utils::Vector& x) { @@ -94,133 +127,6 @@ namespace numerics{ } } - - - - /* - // -------------- Collapse(2) OpenMP ---------------- - template - utils::Vector matvec_omp(const utils::Matrix& A, const utils::Vector& x) { - if (A.cols() != x.size()) { - throw std::runtime_error("matvec: dimension mismatch"); - } - - const uint64_t m = A.rows(); - const uint64_t n = A.cols(); - - utils::Vector y(m, T{0}); // <-- y has length m (rows) - - - const T* xptr = x.data(); - const T* Aptr = A.data(); // row-major: A(i,j) == Aptr[i*n + j] - - // Each row i is an independent dot product: y[i] = dot(A[i,*], x) - #pragma omp parallel for schedule(static) - for (uint64_t i = 0; i < m; ++i) { - const T* row = Aptr + i * n; // contiguous row i - T acc = T{0}; - #pragma omp simd reduction(+:acc) - for (uint64_t j = 0; j < n; ++j) { - acc += row[j] * xptr[j]; - } - y[i] = acc; - } - - return y; - } - - // -------------- Auto OpenMP ---------------- - template - utils::Vector matvec_auto(const utils::Matrix& A, - const utils::Vector& x) { - - - uint64_t work = A.rows() * A.cols(); - - bool can_parallel = omp_config::omp_parallel_allowed(); - #ifdef _OPENMP - int threads = omp_get_max_threads(); - #else - int threads = 1; - #endif - - if (can_parallel || work > static_cast(threads) * 4ull) { - return matvec_omp(A,x); - } - else{ - // Safe fallback - return matvec(A,x); - } - - } - -// ================================================= -// y = x * A (Vector–Matrix product) -// ================================================= - template - utils::Vector vecmat(const utils::Vector& x, const utils::Matrix& A) { - if (x.size() != A.rows()) { - throw std::runtime_error("vecmat: dimension mismatch"); - } - const uint64_t m = A.rows(); - const uint64_t n = A.cols(); - - utils::Vector y(n, T{0}); - - for (uint64_t j = 0; j < n; ++j) { - for (uint64_t i = 0; i < m; ++i) { - y[j] += x[i] * A(i, j); - } - } - return y; - } - - // -------------- Collapse(2) OpenMP ---------------- - template - utils::Vector vecmat_omp(const utils::Vector& x, const utils::Matrix& A) { - if (x.size() != A.rows()) { - throw std::runtime_error("vecmat: dimension mismatch"); - } - const uint64_t m = A.rows(); - const uint64_t n = A.cols(); - - utils::Vector y(n, T{0}); - #pragma omp parallel for schedule(static) - for (uint64_t j = 0; j < n; ++j) { - T acc = T{0}; - for (uint64_t i = 0; i < m; ++i) { - acc += x[i] * A(i, j); - } - y[j] = acc; - } - return y; - } - - // -------------- Auto OpenMP ---------------- - template - utils::Vector vecmat_auto(const utils::Vector& x, - const utils::Matrix& A) { - - uint64_t work = A.rows() * A.cols(); - - bool can_parallel = omp_config::omp_parallel_allowed(); - #ifdef _OPENMP - int threads = omp_get_max_threads(); - #else - int threads = 1; - #endif - - if (can_parallel || work > static_cast(threads) * 4ull) { - return vecmat_omp(x,A); - } - else{ - // Safe fallback - return vecmat(x,A); - } - - } -*/ - } // namespace numerics #endif // _matadd_n_ \ No newline at end of file diff --git a/include/numerics/matargmax.h b/include/numerics/matargmax.h new file mode 100644 index 0000000..7fd411f --- /dev/null +++ b/include/numerics/matargmax.h @@ -0,0 +1,62 @@ +#pragma once + +#include "./utils/matrix.h" + +namespace numerics{ + + template + void inplace_matargmax_row(const utils::Matrix& A, utils::Vector& b){ + + if (b.size() != A.rows()){ + b.resize(A.rows(), Ti{0}); + } + Td value; + + for (uint64_t i = 0; i < A.rows(); ++i){ + value = Td{0}; + for (uint64_t j = 0; j < A.cols(); ++j){ + if (value < A(i,j)){ + value = A(i,j); + b[i] = j; + } + } + } + } + + template + void inplace_matargmax_col(const utils::Matrix& A, utils::Vector& b){ + + if (b.size() != A.cols()){ + b(A.cols(), Ti{0}); + } + Td value; + + for (uint64_t j = 0; j < A.cols(); ++j){ + value = Td{0}; + for (uint64_t i = 0; i < A.cols(); ++i){ + if (value < A(i,j)){ + value = A(i,j); + b[j] = i; + } + } + } + } + + template + utils::Vector matargmax_row(const utils::Matrix& A){ + utils::Vector b(A.rows(), Ti{0}); + inplace_matargmax_row(A, b); + return b; + + } + + template + utils::Vector matargmax_col(const utils::Matrix& A){ + utils::Vector b(A.rows(), Ti{0}); + inplace_matargmax_col(A, b); + return b; + + } + +} // namespace numerics + diff --git a/include/numerics/matscalar.h b/include/numerics/matscalar.h new file mode 100644 index 0000000..a4c2dfb --- /dev/null +++ b/include/numerics/matscalar.h @@ -0,0 +1,32 @@ +#pragma once + +#include "./utils/matrix.h" + +namespace numerics{ + + template + void inplace_matscalar(utils::Matrix& A, const T scalar){ + + const uint64_t rows = A.rows(); + const uint64_t cols = A.cols(); + + for (uint64_t i = 0; i < rows; ++i){ + for (uint64_t j = 0; j < cols; ++j){ + A(i,j) *= scalar; + } + } + } + + template + utils::Matrix matscalar(const utils::Matrix& A, T scalar){ + + utils::Matrix B = A; + inplace_matscalar(B, scalar); + return B; + + } + + + +} // namespace numerics + diff --git a/include/numerics/numerics.h b/include/numerics/numerics.h index d048dab..27f2a10 100644 --- a/include/numerics/numerics.h +++ b/include/numerics/numerics.h @@ -3,11 +3,17 @@ #include "./numerics/max.h" #include "./numerics/exp.h" #include "./numerics/log.h" +#include "./numerics/vecclip.h" +#include "./numerics/vecexp.h" +#include "./numerics/vecmax.h" +#include "./numerics/veclog.h" +#include "./numerics/vecargmax.h" #include "./numerics/initializers/eye.h" #include "./numerics/matequal.h" #include "./numerics/transpose.h" #include "./numerics/inverse.h" #include "./numerics/matmul.h" +#include "./numerics/matscalar.h" #include "./numerics/matmax.h" #include "./numerics/matdiv.h" #include "./numerics/matvec.h" @@ -18,12 +24,10 @@ #include "./numerics/matexp.h" #include "./numerics/matlog.h" #include "./numerics/matdot.h" +#include "./numerics/matargmax.h" #include "./numerics/min.h" #include "./numerics/abs.h" -#include "./numerics/vecclip.h" -#include "./numerics/vecexp.h" -#include "./numerics/vecmax.h" -#include "./numerics/veclog.h" + #include "./numerics/interpolation1d.h" // base diff --git a/include/numerics/vecargmax.h b/include/numerics/vecargmax.h new file mode 100644 index 0000000..f61b99d --- /dev/null +++ b/include/numerics/vecargmax.h @@ -0,0 +1,26 @@ +#pragma once + +#include "./utils/vector.h" + +namespace numerics{ + + template + T vecarmax(const utils::Vector& a){ + + const uint64_t N = a.size(); + uint64_t idx = 0; + T value; + + for (uint64_t i = 0; i < N; ++i){ + if (value > a[i]){ + value = a[i]; + idx = i; + + } + } + return idx; + } + + +} // namespace numerics + diff --git a/include/numerics/vecmean.h b/include/numerics/vecmean.h index 268b5aa..65b0b7a 100644 --- a/include/numerics/vecmean.h +++ b/include/numerics/vecmean.h @@ -22,6 +22,44 @@ namespace numerics{ return mean; } + template + T vecmean_isequal(utils::Vector& a, utils::Vector& b, T tol=1e-12) { + + uint64_t count = T{0}; + + const uint64_t N = a.size(); + + if (a.size() != b.size()){ + throw std::runtime_error("vecmean_equal: dimension mismatch"); + } + + + for (uint64_t i = 0; i < N; ++i) { + if ((a[i]-b[i]) < tol){ + count += 1; + } + } + return static_cast(count)/static_cast(N); + } + template + Td vecmean_equal(utils::Vector& a, utils::Vector& b) { + + Ti count = Ti{0}; + + const uint64_t N = a.size(); + + if (a.size() != b.size()){ + throw std::runtime_error("vecmean_equal: dimension mismatch"); + } + + + for (uint64_t i = 0; i < N; ++i) { + if (a[i]==b[i]){ + count += Ti{1}; + } + } + return static_cast(count)/static_cast(N); + } } // namespace numerics diff --git a/include/utils/matrix.h b/include/utils/matrix.h index c13cb3d..8929e72 100644 --- a/include/utils/matrix.h +++ b/include/utils/matrix.h @@ -23,7 +23,7 @@ namespace utils{ class Matrix{ public: Matrix() : rows_(0), cols_(0), data_() {} // Default constructor - +#include // Constructor to initialize matrix with rows × cols and a fill value Matrix(uint64_t rows, uint64_t cols, const T& value = T()) : rows_(rows), cols_(cols), data_(rows * cols, value) {} diff --git a/include/utils/vector.h b/include/utils/vector.h index 5780b19..03c03ce 100644 --- a/include/utils/vector.h +++ b/include/utils/vector.h @@ -7,7 +7,7 @@ #include - +#include "./utils/random.h" #include #include #include @@ -38,6 +38,24 @@ public: + + + void random(const uint64_t size, const T& lower, const T& higher){ + + v.resize(size, T{0}); + + // Copy data row by row + for (uint64_t i = 0; i < size; ++i) { + v[i] = utils::random(lower, higher); + } + + } + + + + + + //########################################################## //# VECTOR: --- basic properties --- # //########################################################## diff --git a/obj/main.d b/obj/main.d index de6b5cc..b7ed3f9 100644 --- a/obj/main.d +++ b/obj/main.d @@ -1,23 +1,24 @@ obj/main.o: src/main.cpp include/./core/omp_config.h \ include/./utils/utils.h include/./utils/vector.h \ - include/./utils/matrix.h include/./utils/random.h \ + include/./utils/random.h include/./utils/matrix.h \ include/./utils/generators.h include/./utils/generators/linspace.h \ include/utils/vector.h include/./utils/matcast.h \ include/./numerics/numerics.h include/./numerics/max.h \ include/./numerics/exp.h include/./numerics/log.h \ - include/./numerics/initializers/eye.h include/./numerics/matequal.h \ - include/./numerics/abs.h include/./numerics/transpose.h \ - include/./numerics/inverse.h \ - include/./numerics/inverse/inverse_gauss_jordan.h \ - include/./numerics/inverse/inverse_lu.h include/./decomp/lu.h \ - include/./numerics/matmul.h include/./numerics/matmax.h \ - include/./numerics/matdiv.h include/./numerics/matvec.h \ - include/./numerics/matadd.h include/./numerics/matsubtract.h \ - include/./numerics/matsum.h include/./numerics/matclip.h \ - include/./numerics/matexp.h include/./numerics/matlog.h \ - include/./numerics/matdot.h include/./numerics/min.h \ include/./numerics/vecclip.h include/./numerics/vecexp.h \ include/./numerics/vecmax.h include/./numerics/veclog.h \ + include/./numerics/vecargmax.h include/./numerics/initializers/eye.h \ + include/./numerics/matequal.h include/./numerics/abs.h \ + include/./numerics/transpose.h include/./numerics/inverse.h \ + include/./numerics/inverse/inverse_gauss_jordan.h \ + include/./numerics/inverse/inverse_lu.h include/./decomp/lu.h \ + include/./numerics/matmul.h include/./numerics/matscalar.h \ + include/./numerics/matmax.h include/./numerics/matdiv.h \ + include/./numerics/matvec.h include/./numerics/matadd.h \ + include/./numerics/matsubtract.h include/./numerics/matsum.h \ + include/./numerics/matclip.h include/./numerics/matexp.h \ + include/./numerics/matlog.h include/./numerics/matdot.h \ + include/./numerics/matargmax.h include/./numerics/min.h \ include/./numerics/interpolation1d.h \ include/./numerics/interpolation1d/interpolation1d_barycentric.h \ include/./numerics/interpolation1d/interpolation1d_base.h \ @@ -31,6 +32,7 @@ obj/main.o: src/main.cpp include/./core/omp_config.h \ include/utils/matrix.h \ include/./modules/neural_networks/neural_networks.h \ include/./modules/neural_networks/datasets/spiral.h \ + include/./modules/neural_networks/datasets/vertical.h \ include/./modules/neural_networks/layers/Dense_Layer.h \ include/./modules/neural_networks/activation_functions/Activation_ReLU.h \ include/./modules/neural_networks/activation_functions/Activation_Softmax.h \ @@ -40,8 +42,8 @@ obj/main.o: src/main.cpp include/./core/omp_config.h \ include/./core/omp_config.h: include/./utils/utils.h: include/./utils/vector.h: -include/./utils/matrix.h: include/./utils/random.h: +include/./utils/matrix.h: include/./utils/generators.h: include/./utils/generators/linspace.h: include/utils/vector.h: @@ -50,6 +52,11 @@ include/./numerics/numerics.h: include/./numerics/max.h: include/./numerics/exp.h: include/./numerics/log.h: +include/./numerics/vecclip.h: +include/./numerics/vecexp.h: +include/./numerics/vecmax.h: +include/./numerics/veclog.h: +include/./numerics/vecargmax.h: include/./numerics/initializers/eye.h: include/./numerics/matequal.h: include/./numerics/abs.h: @@ -59,6 +66,7 @@ include/./numerics/inverse/inverse_gauss_jordan.h: include/./numerics/inverse/inverse_lu.h: include/./decomp/lu.h: include/./numerics/matmul.h: +include/./numerics/matscalar.h: include/./numerics/matmax.h: include/./numerics/matdiv.h: include/./numerics/matvec.h: @@ -69,11 +77,8 @@ include/./numerics/matclip.h: include/./numerics/matexp.h: include/./numerics/matlog.h: include/./numerics/matdot.h: +include/./numerics/matargmax.h: include/./numerics/min.h: -include/./numerics/vecclip.h: -include/./numerics/vecexp.h: -include/./numerics/vecmax.h: -include/./numerics/veclog.h: include/./numerics/interpolation1d.h: include/./numerics/interpolation1d/interpolation1d_barycentric.h: include/./numerics/interpolation1d/interpolation1d_base.h: @@ -90,6 +95,7 @@ include/core/global_config.h: include/utils/matrix.h: include/./modules/neural_networks/neural_networks.h: include/./modules/neural_networks/datasets/spiral.h: +include/./modules/neural_networks/datasets/vertical.h: include/./modules/neural_networks/layers/Dense_Layer.h: include/./modules/neural_networks/activation_functions/Activation_ReLU.h: include/./modules/neural_networks/activation_functions/Activation_Softmax.h: diff --git a/obj/main.o b/obj/main.o index 4ff8bcb..daee6c6 100644 Binary files a/obj/main.o and b/obj/main.o differ diff --git a/src/main.cpp b/src/main.cpp index 1fcda2f..1ac4389 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -25,31 +25,107 @@ int main(int argc, char const *argv[]) utils::Mf X(10,2, 0); utils::Matrix y(10,1, 0); - neural_networks::create_spital_data(100, 3, X, y); + utils::Vector class_targets; + float loss; + float accuracy; + + //neural_networks::create_spital_data(10000, 3, X, y); + neural_networks::create_vertical_data(100, 3, X, y); neural_networks::Dense_Layer dense1(2, 3); - neural_networks::Activation_ReLU activation1; neural_networks::Dense_Layer dense2(3, 3); neural_networks::Activation_Softmax activation2; neural_networks::Loss_CategoricalCrossentrophy loss_funtion; - dense1.forward(X); - activation1.forward(dense1.outputs); - dense2.forward(activation1.outputs); - activation2.forward(dense2.outputs); + float lowest_loss = 9999999; + utils::Mf best_dense_1_weights = dense1.weights; + utils::Vf best_dense_1_biases = dense1.biases; + utils::Mf best_dense_2_weights = dense2.weights; + utils::Vf best_dense_2_biases = dense2.biases; + utils::Mf matrRND; + utils::Vf vectRND; + + utils::Vector predections; + + + + + + for (uint64_t i = 0; i < 10000; ++i){ + + + // Generate a new set of weights for iteration + matrRND.random(dense1.weights.rows(), dense1.weights.cols(), -1, 1); + numerics::inplace_matscalar(matrRND,0.05f); + numerics::inplace_matadd(dense1.weights, matrRND); + + dense1.biases.random(dense1.biases.size(), -1, 1); + + matrRND.random(dense2.weights.rows(), dense2.weights.cols(), -1, 1); + numerics::inplace_matscalar(matrRND,0.05f); + dense2.weights.random(dense2.weights.rows(), dense2.weights.cols(), -1, 1); + dense2.biases.random(dense2.biases.size(), -1, 1); + + // Perform a forward pass of the training data through this layer + dense1.forward(X); + activation1.forward(dense1.outputs); + dense2.forward(activation1.outputs); + activation2.forward(dense2.outputs); + + // Perform a farward pass through activation function + // it takes the output of the second dense layer here and returns loss + loss = loss_funtion.calculate(activation2.outputs, y); + + predections = numerics::matargmax_row(activation2.outputs); + + if (y.cols() < 1){ + class_targets = numerics::matargmax_row(y); + }else{ + class_targets = y.get_col(0); + } + + accuracy = numerics::vecmean_equal(predections, class_targets); + + if (loss < lowest_loss){ + std::cout << "New set of weights found, iteration:" << i << ", loss:" << loss << ", acc:" << accuracy << std::endl; + best_dense_1_weights = dense1.weights; + best_dense_1_biases = dense1.biases; + best_dense_2_weights = dense2.weights; + best_dense_2_biases = dense2.biases; + lowest_loss = loss; + } + + + } + + + + + + + for (int i = 0; i < 5; ++i){ std::cout << activation2.outputs.get_row(i) << std::endl; } - float loss = loss_funtion.calculate(activation2.outputs, y); std::cout << loss << std::endl; + + //predections.print(); + + + //predections.print(); + //class_targets.print(); + + std::cout << accuracy << std::endl; + +