diff --git a/bin/abc_lab b/bin/abc_lab index ac1a151..0c1f68d 100755 Binary files a/bin/abc_lab and b/bin/abc_lab differ diff --git a/include/modules/neural_networks/activation_functions/ReLU.h b/include/modules/neural_networks/activation_functions/ReLU.h new file mode 100644 index 0000000..9e38e10 --- /dev/null +++ b/include/modules/neural_networks/activation_functions/ReLU.h @@ -0,0 +1,28 @@ +#pragma once + +#include "./core/omp_config.h" + +#include "./utils/vector.h" +#include "./utils/matrix.h" +#include "./utils/random.h" + + +namespace neural_networks{ + + template + struct activation_ReLU{ + + utils::Matrix outputs; + + void forward(utils::Matrix inputs){ + outputs = numerics::max(inputs, T{0}); + //outputs.print(); + } + + + + }; + + + +} // end namespace neural_networks \ No newline at end of file diff --git a/include/modules/neural_networks/activation_functions/Softmax.h b/include/modules/neural_networks/activation_functions/Softmax.h new file mode 100644 index 0000000..837b91c --- /dev/null +++ b/include/modules/neural_networks/activation_functions/Softmax.h @@ -0,0 +1,28 @@ +#pragma once + +#include "./core/omp_config.h" + +#include "./utils/vector.h" +#include "./utils/matrix.h" +#include "./utils/random.h" + + +namespace neural_networks{ + + template + struct activation_softmax{ + + utils::Matrix outputs; + + void forward(utils::Matrix inputs){ + //outputs = numerics::max(inputs, T{0}); + //outputs.print(); + } + + + + }; + + + +} // end namespace neural_networks \ No newline at end of file diff --git a/include/modules/neural_networks/datasets/spiral.h b/include/modules/neural_networks/datasets/spiral.h new file mode 100644 index 0000000..d0dd473 --- /dev/null +++ b/include/modules/neural_networks/datasets/spiral.h @@ -0,0 +1,61 @@ +#pragma once + +#include "./core/omp_config.h" +#include "./utils/matrix.h" +#include "./utils/vector.h" +#include "./utils/random.h" + +//#include + + +namespace neural_networks{ + + template + void create_spital_data(const uint64_t samples, const uint64_t classes, utils::Matrix& X, utils::Vector& y) { + + const uint64_t rows = samples*classes; + T r, t; + uint64_t row_idx; + + + if ((rows != X.rows()) || (X.cols() != 2)){ + X.resize(samples*classes, 2); + } + if (rows != y.size()){ + y.resize(rows); + } + + for (uint64_t i = 0; i < classes; ++i){ + for (uint64_t j = 0; j < samples; ++j){ + r = static_cast(j)/static_cast(samples); + t = static_cast(i)*4.0 + (4.0+r); + row_idx = (i*samples) + j; + + X(row_idx, 0) = r*std::cos(t*2.5) + utils::random(T{-0.15}, T{0.15}); + X(row_idx, 1) = r*std::sin(t*2.5) + utils::random(T{-0.15}, T{0.15}); + y[row_idx] = i; + } + } + + + + /* + + utils::Matrix X(static_cast(samples*classes), 3, T{0}); + + const uint64_t rows = A.rows(); + const uint64_t cols = A.cols(); + + if (rows != x.size()) { + throw std::runtime_error("inplace_matadd_colvec: dimension mismatch"); + } + + for (uint64_t i = 0; i < cols; ++i) { + for (uint64_t j = 0; j < rows; ++j) { + A(j, i) += x[j]; + } + }*/ + } + + +} // end namesoace NN \ No newline at end of file diff --git a/include/modules/neural_networks/layers/dense_layer.h b/include/modules/neural_networks/layers/dense_layer.h new file mode 100644 index 0000000..28aa973 --- /dev/null +++ b/include/modules/neural_networks/layers/dense_layer.h @@ -0,0 +1,42 @@ +#pragma once + +#include "./core/omp_config.h" + +#include "./utils/vector.h" +#include "./utils/matrix.h" +#include "./utils/random.h" + + +namespace neural_networks{ + + template + struct dense_layer{ + utils::Matrix weights; + utils::Vector biases; + utils::Matrix outputs; + + // Default Constructor + dense_layer() = default; + + // Constructor + dense_layer(const uint64_t n_inputs, const uint64_t n_neurons){ + + weights.random(n_inputs, n_neurons, -1, 1); + biases.resize(n_neurons, T{0}); + weights.print(); + //outputs.resize() + + } + + + void forward(utils::Matrix inputs){ + outputs = numerics::matadd(numerics::matmul_auto(inputs, (weights)), biases, "row"); + } + + + + }; + + + +} // end namespace neural_networks \ No newline at end of file diff --git a/include/modules/neural_networks/neural_networks.h b/include/modules/neural_networks/neural_networks.h new file mode 100644 index 0000000..f905c20 --- /dev/null +++ b/include/modules/neural_networks/neural_networks.h @@ -0,0 +1,9 @@ +// #include "./modules/neural_networks/neural_networks.h" +#pragma once + +#include "datasets/spiral.h" + +#include "layers/dense_layer.h" + + +#include "activation_functions/ReLU.h" \ No newline at end of file diff --git a/include/numerics/matadd.h b/include/numerics/matadd.h new file mode 100644 index 0000000..58853e8 --- /dev/null +++ b/include/numerics/matadd.h @@ -0,0 +1,229 @@ +#ifndef _matadd_n_ +#define _matadd_n_ + + +#include "./utils/matrix.h" +#include "./core/omp_config.h" + +namespace numerics{ + +// ================================================= +// y = A * x (Matrix–Vector product) +// ================================================= + template + void inplace_matadd_colvec(utils::Matrix& A, const utils::Vector& x) { + + const uint64_t rows = A.rows(); + const uint64_t cols = A.cols(); + + if (rows != x.size()) { + throw std::runtime_error("inplace_matadd_colvec: dimension mismatch"); + } + + for (uint64_t i = 0; i < cols; ++i) { + for (uint64_t j = 0; j < rows; ++j) { + A(j, i) += x[j]; + } + } + } + + template + void inplace_matadd_rowvec(utils::Matrix& A, const utils::Vector& x) { + + const uint64_t rows = A.rows(); + const uint64_t cols = A.cols(); + + if (cols != x.size()) { + throw std::runtime_error("inplace_matadd_rowvec: dimension mismatch"); + } + + for (uint64_t i = 0; i < cols; ++i) { + for (uint64_t j = 0; j < rows; ++j) { + A(j, i) += x[i]; + } + } + } + + template + utils::Matrix matadd_colvec(const utils::Matrix& A, const utils::Vector& x) { + + //const uint64_t rows = A.rows(); + //const uint64_t cols = A.cols(); + + utils::Matrix B = A; + + inplace_matadd_colvec(B, x); + + return B; + } + + template + utils::Matrix matadd_rowvec(const utils::Matrix& A, const utils::Vector& x) { + + //const uint64_t rows = A.rows(); + //const uint64_t cols = A.cols(); + + utils::Matrix B = A; + + inplace_matadd_rowvec(B, x); + + return B; + } + + template + utils::Matrix matadd(const utils::Matrix& A, const utils::Vector& x, std::string method = "auto"){ + + const uint64_t rows = A.rows(); + const uint64_t cols = A.cols(); + const uint64_t N = x.size(); + + if (method=="auto"){ + + if (rows==cols){ + throw std::runtime_error("matadd: too many options for dimensions"); + } else if (rows == N){ + return matadd_rowvec(A, x); + } else if (cols == N){ + return matadd_colvec(A, x); + }else{ + throw std::runtime_error("matadd: undefined fault - auto"); + } + }else if(method=="row"){ + return matadd_rowvec(A, x); + } else if (method=="col"){ + return matadd_colvec(A, x); + }else{ + throw std::runtime_error("matadd: undefined fault - defined method"); + } + } + + + + + /* + // -------------- Collapse(2) OpenMP ---------------- + template + utils::Vector matvec_omp(const utils::Matrix& A, const utils::Vector& x) { + if (A.cols() != x.size()) { + throw std::runtime_error("matvec: dimension mismatch"); + } + + const uint64_t m = A.rows(); + const uint64_t n = A.cols(); + + utils::Vector y(m, T{0}); // <-- y has length m (rows) + + + const T* xptr = x.data(); + const T* Aptr = A.data(); // row-major: A(i,j) == Aptr[i*n + j] + + // Each row i is an independent dot product: y[i] = dot(A[i,*], x) + #pragma omp parallel for schedule(static) + for (uint64_t i = 0; i < m; ++i) { + const T* row = Aptr + i * n; // contiguous row i + T acc = T{0}; + #pragma omp simd reduction(+:acc) + for (uint64_t j = 0; j < n; ++j) { + acc += row[j] * xptr[j]; + } + y[i] = acc; + } + + return y; + } + + // -------------- Auto OpenMP ---------------- + template + utils::Vector matvec_auto(const utils::Matrix& A, + const utils::Vector& x) { + + + uint64_t work = A.rows() * A.cols(); + + bool can_parallel = omp_config::omp_parallel_allowed(); + #ifdef _OPENMP + int threads = omp_get_max_threads(); + #else + int threads = 1; + #endif + + if (can_parallel || work > static_cast(threads) * 4ull) { + return matvec_omp(A,x); + } + else{ + // Safe fallback + return matvec(A,x); + } + + } + +// ================================================= +// y = x * A (Vector–Matrix product) +// ================================================= + template + utils::Vector vecmat(const utils::Vector& x, const utils::Matrix& A) { + if (x.size() != A.rows()) { + throw std::runtime_error("vecmat: dimension mismatch"); + } + const uint64_t m = A.rows(); + const uint64_t n = A.cols(); + + utils::Vector y(n, T{0}); + + for (uint64_t j = 0; j < n; ++j) { + for (uint64_t i = 0; i < m; ++i) { + y[j] += x[i] * A(i, j); + } + } + return y; + } + + // -------------- Collapse(2) OpenMP ---------------- + template + utils::Vector vecmat_omp(const utils::Vector& x, const utils::Matrix& A) { + if (x.size() != A.rows()) { + throw std::runtime_error("vecmat: dimension mismatch"); + } + const uint64_t m = A.rows(); + const uint64_t n = A.cols(); + + utils::Vector y(n, T{0}); + #pragma omp parallel for schedule(static) + for (uint64_t j = 0; j < n; ++j) { + T acc = T{0}; + for (uint64_t i = 0; i < m; ++i) { + acc += x[i] * A(i, j); + } + y[j] = acc; + } + return y; + } + + // -------------- Auto OpenMP ---------------- + template + utils::Vector vecmat_auto(const utils::Vector& x, + const utils::Matrix& A) { + + uint64_t work = A.rows() * A.cols(); + + bool can_parallel = omp_config::omp_parallel_allowed(); + #ifdef _OPENMP + int threads = omp_get_max_threads(); + #else + int threads = 1; + #endif + + if (can_parallel || work > static_cast(threads) * 4ull) { + return vecmat_omp(x,A); + } + else{ + // Safe fallback + return vecmat(x,A); + } + + } +*/ + +} // namespace numerics + +#endif // _matadd_n_ \ No newline at end of file diff --git a/include/numerics/matmul.h b/include/numerics/matmul.h index 6461e31..c2a25a7 100644 --- a/include/numerics/matmul.h +++ b/include/numerics/matmul.h @@ -11,11 +11,11 @@ namespace numerics{ // ---------------- Serial baseline ---------------- template utils::Matrix matmul(const utils::Matrix& A, const utils::Matrix& B){ - + if(A.cols() != B.rows()){ throw std::runtime_error("matmul: dimension mismatch"); } - + const uint64_t m = A.rows(); const uint64_t n = A.cols(); // also B.rows() const uint64_t p = B.cols(); @@ -98,6 +98,7 @@ utils::Matrix matmul_auto(const utils::Matrix& A, // Tiny problems: serial is cheapest. if (!can_parallel || work < threads*4ull) { + return matmul(A,B); } // Plenty of (i,j) work → collapse(2) is a great default. diff --git a/include/numerics/max.h b/include/numerics/max.h index ca3797b..3f8402f 100644 --- a/include/numerics/max.h +++ b/include/numerics/max.h @@ -17,6 +17,35 @@ namespace numerics{ } } + template + void inplace_max(utils::Matrix& A, const T b){ + + const uint64_t rows = A.rows(); + const uint64_t cols = A.cols(); + + for (uint64_t i = 0; i < rows; ++i){ + for (uint64_t j = 0; j < cols; ++j){ + + if (b > A(i,j)){ + //std::cout << A(i,j) << std::endl; + A(i,j) = b; + //std::cout << A(i,j) << std::endl; + } + } + } + } + + template + utils::Matrix max(const utils::Matrix& A, const T b){ + + utils::Matrix B = A; + + inplace_max(B, b); + + + return B; + } + } // namespace numerics diff --git a/include/numerics/numerics.h b/include/numerics/numerics.h index bb4760e..fb5d2b7 100644 --- a/include/numerics/numerics.h +++ b/include/numerics/numerics.h @@ -7,6 +7,7 @@ #include "./numerics/inverse.h" #include "./numerics/matmul.h" #include "./numerics/matvec.h" +#include "./numerics/matadd.h" #include "./numerics/min.h" #include "./numerics/max.h" #include "./numerics/abs.h" diff --git a/include/numerics/transpose.h b/include/numerics/transpose.h index 3052e56..d4fd38a 100644 --- a/include/numerics/transpose.h +++ b/include/numerics/transpose.h @@ -103,6 +103,7 @@ namespace numerics{ uint64_t threads = 1; #endif + if (can_parallel && work > threads * 4ull) { inplace_transpose_square_omp(A); }else { @@ -118,8 +119,10 @@ namespace numerics{ uint64_t work = A.rows() * A.cols(); + + if (rows==cols){ - utils::Matrix B(rows, cols, T{0}); + utils::Matrix B = A; inplace_transpose_square_auto(B); return B; } diff --git a/include/utils/matrix.h b/include/utils/matrix.h index ad72e9d..c13cb3d 100644 --- a/include/utils/matrix.h +++ b/include/utils/matrix.h @@ -2,12 +2,16 @@ #define _matrix_n_ #include "./utils/vector.h" +#include "./utils/random.h" #ifdef _OPENMP #include #endif +#include + + #include namespace utils{ @@ -25,6 +29,72 @@ public: : rows_(rows), cols_(cols), data_(rows * cols, value) {} + // Construct from list-of-lists: + // utils::Mf A{{1,2,3}, {4,5,6}}; + Matrix(std::initializer_list> init) { + rows_ = static_cast(init.size()); + if (rows_ > 0) { + cols_ = static_cast(init.begin()->size()); + } else { + cols_ = 0; + } + + // Validate all rows have the same length + for (const auto& row : init) { + if (row.size() != cols_) { + throw std::runtime_error("Matrix(list of lists): ragged rows"); + } + } + + data_.resize(rows_ * cols_); + uint64_t r = 0; + for (uint64_t i = 0; i < init.size(); ++i, ++r){ + const std::initializer_list& row = *(init.begin() + i); + + uint64_t c = 0; + for (uint64_t j = 0; j < row.size(); ++j, ++c){ + const T& val = *(row.begin() + j); + data_[r * cols_ + c] = val; + } + } + } + + + // Assign from list-of-lists after default construction: + // utils::Md M; M = {{1,2},{3,4}}; + Matrix& operator=(std::initializer_list> init) { + // Set sizes + rows_ = static_cast(init.size()); + if (rows_ > 0) { + cols_ = static_cast((init.begin())->size()); + } else { + cols_ = 0; + } + + // Validate: all rows must have same length + for (uint64_t i = 0; i < rows_; ++i) { + const std::initializer_list& row = *(init.begin() + i); + if (row.size() != cols_) { + throw std::runtime_error("Matrix(list of lists): ragged rows"); + } + } + + // Allocate storage + data_.resize(rows_ * cols_); + + // Copy data row by row + for (uint64_t i = 0; i < rows_; ++i) { + const std::initializer_list& row = *(init.begin() + i); + for (uint64_t j = 0; j < cols_; ++j) { + const T& val = *(row.begin() + j); + data_[i * cols_ + j] = val; + } + } + + return *this; + } + + //# MATRIX: basic properties # uint64_t rows() const noexcept {return rows_;} uint64_t cols() const noexcept {return cols_;} @@ -45,6 +115,20 @@ public: } + void random(uint64_t rows, uint64_t cols, const T& lower, const T& higher){ + rows_ = rows; + cols_ = cols; + data_.resize(rows_*cols_, 0); + + // Copy data row by row + for (uint64_t i = 0; i < rows_; ++i) { + for (uint64_t j = 0; j < cols_; ++j) { + data_[i * cols_ + j] = utils::random(lower, higher); + } + } + + } + //# MATRIX: row helpers (copy out) # // Read whole row as an owning Vector diff --git a/include/utils/random.h b/include/utils/random.h new file mode 100644 index 0000000..8540982 --- /dev/null +++ b/include/utils/random.h @@ -0,0 +1,37 @@ +#pragma once + +#include "./core/omp_config.h" +#include + +namespace utils{ + + // Shared engine + inline std::mt19937& rng() { + static std::random_device rd; + static std::mt19937 gen(rd()); + return gen; + } + + // Integral overload + template < + typename T, + typename std::enable_if::value, int>::type = 0 + > + T random(T low, T high) { + std::uniform_int_distribution dist(low, high); + return dist(rng()); + } + + // Floating-point overload + template < + typename T, + typename std::enable_if::value, int>::type = 0 + > + T random(T low, T high) { + std::uniform_real_distribution dist(low, high); + return dist(rng()); + } + + +} // end namespace utils + diff --git a/include/utils/utils.h b/include/utils/utils.h index 86904a4..5b7b8bf 100644 --- a/include/utils/utils.h +++ b/include/utils/utils.h @@ -3,4 +3,5 @@ #include "./utils/vector.h" #include "./utils/matrix.h" -#include "./utils/generators.h" \ No newline at end of file +#include "./utils/generators.h" +#include "./utils/random.h" diff --git a/include/utils/vector.h b/include/utils/vector.h index 9b71ac1..5780b19 100644 --- a/include/utils/vector.h +++ b/include/utils/vector.h @@ -5,6 +5,8 @@ #include #include +#include + #include #include @@ -31,6 +33,9 @@ public: v.resize(size, value); } + // Construct from a braced list: utils::Vf v{1,2,3}; + Vector(std::initializer_list init) : v(init) {} + //########################################################## @@ -47,6 +52,13 @@ public: // a = vector[2]; const T& operator[](uint64_t idx) const { return v[idx]; } + + // Assign from a braced list after default construction: + Vector& operator=(std::initializer_list init) { + v = init; + return *this; + } + // vector.size(); uint64_t size() const noexcept { return v.size(); } diff --git a/makefile b/makefile index 3d04849..fb36b99 100644 --- a/makefile +++ b/makefile @@ -51,7 +51,7 @@ OMP_MAX_LEVELS ?= 1 # 1 = no nested teams; set 2+ to allow nesting OMP_THREADS ?= 16 # e.g. "16" or "8,4" for nested (outer,inner) OMP_DYNAMIC ?= TRUE # TRUE/FALSE: let runtime adjust threads OMP_SCHEDULE ?= STATIC # STATIC recommended for matvec/matmul -OMP_DISPLAY_ENV ?= TRUE # TRUE to print runtime config at startup +OMP_DISPLAY_ENV ?= FALSE # TRUE to print runtime config at startup # Export OMP defaults so child makes or tools see them (not strictly required) export OMP_PROC_BIND diff --git a/obj/main.d b/obj/main.d index be73496..2f70e92 100644 --- a/obj/main.d +++ b/obj/main.d @@ -1,15 +1,16 @@ obj/main.o: src/main.cpp include/./core/omp_config.h \ include/./utils/utils.h include/./utils/vector.h \ - include/./utils/matrix.h include/./utils/generators.h \ - include/./utils/generators/linspace.h include/utils/vector.h \ - include/./numerics/numerics.h include/./numerics/initializers/eye.h \ - include/./numerics/matequal.h include/./numerics/abs.h \ - include/./numerics/transpose.h include/./numerics/inverse.h \ + include/./utils/matrix.h include/./utils/random.h \ + include/./utils/generators.h include/./utils/generators/linspace.h \ + include/utils/vector.h include/./numerics/numerics.h \ + include/./numerics/initializers/eye.h include/./numerics/matequal.h \ + include/./numerics/abs.h include/./numerics/transpose.h \ + include/./numerics/inverse.h \ include/./numerics/inverse/inverse_gauss_jordan.h \ include/./numerics/inverse/inverse_lu.h include/./decomp/lu.h \ include/./numerics/matmul.h include/./numerics/matvec.h \ - include/./numerics/min.h include/./numerics/max.h \ - include/./numerics/interpolation1d.h \ + include/./numerics/matadd.h include/./numerics/min.h \ + include/./numerics/max.h include/./numerics/interpolation1d.h \ include/./numerics/interpolation1d/interpolation1d_barycentric.h \ include/./numerics/interpolation1d/interpolation1d_base.h \ include/./numerics/interpolation1d/interpolation1d_cubic_spline.h \ @@ -19,11 +20,16 @@ obj/main.o: src/main.cpp include/./core/omp_config.h \ include/./decomp/decomp.h include/./modules/mesh/mesh.h \ include/modules/mesh/mesh1d.h include/modules/fluids/fluids.h \ include/modules/fluids/diffusion1d.h include/core/global_config.h \ - include/utils/matrix.h + include/utils/matrix.h \ + include/./modules/neural_networks/neural_networks.h \ + include/./modules/neural_networks/datasets/spiral.h \ + include/./modules/neural_networks/layers/dense_layer.h \ + include/./modules/neural_networks/activation_functions/ReLU.h include/./core/omp_config.h: include/./utils/utils.h: include/./utils/vector.h: include/./utils/matrix.h: +include/./utils/random.h: include/./utils/generators.h: include/./utils/generators/linspace.h: include/utils/vector.h: @@ -38,6 +44,7 @@ include/./numerics/inverse/inverse_lu.h: include/./decomp/lu.h: include/./numerics/matmul.h: include/./numerics/matvec.h: +include/./numerics/matadd.h: include/./numerics/min.h: include/./numerics/max.h: include/./numerics/interpolation1d.h: @@ -54,3 +61,7 @@ include/modules/fluids/fluids.h: include/modules/fluids/diffusion1d.h: include/core/global_config.h: include/utils/matrix.h: +include/./modules/neural_networks/neural_networks.h: +include/./modules/neural_networks/datasets/spiral.h: +include/./modules/neural_networks/layers/dense_layer.h: +include/./modules/neural_networks/activation_functions/ReLU.h: diff --git a/obj/main.o b/obj/main.o index 15bd227..debce12 100644 Binary files a/obj/main.o and b/obj/main.o differ diff --git a/src/main.cpp b/src/main.cpp index 5efbf29..83b8280 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -8,6 +8,8 @@ #include "./modules/mesh/mesh.h" #include "modules/fluids/fluids.h" +#include "./modules/neural_networks/neural_networks.h" + //#include @@ -19,7 +21,40 @@ int main(int argc, char const *argv[]) -{ +{ + + utils::Mf X(10,2, 0); + utils::Vf y(10, 0); + neural_networks::create_spital_data(100, 3, X, y); + + neural_networks::dense_layer layer(2, 3); + + neural_networks::activation_ReLU RelU; + + layer.forward(X); + RelU.forward(layer.outputs); + + for (int i = 0; i < 5; ++i){ + std::cout << RelU.outputs.get_row(i) << std::endl; + } + + //layer1_output.print(); + //layer2_output.print(); + + //std::cout << output << std::endl; + + + + + + + + + + + + + /* utils::Vd a = utils::linspace(1, 10, 10, true); a.print(); mesh::Mesh1D mesh(a); @@ -40,7 +75,7 @@ int main(int argc, char const *argv[]) fluids::Diffusion1D diffusion(cfg, mesh, Gamma); diffusion.assemble(A, b, s); - +*/ return 0; } \ No newline at end of file