Regulaization

Started on regulaization in  Loss.h. I need to refactor the matsum.h since I need a total sum over the matrix. Also matmul needs a elementwise matmul function, which is the next this in the ragulaization

examples/dense-neural-network/main.cpp

@@ -19,17 +19,19 @@
 int main(int argc, char const *argv[])
 {   
 
-    uint64_t number_of_classes = 5;
+    uint64_t number_of_classes = 3;
     uint64_t number_of_samples = 100;
-    uint64_t number_of_epochs = 100;
+    uint64_t number_of_epochs = 1000;
 
 
     utils::Mf X;
+    utils::Mf X_test;
     utils::Matrix<int64_t> y;
-    utils::Vector<int64_t> class_targets;
+    utils::Matrix<int64_t> y_test;
     float loss;
     float accuracy;
 
+    utils::Vector<int64_t> class_targets;
     utils::Vector<int64_t> predections;
 
 
@@ -52,7 +54,9 @@ int main(int argc, char const *argv[])
 
     // Create optimizer
     //neural_networks::Optimizer_SGD<float> optimizer(1, 1e-3, 0.5);
-    neural_networks::Optimizer_Adagrad<float> optimizer(1, 1e-3, 1e-6);
+    //neural_networks::Optimizer_Adagrad<float> optimizer(1, 1e-3, 1e-6);
+    //neural_networks::Optimizer_RMSprop<float> optimizer(1, 1e-3, 1e-6, 0.9);
+    neural_networks::Optimizer_Adam<float> optimizer(1, 1e-3, 1e-6, 0.9, 0.999);
     
 
 
@@ -73,6 +77,7 @@ int main(int argc, char const *argv[])
         // Perform a foard pass through the activation/loss function
         // takes the output of the second dense layer here and returns loss
         loss = loss_activation.forward(dense2.outputs, y);
+        loss_activation.loss.regularization_loss(dense1);
 
         // Calculate accuracy from output of activation2 and targets
         predections = numerics::matargmax_row<int64_t, float>(loss_activation.outputs);
@@ -109,6 +114,40 @@ int main(int argc, char const *argv[])
         optimizer.post_update_params();
 
     }
-    
+
+    // Validate the model
+
+    // Create dataset
+    neural_networks::create_spital_data<float, int64_t>(100, number_of_classes, X_test, y_test);
+
+    // Perform a forward pass of our testing data through this layer
+    dense1.forward(X_test);
+
+    // Perform a forward pass thourgh activation function
+    // takes the output fo the first layer here
+    activation1.forward(dense1.outputs);
+
+    // Perform a forward pass through second Dense layer
+    // takes output of activation function of the first layer as input
+    dense2.forward(activation1.outputs);
+
+    // Perform a foard pass through the activation/loss function
+    // takes the output of the second dense layer here and returns loss
+    loss = loss_activation.forward(dense2.outputs, y);
+
+    // Calculate accuracy from output of activation2 and targets
+    predections = numerics::matargmax_row<int64_t, float>(loss_activation.outputs);
+
+    if (y.cols() < 1){
+        class_targets = numerics::matargmax_row<int64_t, int64_t>(y);
+    }else{
+        class_targets = y.get_col(0);
+    }
+
+
+    accuracy = numerics::vecmean_equal<float>(predections, class_targets);
+
+    std::cout << "validation, acc: " << accuracy << ", loss: " << loss << std::endl;
+
     return 0;
 }