Sync public subset from Flux

include/modules/neural_networks/loss/Loss.h

@@ -5,30 +5,28 @@
 #include "./utils/vector.h"
 #include "./utils/matrix.h"
 
+#include "./numerics/vecmean.h"
 
 namespace neural_networks{
 
 	template <typename Td, typename Ti>
 	struct Loss{
 
-			utils::Matrix<Td> sample_losses;
-			Td data_losses;
+			utils::Vector<Td> sample_losses;
+			Td data_loss;
 
 			virtual utils::Vector<Td> forward(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y) = 0;
 
 			Td calculate(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y){
+				
 				// Calculate sample losses
 				sample_losses = forward(output, y);
 
 				// Calculate mean loss
-				data_losses = numerics::mean(sample_losses);
-				return data_losses;
+				data_loss = numerics::vecmean(sample_losses);
+				return data_loss;
 
 			}
-
-
 	};
 
-
-
 } // end namespace neural_networks

include/modules/neural_networks/loss/Loss_CategoricalCrossentrophy.h

@@ -0,0 +1,55 @@
+#pragma once
+
+#include "./core/omp_config.h"
+
+#include "./utils/vector.h"
+#include "./utils/matrix.h"
+#include "./utils/matcast.h"
+
+#include "./numerics/matclip.h"
+#include "./numerics/veclog.h"
+
+#include "./Loss.h"
+
+
+namespace neural_networks{
+
+	template <typename Td, typename Ti>
+	struct Loss_CategoricalCrossentrophy : Loss<Td, Ti>  {
+		
+			utils::Vector<Td> forward(const utils::Matrix<Td>& y_pred, const utils::Matrix<Ti>& y_true) override{
+				
+				utils::Vector<Td> correct_confidences(y_true.rows(), Td{0});
+				utils::Matrix<Td> cast_y_true = utils::matcast<Td, Ti>(y_true);
+
+				// Number of samles in a batch
+				const uint64_t samples = y_true.rows();
+
+				// Clip data to prevent dividning by 0
+				// Clip both sides to not drag mean towards any value
+				utils::Matrix<Td> y_pred_clipped = numerics::matclip(y_pred, Td{1e-7}, Td{1.0} - Td{1e-7});
+
+				// Probabilities for taget values
+				// only if categorical labes
+				if (y_true.cols() == 1){
+					for (uint64_t i = 0; i < y_true.rows(); ++i){
+						const uint64_t idx = static_cast<uint64_t>(y_true(i, 0));
+						correct_confidences[i] = y_pred_clipped(i, idx);
+					}
+				}else{ // Mask values - only for one-hot encoded labels
+					correct_confidences = numerics::matdot_row(y_pred_clipped, cast_y_true);
+
+				}
+				// Losses			
+				utils::Vector<Td> negative_log_likelihoods(samples, Td{0});
+				for (uint64_t i = 0; i < samples; ++i){
+				    negative_log_likelihoods[i] = -std::log(static_cast<Td>(correct_confidences[i]));
+				}
+
+				return negative_log_likelihoods;
+			}
+	};
+
+
+
+} // end namespace neural_networks

include/modules/neural_networks/loss/loss.h

@@ -1,34 +0,0 @@
-#pragma once
-
-#include "./core/omp_config.h"
-
-#include "./utils/vector.h"
-#include "./utils/matrix.h"
-
-
-namespace neural_networks{
-
-	template <typename Td, typename Ti>
-	struct Loss{
-
-			utils::Matrix<Td> sample_losses;
-			Td data_losses;
-
-			virtual utils::Vector<Td> forward(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y) = 0;
-
-			Td calculate(const utils::Matrix<Td>& output, const utils::Matrix<Ti>& y){
-				// Calculate sample losses
-				sample_losses = forward(output, y);
-
-				// Calculate mean loss
-				data_losses = numerics::mean(sample_losses);
-				return data_losses;
-
-			}
-
-
-	};
-
-
-
-} // end namespace neural_networks