From 951e5d84cc5e82fd538e9e184388bff60b8be4a4 Mon Sep 17 00:00:00 2001 From: adamhutchings Date: Tue, 17 Oct 2023 11:05:28 -0400 Subject: [PATCH 1/6] Add basic loss function --- core/include/jml/math/loss_functions.hpp | 46 ++++++++++++++++++++++ core/meson.build | 4 +- core/src/math/loss_functions.cpp | 50 ++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 core/include/jml/math/loss_functions.hpp create mode 100644 core/src/math/loss_functions.cpp diff --git a/core/include/jml/math/loss_functions.hpp b/core/include/jml/math/loss_functions.hpp new file mode 100644 index 0000000..f5a3154 --- /dev/null +++ b/core/include/jml/math/loss_functions.hpp @@ -0,0 +1,46 @@ +/** + * This offers loss functions for a model. Only one such loss function exists + * right now, but we may need more in the future. + * + * Each loss function needs an actual calculation (put in a jml::Vector, get out + * a double), and a derivative with respect to a particular entry. That is, if + * we change the i-th entry of the input vector, how much the loss will change + * by. This will be very important later for gradient descent. + * + * Author: Adam Hutchings + * Date: 10-17-23 +*/ + +#pragma once + +#include + +#include +#include + +namespace jml { + +typedef std::function LF; +typedef std::function DL; + +class JML_API LossFunction { + +private: + LF loss; + DL dl; + +public: + // Create a loss function with a given loss and derivative. + LossFunction(const LF& lf, const DL& dl); + double get_loss(const jml::Vector& actual, const jml::Vector& expected); + double get_loss_derivative( + const jml::Vector& actual, const jml::Vector& expected, int index + ); + +}; + +// Also, we provide an example set of loss functions (L^2 norm) +extern LF l2lf; +extern DL l2dl; + +} diff --git a/core/meson.build b/core/meson.build index 00f7bfc..ef4bcc4 100644 --- a/core/meson.build +++ b/core/meson.build @@ -10,7 +10,9 @@ endif # TODO: pkg-config stuff -core_sources = ['src/math/matrix.cpp', 'src/math/vector.cpp', 'src/math/activation_functions.cpp', 'src/model.cpp', 'src/logger.cpp', 'capi/cjml.cpp'] +core_sources = ['src/math/matrix.cpp', 'src/math/vector.cpp', +'src/math/activation_functions.cpp', 'src/model.cpp', 'src/logger.cpp', +'capi/cjml.cpp', 'src/math/loss_functions.cpp'] jmlcore = library('jmlcore', core_sources, include_directories: core_inc, diff --git a/core/src/math/loss_functions.cpp b/core/src/math/loss_functions.cpp new file mode 100644 index 0000000..3d645f6 --- /dev/null +++ b/core/src/math/loss_functions.cpp @@ -0,0 +1,50 @@ +#include + +#include + +#include + +namespace jml { + +LossFunction::LossFunction(const LF& lf, const DL& dl) { + this->loss = lf; + this->dl = dl; +} + +double LossFunction::get_loss( + const Vector& actual, const Vector& expected +) { + return this->loss(actual, expected); +} + +double LossFunction::get_loss_derivative( + const Vector& actual, const Vector& expected, int index +) { + return this->dl(actual, expected, index); +} + +LF l2lf = [](const Vector& actual, const Vector& expected) { + int a = actual.get_size(), e = expected.get_size(); + if (a != e) { + LOGGER->log(Log(WARN) + << "Tried to compare a vector of length " << a + << "to a vector of length " << e << ".\n"); + } + double total = 0; + double diff; + for (int i = 0; i < a; ++i) { + diff = actual.get_entry(i) - expected.get_entry(i); + diff *= diff; + total += diff; + } + return sqrt(total); +}; + +DL l2dl = [](const Vector& actual, const Vector& expected, int i) { + double l = l2lf(actual, expected); + double ret = 1.0 / (2 * l); + ret *= 2 * (actual.get_entry(i) - expected.get_entry(i)); + return ret; +}; + +} From 336cdbe1965e807dd436eee7796d85e8db96759a Mon Sep 17 00:00:00 2001 From: adamhutchings Date: Tue, 17 Oct 2023 23:47:26 -0400 Subject: [PATCH 2/6] Add test for loss function --- tests/core/math/test_losses.cpp | 17 +++++++++++++++++ tests/meson.build | 1 + 2 files changed, 18 insertions(+) create mode 100644 tests/core/math/test_losses.cpp diff --git a/tests/core/math/test_losses.cpp b/tests/core/math/test_losses.cpp new file mode 100644 index 0000000..922a7dd --- /dev/null +++ b/tests/core/math/test_losses.cpp @@ -0,0 +1,17 @@ +#include "catch2/catch_test_macros.hpp" +#include +#include + +SCENARIO("Loss can be calculated from two vectors") { + GIVEN("Two vectors of the same size") { + jml::Vector vec1(3), vec2(3); + vec1.set_entry(0, 1.0); + vec1.set_entry(1, 2.0); + vec1.set_entry(2, 3.0); + vec2.set_entry(0, 4.0); + vec2.set_entry(1, 2.0); + vec2.set_entry(2,-2.0); + auto loss = jml::l2lf(vec1, vec2); + REQUIRE(loss == 5.0); + } +} diff --git a/tests/meson.build b/tests/meson.build index 5d2749f..c5d8c0c 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -2,6 +2,7 @@ core_test_sources = [ 'core/math/test_matrix.cpp', 'core/math/test_vector.cpp', 'core/math/test_activation_functions.cpp', + 'core/math/test_losses.cpp', 'core/test_model.cpp', ] From 61af64110e93b81b4bce204e683ae11f5db282c7 Mon Sep 17 00:00:00 2001 From: adamhutchings Date: Wed, 18 Oct 2023 02:17:16 -0400 Subject: [PATCH 3/6] Use visibility --- core/include/jml/math/loss_functions.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/include/jml/math/loss_functions.hpp b/core/include/jml/math/loss_functions.hpp index f5a3154..e3f7652 100644 --- a/core/include/jml/math/loss_functions.hpp +++ b/core/include/jml/math/loss_functions.hpp @@ -40,7 +40,7 @@ class JML_API LossFunction { }; // Also, we provide an example set of loss functions (L^2 norm) -extern LF l2lf; -extern DL l2dl; +extern JML_API LF l2lf; +extern JML_API DL l2dl; } From 039410add83764315b6a1161e1d1aa44e58b17e1 Mon Sep 17 00:00:00 2001 From: adamhutchings Date: Wed, 18 Oct 2023 15:17:29 -0400 Subject: [PATCH 4/6] Fix test entries --- tests/core/math/test_losses.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/math/test_losses.cpp b/tests/core/math/test_losses.cpp index 922a7dd..c005cf0 100644 --- a/tests/core/math/test_losses.cpp +++ b/tests/core/math/test_losses.cpp @@ -10,8 +10,8 @@ SCENARIO("Loss can be calculated from two vectors") { vec1.set_entry(2, 3.0); vec2.set_entry(0, 4.0); vec2.set_entry(1, 2.0); - vec2.set_entry(2,-2.0); + vec2.set_entry(2,-1.0); auto loss = jml::l2lf(vec1, vec2); REQUIRE(loss == 5.0); } -} +} \ No newline at end of file From a32c0e3e5ee1c27553f6e4088af58890931336f7 Mon Sep 17 00:00:00 2001 From: adamhutchings Date: Thu, 19 Oct 2023 10:35:02 -0400 Subject: [PATCH 5/6] Does this work? --- core/include/jml/math/loss_functions.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/include/jml/math/loss_functions.hpp b/core/include/jml/math/loss_functions.hpp index e3f7652..c5bca5e 100644 --- a/core/include/jml/math/loss_functions.hpp +++ b/core/include/jml/math/loss_functions.hpp @@ -20,8 +20,8 @@ namespace jml { -typedef std::function LF; -typedef std::function DL; +typedef JML_API std::function LF; +typedef JML_API std::function DL; class JML_API LossFunction { From c7fa101e143ef68b1666dd264681cf977e3c4c1c Mon Sep 17 00:00:00 2001 From: adamhutchings Date: Thu, 19 Oct 2023 11:50:11 -0400 Subject: [PATCH 6/6] Does this work? --- core/include/jml/math/loss_functions.hpp | 4 ++-- core/src/math/loss_functions.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/include/jml/math/loss_functions.hpp b/core/include/jml/math/loss_functions.hpp index c5bca5e..e3f7652 100644 --- a/core/include/jml/math/loss_functions.hpp +++ b/core/include/jml/math/loss_functions.hpp @@ -20,8 +20,8 @@ namespace jml { -typedef JML_API std::function LF; -typedef JML_API std::function DL; +typedef std::function LF; +typedef std::function DL; class JML_API LossFunction { diff --git a/core/src/math/loss_functions.cpp b/core/src/math/loss_functions.cpp index 3d645f6..8cbfdda 100644 --- a/core/src/math/loss_functions.cpp +++ b/core/src/math/loss_functions.cpp @@ -23,7 +23,7 @@ double LossFunction::get_loss_derivative( return this->dl(actual, expected, index); } -LF l2lf = [](const Vector& actual, const Vector& expected) { +JML_API LF l2lf = [](const Vector& actual, const Vector& expected) { int a = actual.get_size(), e = expected.get_size(); if (a != e) { LOGGER->log(Log(WARN) @@ -40,7 +40,7 @@ LF l2lf = [](const Vector& actual, const Vector& expected) { return sqrt(total); }; -DL l2dl = [](const Vector& actual, const Vector& expected, int i) { +JML_API DL l2dl = [](const Vector& actual, const Vector& expected, int i) { double l = l2lf(actual, expected); double ret = 1.0 / (2 * l); ret *= 2 * (actual.get_entry(i) - expected.get_entry(i));