Reactor replace internal optimizer with public shared optimizer (#45)

* refactor out the internal class from public API

* remove todo -- it was incomplete type defintions

* refactored internal optimizers

 while encapulating eigen

* fix spelling

* switch optimizers to shared ptr

* missing CRTP

* fix copy PASTE MISTAKE

CMakeLists.txt

@@ -38,7 +38,7 @@ set(SOURCES
     ${SOURCE_DIR}/internals/functions/internal_function_logsoftmax.cpp
     ${SOURCE_DIR}/internals/functions/internal_function_relu.cpp
     ${SOURCE_DIR}/internals/criterions/internal_criterion_nllloss.cpp
-    ${SOURCE_DIR}/internals/optimizers/internal_optimizer_sgd.cpp)
+)
 
 find_package(Eigen3 REQUIRED CONFIG)
 

examples/layers.cpp

@@ -14,8 +14,8 @@ cmake --build . --target cabernet-examples-layers
 struct Autoencoder : public net::Model<Autoencoder> {
 
     Autoencoder() {
-        encoder.configure_optimizer(encoder_optimizer);
-        decoder.configure_optimizer(decoder_optimizer);
+        encoder.configure_optimizer(std::dynamic_pointer_cast<net::base::Optimizer>(encoder_optimizer));
+        decoder.configure_optimizer(std::dynamic_pointer_cast<net::base::Optimizer>(decoder_optimizer));
     }
 
     net::layer::Sequence encoder {
@@ -38,16 +38,16 @@ struct Autoencoder : public net::Model<Autoencoder> {
     }
 
     void step() {
-        encoder_optimizer.step();
-        decoder_optimizer.step();
+        encoder_optimizer->step();
+        decoder_optimizer->step();
     }
-    /* you can add diferent optimizers to different layers
+    /* you can add different optimizers to different layers
     or the same, doesn't matter, the optimizer has a shared pointer
     to it's implementation so you can pass instances of it with value
     semantics without making deep copies */
 
-    net::optimizer::SGD encoder_optimizer {/*learning rate*/ 0.1};
-    net::optimizer::SGD decoder_optimizer {/*learning rate*/ 0.2};
+    std::shared_ptr<net::optimizer::SGD> encoder_optimizer = std::make_shared<net::optimizer::SGD>(/*learning rate*/ 0.1);
+    std::shared_ptr<net::optimizer::SGD> decoder_optimizer = std::make_shared<net::optimizer::SGD>(/*learning rate*/ 0.2);
 };
 
 int main() {

examples/model.cpp

@@ -1,9 +1,9 @@
 #include <CaberNet.h>
 
 struct Network : public net::Model<Network> {
-    Network() {
-        layers.configure_optimizer(optimizer);
-    }
+    Network() : net::Model<Network>(
+        std::make_shared<net::optimizer::SGD>(/*learning rate*/ 0.1)
+    ) {}
 
     net::layer::Sequence layers {
         net::layer::Linear(784, 128),
@@ -15,8 +15,6 @@ struct Network : public net::Model<Network> {
     net::Tensor<float> forward(net::Tensor<float> x) {
         return layers(x);
     }
-
-    net::optimizer::SGD optimizer {/*learning rate*/ 0.1};
 };
 
 int main() {
@@ -39,7 +37,7 @@ int main() {
 
         std::cout << "Epoch: " << epoch + 1 << std::endl;
 
-        for(int batch = 0; batch < dataset.lenght(); ++batch) {
+        for(int batch = 0; batch < dataset.length(); ++batch) {
             input.copy(dataset.features()[batch].internal()); // I will fix this in the future so it will be prettier and without copies.
             targets.copy(dataset.targets()[batch].internal());
 
@@ -49,4 +47,4 @@ int main() {
         }
 
     }
-}
+}

include/CaberNet/dataset.h

@@ -93,7 +93,7 @@ class Dataset {
         targets_.clear();
     }
 
-    std::size_t lenght() {
+    std::size_t length() {
         return features_.size();
     }
 

include/CaberNet/layers.h

@@ -9,7 +9,6 @@
 
 namespace internal {
     class Tensor;
-    class Optimizer;
 };
 
 namespace net::layer {
@@ -23,7 +22,7 @@ class Linear : public Model<Linear> {
         initializer distribution = initializer::He );
 
     Tensor<float> forward(Tensor<float> x);
-    void set_optimizer(internal::Optimizer* optimizer);
+    void set_optimizer(std::shared_ptr<net::base::Optimizer> optimizer);
 
     private:
     Tensor<float> weight_;
@@ -33,21 +32,21 @@ class Linear : public Model<Linear> {
 struct ReLU : public Model<ReLU> {
     ReLU() = default;
     Tensor<float> forward(Tensor<float> input);
-    void set_optimizer(internal::Optimizer* optimizer) { return; }
+    void set_optimizer(std::shared_ptr<net::base::Optimizer> optimizer) { return; }
 };
 
 struct Softmax : public Model<Softmax> {
     int axis;
     Softmax(int axis);
     Tensor<float> forward(Tensor<float> input);
-    void set_optimizer(internal::Optimizer* optimizer) { return; }
+    void set_optimizer(std::shared_ptr<net::base::Optimizer> optimizer) { return; }
 };
 
 struct LogSoftmax : public Model<LogSoftmax> {
     int axis;
     LogSoftmax(int axis);
     Tensor<float> forward(Tensor<float> input);
-    void set_optimizer(internal::Optimizer* optimizer) { return; }
+    void set_optimizer(std::shared_ptr<net::base::Optimizer> optimizer) { return; }
 };
 
 class Sequence : public Model<Sequence> {
@@ -71,8 +70,7 @@ class Sequence : public Model<Sequence> {
         }
         return input;
     }
-
-    void set_optimizer(internal::Optimizer* optimizer) {
+    void set_optimizer(std::shared_ptr<net::base::Optimizer> optimizer) {
         for (auto& layer : layers_) {
             std::cout << "visited" << std::endl;
             std::visit([optimizer](auto&& argument) { argument.set_optimizer(optimizer); }, layer);

include/CaberNet/model.h

@@ -6,18 +6,10 @@
 #include "tensor.h"
 #include "optimizers.h"
 
-namespace internal {
-    class Optimizer;
-}
-
 namespace net {
 
 template<class Derived>
 class Model {
-    using optimizer_variant = std::variant<
-        optimizer::SGD
-    >;
-
     public:
     using size_type = std::size_t;
     using shape_type = std::vector<size_t>;
@@ -26,24 +18,19 @@ class Model {
         return static_cast<Derived*>(this)->forward(input);
     }
 
-    void configure_optimizer(optimizer_variant instance) {
-        optimizer_ = std::visit([](auto&& argument) { return argument.get(); }, instance);
-        static_cast<Derived*>(this)->set_optimizer(optimizer_);
-    }
-
-    void set_optimizer(internal::Optimizer* optimizer) {
+    void configure_optimizer(std::shared_ptr<net::base::Optimizer> optimizer) {
         static_cast<Derived*>(this)->set_optimizer(optimizer);
+        optimizer_ = optimizer;
     }
 
-    internal::Optimizer* optimizer() const {
-        return optimizer_;
-    }
+    private:
+    std::shared_ptr<net::base::Optimizer> optimizer_ = std::make_shared<net::optimizer::NoOptimization>();
 
+    protected:
     Model() = default;
-
-    private:
-    internal::Optimizer* optimizer_;
-
+    Model(std::shared_ptr<net::base::Optimizer> optimizer) : optimizer_(optimizer) {
+        static_cast<Derived*>(this)->set_optimizer(optimizer);
+    }
 };
 
 } // namespace net

include/CaberNet/optimizers.h

@@ -4,39 +4,56 @@
 #include <vector>
 #include <memory>
 
-#include "tensor.h"
 
-namespace internal { 
-
-class Tensor; 
-class Optimizer;
-
-} // namespace internal
+namespace internal { class Tensor; }
 
 namespace net::base {
 
-class Optimizer {
+struct Optimizer {
+    virtual ~Optimizer() = default;
+    virtual void add_parameter(internal::Tensor* parameter) = 0;
+    virtual void step() = 0;
+};
+
+template<class Derived>
+class Optimize : public Optimizer {
     public:
-    ~Optimizer();
-    void add_parameter(internal::Tensor* parameter);
-    void step();
+    ~Optimize() override = default;
 
-    internal::Optimizer* get() const;
+    void add_parameter(internal::Tensor* parameter) override final {
+        parameters_.push_back(parameter);
+    }
 
-    protected:
-    Optimizer() = default;
-    std::shared_ptr<internal::Optimizer> optimizer_ = nullptr;
+    void step() override final {
+        for(internal::Tensor* parameter : parameters_) {
+            static_cast<Derived*>(this)->update(parameter);
+        }
+    }
+
+    private:
+    std::vector<internal::Tensor*> parameters_;
 };
 
 }
 
 namespace net::optimizer {
 
-class SGD : public base::Optimizer {
-    public:
-    SGD() = default;
-    ~SGD();
-    SGD(float learning_rate);
+class NoOptimization : public base::Optimize<NoOptimization> {
+    public: 
+    ~NoOptimization() = default;
+    void update(internal::Tensor* parameter) {return;}
+};
+
+
+class SGD : public base::Optimize<SGD> {
+    public: 
+    SGD(float learning_rate): learning_rate_{learning_rate} {}
+    ~SGD() = default;
+
+    void update(internal::Tensor* parameter);
+
+    protected:
+    const float learning_rate_;
 };
 
 } // namespace net::optimizer

src/layers.cpp

@@ -1,7 +1,6 @@
 #include "CaberNet/tensor.h"
 #include "CaberNet/layers.h"
 
-#include "internals/optimizers/internal_optimizers.hpp"
 #include "internals/functions/internal_functions.hpp"
 
 
@@ -23,7 +22,7 @@ LogSoftmax::LogSoftmax(int axis) : axis(axis) {}
 
 /// settings
 
-void Linear::set_optimizer(internal::Optimizer* optimizer) {
+void Linear::set_optimizer(std::shared_ptr<net::base::Optimizer> optimizer) {
     optimizer->add_parameter(weight_.internal());
     optimizer->add_parameter(bias_.internal());
 }

src/optimizers.cpp

@@ -1,31 +1,20 @@
 #include "CaberNet/optimizers.h"
-#include "internals/optimizers/internal_optimizers.hpp"
 
-namespace net::base {
+#include "internals/config.h"
+#include "internals/internal_tensor.hpp"
 
-Optimizer::~Optimizer() {}
 
-internal::Optimizer* Optimizer::get() const {
-    return optimizer_.get();
-}
-
-void Optimizer::add_parameter(internal::Tensor* parameter) {
-    optimizer_->add_parameter(parameter);
-    std::cout << "Parameter added" << std::endl;
-}
-
-void Optimizer::step() {
-    if(optimizer_) optimizer_->step();
-}
-
-}
+#if defined(USE_EIGEN_BACKEND)
 
 namespace net::optimizer {
 
-SGD::~SGD() {}
-
-SGD::SGD(float learning_rate) {
-    optimizer_ = std::make_shared<internal::SGD>(learning_rate);
+void SGD::update(internal::Tensor* parameter) {
+    Eigen::Map<Eigen::Array<internal::Tensor::scalar_type, 1, -1>> parameter_map(parameter->data(), parameter->size());
+    Eigen::Map<Eigen::Array<internal::Tensor::scalar_type, 1, -1>> parameter_gradient_map(parameter->gradient()->data(), parameter->size());
+    parameter_map -= learning_rate_ * parameter_gradient_map;
+    parameter_gradient_map = 0;
 }
 
 }
+
+#endif