shared ptr implementation

andrewldesousa · Jun 22, 2024 · 1bd1f8d · 1bd1f8d
1 parent 7c6b3ac
commit 1bd1f8d
Show file tree

Hide file tree

Showing 5 changed files with 243 additions and 168 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -50,6 +50,36 @@
         "vector": "cpp",
         "algorithm": "cpp",
         "iterator": "cpp",
-        "memory_resource": "cpp"
+        "memory_resource": "cpp",
+        "any": "cpp",
+        "charconv": "cpp",
+        "codecvt": "cpp",
+        "condition_variable": "cpp",
+        "deque": "cpp",
+        "fstream": "cpp",
+        "iomanip": "cpp",
+        "list": "cpp",
+        "map": "cpp",
+        "numbers": "cpp",
+        "queue": "cpp",
+        "semaphore": "cpp",
+        "shared_mutex": "cpp",
+        "source_location": "cpp",
+        "span": "cpp",
+        "stack": "cpp",
+        "unordered_set": "cpp",
+        "atomic": "cpp",
+        "*.tcc": "cpp",
+        "compare": "cpp",
+        "concepts": "cpp",
+        "exception": "cpp",
+        "functional": "cpp",
+        "memory": "cpp",
+        "system_error": "cpp",
+        "type_traits": "cpp",
+        "utility": "cpp",
+        "__memory": "cpp",
+        "chrono": "cpp",
+        "format": "cpp"
     }
 }
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -2,5 +2,4 @@ cmake_minimum_required(VERSION 3.12)
 project(backprop)
 
 set(CMAKE_CXX_STANDARD 17)
-
 add_subdirectory(tests)
diff --git a/example/and.cpp b/example/and.cpp
@@ -0,0 +1,54 @@
+#include <iostream>
+#include "../scalar.h"
+
+
+Scalar<double> cross_entropy(Scalar<double> y, Scalar<double> y_hat) {
+    return -y * log(y_hat) - (1 - y) * log(1 - y_hat);
+}
+
+Scalar<double> sigmoid(Scalar<double> x) {
+    return 1 / (1 + exp(-x));
+}
+
+int main() {
+    // 4x2 dataset maxtrix
+    Scalar<double> X[4][2] = {
+        {Scalar<double>(0), Scalar<double>(0)},
+        {Scalar<double>(0), Scalar<double>(1)},
+        {Scalar<double>(1), Scalar<double>(0)},
+        {Scalar<double>(1), Scalar<double>(1)}
+    };
+
+    Scalar<double> Y[4] = {
+        Scalar<double>(0),
+        Scalar<double>(0),
+        Scalar<double>(0),
+        Scalar<double>(1)
+    };
+
+    Scalar<double> w1(.1), w2(.1), b(.1);
+
+
+    int num_epochs = 1000, num_samples = 4;
+    for (int i = 0; i < num_epochs; i++) {
+        Scalar<double> loss(0);
+
+        for (int j = 0; j < num_samples; j++) {
+            Scalar<double> z = X[j][0] * w1 + X[j][1] * w2 + b;
+            Scalar<double> a = sigmoid(z);
+            loss += cross_entropy(Y[j], a);
+        }
+
+        loss.backward();
+
+        w1.value -= w1.grad * 0.1;
+        w2.value -= w2.grad * 0.1;
+        b.value -= b.grad * 0.1;
+
+        w1.grad = 0;
+        w2.grad = 0;
+        b.grad = 0;
+    }
+
+    return 0;
+}
diff --git a/scalar.h b/scalar.h
@@ -5,7 +5,7 @@
 
 
 template <typename T>
-class Scalar {
+class Scalar: public std::enable_shared_from_this<Scalar<T>> {
 public:
     T value;
     T grad = 0;
@@ -18,170 +18,101 @@ class Scalar {
         _backward = []() {};
     }
 
-    void backward() {
+    void backward(bool is_root=true) {
         // if in_degrees is not zero, return
         if (in_degrees != 0) { throw std::runtime_error("in_degrees is not zero"); }
+        if (is_root) { this->grad = 1.0; }
+
         _backward();
 
         // backward children
         for (auto child : children) {
             child->in_degrees--;
-            if (child->in_degrees == 0) { child->backward(); }
+            if (child->in_degrees == 0) { child->backward(false); }
         }
     }
 
-    Scalar<T>& operator+(Scalar<T>& other) {
-        auto result = std::make_shared<Scalar<T>>(this->value + other.value);
+    friend std::shared_ptr<Scalar<T>> operator+(std::shared_ptr<Scalar<T>> lhs, std::shared_ptr<Scalar<T>> rhs) {
+        auto result = std::make_shared<Scalar<T>>(lhs->value + rhs->value);
 
-        auto other_shared_ptr = std::shared_ptr<Scalar<T>>(&other);
-        auto this_shared_ptr = std::shared_ptr<Scalar<T>>(this);
-        result->children.insert(this_shared_ptr);
-        result->children.insert(other_shared_ptr);
-        this->in_degrees++;
-        other.in_degrees++;
+        result->children.insert(lhs);
+        result->children.insert(rhs);
+        lhs->in_degrees++;
+        rhs->in_degrees++;
 
         result->_backward = [&, result]() {
-            this->grad += 1.0 * result->grad;
-            other.grad += 1.0 * result->grad;
+            lhs->grad += 1.0 * result->grad;
+            rhs->grad += 1.0 * result->grad;
         };
 
-        return *result;
+        return result;
     }
 
-    Scalar<T>& operator-(Scalar<T>& other) {
-        auto result = std::make_shared<Scalar<T>>(this->value - other.value);
-
-        auto other_shared_ptr = std::shared_ptr<Scalar<T>>(&other);
-        auto this_shared_ptr = std::shared_ptr<Scalar<T>>(this);
-        result->children.insert(this_shared_ptr);
-        result->children.insert(other_shared_ptr);
+    friend std::shared_ptr<Scalar<T>> operator-(std::shared_ptr<Scalar<T>> lhs, std::shared_ptr<Scalar<T>> rhs) {
+        auto result = std::make_shared<Scalar<T>>(lhs->value - rhs->value);
 
-        this->in_degrees++;
-        other.in_degrees++;
+        result->children.insert(lhs);
+        result->children.insert(rhs);
+        lhs->in_degrees++;
+        rhs->in_degrees++;
 
         result->_backward = [&, result]() {
-            this->grad += 1.0 * result->grad;
-            other.grad -= 1.0 * result->grad;
+            lhs->grad += 1.0 * result->grad;
+            rhs->grad += -1.0 * result->grad;
         };
 
-        return *result;
+        return result;
     }
 
-    Scalar<T>& operator*(Scalar<T>& other) {
-        auto result = std::make_shared<Scalar<T>>(this->value * other.value);
-
-        auto other_shared_ptr = std::shared_ptr<Scalar<T>>(&other);
-        auto this_shared_ptr = std::shared_ptr<Scalar<T>>(this);
-        result->children.insert(this_shared_ptr);
-        result->children.insert(other_shared_ptr);
+    friend std::shared_ptr<Scalar<T>> operator*(std::shared_ptr<Scalar<T>> lhs, std::shared_ptr<Scalar<T>> rhs) {
+        auto result = std::make_shared<Scalar<T>>(lhs->value * rhs->value);
 
-        this->in_degrees++;
-        other.in_degrees++;
+        result->children.insert(lhs);
+        result->children.insert(rhs);
+        lhs->in_degrees++;
+        rhs->in_degrees++;
 
         result->_backward = [&, result]() {
-            this->grad += other.value * result->grad;
-            other.grad += this->value * result->grad;
+            lhs->grad += rhs->value * result->grad;
+            rhs->grad += lhs->value * result->grad;
         };
 
-        return *result;
+        return result;
     }
 
-    Scalar<T>& operator/(Scalar<T>& other) {
-        auto result = std::make_shared<Scalar<T>>(this->value / other.value);
-
-        auto other_shared_ptr = std::shared_ptr<Scalar<T>>(&other);
-        auto this_shared_ptr = std::shared_ptr<Scalar<T>>(this);
-        result->children.insert(this_shared_ptr);
-        result->children.insert(other_shared_ptr);
+    friend std::shared_ptr<Scalar<T>> operator/(std::shared_ptr<Scalar<T>> lhs, std::shared_ptr<Scalar<T>> rhs) {
+        auto result = std::make_shared<Scalar<T>>(lhs->value / rhs->value);
 
-        this->in_degrees++;
-        other.in_degrees++;
+        result->children.insert(lhs);
+        result->children.insert(rhs);
+        lhs->in_degrees++;
+        rhs->in_degrees++;
 
         result->_backward = [&, result]() {
-            this->grad += 1.0 / other.value * result->grad;
-            other.grad -= this->value / (other.value * other.value) * result->grad;
+            lhs->grad += 1.0 / rhs->value * result->grad;
+            rhs->grad += -lhs->value / (rhs->value * rhs->value) * result->grad;
         };
 
-        return *result;
+        return result;
     }
-};
-
-
-// numerical and scalar overload operators
-template <typename T>
-Scalar<T>& operator+(Scalar<T>& s1, T s2) {
-    auto result = std::make_shared<Scalar<T>>(s1.value + s2);
-    auto s1_shared_ptr = std::shared_ptr<Scalar<T>>(&s1);
-    result->children.insert(s1_shared_ptr);
-    s1.in_degrees++;
-
-    result->_backward = [&, result]() {
-        s1.grad += 1.0 * result->grad;
-    };
-
-    return *result;
-}
-
-// other order
-template <typename T>
-Scalar<T>& operator+(T s1, Scalar<T>& s2) {
-    return s2 + s1;
-}
-
-template <typename T>
-Scalar<T>& operator-(Scalar<T>& s1, T s2) {
-    auto result = std::make_shared<Scalar<T>>(s1.value - s2);
-    auto s1_shared_ptr = std::shared_ptr<Scalar<T>>(&s1);
-    result->children.insert(s1_shared_ptr);
-    s1.in_degrees++;
-
-    result->_backward = [&, result]() {
-        s1.grad += 1.0 * result->grad;
-    };
 
-    return *result;
-}
-
-// other order
-template <typename T>
-Scalar<T>& operator-(T s1, Scalar<T>& s2) {
-    return s2 - s1;
-}
-
-template <typename T>
-Scalar<T>& operator*(Scalar<T>& s1, T s2) {
-    auto result = std::make_shared<Scalar<T>>(s1.value * s2);
-    auto s1_shared_ptr = std::shared_ptr<Scalar<T>>(&s1);
-    result->children.insert(s1_shared_ptr);
-    s1.in_degrees++;
-
-    result->_backward = [&, result]() {
-        s1.grad += s2 * result->grad;
-    };
-
-    return *result;
-}
-
-template <typename T>
-Scalar<T>& operator*(T s1, Scalar<T>& s2) {
-    return s2 * s1;
-}
-
-template <typename T>
-Scalar<T>& operator/(Scalar<T>& s1, T s2) {
-    auto result = std::make_shared<Scalar<T>>(s1.value / s2);
-    auto s1_shared_ptr = std::shared_ptr<Scalar<T>>(&s1);
-    result->children.insert(s1_shared_ptr);
-    s1.in_degrees++;
-
-    result->_backward = [&, result]() {
-        s1.grad += 1.0 / s2 * result->grad;
-    };
-
-    return *result;
-}
-
-template <typename T>
-Scalar<T>& operator/(T s1, Scalar<T>& s2) {
-    return s2 / s1;
-}
+    // // unary operators
+    // std::shared_ptr<Scalar<T>> operator-() {
+    //     auto result = std::make_shared<Scalar<T>>(-this->value);
+    //     auto this_shared_ptr = std::shared_ptr<Scalar<T>>(this);
+    //     result->children.insert(this_shared_ptr);
+    //     this->in_degrees++;
+
+    //     result->_backward = [&, result]() {
+    //         this->grad += -1.0 * result->grad;
+    //     };
+
+    //     return *result;
+    // }
+
+    // // += operator
+    // std::shared_ptr<Scalar<T>> operator+=(std::shared_ptr<Scalar<T>> other) {
+    //     *this = *this + other;
+    //     return *this;
+    // }
+};
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,5 +2,4 @@ cmake_minimum_required(VERSION 3.12)
		project(backprop)

		set(CMAKE_CXX_STANDARD 17)

		add_subdirectory(tests)