add optimizer

README.md

@@ -12,7 +12,7 @@ Terox is a tiny Python package that provides some features:
 - [x] Control the parameters and the model.
 - [ ] Provides common computing functions for deep learning.
 - [ ] Provides common deep learning components.
-- [ ] Provides deep learning model optimizer.
+- [x] Provides deep learning model optimizer.
 - [ ] Accelerate computing on CPU and GPU.
 - [ ] Support distributed computing.
 

README_cn.md

@@ -12,7 +12,7 @@ Terox 是一个很精简的 Python 包，它提供了一些特性:
 - [x] 便捷控制参数和模型。
 - [ ] 提供深度学习常用的计算函数。
 - [ ] 提供常用的深度学习组件。
-- [ ] 提供深度学习模型优化器。
+- [x] 提供深度学习模型优化器。
 - [ ] 提高在 CPU 和 GPU 上的计算速度。
 - [ ] 支持分布式计算。
 

project/tensor/model.py

@@ -34,7 +34,7 @@ def forward(self, inputs:Tensor) -> Tensor:
         out = out.sigmoid()
         return out
 
-class SGD():
+class GD():
     def __init__(self, parameters:List[Parameter], lr:float) -> None:
         self.parameters = parameters
         self.lr = lr

project/tensor/run.py

@@ -4,7 +4,7 @@
 from terox.tensor import Tensor
 from terox.module import Module
 
-from model import ScalarIrisClassifyModel, SGD
+from model import ScalarIrisClassifyModel, GD
 from dataset import getDataSet
 from function import MSELoss
 
@@ -25,7 +25,7 @@ def test(model:Module, dataset:List) -> float:
     dataset   = getDataSet(N)
     model     = ScalarIrisClassifyModel(in_feature=2, hidden_feature=128, out_feature=1)
     criterion = MSELoss
-    optimizer = SGD(model.parmeters(), LR)
+    optimizer = GD(model.parmeters(), LR)
     print(f"[INFO] Start Acc:{test(model, dataset) * 100:.2f}%")
 
     for epoch in range(EPOCHS):

terox/optim/__init__.py

@@ -0,0 +1 @@
+from .sgd import SGD

terox/optim/adam.py

@@ -0,0 +1,31 @@
+import math
+from typing import List
+
+from terox.module import Parameter
+from .optim import Optimizer
+
+class Adam(Optimizer):
+    def __init__(self, parameters: List[Parameter], lr: float, beta1: float = 0.9, beta2: float = 0.999, epsilon: float = 1e-8) -> None:
+        super().__init__(parameters, lr)
+        self.beta1 = beta1
+        self.beta2 = beta2
+        self.epsilon = epsilon
+        self.m = {id(param): 0.0 for param in parameters}
+        self.v = {id(param): 0.0 for param in parameters}
+        self.t = 0
+        return
+
+    def step(self) -> None:
+        self.t += 1
+        for parameter in self.parameters:
+            grad = parameter.value()._gradient._item
+            param_id = id(parameter)
+
+            self.m[param_id] = self.beta1 * self.m[param_id] + (1 - self.beta1) * grad
+            self.v[param_id] = self.beta2 * self.v[param_id] + (1 - self.beta2) * (grad ** 2)
+
+            m_hat = self.m[param_id] / (1 - self.beta1 ** self.t)
+            v_hat = self.v[param_id] / (1 - self.beta2 ** self.t)
+
+            parameter.value()._item -= self.lr * m_hat / (math.sqrt(v_hat) + self.epsilon)
+        return

terox/optim/optim.py

@@ -0,0 +1,17 @@
+from typing import List
+
+from terox.module import Parameter
+
+class Optimizer():
+    def __init__(self, parameters: List[Parameter], lr: float) -> None:
+        self.parameters = parameters
+        self.lr = lr
+        return
+
+    def zero_grad(self) -> None:
+        for parameter in self.parameters:
+            parameter.value()._zeroGrad()
+        return
+
+    def step(self) -> None:
+        raise NotImplementedError

terox/optim/sgd.py

@@ -0,0 +1,20 @@
+from typing import List
+
+from terox.module import Parameter
+from .optim import Optimizer
+
+class SGD(Optimizer):
+    def __init__(self, parameters: List[Parameter], lr: float, momentum: float = 0.0) -> None:
+        super().__init__(parameters, lr)
+        self.momentum = momentum
+        self.velocity = {id(param): 0.0 for param in parameters}
+        return
+
+    def step(self) -> None:
+        for parameter in self.parameters:
+            grad = parameter.value()._gradient._item
+            param_id = id(parameter)
+
+            self.velocity[param_id] = self.momentum * self.velocity[param_id] - self.lr * grad
+            parameter.value()._item += self.velocity[param_id]
+        return