diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eae4969 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +cmake-build-debug + +build + +.idea \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..ea73669 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 3.23) +project(MetaAI) + +set(CMAKE_CXX_STANDARD 20) + +add_executable(MetaAI + main.cpp) diff --git a/Components.h b/Components.h new file mode 100644 index 0000000..c8c88e0 --- /dev/null +++ b/Components.h @@ -0,0 +1,569 @@ +// +// Created by iku-iku-iku on 2022/10/21. +// + +#ifndef METAAI_COMPONENTS_H +#define METAAI_COMPONENTS_H + +#include +#include "Type.h" +#include +#include +#include + +// 定义单操作数的MatrixLikeComponent的N和M +#define SINGLE_OPERAND_MATRIX_NM \ +static constexpr std::size_t N = InOperand::N; \ +static constexpr std::size_t M = InOperand::M; \ +// + +// 标量 +template +concept Scalar = std::is_scalar_v; + +// MatrixVariable和Matrix都是MatrixLike +// 输出为矩阵的component也可以看成MatrixLike +template +concept MatrixLike = requires { T::N; T::M; }; + +// 可优化的类型 +template +concept Optimizable = requires(T t) { + &T::step; + &T::clear_grad; +}; + +// 单独的变量 +template +concept IsSingleVariable = requires{ typename T::VariableClassFlag; }; + +// 矩阵变量 +template +concept IsMatrixVariable = requires{ typename T::MatrixVariableClassFlag; }; + +// 变量 +template +concept IsVariable = IsMatrixVariable || IsSingleVariable; + +// 利用Component来构建一棵树 +// 将Variable看成一种trivial的component,并且Variable在叶子节点 +template +concept Component = requires(T t) { + &T::Backward; + &T::Forward; + t.whole.data; // 如果是Variable,则为可优化的参数,如果是component,则为中间结果 + t.whole.grad; // 梯度,反向传播时根据component的求导结果和component的输出来计算得出(链式法则) +}; + +template +concept MatrixLikeComponent = Component && MatrixLike; + +// CRTP,提供静态多态 +template +struct Operator { + void Forward() { + static_cast(this)->ForwardImpl(); + } + + void Backward() { + static_cast(this)->BackwardImpl(); + } +}; + + +template +struct MatMultiply; + +template +struct MatAddition; + +template +struct Variable; + +template +struct MatrixVariable; + +template +struct z_operand_ref { + using type = std::remove_reference_t; +}; + +// 目前是完全在栈上运算,效率更高,因此需要对于栈上的Variable需要引用 +template +struct z_operand_ref> { + using type = std::remove_reference_t> &; +}; + +template +struct z_operand_ref> { + using type = std::remove_reference_t> &; +}; + +template +using operand_ref_t = typename z_operand_ref::type; + + +// 单独的变量 +template +struct Variable : Operator> { + friend struct Operator>; + + using VariableClassFlag = std::nullptr_t; + + using DataType = InDataType; + + DataType data{}; + DataType grad{1}; // 对自己的梯度为1 + + Variable &whole; + + Variable() : whole(*this) {} + + explicit Variable(DataType x) : data(x), whole(*this) {} + + friend auto &operator<<(std::ostream &out, const Variable &var) { + return out << var.data; + } + + DataType value() { return data; } + + DataType gradient() { return grad; } + +// Variable(const Variable &) = delete; +// +// Variable &operator=(const Variable &) = delete; + + // 利用梯度更新参数 + void step(double lr) { + data += -lr * grad; + } + + // 清除梯度 + void clear_grad() { + grad = DataType{}; + } + +private: + void ForwardImpl() {} + + void BackwardImpl() {} +}; + +template +struct VarMultiply : Operator> { + friend struct Operator>; + + using Whole = Variable; + + std::tuple operands_tuple; + + Whole whole{}; // 看成一个整体 + + explicit VarMultiply(Operands &... operands) : operands_tuple(operands...) {} + +private: + template + auto VarForwardImpl(std::index_sequence) { + (get(operands_tuple).Forward(), ...); + return (get(operands_tuple).value() * ...); + } + + void ForwardImpl() { + whole.whole.data = VarForwardImpl(std::index_sequence_for{}); + } + + template + void VarBackwardImpl(std::index_sequence) { + auto product = (get(operands_tuple).value() * ...); + ((get(operands_tuple).whole.grad = (whole.grad * product / get(operands_tuple).value())), ...); + + (get(operands_tuple).Backward(), ...); + } + + void BackwardImpl() { + VarBackwardImpl(std::index_sequence_for{}); + } +}; + +// 多个变量求和 +template +struct VarAddition : Operator> { + friend struct Operator>; + + using Whole = Variable; + + std::tuple...> operands_tuple; + + Whole whole{}; // 看成一个整体 + + explicit VarAddition(const Operands &... operands) : operands_tuple(const_cast(operands)...) {} + +private: + template + auto VarForwardImpl(std::index_sequence) { + (get(operands_tuple).Forward(), ...); + return (get(operands_tuple).whole.data + ...); + } + + void ForwardImpl() { + whole.data = VarForwardImpl(std::index_sequence_for{}); + } + + template + void VarBackwardImpl(std::index_sequence) { + ((get(operands_tuple).whole.grad = whole.grad), ...); + + (get(operands_tuple).Backward(), ...); + } + + void BackwardImpl() { + VarBackwardImpl(std::index_sequence_for{}); + } +}; + + +// 向量看成列数为1的矩阵 +template +using Vector = Matrix; + +// 矩阵变量 +template +struct MatrixVariable : Operator> { + friend struct Operator>; + + using MatrixVariableClassFlag = std::nullptr_t; + + static constexpr std::size_t N = InN; + static constexpr std::size_t M = InM; + using Mat = Matrix; + + Mat data; // 变量的值 + Mat grad; // 变量的梯度 + + MatrixVariable &whole; + + void step(double lr) { + data += -lr * grad; + } + + void clear_grad() { + grad = Mat{}; + } + + void init_grad() { + for (int i = 0; i < N; i++) { + for (int j = 0; j < M; j++) { + grad.data[i][j] = 1; + } + } + } + + MatrixVariable() : whole(*this) { init_grad(); } + + explicit MatrixVariable(const Matrix &in_data) : data(in_data), whole(*this) { init_grad(); } + + void set_value(const Mat &value) { data = value; } + + +private: + void ForwardImpl() {} + + void BackwardImpl() {} +}; + +// 矩阵相乘 +template +struct MatMultiply : Operator> { + friend struct Operator>; + + using Operand1 = InOperand1; + using Operand2 = InOperand2; + + using Whole = MatrixVariable; + + static constexpr std::size_t N = InOperand1::N; + static constexpr std::size_t M = InOperand2::M; + + Whole whole; + + operand_ref_t operand1; // N * K + operand_ref_t operand2; // K * M + + MatMultiply(const InOperand1 &o1, const InOperand2 &o2) : operand1(const_cast(o1)), + operand2(const_cast(o2)) {} + +private: + void ForwardImpl() { + operand1.Forward(); + operand2.Forward(); + whole.set_value(operand1.whole.data * operand2.whole.data); + } + + void BackwardImpl() { + // (N, K) * (K, M) => (N, M) + // (N, M) * (M, K) => (N, K) + // (K, N) * (N, M) => (K, M) + + operand1.whole.grad = whole.grad * operand2.whole.data.T(); + operand2.whole.grad = operand1.whole.data.T() * whole.grad; + + operand1.Backward(); + operand2.Backward(); + } +}; + +// 矩阵求和 +template +struct MatAddition : Operator> { + friend struct Operator>; + + using Operand1 = InOperand1; + using Operand2 = InOperand2; + + using Whole = MatrixVariable; + + static constexpr std::size_t N = InOperand1::N; + static constexpr std::size_t M = InOperand2::M; + + Whole whole; + + operand_ref_t operand1; // N * M + operand_ref_t operand2; // N * M + + MatAddition(const Operand1 &o1, const Operand2 &o2) : operand1(const_cast(o1)), + operand2(const_cast(o2)) {} + +private: + void ForwardImpl() { + operand1.Forward(); + operand2.Forward(); + + whole.data = operand1.whole.data + operand2.whole.data; + } + + void BackwardImpl() { + operand1.whole.grad = whole.grad; + operand2.whole.grad = whole.grad; + + operand1.Backward(); + operand2.Backward(); + } +}; + +// 对矩阵的每一维求指数运算 +template +struct Power : Operator> { + friend struct Operator>; + + using Operand = InOperand; + + SINGLE_OPERAND_MATRIX_NM + + using Whole = MatrixVariable; + + const unsigned int K; // 求K次方 + + operand_ref_t operand; + Whole whole; + + Power(unsigned int k, const InOperand &o) : K(k), operand(const_cast(o)) {} + +private: + void ForwardImpl() { + operand.Forward(); + + auto &val = operand.whole.data; + for (int i = 0; i < N; i++) { + for (int j = 0; j < M; j++) { + whole.data[i][j] = std::pow(val[i][j], K); + } + } + } + + void BackwardImpl() { + auto &val = operand.whole.data; + + for (int i = 0; i < N; i++) { + for (int j = 0; j < M; j++) { + operand.whole.grad[i][j] = whole.grad[i][j] * K * std::pow(val[i][j], K - 1); + } + } + + operand.Backward(); + } +}; + +// 对矩阵的每个元素进行映射 f(x) = 1 / (1 + exp(-x)) +template +struct Sigmoid : Operator> { + friend struct Operator>; + + using Operand = InOperand; + + SINGLE_OPERAND_MATRIX_NM + + using Whole = MatrixVariable; + + operand_ref_t operand; + Whole whole; + + explicit Sigmoid(const InOperand &o) : operand(const_cast(o)) {} + + Matrix &value() { return whole.data; } + +private: + void ForwardImpl() { + operand.Forward(); + + auto &val = operand.whole.data; + for (int i = 0; i < N; i++) { + for (int j = 0; j < M; j++) { + auto &x = val.data[i][j]; + whole.data[i][j] = 1 / (1 + std::exp(-x)); + } + } + } + + void BackwardImpl() { + for (int i = 0; i < N; i++) { + for (int j = 0; j < M; j++) { + operand.whole.grad[i][j] = whole.grad[i][j] * (1 - whole.data[i][j]) * whole.data[i][j]; + } + } + + operand.Backward(); + } +}; + +// 矩阵乘标量(标量不可优化) +template +struct MatScale : Operator> { + friend struct Operator>; + SINGLE_OPERAND_MATRIX_NM + + using Operand = InOperand; + using Whole = MatrixVariable; + + + Whole whole; + + operand_ref_t operand; + ScalarType scalar; + + MatScale(const Operand &o, ScalarType scalar_) : operand(const_cast(o)), scalar(scalar_) {} + +private: + void ForwardImpl() { + operand.Forward(); + + for (int i = 0; i < N; ++i) { + for (int j = 0; j < M; ++j) { + whole.data[i][j] = scalar * operand.whole.data[i][j]; + } + } + } + + void BackwardImpl() { + for (int i = 0; i < N; ++i) { + for (int j = 0; j < M; ++j) { + operand.whole.grad[i][j] = scalar * whole.grad[i][j]; + } + } + + operand.Backward(); + } +}; + +// 将矩阵的所有元素加起来 +template +struct Sum : Operator> { + friend struct Operator>; + + using Operand = InOperand; + + using Whole = Variable; + + Whole whole; + + operand_ref_t operand; + + explicit Sum(const InOperand &o) : operand(const_cast(o)) {} + +private: + void ForwardImpl() { + operand.Forward(); + + whole.whole.data = 0; + + auto &val = operand.whole.data; + for (int i = 0; i < InOperand::N; i++) { + for (int j = 0; j < InOperand::M; j++) { + whole.whole.data = whole.value() + val[i][j]; + } + } + } + + void BackwardImpl() { + for (int i = 0; i < InOperand::N; i++) { + for (int j = 0; j < InOperand::M; j++) { + operand.whole.grad[i][j] = whole.gradient(); + } + } + + operand.Backward(); + } +}; + +template +inline void step(double lr, T &... variables) { + (variables.step(lr), ...); + (variables.clear_grad(), ...); +} + +template +inline auto operator*(const LHS &lhs, const RHS &rhs) { + return MatMultiply{lhs, rhs}; +} + +template +inline auto operator*(ScalarType scalar, const RHS &rhs) { + return MatScale{rhs, scalar}; +} + +template +inline auto operator*(const LHS &lhs, ScalarType scalar) { + return MatScale{lhs, scalar}; +} + +template +requires (IsMatrixVariable || IsMatrixVariable) && + (IsMatrixVariable || IsMatrixVariable) +inline auto operator+(const LHS &lhs, const RHS &rhs) { + return MatAddition{lhs, rhs}; +} + +template +requires (IsSingleVariable || IsSingleVariable) && + (IsSingleVariable || IsSingleVariable) +inline auto operator+(const LHS &lhs, const RHS &rhs) { + return VarAddition{lhs, rhs}; +} + +template +inline auto operator-(const LHS &lhs, const RHS &rhs) { + return lhs + (-rhs); +} + +// 用^模拟指数运算符,注意^的优先级低于+ +template +inline auto operator^(const LHS &lhs, unsigned int k) { + return Power{k, lhs}; +} + +// 乘以-1 +template +inline auto operator-(const LHS &lhs) { + return MatScale{lhs, -1}; +} + +#endif //METAAI_COMPONENTS_H diff --git a/Data.h b/Data.h new file mode 100644 index 0000000..eece080 --- /dev/null +++ b/Data.h @@ -0,0 +1,140 @@ +// +// Created by iku-iku-iku on 2022/10/21. +// + +#ifndef METAAI_DATA_H +#define METAAI_DATA_H + +#include "Components.h" +#include + + +template +inline auto mean(std::vector &vec) { + T sum{}; + for (const auto &x: vec) { + sum += x; + } + + return sum * 1.0 / vec.size(); +} + +template +inline auto deviation(std::vector &vec) { + T sqrt_sum{}; + for (const auto &x: vec) { + sqrt_sum += x * x; + } + auto mean_value = mean(vec); + return sqrt_sum * 1.0 / vec.size() - mean_value * mean_value; +} + +template +inline auto sigma(std::vector &vec) { + return std::sqrt(deviation(vec)); +} + +template +auto read_vector(std::stringstream &ss) { + Vector vec; + + for (int i = 0; i < N; i++) { + ss >> vec.data[i][0]; + } + + return vec; +} + +template +auto read_elem(std::stringstream &ss) { + T t; + ss >> t; + return t; +} + +template +auto one_hot(std::size_t i) { + Vector vec; + vec.data[i][0] = 1; + return vec; +} + +template +auto max_i(Vector &vec) { + std::size_t res = -1; + DataType max_v = -1e9; + for (std::size_t i = 0; i < N; ++i) { + if (vec.data[i][0] > max_v) { + max_v = vec.data[i][0]; + res = i; + } + } + return res; +} + +template +struct Dataset { + using FeatureType = InFeatureType; + using LabelType = InLabelType; + + using VecType = Vector; + + std::vector features; + + std::vector labels; + + void read_from_file(const char *path) { + std::ifstream ifs; + + ifs.open(path, std::ios::in); + + std::stringstream ss; + char buf[256]; + while (ifs.getline(buf, sizeof(buf))) { + ss << buf; + features.push_back(read_vector(ss)); + labels.push_back(read_elem(ss)); + } + + ifs.close(); + } + + void normalize() { + for (int i = 0; i < FeatN; i++) { + FeatureType min_v = 1e9, max_v = -1e9; + for (auto &vec: features) { + min_v = std::min(min_v, vec.get(i)); + max_v = std::max(max_v, vec.get(i)); + } + + for (auto &vec: features) { + vec.get(i) = (vec.get(i) - min_v) / (max_v - min_v); + } + } + } + + + void normalize2() { + for (int i = 0; i < FeatN; i++) { + std::vector vec; + for (auto &feat: features) { + vec.push_back(feat.get(i)); + } + + auto mu = mean(vec); + auto sig = sigma(vec); + + for (auto &feat: features) { + feat.get(i) = (feat.get(i) - mu) / sig; + } + } + } + + VecType &get_feature(std::size_t i) { return features[i]; } + + LabelType &get_label(std::size_t i) { return labels[i]; } + + std::size_t size() { return features.size(); } +}; + +#endif //METAAI_DATA_H diff --git a/Dict.h b/Dict.h new file mode 100644 index 0000000..145c6db --- /dev/null +++ b/Dict.h @@ -0,0 +1,124 @@ +// +// Created by iku-iku-iku on 2022/10/20. +// + +#ifndef METAAI_DICT_H +#define METAAI_DICT_H + +#include +#include + +struct A { +}; +struct B { +}; +struct C { +}; + +template +struct condition { + using type = B; +}; + +template +struct condition { + using type = C; +}; + +template +constexpr auto is_same_type = false; + +template +constexpr auto is_same_type = true; + + +template +struct Dict { + + template + static constexpr auto GetIndex() { + return Find<0, T, VarKeys...>(); + } + + template + static constexpr auto Find() { + return -1; + } + + template + static constexpr auto Find() { + if constexpr (is_same_type) { + return i; + } else { + return Find(); + } + } + template + struct Values { + template + struct Type { + using type = typename Type::type; + }; + + template + struct Type<0, CurType, Types...> { + using type = CurType; + }; + + template + using GetType = typename Type::type; + + + std::shared_ptr m_Tuple[sizeof...(Args)]; + + template + constexpr auto Set(ValueType &&val) { + constexpr auto index = GetIndex(); + using RawValType = std::decay_t; + using New = Replace, Args...>; + auto n = New(); + for (int i = 0; i < sizeof...(Args); i++) { + if (i != index) { n.m_Tuple[i] = std::move(m_Tuple[i]); } + else { + n.m_Tuple[i] = std::shared_ptr(new RawValType(std::forward(val)), [](void *ptr) { + auto p = static_cast(ptr); + delete p; + }); + } + } + return n; + } + + template + [[nodiscard]] auto& Get() const { + constexpr auto index = GetIndex(); + using ValType = GetType; + return *reinterpret_cast(m_Tuple[index].get()); + } + + template + struct Replace_; + + template class PreTypes, + typename... Traveled, typename CurType, typename... RemainTypes> + struct Replace_, CurType, RemainTypes...> { + using type = typename Replace_, RemainTypes...>::type; + }; + template class PreTypes, + typename... Traveled, typename CurType, typename... RemainTypes> + struct Replace_, CurType, RemainTypes...> { + using type = PreTypes; + }; + + template + using Replace = typename Replace_::type; + + }; + + using Keys = Values; + + static constexpr auto Create() { return Keys(); } +}; + +#endif //METAAI_DICT_H diff --git a/README.md b/README.md new file mode 100644 index 0000000..48e5dea --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +C++元编程实现AI框架 + +> 元编程真好玩,就是头有点凉 + +编译器需要支持C++20 + +实现了若干Operator,目前利用Operator来组装网络 + +重载了+*^-操作符方便模型构建 + +主要思路为: +1. Forward时把计算结果存储到whole中 (后序遍历:先对Operand进行Forward,在计算当前Component的结果) +2. Backward时利用whole的grad来计算operand的grad (前序遍历:先计算operand的grad,再对operand进行Backward) + +使用到的新特性 +1. auto返回值类型推导 +2. 类模板参数推导 +3. concept和requires +4. fold expression +5. alias template \ No newline at end of file diff --git a/Train.h b/Train.h new file mode 100644 index 0000000..ef4d278 --- /dev/null +++ b/Train.h @@ -0,0 +1,47 @@ +// +// Created by iku-iku-iku on 2022/10/21. +// + +#ifndef METAAI_TRAIN_H +#define METAAI_TRAIN_H + +#include +#include "Components.h" + +struct LrScheduler { + double lr; + double decay_rate; + + LrScheduler(double init_lr = 0.985, double decay_rate_ = 0.985) : lr(init_lr), decay_rate(decay_rate_) {} + + double get() { return lr; } + + double next() { lr *= decay_rate; return lr;} +}; + +template +struct Plan { + LrScheduler scheduler; + + Model model; + + std::tuple param_tuple; + + + Plan(LrScheduler scheduler_, Model model_, Param &... param) : scheduler(scheduler_), model(model_), param_tuple(param...) {} + + template + void optimize(std::index_sequence) { + auto lr = scheduler.next(); + (get(param_tuple).step(lr), ...); + (get(param_tuple).clear_grad(), ...); + } + + void step() { + model.Forward(); + model.Backward(); + optimize(std::index_sequence_for{}); + } +}; + +#endif //METAAI_TRAIN_H diff --git a/Type.h b/Type.h new file mode 100644 index 0000000..873f84b --- /dev/null +++ b/Type.h @@ -0,0 +1,134 @@ +// +// Created by iku-iku-iku on 2022/10/21. +// + +#ifndef METAAI_TYPE_H +#define METAAI_TYPE_H + +#include + +// 矩阵,可以看成基本类型 +template +struct Matrix { + using DataType = InDataType; + static constexpr std::size_t N = In_N; + static constexpr std::size_t M = In_M; + + DataType data[N][M]{}; + + Matrix() = default; + + Matrix(DataType in_data[N][M]) { std::memcpy(data, in_data, sizeof(data)); } + + // 根据传入的随机数生成器生成矩阵 + template + static Matrix RandomlyCreate(Gen gen) { + Matrix res; + for (int i = 0; i < N; i++) { + for (int j = 0; j < M; j++) { + res.data[i][j] = gen(); + } + } + return res; + } + + DataType& get(std::size_t i , std::size_t j = 0) { + return data[i][j]; + } + + DataType value() { return data; } + + Matrix(const Matrix &) = default; + + Matrix &operator=(const Matrix &) = default; + + void ForwardImpl() {} + + void BackwardImpl() {} + + template + Matrix operator*(const Matrix &rhs) { + Matrix res; + for (int i = 0; i < N; i++) { + for (int j = 0; j < R_M; j++) { + for (int k = 0; k < R_N; k++) { + res.data[i][j] += data[i][k] * rhs.data[k][j]; + } + } + } + return res; + } + + Matrix operator+(const Matrix &rhs) { + Matrix res; + for (int i = 0; i < N; i++) { + for (int j = 0; j < M; j++) { + res.data[i][j] = data[i][j] + rhs.data[i][j]; + } + } + return res; + } + + Matrix operator-() { + Matrix res = *this; + for (int i = 0; i < N; i++) { + for (int j = 0; j < M; j++) { + res.data[i][j] = -data[i][j]; + } + } + return res; + } + + Matrix T() { + Matrix res; + for (int i = 0; i < N; i++) { + for (int j = 0; j < M; j++) { + res.data[j][i] = data[i][j]; + } + } + return res; + } + + DataType *operator[](std::size_t i) { + return data[i]; + } + + Matrix &operator+=(const Matrix &rhs) { + for (int i = 0; i < N; i++) { + for (int j = 0; j < M; j++) { + data[i][j] += rhs.data[i][j]; + } + } + return *this; + } + + template + friend Matrix operator*(const Matrix &mat, const T &scalar) { + Matrix res(mat); + for (int i = 0; i < N; i++) { + for (int j = 0; j < M; j++) { + res.data[i][j] *= scalar; + } + } + return res; + } + + template + friend Matrix operator*(const T &scalar, const Matrix &mat) { + return mat * scalar; + } + + friend auto &operator<<(std::ostream &out, const Matrix &var) { + std::cout << "data:" << std::endl; + for (int i = 0; i < N; i++) { + out << '\t'; + for (int j = 0; j < M; j++) { + out << var.data[i][j] << ' '; + } + out << std::endl; + } + return out; + } +}; + +#endif //METAAI_TYPE_H diff --git a/Util.h b/Util.h new file mode 100644 index 0000000..9613dcd --- /dev/null +++ b/Util.h @@ -0,0 +1,17 @@ +// +// Created by iku-iku-iku on 2022/10/21. +// + +#ifndef METAAI_UTIL_H +#define METAAI_UTIL_H + +#include + +inline auto print() {std::cout << std::endl;} +template +inline auto print(G g, T&&... var) { + std::cout << g << " "; + print(var...); +} + +#endif //METAAI_UTIL_H diff --git a/data/Iris-test.txt b/data/Iris-test.txt new file mode 100644 index 0000000..91a6408 --- /dev/null +++ b/data/Iris-test.txt @@ -0,0 +1,75 @@ +5.0 3.0 1.6 0.2 0 +5.0 3.4 1.6 0.4 0 +5.2 3.5 1.5 0.2 0 +5.2 3.4 1.4 0.2 0 +4.7 3.2 1.6 0.2 0 +4.8 3.1 1.6 0.2 0 +5.4 3.4 1.5 0.4 0 +5.2 4.1 1.5 0.1 0 +5.5 4.2 1.4 0.2 0 +4.9 3.1 1.5 0.1 0 +5.0 3.2 1.2 0.2 0 +5.5 3.5 1.3 0.2 0 +4.9 3.1 1.5 0.1 0 +4.4 3.0 1.3 0.2 0 +5.1 3.4 1.5 0.2 0 +5.0 3.5 1.3 0.3 0 +4.5 2.3 1.3 0.3 0 +4.4 3.2 1.3 0.2 0 +5.0 3.5 1.6 0.6 0 +5.1 3.8 1.9 0.4 0 +4.8 3.0 1.4 0.3 0 +5.1 3.8 1.6 0.2 0 +4.6 3.2 1.4 0.2 0 +5.3 3.7 1.5 0.2 0 +5.0 3.3 1.4 0.2 0 +6.6 3.0 4.4 1.4 1 +6.8 2.8 4.8 1.4 1 +6.7 3.0 5.0 1.7 1 +6.0 2.9 4.5 1.5 1 +5.7 2.6 3.5 1.0 1 +5.5 2.4 3.8 1.1 1 +5.5 2.4 3.7 1.0 1 +5.8 2.7 3.9 1.2 1 +6.0 2.7 5.1 1.6 1 +5.4 3.0 4.5 1.5 1 +6.0 3.4 4.5 1.6 1 +6.7 3.1 4.7 1.5 1 +6.3 2.3 4.4 1.3 1 +5.6 3.0 4.1 1.3 1 +5.5 2.5 4.0 1.3 1 +5.5 2.6 4.4 1.2 1 +6.1 3.0 4.6 1.4 1 +5.8 2.6 4.0 1.2 1 +5.0 2.3 3.3 1.0 1 +5.6 2.7 4.2 1.3 1 +5.7 3.0 4.2 1.2 1 +5.7 2.9 4.2 1.3 1 +6.2 2.9 4.3 1.3 1 +5.1 2.5 3.0 1.1 1 +5.7 2.8 4.1 1.3 1 +7.2 3.2 6.0 1.8 2 +6.2 2.8 4.8 1.8 2 +6.1 3.0 4.9 1.8 2 +6.4 2.8 5.6 2.1 2 +7.2 3.0 5.8 1.6 2 +7.4 2.8 6.1 1.9 2 +7.9 3.8 6.4 2.0 2 +6.4 2.8 5.6 2.2 2 +6.3 2.8 5.1 1.5 2 +6.1 2.6 5.6 1.4 2 +7.7 3.0 6.1 2.3 2 +6.3 3.4 5.6 2.4 2 +6.4 3.1 5.5 1.8 2 +6.0 3.0 4.8 1.8 2 +6.9 3.1 5.4 2.1 2 +6.7 3.1 5.6 2.4 2 +6.9 3.1 5.1 2.3 2 +5.8 2.7 5.1 1.9 2 +6.8 3.2 5.9 2.3 2 +6.7 3.3 5.7 2.5 2 +6.7 3.0 5.2 2.3 2 +6.3 2.5 5.0 1.9 2 +6.5 3.0 5.2 2.0 2 +6.2 3.4 5.4 2.3 2 +5.9 3.0 5.1 1.8 2 \ No newline at end of file diff --git a/data/Iris-train.txt b/data/Iris-train.txt new file mode 100644 index 0000000..997e68d --- /dev/null +++ b/data/Iris-train.txt @@ -0,0 +1,75 @@ +5.1 3.5 1.4 0.2 0 +4.9 3.0 1.4 0.2 0 +4.7 3.2 1.3 0.2 0 +4.6 3.1 1.5 0.2 0 +5.0 3.6 1.4 0.2 0 +5.4 3.9 1.7 0.4 0 +4.6 3.4 1.4 0.3 0 +5.0 3.4 1.5 0.2 0 +4.4 2.9 1.4 0.2 0 +4.9 3.1 1.5 0.1 0 +5.4 3.7 1.5 0.2 0 +4.8 3.4 1.6 0.2 0 +4.8 3.0 1.4 0.1 0 +4.3 3.0 1.1 0.1 0 +5.8 4.0 1.2 0.2 0 +5.7 4.4 1.5 0.4 0 +5.4 3.9 1.3 0.4 0 +5.1 3.5 1.4 0.3 0 +5.7 3.8 1.7 0.3 0 +5.1 3.8 1.5 0.3 0 +5.4 3.4 1.7 0.2 0 +5.1 3.7 1.5 0.4 0 +4.6 3.6 1.0 0.2 0 +5.1 3.3 1.7 0.5 0 +4.8 3.4 1.9 0.2 0 +7.0 3.2 4.7 1.4 1 +6.4 3.2 4.5 1.5 1 +6.9 3.1 4.9 1.5 1 +5.5 2.3 4.0 1.3 1 +6.5 2.8 4.6 1.5 1 +5.7 2.8 4.5 1.3 1 +6.3 3.3 4.7 1.6 1 +4.9 2.4 3.3 1.0 1 +6.6 2.9 4.6 1.3 1 +5.2 2.7 3.9 1.4 1 +5.0 2.0 3.5 1.0 1 +5.9 3.0 4.2 1.5 1 +6.0 2.2 4.0 1.0 1 +6.1 2.9 4.7 1.4 1 +5.6 2.9 3.6 1.3 1 +6.7 3.1 4.4 1.4 1 +5.6 3.0 4.5 1.5 1 +5.8 2.7 4.1 1.0 1 +6.2 2.2 4.5 1.5 1 +5.6 2.5 3.9 1.1 1 +5.9 3.2 4.8 1.8 1 +6.1 2.8 4.0 1.3 1 +6.3 2.5 4.9 1.5 1 +6.1 2.8 4.7 1.2 1 +6.4 2.9 4.3 1.3 1 +6.3 3.3 6.0 2.5 2 +5.8 2.7 5.1 1.9 2 +7.1 3.0 5.9 2.1 2 +6.3 2.9 5.6 1.8 2 +6.5 3.0 5.8 2.2 2 +7.6 3.0 6.6 2.1 2 +4.9 2.5 4.5 1.7 2 +7.3 2.9 6.3 1.8 2 +6.7 2.5 5.8 1.8 2 +7.2 3.6 6.1 2.5 2 +6.5 3.2 5.1 2.0 2 +6.4 2.7 5.3 1.9 2 +6.8 3.0 5.5 2.1 2 +5.7 2.5 5.0 2.0 2 +5.8 2.8 5.1 2.4 2 +6.4 3.2 5.3 2.3 2 +6.5 3.0 5.5 1.8 2 +7.7 3.8 6.7 2.2 2 +7.7 2.6 6.9 2.3 2 +6.0 2.2 5.0 1.5 2 +6.9 3.2 5.7 2.3 2 +5.6 2.8 4.9 2.0 2 +7.7 2.8 6.7 2.0 2 +6.3 2.7 4.9 1.8 2 +6.7 3.3 5.7 2.1 2 diff --git a/main.cpp b/main.cpp new file mode 100644 index 0000000..b377157 --- /dev/null +++ b/main.cpp @@ -0,0 +1,110 @@ +// +// Created by iku-iku-iku on 2022/10/19. +// + +#include +#include +#include +#include "Components.h" +#include "Data.h" +#include "Util.h" + + +auto RandomGenerator(unsigned seed) { + std::default_random_engine gen(seed); + std::normal_distribution dis(0, 1); + return std::make_tuple(dis, gen); +} + +#define MV(NAME, N, M, g) \ +auto NAME##_mat = Matrix::RandomlyCreate(g); \ +MatrixVariable NAME{NAME##_mat}; \ +// + +void train(auto &w1, auto &b1, auto &w2, auto &b2) { + Dataset<4> train_dataset; + train_dataset.read_from_file("../data/Iris-train.txt"); +// train_dataset.normalize(); + + + const int epoch = 200; +// const int epoch = 40; + for (int e_i = 0; e_i < epoch; ++e_i) { + double total_loss = 0; + for (int d_i = 0; d_i < train_dataset.size(); ++d_i) { + MatrixVariable x{train_dataset.get_feature(d_i)}; + MatrixVariable y{one_hot<3>(train_dataset.get_label(d_i))}; + + // 组装 + auto a1 = Sigmoid{w1 * x + b1}; + + auto a2 = Sigmoid{w2 * a1 + b2}; + + auto loss = Sum{((a2 - y) ^ 2) * 0.5}; + + + loss.Forward(); + loss.Backward(); + + step(0.9 * std::pow(0.985, e_i), w2, w1, b2, b1); + + total_loss += loss.whole.value(); + } + } +} + +auto test(auto &w1, auto &b1, auto &w2, auto &b2) { + Dataset<4> test_dataset; + test_dataset.read_from_file("../data/Iris-test.txt"); +// test_dataset.normalize(); + + int total = 0, right = 0; + for (int d_i = 0; d_i < test_dataset.size(); ++d_i) { + MatrixVariable x{test_dataset.get_feature(d_i)}; + + // 组装 + auto a1 = Sigmoid{w1 * x + b1}; + + auto pred = Sigmoid{w2 * a1 + b2}; + + pred.Forward(); + + total++; + right += test_dataset.get_label(d_i) == max_i(pred.whole.data); + } + + return 1.0 * right / total; +} + +#define RANDOM + +int main() { + std::vector accuracy_vec; + for (int i = 0; i < 10; i++) { + +#ifdef RANDOM + auto seed = std::chrono::system_clock::now().time_since_epoch().count(); +#else + auto seed = 0; +#endif + auto t = RandomGenerator(seed); + auto g = [&t] { return get<0>(t)(get<1>(t)); }; + + MV(w1, 10, 4, g) + MV(w2, 3, 10, g) + MV(b1, 10, 1, g) + MV(b2, 3, 1, g) + + train(w1, b1, w2, b2); + auto accuracy = test(w1, b1, w2, b2); + accuracy_vec.push_back(accuracy); + } + + for (const auto& acc : accuracy_vec) { + print("accuracy:", acc); + } + + print("sigma:", sigma(accuracy_vec)); + print("mean:", mean(accuracy_vec)); + +} \ No newline at end of file diff --git a/test.cpp b/test.cpp new file mode 100644 index 0000000..a6f910c --- /dev/null +++ b/test.cpp @@ -0,0 +1,42 @@ +// +// Created by iku-iku-iku on 2022/10/21. +// +#include "Components.h" +#include "Util.h" + +void test1() { + auto v1 = Variable(1); + auto v2 = Variable(2); + auto v3 = Variable(3); + auto v4 = Variable(4); + auto v5 = Variable(5); + auto v6 = Variable(6); + auto v7 = Variable(7); + + VarMultiply layer2{v5, v6, v7}; + + VarAddition layer3{v1, v2, v3, v4, layer2}; + layer3.Forward(); + layer3.Backward(); + + print(v1.gradient(), v2.gradient(), v3.gradient(), v4.gradient(), v5.gradient(), v6.gradient(), v7.gradient()); +} + +void test2() { + double mat1[2][2] = {{2, 0}, + {1, 2}}; + double mat2[2][2] = {{1, 0}, + {3, 0}}; + Matrix<2, 2> m1{mat1}; + Matrix<2, 2> m2{mat2}; + + MatrixVariable mv1{m1}; + MatrixVariable mv2{m2}; + + MatAddition layer{mv1, mv2}; + + layer.Forward(); + layer.Backward(); + std::cout << mv2.grad; +} +