From 8f2907ebf35390b49ebbd9c5d4541fdece6ac1ef Mon Sep 17 00:00:00 2001 From: Az-r-ow Date: Wed, 6 Mar 2024 18:52:19 +0100 Subject: [PATCH 1/3] format: created a .clang-format using google's style and formatted all files --- .clang-format | 280 ++++++++++ .prettierrc | 4 - TODO.md | 1 + src/NeuralNet/Model.hpp | 105 ++-- src/NeuralNet/Network.cpp | 200 +++---- src/NeuralNet/Network.hpp | 652 ++++++++++++---------- src/NeuralNet/activations/Activation.hpp | 50 +- src/NeuralNet/activations/Relu.hpp | 38 +- src/NeuralNet/activations/Sigmoid.hpp | 26 +- src/NeuralNet/activations/Softmax.hpp | 40 +- src/NeuralNet/activations/activations.hpp | 3 +- src/NeuralNet/callbacks/CSVLogger.hpp | 209 ++++--- src/NeuralNet/callbacks/Callback.hpp | 117 ++-- src/NeuralNet/callbacks/EarlyStopping.hpp | 120 ++-- src/NeuralNet/data/Tensor.hpp | 93 ++- src/NeuralNet/data/TrainingData.hpp | 75 ++- src/NeuralNet/layers/Dense.hpp | 37 +- src/NeuralNet/layers/Flatten.hpp | 132 +++-- src/NeuralNet/layers/Layer.hpp | 495 ++++++++-------- src/NeuralNet/losses/Loss.hpp | 49 +- src/NeuralNet/losses/MCE.hpp | 35 +- src/NeuralNet/losses/Quadratic.hpp | 37 +- src/NeuralNet/losses/losses.hpp | 3 +- src/NeuralNet/optimizers/Adam.hpp | 191 +++---- src/NeuralNet/optimizers/Optimizer.hpp | 94 ++-- src/NeuralNet/optimizers/SGD.hpp | 42 +- src/NeuralNet/optimizers/optimizers.hpp | 7 +- src/NeuralNet/utils/Enums.hpp | 40 +- src/NeuralNet/utils/Formatters.hpp | 88 ++- src/NeuralNet/utils/Functions.hpp | 541 +++++++++--------- src/NeuralNet/utils/Gauge.hpp | 262 +++++---- src/NeuralNet/utils/Serialize.hpp | 78 +-- src/bindings/NeuralNetPy.cpp | 147 +++-- src/bindings/TemplateBindings.hpp | 14 +- tests/test-activations.cpp | 70 +-- tests/test-callbacks.cpp | 19 +- tests/test-functions.cpp | 26 +- tests/test-layer.cpp | 13 +- tests/test-macros.hpp | 35 +- tests/test-network.cpp | 138 ++--- tests/test-optimizers.cpp | 11 +- tests/test-tensor.cpp | 14 +- 42 files changed, 2413 insertions(+), 2218 deletions(-) create mode 100644 .clang-format delete mode 100644 .prettierrc diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..7403b0f --- /dev/null +++ b/.clang-format @@ -0,0 +1,280 @@ +--- +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignArrayOfStructures: None +AlignConsecutiveAssignments: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionPointers: false + PadOperators: true +AlignConsecutiveBitFields: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveDeclarations: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveMacros: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveShortCaseStatements: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCaseColons: false +AlignEscapedNewlines: Left +AlignOperands: Align +AlignTrailingComments: + Kind: Always + OverEmptyLines: 0 +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowBreakBeforeNoexceptSpecifier: Never +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortCompoundRequirementOnASingleLine: true +AllowShortEnumsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: WithoutElse +AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: true +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +AttributeMacros: + - __capability +BinPackArguments: true +BinPackParameters: true +BitFieldColonSpacing: Both +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: Never + AfterEnum: false + AfterExternBlock: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakAdjacentStringLiterals: true +BreakAfterAttributes: Leave +BreakAfterJavaFieldAnnotations: false +BreakArrays: true +BreakBeforeBinaryOperators: None +BreakBeforeConceptDeclarations: Always +BreakBeforeBraces: Attach +BreakBeforeInlineASMColon: OnlyMultiline +BreakBeforeTernaryOperators: true +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +BreakStringLiterals: true +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: true +DisableFormat: false +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IfMacros: + - KJ_IF_MAYBE +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^' + Priority: 2 + SortPriority: 0 + CaseSensitive: false + - Regex: '^<.*\.h>' + Priority: 1 + SortPriority: 0 + CaseSensitive: false + - Regex: '^<.*' + Priority: 2 + SortPriority: 0 + CaseSensitive: false + - Regex: '.*' + Priority: 3 + SortPriority: 0 + CaseSensitive: false +IncludeIsMainRegex: '([-_](test|unittest))?$' +IncludeIsMainSourceRegex: '' +IndentAccessModifiers: false +IndentCaseBlocks: false +IndentCaseLabels: true +IndentExternBlock: AfterExternBlock +IndentGotoLabels: true +IndentPPDirectives: None +IndentRequiresClause: true +IndentWidth: 2 +IndentWrappedFunctionNames: false +InsertBraces: false +InsertNewlineAtEOF: false +InsertTrailingCommas: None +IntegerLiteralSeparator: + Binary: 0 + BinaryMinDigits: 0 + Decimal: 0 + DecimalMinDigits: 0 + Hex: 0 + HexMinDigits: 0 +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +KeepEmptyLinesAtEOF: false +LambdaBodyIndentation: Signature +LineEnding: DeriveLF +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PackConstructorInitializers: NextLine +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakScopeResolution: 500 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyIndentedWhitespace: 0 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +PPIndentWidth: -1 +QualifierAlignment: Leave +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + BasedOnStyle: google + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + - ParseTestProto + - ParsePartialTestProto + CanonicalDelimiter: pb + BasedOnStyle: google +ReferenceAlignment: Pointer +ReflowComments: true +RemoveBracesLLVM: false +RemoveParentheses: Leave +RemoveSemicolon: false +RequiresClausePosition: OwnLine +RequiresExpressionIndentation: OuterScope +SeparateDefinitionBlocks: Leave +ShortNamespaceLines: 1 +SkipMacroDefinitionBody: false +SortIncludes: CaseSensitive +SortJavaStaticImport: Before +SortUsingDeclarations: LexicographicNumeric +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceAroundPointerQualifiers: Default +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeJsonColon: false +SpaceBeforeParens: ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterOverloadedOperator: false + AfterPlacementOperator: true + AfterRequiresInClause: false + AfterRequiresInExpression: false + BeforeNonEmptyParentheses: false +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: Never +SpacesInContainerLiterals: true +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 +SpacesInParens: Never +SpacesInParensOptions: + InCStyleCasts: false + InConditionalStatements: false + InEmptyParentheses: false + Other: false +SpacesInSquareBrackets: false +Standard: Auto +StatementAttributeLikeMacros: + - Q_EMIT +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 8 +UseTab: Never +VerilogBreakBetweenInstancePorts: true +WhitespaceSensitiveMacros: + - BOOST_PP_STRINGIZE + - CF_SWIFT_NAME + - NS_SWIFT_NAME + - PP_STRINGIZE + - STRINGIZE +... + diff --git a/.prettierrc b/.prettierrc deleted file mode 100644 index 222861c..0000000 --- a/.prettierrc +++ /dev/null @@ -1,4 +0,0 @@ -{ - "tabWidth": 2, - "useTabs": false -} diff --git a/TODO.md b/TODO.md index 8244a03..c0ba684 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,6 @@ ## TODOS : +- [ ] add automatic versioning in ci - [ ] Setup `clang-format` - [ ] CI versioning - [ ] Find out why the predictions are not accurate on my_samples diff --git a/src/NeuralNet/Model.hpp b/src/NeuralNet/Model.hpp index 6bad027..fefc51e 100644 --- a/src/NeuralNet/Model.hpp +++ b/src/NeuralNet/Model.hpp @@ -1,60 +1,63 @@ #pragma once -#include +#include +#include +#include #include +#include #include -#include -#include -#include -#include "utils/Functions.hpp" -namespace NeuralNet -{ - class Model - { - public: - /** - * @brief This method will save (by serializing) the model passed as argument to a .bin file - * - * @param filename The binary file in which the serialized model will be saved - * @param model The model that will be serialized and saved - */ - template ::value>::type> - static void save_to_file(const std::string &filename, T model) - { - // Serializing model to a binary file - std::ofstream file(filename, std::ios::binary); - cereal::BinaryOutputArchive archive(file); - archive(model); - }; +#include "utils/Functions.hpp" - /** - * @brief This static method loads a Model from a file and assigns it to the supposedly "empty" model passed as argument - * - * @param filename The name of the binary file from which to retrieve the model - * @param model An "empty" model which will be filled with the loaded model's parameters - * - * This function will assign the parameters of the saved model in the binary file to the model passed as a parameter - */ - template ::value>::type> - static void - load_from_file(const std::string &filename, T &model) - { - // Making sure the file exists and is binary - assert(fileExistsWithExtension(filename, ".bin") && "The file doesn't exists or is not binary '.bin'"); +namespace NeuralNet { +class Model { + public: + /** + * @brief This method will save (by serializing) the model passed as argument + * to a .bin file + * + * @param filename The binary file in which the serialized model will be saved + * @param model The model that will be serialized and saved + */ + template ::value>::type> + static void save_to_file(const std::string &filename, T model) { + // Serializing model to a binary file + std::ofstream file(filename, std::ios::binary); + cereal::BinaryOutputArchive archive(file); + archive(model); + }; - // Deserializing the model from the binary file - std::ifstream file(filename, std::ios::binary); - cereal::BinaryInputArchive archive(file); - archive(model); - }; + /** + * @brief This static method loads a Model from a file and assigns it to the + * supposedly "empty" model passed as argument + * + * @param filename The name of the binary file from which to retrieve the + * model + * @param model An "empty" model which will be filled with the loaded model's + * parameters + * + * This function will assign the parameters of the saved model in the binary + * file to the model passed as a parameter + */ + template ::value>::type> + static void load_from_file(const std::string &filename, T &model) { + // Making sure the file exists and is binary + assert(fileExistsWithExtension(filename, ".bin") && + "The file doesn't exists or is not binary '.bin'"); - protected: - void registerSignals() const - { - // Registering signals - signal(SIGINT, signalHandler); - signal(SIGTERM, signalHandler); - } + // Deserializing the model from the binary file + std::ifstream file(filename, std::ios::binary); + cereal::BinaryInputArchive archive(file); + archive(model); }; -} \ No newline at end of file + + protected: + void registerSignals() const { + // Registering signals + signal(SIGINT, signalHandler); + signal(SIGTERM, signalHandler); + } +}; +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/Network.cpp b/src/NeuralNet/Network.cpp index dc4df06..392158a 100644 --- a/src/NeuralNet/Network.cpp +++ b/src/NeuralNet/Network.cpp @@ -4,26 +4,20 @@ using namespace NeuralNet; Network::Network(){}; -size_t Network::getNumLayers() const -{ - return this->layers.size(); -} +size_t Network::getNumLayers() const { return this->layers.size(); } -void Network::setup(const std::shared_ptr &optimizer, LOSS loss) -{ +void Network::setup(const std::shared_ptr &optimizer, LOSS loss) { this->optimizer = optimizer; this->lossFunc = loss; this->setLoss(loss); this->updateOptimizerSetup(this->layers.size()); - this->registerSignals(); // Allows smooth exit of program + this->registerSignals(); // Allows smooth exit of program } -void Network::addLayer(std::shared_ptr &layer) -{ +void Network::addLayer(std::shared_ptr &layer) { size_t numLayers = this->layers.size(); // Init layer with right amount of weights - if (numLayers > 0) - { + if (numLayers > 0) { std::shared_ptr prevLayer = this->layers[this->layers.size() - 1]; layer->init(prevLayer->getNumNeurons()); } @@ -31,113 +25,104 @@ void Network::addLayer(std::shared_ptr &layer) this->layers.push_back(layer); } -void Network::setLoss(LOSS loss) -{ - switch (loss) - { - case LOSS::QUADRATIC: - this->cmpLoss = Quadratic::cmpLoss; - this->cmpLossGrad = Quadratic::cmpLossGrad; - break; - case LOSS::MCE: - this->cmpLoss = MCE::cmpLoss; - this->cmpLossGrad = MCE::cmpLossGrad; - break; - default: - assert(false && "Loss not defined"); - break; +void Network::setLoss(LOSS loss) { + switch (loss) { + case LOSS::QUADRATIC: + this->cmpLoss = Quadratic::cmpLoss; + this->cmpLossGrad = Quadratic::cmpLossGrad; + break; + case LOSS::MCE: + this->cmpLoss = MCE::cmpLoss; + this->cmpLossGrad = MCE::cmpLossGrad; + break; + default: + assert(false && "Loss not defined"); + break; } } -std::shared_ptr Network::getLayer(int index) const -{ +std::shared_ptr Network::getLayer(int index) const { assert(index < this->layers.size() && index >= 0); return this->layers.at(index); } -std::shared_ptr Network::getOutputLayer() const -{ +std::shared_ptr Network::getOutputLayer() const { assert(this->layers.size() > 0); return this->layers[this->layers.size() - 1]; } -double Network::train(std::vector> inputs, std::vector labels, int epochs, std::vector> callbacks) -{ - try - { +double Network::train(std::vector> inputs, + std::vector labels, int epochs, + std::vector> callbacks) { + try { return onlineTraining(inputs, labels, epochs, callbacks); - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { std::cerr << "Training Interrupted : " << e.what() << '\n'; return loss; } } -double Network::train(std::vector>> inputs, std::vector labels, int epochs, std::vector> callbacks) -{ - try - { +double Network::train(std::vector>> inputs, + std::vector labels, int epochs, + std::vector> callbacks) { + try { return onlineTraining(inputs, labels, epochs, callbacks); - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { std::cerr << "Training Interrupted : " << e.what() << '\n'; return loss; } } // Specific implementation of train that takes TrainingData class as input -double Network::train(TrainingData>, std::vector> trainingData, int epochs, std::vector> callbacks) -{ - try - { +double Network::train( + TrainingData>, std::vector> + trainingData, + int epochs, std::vector> callbacks) { + try { return this->trainer(trainingData, epochs, callbacks); - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { std::cerr << "Training Interrupted : " << e.what() << '\n'; return loss; } } -double Network::train(TrainingData>>, std::vector> trainingData, int epochs, std::vector> callbacks) -{ - try - { +double Network::train( + TrainingData>>, + std::vector> + trainingData, + int epochs, std::vector> callbacks) { + try { return this->trainer(trainingData, epochs, callbacks); - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { std::cerr << "Training Interrupted : " << e.what() << '\n'; return loss; } } template -double Network::trainer(TrainingData trainingData, int epochs, std::vector> callbacks) -{ +double Network::trainer(TrainingData trainingData, int epochs, + std::vector> callbacks) { if (trainingData.batched) return this->miniBatchTraining(trainingData, epochs, callbacks); return this->batchTraining(trainingData, epochs, callbacks); } template -double Network::miniBatchTraining(TrainingData trainingData, int epochs, std::vector> callbacks) -{ +double Network::miniBatchTraining( + TrainingData trainingData, int epochs, + std::vector> callbacks) { double sumLoss = 0; trainingCheckpoint("onTrainBegin", callbacks); - for (cEpoch = 0; cEpoch < epochs; cEpoch++) - { + for (cEpoch = 0; cEpoch < epochs; cEpoch++) { trainingCheckpoint("onEpochBegin", callbacks); TrainingGauge g(trainingData.inputs.size(), 0, epochs, (cEpoch + 1)); - for (int b = 0; b < trainingData.inputs.size(); b++) - { + for (int b = 0; b < trainingData.inputs.size(); b++) { trainingCheckpoint("onBatchBegin", callbacks); const int numOutputs = this->getOutputLayer()->getNumNeurons(); const int inputsSize = trainingData.inputs.batches[b].size(); - Eigen::MatrixXd y = formatLabels(trainingData.labels.batches[b], {inputsSize, numOutputs}); + Eigen::MatrixXd y = formatLabels(trainingData.labels.batches[b], + {inputsSize, numOutputs}); // computing outputs from forward propagation Eigen::MatrixXd o = this->forwardProp(trainingData.inputs.batches[b]); @@ -156,16 +141,17 @@ double Network::miniBatchTraining(TrainingData trainingData, int epochs, } template -double Network::batchTraining(TrainingData trainingData, int epochs, std::vector> callbacks) -{ +double Network::batchTraining( + TrainingData trainingData, int epochs, + std::vector> callbacks) { double sumLoss = 0; const int numOutputs = this->getOutputLayer()->getNumNeurons(); const int numInputs = trainingData.inputs.data.size(); - Eigen::MatrixXd y = formatLabels(trainingData.labels.data, {numInputs, numOutputs}); + Eigen::MatrixXd y = + formatLabels(trainingData.labels.data, {numInputs, numOutputs}); trainingCheckpoint("onTrainBegin", callbacks); - for (cEpoch = 0; cEpoch < epochs; cEpoch++) - { + for (cEpoch = 0; cEpoch < epochs; cEpoch++) { trainingCheckpoint("onEpochBegin", callbacks); TrainingGauge g(1, 0, epochs, (cEpoch + 1)); Eigen::MatrixXd o = this->forwardProp(trainingData.inputs.data); @@ -184,8 +170,9 @@ double Network::batchTraining(TrainingData trainingData, int epochs, std } template -double Network::onlineTraining(std::vector inputs, std::vector labels, int epochs, std::vector> callbacks) -{ +double Network::onlineTraining( + std::vector inputs, std::vector labels, int epochs, + std::vector> callbacks) { double sumLoss = 0; int tCorrect = 0; const int numOutputs = this->getOutputLayer()->getNumNeurons(); @@ -195,12 +182,10 @@ double Network::onlineTraining(std::vector inputs, std::vector labels, i // Injecting callbacks trainingCheckpoint("onTrainBegin", callbacks); - for (cEpoch = 0; cEpoch < epochs; cEpoch++) - { + for (cEpoch = 0; cEpoch < epochs; cEpoch++) { trainingCheckpoint("onEpochBegin", callbacks); TrainingGauge tg(inputs.size(), 0, epochs, (cEpoch + 1)); - for (auto &input : inputs) - { + for (auto &input : inputs) { Eigen::MatrixXd o = this->forwardProp(inputs); loss = this->cmpLoss(o, y); sumLoss += loss; @@ -218,35 +203,32 @@ double Network::onlineTraining(std::vector inputs, std::vector labels, i return sumLoss / numInputs; } -Eigen::MatrixXd Network::predict(std::vector> inputs) -{ +Eigen::MatrixXd Network::predict(std::vector> inputs) { Eigen::MatrixXd mInputs = vectorToMatrixXd(inputs); return forwardProp(mInputs); } -Eigen::MatrixXd Network::predict(std::vector>> inputs) -{ +Eigen::MatrixXd Network::predict( + std::vector>> inputs) { return forwardProp(inputs); } /** * Forward propagation */ -Eigen::MatrixXd Network::feedForward(Eigen::MatrixXd inputs, int startIdx) -{ +Eigen::MatrixXd Network::feedForward(Eigen::MatrixXd inputs, int startIdx) { assert(startIdx < this->layers.size()); Eigen::MatrixXd prevLayerOutputs = inputs; - for (int l = startIdx; l < this->layers.size(); l++) - { + for (int l = startIdx; l < this->layers.size(); l++) { prevLayerOutputs = this->layers[l]->feedInputs(prevLayerOutputs); } return prevLayerOutputs; } -Eigen::MatrixXd Network::forwardProp(std::vector>> &inputs) -{ +Eigen::MatrixXd Network::forwardProp( + std::vector>> &inputs) { // Passing the inputs as outputs to the input layer this->layers[0]->feedInputs(inputs); @@ -255,30 +237,26 @@ Eigen::MatrixXd Network::forwardProp(std::vector return feedForward(prevLayerOutputs, 1); } -Eigen::MatrixXd Network::forwardProp(std::vector> &inputs) -{ +Eigen::MatrixXd Network::forwardProp(std::vector> &inputs) { // Previous layer outputs Eigen::MatrixXd prevLayerO = vectorToMatrixXd(inputs); return feedForward(prevLayerO); } -Eigen::MatrixXd Network::forwardProp(Eigen::MatrixXd &inputs) -{ +Eigen::MatrixXd Network::forwardProp(Eigen::MatrixXd &inputs) { // Previous layer outputs Eigen::MatrixXd prevLayerO = inputs; return feedForward(prevLayerO); } -void Network::backProp(Eigen::MatrixXd &outputs, Eigen::MatrixXd &y) -{ +void Network::backProp(Eigen::MatrixXd &outputs, Eigen::MatrixXd &y) { // Next Layer activation der dL/da(l - 1) Eigen::MatrixXd beta = this->cmpLossGrad(outputs, y); int m = beta.rows(); - for (size_t i = this->layers.size(); --i > 0;) - { + for (size_t i = this->layers.size(); --i > 0;) { std::shared_ptr cLayer = this->layers[i]; std::shared_ptr nLayer = this->layers[i - 1]; // a'(L) @@ -300,32 +278,30 @@ void Network::backProp(Eigen::MatrixXd &outputs, Eigen::MatrixXd &y) } } -void Network::updateOptimizerSetup(size_t numLayers) -{ +void Network::updateOptimizerSetup(size_t numLayers) { /** * This is a way to let adams know about the number of layers * With that it can setup the `l` variable and the std::vectors * - * I'm not very proud of this method but so far it seems like the most convenient way + * I'm not very proud of this method but so far it seems like the most + * convenient way */ this->optimizer->insiderInit(numLayers); } -void Network::trainingCheckpoint(std::string checkpointName, std::vector> callbacks) -{ - if (callbacks.size() == 0) - return; +void Network::trainingCheckpoint( + std::string checkpointName, + std::vector> callbacks) { + if (callbacks.size() == 0) return; std::unordered_map logs = getLogs(); - for (std::shared_ptr callback : callbacks) - { + for (std::shared_ptr callback : callbacks) { Callback::callMethod(callback, checkpointName, logs); } } -std::unordered_map Network::getLogs() -{ +std::unordered_map Network::getLogs() { std::unordered_map logs; logs["LOSS"] = loss; @@ -336,15 +312,15 @@ std::unordered_map Network::getLogs() } /** - * @note This function will return the accuracy of the given outputs compared to the labels. + * @note This function will return the accuracy of the given outputs compared to + * the labels. * * @param outputs The outputs from the network * @param y The labels * * @return The accuracy of the network. */ -double Network::computeAccuracy(Eigen::MatrixXd &outputs, Eigen::MatrixXd &y) -{ +double Network::computeAccuracy(Eigen::MatrixXd &outputs, Eigen::MatrixXd &y) { int total = y.rows(); // Hardmax the outputs @@ -357,6 +333,4 @@ double Network::computeAccuracy(Eigen::MatrixXd &outputs, Eigen::MatrixXd &y) return 1.0 - (wrong / static_cast(total)); } -Network::~Network() -{ -} \ No newline at end of file +Network::~Network() {} \ No newline at end of file diff --git a/src/NeuralNet/Network.hpp b/src/NeuralNet/Network.hpp index 58efba3..2818a75 100644 --- a/src/NeuralNet/Network.hpp +++ b/src/NeuralNet/Network.hpp @@ -1,309 +1,363 @@ #pragma once -#include -#include -#include -#include // for defer +#include // for defer #include #include +#include +#include +#include + #include "Model.hpp" +#include "callbacks/Callback.hpp" +#include "data/Tensor.hpp" +#include "data/TrainingData.hpp" +#include "layers/Dense.hpp" +#include "layers/Flatten.hpp" +#include "layers/Layer.hpp" +#include "losses/losses.hpp" +#include "optimizers/Optimizer.hpp" +#include "optimizers/optimizers.hpp" #include "utils/Formatters.hpp" #include "utils/Functions.hpp" #include "utils/Gauge.hpp" -#include "optimizers/Optimizer.hpp" -#include "layers/Layer.hpp" -#include "layers/Flatten.hpp" -#include "layers/Dense.hpp" -#include "optimizers/optimizers.hpp" -#include "losses/losses.hpp" -#include "data/Tensor.hpp" -#include "data/TrainingData.hpp" -#include "callbacks/Callback.hpp" -namespace NeuralNet -{ - class Layer; - - class Network : public Model - { - public: - Network(); - - /** - * @brief Method that sets up the model's hyperparameter - * - * @param optimizer An Optimizer's child class - * @param epochs The number of epochs - * @param loss The loss function - */ - void setup(const std::shared_ptr &optimizer, LOSS loss = LOSS::QUADRATIC); - - /** - * @brief Method to add a layer to the network - * - * @param layer the layer to add to the model it should be of type Layer - */ - void addLayer(std::shared_ptr &layer); - - /** - * @brief This method will set the network's loss function - * - * @param loss The loss function (choose from the list of LOSS enums) - */ - void setLoss(LOSS loss); - - /** - * @brief This method will return the Layer residing at the specified index - * - * @param index The index from which to fetch the layer - * - * @return Layer at specified index - */ - std::shared_ptr getLayer(int index) const; - - /** - * @brief This method will return the output layer (the last layer of the network) - * - * @return The output Layer - */ - std::shared_ptr getOutputLayer() const; - - /** - * @brief This method will get you the number of layers currently in the Network - * - * @return A size_t representing the number of layers in the Network - */ - size_t getNumLayers() const; - - /** - * @brief This method will Train the model with the given inputs and labels - * - * @param inputs The inputs that will be passed to the model - * @param labels The labels that represent the expected outputs of the model - * @param epochs - * @param callbacks A vector of `Callback` that will be called during training stages - * - * @return The last training's loss - */ - double train(std::vector> inputs, std::vector labels, int epochs = 1, const std::vector> callbacks = {}); - - /** - * @brief This method will Train the model with the given inputs and labels - * - * @param inputs The inputs that will be passed to the model - * @param labels The labels that represent the expected outputs of the model - * @param epochs - * @param callbacks A vector of `Callback` that will be called during training stages - * - * @return The last training's loss - */ - double train(std::vector>> inputs, std::vector labels, int epochs = 1, const std::vector> callbacks = {}); - - /** - * @brief This method will train the model with the given TrainingData - * - * @param trainingData the data passed through the TrainingData class - * @param epochs - * @param callbacks A vector of `Callback` that will be called during training stages - * - * @return The last training's loss - */ - double train(TrainingData>, std::vector> trainingData, int epochs = 1, const std::vector> callbacks = {}); - - /** - * @brief This method will train the model with the given TrainingData - * - * @param trainingData the data passed through the TrainingData class - * @param epochs - * @param callbacks A vector of `Callback` that will be called during training stages - * - * @return The last training's loss - */ - double train(TrainingData>>, std::vector> trainingData, int epochs = 1, const std::vector> callbacks = {}); - - /** - * @brief This model will try to make predictions based off the inputs passed - * - * @param inputs The inputs that will be passed through the network - * - * @return This method will return the outputs of the neural network - */ - Eigen::MatrixXd predict(std::vector> inputs); - - /** - * @brief This model will try to make predictions based off the inputs passed - * - * @param inputs The inputs that will be passed through the network - * - * @return This method will return the outputs of the neural network - */ - Eigen::MatrixXd predict(std::vector>> inputs); - - ~Network(); - - private: - // non-public serialization - friend class cereal::access; - - int cEpoch = 0; // Current epoch - double loss = 0, accuracy = 0; - std::vector> layers; - LOSS lossFunc; // Storing the loss function for serialization - bool debugMode = false; - double (*cmpLoss)(const Eigen::MatrixXd &, const Eigen::MatrixXd &); - Eigen::MatrixXd (*cmpLossGrad)(const Eigen::MatrixXd &, const Eigen::MatrixXd &); - std::shared_ptr optimizer; - - template - void save(Archive &archive) const - { - archive(layers, lossFunc); - archive.serializeDeferments(); - }; - - template - void load(Archive &archive) - { - archive(layers, lossFunc); - setLoss(lossFunc); - } - - /** - * @brief online training with given training data - * - * @tparam D1 The type of data passed - * @tparam D2 The type of labels passed - * @param inputs A vector of inputs (features) of type D1 - * @param labels A vector of labels (targets) of type D2. Each element in this vector corresponds to the - * label of the training example at the same index in the inputs vector. - * @param epochs An integer specifying the number of times the training algorithm should iterate over the dataset. - * @param callbacks A vector of `Callback` that will be called during training stages - * - * @return A double value that represents the average loss of the training process. This can be used to gauge the effectiveness of the process. - * - * @note The functions assumes that the inputs and labels will be of the same length. - */ - template - double onlineTraining(std::vector inputs, std::vector labels, int epochs, std::vector> callbacks = {}); - - /** - * @brief mini-batch training with given training data - * - * @tparam D1 The type of data passed - * @tparam D2 The type of labels passed - * @param inputs A vector of inputs (features) of type D1 - * @param labels A vector of labels (targets) of type D2. Each element in this vector corresponds to the - * label of the training example at the same index in the inputs vector. - * @param epochs An integer specifying the number of times the training algorithm should iterate over the dataset. - * @param callbacks A vector of `Callback` that will be called during training stages - * - * @return A double value that represents the average loss of the training process. This can be used to gauge the effectiveness of the process. - * - * @note The functions assumes that the inputs and labels will be of the same length. - */ - template - double trainer(TrainingData trainingData, int epochs, std::vector> callbacks = {}); - - /** - * @brief mini-batch training with given training data - * - * @tparam D1 The type of data passed - * @tparam D2 The type of labels passed - * @param inputs A vector of inputs (features) of type D1 - * @param labels A vector of labels (targets) of type D2. Each element in this vector corresponds to the - * label of the training example at the same index in the inputs vector. - * @param epochs An integer specifying the number of times the training algorithm should iterate over the dataset. - * @param callbacks A vector of `Callback` that will be called during training stages * @return A double value that represents the average loss of the training process. This can be used to gauge the effectiveness of the process. - * - * @note The functions assumes that the inputs and labels will be of the same length. - */ - template - double miniBatchTraining(TrainingData trainingData, int epochs, std::vector> callbacks = {}); - - /** - * @brief batch training with given training data - * - * @tparam D1 The type of data passed - * @tparam D2 The type of labels passed - * @param inputs A vector of inputs (features) of type D1 - * @param labels A vector of labels (targets) of type D2. Each element in this vector corresponds to the - * label of the training example at the same index in the inputs vector. - * @param epochs An integer specifying the number of times the training algorithm should iterate over the dataset. - * @param callbacks A vector of `Callback` that will be called during training stages - * @return A double value that represents the average loss of the training process. This can be used to gauge the effectiveness of the process. - * - * @note The functions assumes that the inputs and labels will be of the same length. - */ - template - double batchTraining(TrainingData trainingData, int epochs, std::vector> callbacks = {}); - - /** - * @brief This method will pass the inputs through the network and return an output - * - * @param inputs The inputs that will be passed through the network - * - * @return The output of the network - */ - Eigen::MatrixXd forwardProp(std::vector>> &inputs); - - /** - * @brief This method will pass the inputs through the network and return an output - * - * @param inputs The inputs that will be passed through the network - * - * @return The output of the network - */ - Eigen::MatrixXd forwardProp(std::vector> &inputs); - - /** - * @brief This method will pass the inputs through the network and return an output - * - * @param inputs The inputs that will be passed through the network - * - * @return The output of the network - */ - Eigen::MatrixXd forwardProp(Eigen::MatrixXd &inputs); - - Eigen::MatrixXd feedForward(Eigen::MatrixXd inputs, int startIdx = 0); - - /** - * @brief This method will compute the loss and backpropagate it through the network whilst doing adjusting the parameters accordingly. - * - * @param outputs The outputs from the forward propagation - * @param y The expected outputs (targets) - */ - void backProp(Eigen::MatrixXd &outputs, Eigen::MatrixXd &y); - - /** - * @brief This method will go over the provided callbacks and trigger the appropriate methods whilst passing the necessary logs. - * - * @param checkpointName The name of the checkpoint (e.g. onTrainBegin, onEpochEnd, etc.) - * @param callbacks A vector of `Callback` that will be called during training stages - */ - void trainingCheckpoint(std::string checkpointName, std::vector> callbacks); - - /** - * @brief This method will compute the accuracy of the model based on the outputs of the model and the expected values. - * - * @param outputs The outputs from the forward propagation - * @param y The expected outputs (targets) - * - * @return The accuracy of the model (percentage of correct predictions) - */ - double computeAccuracy(Eigen::MatrixXd &outputs, Eigen::MatrixXd &y); - - /** - * @brief This method will fetch the logs and return them - * - * @return A map of useful logs - */ - std::unordered_map getLogs(); - - /** - * @brief This method will update the optimizer's setup - * - * @param numLayers The number of layers in the network - */ - void updateOptimizerSetup(size_t numLayers); +namespace NeuralNet { +class Layer; + +class Network : public Model { + public: + Network(); + + /** + * @brief Method that sets up the model's hyperparameter + * + * @param optimizer An Optimizer's child class + * @param epochs The number of epochs + * @param loss The loss function + */ + void setup(const std::shared_ptr &optimizer, + LOSS loss = LOSS::QUADRATIC); + + /** + * @brief Method to add a layer to the network + * + * @param layer the layer to add to the model it should be of type Layer + */ + void addLayer(std::shared_ptr &layer); + + /** + * @brief This method will set the network's loss function + * + * @param loss The loss function (choose from the list of LOSS enums) + */ + void setLoss(LOSS loss); + + /** + * @brief This method will return the Layer residing at the specified index + * + * @param index The index from which to fetch the layer + * + * @return Layer at specified index + */ + std::shared_ptr getLayer(int index) const; + + /** + * @brief This method will return the output layer (the last layer of the + * network) + * + * @return The output Layer + */ + std::shared_ptr getOutputLayer() const; + + /** + * @brief This method will get you the number of layers currently in the + * Network + * + * @return A size_t representing the number of layers in the Network + */ + size_t getNumLayers() const; + + /** + * @brief This method will Train the model with the given inputs and labels + * + * @param inputs The inputs that will be passed to the model + * @param labels The labels that represent the expected outputs of the model + * @param epochs + * @param callbacks A vector of `Callback` that will be called during training + * stages + * + * @return The last training's loss + */ + double train(std::vector> inputs, + std::vector labels, int epochs = 1, + const std::vector> callbacks = {}); + + /** + * @brief This method will Train the model with the given inputs and labels + * + * @param inputs The inputs that will be passed to the model + * @param labels The labels that represent the expected outputs of the model + * @param epochs + * @param callbacks A vector of `Callback` that will be called during training + * stages + * + * @return The last training's loss + */ + double train(std::vector>> inputs, + std::vector labels, int epochs = 1, + const std::vector> callbacks = {}); + + /** + * @brief This method will train the model with the given TrainingData + * + * @param trainingData the data passed through the TrainingData class + * @param epochs + * @param callbacks A vector of `Callback` that will be called during training + * stages + * + * @return The last training's loss + */ + double train( + TrainingData>, std::vector> + trainingData, + int epochs = 1, + const std::vector> callbacks = {}); + + /** + * @brief This method will train the model with the given TrainingData + * + * @param trainingData the data passed through the TrainingData class + * @param epochs + * @param callbacks A vector of `Callback` that will be called during training + * stages + * + * @return The last training's loss + */ + double train(TrainingData>>, + std::vector> + trainingData, + int epochs = 1, + const std::vector> callbacks = {}); + + /** + * @brief This model will try to make predictions based off the inputs passed + * + * @param inputs The inputs that will be passed through the network + * + * @return This method will return the outputs of the neural network + */ + Eigen::MatrixXd predict(std::vector> inputs); + + /** + * @brief This model will try to make predictions based off the inputs passed + * + * @param inputs The inputs that will be passed through the network + * + * @return This method will return the outputs of the neural network + */ + Eigen::MatrixXd predict(std::vector>> inputs); + + ~Network(); + + private: + // non-public serialization + friend class cereal::access; + + int cEpoch = 0; // Current epoch + double loss = 0, accuracy = 0; + std::vector> layers; + LOSS lossFunc; // Storing the loss function for serialization + bool debugMode = false; + double (*cmpLoss)(const Eigen::MatrixXd &, const Eigen::MatrixXd &); + Eigen::MatrixXd (*cmpLossGrad)(const Eigen::MatrixXd &, + const Eigen::MatrixXd &); + std::shared_ptr optimizer; + + template + void save(Archive &archive) const { + archive(layers, lossFunc); + archive.serializeDeferments(); }; -} // namespace NeuralNet + + template + void load(Archive &archive) { + archive(layers, lossFunc); + setLoss(lossFunc); + } + + /** + * @brief online training with given training data + * + * @tparam D1 The type of data passed + * @tparam D2 The type of labels passed + * @param inputs A vector of inputs (features) of type D1 + * @param labels A vector of labels (targets) of type D2. Each element in this + * vector corresponds to the label of the training example at the same index + * in the inputs vector. + * @param epochs An integer specifying the number of times the training + * algorithm should iterate over the dataset. + * @param callbacks A vector of `Callback` that will be called during training + * stages + * + * @return A double value that represents the average loss of the training + * process. This can be used to gauge the effectiveness of the process. + * + * @note The functions assumes that the inputs and labels will be of the same + * length. + */ + template + double onlineTraining(std::vector inputs, std::vector labels, + int epochs, + std::vector> callbacks = {}); + + /** + * @brief mini-batch training with given training data + * + * @tparam D1 The type of data passed + * @tparam D2 The type of labels passed + * @param inputs A vector of inputs (features) of type D1 + * @param labels A vector of labels (targets) of type D2. Each element in this + * vector corresponds to the label of the training example at the same index + * in the inputs vector. + * @param epochs An integer specifying the number of times the training + * algorithm should iterate over the dataset. + * @param callbacks A vector of `Callback` that will be called during training + * stages + * + * @return A double value that represents the average loss of the training + * process. This can be used to gauge the effectiveness of the process. + * + * @note The functions assumes that the inputs and labels will be of the same + * length. + */ + template + double trainer(TrainingData trainingData, int epochs, + std::vector> callbacks = {}); + + /** + * @brief mini-batch training with given training data + * + * @tparam D1 The type of data passed + * @tparam D2 The type of labels passed + * @param inputs A vector of inputs (features) of type D1 + * @param labels A vector of labels (targets) of type D2. Each element in this + * vector corresponds to the label of the training example at the same index + * in the inputs vector. + * @param epochs An integer specifying the number of times the training + * algorithm should iterate over the dataset. + * @param callbacks A vector of `Callback` that will be called during training + * stages * @return A double value that represents the average loss of the + * training process. This can be used to gauge the effectiveness of the + * process. + * + * @note The functions assumes that the inputs and labels will be of the same + * length. + */ + template + double miniBatchTraining( + TrainingData trainingData, int epochs, + std::vector> callbacks = {}); + + /** + * @brief batch training with given training data + * + * @tparam D1 The type of data passed + * @tparam D2 The type of labels passed + * @param inputs A vector of inputs (features) of type D1 + * @param labels A vector of labels (targets) of type D2. Each element in this + * vector corresponds to the label of the training example at the same index + * in the inputs vector. + * @param epochs An integer specifying the number of times the training + * algorithm should iterate over the dataset. + * @param callbacks A vector of `Callback` that will be called during training + * stages + * @return A double value that represents the average loss of the training + * process. This can be used to gauge the effectiveness of the process. + * + * @note The functions assumes that the inputs and labels will be of the same + * length. + */ + template + double batchTraining(TrainingData trainingData, int epochs, + std::vector> callbacks = {}); + + /** + * @brief This method will pass the inputs through the network and return an + * output + * + * @param inputs The inputs that will be passed through the network + * + * @return The output of the network + */ + Eigen::MatrixXd forwardProp( + std::vector>> &inputs); + + /** + * @brief This method will pass the inputs through the network and return an + * output + * + * @param inputs The inputs that will be passed through the network + * + * @return The output of the network + */ + Eigen::MatrixXd forwardProp(std::vector> &inputs); + + /** + * @brief This method will pass the inputs through the network and return an + * output + * + * @param inputs The inputs that will be passed through the network + * + * @return The output of the network + */ + Eigen::MatrixXd forwardProp(Eigen::MatrixXd &inputs); + + Eigen::MatrixXd feedForward(Eigen::MatrixXd inputs, int startIdx = 0); + + /** + * @brief This method will compute the loss and backpropagate it through the + * network whilst doing adjusting the parameters accordingly. + * + * @param outputs The outputs from the forward propagation + * @param y The expected outputs (targets) + */ + void backProp(Eigen::MatrixXd &outputs, Eigen::MatrixXd &y); + + /** + * @brief This method will go over the provided callbacks and trigger the + * appropriate methods whilst passing the necessary logs. + * + * @param checkpointName The name of the checkpoint (e.g. onTrainBegin, + * onEpochEnd, etc.) + * @param callbacks A vector of `Callback` that will be called during training + * stages + */ + void trainingCheckpoint(std::string checkpointName, + std::vector> callbacks); + + /** + * @brief This method will compute the accuracy of the model based on the + * outputs of the model and the expected values. + * + * @param outputs The outputs from the forward propagation + * @param y The expected outputs (targets) + * + * @return The accuracy of the model (percentage of correct predictions) + */ + double computeAccuracy(Eigen::MatrixXd &outputs, Eigen::MatrixXd &y); + + /** + * @brief This method will fetch the logs and return them + * + * @return A map of useful logs + */ + std::unordered_map getLogs(); + + /** + * @brief This method will update the optimizer's setup + * + * @param numLayers The number of layers in the network + */ + void updateOptimizerSetup(size_t numLayers); +}; +} // namespace NeuralNet diff --git a/src/NeuralNet/activations/Activation.hpp b/src/NeuralNet/activations/Activation.hpp index 1ff8313..b553cbc 100644 --- a/src/NeuralNet/activations/Activation.hpp +++ b/src/NeuralNet/activations/Activation.hpp @@ -2,33 +2,25 @@ #include -namespace NeuralNet -{ - class Activation - { - public: - /** - * @brief Activate a layer's outputs - * - * @param z A matrix representing a layer's outputs - * - * @return The activated outputs - */ - static Eigen::MatrixXd activate(const Eigen::MatrixXd &z) - { - return z; - }; +namespace NeuralNet { +class Activation { + public: + /** + * @brief Activate a layer's outputs + * + * @param z A matrix representing a layer's outputs + * + * @return The activated outputs + */ + static Eigen::MatrixXd activate(const Eigen::MatrixXd &z) { return z; }; - /** - * @brief Compute the derivative of the activation function - * - * @param a Activated outputs - * - * @return The activated outputs derivatives - */ - static Eigen::MatrixXd diff(const Eigen::MatrixXd &a) - { - return a; - }; - }; -} + /** + * @brief Compute the derivative of the activation function + * + * @param a Activated outputs + * + * @return The activated outputs derivatives + */ + static Eigen::MatrixXd diff(const Eigen::MatrixXd &a) { return a; }; +}; +} // namespace NeuralNet diff --git a/src/NeuralNet/activations/Relu.hpp b/src/NeuralNet/activations/Relu.hpp index 443ab9f..41712a3 100644 --- a/src/NeuralNet/activations/Relu.hpp +++ b/src/NeuralNet/activations/Relu.hpp @@ -2,30 +2,20 @@ #include "Activation.hpp" -namespace NeuralNet -{ - class Relu : public Activation - { - public: - static Eigen::MatrixXd activate(const Eigen::MatrixXd &z) - { - return z.unaryExpr(&Relu::activateValue); - } +namespace NeuralNet { +class Relu : public Activation { + public: + static Eigen::MatrixXd activate(const Eigen::MatrixXd &z) { + return z.unaryExpr(&Relu::activateValue); + } - static Eigen::MatrixXd diff(const Eigen::MatrixXd &a) - { - return a.unaryExpr(&Relu::diffValue); - } + static Eigen::MatrixXd diff(const Eigen::MatrixXd &a) { + return a.unaryExpr(&Relu::diffValue); + } - private: - static double diffValue(double a) - { - return a > 0 ? 1 : 0; - } + private: + static double diffValue(double a) { return a > 0 ? 1 : 0; } - static double activateValue(double z) - { - return z < 0 ? 0 : z; - } - }; -} \ No newline at end of file + static double activateValue(double z) { return z < 0 ? 0 : z; } +}; +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/activations/Sigmoid.hpp b/src/NeuralNet/activations/Sigmoid.hpp index 9111f7d..38d55f2 100644 --- a/src/NeuralNet/activations/Sigmoid.hpp +++ b/src/NeuralNet/activations/Sigmoid.hpp @@ -2,20 +2,16 @@ #include "Activation.hpp" -namespace NeuralNet -{ - class Sigmoid : public Activation - { - public: - static Eigen::MatrixXd activate(const Eigen::MatrixXd &z) - { - Eigen::MatrixXd negZ = -z; - return 1 / (1 + negZ.array().exp()); - }; +namespace NeuralNet { +class Sigmoid : public Activation { + public: + static Eigen::MatrixXd activate(const Eigen::MatrixXd &z) { + Eigen::MatrixXd negZ = -z; + return 1 / (1 + negZ.array().exp()); + }; - static Eigen::MatrixXd diff(const Eigen::MatrixXd &a) - { - return a.array() * (1.0 - a.array()); - }; + static Eigen::MatrixXd diff(const Eigen::MatrixXd &a) { + return a.array() * (1.0 - a.array()); }; -} +}; +} // namespace NeuralNet diff --git a/src/NeuralNet/activations/Softmax.hpp b/src/NeuralNet/activations/Softmax.hpp index 71bf78c..a2f85c2 100644 --- a/src/NeuralNet/activations/Softmax.hpp +++ b/src/NeuralNet/activations/Softmax.hpp @@ -1,31 +1,27 @@ #pragma once #include -#include "Activation.hpp" -namespace NeuralNet -{ - class Softmax : public Activation - { - public: - static Eigen::MatrixXd activate(const Eigen::MatrixXd &z) - { - Eigen::MatrixXd exp = z.array().exp(); +#include "Activation.hpp" - Eigen::MatrixXd sumExp = exp.rowwise().sum().replicate(1, exp.cols()); +namespace NeuralNet { +class Softmax : public Activation { + public: + static Eigen::MatrixXd activate(const Eigen::MatrixXd &z) { + Eigen::MatrixXd exp = z.array().exp(); - return exp.array() / sumExp.array(); - }; + Eigen::MatrixXd sumExp = exp.rowwise().sum().replicate(1, exp.cols()); - static Eigen::MatrixXd diff(const Eigen::MatrixXd &a) - { - return Eigen::MatrixXd::Constant(a.rows(), a.cols(), 1); - }; + return exp.array() / sumExp.array(); + }; - private: - static Eigen::MatrixXd scale(const Eigen::MatrixXd &z, double scaleFactor) - { - return z * scaleFactor; - } + static Eigen::MatrixXd diff(const Eigen::MatrixXd &a) { + return Eigen::MatrixXd::Constant(a.rows(), a.cols(), 1); }; -} + + private: + static Eigen::MatrixXd scale(const Eigen::MatrixXd &z, double scaleFactor) { + return z * scaleFactor; + } +}; +} // namespace NeuralNet diff --git a/src/NeuralNet/activations/activations.hpp b/src/NeuralNet/activations/activations.hpp index 1d33c7d..9de6c8c 100644 --- a/src/NeuralNet/activations/activations.hpp +++ b/src/NeuralNet/activations/activations.hpp @@ -1,5 +1,6 @@ /** - * The purpose of this file is solely to group the includes of the files in this folder. + * The purpose of this file is solely to group the includes of the files in this + * folder. */ #pragma once diff --git a/src/NeuralNet/callbacks/CSVLogger.hpp b/src/NeuralNet/callbacks/CSVLogger.hpp index d380c86..01079f6 100644 --- a/src/NeuralNet/callbacks/CSVLogger.hpp +++ b/src/NeuralNet/callbacks/CSVLogger.hpp @@ -1,131 +1,122 @@ #pragma once +#include #include #include -#include + #include "Callback.hpp" -#include "utils/Functions.hpp" // fileExistsWithExtension - -namespace NeuralNet -{ - class CSVLogger : public Callback - { - public: - /** - * @brief CSVLogger is a `Callback` that streams epoch results to a csv file - * - * @param filename The name of the csv file - * @param separator The separator used in the csv file (default: ",") - */ - CSVLogger(const std::string &filename, const std::string &separator = ",") - { - assert(fileHasExtension(filename, ".csv") && "Filename must have .csv extension"); - this->filename = filename; - this->separator = separator; - }; +#include "utils/Functions.hpp" // fileExistsWithExtension + +namespace NeuralNet { +class CSVLogger : public Callback { + public: + /** + * @brief CSVLogger is a `Callback` that streams epoch results to a csv file + * + * @param filename The name of the csv file + * @param separator The separator used in the csv file (default: ",") + */ + CSVLogger(const std::string &filename, const std::string &separator = ",") { + assert(fileHasExtension(filename, ".csv") && + "Filename must have .csv extension"); + this->filename = filename; + this->separator = separator; + }; - void onEpochBegin(Logs logs) override{}; + void onEpochBegin(Logs logs) override{}; - /** - * @brief This method will be called at the end of each epoch - * - * In the case of CSVLogger, it will append the logs of the current epoch to data which `onTrainEnd` will be written to the file. - * - * @param logs The logs of the current epoch - */ - void onEpochEnd(Logs logs) override - { - std::vector row; + /** + * @brief This method will be called at the end of each epoch + * + * In the case of CSVLogger, it will append the logs of the current epoch to + * data which `onTrainEnd` will be written to the file. + * + * @param logs The logs of the current epoch + */ + void onEpochEnd(Logs logs) override { + std::vector row; - row.reserve(logs.size()); + row.reserve(logs.size()); - std::transform(logs.begin(), logs.end(), std::back_inserter(row), - [](const auto &log) - { return static_cast(log.second); }); + std::transform( + logs.begin(), logs.end(), std::back_inserter(row), + [](const auto &log) { return static_cast(log.second); }); - data.push_back(row); - }; + data.push_back(row); + }; - /** - * @brief This method will be called at the beginning of the training. - * - * It will initialize the headers with the logs keys. - * - * @param logs The logs of the current epoch - */ - void onTrainBegin(Logs logs) override - { - // Initializing the headers with the logs keys - for (const auto &log : logs) - { - headers.push_back(log.first); - }; + /** + * @brief This method will be called at the beginning of the training. + * + * It will initialize the headers with the logs keys. + * + * @param logs The logs of the current epoch + */ + void onTrainBegin(Logs logs) override { + // Initializing the headers with the logs keys + for (const auto &log : logs) { + headers.push_back(log.first); }; + }; - /** - * @brief This method will be called at the end of the training. - * - * It will write the data in the given csv file. - * - * @param logs The logs of the current epoch - */ - void onTrainEnd(Logs logs) override - { - std::ofstream file(filename); + /** + * @brief This method will be called at the end of the training. + * + * It will write the data in the given csv file. + * + * @param logs The logs of the current epoch + */ + void onTrainEnd(Logs logs) override { + std::ofstream file(filename); - if (!file.is_open()) - { - throw std::runtime_error("Couldn't open csv file"); - } + if (!file.is_open()) { + throw std::runtime_error("Couldn't open csv file"); + } - file << formatRow(headers); + file << formatRow(headers); - std::for_each(data.begin(), data.end(), [&file, this](auto &row) - { file << this->formatRow(row); }); + std::for_each(data.begin(), data.end(), + [&file, this](auto &row) { file << this->formatRow(row); }); - file.close(); - }; + file.close(); + }; - void onBatchBegin(Logs logs) override{}; - void onBatchEnd(Logs logs) override{}; - - private: - std::string filename; - std::string separator; - std::vector headers; - std::vector> data; - - /** - * @brief This method will format a row of the csv file - * - * @tparam T The type of the elements in the row - * @param v The row to format - * - * @return The row in a csv format - */ - template - std::string formatRow(const std::vector &v) - { - std::string csvRow; - - for (T el : v) - { - csvRow += std::to_string(el) + separator; - } - - return csvRow + "\n"; - }; + void onBatchBegin(Logs logs) override{}; + void onBatchEnd(Logs logs) override{}; + + private: + std::string filename; + std::string separator; + std::vector headers; + std::vector> data; + + /** + * @brief This method will format a row of the csv file + * + * @tparam T The type of the elements in the row + * @param v The row to format + * + * @return The row in a csv format + */ + template + std::string formatRow(const std::vector &v) { + std::string csvRow; + + for (T el : v) { + csvRow += std::to_string(el) + separator; + } + + return csvRow + "\n"; + }; - std::string formatRow(const std::vector &v) - { - std::string csvRow; + std::string formatRow(const std::vector &v) { + std::string csvRow; - for (const std::string &el : v) - { - csvRow += el + separator; - } + for (const std::string &el : v) { + csvRow += el + separator; + } - return csvRow + "\n"; - }; + return csvRow + "\n"; }; -} // namespace NeuralNet +}; +} // namespace NeuralNet diff --git a/src/NeuralNet/callbacks/Callback.hpp b/src/NeuralNet/callbacks/Callback.hpp index 6545e06..a49a7a7 100644 --- a/src/NeuralNet/callbacks/Callback.hpp +++ b/src/NeuralNet/callbacks/Callback.hpp @@ -1,69 +1,76 @@ #pragma once -#include -#include +#include +#include #include #include -#include #include -#include +#include +#include -namespace NeuralNet -{ - using Logs = std::unordered_map; +namespace NeuralNet { +using Logs = std::unordered_map; - class Callback - { - public: - virtual void onTrainBegin(Logs logs) = 0; - virtual void onTrainEnd(Logs logs) = 0; - virtual void onEpochBegin(Logs logs) = 0; - virtual void onEpochEnd(Logs logs) = 0; - virtual void onBatchBegin(Logs logs) = 0; - virtual void onBatchEnd(Logs logs) = 0; +class Callback { + public: + virtual void onTrainBegin(Logs logs) = 0; + virtual void onTrainEnd(Logs logs) = 0; + virtual void onEpochBegin(Logs logs) = 0; + virtual void onEpochEnd(Logs logs) = 0; + virtual void onBatchBegin(Logs logs) = 0; + virtual void onBatchEnd(Logs logs) = 0; - virtual ~Callback() = default; + virtual ~Callback() = default; - /** - * @brief Calls the method of the callback with the given logs - * - * @tparam T The type of the callback - * @param callback A shared_ptr to the callback - * @param methodName The name of the method to call (onTrainBegin, onTrainEnd, onEpochBegin, onEpochEnd, onBatchBegin, onBatchEnd) - * @param logs The logs to pass to the method - * - * @warning There should be consistency between the names of the logs and the metrics of the callbacks - */ - template - static void callMethod(std::shared_ptr callback, const std::string &methodName, Logs logs) - { - static const std::unordered_map> methods = { - {"onTrainBegin", [](T *callback, Logs logs) - { return callback->onTrainBegin(logs); }}, - {"onTrainEnd", [](T *callback, Logs logs) - { return callback->onTrainEnd(logs); }}, - {"onEpochBegin", [](T *callback, Logs logs) - { return callback->onEpochBegin(logs); }}, - {"onEpochEnd", [](T *callback, Logs logs) - { return callback->onEpochEnd(logs); }}, - {"onBatchBegin", [](T *callback, Logs logs) - { return callback->onBatchBegin(logs); }}, - {"onBatchEnd", [](T *callback, Logs logs) - { return callback->onBatchEnd(logs); }}}; + /** + * @brief Calls the method of the callback with the given logs + * + * @tparam T The type of the callback + * @param callback A shared_ptr to the callback + * @param methodName The name of the method to call (onTrainBegin, onTrainEnd, + * onEpochBegin, onEpochEnd, onBatchBegin, onBatchEnd) + * @param logs The logs to pass to the method + * + * @warning There should be consistency between the names of the logs and the + * metrics of the callbacks + */ + template + static void callMethod(std::shared_ptr callback, + const std::string &methodName, Logs logs) { + static const std::unordered_map> + methods = { + {"onTrainBegin", + [](T *callback, Logs logs) { + return callback->onTrainBegin(logs); + }}, + {"onTrainEnd", + [](T *callback, Logs logs) { return callback->onTrainEnd(logs); }}, + {"onEpochBegin", + [](T *callback, Logs logs) { + return callback->onEpochBegin(logs); + }}, + {"onEpochEnd", + [](T *callback, Logs logs) { return callback->onEpochEnd(logs); }}, + {"onBatchBegin", + [](T *callback, Logs logs) { + return callback->onBatchBegin(logs); + }}, + {"onBatchEnd", [](T *callback, Logs logs) { + return callback->onBatchEnd(logs); + }}}; - auto it = methods.find(methodName); + auto it = methods.find(methodName); - if (it == methods.end()) - return; + if (it == methods.end()) return; - it->second(callback.get(), logs); - } + it->second(callback.get(), logs); + } - protected: - static void checkMetric(const std::string &metric, const std::vector &metrics) - { - if (std::find(metrics.begin(), metrics.end(), metric) == metrics.end()) - throw std::invalid_argument("Metric not found"); - }; + protected: + static void checkMetric(const std::string &metric, + const std::vector &metrics) { + if (std::find(metrics.begin(), metrics.end(), metric) == metrics.end()) + throw std::invalid_argument("Metric not found"); }; -} // namespace NeuralNet \ No newline at end of file +}; +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/callbacks/EarlyStopping.hpp b/src/NeuralNet/callbacks/EarlyStopping.hpp index c63bc9f..77ad428 100644 --- a/src/NeuralNet/callbacks/EarlyStopping.hpp +++ b/src/NeuralNet/callbacks/EarlyStopping.hpp @@ -1,78 +1,80 @@ #pragma once +#include #include #include -#include + #include "Callback.hpp" #include "utils/Functions.hpp" -namespace NeuralNet -{ - class EarlyStopping : public Callback - { - public: - /** - * @brief EarlyStopping is a `Callback` that stops training when a monitored metric has stopped improving. - * - * @param metric The metric to monitor (default: `LOSS`) - * @param minDelta Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than minDelta, will count as no improvement. (default: 0) - * @param patience Number of epochs with no improvement after which training will be stopped. (default: 0) - */ - EarlyStopping(const std::string &metric = "LOSS", double minDelta = 0, int patience = 0) - { - checkMetric(metric, metrics); - this->metric = metric; - this->minDelta = minDelta; - this->patience = patience; - }; +namespace NeuralNet { +class EarlyStopping : public Callback { + public: + /** + * @brief EarlyStopping is a `Callback` that stops training when a monitored + * metric has stopped improving. + * + * @param metric The metric to monitor (default: `LOSS`) + * @param minDelta Minimum change in the monitored quantity to qualify as an + * improvement, i.e. an absolute change of less than minDelta, will count as + * no improvement. (default: 0) + * @param patience Number of epochs with no improvement after which training + * will be stopped. (default: 0) + */ + EarlyStopping(const std::string &metric = "LOSS", double minDelta = 0, + int patience = 0) { + checkMetric(metric, metrics); + this->metric = metric; + this->minDelta = minDelta; + this->patience = patience; + }; - void onEpochBegin(Logs logs) override{}; + void onEpochBegin(Logs logs) override{}; - /** - * @brief This method will be called at the end of each epoch - * - * @param epoch The current epoch - * @param logs The logs of the current epoch - * @return Returns true if the training should continue otherwise returns false - * - * @warning The order of the logs should be the same as the order of the metrics. - */ - void onEpochEnd(Logs logs) override - { - auto it = logs.find(metric); + /** + * @brief This method will be called at the end of each epoch + * + * @param epoch The current epoch + * @param logs The logs of the current epoch + * @return Returns true if the training should continue otherwise returns + * false + * + * @warning The order of the logs should be the same as the order of the + * metrics. + */ + void onEpochEnd(Logs logs) override { + auto it = logs.find(metric); - if (it == logs.end()) - throw std::invalid_argument("Metric not found"); + if (it == logs.end()) throw std::invalid_argument("Metric not found"); - double currentMetric = it->second; + double currentMetric = it->second; - if (previousMetric == 0) - { - previousMetric = currentMetric; - return; - } + if (previousMetric == 0) { + previousMetric = currentMetric; + return; + } - double absCurrentDelta = std::abs(currentMetric - previousMetric); + double absCurrentDelta = std::abs(currentMetric - previousMetric); - patience = absCurrentDelta <= minDelta ? patience - 1 : patience; - previousMetric = currentMetric; + patience = absCurrentDelta <= minDelta ? patience - 1 : patience; + previousMetric = currentMetric; - if (patience < 0) - throw std::runtime_error("Early stopping"); - }; + if (patience < 0) throw std::runtime_error("Early stopping"); + }; - void onTrainBegin(Logs logs) override{}; - void onTrainEnd(Logs logs) override{}; - void onBatchBegin(Logs logs) override{}; - void onBatchEnd(Logs logs) override{}; + void onTrainBegin(Logs logs) override{}; + void onTrainEnd(Logs logs) override{}; + void onBatchBegin(Logs logs) override{}; + void onBatchEnd(Logs logs) override{}; - ~EarlyStopping() override = default; + ~EarlyStopping() override = default; - private: - std::string metric; - double minDelta, previousMetric = 0; - int patience; - std::vector metrics = {"LOSS", "ACCURACY"}; // Available metrics for this Callback - }; + private: + std::string metric; + double minDelta, previousMetric = 0; + int patience; + std::vector metrics = { + "LOSS", "ACCURACY"}; // Available metrics for this Callback +}; -} // namespace NeuralNet \ No newline at end of file +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/data/Tensor.hpp b/src/NeuralNet/data/Tensor.hpp index 5c26414..ef08186 100644 --- a/src/NeuralNet/data/Tensor.hpp +++ b/src/NeuralNet/data/Tensor.hpp @@ -2,58 +2,47 @@ #include -namespace NeuralNet -{ - template - class Tensor - { - public: - Tensor(T data) : data(data) {} - - T data; - std::vector batches; - bool batched = false; - - /** - * @brief This method will separate the data into batches of the specified size - * - * @param batchSize The number of elements in each batch - */ - void batch(int batchSize) - { - assert(data.size() > 0 && batchSize < data.size()); - - int numBatches = (data.size() + batchSize - 1) / batchSize; - - batches.reserve(numBatches); - - batched = true; - - for (int i = 0; i < numBatches; ++i) - { - int startIdx = i * batchSize; - int endIdx = std::min((i + 1) * batchSize, static_cast(data.size())); - - batches.emplace_back(std::make_move_iterator(data.begin() + startIdx), std::make_move_iterator(data.begin() + endIdx)); - } - - // Remove the the mess from data - data.erase(data.begin(), data.end()); - }; - - size_t size() const - { - return batched ? batches.size() : data.size(); - }; - - std::vector getBatchedData() const - { - return batches; - } +namespace NeuralNet { +template +class Tensor { + public: + Tensor(T data) : data(data) {} + + T data; + std::vector batches; + bool batched = false; + + /** + * @brief This method will separate the data into batches of the specified + * size + * + * @param batchSize The number of elements in each batch + */ + void batch(int batchSize) { + assert(data.size() > 0 && batchSize < data.size()); + + int numBatches = (data.size() + batchSize - 1) / batchSize; + + batches.reserve(numBatches); + + batched = true; - std::vector getData() const - { - return data; + for (int i = 0; i < numBatches; ++i) { + int startIdx = i * batchSize; + int endIdx = std::min((i + 1) * batchSize, static_cast(data.size())); + + batches.emplace_back(std::make_move_iterator(data.begin() + startIdx), + std::make_move_iterator(data.begin() + endIdx)); } + + // Remove the the mess from data + data.erase(data.begin(), data.end()); }; -} \ No newline at end of file + + size_t size() const { return batched ? batches.size() : data.size(); }; + + std::vector getBatchedData() const { return batches; } + + std::vector getData() const { return data; } +}; +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/data/TrainingData.hpp b/src/NeuralNet/data/TrainingData.hpp index 24131f1..79601e1 100644 --- a/src/NeuralNet/data/TrainingData.hpp +++ b/src/NeuralNet/data/TrainingData.hpp @@ -2,44 +2,43 @@ #include "Tensor.hpp" -namespace NeuralNet -{ - template - class TrainingData - { - friend class Network; +namespace NeuralNet { +template +class TrainingData { + friend class Network; - public: - /** - * @brief Construct a new Training Data object. - * This object is used to store the inputs and labels data, - * it comes with a set of methods to manipulate the data to your liking for better training optimization. - * - * @param inputs_data The inputs data - * @param labels_data The labels data - * - * @note The inputs and labels data must have the same size - */ - TrainingData(I inputs_data, L labels_data) : inputs(inputs_data), labels(labels_data) - { - assert(inputs_data.size() == labels_data.size()); - } + public: + /** + * @brief Construct a new Training Data object. + * This object is used to store the inputs and labels data, + * it comes with a set of methods to manipulate the data to your liking for + * better training optimization. + * + * @param inputs_data The inputs data + * @param labels_data The labels data + * + * @note The inputs and labels data must have the same size + */ + TrainingData(I inputs_data, L labels_data) + : inputs(inputs_data), labels(labels_data) { + assert(inputs_data.size() == labels_data.size()); + } - /** - * @brief This method will separate the inputs and labels data into batches of the specified size - * - * @param batchSize The number of elements in each batch - */ - void batch(int batchSize) - { - inputs.batch(batchSize); - labels.batch(batchSize); - batched = true; - }; - - private: - Tensor inputs; - Tensor labels; - bool batched = false; + /** + * @brief This method will separate the inputs and labels data into batches of + * the specified size + * + * @param batchSize The number of elements in each batch + */ + void batch(int batchSize) { + inputs.batch(batchSize); + labels.batch(batchSize); + batched = true; }; -} \ No newline at end of file + + private: + Tensor inputs; + Tensor labels; + bool batched = false; +}; +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/layers/Dense.hpp b/src/NeuralNet/layers/Dense.hpp index 5975acc..2b71c87 100644 --- a/src/NeuralNet/layers/Dense.hpp +++ b/src/NeuralNet/layers/Dense.hpp @@ -1,30 +1,29 @@ #pragma once +#include #include -#include -#include #include -#include +#include + #include "Layer.hpp" -namespace NeuralNet -{ - class Dense : public Layer - { - public: - Dense(int nNeurons, ACTIVATION activation = ACTIVATION::SIGMOID, WEIGHT_INIT weightInit = WEIGHT_INIT::RANDOM, int bias = 0) : Layer(nNeurons, activation, weightInit, bias) - { - type = LayerType::DENSE; - } +namespace NeuralNet { +class Dense : public Layer { + public: + Dense(int nNeurons, ACTIVATION activation = ACTIVATION::SIGMOID, + WEIGHT_INIT weightInit = WEIGHT_INIT::RANDOM, int bias = 0) + : Layer(nNeurons, activation, weightInit, bias) { + type = LayerType::DENSE; + } - ~Dense(){}; + ~Dense(){}; - private: - // non-public serialization - friend class cereal::access; - Dense(){}; // Required for serialization - }; -} + private: + // non-public serialization + friend class cereal::access; + Dense(){}; // Required for serialization +}; +} // namespace NeuralNet CEREAL_REGISTER_TYPE(NeuralNet::Dense); diff --git a/src/NeuralNet/layers/Flatten.hpp b/src/NeuralNet/layers/Flatten.hpp index 640af2d..66e9e4c 100644 --- a/src/NeuralNet/layers/Flatten.hpp +++ b/src/NeuralNet/layers/Flatten.hpp @@ -1,88 +1,86 @@ #pragma once -#include -#include -#include -#include -#include #include +#include +#include +#include #include +#include +#include + #include "Layer.hpp" #include "utils/Functions.hpp" #include "utils/Serialize.hpp" -namespace NeuralNet -{ - class Flatten : public Layer - { - public: - /** - * @brief Construct a new Flatten object. - * - * This type of Layer is perfect as an input layer since its sole purpose is to flatten a given input - * - * @param inputShape The shape of the input to be flattened - */ - Flatten(std::tuple inputShape) : Layer(inputShape), inputShape(inputShape) - { - type = LayerType::FLATTEN; - }; +namespace NeuralNet { +class Flatten : public Layer { + public: + /** + * @brief Construct a new Flatten object. + * + * This type of Layer is perfect as an input layer since its sole purpose is + * to flatten a given input + * + * @param inputShape The shape of the input to be flattened + */ + Flatten(std::tuple inputShape) + : Layer(inputShape), inputShape(inputShape) { + type = LayerType::FLATTEN; + }; - /** - * @brief This method flattens a 3D vector into a 2D Eigen::MatrixXd - * - * @param inputs The 3D vector to be flattened - * @return Eigen::MatrixXd The flattened 2D Eigen::MatrixXd - */ - Eigen::MatrixXd flatten(std::vector>> inputs) - { - int rows = std::get<0>(inputShape); - int cols = std::get<1>(inputShape); + /** + * @brief This method flattens a 3D vector into a 2D Eigen::MatrixXd + * + * @param inputs The 3D vector to be flattened + * @return Eigen::MatrixXd The flattened 2D Eigen::MatrixXd + */ + Eigen::MatrixXd flatten( + std::vector>> inputs) { + int rows = std::get<0>(inputShape); + int cols = std::get<1>(inputShape); - // Flatten the vectors - std::vector flatInputs; - for (const std::vector> &input : inputs) - { - std::vector flattenedInput = flatten2DVector(input, rows, cols); - flatInputs.insert(flatInputs.end(), flattenedInput.begin(), flattenedInput.end()); - } + // Flatten the vectors + std::vector flatInputs; + for (const std::vector> &input : inputs) { + std::vector flattenedInput = flatten2DVector(input, rows, cols); + flatInputs.insert(flatInputs.end(), flattenedInput.begin(), + flattenedInput.end()); + } - const int numRows = inputs.size(); - const int numCols = rows * cols; - return Eigen::Map>(flatInputs.data(), numRows, numCols); - }; + const int numRows = inputs.size(); + const int numCols = rows * cols; + return Eigen::Map< + Eigen::Matrix>( + flatInputs.data(), numRows, numCols); + }; - ~Flatten(){}; + ~Flatten(){}; - private: - // non-public serialization - friend class cereal::access; + private: + // non-public serialization + friend class cereal::access; - Flatten(){}; // Necessary for serialization + Flatten(){}; // Necessary for serialization - std::tuple inputShape; + std::tuple inputShape; - template - void serialize(Archive &ar) - { - ar(cereal::base_class(this), inputShape); - } + template + void serialize(Archive &ar) { + ar(cereal::base_class(this), inputShape); + } - void feedInputs(std::vector>> inputs) - { - Eigen::MatrixXd flattenedInputs = this->flatten(inputs); - this->setOutputs(flattenedInputs); - } - }; -} + void feedInputs(std::vector>> inputs) { + Eigen::MatrixXd flattenedInputs = this->flatten(inputs); + this->setOutputs(flattenedInputs); + } +}; +} // namespace NeuralNet -namespace cereal -{ - template - struct specialize - { - }; -} +namespace cereal { +template +struct specialize {}; +} // namespace cereal CEREAL_REGISTER_TYPE(NeuralNet::Flatten); diff --git a/src/NeuralNet/layers/Layer.hpp b/src/NeuralNet/layers/Layer.hpp index 974495f..b353aa4 100644 --- a/src/NeuralNet/layers/Layer.hpp +++ b/src/NeuralNet/layers/Layer.hpp @@ -1,142 +1,120 @@ #pragma once -#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include + +#include "Network.hpp" +#include "activations/activations.hpp" +#include "utils/Enums.hpp" #include "utils/Functions.hpp" #include "utils/Serialize.hpp" -#include "utils/Enums.hpp" -#include "activations/activations.hpp" -#include "Network.hpp" -namespace NeuralNet -{ - // Should be updated when a new layer type is added - enum class LayerType - { - DEFAULT, - DENSE, - FLATTEN +namespace NeuralNet { +// Should be updated when a new layer type is added +enum class LayerType { DEFAULT, DENSE, FLATTEN }; + +class Layer { + friend class Network; + + public: + Layer(int nNeurons, ACTIVATION activation = ACTIVATION::SIGMOID, + WEIGHT_INIT weightInit = WEIGHT_INIT::RANDOM, int bias = 0) { + this->bias = bias; + this->nNeurons = nNeurons; + this->weightInit = weightInit; + this->activation = activation; + this->setActivation(activation); }; - class Layer - { - friend class Network; - - public: - Layer(int nNeurons, ACTIVATION activation = ACTIVATION::SIGMOID, WEIGHT_INIT weightInit = WEIGHT_INIT::RANDOM, int bias = 0) - { - this->bias = bias; - this->nNeurons = nNeurons; - this->weightInit = weightInit; - this->activation = activation; - this->setActivation(activation); - }; - - /** - * @brief This method get the number of neurons actually in the layer - * - * @return The number of neurons in the layer - */ - int getNumNeurons() const - { - return nNeurons; - }; - - /** - * @brief This method gets the layer's weights - * - * @return an Eigen::MatrixXd representing the weights - */ - Eigen::MatrixXd getWeights() const - { - return weights; - } + /** + * @brief This method get the number of neurons actually in the layer + * + * @return The number of neurons in the layer + */ + int getNumNeurons() const { return nNeurons; }; + + /** + * @brief This method gets the layer's weights + * + * @return an Eigen::MatrixXd representing the weights + */ + Eigen::MatrixXd getWeights() const { return weights; } + + /** + * @brief Return the biases of the layer + * + * @return an Eigen::Matrix representing the biases + */ + Eigen::MatrixXd getBiases() const { return biases; } + + /** + * @brief This method get the layer's outputs + * + * @return an Eigen::Eigen::MatrixXd representing the layer's outputs + */ + Eigen::MatrixXd getOutputs() const { return outputs; }; + + /** + * @brief Method to print layer's weights + */ + void printWeights() { + std::cout << this->weights << "\n"; + return; + }; - /** - * @brief Return the biases of the layer - * - * @return an Eigen::Matrix representing the biases - */ - Eigen::MatrixXd getBiases() const - { - return biases; - } + /** + * @brief Method to print layer's outputs + */ + void printOutputs() { + std::cout << this->outputs << "\n"; + return; + }; - /** - * @brief This method get the layer's outputs - * - * @return an Eigen::Eigen::MatrixXd representing the layer's outputs - */ - Eigen::MatrixXd getOutputs() const - { - return outputs; - }; - - /** - * @brief Method to print layer's weights - */ - void printWeights() - { - std::cout << this->weights << "\n"; - return; - }; - - /** - * @brief Method to print layer's outputs - */ - void printOutputs() - { - std::cout << this->outputs << "\n"; + ~Layer(){}; + + private: + // non-public serialization + friend class cereal::access; + + double bias; + Eigen::MatrixXd biases; + WEIGHT_INIT weightInit; + Eigen::MatrixXd outputs; + Eigen::MatrixXd weights; + Eigen::MatrixXd cachedWeights; + Eigen::MatrixXd cachedBiases; + ACTIVATION activation; + Eigen::MatrixXd (*activate)(const Eigen::MatrixXd &); + Eigen::MatrixXd (*diff)(const Eigen::MatrixXd &); + + /** + * This function will be used to properly initialize the Layer + * It's being done like this because of the number of neurons of the previous + * layer that's unkown prior + */ + void init(int numRows) { + // First and foremost init the biases and the outputs + double mean = 0, stddev = 0; + this->weights = Eigen::MatrixXd::Zero(numRows, nNeurons); + + // This is going to be used for testing + if (this->weightInit == WEIGHT_INIT::CONSTANT) { + this->weights = Eigen::MatrixXd::Constant(numRows, nNeurons, 1); return; - }; - - ~Layer(){}; - - private: - // non-public serialization - friend class cereal::access; - - double bias; - Eigen::MatrixXd biases; - WEIGHT_INIT weightInit; - Eigen::MatrixXd outputs; - Eigen::MatrixXd weights; - Eigen::MatrixXd cachedWeights; - Eigen::MatrixXd cachedBiases; - ACTIVATION activation; - Eigen::MatrixXd (*activate)(const Eigen::MatrixXd &); - Eigen::MatrixXd (*diff)(const Eigen::MatrixXd &); - - /** - * This function will be used to properly initialize the Layer - * It's being done like this because of the number of neurons of the previous layer that's unkown prior - */ - void init(int numRows) - { - // First and foremost init the biases and the outputs - double mean = 0, stddev = 0; - this->weights = Eigen::MatrixXd::Zero(numRows, nNeurons); - - // This is going to be used for testing - if (this->weightInit == WEIGHT_INIT::CONSTANT) - { - this->weights = Eigen::MatrixXd::Constant(numRows, nNeurons, 1); - return; - } - - // calculate mean and stddev based on init algo - switch (this->weightInit) - { + } + + // calculate mean and stddev based on init algo + switch (this->weightInit) { case WEIGHT_INIT::GLOROT: // sqrt(fan_avg) stddev = sqrt(static_cast(2) / (numRows + nNeurons)); @@ -151,101 +129,99 @@ namespace NeuralNet break; default: break; - } + } + + // Init the weights + this->weightInit == WEIGHT_INIT::RANDOM + ? randomWeightInit(&(this->weights), -1, 1) + : randomDistMatrixInit(&(this->weights), mean, stddev); + } + + /** + * @brief This method is used to feed the inputs to the layer + * + * @param inputs A vector of doubles representing the inputs (features) + * + * @return an Eigen::MatrixXd representing the outputs of the layer + */ + virtual Eigen::MatrixXd feedInputs(std::vector inputs) { + return this->feedInputs(Eigen::MatrixXd::Map(&inputs[0], inputs.size(), 1)); + }; + + /** + * @brief This method is used to feed the inputs to the layer + * + * @param inputs An Eigen::MatrixXd representing the inputs (features) + * + * @return an Eigen::MatrixXd representing the outputs of the layer + */ + virtual Eigen::MatrixXd feedInputs(Eigen::MatrixXd inputs) { + // Layer is "input" layer + if (weights.rows() == 0) { + this->setOutputs(inputs); + return inputs; + } + + inputs = inputs.cols() == weights.rows() ? inputs : inputs.transpose(); + + assert(inputs.cols() == weights.rows()); + return this->computeOutputs(inputs); + }; + + /** + * @brief This method is used to feed the inputs to the layer + * + * @param inputs A vector of vectors of doubles representing the inputs + * (features) + * + * @return void + */ + virtual void feedInputs( + std::vector>> inputs) { + assert(false && + "Cannot feed 3d vectors, a Flatten layer could do it though"); + return; + }; - // Init the weights - this->weightInit == WEIGHT_INIT::RANDOM ? randomWeightInit(&(this->weights), -1, 1) : randomDistMatrixInit(&(this->weights), mean, stddev); + /** + * @brief This method is used to feed the inputs to the layer + * + * @param inputs A vector of vectors of doubles representing the inputs + * (features) + * + * @return an Eigen::MatrixXd representing the computed outputs based on the + * layer's parameters + */ + Eigen::MatrixXd computeOutputs(Eigen::MatrixXd inputs) { + // Initialize the biases based on the input's size + if (biases.rows() == 0 && biases.cols() == 0) { + biases = Eigen::MatrixXd::Constant(1, nNeurons, bias); } - /** - * @brief This method is used to feed the inputs to the layer - * - * @param inputs A vector of doubles representing the inputs (features) - * - * @return an Eigen::MatrixXd representing the outputs of the layer - */ - virtual Eigen::MatrixXd feedInputs(std::vector inputs) - { - return this->feedInputs(Eigen::MatrixXd::Map(&inputs[0], inputs.size(), 1)); - }; - - /** - * @brief This method is used to feed the inputs to the layer - * - * @param inputs An Eigen::MatrixXd representing the inputs (features) - * - * @return an Eigen::MatrixXd representing the outputs of the layer - */ - virtual Eigen::MatrixXd feedInputs(Eigen::MatrixXd inputs) - { - // Layer is "input" layer - if (weights.rows() == 0) - { - this->setOutputs(inputs); - return inputs; - } - - inputs = inputs.cols() == weights.rows() ? inputs : inputs.transpose(); - - assert(inputs.cols() == weights.rows()); - return this->computeOutputs(inputs); - }; - - /** - * @brief This method is used to feed the inputs to the layer - * - * @param inputs A vector of vectors of doubles representing the inputs (features) - * - * @return void - */ - virtual void feedInputs(std::vector>> inputs) - { - assert(false && "Cannot feed 3d vectors, a Flatten layer could do it though"); + // Weighted sum + Eigen::MatrixXd wSum = inputs * weights; + + wSum.rowwise() += biases.row(0); + + outputs = activate(wSum); + return outputs; + }; + + /** + * @brief This method is used to set the activation function of the layer + * + * @param activation The activation function to be used + * + * @return void + */ + void setActivation(ACTIVATION activation) { + if (type == LayerType::FLATTEN) { + this->activate = Activation::activate; + this->diff = Activation::diff; return; - }; - - /** - * @brief This method is used to feed the inputs to the layer - * - * @param inputs A vector of vectors of doubles representing the inputs (features) - * - * @return an Eigen::MatrixXd representing the computed outputs based on the layer's parameters - */ - Eigen::MatrixXd computeOutputs(Eigen::MatrixXd inputs) - { - // Initialize the biases based on the input's size - if (biases.rows() == 0 && biases.cols() == 0) - { - biases = Eigen::MatrixXd::Constant(1, nNeurons, bias); - } - - // Weighted sum - Eigen::MatrixXd wSum = inputs * weights; - - wSum.rowwise() += biases.row(0); - - outputs = activate(wSum); - return outputs; - }; - - /** - * @brief This method is used to set the activation function of the layer - * - * @param activation The activation function to be used - * - * @return void - */ - void setActivation(ACTIVATION activation) - { - if (type == LayerType::FLATTEN) - { - this->activate = Activation::activate; - this->diff = Activation::diff; - return; - } - - switch (activation) - { + } + + switch (activation) { case ACTIVATION::SIGMOID: this->activate = Sigmoid::activate; this->diff = Sigmoid::diff; @@ -263,51 +239,52 @@ namespace NeuralNet */ default: assert(false && "Activation not defined"); - } - - return; - }; - - // Necessary function for serializing Layer - template - void save(Archive &archive) const - { - archive(nNeurons, weights, outputs, biases, activation, type); - }; - - template - void load(Archive &archive) - { - archive(nNeurons, weights, outputs, biases, activation, type); - setActivation(activation); } - protected: - Layer(){}; // Necessary for serialization - Layer(std::tuple inputShape) : nNeurons(std::get<0>(inputShape) * std::get<1>(inputShape)){}; // Used in Flatten layer - - int nNeurons; // Number of neurons - LayerType type = LayerType::DEFAULT; - - /** - * @brief This method is used to set the outputs of the layer - * - * @param outputs A vector of doubles representing the outputs of the layer - * - * @return void - * - * @note This method is used for the input layer (the first layer of the network) - */ - void setOutputs(std::vector outputs) - { - assert(outputs.size() == nNeurons); - this->outputs = Eigen::MatrixXd ::Map(&outputs[0], this->getNumNeurons(), 1); - }; - void setOutputs(Eigen::MatrixXd outputs) // used for the Flatten Layer - { - this->outputs = outputs; - }; + return; + }; + + // Necessary function for serializing Layer + template + void save(Archive &archive) const { + archive(nNeurons, weights, outputs, biases, activation, type); + }; + + template + void load(Archive &archive) { + archive(nNeurons, weights, outputs, biases, activation, type); + setActivation(activation); + } + + protected: + Layer(){}; // Necessary for serialization + Layer(std::tuple inputShape) + : nNeurons(std::get<0>(inputShape) * + std::get<1>(inputShape)){}; // Used in Flatten layer + + int nNeurons; // Number of neurons + LayerType type = LayerType::DEFAULT; + + /** + * @brief This method is used to set the outputs of the layer + * + * @param outputs A vector of doubles representing the outputs of the layer + * + * @return void + * + * @note This method is used for the input layer (the first layer of the + * network) + */ + void setOutputs(std::vector outputs) { + assert(outputs.size() == nNeurons); + this->outputs = + Eigen::MatrixXd ::Map(&outputs[0], this->getNumNeurons(), 1); + }; + void setOutputs(Eigen::MatrixXd outputs) // used for the Flatten Layer + { + this->outputs = outputs; }; -} // namespace NeuralNet +}; +} // namespace NeuralNet CEREAL_REGISTER_TYPE(NeuralNet::Layer); \ No newline at end of file diff --git a/src/NeuralNet/losses/Loss.hpp b/src/NeuralNet/losses/Loss.hpp index 190eaa0..ebff7f7 100644 --- a/src/NeuralNet/losses/Loss.hpp +++ b/src/NeuralNet/losses/Loss.hpp @@ -2,29 +2,28 @@ #include -namespace NeuralNet -{ - class Loss - { - public: - /** - * @brief This function computes the loss of the current iteration - * - * @param o The outputs from the output layer - * @param y The labels (the expected values) - * - * @return The loss based on the selected loss function - */ - static double cmpLoss(const Eigen::MatrixXd &o, const Eigen::MatrixXd &y); +namespace NeuralNet { +class Loss { + public: + /** + * @brief This function computes the loss of the current iteration + * + * @param o The outputs from the output layer + * @param y The labels (the expected values) + * + * @return The loss based on the selected loss function + */ + static double cmpLoss(const Eigen::MatrixXd &o, const Eigen::MatrixXd &y); - /** - * @brief This function computes the loss gradient w.r.t the outputs - * - * @param o The outputs from the output layer - * @param y The labels (expected vals) - * - * @return The current iteration's gradient - */ - static Eigen::MatrixXd cmpLossGrad(const Eigen::MatrixXd &o, const Eigen::MatrixXd &y); - }; -} \ No newline at end of file + /** + * @brief This function computes the loss gradient w.r.t the outputs + * + * @param o The outputs from the output layer + * @param y The labels (expected vals) + * + * @return The current iteration's gradient + */ + static Eigen::MatrixXd cmpLossGrad(const Eigen::MatrixXd &o, + const Eigen::MatrixXd &y); +}; +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/losses/MCE.hpp b/src/NeuralNet/losses/MCE.hpp index 7d775a7..ee81f62 100644 --- a/src/NeuralNet/losses/MCE.hpp +++ b/src/NeuralNet/losses/MCE.hpp @@ -2,25 +2,22 @@ #include "Loss.hpp" -namespace NeuralNet -{ - /** - * Multi-class Cross Entropy - */ - class MCE : public Loss - { - public: - static double cmpLoss(const Eigen::MatrixXd &o, const Eigen::MatrixXd &y) - { - Eigen::MatrixXd cMatrix = y.array() * o.array().log(); +namespace NeuralNet { +/** + * Multi-class Cross Entropy + */ +class MCE : public Loss { + public: + static double cmpLoss(const Eigen::MatrixXd &o, const Eigen::MatrixXd &y) { + Eigen::MatrixXd cMatrix = y.array() * o.array().log(); - return -cMatrix.sum(); - }; + return -cMatrix.sum(); + }; - static Eigen::MatrixXd cmpLossGrad(const Eigen::MatrixXd &yHat, const Eigen::MatrixXd &y) - { - assert(yHat.rows() == y.rows() && yHat.cols() == y.cols()); - return yHat.array() - y.array(); - }; + static Eigen::MatrixXd cmpLossGrad(const Eigen::MatrixXd &yHat, + const Eigen::MatrixXd &y) { + assert(yHat.rows() == y.rows() && yHat.cols() == y.cols()); + return yHat.array() - y.array(); }; -} \ No newline at end of file +}; +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/losses/Quadratic.hpp b/src/NeuralNet/losses/Quadratic.hpp index b56a622..802a4de 100644 --- a/src/NeuralNet/losses/Quadratic.hpp +++ b/src/NeuralNet/losses/Quadratic.hpp @@ -2,26 +2,23 @@ #include "Loss.hpp" -namespace NeuralNet -{ - /** - * Quadratic Loss - */ - class Quadratic : public Loss - { - public: - static double cmpLoss(const Eigen::MatrixXd &o, const Eigen::MatrixXd &y) - { - Eigen::MatrixXd cMatrix = o.array() - y.array(); - cMatrix = cMatrix.unaryExpr(&sqr); +namespace NeuralNet { +/** + * Quadratic Loss + */ +class Quadratic : public Loss { + public: + static double cmpLoss(const Eigen::MatrixXd &o, const Eigen::MatrixXd &y) { + Eigen::MatrixXd cMatrix = o.array() - y.array(); + cMatrix = cMatrix.unaryExpr(&sqr); - return cMatrix.sum(); - }; + return cMatrix.sum(); + }; - static Eigen::MatrixXd cmpLossGrad(const Eigen::MatrixXd &yHat, const Eigen::MatrixXd &y) - { - assert(yHat.rows() == y.rows()); - return (yHat.array() - y.array()).matrix() * 2; - }; + static Eigen::MatrixXd cmpLossGrad(const Eigen::MatrixXd &yHat, + const Eigen::MatrixXd &y) { + assert(yHat.rows() == y.rows()); + return (yHat.array() - y.array()).matrix() * 2; }; -} \ No newline at end of file +}; +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/losses/losses.hpp b/src/NeuralNet/losses/losses.hpp index b5ba575..87937f6 100644 --- a/src/NeuralNet/losses/losses.hpp +++ b/src/NeuralNet/losses/losses.hpp @@ -1,5 +1,6 @@ /** - * The purpose of this file is solely to group the includes of the files in this folder. + * The purpose of this file is solely to group the includes of the files in this + * folder. */ #pragma once diff --git a/src/NeuralNet/optimizers/Adam.hpp b/src/NeuralNet/optimizers/Adam.hpp index e2decc4..bd472f0 100644 --- a/src/NeuralNet/optimizers/Adam.hpp +++ b/src/NeuralNet/optimizers/Adam.hpp @@ -2,109 +2,112 @@ #include #include + #include "Optimizer.hpp" -namespace NeuralNet -{ +namespace NeuralNet { +/** + * Adam optimizer + */ +class Adam : public Optimizer { + public: /** - * Adam optimizer + * Adam is an optimization algorithm that can be used instead of the classical + * stochastic gradient descent procedure to update network weights + * iteratively. + * + * @param alpha Learning rate + * @param beta1 Exponential decay rate for the first moment estimates + * @param beta2 Exponential decay rate for the second moment estimates + * @param epsilon A small constant for numerical stability */ - class Adam : public Optimizer - { - public: - /** - * Adam is an optimization algorithm that can be used instead of the classical stochastic gradient descent procedure to update network weights iteratively. - * - * @param alpha Learning rate - * @param beta1 Exponential decay rate for the first moment estimates - * @param beta2 Exponential decay rate for the second moment estimates - * @param epsilon A small constant for numerical stability - */ - Adam(double alpha = 0.001, double beta1 = 0.9, double beta2 = 0.999, double epsilon = 10E-8) : Optimizer(alpha) - { - this->beta1 = beta1; - this->beta2 = beta2; - this->epsilon = epsilon; - }; - - ~Adam() override = default; - - void updateWeights(Eigen::MatrixXd &weights, const Eigen::MatrixXd &weightsGrad) override - { - this->update(weights, weightsGrad, mWeights[cl], vWeights[cl]); - }; - - void updateBiases(Eigen::MatrixXd &biases, const Eigen::MatrixXd &biasesGrad) override - { - this->update(biases, biasesGrad, mBiases[cl], vBiases[cl]); - this->setCurrentL(); - }; - - template - void update(Eigen::MatrixBase ¶m, const Eigen::MatrixBase &gradients, Eigen::MatrixBase &m, Eigen::MatrixBase &v) - { - assert(param.rows() == gradients.rows() && param.cols() == gradients.cols()); - - // increment time step - t = t + 1; - - if (m.rows() == 0 || m.cols() == 0) - { - // Initialize moment matrices m and v - m = Eigen::MatrixBase::Zero(param.rows(), param.cols()); - v = Eigen::MatrixBase::Zero(param.rows(), param.cols()); - } + Adam(double alpha = 0.001, double beta1 = 0.9, double beta2 = 0.999, + double epsilon = 10E-8) + : Optimizer(alpha) { + this->beta1 = beta1; + this->beta2 = beta2; + this->epsilon = epsilon; + }; - // update biased first moment estimate - m = (beta1 * m).array() + ((1 - beta2) * gradients.array()).array(); + ~Adam() override = default; - // updated biased second raw moment estimate - v = (beta2 * v).array() + ((1 - beta2) * (gradients.array() * gradients.array())).array(); + void updateWeights(Eigen::MatrixXd &weights, + const Eigen::MatrixXd &weightsGrad) override { + this->update(weights, weightsGrad, mWeights[cl], vWeights[cl]); + }; - // compute bias-corrected first moment estimate - double beta1_t = std::pow(beta1, t); + void updateBiases(Eigen::MatrixXd &biases, + const Eigen::MatrixXd &biasesGrad) override { + this->update(biases, biasesGrad, mBiases[cl], vBiases[cl]); + this->setCurrentL(); + }; - // compute bias-corrected second raw moment estimate - double beta2_t = std::pow(beta2, t); + template + void update(Eigen::MatrixBase ¶m, + const Eigen::MatrixBase &gradients, + Eigen::MatrixBase &m, Eigen::MatrixBase &v) { + assert(param.rows() == gradients.rows() && + param.cols() == gradients.cols()); - double alpha_t = alpha * (sqrt(1 - beta2_t) / (1 - beta1_t)); + // increment time step + t = t + 1; - // update param - param = param.array() - alpha_t * (m.array() / (v.array().sqrt() + epsilon)); + if (m.rows() == 0 || m.cols() == 0) { + // Initialize moment matrices m and v + m = Eigen::MatrixBase::Zero(param.rows(), param.cols()); + v = Eigen::MatrixBase::Zero(param.rows(), param.cols()); } - private: - double beta1; - double beta2; - double epsilon; - int t = 0; - int cl; // Current layer (should be initialized to the total number of layers - 0) - int ll; // Last layer (should also be initialized to numLayers - 1) - std::vector mWeights; // First-moment vector for weights - std::vector vWeights; // Second-moment vector for weights - std::vector mBiases; // First-moment vector for biases - std::vector vBiases; // Second-moment vector for biases - - void insiderInit(size_t numLayers) override - { - cl = numLayers - 1; - ll = numLayers - 1; - - Eigen::MatrixXd dotMatrix = Eigen::MatrixXd::Zero(0, 0); - - for (int i = mWeights.size(); i < numLayers; i++) - { - mWeights.push_back(dotMatrix); - vWeights.push_back(dotMatrix); - mBiases.push_back(dotMatrix); - vBiases.push_back(dotMatrix); - }; - } - - void setCurrentL() - { - // If current layer is the first layer set it to the last layer - cl = cl == 1 ? ll : cl - 1; - } - }; -} \ No newline at end of file + // update biased first moment estimate + m = (beta1 * m).array() + ((1 - beta2) * gradients.array()).array(); + + // updated biased second raw moment estimate + v = (beta2 * v).array() + + ((1 - beta2) * (gradients.array() * gradients.array())).array(); + + // compute bias-corrected first moment estimate + double beta1_t = std::pow(beta1, t); + + // compute bias-corrected second raw moment estimate + double beta2_t = std::pow(beta2, t); + + double alpha_t = alpha * (sqrt(1 - beta2_t) / (1 - beta1_t)); + + // update param + param = + param.array() - alpha_t * (m.array() / (v.array().sqrt() + epsilon)); + } + + private: + double beta1; + double beta2; + double epsilon; + int t = 0; + int cl; // Current layer (should be initialized to the total number of layers + // - 0) + int ll; // Last layer (should also be initialized to numLayers - 1) + std::vector mWeights; // First-moment vector for weights + std::vector vWeights; // Second-moment vector for weights + std::vector mBiases; // First-moment vector for biases + std::vector vBiases; // Second-moment vector for biases + + void insiderInit(size_t numLayers) override { + cl = numLayers - 1; + ll = numLayers - 1; + + Eigen::MatrixXd dotMatrix = Eigen::MatrixXd::Zero(0, 0); + + for (int i = mWeights.size(); i < numLayers; i++) { + mWeights.push_back(dotMatrix); + vWeights.push_back(dotMatrix); + mBiases.push_back(dotMatrix); + vBiases.push_back(dotMatrix); + }; + } + + void setCurrentL() { + // If current layer is the first layer set it to the last layer + cl = cl == 1 ? ll : cl - 1; + } +}; +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/optimizers/Optimizer.hpp b/src/NeuralNet/optimizers/Optimizer.hpp index 297b129..08bd73d 100644 --- a/src/NeuralNet/optimizers/Optimizer.hpp +++ b/src/NeuralNet/optimizers/Optimizer.hpp @@ -2,47 +2,53 @@ #include -namespace NeuralNet -{ - class Optimizer - { - friend class Network; - - public: - Optimizer(double alpha) : alpha(alpha){}; - - virtual ~Optimizer() = default; // Virtual destructor - - /** - * @brief This function updates the weights passed based on the selected Optimizer and the weights gradients - * - * @param weights The weights that should be updated - * @param weightsGrad The weights gradient - * - * The function will return void, since it only performs an update on the weights passed - */ - virtual void updateWeights(Eigen::MatrixXd &weights, const Eigen::MatrixXd &weightsGrad) = 0; - - /** - * @brief This function updates the biases passed based based on the Optimizer and the biases gradients - * - * @param biases The biases that should be updated - * @param biasesGrad The biases gradient - * - * The function will return void, since it only performs an update on the biases passed - */ - virtual void updateBiases(Eigen::MatrixXd &biases, const Eigen::MatrixXd &biasesGrad) = 0; - - protected: - double alpha; - - /** - * @brief This function's purpose is to provide an interface to perform updates for the Optimizers from within the network - * - * @param size The size (for now it's only used to init the layer's in the Adam optimizer) - * - * This function returns nothing - */ - virtual void insiderInit(size_t size) = 0; - }; -} \ No newline at end of file +namespace NeuralNet { +class Optimizer { + friend class Network; + + public: + Optimizer(double alpha) : alpha(alpha){}; + + virtual ~Optimizer() = default; // Virtual destructor + + /** + * @brief This function updates the weights passed based on the selected + * Optimizer and the weights gradients + * + * @param weights The weights that should be updated + * @param weightsGrad The weights gradient + * + * The function will return void, since it only performs an update on the + * weights passed + */ + virtual void updateWeights(Eigen::MatrixXd &weights, + const Eigen::MatrixXd &weightsGrad) = 0; + + /** + * @brief This function updates the biases passed based based on the Optimizer + * and the biases gradients + * + * @param biases The biases that should be updated + * @param biasesGrad The biases gradient + * + * The function will return void, since it only performs an update on the + * biases passed + */ + virtual void updateBiases(Eigen::MatrixXd &biases, + const Eigen::MatrixXd &biasesGrad) = 0; + + protected: + double alpha; + + /** + * @brief This function's purpose is to provide an interface to perform + * updates for the Optimizers from within the network + * + * @param size The size (for now it's only used to init the layer's in the + * Adam optimizer) + * + * This function returns nothing + */ + virtual void insiderInit(size_t size) = 0; +}; +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/optimizers/SGD.hpp b/src/NeuralNet/optimizers/SGD.hpp index ea4ff72..0b9a717 100644 --- a/src/NeuralNet/optimizers/SGD.hpp +++ b/src/NeuralNet/optimizers/SGD.hpp @@ -2,29 +2,27 @@ #include "Optimizer.hpp" -namespace NeuralNet -{ - /** - * Stochastic Gradient Descent optimizer - */ - class SGD : public Optimizer - { - public: - SGD(double alpha) : Optimizer(alpha){}; +namespace NeuralNet { +/** + * Stochastic Gradient Descent optimizer + */ +class SGD : public Optimizer { + public: + SGD(double alpha) : Optimizer(alpha){}; - ~SGD() override = default; + ~SGD() override = default; - void updateWeights(Eigen::MatrixXd &weights, const Eigen::MatrixXd &weightsGrad) override - { - weights = weights.array() - (this->alpha * weightsGrad).array(); - }; - - void updateBiases(Eigen::MatrixXd &biases, const Eigen::MatrixXd &biasesGrad) override - { - biases = biases.array() - (this->alpha * biasesGrad).array(); - }; + void updateWeights(Eigen::MatrixXd &weights, + const Eigen::MatrixXd &weightsGrad) override { + weights = weights.array() - (this->alpha * weightsGrad).array(); + }; - private: - void insiderInit(size_t size) override{}; + void updateBiases(Eigen::MatrixXd &biases, + const Eigen::MatrixXd &biasesGrad) override { + biases = biases.array() - (this->alpha * biasesGrad).array(); }; -} \ No newline at end of file + + private: + void insiderInit(size_t size) override{}; +}; +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/optimizers/optimizers.hpp b/src/NeuralNet/optimizers/optimizers.hpp index 697d8b1..2d607b0 100644 --- a/src/NeuralNet/optimizers/optimizers.hpp +++ b/src/NeuralNet/optimizers/optimizers.hpp @@ -1,7 +1,8 @@ /** - * The purpose of this file is solely to group the includes of the files in this folder. + * The purpose of this file is solely to group the includes of the files in this + * folder. */ #pragma once -#include "SGD.hpp" -#include "Adam.hpp" \ No newline at end of file +#include "Adam.hpp" +#include "SGD.hpp" \ No newline at end of file diff --git a/src/NeuralNet/utils/Enums.hpp b/src/NeuralNet/utils/Enums.hpp index 1d2e20d..7b75341 100644 --- a/src/NeuralNet/utils/Enums.hpp +++ b/src/NeuralNet/utils/Enums.hpp @@ -1,26 +1,22 @@ #pragma once -namespace NeuralNet -{ - enum class ACTIVATION - { - RELU, - SIGMOID, - SOFTMAX, - }; +namespace NeuralNet { +enum class ACTIVATION { + RELU, + SIGMOID, + SOFTMAX, +}; - enum class WEIGHT_INIT - { - RANDOM, - GLOROT, - HE, - LECUN, - CONSTANT // For testing - }; +enum class WEIGHT_INIT { + RANDOM, + GLOROT, + HE, + LECUN, + CONSTANT // For testing +}; - enum class LOSS - { - MCE, // Multi-class Cross Entropy - QUADRATIC - }; -} +enum class LOSS { + MCE, // Multi-class Cross Entropy + QUADRATIC +}; +} // namespace NeuralNet diff --git a/src/NeuralNet/utils/Formatters.hpp b/src/NeuralNet/utils/Formatters.hpp index 722a2dd..b4622d6 100644 --- a/src/NeuralNet/utils/Formatters.hpp +++ b/src/NeuralNet/utils/Formatters.hpp @@ -2,52 +2,50 @@ #include #include + #include "Functions.hpp" -namespace NeuralNet -{ - /** - * @brief This function transform the labels in 2d or 1d vectors into Matrices - * - * The resulting matrices will be used to evaluate the model's performance and do the necessary adjustments on its parameters - * - * @param labels 1d std::vector or 2d std::vector (based on the task) - * @param shape std::tuple that defines the shape of the resulting matrix - * - * @return The resulting matrix from the passed labels - */ - template - static Eigen::MatrixXd formatLabels(std::vector labels, std::tuple shape) - { - int rows = std::get<0>(shape); - int cols = std::get<1>(shape); - - // When propagating forward, the number of rows will be the number of inputs - assert(labels.size() == rows && "The number of labels don't match the number of inputs"); - - Eigen::MatrixXd mLabels(rows, cols); - - if constexpr (std::is_same>::value) - { - std::vector flattenedVector = flatten2DVector(labels, rows, cols); - mLabels = Eigen::Map(flattenedVector.data(), rows, cols); - } - else if constexpr (std::is_same::value) - { - mLabels = Eigen::MatrixXd::Zero(rows, cols); - - /** - * Setting the cols indexes to 1 - * Notably used in classification tasks - */ - for (int i = 0; i < rows; i++) - { - int colIndex = labels[i]; - assert(colIndex < cols); - mLabels(i, colIndex) = 1; - } - } +namespace NeuralNet { +/** + * @brief This function transform the labels in 2d or 1d vectors into Matrices + * + * The resulting matrices will be used to evaluate the model's performance and + * do the necessary adjustments on its parameters + * + * @param labels 1d std::vector or 2d std::vector (based on the task) + * @param shape std::tuple that defines the shape of the resulting matrix + * + * @return The resulting matrix from the passed labels + */ +template +static Eigen::MatrixXd formatLabels(std::vector labels, + std::tuple shape) { + int rows = std::get<0>(shape); + int cols = std::get<1>(shape); + + // When propagating forward, the number of rows will be the number of inputs + assert(labels.size() == rows && + "The number of labels don't match the number of inputs"); - return mLabels; + Eigen::MatrixXd mLabels(rows, cols); + + if constexpr (std::is_same>::value) { + std::vector flattenedVector = flatten2DVector(labels, rows, cols); + mLabels = Eigen::Map(flattenedVector.data(), rows, cols); + } else if constexpr (std::is_same::value) { + mLabels = Eigen::MatrixXd::Zero(rows, cols); + + /** + * Setting the cols indexes to 1 + * Notably used in classification tasks + */ + for (int i = 0; i < rows; i++) { + int colIndex = labels[i]; + assert(colIndex < cols); + mLabels(i, colIndex) = 1; + } } -} // namespace NeuralNet + + return mLabels; +} +} // namespace NeuralNet diff --git a/src/NeuralNet/utils/Functions.hpp b/src/NeuralNet/utils/Functions.hpp index b7816a5..7f9782c 100644 --- a/src/NeuralNet/utils/Functions.hpp +++ b/src/NeuralNet/utils/Functions.hpp @@ -1,312 +1,295 @@ #pragma once +#include #include -#include -#include #include -#include #include -#include +#include +#include +#include namespace fs = std::filesystem; -namespace NeuralNet -{ - - /** - * @brief Rand double generator that uses the Mersenne Twister algo - * - * @param min Minimum value generated - * @param max Maximum value generated - * - * @return A random number between min and max - */ - inline double mtRand(double min, double max) - { - assert(min < max); - std::random_device rseed; - std::mt19937_64 rng(rseed()); - std::uniform_real_distribution dist(min, max); - - return dist(rng); - }; - - /** - * @brief Generates a vector with random doubles based on the previous mtRand function - * - * @param size The desired size of the resulting vector - * @param min Minimum possible number (default : -10) - * @param max Maximum possible number (default: 10) - * - * @return A vector with random doubles generated with the Mersenne Twister algo - */ - inline std::vector randDVector(int size, double min = -10, double max = 10) - { - std::vector v; - - for (int i = 0; i < size; i++) - { - v.push_back(mtRand(min, max)); - } - - return v; +namespace NeuralNet { + +/** + * @brief Rand double generator that uses the Mersenne Twister algo + * + * @param min Minimum value generated + * @param max Maximum value generated + * + * @return A random number between min and max + */ +inline double mtRand(double min, double max) { + assert(min < max); + std::random_device rseed; + std::mt19937_64 rng(rseed()); + std::uniform_real_distribution dist(min, max); + + return dist(rng); +}; + +/** + * @brief Generates a vector with random doubles based on the previous mtRand + * function + * + * @param size The desired size of the resulting vector + * @param min Minimum possible number (default : -10) + * @param max Maximum possible number (default: 10) + * + * @return A vector with random doubles generated with the Mersenne Twister algo + */ +inline std::vector randDVector(int size, double min = -10, + double max = 10) { + std::vector v; + + for (int i = 0; i < size; i++) { + v.push_back(mtRand(min, max)); } - /** - * @brief This function checks if a file exists and has a specific extension - * - * @param filePath The path of the file - * @param extension The extension that's checked - * - * @return Returns true if the file exists and has the specified extension otherwise returns false - */ - inline bool fileExistsWithExtension(const std::string &filePath, const std::string &extension) - { - // Check if the file exist - if (fs::exists(filePath)) - { - // Check if the file has specified extension - fs::path file(filePath); - return file.has_extension() && file.extension() == extension; - } - - return false; - } - - /** - * @brief This function checks if a file has a specific extension - * - * @param filePath The path of the file - * @param extension The extension that's checked - * - * @return Returns true if the file has the specified extension otherwise returns false - */ - inline bool fileHasExtension(const std::string &filePath, const std::string &extension) - { + return v; +} + +/** + * @brief This function checks if a file exists and has a specific extension + * + * @param filePath The path of the file + * @param extension The extension that's checked + * + * @return Returns true if the file exists and has the specified extension + * otherwise returns false + */ +inline bool fileExistsWithExtension(const std::string &filePath, + const std::string &extension) { + // Check if the file exist + if (fs::exists(filePath)) { + // Check if the file has specified extension fs::path file(filePath); return file.has_extension() && file.extension() == extension; } - /* MATHEMATICAL FUNCTIONS */ - - /** - * @brief Function that calculates the square of a number - * - * @param x the number - * - * @return The square of x - */ - inline double sqr(const double x) - { - return x * x; - }; - - /* VECTOR OPERATIONS */ - - /** - * @brief 2d std::vector memory allocation function - * - * @param v the vector that needs size allocation - * @param rows the number of rows to allocate - * @param cols the number of cols to allocate - * - * This function just simplifies reserving space for a 2 dimensional vector - * It's necessary if we know the size in advance because it can save a lot of unnecessary computations - */ - template - inline void reserve2d(std::vector> &v, int rows, int cols) - { - // reserve space for num rows - v.reserve(rows); - - // reserve space for each row - for (int i = 0; i < rows; i++) - { - v.push_back(std::vector()); - v[i].reserve(cols); - } - }; - - /** - * @brief Find the row index of the max element in a Matrix - * - * @param m The Eigen::Matrix - * - * @return -1 if an error occurs or not found otherwise returns the row index of the element. - */ - inline int findRowIndexOfMaxEl(const Eigen::MatrixXd &m) - { - // Find the maximum value in the matrix - double maxVal = m.maxCoeff(); - - // Find the row index by iterating through rows - for (int i = 0; i < m.rows(); ++i) - { - if ((m.row(i).array() == maxVal).any()) - { - return i; - } - } - - // Return -1 if not found (this can be handled based on your use case) - return -1; - }; - - /** - * @brief This function takes a 2d vector and flattens it into a 1d vector - * - * @param input The 2D vector - * @param rows The number of vectors - * @param cols The number of cols inside each vector - * - * Passing the number of rows and columns makes this function much more efficient - * - * @return The resulting 1D vector - */ - template - inline std::vector flatten2DVector(const std::vector> &input, size_t rows, size_t cols) - { - // Asserting that the inputs respect the declared size - assert(input.size() == rows); - for (const std::vector &row : input) - { - assert(row.size() == cols); + return false; +} + +/** + * @brief This function checks if a file has a specific extension + * + * @param filePath The path of the file + * @param extension The extension that's checked + * + * @return Returns true if the file has the specified extension otherwise + * returns false + */ +inline bool fileHasExtension(const std::string &filePath, + const std::string &extension) { + fs::path file(filePath); + return file.has_extension() && file.extension() == extension; +} + +/* MATHEMATICAL FUNCTIONS */ + +/** + * @brief Function that calculates the square of a number + * + * @param x the number + * + * @return The square of x + */ +inline double sqr(const double x) { return x * x; }; + +/* VECTOR OPERATIONS */ + +/** + * @brief 2d std::vector memory allocation function + * + * @param v the vector that needs size allocation + * @param rows the number of rows to allocate + * @param cols the number of cols to allocate + * + * This function just simplifies reserving space for a 2 dimensional vector + * It's necessary if we know the size in advance because it can save a lot of + * unnecessary computations + */ +template +inline void reserve2d(std::vector> &v, int rows, int cols) { + // reserve space for num rows + v.reserve(rows); + + // reserve space for each row + for (int i = 0; i < rows; i++) { + v.push_back(std::vector()); + v[i].reserve(cols); + } +}; + +/** + * @brief Find the row index of the max element in a Matrix + * + * @param m The Eigen::Matrix + * + * @return -1 if an error occurs or not found otherwise returns the row index of + * the element. + */ +inline int findRowIndexOfMaxEl(const Eigen::MatrixXd &m) { + // Find the maximum value in the matrix + double maxVal = m.maxCoeff(); + + // Find the row index by iterating through rows + for (int i = 0; i < m.rows(); ++i) { + if ((m.row(i).array() == maxVal).any()) { + return i; } + } - std::vector result; - result.reserve(rows * cols); + // Return -1 if not found (this can be handled based on your use case) + return -1; +}; + +/** + * @brief This function takes a 2d vector and flattens it into a 1d vector + * + * @param input The 2D vector + * @param rows The number of vectors + * @param cols The number of cols inside each vector + * + * Passing the number of rows and columns makes this function much more + * efficient + * + * @return The resulting 1D vector + */ +template +inline std::vector flatten2DVector(const std::vector> &input, + size_t rows, size_t cols) { + // Asserting that the inputs respect the declared size + assert(input.size() == rows); + for (const std::vector &row : input) { + assert(row.size() == cols); + } - // Flatten the 2D vector - for (const std::vector &row : input) - { - result.insert(result.end(), row.begin(), row.end()); - } + std::vector result; + result.reserve(rows * cols); - return result; + // Flatten the 2D vector + for (const std::vector &row : input) { + result.insert(result.end(), row.begin(), row.end()); } - /** - * @brief This function takes a vector and a value and returns the index of the value in that vector. - * - * @param v The vector - * @param el The element to look for in the vector - * - * @return The index of the element in the vector `-1` if not found. - */ - template - inline int findIndexOf(const std::vector &v, const T &el) - { - auto it = std::find(v.begin(), v.end(), el); - - if (it == v.end()) - return -1; - - return it - v.begin(); + return result; +} + +/** + * @brief This function takes a vector and a value and returns the index of the + * value in that vector. + * + * @param v The vector + * @param el The element to look for in the vector + * + * @return The index of the element in the vector `-1` if not found. + */ +template +inline int findIndexOf(const std::vector &v, const T &el) { + auto it = std::find(v.begin(), v.end(), el); + + if (it == v.end()) return -1; + + return it - v.begin(); +} + +/* MATRIX OPERATIONS */ +inline Eigen::MatrixXd zeroMatrix(const std::tuple size) { + return Eigen::MatrixXd::Zero(std::get<0>(size), std::get<1>(size)); +} + +inline Eigen::MatrixXd vectorToMatrixXd(std::vector> &v) { + if (v.empty() || v[0].empty()) return Eigen::MatrixXd(0, 0); + + int rows = v.size(); + int cols = v[0].size(); + + // Flatten the vector of vectors into a single vector + std::vector flat; + flat.reserve(rows * cols); + for (const auto &row : v) { + flat.insert(flat.end(), row.begin(), row.end()); } - /* MATRIX OPERATIONS */ - inline Eigen::MatrixXd zeroMatrix(const std::tuple size) - { - return Eigen::MatrixXd::Zero(std::get<0>(size), std::get<1>(size)); + return Eigen::Map< + Eigen::Matrix>( + flat.data(), rows, cols); +}; + +/** + * @brief initialises a matrix with random values that ranges between the + * defined boundaries + * + * @param weightsMatrix a pointer to the weights matrix or any Eigen::MatrixXd + * @param min The min value in the range (default: -1) + * @param max The max value in the range (default: 1) + * + * @return void + */ +static void randomWeightInit(Eigen::MatrixXd *weightsMatrix, double min = -1.0, + double max = 1.0) { + for (int col = 0; col < weightsMatrix->cols(); col++) { + for (int row = 0; row < weightsMatrix->rows(); row++) { + weightsMatrix->operator()(row, col) = mtRand(min, max); + } } - inline Eigen::MatrixXd vectorToMatrixXd(std::vector> &v) - { - if (v.empty() || v[0].empty()) - return Eigen::MatrixXd(0, 0); - - int rows = v.size(); - int cols = v[0].size(); - - // Flatten the vector of vectors into a single vector - std::vector flat; - flat.reserve(rows * cols); - for (const auto &row : v) - { - flat.insert(flat.end(), row.begin(), row.end()); + return; +}; + +/** + * @brief Method that sets the value of a matrix to follow a certain random Dist + * + * @param weightsMatrix A pointer to a weight matrix or any Eigen::MatrixXd + * @param mean The mean for the std dist + * @param stddev The standard deviation + * + * @return void + */ +static void randomDistMatrixInit(Eigen::MatrixXd *weightsMatrix, double mean, + double stddev) { + std::random_device rseed; + std::default_random_engine generator(rseed()); + std::normal_distribution distribution(mean, stddev); + + for (int col = 0; col < weightsMatrix->cols(); col++) { + for (int row = 0; row < weightsMatrix->rows(); row++) { + weightsMatrix->operator()(row, col) = distribution(generator); } + } - return Eigen::Map>(flat.data(), rows, cols); - }; - - /** - * @brief initialises a matrix with random values that ranges between the defined boundaries - * - * @param weightsMatrix a pointer to the weights matrix or any Eigen::MatrixXd - * @param min The min value in the range (default: -1) - * @param max The max value in the range (default: 1) - * - * @return void - */ - static void randomWeightInit(Eigen::MatrixXd *weightsMatrix, double min = -1.0, double max = 1.0) - { - for (int col = 0; col < weightsMatrix->cols(); col++) - { - for (int row = 0; row < weightsMatrix->rows(); row++) - { - weightsMatrix->operator()(row, col) = mtRand(min, max); - } - } + return; +}; - return; - }; - - /** - * @brief Method that sets the value of a matrix to follow a certain random Dist - * - * @param weightsMatrix A pointer to a weight matrix or any Eigen::MatrixXd - * @param mean The mean for the std dist - * @param stddev The standard deviation - * - * @return void - */ - static void randomDistMatrixInit(Eigen::MatrixXd *weightsMatrix, double mean, double stddev) - { - std::random_device rseed; - std::default_random_engine generator(rseed()); - std::normal_distribution distribution(mean, stddev); - - for (int col = 0; col < weightsMatrix->cols(); col++) - { - for (int row = 0; row < weightsMatrix->rows(); row++) - { - weightsMatrix->operator()(row, col) = distribution(generator); - } - } +/** + * @brief takes in a matrix and returns its hardmax equivalent. + * + * @param mat the inputs matrix + * + * @return the hardmax version of the matrix. + */ +static Eigen::MatrixXd hardmax(const Eigen::MatrixXd &mat) { + Eigen::MatrixXd hardmaxMatrix = Eigen::MatrixXd::Zero(mat.rows(), mat.cols()); - return; - }; - - /** - * @brief takes in a matrix and returns its hardmax equivalent. - * - * @param mat the inputs matrix - * - * @return the hardmax version of the matrix. - */ - static Eigen::MatrixXd hardmax(const Eigen::MatrixXd &mat) - { - Eigen::MatrixXd hardmaxMatrix = Eigen::MatrixXd::Zero(mat.rows(), mat.cols()); - - for (int i = 0; i < mat.rows(); ++i) - { - int maxIndex; - mat.row(i).maxCoeff(&maxIndex); - - hardmaxMatrix(i, maxIndex) = 1; - } + for (int i = 0; i < mat.rows(); ++i) { + int maxIndex; + mat.row(i).maxCoeff(&maxIndex); - return hardmaxMatrix; + hardmaxMatrix(i, maxIndex) = 1; } - /* SIGNAL HANDLING */ - static void signalHandler(int signum) - { - std::cout << "Interrupt signal (" << signum << ") received.\n"; + return hardmaxMatrix; +} + +/* SIGNAL HANDLING */ +static void signalHandler(int signum) { + std::cout << "Interrupt signal (" << signum << ") received.\n"; - // cleanup and close up stuff here - // terminate program - exit(signum); - }; + // cleanup and close up stuff here + // terminate program + exit(signum); +}; -} // namespace NeuralNet \ No newline at end of file +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/utils/Gauge.hpp b/src/NeuralNet/utils/Gauge.hpp index 86715ae..d701547 100644 --- a/src/NeuralNet/utils/Gauge.hpp +++ b/src/NeuralNet/utils/Gauge.hpp @@ -1,142 +1,140 @@ #pragma once #include // for text, gauge, operator|, flex, hbox, Element -#include // for Screen -#include // for cout, endl, ostream -#include // for allocator, char_traits, operator+, operator<<, string, to_string, basic_string -#include // for sleep_for +#include // for Screen +#include // for cout, endl, ostream +#include // for allocator, char_traits, operator+, operator<<, string, to_string, basic_string +#include // for sleep_for using namespace ftxui; -namespace NeuralNet -{ - /** - * A general purpose simple use gauge. - * It encompasses the few lines of code to have an operational ftxui::gauge - */ +namespace NeuralNet { +/** + * A general purpose simple use gauge. + * It encompasses the few lines of code to have an operational ftxui::gauge + */ + +class Gauge { + public: + Gauge(std::string preText, int totalIndexes, int currIndex = 0) { + this->totalIndexes = totalIndexes; + this->currIndex = currIndex; + this->preText = preText; + }; - class Gauge - { - public: - Gauge(std::string preText, int totalIndexes, int currIndex = 0) - { - this->totalIndexes = totalIndexes; - this->currIndex = currIndex; - this->preText = preText; - }; - - Gauge &operator++() - { - this->currIndex++; - printGauge(); - return *this; - }; - - protected: - int totalIndexes; - int currIndex; - std::string preText; - // string to be printed in order to reset the cursor position - std::string resetPos; - - private: - void printGauge() - { - std::string ratioStr = std::to_string(this->currIndex) + "/" + std::to_string(this->totalIndexes); - float ratio = static_cast(this->currIndex) / this->totalIndexes; - Element document = hbox({ - text(this->preText), - gauge(ratio) | flex, - text(" " + ratioStr), - }); - - auto screen = Screen::Create(Dimension::Fixed(100), Dimension::Fit(document)); - Render(screen, document); - std::cout << this->resetPos; - screen.Print(); - this->resetPos = screen.ResetPosition(); - }; + Gauge &operator++() { + this->currIndex++; + printGauge(); + return *this; }; - // todo: Refactor this code to not have duplicates - class TrainingGauge : public Gauge - { - public: - TrainingGauge(int totalIndexes, int currIndex = 0, int totalEpochs = 10, int currEpoch = 0) : Gauge("Training : ", totalIndexes, currIndex) - { - this->totalEpochs = totalEpochs; - this->currEpoch = currEpoch; - } - - void printWithLoss(double l) - { - ++this->currIndex; - std::string ratioStr = std::to_string(this->currIndex) + "/" + std::to_string(this->totalIndexes); - std::string epochStr = "Epoch : " + std::to_string(this->currEpoch) + "/" + std::to_string(this->totalEpochs); - std::string errorStr = "Loss : " + std::to_string(static_cast(l)); - float ratio = static_cast(this->currIndex) / this->totalIndexes; - - auto screen = gaugeBuilder({hbox({ - text(epochStr + " "), - gauge(ratio), - text(" " + ratioStr), - }) | flex | - border, - text(errorStr) | border}); - - std::cout << this->resetPos; - screen.Print(); - this->resetPos = screen.ResetPosition(); - } - - /** - * Print with loss and accuracy - */ - void printWithLAndA(double l, double a) - { - ++this->currIndex; - std::string ratioStr = std::to_string(this->currIndex) + "/" + std::to_string(this->totalIndexes); - std::string epochStr = "Epoch : " + std::to_string(this->currEpoch) + "/" + std::to_string(this->totalEpochs); - std::string errorStr = "Loss : " + std::to_string(static_cast(l)); - std::string accStr = "Accuracy : " + std::to_string(static_cast(a)); - - float ratio = static_cast(this->currIndex) / this->totalIndexes; - - auto screen = gaugeBuilder({hbox({ - text(epochStr + " "), - gauge(ratio), - text(" " + ratioStr), - }) | flex | - border, - text(errorStr) | border, - text(accStr) | border}); - - std::cout << this->resetPos; - screen.Print(); - this->resetPos = screen.ResetPosition(); - }; - - /** - * @brief Create a horizontal document with the given elements - * - * @param elements The elements to be added to the document - * - * @return The screen with the given elements - */ - Screen gaugeBuilder(Elements elements) - { - Element document = hbox(elements); - - Screen screen = Screen::Create(Dimension::Full(), Dimension::Fit(document)); - - Render(screen, document); - - return screen; - } - - private: - std::string resetPos; - int totalEpochs; - int currEpoch; + protected: + int totalIndexes; + int currIndex; + std::string preText; + // string to be printed in order to reset the cursor position + std::string resetPos; + + private: + void printGauge() { + std::string ratioStr = std::to_string(this->currIndex) + "/" + + std::to_string(this->totalIndexes); + float ratio = static_cast(this->currIndex) / this->totalIndexes; + Element document = hbox({ + text(this->preText), + gauge(ratio) | flex, + text(" " + ratioStr), + }); + + auto screen = + Screen::Create(Dimension::Fixed(100), Dimension::Fit(document)); + Render(screen, document); + std::cout << this->resetPos; + screen.Print(); + this->resetPos = screen.ResetPosition(); + }; +}; + +// todo: Refactor this code to not have duplicates +class TrainingGauge : public Gauge { + public: + TrainingGauge(int totalIndexes, int currIndex = 0, int totalEpochs = 10, + int currEpoch = 0) + : Gauge("Training : ", totalIndexes, currIndex) { + this->totalEpochs = totalEpochs; + this->currEpoch = currEpoch; + } + + void printWithLoss(double l) { + ++this->currIndex; + std::string ratioStr = std::to_string(this->currIndex) + "/" + + std::to_string(this->totalIndexes); + std::string epochStr = "Epoch : " + std::to_string(this->currEpoch) + "/" + + std::to_string(this->totalEpochs); + std::string errorStr = "Loss : " + std::to_string(static_cast(l)); + float ratio = static_cast(this->currIndex) / this->totalIndexes; + + auto screen = gaugeBuilder({hbox({ + text(epochStr + " "), + gauge(ratio), + text(" " + ratioStr), + }) | flex | + border, + text(errorStr) | border}); + + std::cout << this->resetPos; + screen.Print(); + this->resetPos = screen.ResetPosition(); + } + + /** + * Print with loss and accuracy + */ + void printWithLAndA(double l, double a) { + ++this->currIndex; + std::string ratioStr = std::to_string(this->currIndex) + "/" + + std::to_string(this->totalIndexes); + std::string epochStr = "Epoch : " + std::to_string(this->currEpoch) + "/" + + std::to_string(this->totalEpochs); + std::string errorStr = "Loss : " + std::to_string(static_cast(l)); + std::string accStr = "Accuracy : " + std::to_string(static_cast(a)); + + float ratio = static_cast(this->currIndex) / this->totalIndexes; + + auto screen = + gaugeBuilder({hbox({ + text(epochStr + " "), + gauge(ratio), + text(" " + ratioStr), + }) | flex | + border, + text(errorStr) | border, text(accStr) | border}); + + std::cout << this->resetPos; + screen.Print(); + this->resetPos = screen.ResetPosition(); }; -} // namespace NeuralNet \ No newline at end of file + + /** + * @brief Create a horizontal document with the given elements + * + * @param elements The elements to be added to the document + * + * @return The screen with the given elements + */ + Screen gaugeBuilder(Elements elements) { + Element document = hbox(elements); + + Screen screen = Screen::Create(Dimension::Full(), Dimension::Fit(document)); + + Render(screen, document); + + return screen; + } + + private: + std::string resetPos; + int totalEpochs; + int currEpoch; +}; +} // namespace NeuralNet \ No newline at end of file diff --git a/src/NeuralNet/utils/Serialize.hpp b/src/NeuralNet/utils/Serialize.hpp index 1d25f18..b61b6e8 100644 --- a/src/NeuralNet/utils/Serialize.hpp +++ b/src/NeuralNet/utils/Serialize.hpp @@ -1,46 +1,48 @@ #pragma once #include -#include #include +#include #include #include + #include "Enums.hpp" -namespace cereal -{ - template - void save(Archive &ar, Eigen::Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> const &m) - { - int32_t rows = m.rows(); - int32_t cols = m.cols(); - ar(rows); - ar(cols); - ar(binary_data(m.data(), rows * cols * sizeof(_Scalar))); - } - - template - void load(Archive &ar, Eigen::Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> &m) - { - int32_t rows; - int32_t cols; - ar(rows); - ar(cols); - - m.resize(rows, cols); - - ar(binary_data(m.data(), static_cast(rows * cols * sizeof(_Scalar)))); - } - - template - void save(Archive &ar, const std::tuple &t) - { - ar(std::get<0>(t), std::get<1>(t)); - } - - template - void load(Archive &ar, std::tuple &t) - { - ar(std::get<0>(t), std::get<1>(t)); - } -} // namespace cereal \ No newline at end of file +namespace cereal { +template +void save(Archive &ar, Eigen::Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, + _MaxCols> const &m) { + int32_t rows = m.rows(); + int32_t cols = m.cols(); + ar(rows); + ar(cols); + ar(binary_data(m.data(), rows * cols * sizeof(_Scalar))); +} + +template +void load( + Archive &ar, + Eigen::Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> &m) { + int32_t rows; + int32_t cols; + ar(rows); + ar(cols); + + m.resize(rows, cols); + + ar(binary_data(m.data(), + static_cast(rows * cols * sizeof(_Scalar)))); +} + +template +void save(Archive &ar, const std::tuple &t) { + ar(std::get<0>(t), std::get<1>(t)); +} + +template +void load(Archive &ar, std::tuple &t) { + ar(std::get<0>(t), std::get<1>(t)); +} +} // namespace cereal \ No newline at end of file diff --git a/src/bindings/NeuralNetPy.cpp b/src/bindings/NeuralNetPy.cpp index 3b1a4d9..be044bc 100644 --- a/src/bindings/NeuralNetPy.cpp +++ b/src/bindings/NeuralNetPy.cpp @@ -1,35 +1,34 @@ /** * In this file the essential classes and methods are exposed to Python. * Just enough to be able to setup and manipulate the Neural Network - * When the project is built with the PYBIND_BUILD option, it will create a .so file - * in the build folder. + * When the project is built with the PYBIND_BUILD option, it will create a .so + * file in the build folder. */ +#include #include #include #include -#include -#include "Network.hpp" + #include "Model.hpp" #include "Network.cpp" -#include "layers/Layer.hpp" -#include "layers/Flatten.hpp" +#include "Network.hpp" +#include "TemplateBindings.hpp" // Template classes binding functions +#include "callbacks/CSVLogger.hpp" +#include "callbacks/Callback.hpp" +#include "callbacks/EarlyStopping.hpp" #include "layers/Dense.hpp" +#include "layers/Flatten.hpp" +#include "layers/Layer.hpp" #include "optimizers/Optimizer.hpp" #include "optimizers/optimizers.hpp" -#include "callbacks/Callback.hpp" -#include "callbacks/EarlyStopping.hpp" -#include "callbacks/CSVLogger.hpp" -#include "callbacks/CSVLogger.hpp" #include "utils/Enums.hpp" -#include "TemplateBindings.hpp" // Template classes binding functions namespace py = pybind11; using namespace NeuralNet; -PYBIND11_MODULE(NeuralNetPy, m) -{ +PYBIND11_MODULE(NeuralNetPy, m) { m.doc() = R"pbdoc( Neural Network Library ----------------------- @@ -44,7 +43,8 @@ PYBIND11_MODULE(NeuralNetPy, m) .value("SOFTMAX", ACTIVATION::SOFTMAX, "Softmax Activation Function"); py::enum_(m, "WEIGHT_INIT") - .value("RANDOM", WEIGHT_INIT::RANDOM, "Initialize weights with random values") + .value("RANDOM", WEIGHT_INIT::RANDOM, + "Initialize weights with random values") .value("GLOROT", WEIGHT_INIT::GLOROT, R"pbdoc( Initialize weights with Glorot initialization. @@ -88,10 +88,10 @@ PYBIND11_MODULE(NeuralNetPy, m) :param alpha: The learning rate, defaults to 0.001 :type alpha: float )pbdoc") - .def(py::init(), - py::arg("alpha") = 0.001); + .def(py::init(), py::arg("alpha") = 0.001); - py::class_>(optimizers_m, "Adam", R"pbdoc( + py::class_>(optimizers_m, "Adam", + R"pbdoc( For more information on `Adam optimizer ` :param alpha: The learning rate, defaults to 0.001 @@ -103,10 +103,8 @@ PYBIND11_MODULE(NeuralNetPy, m) :param epsilon: A small constant for numerical stability, defaults to 10E-8 :type epsilon: float )pbdoc") - .def(py::init(), - py::arg("alpha") = 0.001, - py::arg("beta1") = 0.9, - py::arg("beta2") = 0.999, + .def(py::init(), py::arg("alpha") = 0.001, + py::arg("beta1") = 0.9, py::arg("beta2") = 0.999, py::arg("epsilon") = 10E-8); py::module layers_m = m.def_submodule("layers", R"pbdoc( @@ -122,7 +120,10 @@ PYBIND11_MODULE(NeuralNetPy, m) )pbdoc"); py::class_>(layers_m, "Layer") - .def(py::init(), py::arg("nNeurons"), py::arg("activationFunc") = ACTIVATION::SIGMOID, py::arg("weightInit") = WEIGHT_INIT::RANDOM, py::arg("bias") = 0, "This is a simple test") + .def(py::init(), py::arg("nNeurons"), + py::arg("activationFunc") = ACTIVATION::SIGMOID, + py::arg("weightInit") = WEIGHT_INIT::RANDOM, py::arg("bias") = 0, + "This is a simple test") .def("getNumNeurons", &Layer::getNumNeurons); py::class_>(layers_m, "Dense", R"pbdoc( @@ -145,13 +146,12 @@ PYBIND11_MODULE(NeuralNetPy, m) layer = NNP.layers.Dense(3, NNP.ACTIVATION.RELU, NNP.WEIGHT_INIT.HE) )pbdoc") - .def(py::init(), - py::arg("nNeurons"), + .def(py::init(), py::arg("nNeurons"), py::arg("activationFunc") = ACTIVATION::SIGMOID, - py::arg("weightInit") = WEIGHT_INIT::RANDOM, - py::arg("bias") = 0); + py::arg("weightInit") = WEIGHT_INIT::RANDOM, py::arg("bias") = 0); - py::class_>(layers_m, "Flatten", R"pbdoc( + py::class_>(layers_m, "Flatten", + R"pbdoc( Initializes a ``Flatten`` layer. The sole purpose of this layer is to vectorize matrix inputs like images. :param inputShape: The shape of the input matrix (rows, cols or number of pixels per row and column in the case of images) @@ -164,11 +164,11 @@ PYBIND11_MODULE(NeuralNetPy, m) layer = NNP.layers.Flatten((3, 3)) )pbdoc") - .def(py::init>(), - py::arg("inputShape")); + .def(py::init>(), py::arg("inputShape")); py::bind_vector>>(layers_m, "VectorLayer"); - py::bind_vector>>(layers_m, "VectorFlatten"); + py::bind_vector>>(layers_m, + "VectorFlatten"); py::bind_vector>>(layers_m, "VectorDense"); py::module callbacks_m = m.def_submodule("callbacks", R"pbdoc( @@ -184,11 +184,13 @@ PYBIND11_MODULE(NeuralNetPy, m) :recursive: )pbdoc"); - py::class_>(callbacks_m, "Callback", R"pbdoc( + py::class_>(callbacks_m, "Callback", + R"pbdoc( This is the base class for all callbacks. )pbdoc"); - py::class_>(callbacks_m, "EarlyStopping", R"pbdoc( + py::class_>( + callbacks_m, "EarlyStopping", R"pbdoc( Initializes an ``EarlyStopping`` callback. This callback will stop the training if the given metric doesn't improve more than the given delta over a certain number of epochs (patience). :param metric: The metric to be monitored (Either ``LOSS`` or ``ACCURACY``), defaults to ``LOSS`` @@ -204,12 +206,11 @@ PYBIND11_MODULE(NeuralNetPy, m) network.train(inputs, labels, 100, [NNP.callbacks.EarlyStopping("loss", 0.01, 10)]) )pbdoc") - .def(py::init(), - py::arg("metric") = "LOSS", - py::arg("minDelta") = 0.01, - py::arg("patience") = 0); + .def(py::init(), py::arg("metric") = "LOSS", + py::arg("minDelta") = 0.01, py::arg("patience") = 0); - py::class_>(callbacks_m, "CSVLogger", R"pbdoc( + py::class_>( + callbacks_m, "CSVLogger", R"pbdoc( Initializes a ``CSVLogger`` callback. This callback will log the training process in a CSV file. .. highlight: python @@ -218,16 +219,19 @@ PYBIND11_MODULE(NeuralNetPy, m) network.train(inputs, labels, 100, [NNP.callbacks.CSVLogger("logs.csv")]) )pbdoc") - .def(py::init(), - py::arg("filename"), + .def(py::init(), py::arg("filename"), py::arg("separator") = ","); - py::bind_vector>>(callbacks_m, "VectorCallback"); - py::bind_vector>>(callbacks_m, "VectorEarlyStopping"); - py::bind_vector>>(callbacks_m, "VectorCSVLogger"); + py::bind_vector>>(callbacks_m, + "VectorCallback"); + py::bind_vector>>( + callbacks_m, "VectorEarlyStopping"); + py::bind_vector>>(callbacks_m, + "VectorCSVLogger"); // TrainingData with 2 dimensional inputs - bindTrainingData>, std::vector>(m, "TrainingData2dI", R"pbdoc( + bindTrainingData>, std::vector>( + m, "TrainingData2dI", R"pbdoc( Represents training data with 2 dimensional inputs (vectors). This class is supposed to bring the table some methods to easily manipulate the data and prepare it for training. .. highlight: python @@ -253,7 +257,8 @@ PYBIND11_MODULE(NeuralNetPy, m) )pbdoc"); // TrainingData with 3 dimensional inputs - bindTrainingData>>, std::vector>(m, "TrainingData3dI", R"pbdoc( + bindTrainingData>>, + std::vector>(m, "TrainingData3dI", R"pbdoc( Represents training data with 3 dimensional inputs (matrices). This class is supposed to bring the table some methods to easily manipulate the data and prepare it for training. .. highlight: python @@ -290,7 +295,8 @@ PYBIND11_MODULE(NeuralNetPy, m) * * https://github.com/pybind/pybind11/issues/199#issuecomment-220302516 * - * This is why I had to specify the type "Network", I'll have to do so for every type added + * This is why I had to specify the type "Network", I'll have to do so for + * every type added */ py::module models_m = m.def_submodule("models", R"pbdoc( @@ -361,9 +367,7 @@ PYBIND11_MODULE(NeuralNetPy, m) network.setup(optimizer=NNP.SGD(0.01), loss=NNP.LOSS.MCQ) )pbdoc") .def(py::init<>()) - .def("setup", - &Network::setup, - py::arg("optimizer"), + .def("setup", &Network::setup, py::arg("optimizer"), py::arg("loss") = LOSS::QUADRATIC) .def("addLayer", &Network::addLayer, R"pbdoc( Add a layer to the network. @@ -386,9 +390,7 @@ PYBIND11_MODULE(NeuralNetPy, m) .. danger:: Under no circumstances you should add a ``Flatten`` layer as a hidden layer. )pbdoc") - .def("getLayer", - &Network::getLayer, - py::return_value_policy::copy, + .def("getLayer", &Network::getLayer, py::return_value_policy::copy, R"pbdoc( Get a layer from the network by it's index. They're 0-indexed. @@ -409,8 +411,13 @@ PYBIND11_MODULE(NeuralNetPy, m) layer = network.getLayer(1) # Return Dense layer with 2 neurons )pbdoc") - .def("getNumLayers", &Network::getNumLayers, "Return the number of layers in the network.") - .def("train", static_cast>, std::vector, int, const std::vector>)>(&Network::train), R"pbdoc( + .def("getNumLayers", &Network::getNumLayers, + "Return the number of layers in the network.") + .def("train", + static_cast>, std::vector, int, + const std::vector>)>(&Network::train), + R"pbdoc( Train the network by passing it 2 dimensional inputs (vectors). :param inputs: A list of vectors representing the inputs @@ -446,7 +453,12 @@ PYBIND11_MODULE(NeuralNetPy, m) loss = network.train(inputs, labels, 10) )pbdoc") - .def("train", static_cast>>, std::vector, int, const std::vector>)>(&Network::train), R"pbdoc( + .def("train", + static_cast>>, + std::vector, int, + const std::vector>)>(&Network::train), + R"pbdoc( Train the network by passing it a list of 3 dimensional inputs (matrices). :param inputs: A list of matrices representing the inputs @@ -488,7 +500,13 @@ PYBIND11_MODULE(NeuralNetPy, m) loss = network.train(inputs, labels, 10) )pbdoc") - .def("train", static_cast>, std::vector>, int, const std::vector>)>(&Network::train), R"pbdoc( + .def("train", + static_cast>, + std::vector>, + int, const std::vector>)>( + &Network::train), + R"pbdoc( Train the network by passing it a ``TrainingData2dI`` object. :param trainingData: A ``TrainingData2dI`` object @@ -527,7 +545,13 @@ PYBIND11_MODULE(NeuralNetPy, m) loss = network.train(trainingData, 10) )pbdoc") - .def("train", static_cast>>, std::vector>, int, const std::vector>)>(&Network::train), R"pbdoc( + .def("train", + static_cast>>, + std::vector>, + int, const std::vector>)>( + &Network::train), + R"pbdoc( Train the network by passing it a ``TrainingData3dI`` object. :param trainingData: A ``TrainingData3dI`` object @@ -578,7 +602,10 @@ PYBIND11_MODULE(NeuralNetPy, m) loss = network.train(trainingData, 10) )pbdoc") - .def("predict", static_cast>)>(&Network::predict), R"pbdoc( + .def("predict", + static_cast>)>(&Network::predict), + R"pbdoc( Feed forward the given inputs through the network and return the predictions/outputs. :param inputs: A list of vectors representing the inputs @@ -586,7 +613,11 @@ PYBIND11_MODULE(NeuralNetPy, m) :return: A matrix representing the outputs of the network for the given inputs :rtype: numpy.ndarray )pbdoc") - .def("predict", static_cast>>)>(&Network::predict), R"pbdoc( + .def("predict", + static_cast>>)>( + &Network::predict), + R"pbdoc( Feed forward the given inputs through the network and return the predictions/outputs. :param inputs: A list of vectors representing the inputs diff --git a/src/bindings/TemplateBindings.hpp b/src/bindings/TemplateBindings.hpp index eb49a93..c3eea10 100644 --- a/src/bindings/TemplateBindings.hpp +++ b/src/bindings/TemplateBindings.hpp @@ -1,15 +1,15 @@ #include + #include "data/TrainingData.hpp" namespace py = pybind11; template -void bindTrainingData(py::module &m, const char *name, const char *docstring) -{ +void bindTrainingData(py::module &m, const char *name, const char *docstring) { py::class_>(m, name) - .def(py::init(), - py::arg("inputs_data"), - py::arg("labels_data"), - docstring) - .def("batch", &TrainingData::batch, py::arg("batchSize"), "This method will separate the inputs and labels data into batches of the specified size"); + .def(py::init(), py::arg("inputs_data"), + py::arg("labels_data"), docstring) + .def("batch", &TrainingData::batch, py::arg("batchSize"), + "This method will separate the inputs and labels data into batches " + "of the specified size"); }; \ No newline at end of file diff --git a/tests/test-activations.cpp b/tests/test-activations.cpp index 7b231fd..5d41850 100644 --- a/tests/test-activations.cpp +++ b/tests/test-activations.cpp @@ -1,43 +1,34 @@ -#include -#include #include #include #include +#include +#include + #include "test-macros.hpp" using Eigen::MatrixXd; -TEST_CASE("Relu activates correctly", "[function]") -{ +TEST_CASE("Relu activates correctly", "[function]") { MatrixXd inputs(4, 1); MatrixXd expectedOutputs(4, 1); - inputs << -1, - 0, - 4, - 10; + inputs << -1, 0, 4, 10; - expectedOutputs << 0, - 0, - 4, - 10; + expectedOutputs << 0, 0, 4, 10; CHECK(NeuralNet::Relu::activate(inputs) == expectedOutputs); } -TEST_CASE("Relu differentiate correctly", "[function]") -{ +TEST_CASE("Relu differentiate correctly", "[function]") { const int num_cols = 2; const int num_rows = 2; MatrixXd test_m1 = MatrixXd::Random(num_rows, num_cols); MatrixXd expected_m1(num_rows, num_cols); - for (int r = 0; r < num_rows; r++) - { - for (int c = 0; c < num_cols; c++) - { + for (int r = 0; r < num_rows; r++) { + for (int c = 0; c < num_cols; c++) { expected_m1(r, c) = test_m1(r, c) > 0 ? 1 : 0; } } @@ -45,38 +36,29 @@ TEST_CASE("Relu differentiate correctly", "[function]") CHECK(NeuralNet::Relu::diff(test_m1) == expected_m1); } -TEST_CASE("Sigmoid Activates correctly", "[function]") -{ +TEST_CASE("Sigmoid Activates correctly", "[function]") { MatrixXd inputs(4, 1); MatrixXd expectedOutputs(4, 1); - inputs << -7.66, - -1, - 0, - 10; + inputs << -7.66, -1, 0, 10; // Pre-calculated - expectedOutputs << 0, - 0.268, - 0.5, - 1; + expectedOutputs << 0, 0.268, 0.5, 1; - CHECK_MATRIX_APPROX(NeuralNet::Sigmoid::activate(inputs), expectedOutputs, EPSILON); + CHECK_MATRIX_APPROX(NeuralNet::Sigmoid::activate(inputs), expectedOutputs, + EPSILON); } -TEST_CASE("Sigmoid differentiates correctly", "[function]") -{ +TEST_CASE("Sigmoid differentiates correctly", "[function]") { const int num_rows = 2; const int num_cols = 2; MatrixXd test_m1 = MatrixXd::Random(num_rows, num_cols); MatrixXd expected_m1(num_rows, num_cols); - for (int r = 0; r < num_rows; r++) - { - for (int c = 0; c < num_cols; c++) - { + for (int r = 0; r < num_rows; r++) { + for (int c = 0; c < num_cols; c++) { expected_m1(r, c) = test_m1(r, c) * (1 - test_m1(r, c)); } } @@ -84,24 +66,21 @@ TEST_CASE("Sigmoid differentiates correctly", "[function]") CHECK(NeuralNet::Sigmoid::diff(test_m1) == expected_m1); } -TEST_CASE("Softmax activates correctly", "[function]") -{ +TEST_CASE("Softmax activates correctly", "[function]") { /** * Expected outputs should be updated because Softmax now scales the inputs */ MatrixXd inputs(3, 3); - inputs << 0.5, 1.3, -5, - 0.7, 0.3, 1.5, - -0.5, -1.3, -5; + inputs << 0.5, 1.3, -5, 0.7, 0.3, 1.5, -0.5, -1.3, -5; MatrixXd expectedOutputs(3, 3); - expectedOutputs << 0.30963321, 0.68910139, 0.0012654, - 0.25668267, 0.17205954, 0.57125779, - 0.68472611, 0.30766727, 0.00760662; + expectedOutputs << 0.30963321, 0.68910139, 0.0012654, 0.25668267, 0.17205954, + 0.57125779, 0.68472611, 0.30766727, 0.00760662; - CHECK_MATRIX_APPROX(NeuralNet::Softmax::activate(inputs), expectedOutputs, EPSILON); + CHECK_MATRIX_APPROX(NeuralNet::Softmax::activate(inputs), expectedOutputs, + EPSILON); } // TEST_CASE("Softmax differentiates correctly", "[function]") @@ -116,5 +95,6 @@ TEST_CASE("Softmax activates correctly", "[function]") // MatrixXd activatedInputs = NeuralNet::Softmax::activate(inputs); -// CHECK_MATRIX_APPROX(NeuralNet::Softmax::diff(activatedInputs), expectedOutputs, EPSILON); +// CHECK_MATRIX_APPROX(NeuralNet::Softmax::diff(activatedInputs), +// expectedOutputs, EPSILON); // } \ No newline at end of file diff --git a/tests/test-callbacks.cpp b/tests/test-callbacks.cpp index 6d56391..707dde7 100644 --- a/tests/test-callbacks.cpp +++ b/tests/test-callbacks.cpp @@ -1,21 +1,26 @@ -#include #include #include +#include #include using namespace NeuralNet; -TEST_CASE("EarlyStopping callback throws exception when the metric is not found", "[callback]") -{ - std::shared_ptr earlyStopping = std::make_shared("LOSS", 0.1); +TEST_CASE( + "EarlyStopping callback throws exception when the metric is not found", + "[callback]") { + std::shared_ptr earlyStopping = + std::make_shared("LOSS", 0.1); std::unordered_map logs = {{"TEST", 0.2}}; REQUIRE_THROWS(Callback::callMethod(earlyStopping, "onEpochEnd", logs)); } -TEST_CASE("EarlyStopping callback throws exception when metric does not more than the given delta", "[callback]") -{ - std::shared_ptr earlyStopping = std::make_shared("LOSS", 0.1); +TEST_CASE( + "EarlyStopping callback throws exception when metric does not more than " + "the given delta", + "[callback]") { + std::shared_ptr earlyStopping = + std::make_shared("LOSS", 0.1); std::unordered_map logs = {{"LOSS", 0.2}}; std::unordered_map logs2 = {{"LOSS", 0.2}}; diff --git a/tests/test-functions.cpp b/tests/test-functions.cpp index bd8de91..fb3525f 100644 --- a/tests/test-functions.cpp +++ b/tests/test-functions.cpp @@ -1,14 +1,14 @@ #include #include -#include #include +#include + #include "test-macros.hpp" using namespace Catch::Matchers; using namespace NeuralNet; -TEST_CASE("Sqr function returns the right square", "[helper_function]") -{ +TEST_CASE("Sqr function returns the right square", "[helper_function]") { CHECK(sqr(0) == 0); CHECK(sqr(2) == 4); CHECK(sqr(10) == 100); @@ -16,24 +16,18 @@ TEST_CASE("Sqr function returns the right square", "[helper_function]") CHECK(sqr(657666) == 432524567556); } -TEST_CASE("vectorToMatrixXd outputs the correct format", "[helper_function]") -{ - std::vector> v = { - {1, 2, 3}, - {4, 5, 6}, - {7, 8, 9}}; +TEST_CASE("vectorToMatrixXd outputs the correct format", "[helper_function]") { + std::vector> v = {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}; Eigen::MatrixXd expected(3, 3); - expected << 1, 2, 3, - 4, 5, 6, - 7, 8, 9; + expected << 1, 2, 3, 4, 5, 6, 7, 8, 9; CHECK(vectorToMatrixXd(v) == expected); } -TEST_CASE("randomWeightInit initializes value properly", "[weight_initialization]") -{ +TEST_CASE("randomWeightInit initializes value properly", + "[weight_initialization]") { Eigen::MatrixXd weights = Eigen::MatrixXd::Zero(5, 5); double min = -2.0; double max = 2.0; @@ -42,8 +36,8 @@ TEST_CASE("randomWeightInit initializes value properly", "[weight_initialization CHECK_MATRIX_VALUES_IN_RANGE(weights, min, max); } -TEST_CASE("randomDistMatrixInit initializes values properly", "[weight_initialization]") -{ +TEST_CASE("randomDistMatrixInit initializes values properly", + "[weight_initialization]") { Eigen::MatrixXd weights = Eigen::MatrixXd::Zero(5, 5); double mean = -1.0; double stddev = 0; diff --git a/tests/test-layer.cpp b/tests/test-layer.cpp index b38e13c..eac4246 100644 --- a/tests/test-layer.cpp +++ b/tests/test-layer.cpp @@ -1,23 +1,20 @@ -#include -#include #include #include +#include #include +#include using namespace NeuralNet; -SCENARIO("Flatten's flatten() works as expected") -{ +SCENARIO("Flatten's flatten() works as expected") { Flatten flattenLayer({2, 3}); std::vector>> inputs = { - {{1, 2, 3}, {4, 5, 6}}, - {{6, 5, 4}, {3, 2, 1}}}; + {{1, 2, 3}, {4, 5, 6}}, {{6, 5, 4}, {3, 2, 1}}}; Eigen::MatrixXd expected(2, 6); - expected << 1, 2, 3, 4, 5, 6, - 6, 5, 4, 3, 2, 1; + expected << 1, 2, 3, 4, 5, 6, 6, 5, 4, 3, 2, 1; Eigen::MatrixXd outputs = flattenLayer.flatten(inputs); diff --git a/tests/test-macros.hpp b/tests/test-macros.hpp index 059b5af..9405112 100644 --- a/tests/test-macros.hpp +++ b/tests/test-macros.hpp @@ -1,33 +1,28 @@ #pragma once +#include #include -#include -#include #include -#include +#include +#include #define EPSILON 1e-3 -void REQUIRE_MATRIX_VALUES_UNIQUE(Eigen::MatrixXd &matrix) -{ +void REQUIRE_MATRIX_VALUES_UNIQUE(Eigen::MatrixXd &matrix) { std::set seenValues; - for (int i = 0; i < matrix.rows(); i++) - { - for (int j = 0; j < matrix.cols(); j++) - { + for (int i = 0; i < matrix.rows(); i++) { + for (int j = 0; j < matrix.cols(); j++) { REQUIRE(seenValues.find(matrix(i, j)) == seenValues.end()); seenValues.insert(matrix(i, j)); } } } -void CHECK_MATRIX_VALUES_IN_RANGE(Eigen::MatrixXd matrix, double min, double max) -{ - for (int i = 0; i < matrix.rows(); i++) - { - for (int j = 0; j < matrix.rows(); j++) - { +void CHECK_MATRIX_VALUES_IN_RANGE(Eigen::MatrixXd matrix, double min, + double max) { + for (int i = 0; i < matrix.rows(); i++) { + for (int j = 0; j < matrix.rows(); j++) { REQUIRE(matrix(i, j) < max); REQUIRE(matrix(i, j) > min); } @@ -35,14 +30,12 @@ void CHECK_MATRIX_VALUES_IN_RANGE(Eigen::MatrixXd matrix, double min, double max } // Custom test assertion to check it two matrices are approximately equal -void CHECK_MATRIX_APPROX(const Eigen::MatrixXd &matA, const Eigen::MatrixXd &matB, double epsilon = 1e-6) -{ +void CHECK_MATRIX_APPROX(const Eigen::MatrixXd &matA, + const Eigen::MatrixXd &matB, double epsilon = 1e-6) { assert(matA.rows() == matB.rows() && matA.cols() == matB.cols()); - for (int i = 0; i < matA.rows(); ++i) - { - for (int j = 0; j < matA.cols(); ++j) - { + for (int i = 0; i < matA.rows(); ++i) { + for (int j = 0; j < matA.cols(); ++j) { CHECK(std::abs(matA(i, j) - matB(i, j)) < epsilon); } } diff --git a/tests/test-network.cpp b/tests/test-network.cpp index 8fd7874..5c8833a 100644 --- a/tests/test-network.cpp +++ b/tests/test-network.cpp @@ -1,28 +1,28 @@ -#include #include #include +#include #include + #include "test-macros.hpp" using namespace NeuralNet; -SCENARIO("Basic small network functions") -{ - GIVEN("A small neural network") - { - Network sn; // sn - small network +SCENARIO("Basic small network functions") { + GIVEN("A small neural network") { + Network sn; // sn - small network std::shared_ptr optimizer = std::make_shared(1); sn.setup(optimizer, LOSS::QUADRATIC); - std::shared_ptr layer1 = std::make_shared(2, ACTIVATION::RELU, WEIGHT_INIT::HE); - std::shared_ptr layer2 = std::make_shared(2, ACTIVATION::SIGMOID, WEIGHT_INIT::GLOROT); + std::shared_ptr layer1 = + std::make_shared(2, ACTIVATION::RELU, WEIGHT_INIT::HE); + std::shared_ptr layer2 = + std::make_shared(2, ACTIVATION::SIGMOID, WEIGHT_INIT::GLOROT); sn.addLayer(layer1); sn.addLayer(layer2); - WHEN("Network trained") - { + WHEN("Network trained") { std::vector> trainingInputs = {{1.0, 1.0}}; std::vector label = {1}; @@ -35,43 +35,35 @@ SCENARIO("Basic small network functions") } } -SCENARIO("Layers are initialized correctly in the network") -{ - GIVEN("An empty network") - { +SCENARIO("Layers are initialized correctly in the network") { + GIVEN("An empty network") { Network network; std::shared_ptr optimizer = std::make_shared(1); // Setting up the parameters network.setup(optimizer, LOSS::QUADRATIC); - THEN("Number layers == 0") - { - REQUIRE(network.getNumLayers() == 0); - } + THEN("Number layers == 0") { REQUIRE(network.getNumLayers() == 0); } - WHEN("3 layers are added") - { - std::shared_ptr layer1 = std::make_shared(2, ACTIVATION::RELU, WEIGHT_INIT::GLOROT); - std::shared_ptr layer2 = std::make_shared(3, ACTIVATION::RELU, WEIGHT_INIT::GLOROT); - std::shared_ptr layer3 = std::make_shared(1, ACTIVATION::RELU, WEIGHT_INIT::GLOROT); + WHEN("3 layers are added") { + std::shared_ptr layer1 = + std::make_shared(2, ACTIVATION::RELU, WEIGHT_INIT::GLOROT); + std::shared_ptr layer2 = + std::make_shared(3, ACTIVATION::RELU, WEIGHT_INIT::GLOROT); + std::shared_ptr layer3 = + std::make_shared(1, ACTIVATION::RELU, WEIGHT_INIT::GLOROT); network.addLayer(layer1); network.addLayer(layer2); network.addLayer(layer3); - THEN("Number layer == 3") - { - REQUIRE(network.getNumLayers() == 3); - } + THEN("Number layer == 3") { REQUIRE(network.getNumLayers() == 3); } - THEN("Right number neurons in layers") - { + THEN("Right number neurons in layers") { CHECK(layer1->getNumNeurons() == 2); CHECK(layer2->getNumNeurons() == 3); } - THEN("Weights matrices have correct sizes") - { + THEN("Weights matrices have correct sizes") { Eigen::MatrixXd weightsL2 = layer2->getWeights(); Eigen::MatrixXd weightsL3 = layer3->getWeights(); @@ -88,43 +80,41 @@ SCENARIO("Layers are initialized correctly in the network") REQUIRE(weightsL2.rows() == layer1->getNumNeurons()); } - THEN("Weights are not initialized to 0") - { + THEN("Weights are not initialized to 0") { Eigen::MatrixXd weightsL2 = layer2->getWeights(); Eigen::MatrixXd weightsL3 = layer3->getWeights(); - REQUIRE(weightsL2 != Eigen::MatrixXd::Zero(weightsL2.rows(), weightsL2.cols())); - REQUIRE(weightsL3 != Eigen::MatrixXd::Zero(weightsL3.rows(), weightsL3.cols())); + REQUIRE(weightsL2 != + Eigen::MatrixXd::Zero(weightsL2.rows(), weightsL2.cols())); + REQUIRE(weightsL3 != + Eigen::MatrixXd::Zero(weightsL3.rows(), weightsL3.cols())); } } } } -SCENARIO("The network remains the same when trained with null inputs") -{ +SCENARIO("The network remains the same when trained with null inputs") { // Limiting the network with 1 epoch Network network; std::shared_ptr optimizer = std::make_shared(1); // Setting up the parameters network.setup(optimizer, LOSS::QUADRATIC); - std::shared_ptr inputLayer = std::make_shared(2, ACTIVATION::RELU, WEIGHT_INIT::GLOROT); - std::shared_ptr hiddenLayer = std::make_shared(3, ACTIVATION::RELU, WEIGHT_INIT::GLOROT); - std::shared_ptr outputLayer = std::make_shared(1, ACTIVATION::RELU, WEIGHT_INIT::GLOROT); + std::shared_ptr inputLayer = + std::make_shared(2, ACTIVATION::RELU, WEIGHT_INIT::GLOROT); + std::shared_ptr hiddenLayer = + std::make_shared(3, ACTIVATION::RELU, WEIGHT_INIT::GLOROT); + std::shared_ptr outputLayer = + std::make_shared(1, ACTIVATION::RELU, WEIGHT_INIT::GLOROT); network.addLayer(inputLayer); network.addLayer(hiddenLayer); network.addLayer(outputLayer); - GIVEN("A network with 3 layers") - { - THEN("Number of layers = 3") - { - REQUIRE(network.getNumLayers() == 3); - } + GIVEN("A network with 3 layers") { + THEN("Number of layers = 3") { REQUIRE(network.getNumLayers() == 3); } - WHEN("Null inputs are passed") - { + WHEN("Null inputs are passed") { std::vector> nullInputs = {{0, 0}}; std::vector labels = {0}; @@ -135,15 +125,14 @@ SCENARIO("The network remains the same when trained with null inputs") // Training with null weights and inputs network.train(nullInputs, labels, 2); - THEN("Outputs are 0") - { + THEN("Outputs are 0") { Eigen::MatrixXd outputs = network.getOutputLayer()->getOutputs(); - REQUIRE(outputs == Eigen::MatrixXd::Zero(outputs.rows(), outputs.cols())); + REQUIRE(outputs == + Eigen::MatrixXd::Zero(outputs.rows(), outputs.cols())); } - AND_THEN("The weights remain the same") - { + AND_THEN("The weights remain the same") { CHECK(network.getLayer(1)->getWeights() == preTrainW1); CHECK(network.getLayer(2)->getWeights() == preTrainW2); } @@ -151,38 +140,34 @@ SCENARIO("The network remains the same when trained with null inputs") } } -SCENARIO("The network updates the weights and biases as pre-calculated") -{ +SCENARIO("The network updates the weights and biases as pre-calculated") { Network network; std::shared_ptr sgdOptimizer = std::make_shared(1.5); network.setup(sgdOptimizer, LOSS::QUADRATIC); - std::shared_ptr inputLayer = std::make_shared(3, ACTIVATION::RELU); - std::shared_ptr hiddenLayer = std::make_shared(3, ACTIVATION::RELU, WEIGHT_INIT::CONSTANT); - std::shared_ptr outputLayer = std::make_shared(2, ACTIVATION::SIGMOID, WEIGHT_INIT::CONSTANT); + std::shared_ptr inputLayer = + std::make_shared(3, ACTIVATION::RELU); + std::shared_ptr hiddenLayer = + std::make_shared(3, ACTIVATION::RELU, WEIGHT_INIT::CONSTANT); + std::shared_ptr outputLayer = + std::make_shared(2, ACTIVATION::SIGMOID, WEIGHT_INIT::CONSTANT); network.addLayer(inputLayer); network.addLayer(hiddenLayer); network.addLayer(outputLayer); std::vector> inputs = { - {0.7, 0.3, 0.1}, - {0.5, 0.3, 0.1}, - {1.0, 0.2, 0.4}, - {-0.5, 0.3, -1}}; + {0.7, 0.3, 0.1}, {0.5, 0.3, 0.1}, {1.0, 0.2, 0.4}, {-0.5, 0.3, -1}}; std::vector labels = {1, 1, 0, 1}; - WHEN("Predicting without training") - { + WHEN("Predicting without training") { Eigen::MatrixXd predictions = network.predict(inputs); Eigen::MatrixXd expectedPredictions(4, 2); - expectedPredictions << 0.96442881, 0.96442881, - 0.93702664, 0.93702664, - 0.99183743, 0.99183743, - 0.5, 0.5; + expectedPredictions << 0.96442881, 0.96442881, 0.93702664, 0.93702664, + 0.99183743, 0.99183743, 0.5, 0.5; CHECK_MATRIX_APPROX(predictions, expectedPredictions, EPSILON); } @@ -199,8 +184,7 @@ SCENARIO("The network updates the weights and biases as pre-calculated") // Expected Weights Hidden Layer Eigen::MatrixXd EWHL(3, 3); - EWHL << 0.90743867, 0.90743867, 0.90743867, - 0.9583673, 0.9583673, 0.9583673, + EWHL << 0.90743867, 0.90743867, 0.90743867, 0.9583673, 0.9583673, 0.9583673, 0.97931547, 0.97931547, 0.97931547; // Expected Biases Hidden Layer @@ -211,9 +195,8 @@ SCENARIO("The network updates the weights and biases as pre-calculated") // Expected Weights Output Layer Eigen::MatrixXd EWOL(3, 2); - EWOL << 0.87250736, 0.97733271, - 0.87250736, 0.97733271, - 0.87250736, 0.97733271; + EWOL << 0.87250736, 0.97733271, 0.87250736, 0.97733271, 0.87250736, + 0.97733271; // Expected Biases Output Layer Eigen::MatrixXd EBOL(1, 2); @@ -225,15 +208,12 @@ SCENARIO("The network updates the weights and biases as pre-calculated") CHECK_MATRIX_APPROX(oLayer->getWeights(), EWOL, EPSILON); CHECK_MATRIX_APPROX(oLayer->getBiases(), EBOL, EPSILON); - WHEN("Predicting after training") - { + WHEN("Predicting after training") { Eigen::MatrixXd predictions = network.predict(inputs); Eigen::MatrixXd expectedPredictions(4, 2); - expectedPredictions << 0.87919928, 0.93926274, - 0.8190347, 0.90082421, - 0.96141716, 0.98396999, - 0.42418036, 0.54310411; + expectedPredictions << 0.87919928, 0.93926274, 0.8190347, 0.90082421, + 0.96141716, 0.98396999, 0.42418036, 0.54310411; CHECK_MATRIX_APPROX(predictions, expectedPredictions, EPSILON); } diff --git a/tests/test-optimizers.cpp b/tests/test-optimizers.cpp index 1504c0f..e5ed2a8 100644 --- a/tests/test-optimizers.cpp +++ b/tests/test-optimizers.cpp @@ -1,20 +1,17 @@ -#include #include +#include #include #include using namespace NeuralNet; -SCENARIO("Testing SGD Optimizer") -{ +SCENARIO("Testing SGD Optimizer") { SGD optimizer(1); - GIVEN("Weights that initialized to 1") - { + GIVEN("Weights that initialized to 1") { Eigen::MatrixXd weights = Eigen::MatrixXd::Constant(2, 2, 1); - GIVEN("Gradients equal to the weights") - { + GIVEN("Gradients equal to the weights") { Eigen::MatrixXd weightsGrad = Eigen::MatrixXd::Constant(2, 2, 1); optimizer.updateWeights(weights, weightsGrad); diff --git a/tests/test-tensor.cpp b/tests/test-tensor.cpp index a6cfaf6..21e2a78 100644 --- a/tests/test-tensor.cpp +++ b/tests/test-tensor.cpp @@ -3,10 +3,8 @@ using namespace NeuralNet; -SCENARIO("Tensor batches data correctly") -{ - GIVEN("A vector of ints") - { +SCENARIO("Tensor batches data correctly") { + GIVEN("A vector of ints") { std::vector data = {1, 2, 3, 4, 5, 6, 7}; Tensor t(data); @@ -21,13 +19,9 @@ SCENARIO("Tensor batches data correctly") CHECK(batches[1] == std::vector{3, 4}); }; - GIVEN("A vector of vectors of ints") - { + GIVEN("A vector of vectors of ints") { std::vector> data = { - {1, 2, 3}, - {1, 3, 4}, - {1, 3, 5}, - {1, 3, 4}}; + {1, 2, 3}, {1, 3, 4}, {1, 3, 5}, {1, 3, 4}}; NeuralNet::Tensor t(data); From 34a2784a1aabef5dd6c4f13ab06aa08e8035eb38 Mon Sep 17 00:00:00 2001 From: Az-r-ow Date: Wed, 6 Mar 2024 18:58:48 +0100 Subject: [PATCH 2/3] docs: Contribution docs --- .prettierrc | 4 ++++ contribution/README.md | 6 ++++++ 2 files changed, 10 insertions(+) create mode 100644 .prettierrc diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..222861c --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "tabWidth": 2, + "useTabs": false +} diff --git a/contribution/README.md b/contribution/README.md index bd29a3e..3f5ce3f 100644 --- a/contribution/README.md +++ b/contribution/README.md @@ -11,3 +11,9 @@ For the main part following whatever's written [here](https://www.conventionalco ### Non Functional Commit : Non functional commits are when a feature is still under development but you had to create a commit. For example, if you're working on feature A at home on your PC, and all of a sudden you have to save you're changes so you can resume working at a Starbucks on your laptop. You create a commit with the tag : nfc(A) - A should be replaced with the name of the feature (which is the name of the branch). + +## Code Style : + +I used [google](https://google.github.io/styleguide/cppguide.html)'s style for formatting the code, however I don't apply all google's C++ style guides in the project. + +When contributing make sure you format your code with `clang-format`. If you're on VS Code setup the `clang-format` extension and enable `formatOnSave`. From f0b4359b2030110fe58ca22a0c846eeb708b4be9 Mon Sep 17 00:00:00 2001 From: Az-r-ow Date: Wed, 6 Mar 2024 19:19:21 +0100 Subject: [PATCH 3/3] fix: included stdexcep header --- src/NeuralNet/callbacks/CSVLogger.hpp | 1 + src/NeuralNet/callbacks/Callback.hpp | 1 + src/NeuralNet/callbacks/EarlyStopping.hpp | 1 + 3 files changed, 3 insertions(+) diff --git a/src/NeuralNet/callbacks/CSVLogger.hpp b/src/NeuralNet/callbacks/CSVLogger.hpp index 01079f6..2f3aff8 100644 --- a/src/NeuralNet/callbacks/CSVLogger.hpp +++ b/src/NeuralNet/callbacks/CSVLogger.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include diff --git a/src/NeuralNet/callbacks/Callback.hpp b/src/NeuralNet/callbacks/Callback.hpp index a49a7a7..b2e83cd 100644 --- a/src/NeuralNet/callbacks/Callback.hpp +++ b/src/NeuralNet/callbacks/Callback.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/NeuralNet/callbacks/EarlyStopping.hpp b/src/NeuralNet/callbacks/EarlyStopping.hpp index 77ad428..787dddc 100644 --- a/src/NeuralNet/callbacks/EarlyStopping.hpp +++ b/src/NeuralNet/callbacks/EarlyStopping.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include