From afb19c04fea9850f9486467207483fa6a7bfcccd Mon Sep 17 00:00:00 2001 From: manu Date: Sun, 30 Jun 2024 13:54:36 +0200 Subject: [PATCH 01/11] "Rework playbook management and execution system" Refactored playbook management and execution functionalities in the system. Essential changes include renaming "ansible" routes to "playbook" routes, relocating several playbook related files, and modifying ansible related Python scripts. Added functionality to filter results by file extensions in directory tree listing and created a REST service for managing playbook repositories. --- client/.eslintrc.js | 1 + client/Dockerfile | 2 +- client/config/routes.ts | 4 + client/package.json | 15 +- .../DeviceQuickActionDropDown.tsx | 2 +- .../DeviceQuickActionReference.tsx | 16 +- .../OSSoftwaresVersions/SoftwareIcon.tsx | 2 +- client/src/components/Footer/index.tsx | 2 +- client/src/components/Icons/CustomIcons.tsx | 32 ++ .../PlaybookSelectionModal.tsx | 26 +- .../RegistryComponents/RegistryLogo.tsx | 50 ++ client/src/components/Template/Title.tsx | 2 + .../TerminalModal/TerminalHandler.ts | 2 +- client/src/components/TerminalModal/index.tsx | 35 +- client/src/pages/Admin/Settings/Settings.tsx | 22 +- ...ettings.tsx => AuthenticationSettings.tsx} | 77 +-- .../Settings/components/PlaybooksSettings.tsx | 333 +++++++++++++ .../Settings/components/RegistrySettings.tsx | 446 ++++++------------ .../subcomponents/GitRepositoryModal.tsx | 236 +++++++++ .../subcomponents/LocalRepositoryModal.tsx | 164 +++++++ .../subcomponents/RegistryModal.tsx | 144 ++++++ .../CreateFileInRepositoryModalForm.tsx | 74 +++ .../components/DirectoryTreeView.tsx | 96 ++++ .../components/ExtraVarsViewEditor.tsx | 10 +- .../components/FloatingButtonsBar.tsx | 52 ++ .../Playbooks/components/GalaxyStoreModal.tsx | 2 +- .../components/NewFileDrawerForm.tsx | 151 ++++++ .../components/NewPlaybookModalForm.tsx | 68 --- .../components/PlaybookDropdownMenu.tsx | 112 +++++ .../Playbooks/components/TreeComponent.tsx | 188 ++++++++ client/src/pages/Playbooks/index.tsx | 314 ++++++------ client/src/services/rest/index.ts | 2 +- .../services/rest/playbooks-repositories.ts | 284 +++++++++++ .../rest/{ansible.ts => playbooks.ts} | 112 ++--- docker-compose.dev.yml | 1 + docker-compose.prod.yml | 2 + docker-compose.yml | 2 + server/Dockerfile | 6 +- server/nodemon.json | 4 +- server/package.json | 19 +- .../agent}/_checkDeviceBeforeAdd.json | 1 + .../agent}/_checkDeviceBeforeAdd.yml | 0 .../agent}/_installAgent.json | 1 + .../agent}/_installAgent.yml | 0 .../agent}/_reinstallAgent.json | 1 + .../agent}/_reinstallAgent.yml | 0 .../agent/_restartAgent.json | 4 + .../agent}/_restartAgent.yml | 0 .../agent/_retrieveAgentLogs.json | 4 + .../agent}/_retrieveAgentLogs.yml | 0 .../agent/_uninstallAgent.json | 4 + .../agent}/_uninstallAgent.yml | 0 .../agent/_updateAgent.json | 16 + .../agent/_updateAgent.yml | 89 ++++ .../device/_ping.json | 3 + .../device}/_ping.yml | 0 .../device/_reboot.json | 4 + .../device}/_reboot.yml | 0 .../device/_upgrade.json | 4 + .../device}/_upgrade.yml | 0 server/src/ansible/_reboot.json | 3 - server/src/ansible/_restartAgent.json | 3 - server/src/ansible/_uninstallAgent.json | 3 - server/src/ansible/_upgrade.json | 3 - server/src/ansible/ssm-ansible-run.py | 4 +- .../ssm-ansible-vault-password-client.py | 2 +- server/src/core/startup/index.ts | 43 +- server/src/core/system/version.ts | 6 +- server/src/data/database/model/Playbook.ts | 26 + .../database/model/PlaybooksRepository.ts | 88 ++++ .../data/database/repository/PlaybookRepo.ts | 48 +- .../repository/PlaybooksRepositoryRepo.ts | 72 +++ server/src/helpers/directory-tree/README.md | 190 ++++++++ .../helpers/directory-tree/directory-tree.ts | 186 ++++++++ server/src/index.ts | 5 +- server/src/integrations/ansible/AnsibleCmd.ts | 8 +- .../integrations/ansible/AnsibleGalaxyCmd.ts | 2 +- .../ansible/utils/InventoryTransformer.ts | 8 +- .../integrations/docker/core/CustomAgent.ts | 1 + .../integrations/docker/core/WatcherEngine.ts | 2 +- .../git-repository/GitRepositoryComponent.ts | 136 ++++++ .../integrations/git-repository/lib/clone.ts | 76 +++ .../git-repository/lib/commitAndSync.ts | 252 ++++++++++ .../git-repository/lib/credential.ts | 72 +++ .../git-repository/lib/defaultGitInfo.ts | 6 + .../integrations/git-repository/lib/errors.ts | 103 ++++ .../git-repository/lib/forcePull.ts | 109 +++++ .../integrations/git-repository/lib/index.ts | 12 + .../integrations/git-repository/lib/init.ts | 50 ++ .../git-repository/lib/initGit.ts | 86 ++++ .../git-repository/lib/inspect.ts | 391 +++++++++++++++ .../git-repository/lib/interface.ts | 101 ++++ .../integrations/git-repository/lib/sync.ts | 200 ++++++++ .../integrations/git-repository/lib/utils.ts | 2 + .../LocalRepositoryComponent.ts | 28 ++ .../PlaybooksRepositoryComponent.ts | 167 +++++++ .../PlaybooksRepositoryEngine.ts | 129 +++++ .../playbooks-repository/utils.ts | 87 ++++ server/src/integrations/shell/index.ts | 69 ++- server/src/integrations/shell/utils.ts | 17 + server/src/routes/index.ts | 6 +- server/src/routes/playbooks-repository.ts | 86 ++++ .../src/routes/{ansible.ts => playbooks.ts} | 61 +-- .../services/ansible/playbook.validator.ts | 46 -- .../src/services/playbooks-repository/git.ts | 155 ++++++ .../playbooks-repository/git.validator.ts | 28 ++ .../services/playbooks-repository/local.ts | 66 +++ .../playbooks-repository/local.validator.ts | 18 + .../platbooks-repository.validator.ts | 22 + .../playbooks-repository.ts | 71 +++ .../{ansible => playbooks}/execution.ts | 33 +- .../execution.validator.ts | 6 +- .../{ansible => playbooks}/extravar.ts | 0 .../extravar.validator.ts | 0 .../services/{ansible => playbooks}/galaxy.ts | 0 .../galaxy.validator.ts | 0 .../services/{ansible => playbooks}/hook.ts | 8 +- .../{ansible => playbooks}/inventory.ts | 0 .../{ansible => playbooks}/playbook.ts | 76 ++- .../services/playbooks/playbook.validator.ts | 31 ++ .../services/{ansible => playbooks}/vault.ts | 2 +- server/src/tests/helpers/FilterHelper.test.ts | 10 +- .../tests/helpers/directory-tree/constants.ts | 1 + .../directory-tree/depth/fixtureFirstDepth.ts | 32 ++ .../depth/fixtureSecondDepth.ts | 60 +++ .../directory-tree/depth/fixtureZeroDepth.ts | 9 + .../directory-tree/directory-tree.test.ts | 172 +++++++ .../tests/helpers/directory-tree/fixture.ts | 85 ++++ .../helpers/directory-tree/fixtureExclude.ts | 63 +++ .../directory-tree/fixtureMultipleExclude.ts | 48 ++ .../directory-tree/test_data/file_a.txt | 1 + .../directory-tree/test_data/file_b.txt | 9 + .../test_data/some_dir/another_dir/file_a.txt | 1 + .../test_data/some_dir/another_dir/file_b.txt | 9 + .../test_data/some_dir/file_a.txt | 1 + .../test_data/some_dir/file_b.txt | 9 + .../test_data/some_dir_2/.gitkeep | 0 server/src/tests/helpers/utils.test.ts | 1 + .../integrations/ansible/AnsibleCmd.test.ts | 12 +- server/src/types/typings.d.ts | 3 +- server/src/use-cases/DeviceUseCases.ts | 6 +- server/src/use-cases/GitRepositoryUseCases.ts | 84 ++++ .../src/use-cases/LocalRepositoryUseCases.ts | 49 ++ server/src/use-cases/PlaybookUseCases.ts | 87 +--- .../use-cases/PlaybooksRepositoryUseCases.ts | 137 ++++++ shared-lib/package.json | 1 + shared-lib/src/enums/ansible.ts | 1 + shared-lib/src/enums/playbooks.ts | 4 + shared-lib/src/enums/settings.ts | 2 +- shared-lib/src/index.ts | 2 + shared-lib/src/types/api.ts | 48 +- shared-lib/src/types/tree.ts | 20 + site/.vitepress/config.ts | 2 +- site/contribute/index.md | 14 +- site/contribute/release.md | 6 +- site/docs/devmode.md | 2 +- site/docs/manual-install-agent.md | 4 +- site/docs/quickstart.md | 4 +- site/index.md | 4 +- site/package.json | 1 + 160 files changed, 6732 insertions(+), 1026 deletions(-) create mode 100644 client/src/components/RegistryComponents/RegistryLogo.tsx rename client/src/pages/Admin/Settings/components/{UserSettings.tsx => AuthenticationSettings.tsx} (58%) create mode 100644 client/src/pages/Admin/Settings/components/PlaybooksSettings.tsx create mode 100644 client/src/pages/Admin/Settings/components/subcomponents/GitRepositoryModal.tsx create mode 100644 client/src/pages/Admin/Settings/components/subcomponents/LocalRepositoryModal.tsx create mode 100644 client/src/pages/Admin/Settings/components/subcomponents/RegistryModal.tsx create mode 100644 client/src/pages/Playbooks/components/CreateFileInRepositoryModalForm.tsx create mode 100644 client/src/pages/Playbooks/components/DirectoryTreeView.tsx create mode 100644 client/src/pages/Playbooks/components/FloatingButtonsBar.tsx create mode 100644 client/src/pages/Playbooks/components/NewFileDrawerForm.tsx delete mode 100644 client/src/pages/Playbooks/components/NewPlaybookModalForm.tsx create mode 100644 client/src/pages/Playbooks/components/PlaybookDropdownMenu.tsx create mode 100644 client/src/pages/Playbooks/components/TreeComponent.tsx create mode 100644 client/src/services/rest/playbooks-repositories.ts rename client/src/services/rest/{ansible.ts => playbooks.ts} (51%) rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000000/agent}/_checkDeviceBeforeAdd.json (78%) rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000000/agent}/_checkDeviceBeforeAdd.yml (100%) rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000000/agent}/_installAgent.json (87%) rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000000/agent}/_installAgent.yml (100%) rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000000/agent}/_reinstallAgent.json (87%) rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000000/agent}/_reinstallAgent.yml (100%) create mode 100644 server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_restartAgent.json rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000000/agent}/_restartAgent.yml (100%) create mode 100644 server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_retrieveAgentLogs.json rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000000/agent}/_retrieveAgentLogs.yml (100%) create mode 100644 server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_uninstallAgent.json rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000000/agent}/_uninstallAgent.yml (100%) create mode 100644 server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_updateAgent.json create mode 100644 server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_updateAgent.yml create mode 100644 server/src/ansible/00000000-0000-0000-0000-000000000000/device/_ping.json rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000000/device}/_ping.yml (100%) create mode 100644 server/src/ansible/00000000-0000-0000-0000-000000000000/device/_reboot.json rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000000/device}/_reboot.yml (100%) create mode 100644 server/src/ansible/00000000-0000-0000-0000-000000000000/device/_upgrade.json rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000000/device}/_upgrade.yml (100%) delete mode 100644 server/src/ansible/_reboot.json delete mode 100644 server/src/ansible/_restartAgent.json delete mode 100644 server/src/ansible/_uninstallAgent.json delete mode 100644 server/src/ansible/_upgrade.json create mode 100644 server/src/data/database/model/PlaybooksRepository.ts create mode 100644 server/src/data/database/repository/PlaybooksRepositoryRepo.ts create mode 100644 server/src/helpers/directory-tree/README.md create mode 100644 server/src/helpers/directory-tree/directory-tree.ts create mode 100644 server/src/integrations/git-repository/GitRepositoryComponent.ts create mode 100644 server/src/integrations/git-repository/lib/clone.ts create mode 100644 server/src/integrations/git-repository/lib/commitAndSync.ts create mode 100644 server/src/integrations/git-repository/lib/credential.ts create mode 100644 server/src/integrations/git-repository/lib/defaultGitInfo.ts create mode 100644 server/src/integrations/git-repository/lib/errors.ts create mode 100644 server/src/integrations/git-repository/lib/forcePull.ts create mode 100644 server/src/integrations/git-repository/lib/index.ts create mode 100644 server/src/integrations/git-repository/lib/init.ts create mode 100644 server/src/integrations/git-repository/lib/initGit.ts create mode 100644 server/src/integrations/git-repository/lib/inspect.ts create mode 100644 server/src/integrations/git-repository/lib/interface.ts create mode 100644 server/src/integrations/git-repository/lib/sync.ts create mode 100644 server/src/integrations/git-repository/lib/utils.ts create mode 100644 server/src/integrations/local-repository/LocalRepositoryComponent.ts create mode 100644 server/src/integrations/playbooks-repository/PlaybooksRepositoryComponent.ts create mode 100644 server/src/integrations/playbooks-repository/PlaybooksRepositoryEngine.ts create mode 100644 server/src/integrations/playbooks-repository/utils.ts create mode 100644 server/src/integrations/shell/utils.ts create mode 100644 server/src/routes/playbooks-repository.ts rename server/src/routes/{ansible.ts => playbooks.ts} (64%) delete mode 100644 server/src/services/ansible/playbook.validator.ts create mode 100644 server/src/services/playbooks-repository/git.ts create mode 100644 server/src/services/playbooks-repository/git.validator.ts create mode 100644 server/src/services/playbooks-repository/local.ts create mode 100644 server/src/services/playbooks-repository/local.validator.ts create mode 100644 server/src/services/playbooks-repository/platbooks-repository.validator.ts create mode 100644 server/src/services/playbooks-repository/playbooks-repository.ts rename server/src/services/{ansible => playbooks}/execution.ts (66%) rename server/src/services/{ansible => playbooks}/execution.validator.ts (62%) rename server/src/services/{ansible => playbooks}/extravar.ts (100%) rename server/src/services/{ansible => playbooks}/extravar.validator.ts (100%) rename server/src/services/{ansible => playbooks}/galaxy.ts (100%) rename server/src/services/{ansible => playbooks}/galaxy.validator.ts (100%) rename server/src/services/{ansible => playbooks}/hook.ts (86%) rename server/src/services/{ansible => playbooks}/inventory.ts (100%) rename server/src/services/{ansible => playbooks}/playbook.ts (52%) create mode 100644 server/src/services/playbooks/playbook.validator.ts rename server/src/services/{ansible => playbooks}/vault.ts (86%) create mode 100644 server/src/tests/helpers/directory-tree/constants.ts create mode 100644 server/src/tests/helpers/directory-tree/depth/fixtureFirstDepth.ts create mode 100644 server/src/tests/helpers/directory-tree/depth/fixtureSecondDepth.ts create mode 100644 server/src/tests/helpers/directory-tree/depth/fixtureZeroDepth.ts create mode 100644 server/src/tests/helpers/directory-tree/directory-tree.test.ts create mode 100644 server/src/tests/helpers/directory-tree/fixture.ts create mode 100644 server/src/tests/helpers/directory-tree/fixtureExclude.ts create mode 100644 server/src/tests/helpers/directory-tree/fixtureMultipleExclude.ts create mode 100644 server/src/tests/helpers/directory-tree/test_data/file_a.txt create mode 100644 server/src/tests/helpers/directory-tree/test_data/file_b.txt create mode 100644 server/src/tests/helpers/directory-tree/test_data/some_dir/another_dir/file_a.txt create mode 100644 server/src/tests/helpers/directory-tree/test_data/some_dir/another_dir/file_b.txt create mode 100644 server/src/tests/helpers/directory-tree/test_data/some_dir/file_a.txt create mode 100644 server/src/tests/helpers/directory-tree/test_data/some_dir/file_b.txt create mode 100644 server/src/tests/helpers/directory-tree/test_data/some_dir_2/.gitkeep create mode 100644 server/src/use-cases/GitRepositoryUseCases.ts create mode 100644 server/src/use-cases/LocalRepositoryUseCases.ts create mode 100644 server/src/use-cases/PlaybooksRepositoryUseCases.ts create mode 100644 shared-lib/src/enums/playbooks.ts create mode 100644 shared-lib/src/types/tree.ts diff --git a/client/.eslintrc.js b/client/.eslintrc.js index ad5984ff..e0a1d8b5 100644 --- a/client/.eslintrc.js +++ b/client/.eslintrc.js @@ -10,5 +10,6 @@ module.exports = { '@typescript-eslint/no-parameter-properties': 'off', '@typescript-eslint/consistent-type-imports': 'off', '@typescript-eslint/switch-exhaustiveness-check': 'off', + '@typescript-eslint/no-throw-literal': 'off', }, }; diff --git a/client/Dockerfile b/client/Dockerfile index 4644115f..ac844634 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -25,5 +25,5 @@ CMD ["npm", "run", "serve"] FROM base as dev COPY . . -RUN npm install --verbose +RUN npm install --verbose --no-audit CMD ["npm", "run", "start:pre"] diff --git a/client/config/routes.ts b/client/config/routes.ts index d119a61e..c3678d01 100644 --- a/client/config/routes.ts +++ b/client/config/routes.ts @@ -81,4 +81,8 @@ export default [ path: '/admin/inventory/:id', component: './Admin/Inventory', }, + { + path: '/manage/playbooks/:id', + component: './Playbooks', + }, ]; diff --git a/client/package.json b/client/package.json index a2711c62..5c609b7a 100644 --- a/client/package.json +++ b/client/package.json @@ -4,6 +4,7 @@ "private": true, "description": "SSM Client - A simple way to manage all your servers", "author": "Squirrel Team", + "license": "AGPL-3.0 license", "scripts": { "analyze": "cross-env ANALYZE=1 max build", "build": "max build", @@ -47,13 +48,13 @@ ], "dependencies": { "ssm-shared-lib": "file:../shared-lib/", - "antd": "^5.18.2", + "antd": "^5.18.3", "@ant-design/icons": "^5.3.7", "@ant-design/pro-components": "^2.7.10", "@ant-design/use-emotion-css": "1.0.4", "@ant-design/charts": "^2.1.1", "@antv/g2plot": "^2.4.31", - "@umijs/max": "^4.2.11", + "@umijs/max": "^4.2.13", "@umijs/route-utils": "^4.0.1", "@umijs/plugin-antd-dayjs": "^0.3.0", "umi-presets-pro": "^2.0.3", @@ -61,7 +62,7 @@ "lodash": "^4.17.21", "moment": "^2.30.1", "querystring": "^0.2.1", - "rc-menu": "^9.14.0", + "rc-menu": "^9.14.1", "react": "^18.3.1", "react-dev-inspector": "^2.0.1", "react-dom": "^18.3.1", @@ -82,7 +83,7 @@ "devDependencies": { "antd-pro-merge-less": "^3.0.11", "@ant-design/pro-cli": "^3.2.1", - "@ant-design/plots": "^2.2.2", + "@ant-design/plots": "^2.2.4", "@testing-library/react": "^16.0.0", "@types/classnames": "^2.3.1", "@types/express": "^4.17.21", @@ -94,7 +95,7 @@ "@types/react-dom": "^18.3.0", "@types/react-helmet": "^6.1.11", "@umijs/fabric": "^4.0.1", - "@umijs/lint": "^4.2.11", + "@umijs/lint": "^4.2.13", "cross-env": "^7.0.3", "eslint": "^8.57.0", "express": "^4.19.2", @@ -105,9 +106,9 @@ "prettier": "^3.3.2", "swagger-ui-dist": "^5.17.14", "ts-node": "^10.9.2", - "typescript": "^5.4.5", + "typescript": "^5.5.2", "eslint-plugin-react": "^7.34.3", - "@typescript-eslint/eslint-plugin": "^7.13.1", + "@typescript-eslint/eslint-plugin": "^7.14.1", "eslint-plugin-jest": "^28.6.0", "eslint-plugin-react-hooks": "^4.6.2" }, diff --git a/client/src/components/DeviceComponents/DeviceQuickAction/DeviceQuickActionDropDown.tsx b/client/src/components/DeviceComponents/DeviceQuickAction/DeviceQuickActionDropDown.tsx index 44819b93..55b34286 100644 --- a/client/src/components/DeviceComponents/DeviceQuickAction/DeviceQuickActionDropDown.tsx +++ b/client/src/components/DeviceComponents/DeviceQuickAction/DeviceQuickActionDropDown.tsx @@ -31,7 +31,7 @@ const DeviceQuickActionDropDown: React.FC = (props) => { if (DeviceQuickActionReference[idx].type === Types.PLAYBOOK) { props.setTerminal({ isOpen: true, - command: DeviceQuickActionReference[idx].playbookFile, + quickRef: DeviceQuickActionReference[idx].playbookQuickRef, target: props.target, }); } else if ( diff --git a/client/src/components/DeviceComponents/DeviceQuickAction/DeviceQuickActionReference.tsx b/client/src/components/DeviceComponents/DeviceQuickAction/DeviceQuickActionReference.tsx index a3f7a48d..ce0cf9df 100644 --- a/client/src/components/DeviceComponents/DeviceQuickAction/DeviceQuickActionReference.tsx +++ b/client/src/components/DeviceComponents/DeviceQuickAction/DeviceQuickActionReference.tsx @@ -22,7 +22,7 @@ export enum Types { export type QuickActionReferenceType = { type: Types; action?: string; - playbookFile?: string; + playbookQuickRef?: string; label?: React.JSX.Element; onAdvancedMenu: boolean; children?: QuickActionReferenceType[]; @@ -45,7 +45,7 @@ const DeviceQuickActionReference: QuickActionReferenceType[] = [ }, { type: Types.PLAYBOOK, - playbookFile: '_reboot', + playbookQuickRef: 'reboot', label: ( <> Reboot @@ -69,7 +69,7 @@ const DeviceQuickActionReference: QuickActionReferenceType[] = [ }, { type: Types.PLAYBOOK, - playbookFile: '_ping', + playbookQuickRef: 'ping', label: ( <> Ping @@ -83,7 +83,7 @@ const DeviceQuickActionReference: QuickActionReferenceType[] = [ }, { type: Types.PLAYBOOK, - playbookFile: '_updateAgent', + playbookQuickRef: 'updateAgent', onAdvancedMenu: true, label: ( <> @@ -93,7 +93,7 @@ const DeviceQuickActionReference: QuickActionReferenceType[] = [ }, { type: Types.PLAYBOOK, - playbookFile: '_reinstallAgent', + playbookQuickRef: 'reinstallAgent', onAdvancedMenu: true, label: ( <> @@ -103,7 +103,7 @@ const DeviceQuickActionReference: QuickActionReferenceType[] = [ }, { type: Types.PLAYBOOK, - playbookFile: '_restartAgent', + playbookQuickRef: 'restartAgent', onAdvancedMenu: true, label: ( <> @@ -114,7 +114,7 @@ const DeviceQuickActionReference: QuickActionReferenceType[] = [ { type: Types.PLAYBOOK, onAdvancedMenu: true, - playbookFile: '_retrieveAgentLogs', + playbookQuickRef: 'retrieveAgentLogs', label: ( <> Retrieve Agent Logs @@ -127,7 +127,7 @@ const DeviceQuickActionReference: QuickActionReferenceType[] = [ }, { type: Types.PLAYBOOK, - playbookFile: '_uninstallAgent', + playbookQuickRef: 'uninstallAgent', onAdvancedMenu: true, label: ( <> diff --git a/client/src/components/DeviceComponents/OSSoftwaresVersions/SoftwareIcon.tsx b/client/src/components/DeviceComponents/OSSoftwaresVersions/SoftwareIcon.tsx index 97a15f56..70af322f 100644 --- a/client/src/components/DeviceComponents/OSSoftwaresVersions/SoftwareIcon.tsx +++ b/client/src/components/DeviceComponents/OSSoftwaresVersions/SoftwareIcon.tsx @@ -115,7 +115,7 @@ const SoftwareIcon: React.FC = (props) => { /* "systemOpenssl": "3.2.0", - "git": "2.23.0", + "playbooks-repository": "2.23.0", "tsc": "5.3.3", "mysql": "", "cache": "", diff --git a/client/src/components/Footer/index.tsx b/client/src/components/Footer/index.tsx index 8193182a..23371de3 100644 --- a/client/src/components/Footer/index.tsx +++ b/client/src/components/Footer/index.tsx @@ -10,7 +10,7 @@ const Footer: React.FC = () => { style={{ background: 'none', }} - copyright={`${currentYear} Squirrel Team.`} + copyright={`${currentYear} Squirrel Team (AGPL-3.0 license).`} links={[ { key: 'ssm', diff --git a/client/src/components/Icons/CustomIcons.tsx b/client/src/components/Icons/CustomIcons.tsx index 321614e8..17e15f48 100644 --- a/client/src/components/Icons/CustomIcons.tsx +++ b/client/src/components/Icons/CustomIcons.tsx @@ -674,3 +674,35 @@ export const VaadinCubes = (props: any) => ( /> ); + +export const SimpleIconsGit = (props: any) => ( + + + +); + +export const StreamlineLocalStorageFolderSolid = (props: any) => ( + + + +); diff --git a/client/src/components/PlaybookSelectionModal/PlaybookSelectionModal.tsx b/client/src/components/PlaybookSelectionModal/PlaybookSelectionModal.tsx index d8002c3a..38a2aaef 100644 --- a/client/src/components/PlaybookSelectionModal/PlaybookSelectionModal.tsx +++ b/client/src/components/PlaybookSelectionModal/PlaybookSelectionModal.tsx @@ -1,4 +1,4 @@ -import { getPlaybooks } from '@/services/rest/ansible'; +import { getPlaybooks } from '@/services/rest/playbooks'; import { RightSquareOutlined } from '@ant-design/icons'; import { ModalForm, @@ -28,15 +28,18 @@ const PlaybookSelectionModal: React.FC = ( ) => { const [form] = Form.useForm<{ playbook: { value: string } }>(); const [listOfPlaybooks, setListOfPlaybooks] = React.useState< - API.PlaybookFileList[] | undefined + API.PlaybookFile[] | undefined >(); const [selectedPlaybookExtraVars, setSelectedPlaybookExtraVars] = React.useState(); const [overrideExtraVars, setOverrideExtraVars] = React.useState([]); - const handleSelectedPlaybook = (newValue: API.PlaybookFileList) => { + const handleSelectedPlaybook = (newValue: { + label: string; + value: string; + }) => { const selectedPlaybook = listOfPlaybooks?.find( - (e) => e.value === newValue?.value, + (e) => e.uuid === newValue?.value, ); if (selectedPlaybook) { setOverrideExtraVars( @@ -150,9 +153,16 @@ const PlaybookSelectionModal: React.FC = ( return (await getPlaybooks() .then((e) => { setListOfPlaybooks(e.data); - return e.data?.filter(({ value, label }) => { - return value.includes(keyWords) || label.includes(keyWords); - }); + return e.data + ?.filter(({ name, path }: { name: string; path: string }) => { + return name.includes(keyWords) || path.includes(keyWords); + }) + .map((f: API.PlaybookFile) => { + return { + value: f.uuid, + label: f.name, + }; + }); }) .catch((error) => { message.error({ @@ -179,7 +189,7 @@ const PlaybookSelectionModal: React.FC = ( return ( SSM will apply " - {playbook ? playbook.value : '?'} + {playbook ? playbook.label : '?'} " on{' '} {props.itemSelected?.map((e) => { return '[' + e.ip + '] '; diff --git a/client/src/components/RegistryComponents/RegistryLogo.tsx b/client/src/components/RegistryComponents/RegistryLogo.tsx new file mode 100644 index 00000000..2e0b5e1c --- /dev/null +++ b/client/src/components/RegistryComponents/RegistryLogo.tsx @@ -0,0 +1,50 @@ +import { + DeviconAzure, + DeviconGooglecloud, + FluentMdl2RegistryEditor, + LogoHotIo, + LogosAws, + LogosGitlab, + LogosQuay, + SimpleIconsForgejo, + SimpleIconsGitea, + VaadinCubes, + VscodeIconsFileTypeDocker2, + ZmdiGithub, +} from '@/components/Icons/CustomIcons'; +import React from 'react'; + +type RegistryLogoProps = { + provider: string; +}; + +const RegistryLogo: React.FC = (props) => { + switch (props.provider) { + case 'ghcr': + return ; + case 'gcr': + return ; + case 'acr': + return ; + case 'hotio': + return ; + case 'hub': + return ; + case 'ecr': + return ; + case 'quay': + return ; + case 'forgejo': + return ; + case 'gitea': + return ; + case 'lscr': + return ; + case 'gitlab': + return ; + default: + return ; + } +}; + +export default RegistryLogo; diff --git a/client/src/components/Template/Title.tsx b/client/src/components/Template/Title.tsx index 186fe4d8..10c2c1bf 100644 --- a/client/src/components/Template/Title.tsx +++ b/client/src/components/Template/Title.tsx @@ -18,6 +18,8 @@ export enum SettingsSubTitleColors { USER_LOGS = '#6d26a8', API = '#1e6d80', SERVER = '#8e7d50', + GIT = '#336048', + LOCAL = '#4b4a4a', } export type PageContainerTitleProps = { diff --git a/client/src/components/TerminalModal/TerminalHandler.ts b/client/src/components/TerminalModal/TerminalHandler.ts index 79a0c367..720d42bc 100644 --- a/client/src/components/TerminalModal/TerminalHandler.ts +++ b/client/src/components/TerminalModal/TerminalHandler.ts @@ -1,5 +1,5 @@ import taskStatusTimeline from '@/components/TerminalModal/TaskStatusTimeline'; -import { getExecLogs, getTaskStatuses } from '@/services/rest/ansible'; +import { getExecLogs, getTaskStatuses } from '@/services/rest/playbooks'; import { StepsProps } from 'antd'; import React, { ReactNode } from 'react'; import { API } from 'ssm-shared-lib'; diff --git a/client/src/components/TerminalModal/index.tsx b/client/src/components/TerminalModal/index.tsx index e6c2d546..d6a7c659 100644 --- a/client/src/components/TerminalModal/index.tsx +++ b/client/src/components/TerminalModal/index.tsx @@ -2,7 +2,10 @@ import TerminalHandler, { TaskStatusTimelineType, } from '@/components/TerminalModal/TerminalHandler'; import TerminalLogs from '@/components/TerminalModal/TerminalLogs'; -import { executePlaybook } from '@/services/rest/ansible'; +import { + executePlaybook, + executePlaybookByQuickRef, +} from '@/services/rest/playbooks'; import { ClockCircleOutlined, ThunderboltOutlined } from '@ant-design/icons'; import { Button, Col, Modal, Row, Steps, message } from 'antd'; import React, { useEffect, useRef, useState } from 'react'; @@ -12,7 +15,8 @@ import { API } from 'ssm-shared-lib'; export type TerminalStateProps = { isOpen: boolean; - command: string | undefined; + command?: string; + quickRef?: string; target?: API.DeviceItem[]; extraVars?: API.ExtraVars; }; @@ -90,20 +94,32 @@ const TerminalModal = (props: TerminalModalProps) => {
)); - if (!props.terminalProps.command) { + if (!props.terminalProps.command && !props.terminalProps.quickRef) { message.error({ type: 'error', content: 'Error running playbook (internal)', duration: 8, }); + setBufferedContent(() => ( + <> + No command +
+ + )); return; } try { - const res = await executePlaybook( - props.terminalProps.command, - props.terminalProps.target?.map((e) => e.uuid), - props.terminalProps.extraVars, - ); + const res = !props.terminalProps.quickRef + ? await executePlaybook( + props.terminalProps.command, + props.terminalProps.target?.map((e) => e.uuid), + props.terminalProps.extraVars, + ) + : await executePlaybookByQuickRef( + props.terminalProps.quickRef, + props.terminalProps.target?.map((e) => e.uuid), + props.terminalProps.extraVars, + ); setExecId(res.data.execId); message.loading({ content: `Playbook is running with id "${res.data.execId}"`, @@ -176,7 +192,8 @@ const TerminalModal = (props: TerminalModalProps) => { transform: 'translate(0, -50%)', }} > - Executing playbook {props.terminalProps.command}...{' '} + Executing playbook{' '} + {props.terminalProps.command || props.terminalProps.quickRef}...{' '} } diff --git a/client/src/pages/Admin/Settings/Settings.tsx b/client/src/pages/Admin/Settings/Settings.tsx index cd6b1cca..b9b8b5ed 100644 --- a/client/src/pages/Admin/Settings/Settings.tsx +++ b/client/src/pages/Admin/Settings/Settings.tsx @@ -1,7 +1,8 @@ import Title, { PageContainerTitleColors } from '@/components/Template/Title'; import AdvancedSettings from '@/pages/Admin/Settings/components/AdvancedSettings'; +import PlaybookSettings from '@/pages/Admin/Settings/components/PlaybooksSettings'; import RegistrySettings from '@/pages/Admin/Settings/components/RegistrySettings'; -import UserSettings from '@/pages/Admin/Settings/components/UserSettings'; +import AuthenticationSettings from '@/pages/Admin/Settings/components/AuthenticationSettings'; import GeneralSettings from '@/pages/Admin/Settings/components/GeneralSettings'; import Information from '@/pages/Admin/Settings/components/Information'; import { InfoCircleOutlined, SettingOutlined } from '@ant-design/icons'; @@ -23,22 +24,31 @@ const Settings: React.FC = () => { key: '2', label: (
- User settings + Authentication
), - children: , + children: , }, { key: '3', label: (
- Registries + Playbooks
), - children: , + children: , }, { key: '4', + label: ( +
+ Container Registries +
+ ), + children: , + }, + { + key: '5', label: (
Advanced @@ -47,7 +57,7 @@ const Settings: React.FC = () => { children: , }, { - key: '5', + key: '6', label: (
System Information diff --git a/client/src/pages/Admin/Settings/components/UserSettings.tsx b/client/src/pages/Admin/Settings/components/AuthenticationSettings.tsx similarity index 58% rename from client/src/pages/Admin/Settings/components/UserSettings.tsx rename to client/src/pages/Admin/Settings/components/AuthenticationSettings.tsx index 4403f199..b05fc39a 100644 --- a/client/src/pages/Admin/Settings/components/UserSettings.tsx +++ b/client/src/pages/Admin/Settings/components/AuthenticationSettings.tsx @@ -24,26 +24,11 @@ import { } from 'antd'; import React, { useState } from 'react'; -const UserSettings: React.FC = () => { +const AuthenticationSettings: React.FC = () => { const { initialState } = useModel('@@initialState'); const { currentUser } = initialState || {}; - const [inputValue, setInputValue] = useState( - currentUser?.settings.userSpecific.userLogsLevel.terminal, - ); const [apiKey, setApiKey] = useState(currentUser?.settings.apiKey); - const onChange = async (newValue: number | null) => { - if (newValue) { - await postUserLogs({ terminal: newValue }).then(() => { - setInputValue(newValue); - message.success({ - content: 'Setting successfully updated', - duration: 6, - }); - }); - } - }; - const onClickResetApiKey = async () => { await postResetApiKey().then((res) => { setApiKey(res.data.uuid); @@ -53,64 +38,6 @@ const UserSettings: React.FC = () => { return ( - } - /> - } - > - - - - - The verbosity level of Ansible output, as described{' '} - - {' '} - here - - - } - > - - {' '} - Log level of terminal - {' '} - - - setInputValue(newValue)} - onChangeComplete={onChange} - value={typeof inputValue === 'number' ? inputValue : 0} - /> - - - - - - - - - { ); }; -export default UserSettings; +export default AuthenticationSettings; diff --git a/client/src/pages/Admin/Settings/components/PlaybooksSettings.tsx b/client/src/pages/Admin/Settings/components/PlaybooksSettings.tsx new file mode 100644 index 00000000..862486ab --- /dev/null +++ b/client/src/pages/Admin/Settings/components/PlaybooksSettings.tsx @@ -0,0 +1,333 @@ +import { + SimpleIconsGit, + StreamlineLocalStorageFolderSolid, +} from '@/components/Icons/CustomIcons'; +import Title, { SettingsSubTitleColors } from '@/components/Template/Title'; +import GitRepositoryModal from '@/pages/Admin/Settings/components/subcomponents/GitRepositoryModal'; +import LocalRepositoryModal from '@/pages/Admin/Settings/components/subcomponents/LocalRepositoryModal'; +import { + getGitRepositories, + getLocalRepositories, +} from '@/services/rest/playbooks-repositories'; +import { postUserLogs } from '@/services/rest/usersettings'; +import { useModel } from '@@/exports'; +import { + InfoCircleFilled, + LockFilled, + UnorderedListOutlined, +} from '@ant-design/icons'; +import { ProList } from '@ant-design/pro-components'; +import { + Avatar, + Button, + Card, + Col, + Flex, + InputNumber, + message, + Popover, + Row, + Slider, + Space, + Tag, + Tooltip, + Typography, +} from 'antd'; +import { AddCircleOutline } from 'antd-mobile-icons'; +import React, { useEffect, useState } from 'react'; +import { API } from 'ssm-shared-lib'; + +const PlaybookSettings: React.FC = () => { + const { initialState } = useModel('@@initialState'); + const { currentUser } = initialState || {}; + const [inputValue, setInputValue] = useState( + currentUser?.settings.userSpecific.userLogsLevel.terminal, + ); + const [gitRepositories, setGitRepositories] = useState( + [], + ); + const [localRepositories, setLocalRepositories] = useState< + API.LocalRepository[] + >([]); + const asyncFetch = async () => { + await getGitRepositories().then((list) => { + if (list?.data) { + setGitRepositories(list.data); + } + }); + await getLocalRepositories().then((list) => { + if (list?.data) { + setLocalRepositories(list.data); + } + }); + }; + useEffect(() => { + void asyncFetch(); + }, []); + + const [gitModalOpened, setGitModalOpened] = useState(false); + const [selectedGitRecord, setSelectedGitRecord] = useState(); + const [localModalOpened, setLocalModalOpened] = useState(false); + const [selectedLocalRecord, setSelectedLocalRecord] = useState(); + + const onChange = async (newValue: number | null) => { + if (newValue) { + await postUserLogs({ terminal: newValue }).then(() => { + setInputValue(newValue); + message.success({ + content: 'Setting successfully updated', + duration: 6, + }); + }); + } + }; + return ( + + + + } + /> + } + > + + + + + The verbosity level of Ansible output, as described{' '} + + {' '} + here + + + } + > + + {' '} + Log level of terminal + {' '} + + + setInputValue(newValue)} + onChangeComplete={onChange} + value={typeof inputValue === 'number' ? inputValue : 0} + /> + + + + + + + + + + } + /> + } + style={{ marginTop: 16 }} + extra={ + <> + + + + + + } + > + + ghost={true} + itemCardProps={{ + ghost: true, + }} + pagination={ + localRepositories?.length > 8 + ? { + defaultPageSize: 8, + showSizeChanger: false, + showQuickJumper: false, + } + : false + } + rowSelection={false} + grid={{ gutter: 0, column: 4 }} + onItem={(record: API.LocalRepository) => { + return { + onMouseEnter: () => { + console.log(record); + }, + onClick: () => { + setSelectedLocalRecord(record); + setLocalModalOpened(true); + }, + }; + }} + metas={{ + title: { + dataIndex: 'name', + }, + content: { + render: (_, row) => ( + + {row.directory} + + ), + }, + subTitle: { + render: (_, row) => + row.default ? ( + }> + Default + + ) : undefined, + }, + type: {}, + avatar: { + render: () => ( + } /> + ), + }, + actions: { + cardActionProps: 'extra', + }, + }} + dataSource={localRepositories} + /> + + } + /> + } + style={{ marginTop: 16 }} + extra={ + <> + + + + + + } + > + + ghost={true} + itemCardProps={{ + ghost: true, + }} + pagination={ + gitRepositories?.length > 8 + ? { + defaultPageSize: 8, + showSizeChanger: false, + showQuickJumper: false, + } + : false + } + rowSelection={false} + grid={{ gutter: 0, column: 4 }} + onItem={(record: API.GitRepository) => { + return { + onMouseEnter: () => { + console.log(record); + }, + onClick: () => { + setSelectedGitRecord(record); + setGitModalOpened(true); + }, + }; + }} + metas={{ + title: { + dataIndex: 'name', + }, + subTitle: { + render: (_, row) => branch:{row.branch}, + }, + content: { + render: (_, row) => ( + + {row.userName}@{row.remoteUrl} + + ), + }, + type: {}, + avatar: { + render: () => } />, + }, + actions: { + cardActionProps: 'extra', + }, + }} + dataSource={gitRepositories} + /> + + + ); +}; + +export default PlaybookSettings; diff --git a/client/src/pages/Admin/Settings/components/RegistrySettings.tsx b/client/src/pages/Admin/Settings/components/RegistrySettings.tsx index eb14d54e..f7154b93 100644 --- a/client/src/pages/Admin/Settings/components/RegistrySettings.tsx +++ b/client/src/pages/Admin/Settings/components/RegistrySettings.tsx @@ -1,23 +1,9 @@ +import RegistryLogo from '@/components/RegistryComponents/RegistryLogo'; +import RegistryModal from '@/pages/Admin/Settings/components/subcomponents/RegistryModal'; import { - DeviconAzure, - DeviconGooglecloud, - FluentMdl2RegistryEditor, - LogoHotIo, - LogosAws, - LogosGitlab, - LogosQuay, - SimpleIconsForgejo, - SimpleIconsGitea, - VaadinCubes, - VscodeIconsFileTypeDocker2, - ZmdiGithub, -} from '@/components/Icons/CustomIcons'; -import { - createCustomRegistry, getRegistries, removeRegistry, resetRegistry, - updateRegistry, } from '@/services/rest/containers'; import { CheckCircleOutlined, @@ -25,54 +11,46 @@ import { UndoOutlined, UserOutlined, } from '@ant-design/icons'; -import { - ModalForm, - ProForm, - ProFormText, - ProList, -} from '@ant-design/pro-components'; -import { - Alert, - Avatar, - Button, - Card, - message, - Popconfirm, - Space, - Tag, - Tooltip, -} from 'antd'; -import { DeleteOutline } from 'antd-mobile-icons'; +import { ProList } from '@ant-design/pro-components'; +import { Avatar, Button, Card, message, Popconfirm, Tag, Tooltip } from 'antd'; +import { AddCircleOutline, DeleteOutline } from 'antd-mobile-icons'; import React, { useEffect, useState } from 'react'; import { API } from 'ssm-shared-lib'; -const getRegistryLogo = (provider: string) => { - switch (provider) { - case 'ghcr': - return ; - case 'gcr': - return ; - case 'acr': - return ; - case 'hotio': - return ; - case 'hub': - return ; - case 'ecr': - return ; - case 'quay': - return ; - case 'forgejo': - return ; - case 'gitea': - return ; - case 'lscr': - return ; - case 'gitlab': - return ; - default: - return ; - } +const addRecord = { + name: 'custom', + provider: 'custom', + authSet: false, + canAuth: true, + canAnonymous: false, + authScheme: [ + { + name: 'url', + type: 'string', + }, + { + name: 'Connection Type', + type: 'choice', + values: [ + [ + { + name: 'login', + type: 'string', + }, + { + name: 'password', + type: 'string', + }, + ], + [ + { + name: 'auth', + type: 'string', + }, + ], + ], + }, + ], }; const RegistrySettings: React.FC = () => { @@ -90,153 +68,53 @@ const RegistrySettings: React.FC = () => { }); }; useEffect(() => { - asyncFetch(); + void asyncFetch(); }, []); - const [ghost] = useState(false); + const [modalOpened, setModalOpened] = useState(false); const [selectedRecord, setSelectedRecord] = useState(); const onDeleteRegistry = async (item: API.ContainerRegistry) => { await removeRegistry(item.name); - asyncFetch(); + void asyncFetch(); }; const onResetRegistry = async (item: API.ContainerRegistry) => { await resetRegistry(item.name); - asyncFetch(); + void asyncFetch(); }; return ( - - title={ - <> - - Connexion for {selectedRecord?.provider} - - } - open={modalOpened} - autoFocusFirstInput - modalProps={{ - destroyOnClose: true, - onCancel: () => setModalOpened(false), - }} - onFinish={async (values) => { - if (selectedRecord?.name !== 'custom') { - await updateRegistry(selectedRecord.name, values); - setModalOpened(false); - await asyncFetch(); - } else { - await createCustomRegistry( - values.newName, - values, - selectedRecord.authScheme, - ); - setModalOpened(false); - await asyncFetch(); - } - }} - > - {selectedRecord?.authSet && ( - - - - )} - {selectedRecord?.name === 'custom' && ( - <> - - - - - )} - - {selectedRecord?.name === 'custom' && ( - e.name === value) === undefined - ) { - return Promise.resolve(); - } - return Promise.reject('Name already exists'); - }, - }, - ]} - /> - )} - {selectedRecord?.authScheme.map((e: any) => { - if (e?.type === 'choice') { - let i = 1; - return e.values.map((f: any) => { - return ( - - {f.map((g: any) => { - return ( - - ); - })} - - ); - }); - } else { - return ( - - ); - } - })} - - + size={'large'} - ghost={ghost} + ghost={false} itemCardProps={{ - ghost, + ghost: false, }} pagination={{ defaultPageSize: 20, showSizeChanger: false, }} + toolBarRender={() => [ + , + ]} showActions="hover" rowSelection={false} grid={{ gutter: 16, column: 3 }} @@ -274,134 +152,96 @@ const RegistrySettings: React.FC = () => { canAnonymous: false, }} headerTitle="Registries" - dataSource={registries - .concat({ - name: 'custom', - provider: 'custom', - authSet: false, - canAuth: true, - canAnonymous: false, - authScheme: [ - { - name: 'url', - type: 'string', - }, - { - name: 'Connection Type', - type: 'choice', - values: [ - [ - { - name: 'login', - type: 'string', - }, - { - name: 'password', - type: 'string', - }, - ], - [ - { - name: 'auth', - type: 'string', - }, - ], - ], - }, - ], - }) - .map((item: API.ContainerRegistry) => ({ - title: - item.name === 'custom' - ? 'Add a new custom provider' - : item.name?.toUpperCase(), - name: item.name, - authScheme: item.authScheme, - provider: item.provider, - canAnonymous: item.canAnonymous, - subTitle: <>{item.fullName}, - canAuth: item.canAuth, - authSet: item.authSet, - actions: [ - item.name !== 'custom' && item.authSet ? ( - item.provider === 'custom' ? ( - - {' '} - onDeleteRegistry(item)} - > - - - - ) : ( - ({ + title: + item.name === 'custom' + ? 'Add a new custom provider' + : item.name?.toUpperCase(), + name: item.name, + authScheme: item.authScheme, + provider: item.provider, + canAnonymous: item.canAnonymous, + subTitle: <>{item.fullName}, + canAuth: item.canAuth, + authSet: item.authSet, + actions: [ + item.name !== 'custom' && item.authSet ? ( + item.provider === 'custom' ? ( + + {' '} + onDeleteRegistry(item)} > - onResetRegistry(item)} - > - {' '} - - - - ) + + + ) : ( - <> - ), - ], - avatar: ( - + + onResetRegistry(item)} + > + {' '} + + + + ) + ) : ( + <> ), - content: ( + ], + avatar: ( + } + /> + ), + content: ( +
-
-
- {(item.name !== 'custom' && - ((item.authSet && ( - } color={'cyan'}> - Authentified - - )) || - (item.canAnonymous && ( - } color={'magenta'}> - Anonymous - - )))) || ( - } color={'geekblue'}> - Deactivated +
+ {(item.name !== 'custom' && + ((item.authSet && ( + } color={'cyan'}> + Authentified - )} -
+ )) || + (item.canAnonymous && ( + } color={'magenta'}> + Anonymous + + )))) || ( + } color={'geekblue'}> + Deactivated + + )}
- ), - }))} +
+ ), + }))} /> ); diff --git a/client/src/pages/Admin/Settings/components/subcomponents/GitRepositoryModal.tsx b/client/src/pages/Admin/Settings/components/subcomponents/GitRepositoryModal.tsx new file mode 100644 index 00000000..e6789fc1 --- /dev/null +++ b/client/src/pages/Admin/Settings/components/subcomponents/GitRepositoryModal.tsx @@ -0,0 +1,236 @@ +import { SimpleIconsGit } from '@/components/Icons/CustomIcons'; +import { + commitAndSyncGitRepository, + deleteGitRepository, + forceCloneGitRepository, + forcePullGitRepository, + forceRegisterGitRepository, + postGitRepository, + putGitRepository, + syncToDatabaseGitRepository, +} from '@/services/rest/playbooks-repositories'; +import { ModalForm, ProForm, ProFormText } from '@ant-design/pro-components'; +import { Avatar, Button, message, Dropdown, MenuProps } from 'antd'; +import React from 'react'; +import { API } from 'ssm-shared-lib'; + +type GitRepositoryModalProps = { + selectedRecord: Partial; + modalOpened: boolean; + setModalOpened: any; + asyncFetch: () => Promise; + repositories: API.GitRepository[]; +}; + +const items = [ + { + key: '1', + label: 'Force Pull', + }, + { + key: '2', + label: 'Commit And Sync', + }, + { + key: '3', + label: 'Force Clone', + }, + { + key: '4', + label: 'Sync To Database', + }, + { + key: '5', + label: 'Force Register', + }, +]; + +const GitRepositoryModal: React.FC = (props) => { + const onMenuClick: MenuProps['onClick'] = async (e) => { + if (!props.selectedRecord.uuid) { + message.error({ + content: 'Internal error - no uuid', + duration: 6, + }); + return; + } + switch (e.key) { + case '1': + await forcePullGitRepository(props.selectedRecord.uuid).then(() => { + message.success({ + content: 'Force pull command launched', + duration: 6, + }); + }); + return; + case '2': + await commitAndSyncGitRepository(props.selectedRecord.uuid).then(() => { + message.success({ + content: 'Commit and sync command launched', + duration: 6, + }); + }); + return; + case '3': + await forceCloneGitRepository(props.selectedRecord.uuid).then(() => { + message.success({ + content: 'Force clone command launched', + duration: 6, + }); + }); + return; + case '4': + await syncToDatabaseGitRepository(props.selectedRecord.uuid).then( + () => { + message.success({ + content: 'Sync to database command launched', + duration: 6, + }); + }, + ); + return; + case '5': + await forceRegisterGitRepository(props.selectedRecord.uuid).then(() => { + message.success({ + content: 'Force register command launched', + duration: 6, + }); + }); + return; + } + }; + + const editionMode = props.selectedRecord + ? [ + + Actions + , + , + ] + : []; + + return ( + + title={ + <> + } + /> + {(props.selectedRecord && ( + <>Edit repository {props.selectedRecord?.name} + )) || <>Add & sync a new repository} + + } + open={props.modalOpened} + autoFocusFirstInput + modalProps={{ + destroyOnClose: true, + onCancel: () => props.setModalOpened(false), + }} + onFinish={async (values) => { + if (props.selectedRecord) { + await postGitRepository(values); + props.setModalOpened(false); + await props.asyncFetch(); + } else { + await putGitRepository(values); + props.setModalOpened(false); + await props.asyncFetch(); + } + }} + submitter={{ + render: (_, defaultDoms) => { + return [...editionMode, ...defaultDoms]; + }, + }} + > + + e.name === value) === undefined + ) { + return Promise.resolve(); + } + return Promise.reject('Name already exists'); + }, + }, + ]} + /> + + + + + + + + ); +}; + +export default GitRepositoryModal; diff --git a/client/src/pages/Admin/Settings/components/subcomponents/LocalRepositoryModal.tsx b/client/src/pages/Admin/Settings/components/subcomponents/LocalRepositoryModal.tsx new file mode 100644 index 00000000..721c7e1a --- /dev/null +++ b/client/src/pages/Admin/Settings/components/subcomponents/LocalRepositoryModal.tsx @@ -0,0 +1,164 @@ +import { SimpleIconsGit } from '@/components/Icons/CustomIcons'; +import { + deleteLocalRepository, + postLocalRepositories, + putLocalRepositories, + syncToDatabaseLocalRepository, +} from '@/services/rest/playbooks-repositories'; +import { ModalForm, ProForm, ProFormText } from '@ant-design/pro-components'; +import { Avatar, Button, Dropdown, MenuProps, message } from 'antd'; +import React, { FC, useState } from 'react'; +import { API } from 'ssm-shared-lib'; + +type LocalRepositoryModalProps = { + selectedRecord: Partial; + modalOpened: boolean; + setModalOpened: any; + asyncFetch: () => Promise; + repositories: API.LocalRepository[]; +}; + +const items = [ + { + key: '4', + label: 'Sync To Database', + }, +]; + +const LocalRepositoryModal: FC = (props) => { + const [loading, setLoading] = useState(false); + const onMenuClick: MenuProps['onClick'] = async (e) => { + if (!props.selectedRecord.uuid) { + message.error({ + content: 'Internal error - no uuid', + duration: 6, + }); + return; + } + switch (e.key) { + case '4': + await syncToDatabaseLocalRepository(props.selectedRecord.uuid).then( + () => { + message.success({ + content: 'Sync to database command launched', + duration: 6, + }); + }, + ); + return; + } + }; + const editionMode = props.selectedRecord + ? [ + + Actions + , + , + ] + : []; + return ( + + title={ + <> + } + /> + {(props.selectedRecord && ( + <>Edit repository {props.selectedRecord?.name} + )) || <>Add & sync a new repository} + + } + open={props.modalOpened} + autoFocusFirstInput + modalProps={{ + destroyOnClose: true, + onCancel: () => props.setModalOpened(false), + }} + onFinish={async (values) => { + if (!props.selectedRecord?.default) { + if (props.selectedRecord) { + await postLocalRepositories({ + ...props.selectedRecord, + name: values.name, + }); + props.setModalOpened(false); + await props.asyncFetch(); + } else { + await putLocalRepositories(values); + props.setModalOpened(false); + await props.asyncFetch(); + } + } else { + props.setModalOpened(false); + await props.asyncFetch(); + } + }} + submitter={{ + render: (_, defaultDoms) => { + return [...editionMode, ...defaultDoms]; + }, + }} + > + + e.name === value) === + undefined || + props.selectedRecord?.name === value + ) { + return Promise.resolve(); + } + return Promise.reject('Name already exists'); + }, + }, + ]} + /> + + + ); +}; + +export default LocalRepositoryModal; diff --git a/client/src/pages/Admin/Settings/components/subcomponents/RegistryModal.tsx b/client/src/pages/Admin/Settings/components/subcomponents/RegistryModal.tsx new file mode 100644 index 00000000..9867335c --- /dev/null +++ b/client/src/pages/Admin/Settings/components/subcomponents/RegistryModal.tsx @@ -0,0 +1,144 @@ +import RegistryLogo from '@/components/RegistryComponents/RegistryLogo'; +import { + createCustomRegistry, + updateRegistry, +} from '@/services/rest/containers'; +import { ModalForm, ProForm, ProFormText } from '@ant-design/pro-components'; +import { Alert, Avatar, Space } from 'antd'; +import React from 'react'; +import { API } from 'ssm-shared-lib'; + +type RegistryModalProps = { + selectedRecord: any; + modalOpened: boolean; + setModalOpened: any; + asyncFetch: () => Promise; + registries: API.ContainerRegistry[]; +}; + +const RegistryModal: React.FC = (props) => { + return ( + + title={ + <> + } + /> + Connexion for {props.selectedRecord?.provider} + + } + open={props.modalOpened} + autoFocusFirstInput + modalProps={{ + destroyOnClose: true, + onCancel: () => props.setModalOpened(false), + }} + onFinish={async (values) => { + if (props.selectedRecord?.name !== 'custom') { + await updateRegistry(props.selectedRecord.name, values); + props.setModalOpened(false); + await props.asyncFetch(); + } else { + await createCustomRegistry( + values.newName, + values, + props.selectedRecord.authScheme, + ); + props.setModalOpened(false); + await props.asyncFetch(); + } + }} + > + {props.selectedRecord?.authSet && ( + + + + )} + {props.selectedRecord?.name === 'custom' && ( + <> + + + + + )} + + {props.selectedRecord?.name === 'custom' && ( + e.name === value) === undefined + ) { + return Promise.resolve(); + } + return Promise.reject('Name already exists'); + }, + }, + ]} + /> + )} + {props.selectedRecord?.authScheme.map((e: any) => { + if (e?.type === 'choice') { + let i = 1; + return e.values.map((f: any) => { + return ( + + {f.map((g: any) => { + return ( + + ); + })} + + ); + }); + } else { + return ( + + ); + } + })} + + + ); +}; + +export default RegistryModal; diff --git a/client/src/pages/Playbooks/components/CreateFileInRepositoryModalForm.tsx b/client/src/pages/Playbooks/components/CreateFileInRepositoryModalForm.tsx new file mode 100644 index 00000000..1606fb08 --- /dev/null +++ b/client/src/pages/Playbooks/components/CreateFileInRepositoryModalForm.tsx @@ -0,0 +1,74 @@ +import { ModalForm, ProFormText } from '@ant-design/pro-components'; +import React from 'react'; + +export type CreateFileInRepositoryModalFormProps = { + submitNewFile: ( + playbooksRepositoryUuid: string, + fileName: string, + fullPath: string, + mode: 'playbook' | 'directory', + ) => Promise; + path: string; + playbooksRepositoryUuid: string; + playbooksRepositoryName: string; + mode: 'playbook' | 'directory'; + opened: boolean; + setModalOpened: any; + basedPath: string; +}; + +const CreateFileInRepositoryModalForm: React.FC< + CreateFileInRepositoryModalFormProps +> = (props) => { + return ( + + title={`Create a new ${props.mode}`} + open={props.opened} + autoFocusFirstInput + modalProps={{ + destroyOnClose: true, + onCancel: () => props.setModalOpened(false), + }} + onFinish={async (values) => { + return await props + .submitNewFile( + props.playbooksRepositoryUuid, + values.name, + `${props.path}/${values.name}`, + props.mode, + ) + .then(() => { + props.setModalOpened(false); + }); + }} + > + + + ); +}; + +export default CreateFileInRepositoryModalForm; diff --git a/client/src/pages/Playbooks/components/DirectoryTreeView.tsx b/client/src/pages/Playbooks/components/DirectoryTreeView.tsx new file mode 100644 index 00000000..0417f0c7 --- /dev/null +++ b/client/src/pages/Playbooks/components/DirectoryTreeView.tsx @@ -0,0 +1,96 @@ +import GalaxyStoreModal from '@/pages/Playbooks/components/GalaxyStoreModal'; +import CreateFileInRepositoryModalForm from '@/pages/Playbooks/components/CreateFileInRepositoryModalForm'; +import NewFileDrawerForm from '@/pages/Playbooks/components/NewFileDrawerForm'; +import { ClientPlaybooksTrees } from '@/pages/Playbooks/components/TreeComponent'; +import { AppstoreOutlined } from '@ant-design/icons'; +import { Button, Card, Tree } from 'antd'; +import React, { Key } from 'react'; +import { API } from 'ssm-shared-lib'; + +const { DirectoryTree } = Tree; + +type DirectoryTreeViewProps = { + onSelect: ( + selectedKeys: Key[], + info: { + event: 'select'; + selected: boolean; + node: any; + selectedNodes: any; + nativeEvent: MouseEvent; + }, + ) => void; + playbookRepositories: ClientPlaybooksTrees[]; + newRepositoryFileModal: { + opened: boolean; + playbookRepositoryUuid: string; + playbookRepositoryName: string; + playbookRepositoryBasePath: string; + path: string; + mode: string; + }; + setNewRepositoryFileModal: any; + createNewFile: ( + playbooksRepositoryUuid: string, + fileName: string, + fullPath: string, + mode: 'directory' | 'playbook', + ) => Promise; + selectedFile?: API.PlaybookFile; +}; + +const DirectoryTreeView: React.FC = (props) => { + const [storeModal, setStoreModal] = React.useState(false); + const { + onSelect, + selectedFile, + newRepositoryFileModal, + createNewFile, + playbookRepositories, + setNewRepositoryFileModal, + } = props; + // @ts-ignore + return ( + } + onClick={() => setStoreModal(true)} + > + Store + , + ]} + > + + + + + setNewRepositoryFileModal({ + ...newRepositoryFileModal, + opened: false, + }) + } + path={newRepositoryFileModal.path} + submitNewFile={createNewFile} + /> + + ); +}; + +export default DirectoryTreeView; diff --git a/client/src/pages/Playbooks/components/ExtraVarsViewEditor.tsx b/client/src/pages/Playbooks/components/ExtraVarsViewEditor.tsx index f93c7e5a..2da851d6 100644 --- a/client/src/pages/Playbooks/components/ExtraVarsViewEditor.tsx +++ b/client/src/pages/Playbooks/components/ExtraVarsViewEditor.tsx @@ -14,10 +14,10 @@ import { getPlaybooks, postExtraVarValue, postPlaybookExtraVar, -} from '@/services/rest/ansible'; +} from '@/services/rest/playbooks'; export type ExtraVarsViewEditionProps = { - playbook: API.PlaybookFileList; + playbook: API.PlaybookFile; }; const ExtraVarsViewEditor: React.FC = ( @@ -33,7 +33,7 @@ const ExtraVarsViewEditor: React.FC = ( }; const handleRemove = async (extraVarName: string) => { await deletePlaybookExtraVar( - extraVarViewEditorProps.playbook.label, + extraVarViewEditorProps.playbook.uuid, extraVarName, ); setCurrentExtraVars( @@ -47,7 +47,7 @@ const ExtraVarsViewEditor: React.FC = ( return ( <> = ( true, }; await postPlaybookExtraVar( - extraVarViewEditorProps.playbook?.value, + extraVarViewEditorProps.playbook?.uuid, newExtraVar, ); setShowCreateNewVarForm(false); diff --git a/client/src/pages/Playbooks/components/FloatingButtonsBar.tsx b/client/src/pages/Playbooks/components/FloatingButtonsBar.tsx new file mode 100644 index 00000000..346283ef --- /dev/null +++ b/client/src/pages/Playbooks/components/FloatingButtonsBar.tsx @@ -0,0 +1,52 @@ +import { RedoOutlined, SaveOutlined } from '@ant-design/icons'; +import { FloatButton, Popconfirm } from 'antd'; +import { DeleteOutline } from 'antd-mobile-icons'; +import React from 'react'; +import { API } from 'ssm-shared-lib'; + +type FloatingButtonsBarProps = { + onClickSavePlaybook: () => void; + onClickDeletePlaybook: () => void; + onClickUndoPlaybook: () => void; + selectedFile?: API.PlaybookFile; +}; + +const FloatingButtonsBar: React.FC = (props) => { + const { + onClickSavePlaybook, + onClickDeletePlaybook, + onClickUndoPlaybook, + selectedFile, + } = props; + return ( + + } + /> + + } + onClick={onClickUndoPlaybook} + /> + {selectedFile?.custom && ( + + } + /> + + )} + + ); +}; + +export default FloatingButtonsBar; diff --git a/client/src/pages/Playbooks/components/GalaxyStoreModal.tsx b/client/src/pages/Playbooks/components/GalaxyStoreModal.tsx index 87006882..74210913 100644 --- a/client/src/pages/Playbooks/components/GalaxyStoreModal.tsx +++ b/client/src/pages/Playbooks/components/GalaxyStoreModal.tsx @@ -3,7 +3,7 @@ import { getCollection, getCollections, postInstallCollection, -} from '@/services/rest/ansible'; +} from '@/services/rest/playbooks'; import { ProDescriptions, ProForm, diff --git a/client/src/pages/Playbooks/components/NewFileDrawerForm.tsx b/client/src/pages/Playbooks/components/NewFileDrawerForm.tsx new file mode 100644 index 00000000..f1a70df8 --- /dev/null +++ b/client/src/pages/Playbooks/components/NewFileDrawerForm.tsx @@ -0,0 +1,151 @@ +import { getPlaybooksRepositories } from '@/services/rest/playbooks-repositories'; +import { + DrawerForm, + ProFormDependency, + ProFormSelect, + ProFormText, + ProFormRadio, +} from '@ant-design/pro-components'; +import { Button } from 'antd'; +import { AddCircleOutline } from 'antd-mobile-icons'; +import React from 'react'; +import { API } from 'ssm-shared-lib'; + +export type NewFileModalFormProps = { + submitNewFile: ( + playbooksRepositoryUuid: string, + fileName: string, + fullPath: string, + mode: 'playbook' | 'directory', + ) => Promise; +}; + +const NewFileDrawerForm: React.FC = (props) => { + const [repositories, setRepositories] = React.useState< + API.PlaybooksRepository[] | undefined + >(); + const [loading, setLoading] = React.useState(false); + return ( + + title={`Create a new file`} + autoFocusFirstInput + loading={loading} + resize={{ + onResize() { + console.log('resize!'); + }, + maxWidth: window.innerWidth * 0.8, + minWidth: 300, + }} + trigger={ + + } + drawerProps={{ + destroyOnClose: true, + }} + onFinish={async (values) => { + setLoading(true); + await props + .submitNewFile( + values.repository, + values.name, + `${ + repositories?.find((e) => e.uuid === values.repository)?.path + }/${values.name}`, + values.type as 'playbook' | 'directory', + ) + .finally(() => { + setLoading(false); + }); + }} + > + { + return await getPlaybooksRepositories().then((e) => { + setRepositories(e?.data); + return e.data + ? e.data.map((f) => { + return { + label: f.name, + value: f.uuid, + path: f.path, + }; + }) + : []; + }); + }} + /> + + {({ repository }) => { + if (repository) { + return ( + <> + + + {({ type }) => { + if (type) { + return ( + e.uuid === repository) + ?.name + }/`, + suffix: type === 'playbook' ? '.yml' : undefined, + }} + rules={[ + { + required: true, + message: `Please input a name!`, + }, + { + pattern: new RegExp('^[0-9a-zA-Z\\-]{0,100}$'), + message: + 'Please enter a valid name (only alphanumerical and "-" authorized), max 100 chars', + }, + ]} + /> + ); + } + }} + + + ); + } + }} + + + ); +}; + +export default NewFileDrawerForm; diff --git a/client/src/pages/Playbooks/components/NewPlaybookModalForm.tsx b/client/src/pages/Playbooks/components/NewPlaybookModalForm.tsx deleted file mode 100644 index 6dae3a9a..00000000 --- a/client/src/pages/Playbooks/components/NewPlaybookModalForm.tsx +++ /dev/null @@ -1,68 +0,0 @@ -import { ModalForm, ProFormText } from '@ant-design/pro-components'; -import { Button } from 'antd'; -import { AddCircleOutline } from 'antd-mobile-icons'; -import React from 'react'; -import { API } from 'ssm-shared-lib'; - -export type NewPlaybookModalFormProps = { - submitNewPlaybook: (name: string) => Promise; - playbookFilesList: API.PlaybookFileList[]; -}; - -const NewPlaybookModalForm: React.FC = (props) => { - return ( - - title={'Create a new playbook'} - trigger={ - - } - autoFocusFirstInput - modalProps={{ - destroyOnClose: true, - }} - onFinish={async (values) => { - return await props.submitNewPlaybook(values.name); - }} - > - e.label === value) === - -1 - ) { - return Promise.resolve(); - } - return Promise.reject('Playbook name already exists'); - }, - }, - ]} - /> - - ); -}; - -export default NewPlaybookModalForm; diff --git a/client/src/pages/Playbooks/components/PlaybookDropdownMenu.tsx b/client/src/pages/Playbooks/components/PlaybookDropdownMenu.tsx new file mode 100644 index 00000000..4a5d97a1 --- /dev/null +++ b/client/src/pages/Playbooks/components/PlaybookDropdownMenu.tsx @@ -0,0 +1,112 @@ +import { Callbacks } from '@/pages/Playbooks/components/TreeComponent'; +import { + DeleteOutlined, + FileOutlined, + FolderOpenOutlined, +} from '@ant-design/icons'; +import { Dropdown, MenuProps, Popconfirm } from 'antd'; +import React from 'react'; +import { DirectoryTree } from 'ssm-shared-lib'; + +type PlaybookDropdownMenuType = { + path: string; + type: DirectoryTree.CONSTANTS; + playbookRepository: { uuid: string; name: string; basePath: string }; + children: React.ReactNode; + callbacks: Callbacks; + cannotDelete?: boolean; +}; + +type PlaybookDrownMenuItemType = { + fileType: DirectoryTree.CONSTANTS | 'any'; + playbookQuickRef?: string; + icon?: React.JSX.Element; + label: string; + key: string; +}; + +const menuItems: PlaybookDrownMenuItemType[] = [ + { + label: 'Create new directory', + icon: , + key: '1', + fileType: DirectoryTree.CONSTANTS.DIRECTORY, + }, + { + label: 'Create an empty playbook', + icon: , + key: '2', + fileType: DirectoryTree.CONSTANTS.DIRECTORY, + }, + { + label: 'Delete', + icon: , + key: '3', + fileType: 'any', + }, +]; + +const PlaybookDropdownMenu: React.FC = (props) => { + const [open, setOpen] = React.useState(false); + const items = menuItems + .filter( + (e) => + (e.fileType === 'any' && !props.cannotDelete) || + e.fileType === props.type, + ) + .map((e) => { + return { + key: e.key, + label: e.label, + icon: e.icon, + }; + }) as MenuProps['items']; + + const onClick: MenuProps['onClick'] = async ({ key, domEvent }) => { + domEvent.stopPropagation(); + switch (key) { + case '1': + props.callbacks.callbackCreateDirectory( + props.path, + props.playbookRepository.uuid, + props.playbookRepository.name, + props.playbookRepository.basePath, + ); + break; + case '2': + props.callbacks.callbackCreatePlaybook( + props.path, + props.playbookRepository.uuid, + props.playbookRepository.name, + props.playbookRepository.basePath, + ); + break; + case '3': + setOpen(true); + break; + } + }; + + return ( + <> + + props.callbacks.callbackDeleteFile( + props.playbookRepository.uuid, + props.path, + ) + } + onCancel={() => setOpen(false)} + okText="Yes" + cancelText="No" + /> + + {props.children} + + + ); +}; +export default PlaybookDropdownMenu; diff --git a/client/src/pages/Playbooks/components/TreeComponent.tsx b/client/src/pages/Playbooks/components/TreeComponent.tsx new file mode 100644 index 00000000..8e95514d --- /dev/null +++ b/client/src/pages/Playbooks/components/TreeComponent.tsx @@ -0,0 +1,188 @@ +import { + SimpleIconsGit, + StreamlineLocalStorageFolderSolid, +} from '@/components/Icons/CustomIcons'; +import { Typography } from 'antd'; +import React, { ReactNode } from 'react'; +import { + DirectoryTree, + API, + Playbooks, + DirectoryTree as DT, +} from 'ssm-shared-lib'; +import PlaybookDropdownMenu from './PlaybookDropdownMenu'; + +export type ClientPlaybooksTrees = { + isLeaf?: boolean; + _name: string; + title: ReactNode; + children?: ClientPlaybooksTrees[]; + key: string; + uuid?: string; + extraVars?: API.ExtraVar[]; + custom?: boolean; + selectable?: boolean; +}; + +export type Callbacks = { + callbackCreateDirectory: ( + path: string, + playbookRepositoryUuid: string, + playbookRepositoryName: string, + playbookRepositoryBasePath: string, + ) => void; + callbackCreatePlaybook: ( + path: string, + playbookRepositoryUuid: string, + playbookRepositoryName: string, + playbookRepositoryBasePath: string, + ) => void; + callbackDeleteFile: (path: string, playbookRepositoryUuid: string) => void; +}; + +export type RootNode = {}; + +export function buildTree( + rootNode: API.PlaybooksRepository, + callbacks: Callbacks, +) { + return { + _name: rootNode.name, + title: ( + + + {rootNode.name} + + + ), + key: rootNode.name, + icon: + rootNode.type === Playbooks.PlaybooksRepositoryType.LOCAL ? ( + + ) : ( + + ), + selectable: false, + children: rootNode.children + ? recursiveTreeTransform( + { + ...rootNode, + type: DT.CONSTANTS.DIRECTORY, + path: rootNode.name, + }, + { uuid: rootNode.uuid, name: rootNode.name, basePath: rootNode.path }, + callbacks, + ) + : undefined, + }; +} + +export function recursiveTreeTransform( + tree: DirectoryTree.ExtendedTreeNode, + playbookRepository: { uuid: string; name: string; basePath: string }, + callbacks: Callbacks, + depth = 0, +): ClientPlaybooksTrees[] { + const node = tree; + const newTree: ClientPlaybooksTrees[] = []; + if (node.children) { + for (const child of node.children) { + if (child && child.type === DirectoryTree.CONSTANTS.DIRECTORY) { + if (depth > 20) { + throw new Error( + 'Depth is too high, to prevent any infinite loop, directories depth is limited to 20', + ); + } + newTree.push({ + key: child.path, + _name: child.name, + title: ( + + + {child.name} + + + ), + selectable: false, + children: recursiveTreeTransform( + child, + playbookRepository, + callbacks, + depth + 1, + ), + }); + } else { + if (child) { + newTree.push({ + key: child.path, + _name: child.name, + title: ( + + + {child.name} + + + ), + uuid: (child as DirectoryTree.ExtendedTreeNode).uuid, + extraVars: (child as DirectoryTree.ExtendedTreeNode).extraVars, + custom: (child as DirectoryTree.ExtendedTreeNode).custom, + isLeaf: true, + }); + } + } + } + } else { + newTree.push({ + key: node.path, + _name: node.name, + title: ( + + + {node.name} + + + ), + uuid: node.uuid, + extraVars: node.extraVars, + custom: node.custom, + isLeaf: true, + }); + } + return newTree; +} diff --git a/client/src/pages/Playbooks/index.tsx b/client/src/pages/Playbooks/index.tsx index eacfd7ae..ab5ca4e5 100644 --- a/client/src/pages/Playbooks/index.tsx +++ b/client/src/pages/Playbooks/index.tsx @@ -1,42 +1,26 @@ import Title, { PageContainerTitleColors } from '@/components/Template/Title'; +import DirectoryTreeView from '@/pages/Playbooks/components/DirectoryTreeView'; import ExtraVarsViewEditor from '@/pages/Playbooks/components/ExtraVarsViewEditor'; -import GalaxyStoreModal from '@/pages/Playbooks/components/GalaxyStoreModal'; -import NewPlaybookModalForm from '@/pages/Playbooks/components/NewPlaybookModalForm'; +import FloatingButtonsBar from '@/pages/Playbooks/components/FloatingButtonsBar'; +import { + buildTree, + ClientPlaybooksTrees, +} from '@/pages/Playbooks/components/TreeComponent'; import { deletePlaybook, - getPlaybooks, - newPlaybook, patchPlaybook, readPlaybookContent, -} from '@/services/rest/ansible'; -import { - AppstoreOutlined, - FileOutlined, - FileSearchOutlined, - PlaySquareOutlined, - RedoOutlined, - SaveOutlined, -} from '@ant-design/icons'; +} from '@/services/rest/playbooks'; import { - ModalForm, - PageContainer, - ProFormText, -} from '@ant-design/pro-components'; + createDirectoryInRepository, + createEmptyPlaybookInRepository, + deleteAnyInRepository, + getPlaybooksRepositories, +} from '@/services/rest/playbooks-repositories'; +import { PlaySquareOutlined } from '@ant-design/icons'; +import { PageContainer } from '@ant-design/pro-components'; import Editor, { Monaco } from '@monaco-editor/react'; -import { - Button, - Card, - Col, - FloatButton, - message, - Popconfirm, - Result, - Row, - Spin, - Tree, - Typography, -} from 'antd'; -import { AddCircleOutline, DeleteOutline } from 'antd-mobile-icons'; +import { Col, message, Result, Row, Spin, Typography } from 'antd'; import type { DirectoryTreeProps } from 'antd/es/tree'; import { editor } from 'monaco-editor'; import { configureMonacoYaml } from 'monaco-yaml'; @@ -59,38 +43,114 @@ window.MonacoEnvironment = { }, }; -const { DirectoryTree } = Tree; - const { Paragraph, Text } = Typography; const Index: React.FC = () => { - const [playbookFilesList, setPlaybookFilesList] = React.useState< - API.PlaybookFileList[] + const [playbookRepositories, setPlaybookRepositories] = React.useState< + ClientPlaybooksTrees[] >([]); const [selectedFile, setSelectedFile] = React.useState< - API.PlaybookFileList | undefined + API.PlaybookFile | undefined >(); const [downloadedContent, setDownloadedContent] = React.useState< string | undefined >(); const [isLoading, setIsLoading] = React.useState(false); const editorRef = React.useRef(null); - const [storeModal, setStoreModal] = React.useState(false); + const [newRepositoryFileModal, setNewRepositoryFileModal] = React.useState({ + opened: false, + playbookRepositoryUuid: '', + playbookRepositoryName: '', + playbookRepositoryBasePath: '', + path: '', + mode: '', + }); const asyncFetchPlaybookContent = async () => { if (selectedFile) { - await readPlaybookContent(selectedFile.value).then((content) => { - setDownloadedContent(content.data); - }); + if (selectedFile.uuid) { + await readPlaybookContent(selectedFile.uuid) + .then((content) => { + setDownloadedContent(content.data); + }) + .catch(() => { + setIsLoading(false); + }); + } else { + message.error( + 'Selected file has no uuid - try to synchronize again the repo', + ); + } } setIsLoading(false); }; - const asyncFetch = async (createdPlaybook?: string) => { - await getPlaybooks().then((list) => { + + const handleShouldCreatePlaybook = ( + path: string, + playbookRepositoryUuid: string, + playbookRepositoryName: string, + playbookRepositoryBasePath: string, + ) => { + setNewRepositoryFileModal({ + opened: true, + mode: 'playbook', + path: path, + playbookRepositoryUuid: playbookRepositoryUuid, + playbookRepositoryName: playbookRepositoryName, + playbookRepositoryBasePath: playbookRepositoryBasePath, + }); + }; + + const handleShouldCreateRepository = ( + path: string, + playbookRepositoryUuid: string, + playbookRepositoryName: string, + playbookRepositoryBasePath: string, + ) => { + setNewRepositoryFileModal({ + opened: true, + mode: 'directory', + path: path, + playbookRepositoryUuid: playbookRepositoryUuid, + playbookRepositoryName: playbookRepositoryName, + playbookRepositoryBasePath: playbookRepositoryBasePath, + }); + }; + + const handleShouldDeleteFile = async ( + playbooksRepositoryUuid: string, + path: string, + ) => { + setIsLoading(true); + await deleteAnyInRepository(playbooksRepositoryUuid, path) + .then(async () => { + message.warning(`File deleted`); + if (selectedFile?.path === path) { + setSelectedFile(undefined); + } + // eslint-disable-next-line @typescript-eslint/no-use-before-define + await asyncFetch(); + setIsLoading(false); + }) + .catch(() => { + setIsLoading(false); + }); + }; + + const asyncFetch = async (createdPlaybook?: API.PlaybookFile) => { + await getPlaybooksRepositories().then((list) => { if (list?.data) { - setPlaybookFilesList(list.data); + setPlaybookRepositories( + list.data.map((e: API.PlaybooksRepository) => { + return buildTree(e, { + callbackCreatePlaybook: handleShouldCreatePlaybook, + callbackCreateDirectory: handleShouldCreateRepository, + callbackDeleteFile: handleShouldDeleteFile, + }); + }), + ); if (createdPlaybook) { - setSelectedFile(list.data.find((e) => e.label === createdPlaybook)); + setSelectedFile(createdPlaybook); } } else { message.error({ @@ -101,27 +161,27 @@ const Index: React.FC = () => { }); }; useEffect(() => { - asyncFetch(); + void asyncFetch(); }, []); useEffect(() => { - asyncFetchPlaybookContent(); + void asyncFetchPlaybookContent(); }, [selectedFile]); const onSelect: DirectoryTreeProps['onSelect'] = (keys, info) => { - if (keys[0] !== selectedFile?.label) { + if (keys[0] !== selectedFile?.path) { setIsLoading(true); - setSelectedFile( - playbookFilesList.find((e) => e.value === (keys[0] as string)), - ); + const playbook = info.node as unknown as ClientPlaybooksTrees; + setSelectedFile({ + name: playbook._name, + extraVars: playbook.extraVars || [], + uuid: playbook.uuid || '', + path: playbook.key, + custom: playbook.custom, + }); } console.log('Trigger Select', keys, info); }; - const onExpand: DirectoryTreeProps['onExpand'] = (keys, info) => { - console.log('Trigger Expand', keys, info); - }; - const keywords = ['ansible', 'test', 'ggg']; - // eslint-disable-next-line @typescript-eslint/no-unused-vars,@typescript-eslint/no-shadow const editorDidMount = (editor: IStandaloneCodeEditor, monaco: Monaco) => { // @ts-ignore @@ -143,9 +203,9 @@ const Index: React.FC = () => { const onClickDeletePlaybook = async () => { if (selectedFile) { setIsLoading(true); - await deletePlaybook(selectedFile.value) + await deletePlaybook(selectedFile.uuid) .then(async () => { - message.warning(`Playbook '${selectedFile.value}' deleted`); + message.warning(`Playbook '${selectedFile.name}' deleted`); setSelectedFile(undefined); await asyncFetch(); setIsLoading(false); @@ -163,9 +223,9 @@ const Index: React.FC = () => { if (selectedFile && editorRef.current?.getValue()) { setIsLoading(true); // @ts-ignore - await patchPlaybook(selectedFile.value, editorRef.current.getValue()) + await patchPlaybook(selectedFile.uuid, editorRef.current.getValue()) .then(() => { - message.success(`Playbook '${selectedFile.value}' saved`); + message.success(`Playbook '${selectedFile.name}' saved`); setIsLoading(false); }) .catch(() => { @@ -185,20 +245,50 @@ const Index: React.FC = () => { } }; - const submitNewPlaybook = async (name: string) => { + const createNewFile = async ( + playbooksRepositoryUuid: string, + fileName: string, + fullPath: string, + mode: 'directory' | 'playbook', + ): Promise => { setIsLoading(true); - return await newPlaybook(name) - .then(async () => { - message.success(`Playbook '${name}' successfully created`); - await asyncFetch(name); - setIsLoading(false); - return true; - }) - .catch(async () => { - await asyncFetch(); - setIsLoading(false); - return false; - }); + if (mode === 'playbook') { + return await createEmptyPlaybookInRepository( + playbooksRepositoryUuid, + fileName, + fullPath, + ) + .then(async (e) => { + message.success(`Playbook '${fileName}' successfully created`); + await asyncFetch(e.data); + setIsLoading(false); + return true; + }) + .catch(async () => { + await asyncFetch(); + setIsLoading(false); + return false; + }); + } + if (mode === 'directory') { + return await createDirectoryInRepository( + playbooksRepositoryUuid, + fileName, + fullPath, + ) + .then(async () => { + message.success(`Directory '${fileName}' successfully created`); + await asyncFetch(); + setIsLoading(false); + return true; + }) + .catch(async () => { + await asyncFetch(); + setIsLoading(false); + return false; + }); + } + throw new Error('Mode is unknown'); }; return ( @@ -216,81 +306,31 @@ const Index: React.FC = () => { - } - onClick={() => setStoreModal(true)} - > - Store - , - ]} - > - - { - return { - title: e.label, - key: e.value, - icon: ({ selected }) => - selected ? : , - }; - })} - selectedKeys={[selectedFile?.value as React.Key]} - /> - - + {(selectedFile && ( <> - - } - /> - - } - onClick={onClickUndoPlaybook} - /> - {!selectedFile?.value.startsWith('_') && ( - - } - /> - - )} - + diff --git a/client/src/services/rest/index.ts b/client/src/services/rest/index.ts index b3b48eee..bd7184a6 100644 --- a/client/src/services/rest/index.ts +++ b/client/src/services/rest/index.ts @@ -3,7 +3,7 @@ import * as user from './user'; import * as device from './device'; import * as cron from './cron'; -import * as ansible from './ansible'; +import * as ansible from './playbooks'; import * as logs from './logs'; import * as deviceauth from './deviceauth'; import * as usersettings from './usersettings'; diff --git a/client/src/services/rest/playbooks-repositories.ts b/client/src/services/rest/playbooks-repositories.ts new file mode 100644 index 00000000..90a91392 --- /dev/null +++ b/client/src/services/rest/playbooks-repositories.ts @@ -0,0 +1,284 @@ +import { request } from '@umijs/max'; +import { API } from 'ssm-shared-lib'; + +export async function getPlaybooksRepositories(): Promise { + return request('/api/playbooks-repository/', { + method: 'GET', + ...{}, + }); +} + +export async function getGitRepositories( + params?: any, + options?: Record, +) { + return request>( + `/api/playbooks-repository/git/`, + { + method: 'GET', + params: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export async function getLocalRepositories( + params?: any, + options?: Record, +) { + return request>( + `/api/playbooks-repository/local/`, + { + method: 'GET', + params: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export async function postLocalRepositories( + repository: Partial, + params?: any, + options?: Record, +) { + return request>( + `/api/playbooks-repository/local/${repository.uuid}`, + { + data: { ...repository }, + method: 'POST', + params: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export async function putLocalRepositories( + repository: API.LocalRepository, + params?: any, + options?: Record, +) { + return request>( + `/api/playbooks-repository/local/`, + { + data: { ...repository }, + method: 'PUT', + params: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export async function deleteLocalRepository( + uuid: string, + params?: any, + options?: Record, +) { + return request(`/api/playbooks-repository/local/${uuid}`, { + method: 'DELETE', + params: { + ...params, + }, + ...(options || {}), + }); +} + +export async function syncToDatabaseLocalRepository( + uuid: string, + params?: any, + options?: Record, +) { + return request( + `/api/playbooks-repository/local/${uuid}/sync-to-database-repository`, + { + method: 'POST', + params: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export async function postGitRepository( + repository: API.GitRepository, + params?: any, + options?: Record, +) { + return request( + `/api/playbooks-repository/git/${repository.uuid}`, + { + data: { ...repository }, + method: 'POST', + params: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export async function putGitRepository( + repository: API.GitRepository, + params?: any, + options?: Record, +) { + return request(`/api/playbooks-repository/git/`, { + data: { ...repository }, + method: 'PUT', + params: { + ...params, + }, + ...(options || {}), + }); +} + +export async function deleteGitRepository( + uuid: string, + params?: any, + options?: Record, +) { + return request(`/api/playbooks-repository/git/${uuid}`, { + method: 'DELETE', + params: { + ...params, + }, + ...(options || {}), + }); +} + +export async function syncToDatabaseGitRepository( + uuid: string, + params?: any, + options?: Record, +) { + return request( + `/api/playbooks-repository/git/${uuid}/sync-to-database-repository`, + { + method: 'POST', + params: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export async function forcePullGitRepository( + uuid: string, + params?: any, + options?: Record, +) { + return request( + `/api/playbooks-repository/git/${uuid}/force-pull-repository`, + { + method: 'POST', + params: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export async function forceCloneGitRepository( + uuid: string, + params?: any, + options?: Record, +) { + return request( + `/api/playbooks-repository/git/${uuid}/force-clone-repository`, + { + method: 'POST', + params: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export async function commitAndSyncGitRepository( + uuid: string, + params?: any, + options?: Record, +) { + return request( + `/api/playbooks-repository/git/${uuid}/commit-and-sync-repository`, + { + method: 'POST', + params: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export async function forceRegisterGitRepository( + uuid: string, + params?: any, + options?: Record, +) { + return request( + `/api/playbooks-repository/git/${uuid}/force-register`, + { + method: 'POST', + params: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export async function createEmptyPlaybookInRepository( + playbooksRepositoryUuid: string, + playbookName: string, + fullPath: string, +) { + return request>( + `/api/playbooks-repository/${playbooksRepositoryUuid}/playbook/${playbookName}/`, + { + method: 'PUT', + data: { fullPath: fullPath }, + ...{}, + }, + ); +} + +export async function createDirectoryInRepository( + playbooksRepositoryUuid: string, + directoryName: string, + fullPath: string, +) { + return request( + `/api/playbooks-repository/${playbooksRepositoryUuid}/directory/${directoryName}/`, + { + method: 'PUT', + data: { fullPath: fullPath }, + ...{}, + }, + ); +} + +export async function deleteAnyInRepository( + playbooksRepositoryUuid: string, + fullPath: string, +) { + return request( + `/api/playbooks-repository/${playbooksRepositoryUuid}`, + { + method: 'DELETE', + data: { fullPath: fullPath }, + ...{}, + }, + ); +} diff --git a/client/src/services/rest/ansible.ts b/client/src/services/rest/playbooks.ts similarity index 51% rename from client/src/services/rest/ansible.ts rename to client/src/services/rest/playbooks.ts index 2646c861..6b256877 100644 --- a/client/src/services/rest/ansible.ts +++ b/client/src/services/rest/playbooks.ts @@ -1,84 +1,83 @@ import { request } from '@umijs/max'; import { API } from 'ssm-shared-lib'; +export async function getPlaybooks( + options?: Record, +): Promise> { + return request>(`/api/playbooks/`, { + method: 'GET', + ...(options || {}), + }); +} + +export async function readPlaybookContent(playbookUuid: string) { + return request(`/api/playbooks/${playbookUuid}`, { + method: 'GET', + ...{}, + }); +} + +export async function patchPlaybook(playbookUuid: string, content: string) { + return request(`/api/playbooks/${playbookUuid}/`, { + method: 'PATCH', + data: { content: content }, + ...{}, + }); +} + export async function executePlaybook( playbook: string, target: string[] | undefined, extraVars?: API.ExtraVars, options?: Record, ) { - return request(`/api/ansible/exec/playbook/${playbook}`, { + return request(`/api/playbooks/exec/${playbook}`, { method: 'POST', data: { playbook: playbook, target: target, extraVars: extraVars }, ...(options || {}), }); } -export async function getExecLogs(execId: string) { - return request(`/api/ansible/exec/${execId}/logs`, { - method: 'GET', - ...{}, +export async function executePlaybookByQuickRef( + quickRef: string, + target: string[] | undefined, + extraVars?: API.ExtraVars, + options?: Record, +) { + return request(`/api/playbooks/exec/quick-ref/${quickRef}`, { + method: 'POST', + data: { quickRef: quickRef, target: target, extraVars: extraVars }, + ...(options || {}), }); } -export async function getTaskStatuses(execId: string) { - return request(`/api/ansible/exec/${execId}/status`, { +export async function getExecLogs(execId: string) { + return request(`/api/playbooks/exec/${execId}/logs/`, { method: 'GET', ...{}, }); } -export async function getPlaybooks(): Promise { - return request('/api/ansible/playbooks', { +export async function getTaskStatuses(execId: string) { + return request(`/api/playbooks/exec/${execId}/status/`, { method: 'GET', ...{}, }); } -export async function readPlaybookContent(playbook: string) { - return request(`/api/ansible/playbooks/${playbook}`, { - method: 'GET', +export async function deletePlaybook(playbookUuid: string) { + return request(`/api/playbooks/${playbookUuid}/`, { + method: 'DELETE', ...{}, }); } -export async function patchPlaybook(playbook: string, content: string) { - return request( - `/api/ansible/playbooks/${playbook}/`, - { - method: 'PATCH', - data: { content: content }, - ...{}, - }, - ); -} - -export async function newPlaybook(playbook: string) { - return request( - `/api/ansible/playbooks/${playbook}/`, - { - method: 'PUT', - ...{}, - }, - ); -} - -export async function deletePlaybook(playbook: string) { - return request( - `/api/ansible/playbooks/${playbook}/`, - { - method: 'DELETE', - ...{}, - }, - ); -} - export async function postPlaybookExtraVar( - playbook: string, + playbookUuid: string, extraVar: API.ExtraVar, ) { return request( - `/api/ansible/playbooks/${playbook}/extravars`, + `/api/playbooks/${playbookUuid}/extravars`, { data: { extraVar: extraVar }, method: 'POST', @@ -88,11 +87,11 @@ export async function postPlaybookExtraVar( } export async function deletePlaybookExtraVar( - playbook: string, + playbookUuid: string, extraVar: string, ) { return request( - `/api/ansible/playbooks/${playbook}/extravars/${extraVar}`, + `/api/playbooks/${playbookUuid}/extravars/${extraVar}`, { method: 'DELETE', ...{}, @@ -101,18 +100,21 @@ export async function deletePlaybookExtraVar( } export async function postExtraVarValue(extraVar: string, value: string) { - return request(`/api/ansible/extravars/${extraVar}`, { - data: { value: value }, - method: 'POST', - ...{}, - }); + return request( + `/api/playbooks/extravars/${extraVar}`, + { + data: { value: value }, + method: 'POST', + ...{}, + }, + ); } export async function getCollections( params?: any, options?: Record, ) { - return request('/api/ansible/galaxy/collection', { + return request('/api/playbooks/galaxy/collection', { method: 'GET', params: { ...params, @@ -125,7 +127,7 @@ export async function getCollection( params: { name: string; namespace: string; version: string }, options?: Record, ) { - return request('/api/ansible/galaxy/collection/details', { + return request('/api/playbooks/galaxy/collection/details', { method: 'GET', params: { ...params, @@ -139,7 +141,7 @@ export async function postInstallCollection( params?: any, options?: Record, ) { - return request('/api/ansible/galaxy/collection/install', { + return request('/api/playbooks/galaxy/collection/install', { method: 'POST', data: body, params: { diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 8b6050df..b941f6e8 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -46,6 +46,7 @@ services: - ./.env.dev volumes: - ./server/src:/server/src + - ./.data.dev/playbooks:/playbooks environment: NODE_ENV: development DEBUG: nodejs-docker-express:* diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index d07ffb2e..d4dd89f0 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -33,6 +33,8 @@ services: depends_on: - mongo - redis + volumes: + - ./.data.prod/playbooks:/playbooks build: context: ./server additional_contexts: diff --git a/docker-compose.yml b/docker-compose.yml index 06cd58b9..a3af7b57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,8 @@ services: env_file: .env environment: NODE_ENV: production + volumes: + - ./.data.prod/playbooks:/playbooks client: image: "ghcr.io/squirrelcorporation/squirrelserversmanager-client:latest" restart: unless-stopped diff --git a/server/Dockerfile b/server/Dockerfile index 125fec3d..1de2b7b5 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -4,7 +4,7 @@ LABEL org.opencontainers.image.description="SSM Server" LABEL org.opencontainers.image.licenses="GNU AFFERO GENERAL PUBLIC LICENSE" WORKDIR /server -RUN apk update && apk add ansible nmap sudo openssh sshpass py3-pip expect +RUN apk update && apk add ansible nmap sudo openssh sshpass py3-pip expect gcompat libcurl RUN npm install -g npm@latest RUN rm -f /usr/lib/python3.12/EXTERNALLY-MANAGED RUN pip install ansible-runner \ @@ -23,7 +23,7 @@ FROM base as production ENV NODE_ENV=production WORKDIR /server COPY . . -RUN npm install --verbose --no-audit +RUN npm install --verbose --no-audit RUN npm run build CMD ["node", "./dist/src/index.js"] @@ -31,5 +31,5 @@ FROM base as dev ENV NODE_ENV=development WORKDIR /server COPY ./nodemon.json . -RUN npm install -g nodemon && npm install --verbose +RUN npm install -g nodemon && npm install --verbose --no-audit CMD ["npm", "run", "dev"] diff --git a/server/nodemon.json b/server/nodemon.json index 5bd48a52..b527abf1 100644 --- a/server/nodemon.json +++ b/server/nodemon.json @@ -9,8 +9,10 @@ "src/logs/*", "src/**/*.{spec,test}.ts", "src/**/ansible/artifacts/*", - "src/**/ansible/inventory/*.json" + "src/**/ansible/inventory/*.json", + "src/tests/helpers/directory-tree/test_data/*" ], "exec": "ts-node --transpile-only src/index.ts" } + diff --git a/server/package.json b/server/package.json index 4bbef8e4..c3159320 100644 --- a/server/package.json +++ b/server/package.json @@ -2,6 +2,7 @@ "name": "ssm-server", "description": "SSM Server - A simple way to manage all your servers", "main": "index", + "license": "AGPL-3.0 license", "scripts": { "dev": "pwd && nodemon", "start": "npm run build-shared && npm run build && cross-env NODE_ENV=production node dist/index.js", @@ -23,7 +24,7 @@ "events": "^3.3.0", "express": "^4.19.2", "express-validator": "^7.1.0", - "joi": "^17.13.1", + "joi": "^17.13.3", "jsonwebtoken": "^9.0.2", "luxon": "^3.4.4", "mongoose": "^8.4.3", @@ -44,7 +45,11 @@ "semver": "^7.6.2", "shelljs": "^0.8.5", "ssm-shared-lib": "file:../shared-lib/", - "@aws-sdk/client-ecr": "^3.598.0" + "@aws-sdk/client-ecr": "^3.600.0", + "dugite": "^2.7.1", + "fs-extra": "^11.2.0", + "isomorphic-git": "^1.26.3", + "lodash": "^4.17.21" }, "overrides": { "minimatch": "5.1.2", @@ -65,17 +70,19 @@ "@types/passport-http-bearer": "^1.0.41", "@types/passport-jwt": "^4.0.1", "@types/shelljs": "^0.8.15", - "@types/uuid": "^9.0.8", + "@types/uuid": "^10.0.0", + "@types/lodash": "^4.17.5", + "@types/fs-extra": "^11.0.4", "@vitest/coverage-v8": "^1.6.0", "eslint": "^8.57.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.1.3", - "@typescript-eslint/eslint-plugin": "^7.13.1", + "@typescript-eslint/eslint-plugin": "^7.14.1", "eslint-plugin-import": "^2.29.1", - "node-mocks-http": "^1.14.1", + "node-mocks-http": "^1.15.0", "prettier": "^3.3.2", "ts-node": "^10.9.2", - "typescript": "^5.4.5", + "typescript": "^5.5.2", "vitest": "^1.6.0" } } diff --git a/server/src/ansible/_checkDeviceBeforeAdd.json b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_checkDeviceBeforeAdd.json similarity index 78% rename from server/src/ansible/_checkDeviceBeforeAdd.json rename to server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_checkDeviceBeforeAdd.json index 501f29be..215ea69d 100644 --- a/server/src/ansible/_checkDeviceBeforeAdd.json +++ b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_checkDeviceBeforeAdd.json @@ -1,5 +1,6 @@ { "playableInBatch": false, + "uniqueQuickRef": "checkDeviceBeforeAdd", "extraVars": [ { "extraVar": "_ssm_masterNodeUrl", diff --git a/server/src/ansible/_checkDeviceBeforeAdd.yml b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_checkDeviceBeforeAdd.yml similarity index 100% rename from server/src/ansible/_checkDeviceBeforeAdd.yml rename to server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_checkDeviceBeforeAdd.yml diff --git a/server/src/ansible/_installAgent.json b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_installAgent.json similarity index 87% rename from server/src/ansible/_installAgent.json rename to server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_installAgent.json index 95618786..7b214526 100644 --- a/server/src/ansible/_installAgent.json +++ b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_installAgent.json @@ -1,5 +1,6 @@ { "playableInBatch": false, + "uniqueQuickRef": "installAgent", "extraVars": [ { "extraVar": "_ssm_masterNodeUrl", diff --git a/server/src/ansible/_installAgent.yml b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_installAgent.yml similarity index 100% rename from server/src/ansible/_installAgent.yml rename to server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_installAgent.yml diff --git a/server/src/ansible/_reinstallAgent.json b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_reinstallAgent.json similarity index 87% rename from server/src/ansible/_reinstallAgent.json rename to server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_reinstallAgent.json index 95618786..d781523b 100644 --- a/server/src/ansible/_reinstallAgent.json +++ b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_reinstallAgent.json @@ -1,5 +1,6 @@ { "playableInBatch": false, + "uniqueQuickRef": "reinstallAgent", "extraVars": [ { "extraVar": "_ssm_masterNodeUrl", diff --git a/server/src/ansible/_reinstallAgent.yml b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_reinstallAgent.yml similarity index 100% rename from server/src/ansible/_reinstallAgent.yml rename to server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_reinstallAgent.yml diff --git a/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_restartAgent.json b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_restartAgent.json new file mode 100644 index 00000000..23b8d108 --- /dev/null +++ b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_restartAgent.json @@ -0,0 +1,4 @@ +{ + "playableInBatch": true, + "uniqueQuickRef": "restartAgent" +} diff --git a/server/src/ansible/_restartAgent.yml b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_restartAgent.yml similarity index 100% rename from server/src/ansible/_restartAgent.yml rename to server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_restartAgent.yml diff --git a/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_retrieveAgentLogs.json b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_retrieveAgentLogs.json new file mode 100644 index 00000000..4871a4df --- /dev/null +++ b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_retrieveAgentLogs.json @@ -0,0 +1,4 @@ +{ + "playableInBatch": false, + "uniqueQuickRef": "retrieveAgentLogs" +} diff --git a/server/src/ansible/_retrieveAgentLogs.yml b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_retrieveAgentLogs.yml similarity index 100% rename from server/src/ansible/_retrieveAgentLogs.yml rename to server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_retrieveAgentLogs.yml diff --git a/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_uninstallAgent.json b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_uninstallAgent.json new file mode 100644 index 00000000..5a264ced --- /dev/null +++ b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_uninstallAgent.json @@ -0,0 +1,4 @@ +{ + "playableInBatch": false, + "uniqueQuickRef": "uninstallAgent" +} diff --git a/server/src/ansible/_uninstallAgent.yml b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_uninstallAgent.yml similarity index 100% rename from server/src/ansible/_uninstallAgent.yml rename to server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_uninstallAgent.yml diff --git a/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_updateAgent.json b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_updateAgent.json new file mode 100644 index 00000000..7f16bf8a --- /dev/null +++ b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_updateAgent.json @@ -0,0 +1,16 @@ +{ + "playableInBatch": false, + "uniqueQuickRef": "updateAgent", + "extraVars": [ + { + "extraVar": "_ssm_masterNodeUrl", + "required": true, + "canBeOverride": true + }, + { + "extraVar": "_ssm_deviceId", + "required": true, + "canBeOverride": false + } + ] +} diff --git a/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_updateAgent.yml b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_updateAgent.yml new file mode 100644 index 00000000..06d58be8 --- /dev/null +++ b/server/src/ansible/00000000-0000-0000-0000-000000000000/agent/_updateAgent.yml @@ -0,0 +1,89 @@ +- name: Update agent on targeted device + hosts: all + become: true + gather_facts: false + vars: + base_path: /opt/squirrelserversmanager + + tasks: + - name: Install agent on targeted device + ansible.builtin.debug: + msg: Host ID {{ _ssm_deviceId }} with API URL {{ _ssm_masterNodeUrl }} + + - name: Check if NodeJS does exist + ansible.builtin.raw: which node + check_mode: false + changed_when: false + failed_when: which_res_node.rc > 1 + register: which_res_node + + - name: Check if NPM does exist + ansible.builtin.raw: which npm + check_mode: false + changed_when: false + failed_when: which_res_npm.rc > 1 + register: which_res_npm + + - name: Install "PM2" node.js package globally. + community.general.npm: + name: pm2 + global: true + state: latest + + - name: Install PM2 LogRotate + command: pm2 install pm2-logrotate + + - name: Check out agent + ansible.builtin.git: + force: true + repo: 'https://github.com/SquirrelCorporation/SquirrelServersManager-Agent.git' + dest: "{{ base_path }}" + timeout: 600 + + - name: Stop PM2 Agent if running + command: pm2 stop agent + ignore_errors: yes + + - name: Delete PM2 Agent if present + command: pm2 delete agent + ignore_errors: yes + + - name: Clean directory + command: + chdir: "{{ base_path }}" + cmd: rm -f ssm-agent && rm -f agent.blob && rm -rf ./build && rm -f ./hostid.txt + + - name: Write Node Url in .env file + copy: + content: "API_URL_MASTER={{ _ssm_masterNodeUrl }}" + dest: "{{ base_path }}/.env" + + - name: Write HostId in hostid.txt file + copy: + content: "{{ _ssm_deviceId }}" + dest: "{{ base_path }}/hostid.txt" + + - name: NPM install + command: + chdir: "{{ base_path }}" + cmd: npm install + + - name: NPM run build + command: + chdir: "{{ base_path }}" + cmd: npm run build + + - name: Start PM2 Agent + command: + chdir: "{{ base_path }}" + cmd: pm2 start -f "./build/agent.js" + + - name: Install PM2 on startup + command: pm2 startup + + - name: Save Agent on startup + command: pm2 save + + - name: Save Agent on startup + command: pm2 update + diff --git a/server/src/ansible/00000000-0000-0000-0000-000000000000/device/_ping.json b/server/src/ansible/00000000-0000-0000-0000-000000000000/device/_ping.json new file mode 100644 index 00000000..3608de0d --- /dev/null +++ b/server/src/ansible/00000000-0000-0000-0000-000000000000/device/_ping.json @@ -0,0 +1,3 @@ +{ + "uniqueQuickRef": "ping" +} diff --git a/server/src/ansible/_ping.yml b/server/src/ansible/00000000-0000-0000-0000-000000000000/device/_ping.yml similarity index 100% rename from server/src/ansible/_ping.yml rename to server/src/ansible/00000000-0000-0000-0000-000000000000/device/_ping.yml diff --git a/server/src/ansible/00000000-0000-0000-0000-000000000000/device/_reboot.json b/server/src/ansible/00000000-0000-0000-0000-000000000000/device/_reboot.json new file mode 100644 index 00000000..687ce88e --- /dev/null +++ b/server/src/ansible/00000000-0000-0000-0000-000000000000/device/_reboot.json @@ -0,0 +1,4 @@ +{ + "playableInBatch": true, + "uniqueQuickRef": "reboot" +} diff --git a/server/src/ansible/_reboot.yml b/server/src/ansible/00000000-0000-0000-0000-000000000000/device/_reboot.yml similarity index 100% rename from server/src/ansible/_reboot.yml rename to server/src/ansible/00000000-0000-0000-0000-000000000000/device/_reboot.yml diff --git a/server/src/ansible/00000000-0000-0000-0000-000000000000/device/_upgrade.json b/server/src/ansible/00000000-0000-0000-0000-000000000000/device/_upgrade.json new file mode 100644 index 00000000..b68a4fe0 --- /dev/null +++ b/server/src/ansible/00000000-0000-0000-0000-000000000000/device/_upgrade.json @@ -0,0 +1,4 @@ +{ + "playableInBatch": true, + "uniqueQuickRef": "upgrade" +} diff --git a/server/src/ansible/_upgrade.yml b/server/src/ansible/00000000-0000-0000-0000-000000000000/device/_upgrade.yml similarity index 100% rename from server/src/ansible/_upgrade.yml rename to server/src/ansible/00000000-0000-0000-0000-000000000000/device/_upgrade.yml diff --git a/server/src/ansible/_reboot.json b/server/src/ansible/_reboot.json deleted file mode 100644 index 00df3760..00000000 --- a/server/src/ansible/_reboot.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "playableInBatch": true -} diff --git a/server/src/ansible/_restartAgent.json b/server/src/ansible/_restartAgent.json deleted file mode 100644 index 00df3760..00000000 --- a/server/src/ansible/_restartAgent.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "playableInBatch": true -} diff --git a/server/src/ansible/_uninstallAgent.json b/server/src/ansible/_uninstallAgent.json deleted file mode 100644 index edd2be0d..00000000 --- a/server/src/ansible/_uninstallAgent.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "playableInBatch": false -} diff --git a/server/src/ansible/_upgrade.json b/server/src/ansible/_upgrade.json deleted file mode 100644 index 00df3760..00000000 --- a/server/src/ansible/_upgrade.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "playableInBatch": true -} diff --git a/server/src/ansible/ssm-ansible-run.py b/server/src/ansible/ssm-ansible-run.py index 013224fe..91af7eb4 100644 --- a/server/src/ansible/ssm-ansible-run.py +++ b/server/src/ansible/ssm-ansible-run.py @@ -31,7 +31,7 @@ def get_configuration(url): def status_handler(data, runner_config): - plugin_config = get_configuration('http://localhost:3000/ansible/hook/task/status') + plugin_config = get_configuration('http://localhost:3000/playbooks/hook/task/status') if plugin_config['runner_url'] is not None: status = send_request(plugin_config['runner_url'], data=data, @@ -42,7 +42,7 @@ def status_handler(data, runner_config): logger.info("HTTP Plugin Skipped") def event_handler(data): - plugin_config = get_configuration('http://localhost:3000/ansible/hook/task/event') + plugin_config = get_configuration('http://localhost:3000/playbooks/hook/task/event') if plugin_config['runner_url'] is not None: status = send_request(plugin_config['runner_url'], data=data, diff --git a/server/src/ansible/ssm-ansible-vault-password-client.py b/server/src/ansible/ssm-ansible-vault-password-client.py index 4e78a27d..c2aef3a6 100755 --- a/server/src/ansible/ssm-ansible-vault-password-client.py +++ b/server/src/ansible/ssm-ansible-vault-password-client.py @@ -8,7 +8,7 @@ logger = logging.getLogger('ansible-runner') def send_request(): - url_actual = "http://localhost:3000/ansible/vault" + url_actual = "http://localhost:3000/playbooks/vault" headers = headers = { 'Authorization': "Bearer {}".format(os.getenv("SSM_API_KEY"))} session = requests.Session() logger.debug("Getting {}".format(url_actual)) diff --git a/server/src/core/startup/index.ts b/server/src/core/startup/index.ts index 62723126..33da2282 100644 --- a/server/src/core/startup/index.ts +++ b/server/src/core/startup/index.ts @@ -1,26 +1,51 @@ -import { SettingsKeys } from 'ssm-shared-lib'; +import { Playbooks, SettingsKeys } from 'ssm-shared-lib'; import { getFromCache } from '../../data/cache'; import initRedisValues from '../../data/cache/defaults'; +import PlaybooksRepositoryRepo from '../../data/database/repository/PlaybooksRepositoryRepo'; +import Crons from '../../integrations/crons'; +import WatcherEngine from '../../integrations/docker/core/WatcherEngine'; import providerConf from '../../integrations/docker/registries/providers/provider.conf'; +import PlaybooksRepositoryEngine from '../../integrations/playbooks-repository/PlaybooksRepositoryEngine'; import logger from '../../logger'; import ContainerRegistryUseCases from '../../use-cases/ContainerRegistryUseCases'; import DeviceAuthUseCases from '../../use-cases/DeviceAuthUseCases'; -import PlaybookUseCases from '../../use-cases/PlaybookUseCases'; import { setAnsibleVersion } from '../system/version'; -async function needConfigurationInit() { - logger.info(`[CONFIGURATION] - needInit`); - void DeviceAuthUseCases.saveAllDeviceAuthSshKeys(); +const corePlaybooksRepository = { + name: 'ssm-core', + uuid: '00000000-0000-0000-0000-000000000000', + enabled: true, + type: Playbooks.PlaybooksRepositoryType.LOCAL, + directory: '/server/src/ansible', + default: true, +}; + +const toolsPlaybooksRepository = { + name: 'ssm-tools', + uuid: '00000000-0000-0000-0000-000000000001', + enabled: true, + type: Playbooks.PlaybooksRepositoryType.LOCAL, + directory: '/server/src/ansible', + default: true, +}; +async function init() { const version = await getFromCache(SettingsKeys.GeneralSettingsKeys.SCHEME_VERSION); + logger.info(`[CONFIGURATION] - initialization`); + logger.info(`[CONFIGURATION] - initialization - Scheme Version: ${version}`); - logger.info(`[CONFIGURATION] - needInit - Scheme Version: ${version}`); + await PlaybooksRepositoryRepo.updateOrCreate(corePlaybooksRepository); + await PlaybooksRepositoryRepo.updateOrCreate(toolsPlaybooksRepository); + await PlaybooksRepositoryEngine.init(); + void DeviceAuthUseCases.saveAllDeviceAuthSshKeys(); + void Crons.initScheduledJobs(); + void WatcherEngine.init(); if (version !== SettingsKeys.DefaultValue.SCHEME_VERSION) { + logger.warn(`[CONFIGURATION] - Scheme version differed, starting writing updates`); await initRedisValues(); - await PlaybookUseCases.initPlaybook(); void setAnsibleVersion(); - + await PlaybooksRepositoryEngine.syncAllRegistered(); providerConf .filter(({ persist }) => persist) .map((e) => { @@ -30,5 +55,5 @@ async function needConfigurationInit() { } export default { - needConfigurationInit, + init, }; diff --git a/server/src/core/system/version.ts b/server/src/core/system/version.ts index 85015ce8..b4aa64d7 100644 --- a/server/src/core/system/version.ts +++ b/server/src/core/system/version.ts @@ -2,13 +2,13 @@ import { getFromCache, setToCache } from '../../data/cache'; import Shell from '../../integrations/shell'; export async function getAnsibleVersion() { - const ansibleVersion = await getFromCache('ansible-version'); + const ansibleVersion = await getFromCache('playbooks-version'); if (ansibleVersion) { return ansibleVersion; } else { const retrievedAnsibleVersion = await Shell.getAnsibleVersion(); if (retrievedAnsibleVersion) { - await setToCache('ansible-version', retrievedAnsibleVersion); + await setToCache('playbooks-version', retrievedAnsibleVersion); } return retrievedAnsibleVersion; } @@ -17,6 +17,6 @@ export async function getAnsibleVersion() { export async function setAnsibleVersion() { const retrievedAnsibleVersion = await Shell.getAnsibleVersion(); if (retrievedAnsibleVersion) { - await setToCache('ansible-version', retrievedAnsibleVersion); + await setToCache('playbooks-version', retrievedAnsibleVersion); } } diff --git a/server/src/data/database/model/Playbook.ts b/server/src/data/database/model/Playbook.ts index a16c855a..333e63e8 100644 --- a/server/src/data/database/model/Playbook.ts +++ b/server/src/data/database/model/Playbook.ts @@ -1,13 +1,19 @@ import { Schema, model } from 'mongoose'; +import { v4 as uuidv4 } from 'uuid'; +import PlaybooksRepository from './PlaybooksRepository'; export const DOCUMENT_NAME = 'Playbook'; export const COLLECTION_NAME = 'playbooks'; export default interface Playbook { custom: boolean; + uuid?: string; name: string; + path: string; extraVars?: [{ extraVar: string; required: boolean }]; playableInBatch?: boolean; + playbooksRepository?: PlaybooksRepository; + uniqueQuickRef?: string; } const schema = new Schema( @@ -15,8 +21,21 @@ const schema = new Schema( name: { type: Schema.Types.String, required: true, + }, + uuid: { + type: Schema.Types.String, + default: uuidv4, + required: true, + unique: true, + }, + uniqueQuickRef: { + type: Schema.Types.String, unique: true, }, + path: { + type: Schema.Types.String, + required: true, + }, playableInBatch: { type: Schema.Types.Boolean, default: true, @@ -30,6 +49,13 @@ const schema = new Schema( type: [Object], required: false, }, + playbooksRepository: { + type: Schema.Types.ObjectId, + ref: 'PlaybooksRepository', + required: true, + select: true, + index: true, + }, }, { timestamps: true, diff --git a/server/src/data/database/model/PlaybooksRepository.ts b/server/src/data/database/model/PlaybooksRepository.ts new file mode 100644 index 00000000..905f2bd9 --- /dev/null +++ b/server/src/data/database/model/PlaybooksRepository.ts @@ -0,0 +1,88 @@ +import { Schema, model } from 'mongoose'; +import { Playbooks } from 'ssm-shared-lib'; + +export const DOCUMENT_NAME = 'PlaybooksRepository'; +export const COLLECTION_NAME = 'playbooksrepository'; + +export default interface PlaybooksRepository { + _id?: string; + uuid: string; + type: Playbooks.PlaybooksRepositoryType; + name: string; + accessToken?: string; + branch?: string; + email?: string; + userName?: string; + remoteUrl?: string; + directory?: string; + enabled: boolean; + default?: boolean; + tree?: any; + createdAt?: Date; + updatedAt?: Date; +} + +const schema = new Schema( + { + uuid: { + type: Schema.Types.String, + required: true, + unique: true, + }, + type: { + type: Schema.Types.String, + required: true, + }, + default: { + type: Schema.Types.Boolean, + required: true, + default: false, + }, + name: { + type: Schema.Types.String, + required: true, + }, + accessToken: { + type: Schema.Types.String, + required: false, + }, + branch: { + type: Schema.Types.String, + required: false, + }, + email: { + type: Schema.Types.String, + required: false, + }, + userName: { + type: Schema.Types.String, + required: false, + }, + remoteUrl: { + type: Schema.Types.String, + required: false, + }, + directory: { + type: Schema.Types.String, + required: true, + }, + enabled: { + type: Schema.Types.Boolean, + required: true, + default: true, + }, + tree: { + type: Object, + }, + }, + { + timestamps: true, + versionKey: false, + }, +); + +export const PlaybooksRepositoryModel = model( + DOCUMENT_NAME, + schema, + COLLECTION_NAME, +); diff --git a/server/src/data/database/repository/PlaybookRepo.ts b/server/src/data/database/repository/PlaybookRepo.ts index d2e3de53..ed364765 100644 --- a/server/src/data/database/repository/PlaybookRepo.ts +++ b/server/src/data/database/repository/PlaybookRepo.ts @@ -1,4 +1,5 @@ import Playbook, { PlaybookModel } from '../model/Playbook'; +import PlaybooksRepository from '../model/PlaybooksRepository'; async function create(playbook: Playbook): Promise { const created = await PlaybookModel.create(playbook); @@ -6,7 +7,7 @@ async function create(playbook: Playbook): Promise { } async function updateOrCreate(playbook: Playbook): Promise { - return PlaybookModel.findOneAndUpdate({ name: playbook.name }, playbook, { upsert: true }) + return PlaybookModel.findOneAndUpdate({ path: playbook.path }, playbook, { upsert: true }) .lean() .exec(); } @@ -15,13 +16,54 @@ async function findAll(): Promise { return await PlaybookModel.find().sort({ createdAt: -1 }).lean().exec(); } -async function findOne(name: string): Promise { +async function findAllWithActiveRepositories(): Promise { + return await PlaybookModel.find() + .populate({ path: 'playbooksRepository', match: { enabled: { $eq: true } } }) + .sort({ createdAt: -1 }) + .lean() + .exec(); +} + +async function findOneByName(name: string): Promise { return await PlaybookModel.findOne({ name: name }).lean().exec(); } +async function findOneByUuid(uuid: string): Promise { + return await PlaybookModel.findOne({ uuid: uuid }).lean().exec(); +} + +async function listAllByRepository( + playbooksRepository: PlaybooksRepository, +): Promise { + return await PlaybookModel.find({ playbooksRepository: playbooksRepository }).lean().exec(); +} + +async function deleteByUuid(uuid: string): Promise { + await PlaybookModel.deleteOne({ uuid: uuid }).lean().exec(); +} + +async function findOneByPath(path: string): Promise { + return await PlaybookModel.findOne({ path: path }).lean().exec(); +} + +async function findOneByUniqueQuickReference(quickRef: string): Promise { + return await PlaybookModel.findOne({ uniqueQuickRef: quickRef }).lean().exec(); +} + +async function deleteByRepository(playbooksRepository: PlaybooksRepository): Promise { + await PlaybookModel.deleteOne({ playbooksRepository: playbooksRepository }).exec(); +} + export default { create, findAll, updateOrCreate, - findOne, + findOneByName, + findOneByUuid, + listAllByRepository, + deleteByUuid, + findOneByPath, + findAllWithActiveRepositories, + findOneByUniqueQuickReference, + deleteByRepository, }; diff --git a/server/src/data/database/repository/PlaybooksRepositoryRepo.ts b/server/src/data/database/repository/PlaybooksRepositoryRepo.ts new file mode 100644 index 00000000..b57dea81 --- /dev/null +++ b/server/src/data/database/repository/PlaybooksRepositoryRepo.ts @@ -0,0 +1,72 @@ +import { Playbooks } from 'ssm-shared-lib'; +import PlaybooksRepository, { PlaybooksRepositoryModel } from '../model/PlaybooksRepository'; + +async function update( + playbooksRepository: PlaybooksRepository, +): Promise { + playbooksRepository.updatedAt = new Date(); + return PlaybooksRepositoryModel.findOneAndUpdate( + { uuid: playbooksRepository.uuid }, + playbooksRepository, + ) + .lean() + .exec(); +} + +async function updateOrCreate( + playbooksRepository: PlaybooksRepository, +): Promise { + return PlaybooksRepositoryModel.findOneAndUpdate( + { uuid: playbooksRepository.uuid }, + playbooksRepository, + { upsert: true }, + ) + .lean() + .exec(); +} + +async function create(playbooksRepository: PlaybooksRepository): Promise { + const created = await PlaybooksRepositoryModel.create(playbooksRepository); + return created.toObject(); +} + +async function findAllActiveWithType( + type: Playbooks.PlaybooksRepositoryType, +): Promise { + return await PlaybooksRepositoryModel.find({ enabled: true, type: type }).lean().exec(); +} + +async function findAllActive(): Promise { + return await PlaybooksRepositoryModel.find({ enabled: true }).lean().exec(); +} + +async function findAllWithType( + type: Playbooks.PlaybooksRepositoryType, +): Promise { + return await PlaybooksRepositoryModel.find({ type: type }).lean().exec(); +} + +async function deleteByUuid(uuid: string): Promise { + await PlaybooksRepositoryModel.deleteOne({ uuid: uuid }).exec(); +} + +async function findByUuid(uuid: string): Promise { + return await PlaybooksRepositoryModel.findOne({ uuid: uuid }).lean().exec(); +} + +async function saveTree(playbooksRepository: PlaybooksRepository, tree: any): Promise { + playbooksRepository.tree = tree; + await PlaybooksRepositoryModel.updateOne({ _id: playbooksRepository._id }, playbooksRepository); +} + +export default { + create, + findAllActiveWithType, + findAllActive, + findAllWithType, + update, + deleteByUuid, + findByUuid, + saveTree, + updateOrCreate, +}; diff --git a/server/src/helpers/directory-tree/README.md b/server/src/helpers/directory-tree/README.md new file mode 100644 index 00000000..684a947d --- /dev/null +++ b/server/src/helpers/directory-tree/README.md @@ -0,0 +1,190 @@ +## Usage + +```js +const dirTree = require("directory-tree"); +const tree = dirTree("/some/path"); +``` + +And you can also filter by an extensions regex: +This is useful for including only certain types of files. + +```js +const dirTree = require("directory-tree"); +const filteredTree = dirTree("/some/path", { extensions: /\.txt/ }); +``` + +Example for filtering multiple extensions with Regex. + +```js +const dirTree = require("directory-tree"); +const filteredTree = dirTree("/some/path", { + extensions: /\.(md|js|html|java|py|rb)$/ +}); +``` + +You can also exclude paths from the tree using a regex: + +```js +const dirTree = require("directory-tree"); +const filteredTree = dirTree("/some/path", { exclude: /some_path_to_exclude/ }); +``` + +You can also specify which additional attributes you would like to be included about each file/directory: + +```js +const dirTree = require('directory-tree'); +const filteredTree = dirTree('/some/path', {attributes:['mode', 'mtime']}); +``` + +The default attributes are `[name, path]` for Files and `[name, path, children]` for Directories + +A callback function can be executed with each file that matches the extensions provided: + +```js +const PATH = require('path'); +const dirTree = require('directory-tree'); + +const tree = dirTree('./test/test_data', {extensions:/\.txt$/}, (item, PATH, stats) => { + console.log(item); +}); +``` + +The callback function takes the directory item (has path, name, size, and extension) and an instance of [node path](https://nodejs.org/api/path.html) and an instance of [node FS.stats](https://nodejs.org/api/fs.html#fs_class_fs_stats). + +You can also pass a callback function for directories: +```js +const PATH = require('path'); +const dirTree = require('directory-tree'); + +const tree = dirTree('./test/test_data', {extensions:/\.txt$/}, null, (item, PATH, stats) => { + console.log(item); +}); +``` + +## Options + +`exclude` : `RegExp|RegExp[]` - A RegExp or an array of RegExp to test for exclusion of directories. + +`extensions` : `RegExp` - A RegExp to test for exclusion of files with the matching extension. + +`attributes` : `string[]` - Array of [FS.stats](https://nodejs.org/api/fs.html#fs_class_fs_stats) attributes. + +`normalizePath` : `Boolean` - If true, windows style paths will be normalized to unix style pathes (/ instead of \\). + +`depth` : `number` - If presented, reads so many nested dirs as specified in argument. Usage of size attribute with depth option is prohibited. + +## Result + +Given a directory structured like this: + +``` +photos +├── summer +│ └── june +│ └── windsurf.jpg +└── winter + └── january + ├── ski.png + └── snowboard.jpg +``` + +`directory-tree` with `attributes: ["size", "type", "extension"]` will return this JS object: + +```json +{ + "path": "photos", + "name": "photos", + "size": 600, + "type": "directory", + "children": [ + { + "path": "photos/summer", + "name": "summer", + "size": 400, + "type": "directory", + "children": [ + { + "path": "photos/summer/june", + "name": "june", + "size": 400, + "type": "directory", + "children": [ + { + "path": "photos/summer/june/windsurf.jpg", + "name": "windsurf.jpg", + "size": 400, + "type": "file", + "extension": ".jpg" + } + ] + } + ] + }, + { + "path": "photos/winter", + "name": "winter", + "size": 200, + "type": "directory", + "children": [ + { + "path": "photos/winter/january", + "name": "january", + "size": 200, + "type": "directory", + "children": [ + { + "path": "photos/winter/january/ski.png", + "name": "ski.png", + "size": 100, + "type": "file", + "extension": ".png" + }, + { + "path": "photos/winter/january/snowboard.jpg", + "name": "snowboard.jpg", + "size": 100, + "type": "file", + "extension": ".jpg" + } + ] + } + ] + } + ] +} +``` + +## Adding custom fields +You can easily extend a `DirectoryTree` object with custom fields by adding them to the custom field. +For example add an `id` based on the path of a `DirectoryTree` object for each directory and file like so: +``` +import { createHash } from 'crypto'; +import * as directoryTree from 'directory-tree'; +import { DirectoryTree, DirectoryTreeOptions, DirectoryTreeCallback } from 'directory-tree'; + +const callback: DirectoryTreeCallback = ( + item: DirectoryTree, + path: string + ) => { + item.custom = {id: createHash('sha1').update(path).digest('base64')}; + }; + +const dirTree: DirectoryTree & { id?: string } = directoryTree( + "", + {}, + callback, + callback +); + +// to explore the object with the new custom fields +console.log(JSON.stringify(dirTree, null, 2)); + +``` + +## Note + +Device, FIFO and socket files are ignored. + +Files to which the user does not have permissions are included in the directory +tree, however, directories to which the user does not have permissions, along +with all of its contained files, are completely ignored. diff --git a/server/src/helpers/directory-tree/directory-tree.ts b/server/src/helpers/directory-tree/directory-tree.ts new file mode 100644 index 00000000..070eac3d --- /dev/null +++ b/server/src/helpers/directory-tree/directory-tree.ts @@ -0,0 +1,186 @@ +import * as FS from 'node:fs'; +import * as PATH from 'node:path'; +import { Stats } from 'fs-extra'; +import { DirectoryTree } from 'ssm-shared-lib'; + +function safeReadDirSync(path: FS.PathLike) { + let dirData: string[] = []; + try { + dirData = FS.readdirSync(path); + } catch (ex: any) { + if (ex.code === 'EACCES' || ex.code === 'EPERM') { + //User does not have permissions, ignore directory + return null; + } else { + throw ex; + } + } + return dirData; +} + +/** + * Normalizes windows style paths by replacing double backslashes with single forward slashes (unix style). + * @param {string} path + * @return {string} + */ +function normalizePath(path: string): string { + return path.replace(/\\/g, '/'); +} + +/** + * Tests if the supplied parameter is of type RegExp + * @param {any} regExp + * @return {Boolean} + */ +function isRegExp(regExp: any): boolean { + return typeof regExp === 'object' && regExp.constructor === RegExp; +} + +/** + * Collects the files and folders for a directory path into an Object, subject + * to the options supplied, and invoking optional + * @param {String} path + * @param {Object} options + * @param {function} onEachFile + * @param {function} onEachDirectory + * @param currentDepth + */ +function directoryTree( + path: string, + options?: { + depth?: any; + attributes?: string[]; + normalizePath?: any; + exclude?: any; + followSymlinks?: any; + symlinks?: any; + extensions?: any; + }, + onEachFile?: (arg0: DirectoryTree.TreeNode, arg1: string, arg2: FS.Stats) => void, + onEachDirectory?: (arg0: DirectoryTree.TreeNode, arg1: string, arg2: FS.Stats) => void, + currentDepth = 0, +) { + options = options || {}; + + if ( + options.depth !== undefined && + options.attributes && + options.attributes.indexOf('size') !== -1 + ) { + throw new Error('usage of size attribute with depth option is prohibited'); + } + + const name = PATH.basename(path); + path = options.normalizePath ? normalizePath(path) : path; + const item: DirectoryTree.TreeNode = { path, name }; + let stats; + let lstat; + + try { + stats = FS.statSync(path); + lstat = FS.lstatSync(path); + } catch (e) { + return null; + } + + // Skip if it matches the exclude regex + if (options.exclude) { + const excludes = isRegExp(options.exclude) ? [options.exclude] : options.exclude; + if (excludes.some((exclusion: RegExp) => exclusion.test(path))) { + return null; + } + } + + if (lstat.isSymbolicLink()) { + item.isSymbolicLink = true; + // Skip if symbolic links should not be followed + if (options.followSymlinks === false) { + return null; + } + // Initialize the symbolic links array to avoid infinite loops + if (!options.symlinks) { + options = { ...options, symlinks: [] }; + } + // Skip if a cyclic symbolic link has been found + if (options.symlinks.find((ino: number) => ino === lstat.ino)) { + return null; + } else { + options.symlinks.push(lstat.ino); + } + } + + if (stats.isFile()) { + const ext = PATH.extname(path).toLowerCase(); + + // Skip if it does not match the extension regex + if (options.extensions && !options.extensions.test(ext)) { + return null; + } + + if (options.attributes) { + options.attributes.forEach((attribute) => { + switch (attribute) { + case 'extension': + item.extension = ext; + break; + case 'type': + item.type = DirectoryTree.CONSTANTS.FILE; + break; + default: + item[attribute] = stats[attribute as keyof Stats]; + break; + } + }); + } + + if (onEachFile) { + onEachFile(item, path, stats); + } + } else if (stats.isDirectory()) { + const dirData = safeReadDirSync(path); + if (dirData === null) { + return null; + } + + if (options.depth === undefined || options.depth > currentDepth) { + item.children = dirData + .map((child) => + directoryTree( + PATH.join(path, child), + options, + onEachFile, + onEachDirectory, + currentDepth + 1, + ), + ) + .filter((e) => !!e); + } + + if (options.attributes) { + options.attributes.forEach((attribute) => { + switch (attribute) { + case 'size': + item.size = item.children?.reduce((prev, cur) => prev + (cur?.size || 0), 0); + break; + case 'type': + item.type = DirectoryTree.CONSTANTS.DIRECTORY; + break; + case 'extension': + break; + default: + item[attribute] = stats[attribute as keyof Stats]; + break; + } + }); + } + + if (onEachDirectory) { + onEachDirectory(item, path, stats); + } + } else { + return null; // Or set item.size = 0 for devices, FIFO and sockets ? + } + return item; +} + +export default directoryTree; diff --git a/server/src/index.ts b/server/src/index.ts index 70c83b14..348ee2f6 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -4,7 +4,6 @@ import cookieParser from 'cookie-parser'; import { SECRET } from './config'; import { connection } from './data/database'; import WatcherEngine from './integrations/docker/core/WatcherEngine'; -import Shell from './integrations/shell'; import { errorHandler } from './middlewares/errorHandler'; import routes from './routes'; import logger from './logger'; @@ -29,8 +28,7 @@ let server: any; const start = () => { logger.info(`Starting server...`); connection().then(async () => { - await Configuration.needConfigurationInit(); - Crons.initScheduledJobs(); + await Configuration.init(); app.use('/', routes); app.use(errorHandler); server = app.listen(3000, () => @@ -38,7 +36,6 @@ const start = () => { 🐿 Squirrel Servers Manager 🚀 Server ready at: http://localhost:3000`), ); - await WatcherEngine.init(); }); }; start(); diff --git a/server/src/integrations/ansible/AnsibleCmd.ts b/server/src/integrations/ansible/AnsibleCmd.ts index 89e153d4..5a0ac346 100644 --- a/server/src/integrations/ansible/AnsibleCmd.ts +++ b/server/src/integrations/ansible/AnsibleCmd.ts @@ -1,6 +1,6 @@ import { API } from 'ssm-shared-lib'; import User from '../../data/database/model/User'; -import { Ansible } from '../../types/typings'; +import { Playbooks } from '../../types/typings'; import ExtraVarsTransformer from './utils/ExtraVarsTransformer'; class AnsibleCommandBuilder { @@ -9,11 +9,11 @@ class AnsibleCommandBuilder { static readonly ansibleRunner = 'ssm-ansible-run.py'; static readonly ssmApiKeyEnv = 'SSM_API_KEY'; - sanitizeInventory(inventoryTargets: Ansible.All & Ansible.HostGroups) { + sanitizeInventory(inventoryTargets: Playbooks.All & Playbooks.HostGroups) { return "'" + JSON.stringify(inventoryTargets).replaceAll('\\\\', '\\') + "'"; } - getInventoryTargets(inventoryTargets: (Ansible.All & Ansible.HostGroups) | undefined) { + getInventoryTargets(inventoryTargets: (Playbooks.All & Playbooks.HostGroups) | undefined) { return `${inventoryTargets ? '--specific-host ' + this.sanitizeInventory(inventoryTargets) : ''}`; } @@ -28,7 +28,7 @@ class AnsibleCommandBuilder { buildAnsibleCmd( playbook: string, uuid: string, - inventoryTargets: (Ansible.All & Ansible.HostGroups) | undefined, + inventoryTargets: (Playbooks.All & Playbooks.HostGroups) | undefined, user: User, extraVars?: API.ExtraVars, ) { diff --git a/server/src/integrations/ansible/AnsibleGalaxyCmd.ts b/server/src/integrations/ansible/AnsibleGalaxyCmd.ts index a0491f84..efabe9ba 100644 --- a/server/src/integrations/ansible/AnsibleGalaxyCmd.ts +++ b/server/src/integrations/ansible/AnsibleGalaxyCmd.ts @@ -1,5 +1,5 @@ class AnsibleGalaxyCommandBuilder { - static readonly ansibleGalaxy = 'ansible-galaxy'; + static readonly ansibleGalaxy = 'playbooks-galaxy'; static readonly collection = 'collection'; getInstallCollectionCmd(name: string, namespace: string) { diff --git a/server/src/integrations/ansible/utils/InventoryTransformer.ts b/server/src/integrations/ansible/utils/InventoryTransformer.ts index 14428999..4e96a5d2 100644 --- a/server/src/integrations/ansible/utils/InventoryTransformer.ts +++ b/server/src/integrations/ansible/utils/InventoryTransformer.ts @@ -1,7 +1,7 @@ import { SsmAnsible } from 'ssm-shared-lib'; import DeviceAuth from '../../../data/database/model/DeviceAuth'; import logger from '../../../logger'; -import { Ansible } from '../../../types/typings'; +import { Playbooks } from '../../../types/typings'; function generateDeviceKey(uuid: string) { return `device${uuid.replaceAll('-', '')}`; @@ -10,7 +10,7 @@ function generateDeviceKey(uuid: string) { function inventoryBuilder(devicesAuth: DeviceAuth[]) { logger.info(`[TRANSFORMERS][INVENTORY] - Inventory for ${devicesAuth.length} device(s)`); // @ts-expect-error - const ansibleInventory: Ansible.Hosts = { + const ansibleInventory: Playbooks.Hosts = { _meta: { hostvars: {} }, all: { children: [] }, }; @@ -36,7 +36,7 @@ function inventoryBuilder(devicesAuth: DeviceAuth[]) { function inventoryBuilderForTarget(devicesAuth: Partial[]) { logger.info(`[TRANSFORMERS][INVENTORY] - Inventory for ${devicesAuth.length} device(s)`); - const ansibleInventory: Ansible.All & Ansible.HostGroups = { + const ansibleInventory: Playbooks.All & Playbooks.HostGroups = { // @ts-expect-error I cannot comprehend generic typescript type all: {}, }; @@ -50,7 +50,7 @@ function inventoryBuilderForTarget(devicesAuth: Partial[]) { vars: getInventoryConnectionVars(e), }; }); - logger.info(ansibleInventory); + logger.debug(ansibleInventory); return ansibleInventory; } diff --git a/server/src/integrations/docker/core/CustomAgent.ts b/server/src/integrations/docker/core/CustomAgent.ts index edabb2ca..ffb29b34 100644 --- a/server/src/integrations/docker/core/CustomAgent.ts +++ b/server/src/integrations/docker/core/CustomAgent.ts @@ -1,3 +1,4 @@ +// eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-nocheck import pino from 'pino'; import { Client } from 'ssh2'; diff --git a/server/src/integrations/docker/core/WatcherEngine.ts b/server/src/integrations/docker/core/WatcherEngine.ts index b15edb8e..7f5f88c2 100644 --- a/server/src/integrations/docker/core/WatcherEngine.ts +++ b/server/src/integrations/docker/core/WatcherEngine.ts @@ -38,7 +38,7 @@ function getStates() { * Return all supported registries * @returns {*} */ -export function getRegistries() { +export function getRegistries(): Registry[] { return getStates().registry; } diff --git a/server/src/integrations/git-repository/GitRepositoryComponent.ts b/server/src/integrations/git-repository/GitRepositoryComponent.ts new file mode 100644 index 00000000..5edce628 --- /dev/null +++ b/server/src/integrations/git-repository/GitRepositoryComponent.ts @@ -0,0 +1,136 @@ +import PlaybooksRepositoryComponent, { + AbstractComponent, + DIRECTORY_ROOT, + FILE_PATTERN, +} from '../playbooks-repository/PlaybooksRepositoryComponent'; +import { createDirectoryWithFullPath, findFilesInDirectory } from '../shell/utils'; +import { + GitStep, + IGitUserInfos, + IInitGitOptionsSyncImmediately, + ILoggerContext, + clone, + commitAndSync, + forcePull, +} from './lib'; + +class GitRepositoryComponent extends PlaybooksRepositoryComponent implements AbstractComponent { + private readonly options: IInitGitOptionsSyncImmediately; + + constructor( + uuid: string, + logger: any, + name: string, + branch: string, + email: string, + gitUserName: string, + accessToken: string, + remoteUrl: string, + ) { + super(uuid, name, DIRECTORY_ROOT); + this.uuid = uuid; + this.name = name; + const userInfo: IGitUserInfos = { + email: email, + gitUserName: gitUserName, + branch: branch, + accessToken: accessToken, + }; + this.options = { + dir: this.directory, + syncImmediately: true, + userInfo: userInfo, + remoteUrl: remoteUrl, + }; + + this.childLogger = logger.child( + { module: `git-repository/${this.name}/${branch}` }, + { msgPrefix: `[GIT_REPOSITORY] - ` }, + ); + } + + async clone() { + this.childLogger.info('Clone starting...'); + try { + void createDirectoryWithFullPath(this.directory); + await clone({ + ...this.options, + logger: { + debug: (message: string, context: ILoggerContext): unknown => + this.childLogger.info(message, { callerFunction: 'clone', ...context }), + warn: (message: string, context: ILoggerContext): unknown => + this.childLogger.warn(message, { callerFunction: 'clone', ...context }), + info: (message: GitStep, context: ILoggerContext): void => { + this.childLogger.info(message, { + callerFunction: 'clone', + ...context, + }); + }, + }, + }); + } catch (error) { + this.childLogger.error(error); + } + } + + async commitAndSync() { + try { + await commitAndSync({ + ...this.options, + logger: { + debug: (message: string, context: ILoggerContext): unknown => + this.childLogger.debug(message, { callerFunction: 'commitAndSync', ...context }), + warn: (message: string, context: ILoggerContext): unknown => + this.childLogger.warn(message, { callerFunction: 'commitAndSync', ...context }), + info: (message: GitStep, context: ILoggerContext): void => { + this.childLogger.info(message, { + callerFunction: 'commitAndSync', + ...context, + }); + }, + }, + }); + } catch (error) { + this.childLogger.error(error); + } + } + + async forcePull() { + try { + await forcePull({ + ...this.options, + logger: { + debug: (message: string, context: ILoggerContext): unknown => + this.childLogger.debug(message, { callerFunction: 'forcePull', ...context }), + warn: (message: string, context: ILoggerContext): unknown => + this.childLogger.warn(message, { callerFunction: 'forcePull', ...context }), + info: (message: GitStep, context: ILoggerContext): void => { + this.childLogger.info(message, { + callerFunction: 'forcePull', + ...context, + }); + }, + }, + }); + } catch (error) { + this.childLogger.error(error); + } + } + + async init() { + await this.clone(); + } + + async syncToRepository() { + const files = await findFilesInDirectory(this.directory, FILE_PATTERN); + for (const file of files) { + this.childLogger.info(`syncToDatabase --> ${file}`); + } + } + + async syncFromRepository() { + await this.forcePull(); + } +} + +export default GitRepositoryComponent; diff --git a/server/src/integrations/git-repository/lib/clone.ts b/server/src/integrations/git-repository/lib/clone.ts new file mode 100644 index 00000000..f12fcbf4 --- /dev/null +++ b/server/src/integrations/git-repository/lib/clone.ts @@ -0,0 +1,76 @@ +import { GitProcess } from 'dugite'; +import { truncate } from 'lodash'; +import { credentialOff, credentialOn } from './credential'; +import { defaultGitInfo as defaultDefaultGitInfo } from './defaultGitInfo'; +import { GitPullPushError, SyncParameterMissingError } from './errors'; +import { initGitWithBranch } from './init'; +import { getRemoteName } from './inspect'; +import { GitStep, IGitUserInfos, ILogger } from './interface'; + +export async function clone(options: { + /** Optional fallback of userInfo. If some info is missing in userInfo, will use defaultGitInfo instead. */ + defaultGitInfo?: typeof defaultDefaultGitInfo; + /** wiki folder path, can be relative, should exist before function call */ + dir: string; + logger?: ILogger; + /** the storage service url we are sync to, for example your github repo url */ + remoteUrl?: string; + /** user info used in the commit message */ + userInfo?: IGitUserInfos; +}): Promise { + const { dir, remoteUrl, userInfo, logger, defaultGitInfo = defaultDefaultGitInfo } = options; + const { gitUserName, branch } = userInfo ?? defaultGitInfo; + const { accessToken } = userInfo ?? {}; + + if (accessToken === '' || accessToken === undefined) { + throw new SyncParameterMissingError('accessToken'); + } + if (remoteUrl === '' || remoteUrl === undefined) { + throw new SyncParameterMissingError('remoteUrl'); + } + + const logProgress = (step: GitStep): unknown => + logger?.info(step, { + functionName: 'clone', + step, + dir, + remoteUrl, + }); + const logDebug = (message: string, step: GitStep): unknown => + logger?.debug(message, { + functionName: 'clone', + step, + dir, + remoteUrl, + }); + + logProgress(GitStep.PrepareCloneOnlineWiki); + + logDebug( + JSON.stringify({ + remoteUrl, + gitUserName, + accessToken: truncate(accessToken, { + length: 24, + }), + }), + GitStep.PrepareCloneOnlineWiki, + ); + logDebug(`Running git init for clone in dir ${dir}`, GitStep.PrepareCloneOnlineWiki); + await initGitWithBranch(dir, branch, { initialCommit: false }); + const remoteName = await getRemoteName(dir, branch); + logDebug(`Successfully Running git init for clone in dir ${dir}`, GitStep.PrepareCloneOnlineWiki); + logProgress(GitStep.StartConfiguringGithubRemoteRepository); + await credentialOn(dir, remoteUrl, gitUserName, accessToken, remoteName); + try { + logProgress(GitStep.StartFetchingFromGithubRemote); + const { stderr: pullStdError, exitCode } = await GitProcess.exec(['pull', remoteName, `${branch}:${branch}`], dir); + if (exitCode === 0) { + logProgress(GitStep.SynchronizationFinish); + } else { + throw new GitPullPushError(options, pullStdError); + } + } finally { + await credentialOff(dir, remoteName, remoteUrl); + } +} diff --git a/server/src/integrations/git-repository/lib/commitAndSync.ts b/server/src/integrations/git-repository/lib/commitAndSync.ts new file mode 100644 index 00000000..9bb7f29c --- /dev/null +++ b/server/src/integrations/git-repository/lib/commitAndSync.ts @@ -0,0 +1,252 @@ +import { GitProcess } from 'dugite'; +import { credentialOff, credentialOn } from './credential'; +import { defaultGitInfo as defaultDefaultGitInfo } from './defaultGitInfo'; +import { + CantSyncGitNotInitializedError, + GitPullPushError, + SyncParameterMissingError, +} from './errors'; +import { + assumeSync, + getDefaultBranchName, + getGitRepositoryState, + getRemoteName, + getSyncState, + haveLocalChanges, +} from './inspect'; +import { GitStep, IGitUserInfos, ILogger } from './interface'; +import { commitFiles, continueRebase, fetchRemote, mergeUpstream, pushUpstream } from './sync'; + +export interface ICommitAndSyncOptions { + /** the commit message */ + commitMessage?: string; + commitOnly?: boolean; + /** Optional fallback of userInfo. If some info is missing in userInfo, will use defaultGitInfo instead. */ + defaultGitInfo?: typeof defaultDefaultGitInfo; + /** wiki folder path, can be relative */ + dir: string; + /** if you want to use a dynamic .gitignore, you can passing an array contains filepaths that want to ignore */ + filesToIgnore?: string[]; + logger?: ILogger; + /** the storage service url we are sync to, for example your github repo url + * When empty, and commitOnly===true, it means we just want commit, without sync + */ + remoteUrl?: string; + /** user info used in the commit message + * When empty, and commitOnly===true, it means we just want commit, without sync + */ + userInfo?: IGitUserInfos; +} +/** + * `playbooks-repository add .` + `playbooks-repository commit` + `playbooks-repository rebase` or something that can sync bi-directional + */ +export async function commitAndSync(options: ICommitAndSyncOptions): Promise { + const { + dir, + remoteUrl, + commitMessage = 'Updated with Git-Sync', + userInfo, + logger, + defaultGitInfo = defaultDefaultGitInfo, + filesToIgnore, + commitOnly, + } = options; + const { gitUserName, email, branch } = userInfo ?? defaultGitInfo; + const { accessToken } = userInfo ?? {}; + + const defaultBranchName = (await getDefaultBranchName(dir)) ?? branch; + const remoteName = await getRemoteName(dir, defaultBranchName); + + const logProgress = (step: GitStep): unknown => + logger?.info?.(step, { + functionName: 'commitAndSync', + step, + dir, + remoteUrl, + branch: defaultBranchName, + }); + const logDebug = (message: string, step: GitStep): unknown => + logger?.debug?.(message, { + functionName: 'commitAndSync', + step, + dir, + remoteUrl, + branch: defaultBranchName, + }); + const logWarn = (message: string, step: GitStep): unknown => + logger?.warn?.(message, { + functionName: 'commitAndSync', + step, + dir, + remoteUrl, + branch: defaultBranchName, + }); + + // preflight check + await syncPreflightCheck({ + dir, + logger, + logProgress, + logDebug, + defaultGitInfo, + userInfo, + }); + + if (await haveLocalChanges(dir)) { + logProgress(GitStep.HaveThingsToCommit); + logDebug(commitMessage, GitStep.HaveThingsToCommit); + const { exitCode: commitExitCode, stderr: commitStdError } = await commitFiles( + dir, + gitUserName, + email ?? defaultGitInfo.email, + commitMessage, + filesToIgnore, + ); + if (commitExitCode !== 0) { + logWarn(`commit failed ${commitStdError}`, GitStep.CommitComplete); + } + logProgress(GitStep.CommitComplete); + } + if (commitOnly === true) { + return; + } + logProgress(GitStep.PreparingUserInfo); + if (accessToken === '' || accessToken === undefined) { + throw new SyncParameterMissingError('accessToken'); + } + if (remoteUrl === '' || remoteUrl === undefined) { + throw new SyncParameterMissingError('remoteUrl'); + } + await credentialOn(dir, remoteUrl, gitUserName, accessToken, remoteName); + logProgress(GitStep.FetchingData); + try { + await fetchRemote(dir, remoteName, defaultBranchName); + let exitCode = 0; + let stderr: string | undefined; + const syncStateAfterCommit = await getSyncState(dir, defaultBranchName, remoteName, logger); + switch (syncStateAfterCommit) { + case 'equal': { + logProgress(GitStep.NoNeedToSync); + return; + } + case 'noUpstreamOrBareUpstream': { + logProgress(GitStep.NoUpstreamCantPush); + // try push, if success, means it is bare, otherwise, it is no upstream + try { + await pushUpstream(dir, defaultBranchName, remoteName, userInfo, logger); + break; + } catch (error) { + logWarn( + `${JSON.stringify({ dir, remoteUrl, userInfo })}, remoteUrl may be not valid, noUpstreamOrBareUpstream after credentialOn`, + GitStep.NoUpstreamCantPush, + ); + throw error; + } + } + case 'ahead': { + logProgress(GitStep.LocalAheadStartUpload); + await pushUpstream(dir, defaultBranchName, remoteName, userInfo, logger); + break; + } + case 'behind': { + logProgress(GitStep.LocalStateBehindSync); + await mergeUpstream(dir, defaultBranchName, remoteName, userInfo, logger); + break; + } + case 'diverged': { + logProgress(GitStep.LocalStateDivergeRebase); + ({ exitCode, stderr } = await GitProcess.exec( + ['rebase', `${remoteName}/${defaultBranchName}`], + dir, + )); + logProgress(GitStep.RebaseResultChecking); + if (exitCode !== 0) { + logWarn( + `exitCode: ${exitCode}, stderr of git rebase: ${stderr}`, + GitStep.RebaseResultChecking, + ); + } + if ( + exitCode === 0 && + (await getGitRepositoryState(dir, logger)).length === 0 && + (await getSyncState(dir, defaultBranchName, remoteName, logger)) === 'ahead' + ) { + logProgress(GitStep.RebaseSucceed); + } else { + await continueRebase(dir, gitUserName, email ?? defaultGitInfo.email, logger); + logProgress(GitStep.RebaseConflictNeedsResolve); + } + await pushUpstream(dir, defaultBranchName, remoteName, userInfo, logger); + break; + } + default: { + logProgress(GitStep.SyncFailedAlgorithmWrong); + } + } + + if (exitCode === 0) { + logProgress(GitStep.PerformLastCheckBeforeSynchronizationFinish); + await assumeSync(dir, defaultBranchName, remoteName, logger); + logProgress(GitStep.SynchronizationFinish); + } else { + switch (exitCode) { + // "message":"exitCode: 128, stderr of playbooks-repository push: fatal: unable to access 'https://github.com/tiddly-gittly/TiddlyWiki-Chinese-Tutorial.git/': LibreSSL SSL_connect: SSL_ERROR_SYSCALL in connection to github.com:443 \n" + case 128: { + throw new GitPullPushError(options, stderr ?? ''); + } + // TODO: handle auth expire and throw here + default: { + throw new GitPullPushError(options, stderr ?? ''); + } + } + } + } finally { + // always restore original remoteUrl without token + await credentialOff(dir, remoteName, remoteUrl); + } +} + +/** + * Check for playbooks-repository repo state, if it is not clean, try fix it. If not init will throw error. + * This method is used by commitAndSync and forcePull before they doing anything. + */ +export async function syncPreflightCheck(configs: { + /** defaultGitInfo from ICommitAndSyncOptions */ + defaultGitInfo?: typeof defaultDefaultGitInfo; + dir: string; + logDebug?: (message: string, step: GitStep) => unknown; + logProgress?: (step: GitStep) => unknown; + logger?: ILogger; + /** userInfo from ICommitAndSyncOptions */ + userInfo?: IGitUserInfos; +}) { + const { + dir, + logger, + logProgress, + logDebug, + defaultGitInfo = defaultDefaultGitInfo, + userInfo, + } = configs; + const { gitUserName, email } = userInfo ?? defaultGitInfo; + + const repoStartingState = await getGitRepositoryState(dir, logger); + if (repoStartingState.length === 0 || repoStartingState === '|DIRTY') { + logProgress?.(GitStep.PrepareSync); + logDebug?.( + `${dir} repoStartingState: ${repoStartingState}, ${gitUserName} <${email ?? defaultGitInfo.email}>`, + GitStep.PrepareSync, + ); + } else if (repoStartingState === 'NOGIT') { + throw new CantSyncGitNotInitializedError(dir); + } else { + // we may be in middle of a rebase, try fix that + await continueRebase( + dir, + gitUserName, + email ?? defaultGitInfo.email, + logger, + repoStartingState, + ); + } +} diff --git a/server/src/integrations/git-repository/lib/credential.ts b/server/src/integrations/git-repository/lib/credential.ts new file mode 100644 index 00000000..1e2de6aa --- /dev/null +++ b/server/src/integrations/git-repository/lib/credential.ts @@ -0,0 +1,72 @@ +import { GitProcess } from 'dugite'; +import { trim } from 'lodash'; +import { getRemoteUrl } from './inspect'; + +export enum ServiceType { + Github = 'github', +} + +// TODO: support folderLocation as rawUrl like `/Users/linonetwo/Desktop/repo/playbooks-repository-sync-js/test/mockUpstreamRepo/credential` for test, or gitlab url. +export const getGitHubUrlWithCredential = ( + rawUrl: string, + username: string, + accessToken: string, +): string => + trim( + rawUrl + .replaceAll('\n', '') + .replace('https://github.com/', `https://${username}:${accessToken}@github.com/`), + ); +const getGitHubUrlWithOutCredential = (urlWithCredential: string): string => + trim(urlWithCredential.replace(/.+@/, 'https://')); + +/** + * Add remote with credential + * @param {string} directory + * @param {string} remoteUrl + * @param userName + * @param accessToken + * @param remoteName + * @param serviceType + */ +export async function credentialOn( + directory: string, + remoteUrl: string, + userName: string, + accessToken: string, + remoteName: string, + serviceType = ServiceType.Github, +): Promise { + let gitUrlWithCredential; + switch (serviceType) { + case ServiceType.Github: { + gitUrlWithCredential = getGitHubUrlWithCredential(remoteUrl, userName, accessToken); + break; + } + } + await GitProcess.exec(['remote', 'add', remoteName, gitUrlWithCredential], directory); + await GitProcess.exec(['remote', 'set-url', remoteName, gitUrlWithCredential], directory); +} +/** + * Add remote without credential + * @param {string} directory + * @param remoteName + * @param remoteUrl + * @param serviceType + */ +export async function credentialOff( + directory: string, + remoteName: string, + remoteUrl?: string, + serviceType = ServiceType.Github, +): Promise { + const gitRepoUrl = remoteUrl ?? (await getRemoteUrl(directory, remoteName)); + let gitUrlWithOutCredential; + switch (serviceType) { + case ServiceType.Github: { + gitUrlWithOutCredential = getGitHubUrlWithOutCredential(gitRepoUrl); + break; + } + } + await GitProcess.exec(['remote', 'set-url', remoteName, gitUrlWithOutCredential], directory); +} diff --git a/server/src/integrations/git-repository/lib/defaultGitInfo.ts b/server/src/integrations/git-repository/lib/defaultGitInfo.ts new file mode 100644 index 00000000..40ddb6b5 --- /dev/null +++ b/server/src/integrations/git-repository/lib/defaultGitInfo.ts @@ -0,0 +1,6 @@ +export const defaultGitInfo = { + email: 'gitsync@gmail.com', + gitUserName: 'gitsync', + branch: 'main', + remote: 'origin', +}; diff --git a/server/src/integrations/git-repository/lib/errors.ts b/server/src/integrations/git-repository/lib/errors.ts new file mode 100644 index 00000000..b46a02e1 --- /dev/null +++ b/server/src/integrations/git-repository/lib/errors.ts @@ -0,0 +1,103 @@ +/** + * Custom errors, for user to catch and `instanceof`. So you can show your custom translated message for each error type. + * `Object.setPrototypeOf(this, AssumeSyncError.prototype);` to fix https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work + */ +import { truncate } from 'lodash'; +import { SyncState } from './inspect'; +import { IGitUserInfos, IGitUserInfosWithoutToken } from './interface'; + +export class AssumeSyncError extends Error { + constructor(state: SyncState, extraMessage?: string) { + super(extraMessage); + Object.setPrototypeOf(this, AssumeSyncError.prototype); + this.name = 'AssumeSyncError'; + this.message = `E-1 In this state, git should have been sync with the remote, but it is "${state}", this is caused by procedural bug in the git-sync-js. ${extraMessage ?? ''}`; + } +} +export class SyncParameterMissingError extends Error { + /** the missing parameterName */ + parameterName: string; + constructor(parameterName = 'accessToken') { + super(parameterName); + Object.setPrototypeOf(this, SyncParameterMissingError.prototype); + this.name = 'SyncParameterMissingError'; + this.parameterName = parameterName; + this.message = `E-2 We need ${parameterName} to sync to the cloud, you should pass ${parameterName} as parameters in options.`; + } +} + +export class GitPullPushError extends Error { + constructor( + configuration: { + branch?: string; + /** wiki folder path, can be relative */ + dir: string; + /** for example, origin */ + remote?: string; + /** the storage service url we are sync to, for example your github repo url */ + remoteUrl?: string; + /** user info used in the commit message */ + userInfo?: IGitUserInfos | IGitUserInfosWithoutToken; + }, + extraMessages: string, + ) { + super(extraMessages); + Object.setPrototypeOf(this, GitPullPushError.prototype); + this.name = 'GitPullPushError'; + this.message = `E-3 failed to config git to successfully pull from or push to remote with configuration ${ + JSON.stringify({ + ...configuration, + userInfo: { + ...configuration.userInfo, + accessToken: truncate((configuration?.userInfo as IGitUserInfos)?.accessToken, { + length: 24, + }), + }, + }) + }.\nerrorMessages: ${extraMessages}`; + } +} + +export class CantSyncGitNotInitializedError extends Error { + /** the directory that should have a playbooks-repository repo */ + directory: string; + constructor(directory: string) { + super(directory); + Object.setPrototypeOf(this, CantSyncGitNotInitializedError.prototype); + this.directory = directory; + this.name = 'CantSyncGitNotInitializedError'; + this.message = `E-4 we can't sync on a git repository that is not initialized, maybe this folder is not a git repository. ${directory}`; + } +} + +export class SyncScriptIsInDeadLoopError extends Error { + constructor() { + super(); + Object.setPrototypeOf(this, SyncScriptIsInDeadLoopError.prototype); + this.name = 'SyncScriptIsInDeadLoopError'; + this.message = `E-5 Unable to sync, and Sync script is in a dead loop, this is caused by procedural bug in the git-sync-js.`; + } +} + +export class CantSyncInSpecialGitStateAutoFixFailed extends Error { + stateMessage: string; + constructor(stateMessage: string) { + super(stateMessage); + Object.setPrototypeOf(this, CantSyncInSpecialGitStateAutoFixFailed.prototype); + this.stateMessage = stateMessage; + this.name = 'CantSyncInSpecialGitStateAutoFixFailed'; + this.message = + `E-6 Unable to Sync, this folder is in special condition, thus can't Sync directly. An auto-fix has been tried, but error still remains. Please resolve all the conflict manually (For example, use VSCode to open the wiki folder), if this still don't work out, please use professional Git tools (Source Tree, GitKraken) to solve this. This is caused by procedural bug in the git-sync-js.\n${stateMessage}`; + } +} + +export class CantForcePullError extends Error { + stateMessage: string; + constructor(stateMessage: string) { + super(stateMessage); + Object.setPrototypeOf(this, CantForcePullError.prototype); + this.stateMessage = stateMessage; + this.name = 'CantForcePullError'; + this.message = `E-7 Unable to force pull remote. This is caused by procedural bug in the git-sync-js.\n${stateMessage}`; + } +} diff --git a/server/src/integrations/git-repository/lib/forcePull.ts b/server/src/integrations/git-repository/lib/forcePull.ts new file mode 100644 index 00000000..e3550f5e --- /dev/null +++ b/server/src/integrations/git-repository/lib/forcePull.ts @@ -0,0 +1,109 @@ +import { GitProcess } from 'dugite'; +import { syncPreflightCheck } from './commitAndSync'; +import { credentialOff, credentialOn } from './credential'; +import { defaultGitInfo as defaultDefaultGitInfo } from './defaultGitInfo'; +import { CantForcePullError, SyncParameterMissingError } from './errors'; +import { getDefaultBranchName, getRemoteName, getSyncState } from './inspect'; +import { GitStep, IGitUserInfos, ILogger } from './interface'; +import { fetchRemote } from './sync'; + +export interface IForcePullOptions { + /** Optional fallback of userInfo. If some info is missing in userInfo, will use defaultGitInfo instead. */ + defaultGitInfo?: typeof defaultDefaultGitInfo; + /** wiki folder path, can be relative */ + dir: string; + logger?: ILogger; + /** the storage service url we are sync to, for example your github repo url + * When empty, and commitOnly===true, it means we just want commit, without sync + */ + remoteUrl?: string; + /** user info used in the commit message + * When empty, and commitOnly===true, it means we just want commit, without sync + */ + userInfo?: IGitUserInfos; +} + +/** + * Ignore all local changes, force reset local to remote. + * This is usually used in readonly blog, that will fetch content from a remote repo. And you can push content to the remote repo, let the blog update. + */ +export async function forcePull(options: IForcePullOptions) { + const { dir, logger, defaultGitInfo = defaultDefaultGitInfo, userInfo, remoteUrl } = options; + const { gitUserName, branch } = userInfo ?? defaultGitInfo; + const { accessToken } = userInfo ?? {}; + const defaultBranchName = (await getDefaultBranchName(dir)) ?? branch; + const remoteName = await getRemoteName(dir, branch); + + if (accessToken === '' || accessToken === undefined) { + throw new SyncParameterMissingError('accessToken'); + } + if (remoteUrl === '' || remoteUrl === undefined) { + throw new SyncParameterMissingError('remoteUrl'); + } + + const logProgress = (step: GitStep): unknown => + logger?.info(step, { + functionName: 'forcePull', + step, + dir, + remoteUrl, + branch: defaultBranchName, + }); + const logDebug = (message: string, step: GitStep): unknown => + logger?.debug(message, { + functionName: 'forcePull', + step, + dir, + remoteUrl, + branch: defaultBranchName, + }); + + logProgress(GitStep.StartForcePull); + logDebug(`Do preflight Check before force pull in dir ${dir}`, GitStep.StartForcePull); + // preflight check + await syncPreflightCheck({ + dir, + logger, + logProgress, + logDebug, + defaultGitInfo, + userInfo, + }); + logProgress(GitStep.StartConfiguringGithubRemoteRepository); + await credentialOn(dir, remoteUrl, gitUserName, accessToken, remoteName); + try { + logProgress(GitStep.StartFetchingFromGithubRemote); + await fetchRemote(dir, defaultGitInfo.remote, defaultGitInfo.branch); + const syncState = await getSyncState(dir, defaultBranchName, remoteName, logger); + logDebug(`syncState in dir ${dir} is ${syncState}`, GitStep.StartFetchingFromGithubRemote); + if (syncState === 'equal') { + // if there is no new commit in remote (and nothing messy in local), we don't need to pull. + logProgress(GitStep.SkipForcePull); + return; + } + logProgress(GitStep.StartResettingLocalToRemote); + await hardResetLocalToRemote(dir, branch, remoteName); + logProgress(GitStep.FinishForcePull); + } catch (error) { + if (error instanceof CantForcePullError) { + throw error; + } else { + throw new CantForcePullError(`${(error as Error).message} ${(error as Error).stack ?? ''}`); + } + } finally { + await credentialOff(dir, remoteName, remoteUrl); + } +} + +/** + * Internal method used by forcePull, does the `reset --hard`. + */ +export async function hardResetLocalToRemote(dir: string, branch: string, remoteName: string) { + const { exitCode, stderr } = await GitProcess.exec( + ['reset', '--hard', `${remoteName}/${branch}`], + dir, + ); + if (exitCode !== 0) { + throw new CantForcePullError(`${remoteName}/${branch} ${stderr}`); + } +} diff --git a/server/src/integrations/git-repository/lib/index.ts b/server/src/integrations/git-repository/lib/index.ts new file mode 100644 index 00000000..8126624d --- /dev/null +++ b/server/src/integrations/git-repository/lib/index.ts @@ -0,0 +1,12 @@ +/** primary functions */ +export * from './clone'; +export * from './commitAndSync'; +export * from './forcePull'; +export * from './initGit'; +/** utils */ +export * from './credential'; +export * from './defaultGitInfo'; +export * from './errors'; +export * from './inspect'; +export * from './interface'; +export * from './sync'; diff --git a/server/src/integrations/git-repository/lib/init.ts b/server/src/integrations/git-repository/lib/init.ts new file mode 100644 index 00000000..c6ab95ad --- /dev/null +++ b/server/src/integrations/git-repository/lib/init.ts @@ -0,0 +1,50 @@ +import path from 'path'; +import fs from 'fs-extra'; +import { GitProcess } from 'dugite'; +import { defaultGitInfo } from './defaultGitInfo'; + +export interface IGitInitOptions { + /** + * Whether create a bare repo, useful as an upstream repo + */ + bare?: boolean; + + /** + * Default to true, to try to fix https://stackoverflow.com/questions/12267912/git-error-fatal-ambiguous-argument-head-unknown-revision-or-path-not-in-the + * + * Following techniques are not working: + * + * ```js + * await GitProcess.exec(['symbolic-ref', 'HEAD', `refs/heads/${branch}`], dir); + * await GitProcess.exec(['checkout', `-b`, branch], dir); + * ``` + * + * This works: + * https://stackoverflow.com/a/51527691/4617295 + */ + initialCommit?: boolean; +} + +/** + * Init and immediately checkout the branch, other wise the branch will be HEAD, which is annoying in the later steps + */ +export async function initGitWithBranch( + dir: string, + branch = defaultGitInfo.branch, + options?: IGitInitOptions, +): Promise { + if (options?.bare === true) { + const bareGitPath = path.join(dir, '.playbooks-repository'); + await fs.mkdirp(bareGitPath); + await GitProcess.exec(['init', `--initial-branch=${branch}`, '--bare'], bareGitPath); + } else { + await GitProcess.exec(['init', `--initial-branch=${branch}`], dir); + } + + if (options?.initialCommit !== false) { + await GitProcess.exec( + ['commit', `--allow-empty`, '-n', '-m', 'Initial commit when init a new playbooks-repository.'], + dir, + ); + } +} diff --git a/server/src/integrations/git-repository/lib/initGit.ts b/server/src/integrations/git-repository/lib/initGit.ts new file mode 100644 index 00000000..609ffdfa --- /dev/null +++ b/server/src/integrations/git-repository/lib/initGit.ts @@ -0,0 +1,86 @@ +import { truncate } from 'lodash'; +import { commitAndSync } from './commitAndSync'; +import { defaultGitInfo as defaultDefaultGitInfo } from './defaultGitInfo'; +import { SyncParameterMissingError } from './errors'; +import { initGitWithBranch } from './init'; +import { GitStep, IGitUserInfos, IGitUserInfosWithoutToken, ILogger } from './interface'; +import { commitFiles } from './sync'; + +export interface IInitGitOptionsSyncImmediately { + /** Optional fallback of userInfo. If some info is missing in userInfo, will use defaultGitInfo instead. */ + defaultGitInfo?: typeof defaultDefaultGitInfo; + /** wiki folder path, can be relative */ + dir: string; + logger?: ILogger; + /** only required if syncImmediately is true, the storage service url we are sync to, for example your github repo url */ + remoteUrl: string; + /** should we sync after playbooks-repository init? */ + syncImmediately: true; + /** user info used in the commit message */ + userInfo: IGitUserInfos; +} +export interface IInitGitOptionsNotSync { + defaultGitInfo?: typeof defaultDefaultGitInfo; + /** wiki folder path, can be relative */ + dir: string; + logger?: ILogger; + /** should we sync after playbooks-repository init? */ + syncImmediately?: false; + userInfo?: IGitUserInfosWithoutToken | IGitUserInfos; +} + +export type IInitGitOptions = IInitGitOptionsSyncImmediately | IInitGitOptionsNotSync; + +export async function initGit(options: IInitGitOptions): Promise { + const { + dir, + userInfo, + syncImmediately, + logger, + defaultGitInfo = defaultDefaultGitInfo, + } = options; + + const logProgress = (step: GitStep): unknown => + logger?.info(step, { + functionName: 'initGit', + step, + }); + const logDebug = (message: string, step: GitStep): unknown => + logger?.debug(message, { functionName: 'initGit', step }); + + logProgress(GitStep.StartGitInitialization); + const { gitUserName, email, branch } = userInfo ?? defaultGitInfo; + logDebug(`Running git init in dir ${dir}`, GitStep.StartGitInitialization); + await initGitWithBranch(dir, branch); + logDebug(`Successfully Running git init in dir ${dir}`, GitStep.StartGitInitialization); + await commitFiles(dir, gitUserName, email ?? defaultGitInfo.email); + + // if we are config local note playbooks-repository, we are done here + if (syncImmediately !== true) { + logProgress(GitStep.GitRepositoryConfigurationFinished); + return; + } + // sync to remote, start config synced note + if ( + userInfo === undefined || + !('accessToken' in userInfo) || + userInfo?.accessToken?.length === 0 + ) { + throw new SyncParameterMissingError('accessToken'); + } + const { remoteUrl } = options; + if (remoteUrl === undefined || remoteUrl.length === 0) { + throw new SyncParameterMissingError('remoteUrl'); + } + logDebug( + `Calling commitAndSync() from initGit() Using gitUrl ${remoteUrl} with gitUserName ${gitUserName} and accessToken ${truncate( + userInfo?.accessToken, + { + length: 24, + }, + )}`, + GitStep.StartConfiguringGithubRemoteRepository, + ); + logProgress(GitStep.StartConfiguringGithubRemoteRepository); + await commitAndSync(options); +} diff --git a/server/src/integrations/git-repository/lib/inspect.ts b/server/src/integrations/git-repository/lib/inspect.ts new file mode 100644 index 00000000..19067ef5 --- /dev/null +++ b/server/src/integrations/git-repository/lib/inspect.ts @@ -0,0 +1,391 @@ +/* eslint-disable security/detect-non-literal-fs-filename */ +/* eslint-disable unicorn/prevent-abbreviations */ +import path from 'path'; +import url from 'url'; +import { GitProcess } from 'dugite'; +import fs from 'fs-extra'; +import { compact } from 'lodash'; +import { AssumeSyncError, CantSyncGitNotInitializedError } from './errors'; +import { GitStep, ILogger } from './interface'; +// eslint-disable-next-line @typescript-eslint/no-require-imports,@typescript-eslint/no-var-requires +const { listRemotes } = require('isomorphic-git'); + +const gitEscapeToEncodedUri = (str: string): string => + str.replaceAll( + /\\(\d{3})/g, + (_: unknown, $1: string) => `%${Number.parseInt($1, 8).toString(16)}`, + ); +const decodeGitEscape = (rawString: string): string => + decodeURIComponent(gitEscapeToEncodedUri(rawString)); + +export interface ModifiedFileList { + filePath: string; + fileRelativePath: string; + type: string; +} +/** + * Get modified files and modify type in a folder + * @param {string} wikiFolderPath location to scan playbooks-repository modify state + */ +export async function getModifiedFileList(wikiFolderPath: string): Promise { + const { stdout } = await GitProcess.exec(['status', '--porcelain'], wikiFolderPath); + const stdoutLines = stdout.split('\n'); + const nonEmptyLines = compact(stdoutLines); + const statusMatrixLines = compact( + nonEmptyLines.map((line: string) => /^\s?(\?\?|[ACMR]|[ACMR][DM])\s?(\S+.*\S+)$/.exec(line)), + ).filter( + ([_, type, fileRelativePath]) => type !== undefined && fileRelativePath !== undefined, + ) as unknown as Array<[unknown, string, string]>; + return statusMatrixLines + .map(([_, type, rawFileRelativePath]) => { + /** + * If filename contains Chinese, it will becomes: + * ```js + * fileRelativePath: "\"tiddlers/\\346\\226\\260\\346\\235\\241\\347\\233\\256.tid\""` + * ``` + * which is actually `"tiddlers/\\346\\226\\260\\346\\235\\241\\347\\233\\256.tid"` (if you try to type it in the console manually). If you console log it, it will become + * ```js + * > temp1[1].fileRelativePath + * '"tiddlers/\346\226\260\346\235\241\347\233\256.tid"' + * ``` + * + * So simply `decodeURIComponent(escape` will work on `tiddlers/\346\226\260\346\235\241\347\233\256.tid` (the logged string), but not on `tiddlers/\\346\\226\\260\\346\\235\\241\\347\\233\\256.tid` (the actual string). + * So how to transform actual string to logged string? Answer is `eval()` it. But we have to check is there any evil script use `;` or `,` mixed into the filename. + * + * But actually those 346 226 are in radix 8 , if we transform it to radix 16 and add prefix % we can make it uri component. + * And it should not be parsed in groups of three, because only the CJK between 0x0800 - 0xffff are encoded into three bytes; so we should just replace all the \\\d{3} with hexadecimal, and then give it to the decodeURIComponent to parse. + */ + const isSafeUtf8UnescapedString = + rawFileRelativePath.startsWith('"') && + rawFileRelativePath.endsWith('"') && + !rawFileRelativePath.includes(';') && + !rawFileRelativePath.includes(','); + const fileRelativePath = isSafeUtf8UnescapedString + ? decodeGitEscape(rawFileRelativePath).replace(/^"/, '').replace(/"$/, '') + : rawFileRelativePath; + return { + type, + fileRelativePath, + filePath: path.normalize(path.join(wikiFolderPath, fileRelativePath)), + }; + }) + .sort((item, item2) => item.fileRelativePath.localeCompare(item2.fileRelativePath, 'zh')); +} + +/** + * Inspect playbooks-repository's remote url from folder's .playbooks-repository config + * @param dir wiki folder path, playbooks-repository folder to inspect + * @param remoteName + * @returns remote url, without `'.playbooks-repository'` + * @example ```ts + const githubRepoUrl = await getRemoteUrl(directory); + const gitUrlWithOutCredential = getGitUrlWithOutCredential(githubRepoUrl); + await GitProcess.exec(['remote', 'set-url', 'origin', gitUrlWithOutCredential], directory); + ``` + */ +export async function getRemoteUrl(dir: string, remoteName: string): Promise { + const remotes = await listRemotes({ fs, dir }); + const githubRemote = remotes.find(({ remote }) => remote === remoteName) ?? remotes[0]; + if ((githubRemote?.url?.length ?? 0) > 0) { + return githubRemote!.url; + } + return ''; +} + +/** + * Get the Github Repo Name, which is similar to "linonetwo/wiki", that is the string after "https://github.com/", so we basically just get the pathname of URL. + * @param remoteUrl full github repository url or other repository url + * @returns + */ +export function getRemoteRepoName(remoteUrl: string): string | undefined { + let wikiRepoName = new url.URL(remoteUrl).pathname; + if (wikiRepoName.startsWith('/')) { + // deepcode ignore GlobalReplacementRegex: change only the first match + wikiRepoName = wikiRepoName.replace('/', ''); + } + if (wikiRepoName.length > 0) { + return wikiRepoName; + } + return undefined; +} + +/** + * See if there is any file not being committed + * @param {string} wikiFolderPath repo path to test + * @example ```ts +if (await haveLocalChanges(dir)) { + // ... do commit and push +``` + */ +export async function haveLocalChanges(wikiFolderPath: string): Promise { + const { stdout } = await GitProcess.exec(['status', '--porcelain'], wikiFolderPath); + const matchResult = stdout.match(/^(\?\?|[ACMR] |[ ACMR][DM])*/gm); + // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions + return !!matchResult?.some?.(Boolean); +} + +/** + * Get "master" or "main" from playbooks-repository repo + * + * https://github.com/simonthum/git-sync/blob/31cc140df2751e09fae2941054d5b61c34e8b649/git-sync#L228-L232 + * @param wikiFolderPath + */ +export async function getDefaultBranchName(wikiFolderPath: string): Promise { + try { + const { stdout } = await GitProcess.exec(['rev-parse', '--abbrev-ref', 'HEAD'], wikiFolderPath); + const [branchName] = stdout.split('\n'); + // don't return empty string, so we can use ?? syntax + if (branchName === '') { + return undefined; + } + return branchName; + } catch { + /** + * Catch "Unable to find path to repository on disk." + at node_modules/dugite/lib/playbooks-repository-process.ts:226:29 + */ + return undefined; + } +} + +export type SyncState = 'noUpstreamOrBareUpstream' | 'equal' | 'ahead' | 'behind' | 'diverged'; +/** + * determine sync state of repository, i.e. how the remote relates to our HEAD + * 'ahead' means our local state is ahead of remote, 'behind' means local state is behind of the remote + * @param dir repo path to test + * @param defaultBranchName + * @param remoteName + * @param logger + */ +export async function getSyncState( + dir: string, + defaultBranchName: string, + remoteName: string, + logger?: ILogger, +): Promise { + const logDebug = (message: string, step: GitStep): unknown => + logger?.debug?.(message, { functionName: 'getSyncState', step, dir }); + const logProgress = (step: GitStep): unknown => + logger?.info?.(step, { + functionName: 'getSyncState', + step, + dir, + }); + logProgress(GitStep.CheckingLocalSyncState); + remoteName = remoteName ?? (await getRemoteName(dir, defaultBranchName)); + const gitArgs = [ + 'rev-list', + '--count', + '--left-right', + `${remoteName}/${defaultBranchName}...HEAD`, + ]; + const { stdout, stderr } = await GitProcess.exec(gitArgs, dir); + logDebug( + `Checking sync state with upstream, command: \`git ${gitArgs.join(' ')}\` , stdout:\n${stdout}\n(stdout end)`, + GitStep.CheckingLocalSyncState, + ); + if (stderr.length > 0) { + logDebug( + `Have problem checking sync state with upstream,stderr:\n${stderr}\n(stderr end)`, + GitStep.CheckingLocalSyncState, + ); + } + if (stdout === '') { + return 'noUpstreamOrBareUpstream'; + } + /** + * checks for the output 0 0, which means there are no differences between the local and remote branches. If this is the case, the function returns 'equal'. + */ + if (/0\t0/.exec(stdout) !== null) { + return 'equal'; + } + /** + * The pattern /0\t\d+/ checks if there are commits on the current HEAD that are not on the remote branch (e.g., 0 2). If this pattern matches, the function returns 'ahead'. + */ + if (/0\t\d+/.exec(stdout) !== null) { + return 'ahead'; + } + /** + * The pattern /\d+\t0/ checks if there are commits on the remote branch that are not on the current HEAD (e.g., 2 0). If this pattern matches, the function returns 'behind'. + */ + if (/\d+\t0/.exec(stdout) !== null) { + return 'behind'; + } + /** + * If none of these patterns match, the function returns 'diverged'. For example, the output `1 1` will indicates that there is one commit on the origin/main branch that is not on your current HEAD, and also one commit on your current HEAD that is not on the origin/main branch. + */ + return 'diverged'; +} + +export async function assumeSync( + wikiFolderPath: string, + defaultBranchName: string, + remoteName: string, + logger?: ILogger, +): Promise { + const syncState = await getSyncState(wikiFolderPath, defaultBranchName, remoteName, logger); + if (syncState === 'equal') { + return; + } + throw new AssumeSyncError(syncState); +} + +/** + * get various repo state in string format + * @param wikiFolderPath repo path to check + * @param logger + * @returns gitState + * // TODO: use template literal type to get exact type of playbooks-repository state + */ +export async function getGitRepositoryState( + wikiFolderPath: string, + logger?: ILogger, +): Promise { + if (!(await hasGit(wikiFolderPath))) { + return 'NOGIT'; + } + const gitDirectory = await getGitDirectory(wikiFolderPath, logger); + const [isRebaseI, isRebaseM, isAMRebase, isMerging, isCherryPicking, isBisecting] = + await Promise.all([ + // isRebaseI + ( + (await fs + .lstat(path.join(gitDirectory, 'rebase-merge', 'interactive')) + .catch(() => ({}))) as fs.Stats + )?.isFile?.(), + // isRebaseM + ( + (await fs.lstat(path.join(gitDirectory, 'rebase-merge')).catch(() => ({}))) as fs.Stats + )?.isDirectory?.(), + // isAMRebase + ( + (await fs.lstat(path.join(gitDirectory, 'rebase-apply')).catch(() => ({}))) as fs.Stats + )?.isDirectory?.(), + // isMerging + ( + (await fs.lstat(path.join(gitDirectory, 'MERGE_HEAD')).catch(() => ({}))) as fs.Stats + )?.isFile?.(), + // isCherryPicking + ( + (await fs.lstat(path.join(gitDirectory, 'CHERRY_PICK_HEAD')).catch(() => ({}))) as fs.Stats + )?.isFile?.(), + // isBisecting + ( + (await fs.lstat(path.join(gitDirectory, 'BISECT_LOG')).catch(() => ({}))) as fs.Stats + )?.isFile?.(), + ]); + let result = ''; + /* eslint-disable @typescript-eslint/strict-boolean-expressions */ + if (isRebaseI) { + result += 'REBASE-i'; + } else if (isRebaseM) { + result += 'REBASE-m'; + } else { + if (isAMRebase) { + result += 'AM/REBASE'; + } + if (isMerging) { + result += 'MERGING'; + } + if (isCherryPicking) { + result += 'CHERRY-PICKING'; + } + if (isBisecting) { + result += 'BISECTING'; + } + } + result += ( + await GitProcess.exec(['rev-parse', '--is-bare-repository', wikiFolderPath], wikiFolderPath) + ).stdout.startsWith('true') + ? '|BARE' + : ''; + + /* if ((await GitProcess.exec(['rev-parse', '--is-inside-work-tree', wikiFolderPath], wikiFolderPath)).stdout.startsWith('true')) { + const { exitCode } = await GitProcess.exec(['diff', '--no-ext-diff', '--quiet', '--exit-code'], wikiFolderPath); + // 1 if there were differences and 0 means no differences. + if (exitCode !== 0) { + result += '|DIRTY'; + } + } */ + // previous above `playbooks-repository diff --no-ext-diff --quiet --exit-code` logic from playbooks-repository-sync script can only detect if an existed file changed, can't detect newly added file, so we use `haveLocalChanges` instead + if (await haveLocalChanges(wikiFolderPath)) { + result += '|DIRTY'; + } + + return result; +} + +/** + * echo the playbooks-repository dir + * @param dir repo path + * @param logger + */ +export async function getGitDirectory(dir: string, logger?: ILogger): Promise { + const logDebug = (message: string, step: GitStep): unknown => + logger?.debug?.(message, { functionName: 'getGitDirectory', step, dir }); + const logProgress = (step: GitStep): unknown => + logger?.info?.(step, { + functionName: 'getGitDirectory', + step, + dir, + }); + + logProgress(GitStep.CheckingLocalGitRepoSanity); + const { stdout, stderr } = await GitProcess.exec( + ['rev-parse', '--is-inside-work-tree', dir], + dir, + ); + if (typeof stderr === 'string' && stderr.length > 0) { + logDebug(stderr, GitStep.CheckingLocalGitRepoSanity); + throw new CantSyncGitNotInitializedError(dir); + } + if (stdout.startsWith('true')) { + const { stdout: stdout2 } = await GitProcess.exec(['rev-parse', '--playbooks-repository-dir', dir], dir); + const [gitPath2, gitPath1] = compact(stdout2.split('\n')); + if (gitPath2 !== undefined && gitPath1 !== undefined) { + return path.resolve(gitPath1, gitPath2); + } + } + throw new CantSyncGitNotInitializedError(dir); +} + +/** + * Check if dir has `.playbooks-repository`. + * @param dir folder that may contains a playbooks-repository + * @param strict if is true, then dir should be the root of the playbooks-repository repo. Default is true + * @returns + */ +export async function hasGit(dir: string, strict = true): Promise { + try { + const resultDir = await getGitDirectory(dir); + if (strict && path.dirname(resultDir) !== dir) { + return false; + } + } catch (error) { + if (error instanceof CantSyncGitNotInitializedError) { + return false; + } + } + return true; +} + +/** + * get things like "origin" + * + * https://github.com/simonthum/git-sync/blob/31cc140df2751e09fae2941054d5b61c34e8b649/git-sync#L238-L257 + */ +export async function getRemoteName(dir: string, branch: string): Promise { + let { stdout } = await GitProcess.exec(['config', '--get', `branch.${branch}.pushRemote`], dir); + if (stdout.trim()) { + return stdout.trim(); + } + ({ stdout } = await GitProcess.exec(['config', '--get', `remote.pushDefault`], dir)); + if (stdout.trim()) { + return stdout.trim(); + } + ({ stdout } = await GitProcess.exec(['config', '--get', `branch.${branch}.remote`], dir)); + if (stdout.trim()) { + return stdout.trim(); + } + return 'origin'; +} diff --git a/server/src/integrations/git-repository/lib/interface.ts b/server/src/integrations/git-repository/lib/interface.ts new file mode 100644 index 00000000..8d550d2b --- /dev/null +++ b/server/src/integrations/git-repository/lib/interface.ts @@ -0,0 +1,101 @@ +export interface IGitUserInfosWithoutToken { + branch: string; + /** Git commit message email */ + email: string | null | undefined; + /** Github Login: username , this is also used to filter user's repo when searching repo */ + gitUserName: string; +} + +export interface IGitUserInfos extends IGitUserInfosWithoutToken { + /** Github Login: token */ + accessToken: string; +} + +export enum GitStep { + AddComplete = 'AddComplete', + AddingFiles = 'AddingFiles', + CantSyncInSpecialGitStateAutoFixSucceed = 'CantSyncInSpecialGitStateAutoFixSucceed', + CheckingLocalGitRepoSanity = 'CheckingLocalGitRepoSanity', + CheckingLocalSyncState = 'CheckingLocalSyncState', + CommitComplete = 'CommitComplete', + FetchingData = 'FetchingData', + FinishForcePull = 'FinishForcePull', + GitMerge = 'GitMerge', + GitMergeComplete = 'GitMergeComplete', + GitMergeFailed = 'GitMergeFailed', + GitPush = 'GitPush', + GitPushComplete = 'GitPushComplete', + GitPushFailed = 'GitPushFailed', + GitRepositoryConfigurationFinished = 'GitRepositoryConfigurationFinished', + HaveThingsToCommit = 'HaveThingsToCommit', + LocalAheadStartUpload = 'LocalAheadStartUpload', + LocalStateBehindSync = 'LocalStateBehindSync', + LocalStateDivergeRebase = 'LocalStateDivergeRebase', + NoNeedToSync = 'NoNeedToSync', + NoUpstreamCantPush = 'NoUpstreamCantPush', + PerformLastCheckBeforeSynchronizationFinish = 'PerformLastCheckBeforeSynchronizationFinish', + PrepareCloneOnlineWiki = 'PrepareCloneOnlineWiki', + PrepareSync = 'PrepareSync', + PreparingUserInfo = 'PreparingUserInfo', + RebaseConflictNeedsResolve = 'RebaseConflictNeedsResolve', + RebaseResultChecking = 'RebaseResultChecking', + RebaseSucceed = 'RebaseSucceed', + SkipForcePull = 'SkipForcePull', + StartBackupToGitRemote = 'StartBackupToGitRemote', + StartConfiguringGithubRemoteRepository = 'StartConfiguringGithubRemoteRepository', + StartFetchingFromGithubRemote = 'StartFetchingFromGithubRemote', + StartForcePull = 'StartForcePull', + StartGitInitialization = 'StartGitInitialization', + StartResettingLocalToRemote = 'StartResettingLocalToRemote', + /** this means our algorithm have some problems */ + SyncFailedAlgorithmWrong = 'SyncFailedAlgorithmWrong', + SynchronizationFinish = 'SynchronizationFinish', +} + +/** context to tell logger which function we are in */ +export interface ILoggerContext { + branch?: string; + dir?: string; + functionName: string; + remoteUrl?: string; + step: GitStep; +} + +/** custom logger to report progress on each step + * we don't use logger to report error, we throw errors. + */ +export interface ILogger { + /** used to report debug logs */ + debug: (message: string, context: ILoggerContext) => unknown; + /** used to report progress for human user to read */ + info: (message: GitStep, context: ILoggerContext) => unknown; + /** used to report failed optional progress */ + warn: (message: string, context: ILoggerContext) => unknown; +} +/** + * Steps that indicate we have new files, so we can restart our wiki to reload changes. + * + * @example 

+ * // (inside a promise)
+ * let hasChanges = false;
+    observable?.subscribe({
+      next: (messageObject) => {
+        if (messageObject.level === 'error') {
+          return;
+        }
+        const { meta } = messageObject;
+        if (typeof meta === 'object' && meta !== null && 'step' in meta && stepsAboutChange.includes((meta as { step: GitStep }).step)) {
+          hasChanges = true;
+        }
+      },
+      complete: () => {
+        resolve(hasChanges);
+      },
+    });
+
+ */ +export const stepsAboutChange = [ + GitStep.GitMergeComplete, + GitStep.RebaseSucceed, + GitStep.FinishForcePull, +]; diff --git a/server/src/integrations/git-repository/lib/sync.ts b/server/src/integrations/git-repository/lib/sync.ts new file mode 100644 index 00000000..811b8997 --- /dev/null +++ b/server/src/integrations/git-repository/lib/sync.ts @@ -0,0 +1,200 @@ +import { GitProcess, IGitResult } from 'dugite'; +import fs from 'fs-extra'; +import { + CantSyncInSpecialGitStateAutoFixFailed, + GitPullPushError, + SyncScriptIsInDeadLoopError, +} from './errors'; +import { getGitRepositoryState } from './inspect'; +import { GitStep, IGitUserInfos, IGitUserInfosWithoutToken, ILogger } from './interface'; + +/** + * Git add and commit all file + * @param dir + * @param username + * @param email + * @param message + * @param filesToIgnore + * @param logger + */ +export async function commitFiles( + dir: string, + username: string, + email: string, + message = 'Commit with SSM', + filesToIgnore: string[] = [], + logger?: ILogger, +): Promise { + const logProgress = (step: GitStep): unknown => + logger?.info(step, { + functionName: 'commitFiles', + step, + dir, + }); + + logProgress(GitStep.AddingFiles); + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { listFiles, remove } = await require('isomorphic-git'); + + await GitProcess.exec(['add', '.'], dir); + // find and unStage files that are in the ignore list + const stagedFiles = await listFiles({ fs, dir }); + if (filesToIgnore.length > 0) { + const stagedFilesToIgnore = filesToIgnore.filter((file) => stagedFiles.includes(file)); + if (stagedFilesToIgnore.length > 0) { + await Promise.all( + stagedFilesToIgnore.map(async (file) => { + await remove({ dir, filepath: file, fs }); + }), + ); + } + } + + logProgress(GitStep.AddComplete); + return await GitProcess.exec(['commit', '-m', message, `--author="${username} <${email}>"`], dir); +} + +/** + * Git push -f origin master + * This does force push, to deal with `--allow-unrelated-histories` case + * @param dir + * @param branch + * @param remoteName + * @param userInfo + * @param logger + */ +export async function pushUpstream( + dir: string, + branch: string, + remoteName: string, + userInfo?: IGitUserInfos | IGitUserInfosWithoutToken | undefined, + logger?: ILogger, +): Promise { + const logProgress = (step: GitStep): unknown => + logger?.info(step, { + functionName: 'pushUpstream', + step, + dir, + }); + /** when push to remote, we need to specify the local branch name and remote branch name */ + const branchMapping = `${branch}:${branch}`; + logProgress(GitStep.GitPush); + const pushResult = await GitProcess.exec(['push', remoteName, branchMapping], dir); + logProgress(GitStep.GitPushComplete); + if (pushResult.exitCode !== 0) { + throw new GitPullPushError( + { dir, branch, remote: remoteName, userInfo }, + pushResult.stdout + pushResult.stderr, + ); + } + return pushResult; +} + +/** + * Git merge origin master + * @param dir + * @param branch + * @param remoteName + * @param userInfo + * @param logger + */ +export async function mergeUpstream( + dir: string, + branch: string, + remoteName: string, + userInfo?: IGitUserInfos | IGitUserInfosWithoutToken | undefined, + logger?: ILogger, +): Promise { + const logProgress = (step: GitStep): unknown => + logger?.info(step, { + functionName: 'mergeUpstream', + step, + dir, + }); + logProgress(GitStep.GitMerge); + const mergeResult = await GitProcess.exec( + ['merge', '--ff', '--ff-only', `${remoteName}/${branch}`], + dir, + ); + logProgress(GitStep.GitMergeComplete); + if (mergeResult.exitCode !== 0) { + throw new GitPullPushError( + { dir, branch, remote: remoteName, userInfo }, + mergeResult.stdout + mergeResult.stderr, + ); + } + + return mergeResult; +} + +/** + * try to continue rebase, simply adding and committing all things, leave them to user to resolve in the TiddlyWiki later. + * @param dir + * @param username + * @param email + * @param logger + * @param providedRepositoryState result of `await getGitRepositoryState(dir, logger)`, optional, if not provided, we will run `await getGitRepositoryState(dir, logger)` by ourself. + */ +export async function continueRebase( + dir: string, + username: string, + email: string, + logger?: ILogger, + providedRepositoryState?: string, +): Promise { + const logProgress = (step: GitStep): unknown => + logger?.info(step, { + functionName: 'continueRebase', + step, + dir, + }); + + let hasNotCommittedConflict = true; + let rebaseContinueExitCode = 0; + let rebaseContinueStdError = ''; + let repositoryState: string = + providedRepositoryState ?? (await getGitRepositoryState(dir, logger)); + // prevent infin loop, if there is some bug that I miss + let loopCount = 0; + while (hasNotCommittedConflict) { + loopCount += 1; + if (loopCount > 1000) { + throw new SyncScriptIsInDeadLoopError(); + } + const { exitCode: commitExitCode, stderr: commitStdError } = await commitFiles( + dir, + username, + email, + 'Conflict files committed', + ); + const rebaseContinueResult = await GitProcess.exec(['rebase', '--continue'], dir); + // get info for logging + rebaseContinueExitCode = rebaseContinueResult.exitCode; + rebaseContinueStdError = rebaseContinueResult.stderr; + const rebaseContinueStdOut = rebaseContinueResult.stdout; + repositoryState = await getGitRepositoryState(dir, logger); + // if playbooks-repository add . + playbooks-repository commit failed or playbooks-repository rebase --continue failed + if (commitExitCode !== 0 || rebaseContinueExitCode !== 0) { + throw new CantSyncInSpecialGitStateAutoFixFailed( + `rebaseContinueStdError when ${repositoryState}: ${rebaseContinueStdError}\ncommitStdError when ${repositoryState}: ${commitStdError}\n${rebaseContinueStdError}`, + ); + } + hasNotCommittedConflict = + rebaseContinueStdError.startsWith('CONFLICT') || rebaseContinueStdOut.startsWith('CONFLICT'); + } + logProgress(GitStep.CantSyncInSpecialGitStateAutoFixSucceed); +} + +/** + * Simply calling playbooks-repository fetch. + * @param dir + * @param remoteName + * @param branch if not provided, will fetch all branches + */ +export async function fetchRemote(dir: string, remoteName: string, branch?: string) { + if (branch === undefined) { + await GitProcess.exec(['fetch', remoteName], dir); + } else { + await GitProcess.exec(['fetch', remoteName, branch], dir); + } +} diff --git a/server/src/integrations/git-repository/lib/utils.ts b/server/src/integrations/git-repository/lib/utils.ts new file mode 100644 index 00000000..eb69626c --- /dev/null +++ b/server/src/integrations/git-repository/lib/utils.ts @@ -0,0 +1,2 @@ +export const getGitUrlWithGitSuffix = (url: string): string => `${url}.git`; +export const getGitUrlWithOutGitSuffix = (url: string): string => url.replace(/\.git$/, ''); diff --git a/server/src/integrations/local-repository/LocalRepositoryComponent.ts b/server/src/integrations/local-repository/LocalRepositoryComponent.ts new file mode 100644 index 00000000..a02b423f --- /dev/null +++ b/server/src/integrations/local-repository/LocalRepositoryComponent.ts @@ -0,0 +1,28 @@ +import PlaybooksRepositoryComponent, { + AbstractComponent, +} from '../playbooks-repository/PlaybooksRepositoryComponent'; +import { createDirectoryWithFullPath } from '../shell/utils'; + +class LocalRepositoryComponent extends PlaybooksRepositoryComponent implements AbstractComponent { + constructor(uuid: string, logger: any, name: string, path: string) { + super(uuid, name, path); + this.childLogger = logger.child( + { module: `local-repository/${this.name}` }, + { msgPrefix: `[LOCAL_REPOSITORY] - ` }, + ); + } + + async init(): Promise { + await createDirectoryWithFullPath(this.directory); + } + + async syncFromRepository(): Promise { + await this.syncToDatabase(); + } + + async syncToRepository(): Promise { + await this.syncToDatabase(); + } +} + +export default LocalRepositoryComponent; diff --git a/server/src/integrations/playbooks-repository/PlaybooksRepositoryComponent.ts b/server/src/integrations/playbooks-repository/PlaybooksRepositoryComponent.ts new file mode 100644 index 00000000..c432cf80 --- /dev/null +++ b/server/src/integrations/playbooks-repository/PlaybooksRepositoryComponent.ts @@ -0,0 +1,167 @@ +import pino from 'pino'; +import shell from 'shelljs'; +import { NotFoundError } from '../../core/api/ApiError'; +import Playbook from '../../data/database/model/Playbook'; +import PlaybooksRepository from '../../data/database/model/PlaybooksRepository'; +import PlaybookRepo from '../../data/database/repository/PlaybookRepo'; +import PlaybooksRepositoryRepo from '../../data/database/repository/PlaybooksRepositoryRepo'; +import directoryTree from '../../helpers/directory-tree/directory-tree'; +import logger from '../../logger'; +import { Playbooks } from '../../types/typings'; +import Shell from '../shell'; +import { deleteFilesAndDirectory } from '../shell/utils'; +import { recursivelyFlattenTree } from './utils'; + +export const DIRECTORY_ROOT = '/playbooks'; +export const FILE_PATTERN = /\.yml$/; + +abstract class PlaybooksRepositoryComponent { + public name: string; + public directory: string; + public uuid: string; + public childLogger: pino.Logger; + + protected constructor(uuid: string, name: string, path: string) { + const dir = `${path}/${uuid}`; + this.uuid = uuid; + this.directory = dir; + this.name = name; + // @ts-expect-error mock + this.childLogger = () => {}; + } + + public async delete() { + await deleteFilesAndDirectory(this.directory); + } + + public async save(playbookUuid: string, content: string) { + const playbook = await PlaybookRepo.findOneByUuid(playbookUuid); + if (!playbook) { + throw new NotFoundError(`Playbook ${playbookUuid} not found`); + } + shell.ShellString(content).to(playbook.path); + } + + public async syncToDatabase() { + this.childLogger.info('saving to database...'); + const playbooksRepository = await this.getPlaybookRepository(); + const filteredTree = await this.updateDirectoriesTree(); + if (!filteredTree) { + return; + } + const playbooksListFromDatabase = await PlaybookRepo.listAllByRepository(playbooksRepository); + this.childLogger.info( + `Found ${playbooksListFromDatabase?.length || 0} playbooks from database`, + ); + const playbooksListFromDirectory = recursivelyFlattenTree(filteredTree).map((treeNode) => { + if (treeNode && treeNode.extension?.match(FILE_PATTERN)) { + this.childLogger.info(`Found child : ${JSON.stringify(treeNode)}`); + const { name, path } = treeNode; + return { name, path } as Playbook; + } + }); + this.childLogger.error(playbooksListFromDirectory); + this.childLogger.info( + `Found ${playbooksListFromDirectory?.length || 0} playbooks from directory`, + ); + const playbooksListToDelete = playbooksListFromDatabase?.filter((playbook) => { + return !playbooksListFromDirectory?.some((p) => p?.path === playbook.path); + }); + if (playbooksListToDelete && playbooksListToDelete.length > 0) { + await Promise.all( + playbooksListToDelete?.map((playbook) => { + if (playbook && playbook.uuid) { + return PlaybookRepo.deleteByUuid(playbook.uuid); + } + }), + ); + } + const playbooksToSync = playbooksListFromDirectory?.filter((playbook) => { + return playbook !== undefined; + }) as Playbook[]; + this.childLogger.info(`Playbooks to sync : ${playbooksToSync.length}`); + await Promise.all( + playbooksToSync.map(async (playbook) => { + return this.updateOrCreateAssociatedPlaybook(playbook, playbooksRepository); + }), + ); + this.childLogger.info( + `Updating Playbooks Repository ${playbooksRepository.name} - ${playbooksRepository._id}`, + ); + } + + private async getPlaybookRepository() { + const playbooksRepository = await PlaybooksRepositoryRepo.findByUuid(this.uuid); + if (!playbooksRepository) { + throw new NotFoundError(`Playbooks repository ${this.uuid} not found`); + } + return playbooksRepository; + } + + public async updateDirectoriesTree() { + const playbooksRepository = await this.getPlaybookRepository(); + const filteredTree = directoryTree(this.directory, { + extensions: FILE_PATTERN, + attributes: ['type', 'extension'], + exclude: /\.git/, + }); + if (!filteredTree) { + this.childLogger.error(`No playbooks found in directory ${this.directory}`); + return; + } + this.childLogger.debug(JSON.stringify(filteredTree)); + await PlaybooksRepositoryRepo.saveTree(playbooksRepository, filteredTree); + return filteredTree; + } + + private async updateOrCreateAssociatedPlaybook( + foundPlaybook: Playbook, + playbooksRepository: PlaybooksRepository, + ): Promise { + const configurationFileContent = await Shell.readPlaybookConfigurationFileIfExists( + foundPlaybook.path.replace('.yml', '.json'), + ); + const isCustomPlaybook = !foundPlaybook.name.startsWith('_'); + const playbookFoundInDatabase = await PlaybookRepo.findOneByPath(foundPlaybook.path); + const playbookData: Playbook = { + path: foundPlaybook.path, + name: foundPlaybook.name, + custom: isCustomPlaybook, + playbooksRepository: playbooksRepository, + uuid: playbookFoundInDatabase?.uuid, + }; + + if (configurationFileContent) { + this.childLogger.info(`playbook ${foundPlaybook.name} has configuration file`); + + const playbookConfiguration = JSON.parse( + configurationFileContent, + ) as Playbooks.PlaybookConfigurationFile; + + playbookData.playableInBatch = playbookConfiguration.playableInBatch; + playbookData.extraVars = playbookConfiguration.extraVars; + playbookData.uniqueQuickRef = playbookConfiguration.uniqueQuickRef; + } + + await PlaybookRepo.updateOrCreate(playbookData); + } + + public fileBelongToRepository(path: string) { + logger.info(`rootPath: ${this.directory?.split('/')[0]} versus ${path.split('/')[0]}`); + return this.directory?.split('/')[0] === path.split('/')[0]; + } + + getDirectory() { + return this.directory; + } +} + +export interface AbstractComponent extends PlaybooksRepositoryComponent { + save(playbookUuid: string, content: string): Promise; + init(): Promise; + delete(): Promise; + syncToRepository(): Promise; + syncFromRepository(): Promise; +} + +export default PlaybooksRepositoryComponent; diff --git a/server/src/integrations/playbooks-repository/PlaybooksRepositoryEngine.ts b/server/src/integrations/playbooks-repository/PlaybooksRepositoryEngine.ts new file mode 100644 index 00000000..75bddfba --- /dev/null +++ b/server/src/integrations/playbooks-repository/PlaybooksRepositoryEngine.ts @@ -0,0 +1,129 @@ +import { Playbooks } from 'ssm-shared-lib'; +import PlaybooksRepository from '../../data/database/model/PlaybooksRepository'; +import PlaybooksRepositoryRepo from '../../data/database/repository/PlaybooksRepositoryRepo'; +import logger from '../../logger'; +import { DEFAULT_VAULT_ID, vaultDecrypt } from '../ansible-vault/vault'; +import GitRepositoryComponent from '../git-repository/GitRepositoryComponent'; +import LocalRepositoryComponent from '../local-repository/LocalRepositoryComponent'; +import { AbstractComponent } from './PlaybooksRepositoryComponent'; + +type stateType = { + playbooksRepository: AbstractComponent[]; +}; + +const state: stateType = { + playbooksRepository: [], +}; + +/** + * Return all registered repositories + * @returns {*} + */ +export function getState(): stateType { + return state; +} + +async function registerGitRepository(playbookRepository: PlaybooksRepository) { + const { uuid, name, branch, email, userName, accessToken, remoteUrl } = playbookRepository; + if (!accessToken) { + throw new Error('accessToken is required'); + } + const decryptedAccessToken = await vaultDecrypt(accessToken, DEFAULT_VAULT_ID); + if (!decryptedAccessToken) { + throw new Error('Error decrypting access token'); + } + return new GitRepositoryComponent( + uuid, + logger, + name, + // @ts-expect-error partial type to fix + branch, + email, + userName, + decryptedAccessToken, + remoteUrl, + ); +} + +async function registerLocalRepository(playbookRepository: PlaybooksRepository) { + if (!playbookRepository.directory) { + throw new Error('playbookRepository.directory is required'); + } + return new LocalRepositoryComponent( + playbookRepository.uuid, + logger, + playbookRepository.name, + playbookRepository.directory, + ); +} + +async function registerRepository(playbookRepository: PlaybooksRepository) { + logger.info( + `[PLAYBOOKS_REPOSITORY_ENGINE] - Registering ${playbookRepository.name}/${playbookRepository.uuid}`, + ); + + switch (playbookRepository.type) { + case Playbooks.PlaybooksRepositoryType.GIT: + state.playbooksRepository[playbookRepository.uuid] = + await registerGitRepository(playbookRepository); + break; + case Playbooks.PlaybooksRepositoryType.LOCAL: + state.playbooksRepository[playbookRepository.uuid] = + await registerLocalRepository(playbookRepository); + break; + default: + throw new Error('Unknown playbook type'); + } + return state.playbooksRepository[playbookRepository.uuid]; +} + +async function registerRepositories() { + const repos = await PlaybooksRepositoryRepo.findAllActive(); + logger.info(`[PLAYBOOKS_REPOSITORY_ENGINE] Found ${repos?.length} active repositories`); + const repositoriesToRegister: any = []; + repos?.map((repo) => { + repositoriesToRegister.push(registerRepository(repo)); + }); + await Promise.all(repositoriesToRegister); +} + +async function deregisterRepository(uuid: string) { + const repository = getState().playbooksRepository[uuid]; + if (!repository) { + throw new Error('Repository not found'); + } + delete state.playbooksRepository[uuid]; +} + +async function clone(uuid: string) { + const gitRepository = getState().playbooksRepository[uuid]; + if (!gitRepository) { + throw new Error("Repository not registered / doesn't exist"); + } + await gitRepository.clone(); +} + +async function init() { + await registerRepositories(); +} + +async function syncAllRegistered() { + logger.warn( + `[PLAYBOOKS_REPOSITORY_ENGINE] - syncAllRegistered, ${getState().playbooksRepository.length} registered`, + ); + await Promise.all( + Object.values(getState().playbooksRepository).map((component) => { + return component.syncFromRepository(); + }), + ); +} + +export default { + registerRepositories, + syncAllRegistered, + registerRepository, + clone, + init, + deregisterRepository, + getState, +}; diff --git a/server/src/integrations/playbooks-repository/utils.ts b/server/src/integrations/playbooks-repository/utils.ts new file mode 100644 index 00000000..c26684dc --- /dev/null +++ b/server/src/integrations/playbooks-repository/utils.ts @@ -0,0 +1,87 @@ +import { DirectoryTree } from 'ssm-shared-lib'; +import PlaybookRepo from '../../data/database/repository/PlaybookRepo'; +import logger from '../../logger'; +import ExtraVars from '../ansible/utils/ExtraVars'; +import { FILE_PATTERN } from './PlaybooksRepositoryComponent'; + +export function recursivelyFlattenTree( + tree: DirectoryTree.TreeNode, + depth = 0, +): (DirectoryTree.TreeNode | undefined)[] { + const node = tree; + if (node.children) { + return node.children + .map((child) => { + if (child && child.type === DirectoryTree.CONSTANTS.DIRECTORY) { + if (depth > 20) { + throw new Error( + 'Depth is too high, to prevent any infinite loop, directories depth is limited to 20', + ); + } + if (child.children) { + return child.children + .map((e) => (e === null ? [] : recursivelyFlattenTree(e, depth + 1))) + .flat(); + } + } else { + return child || []; + } + }) + .flat(); + } + return [node.children ? node.children : node]; +} + +export async function recursiveTreeCompletion( + tree: DirectoryTree.TreeNode, + depth = 0, +): Promise { + const node = tree; + const newTree: DirectoryTree.ExtendedTreeNode[] = []; + if (node?.children) { + for (const child of node.children) { + if (child && child.type === DirectoryTree.CONSTANTS.DIRECTORY) { + if (depth > 20) { + throw new Error( + 'Depth is too high, to prevent any infinite loop, directories depth is limited to 20', + ); + } + newTree.push({ ...child, children: await recursiveTreeCompletion(child, depth + 1) }); + } else { + if (child?.extension?.match(FILE_PATTERN)) { + try { + newTree.push(await completeNode(child)); + } catch (error: any) { + logger.error(error); + } + } else { + newTree.push(node); + } + } + } + } else { + try { + newTree.push(await completeNode(node)); + } catch (error: any) { + logger.error(error); + } + } + return newTree; +} + +async function completeNode(node: DirectoryTree.ExtendedTreeNode) { + const { path } = node; + const playbook = await PlaybookRepo.findOneByPath(path); + const extraVars = playbook?.extraVars + ? await ExtraVars.findValueOfExtraVars(playbook.extraVars, undefined, true) + : undefined; + if (!playbook) { + throw new Error(`Unable to find any playbook for path ${path}`); + } + return { + ...node, + uuid: playbook?.uuid, + extraVars: extraVars, + custom: playbook?.custom, + }; +} diff --git a/server/src/integrations/shell/index.ts b/server/src/integrations/shell/index.ts index 21d15c16..a910f115 100644 --- a/server/src/integrations/shell/index.ts +++ b/server/src/integrations/shell/index.ts @@ -7,7 +7,7 @@ import DeviceAuthRepo from '../../data/database/repository/DeviceAuthRepo'; import logger from '../../logger'; import AnsibleGalaxyCmd from '../ansible/AnsibleGalaxyCmd'; import Inventory from '../ansible/utils/InventoryTransformer'; -import { Ansible } from '../../types/typings'; +import { Playbooks } from '../../types/typings'; import ansibleCmd from '../ansible/AnsibleCmd'; export const ANSIBLE_PATH = '/server/src/ansible/'; @@ -17,41 +17,40 @@ function timeout(ms: number) { } async function executePlaybook( - playbook: string, + playbookPath: string, user: User, target?: string[], extraVars?: API.ExtraVars, ) { logger.info('[SHELL]-[ANSIBLE] - executePlaybook - Starting...'); - let inventoryTargets: (Ansible.All & Ansible.HostGroups) | undefined; + let inventoryTargets: (Playbooks.All & Playbooks.HostGroups) | undefined; if (target) { logger.info(`[SHELL]-[ANSIBLE] - executePlaybook - called with target: ${target}`); const devicesAuth = await DeviceAuthRepo.findManyByDevicesUuid(target); if (!devicesAuth || devicesAuth.length === 0) { - logger.error(`[SHELL]-[ANSIBLE] - executePlaybook - Target not found`); - throw new Error('Exec failed, no matching target'); + logger.error( + `[SHELL]-[ANSIBLE] - executePlaybook - Target not found (Authentication not found)`, + ); + throw new Error('Exec failed, no matching target (Authentication not found)'); } inventoryTargets = Inventory.inventoryBuilderForTarget(devicesAuth); } - return await executePlaybookOnInventory(playbook, user, inventoryTargets, extraVars); + return await executePlaybookOnInventory(playbookPath, user, inventoryTargets, extraVars); } async function executePlaybookOnInventory( - playbook: string, + playbookPath: string, user: User, - inventoryTargets?: Ansible.All & Ansible.HostGroups, + inventoryTargets?: Playbooks.All & Playbooks.HostGroups, extraVars?: API.ExtraVars, ) { - if (!playbook.endsWith('.yml')) { - playbook += '.yml'; - } shell.cd(ANSIBLE_PATH); - shell.rm('/server/src/ansible/inventory/hosts'); - shell.rm('/server/src/ansible/env/_extravars'); + shell.rm('/server/src/playbooks/inventory/hosts'); + shell.rm('/server/src/playbooks/env/_extravars'); const uuid = uuidv4(); const result = await new Promise((resolve) => { - const cmd = ansibleCmd.buildAnsibleCmd(playbook, uuid, inventoryTargets, user, extraVars); + const cmd = ansibleCmd.buildAnsibleCmd(playbookPath, uuid, inventoryTargets, user, extraVars); logger.info(`[SHELL]-[ANSIBLE] - executePlaybook - Executing ${cmd}`); const child = shell.exec(cmd, { async: true, @@ -66,7 +65,11 @@ async function executePlaybookOnInventory( logger.info('[SHELL]-[ANSIBLE] - executePlaybook - launched'); if (result) { logger.info(`[SHELL]-[ANSIBLE] - executePlaybook - ExecId is ${uuid}`); - await AnsibleTaskRepo.create({ ident: uuid, status: 'created', cmd: `playbook ${playbook}` }); + await AnsibleTaskRepo.create({ + ident: uuid, + status: 'created', + cmd: `playbook ${playbookPath}`, + }); return result; } else { logger.error('[SHELL]-[ANSIBLE] - executePlaybook - Result was not properly set'); @@ -93,7 +96,6 @@ async function listPlaybooks() { async function readPlaybook(playbook: string) { try { logger.info(`[SHELL]-[ANSIBLE] - readPlaybook - ${playbook} - Starting...`); - shell.cd(ANSIBLE_PATH); return shell.cat(playbook).toString(); } catch (error) { logger.error('[SHELL]-[ANSIBLE] - readPlaybook'); @@ -101,28 +103,24 @@ async function readPlaybook(playbook: string) { } } -async function readPlaybookConfiguration(playbookConfigurationFile: string) { +async function readPlaybookConfigurationFileIfExists(path: string) { try { - logger.info( - `[SHELL]-[ANSIBLE] - readPlaybookConfiguration - ${playbookConfigurationFile} - Starting...`, - ); - shell.cd(ANSIBLE_PATH); - if (!shell.test('-f', ANSIBLE_PATH + playbookConfigurationFile)) { + logger.info(`[SHELL]-[ANSIBLE] - readPlaybookConfiguration - ${path} - Starting...`); + if (!shell.test('-f', `${path}`)) { logger.info(`[SHELL]-[ANSIBLE] - readPlaybookConfiguration - not found`); return undefined; } - return shell.cat(playbookConfigurationFile).toString(); + return shell.cat(`${path}`).toString(); } catch (error) { logger.error('[SHELL]-[ANSIBLE] - readPlaybookConfiguration'); throw new Error('readPlaybookConfiguration failed'); } } -async function editPlaybook(playbook: string, content: string) { +async function editPlaybook(playbookPath: string, content: string) { try { logger.info('[SHELL]-[ANSIBLE] - editPlaybook - Starting...'); - shell.cd(ANSIBLE_PATH); - shell.ShellString(content).to(playbook); + shell.ShellString(content).to(playbookPath); } catch (error) { logger.error('[SHELL]-[ANSIBLE] - editPlaybook'); throw new Error('editPlaybook failed'); @@ -140,13 +138,12 @@ async function newPlaybook(playbook: string) { } } -async function deletePlaybook(playbook: string) { +async function deletePlaybook(playbookPath: string) { try { - logger.info('[SHELL]-[ANSIBLE] - newPlaybook - Starting...'); - shell.cd(ANSIBLE_PATH); - shell.rm(playbook); + logger.info('[SHELL]-[ANSIBLE] - deletePlaybook - Starting...'); + shell.rm(playbookPath); } catch (error) { - logger.error('[SHELL]-[ANSIBLE] - newPlaybook'); + logger.error('[SHELL]-[ANSIBLE] - deletePlaybook'); throw new Error('deletePlaybook failed'); } } @@ -154,7 +151,7 @@ async function deletePlaybook(playbook: string) { async function getAnsibleVersion() { try { logger.info('[SHELL] - getAnsibleVersion - Starting...'); - return shell.exec('ansible --version').toString(); + return shell.exec('playbooks --version').toString(); } catch (error) { logger.error('[SHELL]- - getAnsibleVersion'); } @@ -186,12 +183,12 @@ async function installAnsibleGalaxyCollection(name: string, namespace: string) { let i = 0; while (!collectionList.includes(`${namespace}.${name}`) && i++ < 60) { await timeout(2000); - const resultList = shell.exec( + shell.exec( AnsibleGalaxyCmd.getListCollectionsCmd(name, namespace) + - ' > /tmp/ansible-collection-output.tmp.txt', + ' > /tmp/playbooks-collection-output.tmp.txt', ); await timeout(2000); - collectionList = shell.cat('/tmp/ansible-collection-output.tmp.txt').toString(); + collectionList = shell.cat('/tmp/playbooks-collection-output.tmp.txt').toString(); } if (!collectionList.includes(`${namespace}.${name}`)) { throw new Error('[SHELL] - installAnsibleGalaxyCollection has failed'); @@ -209,7 +206,7 @@ export default { editPlaybook, newPlaybook, deletePlaybook, - readPlaybookConfiguration, + readPlaybookConfigurationFileIfExists, getAnsibleVersion, saveSshKey, executePlaybookOnInventory, diff --git a/server/src/integrations/shell/utils.ts b/server/src/integrations/shell/utils.ts new file mode 100644 index 00000000..f053d572 --- /dev/null +++ b/server/src/integrations/shell/utils.ts @@ -0,0 +1,17 @@ +import shell from 'shelljs'; +import logger from '../../logger'; + +export async function createDirectoryWithFullPath(fullPath: string) { + return shell.mkdir('-p', fullPath); +} + +export async function findFilesInDirectory(directory: string, pattern: RegExp) { + return shell.find(directory).filter(function (file) { + logger.debug(`[SHELL][UTILS] - findFilesInDirectory - checking ${file} for pattern ${pattern}`); + return file.match(pattern); + }); +} + +export async function deleteFilesAndDirectory(directory: string) { + shell.rm('-rf', directory); +} diff --git a/server/src/routes/index.ts b/server/src/routes/index.ts index 9c0c35dd..b3518ce7 100644 --- a/server/src/routes/index.ts +++ b/server/src/routes/index.ts @@ -3,10 +3,11 @@ import containers from './containers'; import ping from './ping'; import devices from './devices'; import admin from './admin'; -import ansible from './ansible'; +import playbooks from './playbooks'; import logs from './logs'; import user from './user'; import settings from './settings'; +import playbooksRepository from './playbooks-repository'; const router = express.Router(); @@ -14,10 +15,11 @@ const router = express.Router(); router.use('/devices', devices); router.use('/ping', ping); router.use('/admin', admin); -router.use('/ansible', ansible); +router.use('/playbooks', playbooks); router.use('/logs/', logs); router.use('/settings', settings); router.use('/', user); router.use('/containers', containers); +router.use('/playbooks-repository', playbooksRepository); export default router; diff --git a/server/src/routes/playbooks-repository.ts b/server/src/routes/playbooks-repository.ts new file mode 100644 index 00000000..81714464 --- /dev/null +++ b/server/src/routes/playbooks-repository.ts @@ -0,0 +1,86 @@ +import express from 'express'; +import passport from 'passport'; +import { + addGitRepository, + commitAndSyncRepository, + deleteGitRepository, + forceCloneRepository, + forcePullRepository, + forceRegister, + getGitRepositories, + syncToDatabaseRepository, + updateGitRepository, +} from '../services/playbooks-repository/git'; +import { + addGitRepositoryValidator, + genericGitRepositoryActionValidator, + updateGitRepositoryValidator, +} from '../services/playbooks-repository/git.validator'; +import { + addLocalRepository, + deleteLocalRepository, + getLocalRepositories, + syncToDatabaseLocalRepository, + updateLocalRepository, +} from '../services/playbooks-repository/local'; +import { + addLocalRepositoryValidator, + genericActionLocalRepositoryValidator, + updateLocalRepositoryValidator, +} from '../services/playbooks-repository/local.validator'; +import { + addDirectoryToPlaybookRepositoryValidator, + addPlaybookToRepositoryValidator, + deleteAnyFromRepositoryValidator, +} from '../services/playbooks-repository/platbooks-repository.validator'; +import { + addDirectoryToPlaybookRepository, + addPlaybookToRepository, + deleteAnyFromRepository, + getPlaybooksRepositories, +} from '../services/playbooks-repository/playbooks-repository'; + +const router = express.Router(); + +router.use(passport.authenticate('jwt', { session: false })); + +router.get(`/`, getPlaybooksRepositories); + +router.route('/git/').get(getGitRepositories).put(addGitRepositoryValidator, addGitRepository); +router + .route('/local/') + .get(getLocalRepositories) + .put(addLocalRepositoryValidator, addLocalRepository); +router + .route('/local/:uuid') + .post(updateLocalRepositoryValidator, updateLocalRepository) + .delete(genericActionLocalRepositoryValidator, deleteLocalRepository); +router + .route('/git/:uuid') + .post(updateGitRepositoryValidator, updateGitRepository) + .delete(genericGitRepositoryActionValidator, deleteGitRepository); +router + .route('/git/:uuid/sync-to-database-repository') + .post(genericGitRepositoryActionValidator, syncToDatabaseRepository); +router + .route('/local/:uuid/sync-to-database-repository') + .post(genericActionLocalRepositoryValidator, syncToDatabaseLocalRepository); +router + .route('/git/:uuid/force-pull-repository') + .post(genericGitRepositoryActionValidator, forcePullRepository); +router + .route('/git/:uuid/force-clone-repository') + .post(genericGitRepositoryActionValidator, forceCloneRepository); +router + .route('/git/:uuid/commit-and-sync-repository') + .post(genericGitRepositoryActionValidator, commitAndSyncRepository); +router.route('/git/:uuid/force-register').post(genericGitRepositoryActionValidator, forceRegister); +router + .route('/:uuid/directory/:directoryName/') + .put(addDirectoryToPlaybookRepositoryValidator, addDirectoryToPlaybookRepository); +router + .route('/:uuid/playbook/:playbookName/') + .put(addPlaybookToRepositoryValidator, addPlaybookToRepository); +router.route('/:uuid/').delete(deleteAnyFromRepositoryValidator, deleteAnyFromRepository); + +export default router; diff --git a/server/src/routes/ansible.ts b/server/src/routes/playbooks.ts similarity index 64% rename from server/src/routes/ansible.ts rename to server/src/routes/playbooks.ts index 05af72be..246697ac 100644 --- a/server/src/routes/ansible.ts +++ b/server/src/routes/playbooks.ts @@ -1,43 +1,47 @@ import express from 'express'; import passport from 'passport'; -import { execPlaybook, getLogs, getStatus } from '../services/ansible/execution'; import { + execPlaybook, + execPlaybookByQuickRef, + getLogs, + getStatus, +} from '../services/playbooks/execution'; +import { + execPlaybookByQuickRefValidator, execPlaybookValidator, getLogsValidator, getStatusValidator, -} from '../services/ansible/execution.validator'; -import { addOrUpdateExtraVarValue } from '../services/ansible/extravar'; -import { addOrUpdateExtraVarValueValidator } from '../services/ansible/extravar.validator'; +} from '../services/playbooks/execution.validator'; +import { addOrUpdateExtraVarValue } from '../services/playbooks/extravar'; +import { addOrUpdateExtraVarValueValidator } from '../services/playbooks/extravar.validator'; import { getAnsibleGalaxyCollection, getAnsibleGalaxyCollections, postInstallAnsibleGalaxyCollection, -} from '../services/ansible/galaxy'; +} from '../services/playbooks/galaxy'; import { - getAnsibleGalaxyCollectionsValidator, getAnsibleGalaxyCollectionValidator, + getAnsibleGalaxyCollectionsValidator, postInstallAnsibleGalaxyCollectionValidator, -} from '../services/ansible/galaxy.validator'; -import { addTaskEvent, addTaskStatus } from '../services/ansible/hook'; -import { getInventory } from '../services/ansible/inventory'; +} from '../services/playbooks/galaxy.validator'; +import { addTaskEvent, addTaskStatus } from '../services/playbooks/hook'; +import { getInventory } from '../services/playbooks/inventory'; import { addExtraVarToPlaybook, - addPlaybook, deleteExtraVarFromPlaybook, deletePlaybook, editPlaybook, getPlaybook, getPlaybooks, -} from '../services/ansible/playbook'; +} from '../services/playbooks/playbook'; import { addExtraVarToPlaybookValidator, - addPlaybookValidator, deleteExtraVarFromPlaybookValidator, deletePlaybookValidator, editPlaybookValidator, getPlaybookValidator, -} from '../services/ansible/playbook.validator'; -import { getVaultPwd } from '../services/ansible/vault'; +} from '../services/playbooks/playbook.validator'; +import { getVaultPwd } from '../services/playbooks/vault'; const router = express.Router(); @@ -52,6 +56,8 @@ router.get(`/inventory`, passport.authenticate('bearer', { session: false }), ge router.use(passport.authenticate('jwt', { session: false })); +router.get('/', getPlaybooks); + router.get(`/galaxy/collection`, getAnsibleGalaxyCollectionsValidator, getAnsibleGalaxyCollections); router.get( `/galaxy/collection/details`, @@ -64,26 +70,25 @@ router.post( postInstallAnsibleGalaxyCollection, ); -router.post(`/exec/playbook/:playbook`, execPlaybookValidator, execPlaybook); +// Playbooks execution router.get(`/exec/:id/logs`, getLogsValidator, getLogs); router.get(`/exec/:id/status`, getStatusValidator, getStatus); +router.post(`/exec/quick-ref/:quickRef`, execPlaybookByQuickRefValidator, execPlaybookByQuickRef); +router.post(`/exec/:uuid`, execPlaybookValidator, execPlaybook); + router.post(`/extravars/:varname`, addOrUpdateExtraVarValueValidator, addOrUpdateExtraVarValue); -router - .route('/playbooks/:playbook/') - .get(getPlaybookValidator, getPlaybook) - .patch(editPlaybookValidator, editPlaybook) - .put(addPlaybookValidator, addPlaybook) - .delete(deletePlaybookValidator, deletePlaybook); -router.get(`/playbooks`, getPlaybooks); -router.post( - `/playbooks/:playbook/extravars`, - addExtraVarToPlaybookValidator, - addExtraVarToPlaybook, -); + +router.post(`/:uuid/extravars`, addExtraVarToPlaybookValidator, addExtraVarToPlaybook); router.delete( - `/playbooks/:playbook/extravars/:varname`, + `/:uuid/extravars/:varname`, deleteExtraVarFromPlaybookValidator, deleteExtraVarFromPlaybook, ); +router + .route('/:uuid/') + .get(getPlaybookValidator, getPlaybook) + .patch(editPlaybookValidator, editPlaybook) + .delete(deletePlaybookValidator, deletePlaybook); + export default router; diff --git a/server/src/services/ansible/playbook.validator.ts b/server/src/services/ansible/playbook.validator.ts deleted file mode 100644 index e8e088f7..00000000 --- a/server/src/services/ansible/playbook.validator.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { body, param } from 'express-validator'; -import { Validation } from 'ssm-shared-lib'; -import validator from '../../middlewares/validator'; - -export const getPlaybookValidator = [ - param('playbook').exists().notEmpty().withMessage('Playbook name required'), - validator, -]; - -export const editPlaybookValidator = [ - param('playbook').exists().notEmpty().withMessage('Playbook name required'), - body('content').exists().notEmpty().withMessage('Content of playbook in body required'), - validator, -]; - -export const addPlaybookValidator = [ - param('playbook') - .exists() - .notEmpty() - .withMessage('Playbook name required') - .matches(Validation?.playbookNameRegexp) - .withMessage('Forbidden characters in playbook name'), - validator, -]; - -export const deletePlaybookValidator = [ - param('playbook') - .exists() - .notEmpty() - .withMessage('Playbook name required') - .blacklist('_') - .withMessage('Cannot delete playbook with name that starts with _'), - validator, -]; - -export const addExtraVarToPlaybookValidator = [ - param('playbook').exists().notEmpty().withMessage('Playbook name required'), - body('extraVar').exists().notEmpty().withMessage('ExtraVar in body required'), - validator, -]; - -export const deleteExtraVarFromPlaybookValidator = [ - param('playbook').exists().notEmpty().withMessage('Playbook name required'), - param('varname').exists().notEmpty().withMessage('Varname name required'), - validator, -]; diff --git a/server/src/services/playbooks-repository/git.ts b/server/src/services/playbooks-repository/git.ts new file mode 100644 index 00000000..da0da251 --- /dev/null +++ b/server/src/services/playbooks-repository/git.ts @@ -0,0 +1,155 @@ +import { Playbooks } from 'ssm-shared-lib'; +import { NotFoundError } from '../../core/api/ApiError'; +import { SuccessResponse } from '../../core/api/ApiResponse'; +import PlaybooksRepositoryRepo from '../../data/database/repository/PlaybooksRepositoryRepo'; +import asyncHandler from '../../helpers/AsyncHandler'; +import { DEFAULT_VAULT_ID, vaultEncrypt } from '../../integrations/ansible-vault/vault'; +import GitRepositoryComponent from '../../integrations/git-repository/GitRepositoryComponent'; +import PlaybooksRepositoryEngine from '../../integrations/playbooks-repository/PlaybooksRepositoryEngine'; +import logger from '../../logger'; +import GitRepositoryUseCases from '../../use-cases/GitRepositoryUseCases'; +import PlaybooksRepositoryUseCases from '../../use-cases/PlaybooksRepositoryUseCases'; + +export const addGitRepository = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - PUT - /git/`); + const { + name, + accessToken, + branch, + email, + userName, + remoteUrl, + }: { + name: string; + accessToken: string; + branch: string; + email: string; + userName: string; + remoteUrl: string; + } = req.body; + await GitRepositoryUseCases.addGitRepository( + name, + await vaultEncrypt(accessToken, DEFAULT_VAULT_ID), + branch, + email, + userName, + remoteUrl, + ); + return new SuccessResponse('Added playbooks git repository').send(res); +}); + +export const getGitRepositories = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - GET - /git/`); + const repositories = await PlaybooksRepositoryRepo.findAllWithType( + Playbooks.PlaybooksRepositoryType.GIT, + ); + const encryptedRepositories = repositories?.map((repo) => ({ + ...repo, + accessToken: 'REDACTED', + })); + return new SuccessResponse('Got playbooks git repositories', encryptedRepositories).send(res); +}); + +export const updateGitRepository = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - POST - /git/`); + const { uuid } = req.params; + const { + name, + accessToken, + branch, + email, + gitUserName, + remoteUrl, + }: { + name: string; + accessToken: string; + branch: string; + email: string; + gitUserName: string; + remoteUrl: string; + } = req.body; + await GitRepositoryUseCases.updateGitRepository( + uuid, + name, + await vaultEncrypt(accessToken, DEFAULT_VAULT_ID), + branch, + email, + gitUserName, + remoteUrl, + ); + return new SuccessResponse('Updated playbooks git repository').send(res); +}); + +export const deleteGitRepository = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - DELETE - /git/:uuid`); + const { uuid } = req.params; + const repository = await PlaybooksRepositoryRepo.findByUuid(uuid); + if (!repository) { + throw new NotFoundError(); + } + await PlaybooksRepositoryUseCases.deleteRepository(repository); + return new SuccessResponse('Deleted playbooks-repository repository').send(res); +}); + +export const forcePullRepository = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - POST - /git/:uuid/force-pull-repository`); + const { uuid } = req.params; + const repository = PlaybooksRepositoryEngine.getState().playbooksRepository[ + uuid + ] as GitRepositoryComponent; + if (!repository) { + throw new NotFoundError(); + } + await repository.forcePull(); + return new SuccessResponse('Forced pull playbooks git repository').send(res); +}); + +export const forceCloneRepository = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - POST - /git/:uuid/force-clone-repository`); + const { uuid } = req.params; + const repository = PlaybooksRepositoryEngine.getState().playbooksRepository[ + uuid + ] as GitRepositoryComponent; + if (!repository) { + throw new NotFoundError(); + } + await repository.clone(); + return new SuccessResponse('Forced cloned playbooks git repository').send(res); +}); + +export const commitAndSyncRepository = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - POST - /git/:uuid/commit-and-sync-repository`); + const { uuid } = req.params; + const repository = PlaybooksRepositoryEngine.getState().playbooksRepository[ + uuid + ] as GitRepositoryComponent; + if (!repository) { + throw new NotFoundError(); + } + await repository.commitAndSync(); + return new SuccessResponse('Commit And Synced playbooks git repository').send(res); +}); + +export const syncToDatabaseRepository = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - POST - /git/:uuid/sync-to-database-repository`); + const { uuid } = req.params; + const repository = PlaybooksRepositoryEngine.getState().playbooksRepository[ + uuid + ] as GitRepositoryComponent; + if (!repository) { + throw new NotFoundError(); + } + await repository.syncToDatabase(); + return new SuccessResponse('Synced to database playbooks git repository').send(res); +}); + +export const forceRegister = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - POST - /git/:uuid/force-register`); + const { uuid } = req.params; + const repository = await PlaybooksRepositoryRepo.findByUuid(uuid); + if (!repository) { + throw new NotFoundError(); + } + await PlaybooksRepositoryEngine.registerRepository(repository); + return new SuccessResponse('Synced to database playbooks git repository').send(res); +}); diff --git a/server/src/services/playbooks-repository/git.validator.ts b/server/src/services/playbooks-repository/git.validator.ts new file mode 100644 index 00000000..dbf90134 --- /dev/null +++ b/server/src/services/playbooks-repository/git.validator.ts @@ -0,0 +1,28 @@ +import { body, param } from 'express-validator'; +import validator from '../../middlewares/validator'; + +export const addGitRepositoryValidator = [ + body('name').exists().isString().withMessage('Name is incorrect'), + body('accessToken').exists().isString().withMessage('Access token is incorrect'), + body('branch').exists().isString().withMessage('Branch is incorrect'), + body('email').exists().isEmail().withMessage('Email is incorrect'), + body('userName').exists().isString().withMessage('userName is incorrect'), + body('remoteUrl').exists().isURL().withMessage('remoteUrl is incorrect'), + validator, +]; + +export const updateGitRepositoryValidator = [ + param('uuid').exists().isString().isUUID().withMessage('Uuid is incorrect'), + body('name').exists().isString().withMessage('Name is incorrect'), + body('accessToken').exists().isString().withMessage('Access token is incorrect'), + body('branch').exists().isString().withMessage('Branch is incorrect'), + body('email').exists().isEmail().withMessage('Email is incorrect'), + body('userName').exists().isString().withMessage('userName is incorrect'), + body('remoteUrl').exists().isURL().withMessage('remoteUrl is incorrect'), + validator, +]; + +export const genericGitRepositoryActionValidator = [ + param('uuid').exists().isString().withMessage('Uuid is incorrect'), + validator, +]; diff --git a/server/src/services/playbooks-repository/local.ts b/server/src/services/playbooks-repository/local.ts new file mode 100644 index 00000000..a6f6d222 --- /dev/null +++ b/server/src/services/playbooks-repository/local.ts @@ -0,0 +1,66 @@ +import { Playbooks } from 'ssm-shared-lib'; +import { NotFoundError } from '../../core/api/ApiError'; +import { SuccessResponse } from '../../core/api/ApiResponse'; +import PlaybooksRepositoryRepo from '../../data/database/repository/PlaybooksRepositoryRepo'; +import asyncHandler from '../../helpers/AsyncHandler'; +import GitRepositoryComponent from '../../integrations/git-repository/GitRepositoryComponent'; +import LocalRepositoryComponent from '../../integrations/local-repository/LocalRepositoryComponent'; +import PlaybooksRepositoryEngine from '../../integrations/playbooks-repository/PlaybooksRepositoryEngine'; +import logger from '../../logger'; +import LocalRepositoryUseCases from '../../use-cases/LocalRepositoryUseCases'; +import PlaybooksRepositoryUseCases from '../../use-cases/PlaybooksRepositoryUseCases'; + +export const getLocalRepositories = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - GET - /local/`); + const repositories = await PlaybooksRepositoryRepo.findAllWithType( + Playbooks.PlaybooksRepositoryType.LOCAL, + ); + return new SuccessResponse('Got playbooks local repositories', repositories).send(res); +}); + +export const updateLocalRepository = asyncHandler(async (req, res) => { + const { uuid } = req.params; + logger.info(`[CONTROLLER] - POST - /local/:uuid`); + const { + name, + }: { + name: string; + } = req.body; + await LocalRepositoryUseCases.updateLocalRepository(uuid, name); + return new SuccessResponse('Updated playbooks local repository').send(res); +}); + +export const deleteLocalRepository = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - DELETE - /local/:uuid`); + const { uuid } = req.params; + const repository = await PlaybooksRepositoryRepo.findByUuid(uuid); + if (!repository) { + throw new NotFoundError(); + } + await PlaybooksRepositoryUseCases.deleteRepository(repository); + return new SuccessResponse('Deleted playbooks local repository').send(res); +}); + +export const addLocalRepository = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - PUT - /local/`); + const { + name, + }: { + name: string; + } = req.body; + await LocalRepositoryUseCases.addLocalRepository(name); + return new SuccessResponse('Added playbooks local repository').send(res); +}); + +export const syncToDatabaseLocalRepository = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - POST - /local/:uuid/sync-to-database-repository`); + const { uuid } = req.params; + const repository = PlaybooksRepositoryEngine.getState().playbooksRepository[ + uuid + ] as LocalRepositoryComponent; + if (!repository) { + throw new NotFoundError(); + } + await repository.syncToDatabase(); + return new SuccessResponse('Synced to database playbooks local repository').send(res); +}); diff --git a/server/src/services/playbooks-repository/local.validator.ts b/server/src/services/playbooks-repository/local.validator.ts new file mode 100644 index 00000000..38fc098e --- /dev/null +++ b/server/src/services/playbooks-repository/local.validator.ts @@ -0,0 +1,18 @@ +import { body, param } from 'express-validator'; +import validator from '../../middlewares/validator'; + +export const addLocalRepositoryValidator = [ + body('name').exists().isString().withMessage('Name is incorrect'), + validator, +]; + +export const updateLocalRepositoryValidator = [ + param('uuid').exists().isString().isUUID().withMessage('Uuid is incorrect'), + body('name').exists().isString().withMessage('Name is incorrect'), + validator, +]; + +export const genericActionLocalRepositoryValidator = [ + param('uuid').exists().isString().withMessage('Uuid is incorrect'), + validator, +]; diff --git a/server/src/services/playbooks-repository/platbooks-repository.validator.ts b/server/src/services/playbooks-repository/platbooks-repository.validator.ts new file mode 100644 index 00000000..49654488 --- /dev/null +++ b/server/src/services/playbooks-repository/platbooks-repository.validator.ts @@ -0,0 +1,22 @@ +import { body, param } from 'express-validator'; +import validator from '../../middlewares/validator'; + +export const addDirectoryToPlaybookRepositoryValidator = [ + param('uuid').exists().isString().isUUID().withMessage('Uuid is incorrect'), + param('directoryName').exists().isString().withMessage('Name is incorrect'), + body('fullPath').exists().isString().withMessage('path is incorrect'), + validator, +]; + +export const addPlaybookToRepositoryValidator = [ + param('uuid').exists().isString().isUUID().withMessage('Uuid is incorrect'), + param('playbookName').exists().isString().withMessage('Name is incorrect'), + body('fullPath').exists().isString().withMessage('path is incorrect'), + validator, +]; + +export const deleteAnyFromRepositoryValidator = [ + param('uuid').exists().isString().isUUID().withMessage('Uuid is incorrect'), + body('fullPath').exists().isString().withMessage('path is incorrect'), + validator, +]; diff --git a/server/src/services/playbooks-repository/playbooks-repository.ts b/server/src/services/playbooks-repository/playbooks-repository.ts new file mode 100644 index 00000000..a9867e54 --- /dev/null +++ b/server/src/services/playbooks-repository/playbooks-repository.ts @@ -0,0 +1,71 @@ +import { InternalError, NotFoundError } from '../../core/api/ApiError'; +import { SuccessResponse } from '../../core/api/ApiResponse'; +import PlaybooksRepositoryRepo from '../../data/database/repository/PlaybooksRepositoryRepo'; +import asyncHandler from '../../helpers/AsyncHandler'; +import logger from '../../logger'; +import PlaybooksRepositoryUseCases from '../../use-cases/PlaybooksRepositoryUseCases'; + +export const getPlaybooksRepositories = asyncHandler(async (req, res) => { + logger.info(`[CONTROLLER] - GET - /playbook-repositories`); + try { + const listOfPlaybooksToSelect = await PlaybooksRepositoryUseCases.getAllPlaybooksRepositories(); + new SuccessResponse('Get playbooks successful', listOfPlaybooksToSelect).send(res); + } catch (error: any) { + throw new InternalError(error.message); + } +}); + +export const addDirectoryToPlaybookRepository = asyncHandler(async (req, res) => { + const { uuid, directoryName } = req.params; + const { fullPath } = req.body; + logger.info(`[CONTROLLER] - PUT - /playbook-repositories/${uuid}/directory/${directoryName}`); + const playbookRepository = await PlaybooksRepositoryRepo.findByUuid(uuid); + if (!playbookRepository) { + throw new NotFoundError(`PlaybookRepository ${uuid} not found`); + } + try { + await PlaybooksRepositoryUseCases.createDirectoryInPlaybookRepository( + playbookRepository, + fullPath, + ); + new SuccessResponse('Created directory successfully').send(res); + } catch (error: any) { + throw new InternalError(error.message); + } +}); + +export const addPlaybookToRepository = asyncHandler(async (req, res) => { + const { uuid, playbookName } = req.params; + const { fullPath } = req.body; + logger.info(`[CONTROLLER] - PUT - /playbook-repositories/${uuid}/playbook/${playbookName}`); + const playbookRepository = await PlaybooksRepositoryRepo.findByUuid(uuid); + if (!playbookRepository) { + throw new NotFoundError(`PlaybookRepository ${uuid} not found`); + } + try { + const createdPlaybook = await PlaybooksRepositoryUseCases.createPlaybookInRepository( + playbookRepository, + fullPath, + playbookName, + ); + new SuccessResponse('Add playbook successful', createdPlaybook).send(res); + } catch (error: any) { + throw new InternalError(error.message); + } +}); + +export const deleteAnyFromRepository = asyncHandler(async (req, res) => { + const { uuid } = req.params; + const { fullPath } = req.body; + logger.info(`[CONTROLLER] - DELETE - /playbook-repositories/${uuid}/playbook/`); + const playbookRepository = await PlaybooksRepositoryRepo.findByUuid(uuid); + if (!playbookRepository) { + throw new NotFoundError(`PlaybookRepository ${uuid} not found`); + } + try { + await PlaybooksRepositoryUseCases.deleteAnyInPlaybooksRepository(playbookRepository, fullPath); + new SuccessResponse('Deletion successful').send(res); + } catch (error: any) { + throw new InternalError(error.message); + } +}); diff --git a/server/src/services/ansible/execution.ts b/server/src/services/playbooks/execution.ts similarity index 66% rename from server/src/services/ansible/execution.ts rename to server/src/services/playbooks/execution.ts index af9b9811..a93f1c03 100644 --- a/server/src/services/ansible/execution.ts +++ b/server/src/services/playbooks/execution.ts @@ -9,12 +9,35 @@ import logger from '../../logger'; import PlaybookUseCases from '../../use-cases/PlaybookUseCases'; export const execPlaybook = asyncHandler(async (req, res) => { - logger.info(`[CONTROLLER] - POST - ansible/exec/playbook - '${req.params.playbook}'`); - const playbook = await PlaybookRepo.findOne( - req.params.playbook + (req.params.playbook.endsWith('.yml') ? '' : '.yml'), - ); + const { uuid } = req.params; + logger.info(`[CONTROLLER] - POST - playbooks/exec/${uuid}`); + const playbook = await PlaybookRepo.findOneByUuid(uuid); if (!playbook) { - throw new NotFoundError('Playbook not found'); + throw new NotFoundError(`Playbook ${uuid} not found`); + } + if (!req.user) { + throw new NotFoundError('No user'); + } + try { + const execId = await PlaybookUseCases.executePlaybook( + playbook, + req.user, + req.body.target, + req.body.extraVars as API.ExtraVars, + ); + new SuccessResponse('Execution succeeded', { execId: execId } as API.ExecId).send(res); + } catch (error: any) { + logger.error(error); + throw new InternalError(error.message); + } +}); + +export const execPlaybookByQuickRef = asyncHandler(async (req, res) => { + const { quickRef } = req.params; + logger.info(`[CONTROLLER] - POST - playbooks/exec/quick-ref/${quickRef}`); + const playbook = await PlaybookRepo.findOneByUniqueQuickReference(quickRef); + if (!playbook) { + throw new NotFoundError(`Playbook ${quickRef} not found`); } if (!req.user) { throw new NotFoundError('No user'); diff --git a/server/src/services/ansible/execution.validator.ts b/server/src/services/playbooks/execution.validator.ts similarity index 62% rename from server/src/services/ansible/execution.validator.ts rename to server/src/services/playbooks/execution.validator.ts index 0ed2a944..ca39a2be 100644 --- a/server/src/services/ansible/execution.validator.ts +++ b/server/src/services/playbooks/execution.validator.ts @@ -2,10 +2,14 @@ import { param } from 'express-validator'; import validator from '../../middlewares/validator'; export const execPlaybookValidator = [ - param('playbook').exists().notEmpty().withMessage('Playbook required'), + param('uuid').exists().notEmpty().isUUID().withMessage('Playbook uuid required'), validator, ]; +export const execPlaybookByQuickRefValidator = [ + param('quickRef').exists().notEmpty().withMessage('Playbook quickRef required'), + validator, +]; export const getLogsValidator = [ param('id').exists().notEmpty().withMessage('Exec Id required'), validator, diff --git a/server/src/services/ansible/extravar.ts b/server/src/services/playbooks/extravar.ts similarity index 100% rename from server/src/services/ansible/extravar.ts rename to server/src/services/playbooks/extravar.ts diff --git a/server/src/services/ansible/extravar.validator.ts b/server/src/services/playbooks/extravar.validator.ts similarity index 100% rename from server/src/services/ansible/extravar.validator.ts rename to server/src/services/playbooks/extravar.validator.ts diff --git a/server/src/services/ansible/galaxy.ts b/server/src/services/playbooks/galaxy.ts similarity index 100% rename from server/src/services/ansible/galaxy.ts rename to server/src/services/playbooks/galaxy.ts diff --git a/server/src/services/ansible/galaxy.validator.ts b/server/src/services/playbooks/galaxy.validator.ts similarity index 100% rename from server/src/services/ansible/galaxy.validator.ts rename to server/src/services/playbooks/galaxy.validator.ts diff --git a/server/src/services/ansible/hook.ts b/server/src/services/playbooks/hook.ts similarity index 86% rename from server/src/services/ansible/hook.ts rename to server/src/services/playbooks/hook.ts index 7dc5d070..dca29bee 100644 --- a/server/src/services/ansible/hook.ts +++ b/server/src/services/playbooks/hook.ts @@ -8,9 +8,9 @@ import asyncHandler from '../../helpers/AsyncHandler'; import logger from '../../logger'; export const addTaskStatus = asyncHandler(async (req, res) => { - logger.info('[CONTROLLER] - POST - /ansible/hook/task/status'); + logger.info('[CONTROLLER] - POST - /playbooks/hook/task/status'); if (!req.body.runner_ident || !req.body.status) { - logger.error('[CONTROLLER] ansible/hook/task/status - malformed request'); + logger.error('[CONTROLLER] playbooks/hook/task/status - malformed request'); logger.error(req.body); res.status(400).send({ success: false, @@ -33,9 +33,9 @@ export const addTaskStatus = asyncHandler(async (req, res) => { }); export const addTaskEvent = asyncHandler(async (req, res) => { - logger.info('[CONTROLLER] - POST - ansible/hook/task/event'); + logger.info('[CONTROLLER] - POST - playbooks/hook/task/event'); if (!req.body.runner_ident) { - logger.error('[CONTROLLER] ansible/hook/tasks/events - malformed request'); + logger.error('[CONTROLLER] playbooks/hook/tasks/events - malformed request'); logger.error(req.body); res.status(400).send({ success: false, diff --git a/server/src/services/ansible/inventory.ts b/server/src/services/playbooks/inventory.ts similarity index 100% rename from server/src/services/ansible/inventory.ts rename to server/src/services/playbooks/inventory.ts diff --git a/server/src/services/ansible/playbook.ts b/server/src/services/playbooks/playbook.ts similarity index 52% rename from server/src/services/ansible/playbook.ts rename to server/src/services/playbooks/playbook.ts index ae8ffaf5..34d1f929 100644 --- a/server/src/services/ansible/playbook.ts +++ b/server/src/services/playbooks/playbook.ts @@ -4,22 +4,27 @@ import PlaybookRepo from '../../data/database/repository/PlaybookRepo'; import asyncHandler from '../../helpers/AsyncHandler'; import logger from '../../logger'; import shell from '../../integrations/shell'; +import PlaybooksRepositoryUseCases from '../../use-cases/PlaybooksRepositoryUseCases'; import PlaybookUseCases from '../../use-cases/PlaybookUseCases'; export const getPlaybooks = asyncHandler(async (req, res) => { - logger.info(`[CONTROLLER] - GET - /ansible/playbooks`); try { - const listOfPlaybooksToSelect = await PlaybookUseCases.getAllPlaybooks(); - new SuccessResponse('Get playbooks successful', listOfPlaybooksToSelect).send(res); + const playbooks = await PlaybookRepo.findAllWithActiveRepositories(); + new SuccessResponse('Got Playbooks successfuly', playbooks).send(res); } catch (error: any) { throw new InternalError(error.message); } }); export const getPlaybook = asyncHandler(async (req, res) => { - logger.info(`[CONTROLLER] - GET - /ansible/playbooks/${req.params.playbook}`); + const { uuid } = req.params; + logger.info(`[CONTROLLER] - GET - /playbooks/${uuid}`); + const playbook = await PlaybookRepo.findOneByUuid(uuid); + if (!playbook) { + throw new NotFoundError(`Playbook ${uuid} not found`); + } try { - const content = await shell.readPlaybook(req.params.playbook); + const content = await shell.readPlaybook(playbook.path); new SuccessResponse('Get Playbook successful', content).send(res); } catch (error: any) { throw new InternalError(error.message); @@ -27,69 +32,60 @@ export const getPlaybook = asyncHandler(async (req, res) => { }); export const editPlaybook = asyncHandler(async (req, res) => { - logger.info(`[CONTROLLER][ANSIBLE] - PATCH - /ansible/playbooks/${req.params.playbook}`); - const playbook = await PlaybookRepo.findOne(req.params.playbook); + const { uuid } = req.params; + logger.info(`[CONTROLLER][ANSIBLE] - PATCH - /playbooks/${uuid}`); + const playbook = await PlaybookRepo.findOneByUuid(uuid); if (!playbook) { - throw new NotFoundError('Playbook not found'); + throw new NotFoundError(`Playbook ${uuid} not found`); } try { - await shell.editPlaybook(playbook.name, req.body.content); + await shell.editPlaybook(playbook.path, req.body.content); new SuccessResponse('Edit playbook successful').send(res); } catch (error: any) { throw new InternalError(error.message); } }); -export const addPlaybook = asyncHandler(async (req, res) => { - const { playbook } = req.params; - logger.info(`[CONTROLLER] - PUT - /ansible/playbooks/${playbook}`); - try { - await PlaybookUseCases.createCustomPlaybook(playbook); - new SuccessResponse('Add playbook successful').send(res); - } catch (error: any) { - throw new InternalError(error.message); - } -}); - -export const deletePlaybook = asyncHandler(async (req, res) => { - logger.info(`[CONTROLLER] - DELETE - /ansible/playbooks/${req.params.playbook}`); - const playbook = await PlaybookRepo.findOne(req.params.playbook); +export const addExtraVarToPlaybook = asyncHandler(async (req, res) => { + const { uuid } = req.params; + logger.info(`[CONTROLLER] - POST - /${uuid}/extravars`); + const playbook = await PlaybookRepo.findOneByUuid(uuid); if (!playbook) { - throw new NotFoundError('Playbook not found'); + throw new NotFoundError(`Playbook ${uuid} not found`); } try { - await PlaybookUseCases.deleteCustomPlaybook(playbook); - new SuccessResponse('Delete playbook successful').send(res); + await PlaybookUseCases.addExtraVarToPlaybook(playbook, req.body.extraVar); + new SuccessResponse('Add extra var to playbook successful').send(res); } catch (error: any) { throw new InternalError(error.message); } }); -export const addExtraVarToPlaybook = asyncHandler(async (req, res) => { - logger.info(`[CONTROLLER] - POST - /ansible/playbooks/${req.params.playbook}/extravars`); - const playbook = await PlaybookRepo.findOne(req.params.playbook); +export const deleteExtraVarFromPlaybook = asyncHandler(async (req, res) => { + const { uuid, varname } = req.params; + logger.info(`[CONTROLLER] - DELETE - /${uuid}/extravars/${varname}`); + const playbook = await PlaybookRepo.findOneByUuid(uuid); if (!playbook) { - throw new NotFoundError('Playbook not found '); + throw new NotFoundError(`Playbook ${uuid} not found`); } try { - await PlaybookUseCases.addExtraVarToPlaybook(playbook, req.body.extraVar); - new SuccessResponse('Add extra var to playbook successful').send(res); + await PlaybookUseCases.deleteExtraVarFromPlaybook(playbook, varname); + new SuccessResponse('Delete extra var from playbook successful').send(res); } catch (error: any) { throw new InternalError(error.message); } }); -export const deleteExtraVarFromPlaybook = asyncHandler(async (req, res) => { - logger.info( - `[CONTROLLER] - DELETE - /ansible/playbooks/${req.params.playbook}/extravars/${req.params.varname}`, - ); - const playbook = await PlaybookRepo.findOne(req.params.playbook + '.yml'); +export const deletePlaybook = asyncHandler(async (req, res) => { + const { uuid } = req.params; + logger.info(`[CONTROLLER] - DELETE - /playbook/${uuid}`); + const playbook = await PlaybookRepo.findOneByUuid(uuid); if (!playbook) { - throw new NotFoundError('Playbook not found'); + throw new NotFoundError(`Playbook ${uuid} not found`); } try { - await PlaybookUseCases.deleteExtraVarFromPlaybook(playbook, req.params.varname); - new SuccessResponse('Delete extra var from playbook successful').send(res); + await PlaybooksRepositoryUseCases.deletePlaybooksInRepository(playbook); + new SuccessResponse('Delete playbook successful').send(res); } catch (error: any) { throw new InternalError(error.message); } diff --git a/server/src/services/playbooks/playbook.validator.ts b/server/src/services/playbooks/playbook.validator.ts new file mode 100644 index 00000000..5c92eb4f --- /dev/null +++ b/server/src/services/playbooks/playbook.validator.ts @@ -0,0 +1,31 @@ +import { body, param } from 'express-validator'; +import { Validation } from 'ssm-shared-lib'; +import validator from '../../middlewares/validator'; + +export const getPlaybookValidator = [ + param('uuid').exists().notEmpty().isUUID().withMessage('Playbook uuid required'), + validator, +]; + +export const editPlaybookValidator = [ + param('uuid').exists().notEmpty().isUUID().withMessage('Playbook uuid required'), + body('content').exists().notEmpty().withMessage('Content of playbook in body required'), + validator, +]; + +export const deletePlaybookValidator = [ + param('uuid').exists().notEmpty().isUUID().withMessage('Playbook uuid required'), + validator, +]; + +export const addExtraVarToPlaybookValidator = [ + param('uuid').exists().notEmpty().isUUID().withMessage('Playbook uuid required'), + body('extraVar').exists().notEmpty().withMessage('ExtraVar in body required'), + validator, +]; + +export const deleteExtraVarFromPlaybookValidator = [ + param('uuid').exists().notEmpty().isUUID().withMessage('Playbook uuid required'), + param('varname').exists().notEmpty().withMessage('Varname required'), + validator, +]; diff --git a/server/src/services/ansible/vault.ts b/server/src/services/playbooks/vault.ts similarity index 86% rename from server/src/services/ansible/vault.ts rename to server/src/services/playbooks/vault.ts index 80840749..bd3baa9f 100644 --- a/server/src/services/ansible/vault.ts +++ b/server/src/services/playbooks/vault.ts @@ -4,7 +4,7 @@ import asyncHandler from '../../helpers/AsyncHandler'; import logger from '../../logger'; export const getVaultPwd = asyncHandler(async (req, res) => { - logger.info('[CONTROLLER] - GET - ansible/vault'); + logger.info('[CONTROLLER] - GET - playbooks/vault'); new SuccessResponse('Successfully got vault pwd', { pwd: VAULT_PWD }).send(res); }); diff --git a/server/src/tests/helpers/FilterHelper.test.ts b/server/src/tests/helpers/FilterHelper.test.ts index 15c7e556..98461354 100644 --- a/server/src/tests/helpers/FilterHelper.test.ts +++ b/server/src/tests/helpers/FilterHelper.test.ts @@ -1,5 +1,5 @@ +import { describe, expect, test } from 'vitest'; import { filterByFields, filterByQueryParams } from '../../helpers/FilterHelper'; -import { describe, test, expect } from 'vitest'; const data = [ { name: 'John Doe', age: 30, registered: true }, @@ -50,8 +50,8 @@ const params2 = { describe('filterByQueryParams', () => { test('filters by the given query param', () => { - let params = { name: 'doe' }; - let result = filterByQueryParams(data, params, authorizedParams); + const params = { name: 'doe' }; + const result = filterByQueryParams(data, params, authorizedParams); expect(result.length).toBe(2); result.forEach((r) => expect(r.name.toLowerCase()).toContain(params.name)); }); @@ -72,7 +72,7 @@ describe('filterByQueryParams', () => { }); test('filters by multiple given query params', () => { - let result = filterByQueryParams(data, params2, authorizedParams); + const result = filterByQueryParams(data, params2, authorizedParams); expect(result.length).toBe(1); expect(result[0].name.toLowerCase()).toContain(params2.name); expect(result[0].age).toBe(parseInt(params2.age)); @@ -80,7 +80,7 @@ describe('filterByQueryParams', () => { test('processes numeric params correctly', () => { const params = { age: '18' }; // This should match Donald Duck - let result = filterByQueryParams(data, params, authorizedParams); + const result = filterByQueryParams(data, params, authorizedParams); expect(result.length).toBe(1); expect(result[0].name).toBe('Donald Duck'); }); diff --git a/server/src/tests/helpers/directory-tree/constants.ts b/server/src/tests/helpers/directory-tree/constants.ts new file mode 100644 index 00000000..54be888e --- /dev/null +++ b/server/src/tests/helpers/directory-tree/constants.ts @@ -0,0 +1 @@ +export const TEST_DATA_DIRECTORY = `${__dirname}/test_data`; diff --git a/server/src/tests/helpers/directory-tree/depth/fixtureFirstDepth.ts b/server/src/tests/helpers/directory-tree/depth/fixtureFirstDepth.ts new file mode 100644 index 00000000..f41c0121 --- /dev/null +++ b/server/src/tests/helpers/directory-tree/depth/fixtureFirstDepth.ts @@ -0,0 +1,32 @@ +import { TEST_DATA_DIRECTORY } from '../constants'; + +const tree = { + path: `${TEST_DATA_DIRECTORY}`, + name: 'test_data', + type: 'directory', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/file_a.txt`, + name: 'file_a.txt', + type: 'file', + extension: '.txt', + }, + { + path: `${TEST_DATA_DIRECTORY}/file_b.txt`, + name: 'file_b.txt', + type: 'file', + extension: '.txt', + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir`, + name: 'some_dir', + type: 'directory', + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir_2`, + name: 'some_dir_2', + type: 'directory', + }, + ], +}; +export default tree; diff --git a/server/src/tests/helpers/directory-tree/depth/fixtureSecondDepth.ts b/server/src/tests/helpers/directory-tree/depth/fixtureSecondDepth.ts new file mode 100644 index 00000000..1ab16956 --- /dev/null +++ b/server/src/tests/helpers/directory-tree/depth/fixtureSecondDepth.ts @@ -0,0 +1,60 @@ +import { TEST_DATA_DIRECTORY } from '../constants'; + +const tree = { + path: `${TEST_DATA_DIRECTORY}`, + name: 'test_data', + type: 'directory', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/file_a.txt`, + name: 'file_a.txt', + type: 'file', + extension: '.txt', + }, + { + path: `${TEST_DATA_DIRECTORY}/file_b.txt`, + name: 'file_b.txt', + type: 'file', + extension: '.txt', + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir`, + name: 'some_dir', + type: 'directory', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/some_dir/another_dir`, + name: 'another_dir', + type: 'directory', + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir/file_a.txt`, + name: 'file_a.txt', + type: 'file', + extension: '.txt', + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir/file_b.txt`, + name: 'file_b.txt', + type: 'file', + extension: '.txt', + }, + ], + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir_2`, + name: 'some_dir_2', + type: 'directory', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/some_dir_2/.gitkeep`, + name: '.gitkeep', + type: 'file', + extension: '', + }, + ], + }, + ], +}; + +export default tree; diff --git a/server/src/tests/helpers/directory-tree/depth/fixtureZeroDepth.ts b/server/src/tests/helpers/directory-tree/depth/fixtureZeroDepth.ts new file mode 100644 index 00000000..93b372fe --- /dev/null +++ b/server/src/tests/helpers/directory-tree/depth/fixtureZeroDepth.ts @@ -0,0 +1,9 @@ +import { TEST_DATA_DIRECTORY } from '../constants'; + +const tree = { + path: `${TEST_DATA_DIRECTORY}`, + name: 'test_data', + type: 'directory', +}; + +export default tree; diff --git a/server/src/tests/helpers/directory-tree/directory-tree.test.ts b/server/src/tests/helpers/directory-tree/directory-tree.test.ts new file mode 100644 index 00000000..a5367a31 --- /dev/null +++ b/server/src/tests/helpers/directory-tree/directory-tree.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, test } from 'vitest'; +import directoryTree from '../../../helpers/directory-tree/directory-tree'; +import { TEST_DATA_DIRECTORY } from './constants'; +import testTree from './fixture.js'; +import testTreeZeroDepth from './depth/fixtureZeroDepth.js'; +import testTreeFirstDepth from './depth/fixtureFirstDepth.js'; +import testTreeSecondDepth from './depth/fixtureSecondDepth.js'; +import excludeTree from './fixtureExclude'; +import excludeTree2 from './fixtureMultipleExclude'; + +describe('directoryTree', () => { + test('should not crash with empty options', () => { + expect(directoryTree(TEST_DATA_DIRECTORY)).not.null; + }); + + test('should return an Object', () => { + const tree = directoryTree(TEST_DATA_DIRECTORY, { + extensions: /\.txt$/, + followSymlinks: false, + }); + expect(tree).to.be.an('object'); + }); + + test('should list the children in a directory', () => { + const tree = directoryTree(TEST_DATA_DIRECTORY, { + extensions: /\.txt$/, + followSymlinks: false, + }); + + // 4 including the empty `some_dir_2`. + expect(tree?.children?.length).to.equal(4); + }); + + test('should execute a callback function for each file with no specified extensions', () => { + const number_of_files = 7; + let callback_executed_times = 0; + + directoryTree(TEST_DATA_DIRECTORY, { followSymlinks: false }, function () { + callback_executed_times++; + }); + + expect(callback_executed_times).to.equal(number_of_files); + }); + + test('should execute a callback function for each directory', () => { + const number_of_directories = 4; + let callback_executed_times = 0; + + directoryTree(TEST_DATA_DIRECTORY, { followSymlinks: false }, undefined, function () { + callback_executed_times++; + }); + + expect(callback_executed_times).to.equal(number_of_directories); + }); + + test('should execute a callback function for each file with specified extensions', () => { + const number_of_files = 6; + let callback_executed_times = 0; + + directoryTree( + TEST_DATA_DIRECTORY, + { extensions: /\.txt$/, followSymlinks: false }, + function () { + callback_executed_times++; + }, + ); + expect(callback_executed_times).to.equal(number_of_files); + }); + + test('should display the size of a directory (summing up the children)', () => { + const tree = directoryTree(TEST_DATA_DIRECTORY, { extensions: /\.txt$/, attributes: ['size'] }); + expect(tree?.size).to.be.above(11000); + }); + + test('should not crash with directories where the user does not have necessary permissions', () => { + const tree = directoryTree('/root/', { extensions: /\.txt$/ }); + expect(tree).to.equal(null); + }); + + test('should return the correct exact result', () => { + const tree = directoryTree(TEST_DATA_DIRECTORY, { + normalizePath: true, + followSymlinks: false, + attributes: ['size', 'type', 'extension'], + }); + expect(tree).to.deep.equal(testTree); + }); + + test('should not swallow exceptions thrown in the callback function', () => { + const error = new Error('Something happened!'); + const badFunction = function () { + directoryTree(TEST_DATA_DIRECTORY, { extensions: /\.txt$/ }, function () { + throw error; + }); + }; + expect(badFunction).to.throw(error); + }); + + test('should exclude the correct folders', () => { + const tree = directoryTree(TEST_DATA_DIRECTORY, { + exclude: /another_dir/, + normalizePath: true, + followSymlinks: false, + attributes: ['size', 'type', 'extension'], + }); + expect(tree).to.deep.equal(excludeTree); + }); + + test('should exclude multiple folders', () => { + const tree = directoryTree(TEST_DATA_DIRECTORY, { + exclude: [/another_dir/, /some_dir_2/], + normalizePath: true, + followSymlinks: false, + attributes: ['size', 'type', 'extension'], + }); + expect(tree).to.deep.equal(excludeTree2); + }); + + test('should include attributes', () => { + const tree = directoryTree(TEST_DATA_DIRECTORY, { + attributes: ['mtime', 'ctime'], + followSymlinks: false, + }); + tree?.children?.forEach((child) => { + if (child?.type === 'file') { + expect(child).to.have.property('mtime'); + expect(child).to.have.property('ctime'); + } + }); + }); + + test('should respect "depth = 0" argument', () => { + const tree = directoryTree(TEST_DATA_DIRECTORY, { + depth: 0, + normalizePath: true, + followSymlinks: false, + attributes: ['type', 'extension'], + }); + expect(tree).to.deep.equal(testTreeZeroDepth); + }); + + test('should respect "depth = 1" argument', () => { + const tree = directoryTree(TEST_DATA_DIRECTORY, { + depth: 1, + normalizePath: true, + followSymlinks: false, + attributes: ['type', 'extension'], + }); + expect(tree).to.deep.equal(testTreeFirstDepth); + }); + + test('should respect "depth = 2" argument', () => { + const tree = directoryTree(TEST_DATA_DIRECTORY, { + depth: 2, + normalizePath: true, + followSymlinks: false, + attributes: ['type', 'extension'], + }); + expect(tree).to.deep.equal(testTreeSecondDepth); + }); + + test('should throw error when combines size attribute with depth option', () => { + expect( + directoryTree.bind(directoryTree, TEST_DATA_DIRECTORY, { + depth: 2, + normalizePath: true, + followSymlinks: false, + attributes: ['size', 'type', 'extension'], + }), + ).to.throw('usage of size attribute with depth option is prohibited'); + }); +}); diff --git a/server/src/tests/helpers/directory-tree/fixture.ts b/server/src/tests/helpers/directory-tree/fixture.ts new file mode 100644 index 00000000..c625b0c2 --- /dev/null +++ b/server/src/tests/helpers/directory-tree/fixture.ts @@ -0,0 +1,85 @@ +import { TEST_DATA_DIRECTORY } from './constants'; + +const tree = { + path: `${TEST_DATA_DIRECTORY}`, + name: 'test_data', + type: 'directory', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/file_a.txt`, + name: 'file_a.txt', + size: 12, + type: 'file', + extension: '.txt', + }, + { + path: `${TEST_DATA_DIRECTORY}/file_b.txt`, + name: 'file_b.txt', + size: 3756, + type: 'file', + extension: '.txt', + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir`, + name: 'some_dir', + type: 'directory', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/some_dir/another_dir`, + name: 'another_dir', + type: 'directory', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/some_dir/another_dir/file_a.txt`, + name: 'file_a.txt', + size: 12, + type: 'file', + extension: '.txt', + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir/another_dir/file_b.txt`, + name: 'file_b.txt', + size: 3756, + type: 'file', + extension: '.txt', + }, + ], + size: 3768, + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir/file_a.txt`, + name: 'file_a.txt', + size: 12, + type: 'file', + extension: '.txt', + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir/file_b.txt`, + name: 'file_b.txt', + size: 3756, + type: 'file', + extension: '.txt', + }, + ], + size: 7536, + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir_2`, + name: 'some_dir_2', + type: 'directory', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/some_dir_2/.gitkeep`, + name: '.gitkeep', + size: 0, + type: 'file', + extension: '', + }, + ], + size: 0, + }, + ], + size: 11304, +}; + +export default tree; diff --git a/server/src/tests/helpers/directory-tree/fixtureExclude.ts b/server/src/tests/helpers/directory-tree/fixtureExclude.ts new file mode 100644 index 00000000..ff280d7a --- /dev/null +++ b/server/src/tests/helpers/directory-tree/fixtureExclude.ts @@ -0,0 +1,63 @@ +import { TEST_DATA_DIRECTORY } from './constants'; + +const tree = { + path: `${TEST_DATA_DIRECTORY}`, + name: 'test_data', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/file_a.txt`, + name: 'file_a.txt', + size: 12, + extension: '.txt', + type: 'file', + }, + { + path: `${TEST_DATA_DIRECTORY}/file_b.txt`, + name: 'file_b.txt', + size: 3756, + extension: '.txt', + type: 'file', + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir`, + name: 'some_dir', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/some_dir/file_a.txt`, + name: 'file_a.txt', + size: 12, + extension: '.txt', + type: 'file', + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir/file_b.txt`, + name: 'file_b.txt', + size: 3756, + extension: '.txt', + type: 'file', + }, + ], + type: 'directory', + size: 3768, + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir_2`, + name: 'some_dir_2', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/some_dir_2/.gitkeep`, + name: '.gitkeep', + size: 0, + extension: '', + type: 'file', + }, + ], + size: 0, + type: 'directory', + }, + ], + size: 7536, + type: 'directory', +}; + +export default tree; diff --git a/server/src/tests/helpers/directory-tree/fixtureMultipleExclude.ts b/server/src/tests/helpers/directory-tree/fixtureMultipleExclude.ts new file mode 100644 index 00000000..483c5e48 --- /dev/null +++ b/server/src/tests/helpers/directory-tree/fixtureMultipleExclude.ts @@ -0,0 +1,48 @@ +import { TEST_DATA_DIRECTORY } from './constants'; + +const tree = { + path: `${TEST_DATA_DIRECTORY}`, + name: 'test_data', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/file_a.txt`, + name: 'file_a.txt', + size: 12, + extension: '.txt', + type: 'file', + }, + { + path: `${TEST_DATA_DIRECTORY}/file_b.txt`, + name: 'file_b.txt', + size: 3756, + extension: '.txt', + type: 'file', + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir`, + name: 'some_dir', + children: [ + { + path: `${TEST_DATA_DIRECTORY}/some_dir/file_a.txt`, + name: 'file_a.txt', + size: 12, + extension: '.txt', + type: 'file', + }, + { + path: `${TEST_DATA_DIRECTORY}/some_dir/file_b.txt`, + name: 'file_b.txt', + size: 3756, + extension: '.txt', + type: 'file', + }, + ], + type: 'directory', + size: 3768, + }, + ], + size: 7536, + type: 'directory', +}; + +export default tree; diff --git a/server/src/tests/helpers/directory-tree/test_data/file_a.txt b/server/src/tests/helpers/directory-tree/test_data/file_a.txt new file mode 100644 index 00000000..802992c4 --- /dev/null +++ b/server/src/tests/helpers/directory-tree/test_data/file_a.txt @@ -0,0 +1 @@ +Hello world diff --git a/server/src/tests/helpers/directory-tree/test_data/file_b.txt b/server/src/tests/helpers/directory-tree/test_data/file_b.txt new file mode 100644 index 00000000..30386d20 --- /dev/null +++ b/server/src/tests/helpers/directory-tree/test_data/file_b.txt @@ -0,0 +1,9 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam ultrices ligula at metus iaculis accumsan. Donec at aliquam justo. Phasellus eu orci gravida, rutrum eros eget, ultrices lacus. Mauris lacinia tortor eu vestibulum vestibulum. Sed eleifend nec ex vel egestas. Donec viverra nibh eu neque gravida, in tincidunt nisi ultricies. Praesent dictum molestie magna mollis accumsan. Proin sed egestas quam. Mauris luctus ante ipsum, non porta ex ultrices eu. Quisque gravida lectus ut diam porta, ut tempus magna fermentum. In hac habitasse platea dictumst. Phasellus sollicitudin tempor feugiat. Nullam imperdiet cursus arcu, sit amet interdum enim feugiat eget. Ut elit metus, semper in turpis luctus, aliquam pharetra libero. Morbi eget lectus vel enim suscipit tempus eget et ipsum. + +Curabitur imperdiet tortor turpis. Ut eu faucibus sem, ut suscipit tellus. Quisque vitae nunc et felis pretium pellentesque sed in mi. Fusce eu eros nulla. Integer sagittis a ligula non vehicula. Pellentesque consequat lacinia justo ac tempor. Cras accumsan nibh dictum eros blandit, nec tempor ante egestas. Proin vitae augue vitae lorem eleifend faucibus cursus a eros. Maecenas quam velit, tincidunt nec quam quis, cursus finibus eros. Suspendisse dignissim tempus tortor, vel imperdiet mi gravida sed. Nam placerat sapien vel quam efficitur mattis. Mauris vehicula dolor id lacus viverra, eu vehicula arcu suscipit. + +Nunc finibus, erat at hendrerit scelerisque, massa mi cursus mauris, eget suscipit est neque non elit. Suspendisse potenti. Quisque a fermentum ligula, eget luctus felis. Aliquam et lorem risus. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla scelerisque lectus ut lorem molestie accumsan. Etiam pulvinar volutpat augue id maximus. Curabitur dapibus augue magna, in ullamcorper urna lobortis id. Curabitur molestie accumsan sapien quis vestibulum. Sed quis nisi condimentum, fringilla nulla sed, aliquet elit. Donec non turpis odio. Donec sodales volutpat lectus, in rutrum erat tempor vel. Nullam auctor erat a turpis dignissim imperdiet. Fusce in tellus nec mi mattis bibendum id nec metus. + +Nullam efficitur, risus eu ornare pretium, sapien dui rhoncus est, non vulputate purus ipsum non sem. Proin a sem id lectus porttitor auctor ut eget sem. Nullam sodales odio enim, at tincidunt libero tempor vulputate. Nulla facilisi. Maecenas semper tincidunt congue. Phasellus dictum nisi nec nunc finibus finibus. Pellentesque lacinia ante pulvinar justo fermentum tristique. Praesent sit amet arcu lacus. Sed in pharetra nisl. Nunc iaculis ipsum id diam rutrum, eu feugiat lectus euismod. Aenean nunc nunc, lacinia in elementum sed, sagittis at nulla. Curabitur ut posuere urna. Nulla augue ex, cursus eu arcu eu, suscipit ultrices risus. + +Phasellus varius tincidunt est, accumsan hendrerit justo feugiat non. Proin hendrerit, nibh lobortis auctor suscipit, felis nibh malesuada erat, at venenatis ex risus id enim. Nulla rutrum velit ut rhoncus molestie. Quisque ac accumsan risus. Sed non nisi non libero volutpat lobortis. Aliquam viverra felis non lacus efficitur rutrum. Ut sagittis metus dolor, non efficitur turpis porta eget. Praesent ullamcorper, lacus congue suscipit accumsan, leo magna rhoncus nisl, vitae rhoncus dolor odio in odio. Nam neque odio, auctor et lacinia id, posuere sed enim. Etiam sit amet purus viverra, ultrices nisl sit amet, porttitor leo. Ut imperdiet congue pretium. Cras quis neque et lorem scelerisque malesuada. Maecenas et vestibulum erat. Cras faucibus tristique purus at dapibus. Phasellus auctor justo ante, vel feugiat arcu lobortis eu. Sed arcu diam, tincidunt vel leo ac, iaculis vehicula lacus. \ No newline at end of file diff --git a/server/src/tests/helpers/directory-tree/test_data/some_dir/another_dir/file_a.txt b/server/src/tests/helpers/directory-tree/test_data/some_dir/another_dir/file_a.txt new file mode 100644 index 00000000..802992c4 --- /dev/null +++ b/server/src/tests/helpers/directory-tree/test_data/some_dir/another_dir/file_a.txt @@ -0,0 +1 @@ +Hello world diff --git a/server/src/tests/helpers/directory-tree/test_data/some_dir/another_dir/file_b.txt b/server/src/tests/helpers/directory-tree/test_data/some_dir/another_dir/file_b.txt new file mode 100644 index 00000000..30386d20 --- /dev/null +++ b/server/src/tests/helpers/directory-tree/test_data/some_dir/another_dir/file_b.txt @@ -0,0 +1,9 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam ultrices ligula at metus iaculis accumsan. Donec at aliquam justo. Phasellus eu orci gravida, rutrum eros eget, ultrices lacus. Mauris lacinia tortor eu vestibulum vestibulum. Sed eleifend nec ex vel egestas. Donec viverra nibh eu neque gravida, in tincidunt nisi ultricies. Praesent dictum molestie magna mollis accumsan. Proin sed egestas quam. Mauris luctus ante ipsum, non porta ex ultrices eu. Quisque gravida lectus ut diam porta, ut tempus magna fermentum. In hac habitasse platea dictumst. Phasellus sollicitudin tempor feugiat. Nullam imperdiet cursus arcu, sit amet interdum enim feugiat eget. Ut elit metus, semper in turpis luctus, aliquam pharetra libero. Morbi eget lectus vel enim suscipit tempus eget et ipsum. + +Curabitur imperdiet tortor turpis. Ut eu faucibus sem, ut suscipit tellus. Quisque vitae nunc et felis pretium pellentesque sed in mi. Fusce eu eros nulla. Integer sagittis a ligula non vehicula. Pellentesque consequat lacinia justo ac tempor. Cras accumsan nibh dictum eros blandit, nec tempor ante egestas. Proin vitae augue vitae lorem eleifend faucibus cursus a eros. Maecenas quam velit, tincidunt nec quam quis, cursus finibus eros. Suspendisse dignissim tempus tortor, vel imperdiet mi gravida sed. Nam placerat sapien vel quam efficitur mattis. Mauris vehicula dolor id lacus viverra, eu vehicula arcu suscipit. + +Nunc finibus, erat at hendrerit scelerisque, massa mi cursus mauris, eget suscipit est neque non elit. Suspendisse potenti. Quisque a fermentum ligula, eget luctus felis. Aliquam et lorem risus. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla scelerisque lectus ut lorem molestie accumsan. Etiam pulvinar volutpat augue id maximus. Curabitur dapibus augue magna, in ullamcorper urna lobortis id. Curabitur molestie accumsan sapien quis vestibulum. Sed quis nisi condimentum, fringilla nulla sed, aliquet elit. Donec non turpis odio. Donec sodales volutpat lectus, in rutrum erat tempor vel. Nullam auctor erat a turpis dignissim imperdiet. Fusce in tellus nec mi mattis bibendum id nec metus. + +Nullam efficitur, risus eu ornare pretium, sapien dui rhoncus est, non vulputate purus ipsum non sem. Proin a sem id lectus porttitor auctor ut eget sem. Nullam sodales odio enim, at tincidunt libero tempor vulputate. Nulla facilisi. Maecenas semper tincidunt congue. Phasellus dictum nisi nec nunc finibus finibus. Pellentesque lacinia ante pulvinar justo fermentum tristique. Praesent sit amet arcu lacus. Sed in pharetra nisl. Nunc iaculis ipsum id diam rutrum, eu feugiat lectus euismod. Aenean nunc nunc, lacinia in elementum sed, sagittis at nulla. Curabitur ut posuere urna. Nulla augue ex, cursus eu arcu eu, suscipit ultrices risus. + +Phasellus varius tincidunt est, accumsan hendrerit justo feugiat non. Proin hendrerit, nibh lobortis auctor suscipit, felis nibh malesuada erat, at venenatis ex risus id enim. Nulla rutrum velit ut rhoncus molestie. Quisque ac accumsan risus. Sed non nisi non libero volutpat lobortis. Aliquam viverra felis non lacus efficitur rutrum. Ut sagittis metus dolor, non efficitur turpis porta eget. Praesent ullamcorper, lacus congue suscipit accumsan, leo magna rhoncus nisl, vitae rhoncus dolor odio in odio. Nam neque odio, auctor et lacinia id, posuere sed enim. Etiam sit amet purus viverra, ultrices nisl sit amet, porttitor leo. Ut imperdiet congue pretium. Cras quis neque et lorem scelerisque malesuada. Maecenas et vestibulum erat. Cras faucibus tristique purus at dapibus. Phasellus auctor justo ante, vel feugiat arcu lobortis eu. Sed arcu diam, tincidunt vel leo ac, iaculis vehicula lacus. \ No newline at end of file diff --git a/server/src/tests/helpers/directory-tree/test_data/some_dir/file_a.txt b/server/src/tests/helpers/directory-tree/test_data/some_dir/file_a.txt new file mode 100644 index 00000000..802992c4 --- /dev/null +++ b/server/src/tests/helpers/directory-tree/test_data/some_dir/file_a.txt @@ -0,0 +1 @@ +Hello world diff --git a/server/src/tests/helpers/directory-tree/test_data/some_dir/file_b.txt b/server/src/tests/helpers/directory-tree/test_data/some_dir/file_b.txt new file mode 100644 index 00000000..30386d20 --- /dev/null +++ b/server/src/tests/helpers/directory-tree/test_data/some_dir/file_b.txt @@ -0,0 +1,9 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam ultrices ligula at metus iaculis accumsan. Donec at aliquam justo. Phasellus eu orci gravida, rutrum eros eget, ultrices lacus. Mauris lacinia tortor eu vestibulum vestibulum. Sed eleifend nec ex vel egestas. Donec viverra nibh eu neque gravida, in tincidunt nisi ultricies. Praesent dictum molestie magna mollis accumsan. Proin sed egestas quam. Mauris luctus ante ipsum, non porta ex ultrices eu. Quisque gravida lectus ut diam porta, ut tempus magna fermentum. In hac habitasse platea dictumst. Phasellus sollicitudin tempor feugiat. Nullam imperdiet cursus arcu, sit amet interdum enim feugiat eget. Ut elit metus, semper in turpis luctus, aliquam pharetra libero. Morbi eget lectus vel enim suscipit tempus eget et ipsum. + +Curabitur imperdiet tortor turpis. Ut eu faucibus sem, ut suscipit tellus. Quisque vitae nunc et felis pretium pellentesque sed in mi. Fusce eu eros nulla. Integer sagittis a ligula non vehicula. Pellentesque consequat lacinia justo ac tempor. Cras accumsan nibh dictum eros blandit, nec tempor ante egestas. Proin vitae augue vitae lorem eleifend faucibus cursus a eros. Maecenas quam velit, tincidunt nec quam quis, cursus finibus eros. Suspendisse dignissim tempus tortor, vel imperdiet mi gravida sed. Nam placerat sapien vel quam efficitur mattis. Mauris vehicula dolor id lacus viverra, eu vehicula arcu suscipit. + +Nunc finibus, erat at hendrerit scelerisque, massa mi cursus mauris, eget suscipit est neque non elit. Suspendisse potenti. Quisque a fermentum ligula, eget luctus felis. Aliquam et lorem risus. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla scelerisque lectus ut lorem molestie accumsan. Etiam pulvinar volutpat augue id maximus. Curabitur dapibus augue magna, in ullamcorper urna lobortis id. Curabitur molestie accumsan sapien quis vestibulum. Sed quis nisi condimentum, fringilla nulla sed, aliquet elit. Donec non turpis odio. Donec sodales volutpat lectus, in rutrum erat tempor vel. Nullam auctor erat a turpis dignissim imperdiet. Fusce in tellus nec mi mattis bibendum id nec metus. + +Nullam efficitur, risus eu ornare pretium, sapien dui rhoncus est, non vulputate purus ipsum non sem. Proin a sem id lectus porttitor auctor ut eget sem. Nullam sodales odio enim, at tincidunt libero tempor vulputate. Nulla facilisi. Maecenas semper tincidunt congue. Phasellus dictum nisi nec nunc finibus finibus. Pellentesque lacinia ante pulvinar justo fermentum tristique. Praesent sit amet arcu lacus. Sed in pharetra nisl. Nunc iaculis ipsum id diam rutrum, eu feugiat lectus euismod. Aenean nunc nunc, lacinia in elementum sed, sagittis at nulla. Curabitur ut posuere urna. Nulla augue ex, cursus eu arcu eu, suscipit ultrices risus. + +Phasellus varius tincidunt est, accumsan hendrerit justo feugiat non. Proin hendrerit, nibh lobortis auctor suscipit, felis nibh malesuada erat, at venenatis ex risus id enim. Nulla rutrum velit ut rhoncus molestie. Quisque ac accumsan risus. Sed non nisi non libero volutpat lobortis. Aliquam viverra felis non lacus efficitur rutrum. Ut sagittis metus dolor, non efficitur turpis porta eget. Praesent ullamcorper, lacus congue suscipit accumsan, leo magna rhoncus nisl, vitae rhoncus dolor odio in odio. Nam neque odio, auctor et lacinia id, posuere sed enim. Etiam sit amet purus viverra, ultrices nisl sit amet, porttitor leo. Ut imperdiet congue pretium. Cras quis neque et lorem scelerisque malesuada. Maecenas et vestibulum erat. Cras faucibus tristique purus at dapibus. Phasellus auctor justo ante, vel feugiat arcu lobortis eu. Sed arcu diam, tincidunt vel leo ac, iaculis vehicula lacus. \ No newline at end of file diff --git a/server/src/tests/helpers/directory-tree/test_data/some_dir_2/.gitkeep b/server/src/tests/helpers/directory-tree/test_data/some_dir_2/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/server/src/tests/helpers/utils.test.ts b/server/src/tests/helpers/utils.test.ts index e06faa4a..c260057c 100644 --- a/server/src/tests/helpers/utils.test.ts +++ b/server/src/tests/helpers/utils.test.ts @@ -52,6 +52,7 @@ describe('findIpAddress()', () => { id: 42, }, }); + // @ts-expect-error for test mockExpressRequest.ip = expectedIp; const resultIp = findIpAddress(mockExpressRequest); diff --git a/server/src/tests/integrations/ansible/AnsibleCmd.test.ts b/server/src/tests/integrations/ansible/AnsibleCmd.test.ts index c8ce4c2f..66aeeae0 100644 --- a/server/src/tests/integrations/ansible/AnsibleCmd.test.ts +++ b/server/src/tests/integrations/ansible/AnsibleCmd.test.ts @@ -1,7 +1,7 @@ import { API } from 'ssm-shared-lib'; import { describe, expect, test } from 'vitest'; import User from '../../../data/database/model/User'; -import type { Ansible } from '../../../types/typings'; +import type { Playbooks } from '../../../types/typings'; import AnsibleCmd from '../../../integrations/ansible/AnsibleCmd'; // note: replace with actual file path describe('helper functions', () => { @@ -14,7 +14,7 @@ describe('helper functions', () => { hosts: ['host1\\example', 'host2\\example'], }, }, - } as Ansible.All & Ansible.HostGroups; + } as Playbooks.All & Playbooks.HostGroups; const result = AnsibleCmd.sanitizeInventory(inventory); const expectedResult = '\'{"all":{"example_group":{"hosts":["host1\\example","host2\\example"]}}}\''; @@ -22,7 +22,7 @@ describe('helper functions', () => { }); test('sanitizeInventory handles empty inventory', () => { - const inventory = {} as Ansible.All & Ansible.HostGroups; + const inventory = {} as Playbooks.All & Playbooks.HostGroups; const result = AnsibleCmd.sanitizeInventory(inventory); const expectedResult = "'{}'"; expect(result).toEqual(expectedResult); @@ -33,13 +33,13 @@ describe('helper functions', () => { // @ts-expect-error partial type const inventory = { all: { hosts: ['testhost1', 'testhost2'] }, - } as Ansible.All & Ansible.HostGroups; + } as Playbooks.All & Playbooks.HostGroups; const result = AnsibleCmd.getInventoryTargets(inventory); expect(result).toEqual(`--specific-host ${AnsibleCmd.sanitizeInventory(inventory)}`); }); test('getInventoryTargets handles empty targets', () => { - const inventory = {} as Ansible.All & Ansible.HostGroups; + const inventory = {} as Playbooks.All & Playbooks.HostGroups; const result = AnsibleCmd.getInventoryTargets(inventory); expect(result).toEqual("--specific-host '{}'"); }); @@ -84,7 +84,7 @@ describe('buildAnsibleCmd() function', () => { hosts: ['host1', 'host2'], }, }, - } as Ansible.All & Ansible.HostGroups; + } as Playbooks.All & Playbooks.HostGroups; const extraVars: API.ExtraVars = [ { diff --git a/server/src/types/typings.d.ts b/server/src/types/typings.d.ts index 32834f1e..ebadfb17 100644 --- a/server/src/types/typings.d.ts +++ b/server/src/types/typings.d.ts @@ -1,7 +1,7 @@ import { AxiosRequestConfig } from 'axios'; import Container from '../data/database/model/Container'; -export declare namespace Ansible { +export declare namespace Playbooks { type HostVar = { ip: string[]; }; @@ -42,6 +42,7 @@ export declare namespace Ansible { type PlaybookConfigurationFile = { playableInBatch: boolean; extraVars?: [{ extraVar: string; required: boolean }]; + uniqueQuickRef?: string; }; } diff --git a/server/src/use-cases/DeviceUseCases.ts b/server/src/use-cases/DeviceUseCases.ts index 84ba7885..9eb39af9 100644 --- a/server/src/use-cases/DeviceUseCases.ts +++ b/server/src/use-cases/DeviceUseCases.ts @@ -1,6 +1,6 @@ import DockerModem from 'docker-modem'; import Dockerode from 'dockerode'; -import { API, SsmAnsible, SsmStatus, SettingsKeys } from 'ssm-shared-lib'; +import { API, SettingsKeys, SsmAnsible, SsmStatus } from 'ssm-shared-lib'; import { InternalError } from '../core/api/ApiError'; import { setToCache } from '../data/cache'; import Device, { DeviceModel } from '../data/database/model/Device'; @@ -161,7 +161,7 @@ async function checkAnsibleConnection( sshKeyPass: sshKeyPass ? await vaultEncrypt(sshKeyPass, DEFAULT_VAULT_ID) : undefined, }, ]); - const playbook = await PlaybookRepo.findOne('_checkDeviceBeforeAdd.yml'); + const playbook = await PlaybookRepo.findOneByUniqueQuickReference('checkDeviceBeforeAdd'); if (!playbook) { throw new InternalError('_checkDeviceBeforeAdd.yml not found.'); } @@ -251,7 +251,7 @@ async function checkDeviceDockerConnection(device: Device, deviceAuth: DeviceAut } async function checkDeviceAnsibleConnection(user: User, device: Device) { - const playbook = await PlaybookRepo.findOne('_checkDeviceBeforeAdd.yml'); + const playbook = await PlaybookRepo.findOneByUniqueQuickReference('checkDeviceBeforeAdd'); if (!playbook) { throw new InternalError('_checkDeviceBeforeAdd.yml not found.'); } diff --git a/server/src/use-cases/GitRepositoryUseCases.ts b/server/src/use-cases/GitRepositoryUseCases.ts new file mode 100644 index 00000000..e22dca50 --- /dev/null +++ b/server/src/use-cases/GitRepositoryUseCases.ts @@ -0,0 +1,84 @@ +import { Error } from 'mongoose'; +import { v4 as uuidv4 } from 'uuid'; +import { Playbooks } from 'ssm-shared-lib'; +import PlaybooksRepository from '../data/database/model/PlaybooksRepository'; +import PlaybookRepo from '../data/database/repository/PlaybookRepo'; +import PlaybooksRepositoryRepo from '../data/database/repository/PlaybooksRepositoryRepo'; +import PlaybooksRepositoryEngine from '../integrations/playbooks-repository/PlaybooksRepositoryEngine'; +import playbooksRepository from '../routes/playbooks-repository'; + +async function addGitRepository( + name: string, + accessToken: string, + branch: string, + email: string, + userName: string, + remoteUrl: string, +) { + const uuid = uuidv4(); + const gitRepository = await PlaybooksRepositoryEngine.registerRepository({ + uuid, + type: Playbooks.PlaybooksRepositoryType.GIT, + name, + branch, + email, + userName, + accessToken, + remoteUrl, + enabled: true, + }); + await PlaybooksRepositoryRepo.create({ + uuid, + type: Playbooks.PlaybooksRepositoryType.GIT, + name, + remoteUrl, + accessToken, + branch, + email, + userName, + directory: gitRepository.getDirectory(), + enabled: true, + }); + void gitRepository.clone(); + void gitRepository.syncToDatabase(); +} + +async function updateGitRepository( + uuid: string, + name: string, + accessToken: string, + branch: string, + email: string, + userName: string, + remoteUrl: string, +) { + await PlaybooksRepositoryEngine.deregisterRepository(uuid); + const gitRepository = await PlaybooksRepositoryEngine.registerRepository({ + uuid, + type: Playbooks.PlaybooksRepositoryType.GIT, + name, + branch, + email, + userName, + accessToken, + remoteUrl, + enabled: true, + }); + await PlaybooksRepositoryRepo.update({ + uuid, + type: Playbooks.PlaybooksRepositoryType.GIT, + name, + remoteUrl, + accessToken, + branch, + email, + userName, + directory: gitRepository.getDirectory(), + enabled: true, + }); +} + +export default { + addGitRepository, + updateGitRepository, +}; diff --git a/server/src/use-cases/LocalRepositoryUseCases.ts b/server/src/use-cases/LocalRepositoryUseCases.ts new file mode 100644 index 00000000..e95f6b95 --- /dev/null +++ b/server/src/use-cases/LocalRepositoryUseCases.ts @@ -0,0 +1,49 @@ +import { v4 as uuidv4 } from 'uuid'; +import { Playbooks } from 'ssm-shared-lib'; +import { NotFoundError } from '../core/api/ApiError'; +import PlaybooksRepositoryRepo from '../data/database/repository/PlaybooksRepositoryRepo'; +import { DIRECTORY_ROOT } from '../integrations/playbooks-repository/PlaybooksRepositoryComponent'; +import PlaybooksRepositoryEngine from '../integrations/playbooks-repository/PlaybooksRepositoryEngine'; +import { createDirectoryWithFullPath } from '../integrations/shell/utils'; +import logger from '../logger'; + +async function addLocalRepository(name: string) { + const uuid = uuidv4(); + const localRepository = await PlaybooksRepositoryEngine.registerRepository({ + uuid, + type: Playbooks.PlaybooksRepositoryType.LOCAL, + name, + enabled: true, + directory: DIRECTORY_ROOT, + }); + logger.info(localRepository.uuid); + await PlaybooksRepositoryRepo.create({ + uuid, + type: Playbooks.PlaybooksRepositoryType.LOCAL, + name, + directory: DIRECTORY_ROOT, + enabled: true, + }); + try { + await createDirectoryWithFullPath(localRepository.getDirectory()); + void localRepository.syncToDatabase(); + } catch (error: any) { + logger.warn(error); + } +} + +async function updateLocalRepository(uuid: string, name: string) { + const playbooksRepository = await PlaybooksRepositoryRepo.findByUuid(uuid); + if (!playbooksRepository) { + throw new NotFoundError(); + } + await PlaybooksRepositoryEngine.deregisterRepository(uuid); + playbooksRepository.name = name; + await PlaybooksRepositoryEngine.registerRepository(playbooksRepository); + await PlaybooksRepositoryRepo.update(playbooksRepository); +} + +export default { + updateLocalRepository, + addLocalRepository, +}; diff --git a/server/src/use-cases/PlaybookUseCases.ts b/server/src/use-cases/PlaybookUseCases.ts index d32479f4..d1772607 100644 --- a/server/src/use-cases/PlaybookUseCases.ts +++ b/server/src/use-cases/PlaybookUseCases.ts @@ -2,11 +2,9 @@ import { API } from 'ssm-shared-lib'; import { setToCache } from '../data/cache'; import Playbook, { PlaybookModel } from '../data/database/model/Playbook'; import User from '../data/database/model/User'; -import PlaybookRepo from '../data/database/repository/PlaybookRepo'; import ExtraVars from '../integrations/ansible/utils/ExtraVars'; import shell from '../integrations/shell'; -import logger from '../logger'; -import { Ansible } from '../types/typings'; +import { Playbooks } from '../types/typings'; async function completeExtraVar( playbook: Playbook, @@ -30,102 +28,33 @@ async function executePlaybook( target: string[] | undefined, extraVarsForcedValues?: API.ExtraVars, ) { - let substitutedExtraVars: API.ExtraVars | undefined = await completeExtraVar( + const substitutedExtraVars: API.ExtraVars | undefined = await completeExtraVar( playbook, target, extraVarsForcedValues, ); - return await shell.executePlaybook(playbook.name, user, target, substitutedExtraVars); + return await shell.executePlaybook(playbook.path, user, target, substitutedExtraVars); } async function executePlaybookOnInventory( playbook: Playbook, user: User, - inventoryTargets?: Ansible.All & Ansible.HostGroups, + inventoryTargets?: Playbooks.All & Playbooks.HostGroups, extraVarsForcedValues?: API.ExtraVars, ) { - let substitutedExtraVars: API.ExtraVars | undefined = await completeExtraVar( + const substitutedExtraVars: API.ExtraVars | undefined = await completeExtraVar( playbook, undefined, extraVarsForcedValues, ); return await shell.executePlaybookOnInventory( - playbook.name, + playbook.path, user, inventoryTargets, substitutedExtraVars, ); } -async function initPlaybook() { - logger.info(`[USECASES][PLAYBOOK] - initPlaybook`); - - const playbooks = await shell.listPlaybooks(); - const playbookPromises = playbooks?.map(async (playbook) => { - const configurationFileContent = await shell.readPlaybookConfiguration( - playbook.replaceAll('.yml', '.json'), - ); - - const isCustomPlaybook = !playbook.startsWith('_'); - const playbookData: Playbook = { - name: playbook, - custom: isCustomPlaybook, - }; - - if (configurationFileContent) { - logger.info(`[USECASES][PLAYBOOK] - playbook has configuration file`); - - const playbookConfiguration = JSON.parse( - configurationFileContent, - ) as Ansible.PlaybookConfigurationFile; - - playbookData.playableInBatch = playbookConfiguration.playableInBatch; - playbookData.extraVars = playbookConfiguration.extraVars; - } - - await PlaybookRepo.updateOrCreate(playbookData); - }); - - await Promise.all(playbookPromises); -} - -async function getAllPlaybooks() { - const listOfPlaybooks = await PlaybookRepo.findAll(); - if (!listOfPlaybooks) { - return []; - } - - const substitutedListOfPlaybooks = listOfPlaybooks.map(async (playbook) => { - const extraVars = playbook.extraVars - ? await ExtraVars.findValueOfExtraVars(playbook.extraVars, undefined, true) - : undefined; - return { - value: playbook.name, - label: playbook.name.replaceAll('.yml', ''), - extraVars, - custom: playbook.custom, - }; - }); - - return (await Promise.all(substitutedListOfPlaybooks)).sort((a) => - a.value.startsWith('_') ? -1 : 1, - ); -} - -async function createCustomPlaybook(name: string) { - if (!name.endsWith('.yml')) { - name += '.yml'; - } - await PlaybookModel.create({ name: name, custom: true }).then(async () => { - await shell.newPlaybook(name); - }); -} - -async function deleteCustomPlaybook(playbook: Playbook) { - await PlaybookModel.deleteOne({ name: playbook.name }); - await shell.deletePlaybook(playbook.name); -} - async function addExtraVarToPlaybook(playbook: Playbook, extraVar: API.ExtraVar) { playbook.extraVars?.forEach((e) => { if (e.extraVar === extraVar.extraVar) { @@ -152,10 +81,6 @@ async function deleteExtraVarFromPlaybook(playbook: Playbook, extraVarName: stri } export default { - initPlaybook, - getAllPlaybooks, - createCustomPlaybook, - deleteCustomPlaybook, executePlaybook, executePlaybookOnInventory, addExtraVarToPlaybook, diff --git a/server/src/use-cases/PlaybooksRepositoryUseCases.ts b/server/src/use-cases/PlaybooksRepositoryUseCases.ts new file mode 100644 index 00000000..445faee7 --- /dev/null +++ b/server/src/use-cases/PlaybooksRepositoryUseCases.ts @@ -0,0 +1,137 @@ +import { API } from 'ssm-shared-lib'; +import { ForbiddenError, InternalError } from '../core/api/ApiError'; +import Playbook, { PlaybookModel } from '../data/database/model/Playbook'; +import PlaybooksRepository from '../data/database/model/PlaybooksRepository'; +import PlaybooksRepositoryRepo from '../data/database/repository/PlaybooksRepositoryRepo'; +import PlaybooksRepositoryComponent from '../integrations/playbooks-repository/PlaybooksRepositoryComponent'; +import PlaybooksRepositoryEngine from '../integrations/playbooks-repository/PlaybooksRepositoryEngine'; +import { recursiveTreeCompletion } from '../integrations/playbooks-repository/utils'; +import shell from '../integrations/shell'; +import { createDirectoryWithFullPath, deleteFilesAndDirectory } from '../integrations/shell/utils'; +import logger from '../logger'; + +async function getAllPlaybooksRepositories() { + try { + const listOfPlaybooksRepositories = await PlaybooksRepositoryRepo.findAllActive(); + if (!listOfPlaybooksRepositories) { + return []; + } + + const substitutedListOfPlaybooks = listOfPlaybooksRepositories.map( + async (playbookRepository) => { + logger.info( + `[PLAYBOOKS_REPOSITORY_USECASES] - getAllPlaybooksRepositories - processing ${playbookRepository.name}`, + ); + return { + name: playbookRepository.name, + children: await recursiveTreeCompletion(playbookRepository.tree), + type: playbookRepository.type, + uuid: playbookRepository.uuid, + path: playbookRepository.directory + '/' + playbookRepository.uuid, + default: playbookRepository.default, + }; + }, + ); + + return (await Promise.all(substitutedListOfPlaybooks)).sort((a, b) => + b.default ? 1 : a.default ? 1 : a.name.localeCompare(b.name), + ) as API.PlaybooksRepository[]; + } catch (error: any) { + logger.error(error); + logger.error(`[PLAYBOOKS_REPOSITORY_USECASES] - Error during processing`); + } +} + +async function createDirectoryInPlaybookRepository( + playbooksRepository: PlaybooksRepository, + path: string, +) { + const playbooksRepositoryComponent = PlaybooksRepositoryEngine.getState().playbooksRepository[ + playbooksRepository.uuid + ] as PlaybooksRepositoryComponent; + if (!playbooksRepositoryComponent) { + throw new InternalError('Repository is not registered, try restarting or force sync'); + } + if (!playbooksRepositoryComponent.fileBelongToRepository(path)) { + throw new ForbiddenError('The selected path doesnt seems to belong to the repository'); + } + await createDirectoryWithFullPath(path); + await playbooksRepositoryComponent.updateDirectoriesTree(); +} + +async function createPlaybookInRepository( + playbooksRepository: PlaybooksRepository, + fullPath: string, + name: string, +) { + const playbooksRepositoryComponent = PlaybooksRepositoryEngine.getState().playbooksRepository[ + playbooksRepository.uuid + ] as PlaybooksRepositoryComponent; + if (!playbooksRepositoryComponent) { + throw new InternalError(`PlaybookRepository doesn't seem registered`); + } + if (!playbooksRepositoryComponent.fileBelongToRepository(fullPath)) { + throw new ForbiddenError('The selected path doesnt seems to belong to the repository'); + } + const playbook = await PlaybookModel.create({ + name: name, + custom: true, + path: fullPath + '.yml', + playbooksRepository: playbooksRepository, + playableInBatch: true, + }); + await shell.newPlaybook(fullPath); + await playbooksRepositoryComponent.syncToDatabase(); + return playbook; +} + +async function deletePlaybooksInRepository(playbook: Playbook) { + const playbooksRepositoryComponent = PlaybooksRepositoryEngine.getState().playbooksRepository[ + (playbook.playbooksRepository as PlaybooksRepository).uuid + ] as PlaybooksRepositoryComponent; + if (!playbooksRepositoryComponent) { + throw new InternalError(`PlaybookRepository doesnt seem registered`); + } + await PlaybookModel.deleteOne({ uuid: playbook.uuid }); + await shell.deletePlaybook(playbook.path); + await playbooksRepositoryComponent.syncToDatabase(); +} + +async function deleteAnyInPlaybooksRepository( + playbooksRepository: PlaybooksRepository, + fullPath: string, +) { + const playbooksRepositoryComponent = PlaybooksRepositoryEngine.getState().playbooksRepository[ + playbooksRepository.uuid + ] as PlaybooksRepositoryComponent; + if (!playbooksRepositoryComponent) { + throw new InternalError(`PlaybookRepository doesnt seem registered`); + } + if (!playbooksRepositoryComponent.fileBelongToRepository(fullPath)) { + throw new ForbiddenError('The selected path doesnt seems to belong to the repository'); + } + await deleteFilesAndDirectory(fullPath); + await playbooksRepositoryComponent.syncToDatabase(); +} + +async function deleteRepository(repository: PlaybooksRepository): Promise { + const playbooksRepositoryComponent = PlaybooksRepositoryEngine.getState().playbooksRepository[ + repository.uuid + ] as PlaybooksRepositoryComponent; + if (!playbooksRepositoryComponent) { + throw new InternalError(`PlaybookRepository doesnt seem registered`); + } + const directory = playbooksRepositoryComponent.getDirectory(); + await PlaybooksRepositoryEngine.deregisterRepository(repository.uuid); + await PlaybooksRepositoryRepo.deleteByUuid(repository.uuid); + await deleteFilesAndDirectory(directory); +} + +export default { + getAllPlaybooksRepositories, + createDirectoryInPlaybookRepository, + createPlaybookInRepository, + deletePlaybooksInRepository, + deleteAnyInPlaybooksRepository, + deleteRepository, +}; diff --git a/shared-lib/package.json b/shared-lib/package.json index 60546e09..8b9e148c 100644 --- a/shared-lib/package.json +++ b/shared-lib/package.json @@ -4,6 +4,7 @@ "description": "", "main": "./distribution/index.js", "author": "Squirrel Team", + "license": "AGPL-3.0 license", "scripts": { "build": "npm run clean && tsc -p ./tsconfig.json", "build:check": "tsc -p ./tsconfig.json --noEmit", diff --git a/shared-lib/src/enums/ansible.ts b/shared-lib/src/enums/ansible.ts index 4a63f233..9f269b43 100644 --- a/shared-lib/src/enums/ansible.ts +++ b/shared-lib/src/enums/ansible.ts @@ -19,3 +19,4 @@ export enum AnsibleBecomeMethod { RUNAS = 'runas', MACHINECTL = 'machinectl', } + diff --git a/shared-lib/src/enums/playbooks.ts b/shared-lib/src/enums/playbooks.ts new file mode 100644 index 00000000..e24a5a11 --- /dev/null +++ b/shared-lib/src/enums/playbooks.ts @@ -0,0 +1,4 @@ +export enum PlaybooksRepositoryType { + LOCAL = 'local', + GIT = 'git', +} diff --git a/shared-lib/src/enums/settings.ts b/shared-lib/src/enums/settings.ts index 8756e883..cf4e7c84 100644 --- a/shared-lib/src/enums/settings.ts +++ b/shared-lib/src/enums/settings.ts @@ -15,7 +15,7 @@ export enum AnsibleReservedExtraVarsKeys { } export enum DefaultValue { - SCHEME_VERSION = '4', + SCHEME_VERSION = '6', SERVER_LOG_RETENTION_IN_DAYS = '30', CONSIDER_DEVICE_OFFLINE_AFTER_IN_MINUTES = '3', CONSIDER_PERFORMANCE_GOOD_MEM_IF_GREATER = '10', diff --git a/shared-lib/src/index.ts b/shared-lib/src/index.ts index 92e46ab7..cc485ad4 100644 --- a/shared-lib/src/index.ts +++ b/shared-lib/src/index.ts @@ -5,3 +5,5 @@ export * as SsmStatus from './enums/status'; export * as SsmAnsible from './enums/ansible'; export * as Validation from './validation/index'; export * as StatsType from './enums/stats'; +export * as DirectoryTree from './types/tree' +export * as Playbooks from './enums/playbooks' diff --git a/shared-lib/src/types/api.ts b/shared-lib/src/types/api.ts index 49e72582..b9a8ed57 100644 --- a/shared-lib/src/types/api.ts +++ b/shared-lib/src/types/api.ts @@ -1,4 +1,12 @@ import { SSHType } from '../enums/ansible'; +import { PlaybooksRepositoryType } from '../enums/playbooks'; +import { ExtendedTreeNode } from './tree'; + +export type Response = { + success: boolean; + message: string; + data: T +}; export type HasUsers = { success?: boolean; @@ -326,16 +334,26 @@ export type AveragedDeviceStat = { data?: [{ value: number; name: string }]; success?: boolean; }; -export type PlaybookFileList = { - label: string; - value: string; + +export type PlaybookFile = { + name: string; + uuid: string; + path: string; custom?: boolean; extraVars?: ExtraVars; }; -export type Playbooks = { +export type PlaybooksRepository = { + name: string; + uuid: string; + path: string; + type: PlaybooksRepositoryType; + children?: ExtendedTreeNode[]; +}; + +export type PlaybooksRepositories = { success?: boolean; - data?: PlaybookFileList[]; + data?: PlaybooksRepository[]; }; export type PlaybookContent = { @@ -594,7 +612,7 @@ export type ContainerRegistryResponse = SimpleResult & { data?: ContainerRegistries; } -export type ContainerRegistry ={ +export type ContainerRegistry = { name: string; fullName: string; authScheme: any; @@ -604,4 +622,22 @@ export type ContainerRegistry ={ canAnonymous: boolean; } +export type GitRepository = { + uuid?: string; + name: string; + email: string; + branch: string; + userName: string; + remoteUrl: string; + default: boolean; +} + +export type LocalRepository = { + uuid: string; + name: string; + directory: string; + enabled: boolean; + default: boolean; +} + export type ExtraVars = ExtraVar[]; diff --git a/shared-lib/src/types/tree.ts b/shared-lib/src/types/tree.ts new file mode 100644 index 00000000..b4b78a86 --- /dev/null +++ b/shared-lib/src/types/tree.ts @@ -0,0 +1,20 @@ +export enum CONSTANTS { + DIRECTORY = 'directory', + FILE = 'file', +} + +export type TreeNode = { + path: string; + name: string; + isSymbolicLink?: boolean; + extension?: string; + type?: CONSTANTS.DIRECTORY | CONSTANTS.FILE; + children?: (TreeNode | ExtendedTreeNode | null)[]; + size?: number; +}; + +export type ExtendedTreeNode = TreeNode & { + uuid?: string; + extraVars?: any; + custom?: boolean; +}; diff --git a/site/.vitepress/config.ts b/site/.vitepress/config.ts index e88edf66..689c30bc 100644 --- a/site/.vitepress/config.ts +++ b/site/.vitepress/config.ts @@ -114,7 +114,7 @@ export default defineConfig({ }, { text: 'Technical Guide', link: '/docs/technical-guide.md', items: [ - { text: 'Ansible', link: '/docs/ansible.md' }, + { text: 'Ansible', link: '/docs/playbooks.md' }, { text: 'Docker', link: '/docs/docker.md' }, { text: 'Manually installing the agent', link: '/docs/manual-install-agent.md' diff --git a/site/contribute/index.md b/site/contribute/index.md index 33a7e291..42383bfe 100644 --- a/site/contribute/index.md +++ b/site/contribute/index.md @@ -21,19 +21,19 @@ Firstly, start by creating your own 'fork' of our repository. ```sh # Clone the repository -git clone https://github.com/SquirrelCorporation/SquirrelServersManager.git +playbooks-repository clone https://github.com/SquirrelCorporation/SquirrelServersManager.git # Go to your local repository cd SquirrelCorporation # Ensure you are in the 'master' branch -git checkout master +playbooks-repository checkout master # Pull the latest updates -git pull +playbooks-repository pull # Create a new branch for your changes -git checkout -b +playbooks-repository checkout -b ``` ## Making Changes @@ -51,13 +51,13 @@ After your changes, commit and push them to your GitHub repository. ```sh # Add changes -git add . +playbooks-repository add . # Commit the changes -git commit -m "" +playbooks-repository commit -m "" # Push your changes to your repository -git push origin +playbooks-repository push origin ``` ## Submitting a Pull Request diff --git a/site/contribute/release.md b/site/contribute/release.md index e9c6bdf3..f432349e 100644 --- a/site/contribute/release.md +++ b/site/contribute/release.md @@ -8,7 +8,7 @@ Before initiating a new release, ensure you have pulled the latest changes from ```sh # Pull the latest changes from the remote repository -git pull origin master +playbooks-repository pull origin master # Run your tests to ensure everything works as expected npm test @@ -21,11 +21,11 @@ Once your master branch is up to date and tests are passing, create a new Git ta ```sh # The format is v.. # For example: -git tag v1.0.0 +playbooks-repository tag v1.0.0 ``` After creating the new tag, it needs to be pushed to the GitHub repository. ```sh -git push origin v1.0.0 +playbooks-repository push origin v1.0.0 ``` diff --git a/site/docs/devmode.md b/site/docs/devmode.md index 10960454..2535eb70 100644 --- a/site/docs/devmode.md +++ b/site/docs/devmode.md @@ -9,7 +9,7 @@ If you don't understand React or TypeScript, or have security concerns, don't us ### 1. Clone the main repo ```shell -git clone https://github.com/SquirrelCorporation/SquirrelServersManager +playbooks-repository clone https://github.com/SquirrelCorporation/SquirrelServersManager ``` ### 2. CD to the directory and open with your favorite editor: diff --git a/site/docs/manual-install-agent.md b/site/docs/manual-install-agent.md index fb69ec62..30f94816 100644 --- a/site/docs/manual-install-agent.md +++ b/site/docs/manual-install-agent.md @@ -8,7 +8,7 @@ If you have difficulties installing the agent with UI, you can install it manual ### Method 1: Installing the agent with the provided shell script ```shell -git clone https://github.com/SquirrelCorporation/SquirrelServersManager-Agent +playbooks-repository clone https://github.com/SquirrelCorporation/SquirrelServersManager-Agent cd ./SquirrelServersManager-Agent ``` Replace below MASTER_NODE_URL by the URL to SSM @@ -27,7 +27,7 @@ and replace DEVICE_ID by the uuid of the device in SSM (Inventory, click on the ### Method 2: Building & Installing the agent manually ```shell -git clone https://github.com/SquirrelCorporation/SquirrelServersManager-Agent +playbooks-repository clone https://github.com/SquirrelCorporation/SquirrelServersManager-Agent cd ./SquirrelServersManager-Agent ``` diff --git a/site/docs/quickstart.md b/site/docs/quickstart.md index 9e4dfff4..4e3eba29 100644 --- a/site/docs/quickstart.md +++ b/site/docs/quickstart.md @@ -92,7 +92,7 @@ docker compose up ### 1. Clone the main repo ```shell -git clone https://github.com/SquirrelCorporation/SquirrelServersManager +playbooks-repository clone https://github.com/SquirrelCorporation/SquirrelServersManager ``` ### 2. CD to the directory: ```shell @@ -138,7 +138,7 @@ Docker will create a volume directory *.data.prod* in the directory for persiste In SSM cloned directory: ```shell -git pull +playbooks-repository pull ``` ```shell diff --git a/site/index.md b/site/index.md index 83eaf1a4..b6e8789b 100644 --- a/site/index.md +++ b/site/index.md @@ -27,7 +27,7 @@ features: - title: Ansible & Docker Compatible details: Thanks to the power of Ansible and Docker, you can totally manage your servers, services and configuration from SSM icon: - src: /ansible.svg + src: /playbooks.svg - title: Simple to use, yet powerful details: Even is SSM is focused on easiness of use, it flexibility enables you to make powerful and complex setups icon: @@ -80,7 +80,7 @@ SSM is currently in active development and not usable for production yet. We enc   ::: -## Experience the immense power encapsulated within these tools, now exclusitely presented through a user-friendly interface. +## Experience the immense power encapsulated within these tools, now presented through a user-friendly interface. 🔌 We blend the automation powerhouse of Ansible with the portable setup of Docker in a clean and engaging interface. diff --git a/site/package.json b/site/package.json index 7187e7ef..0d9c3009 100644 --- a/site/package.json +++ b/site/package.json @@ -3,6 +3,7 @@ "type": "module", "private": true, "author": "Squirrel Team", + "license": "AGPL-3.0 license", "scripts": { "dev": "vitepress dev", "build": "vitepress build", From c0c82713ad619394bbec26b40b368333cd1b9664 Mon Sep 17 00:00:00 2001 From: manu Date: Sun, 30 Jun 2024 13:55:34 +0200 Subject: [PATCH 02/11] "Rework playbook management and execution system" Refactored playbook management and execution functionalities in the system. Essential changes include renaming "ansible" routes to "playbook" routes, relocating several playbook related files, and modifying ansible related Python scripts. Added functionality to filter results by file extensions in directory tree listing and created a REST service for managing playbook repositories. --- .../{ => 00000000-0000-0000-0000-000000000001}/factScan.yml | 0 server/src/integrations/docker/utils/utils.ts | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename server/src/ansible/{ => 00000000-0000-0000-0000-000000000001}/factScan.yml (100%) diff --git a/server/src/ansible/factScan.yml b/server/src/ansible/00000000-0000-0000-0000-000000000001/factScan.yml similarity index 100% rename from server/src/ansible/factScan.yml rename to server/src/ansible/00000000-0000-0000-0000-000000000001/factScan.yml diff --git a/server/src/integrations/docker/utils/utils.ts b/server/src/integrations/docker/utils/utils.ts index 4d8727e3..7e7a9b62 100644 --- a/server/src/integrations/docker/utils/utils.ts +++ b/server/src/integrations/docker/utils/utils.ts @@ -64,7 +64,7 @@ export function normalizeContainer(container: Container) { logger.info(`[UTILS] - normalizeContainer - for name: ${container.image?.name}`); const registryProvider = Object.values(getRegistries()).find((provider) => provider.match(container.image), - ); + ) as Registry; if (!registryProvider) { logger.warn(`${fullName(container)} - No Registry Provider found`); containerWithNormalizedImage.image.registry.name = 'unknown'; From 46ff56d31cf338e8a1373dadc6b93c8ae9173376 Mon Sep 17 00:00:00 2001 From: manu Date: Sun, 30 Jun 2024 14:02:13 +0200 Subject: [PATCH 03/11] Update clone command in documentation The documentation files previously suggested using the 'playbooks-repository clone' command to copy the repositories. These instructions have been updated to use the more standard 'git clone' command instead, providing more general compatibility. --- site/contribute/index.md | 2 +- site/docs/devmode.md | 2 +- site/docs/manual-install-agent.md | 4 ++-- site/docs/quickstart.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/site/contribute/index.md b/site/contribute/index.md index 42383bfe..23b118b4 100644 --- a/site/contribute/index.md +++ b/site/contribute/index.md @@ -21,7 +21,7 @@ Firstly, start by creating your own 'fork' of our repository. ```sh # Clone the repository -playbooks-repository clone https://github.com/SquirrelCorporation/SquirrelServersManager.git +git clone https://github.com/SquirrelCorporation/SquirrelServersManager.git # Go to your local repository cd SquirrelCorporation diff --git a/site/docs/devmode.md b/site/docs/devmode.md index 2535eb70..10960454 100644 --- a/site/docs/devmode.md +++ b/site/docs/devmode.md @@ -9,7 +9,7 @@ If you don't understand React or TypeScript, or have security concerns, don't us ### 1. Clone the main repo ```shell -playbooks-repository clone https://github.com/SquirrelCorporation/SquirrelServersManager +git clone https://github.com/SquirrelCorporation/SquirrelServersManager ``` ### 2. CD to the directory and open with your favorite editor: diff --git a/site/docs/manual-install-agent.md b/site/docs/manual-install-agent.md index 30f94816..fb69ec62 100644 --- a/site/docs/manual-install-agent.md +++ b/site/docs/manual-install-agent.md @@ -8,7 +8,7 @@ If you have difficulties installing the agent with UI, you can install it manual ### Method 1: Installing the agent with the provided shell script ```shell -playbooks-repository clone https://github.com/SquirrelCorporation/SquirrelServersManager-Agent +git clone https://github.com/SquirrelCorporation/SquirrelServersManager-Agent cd ./SquirrelServersManager-Agent ``` Replace below MASTER_NODE_URL by the URL to SSM @@ -27,7 +27,7 @@ and replace DEVICE_ID by the uuid of the device in SSM (Inventory, click on the ### Method 2: Building & Installing the agent manually ```shell -playbooks-repository clone https://github.com/SquirrelCorporation/SquirrelServersManager-Agent +git clone https://github.com/SquirrelCorporation/SquirrelServersManager-Agent cd ./SquirrelServersManager-Agent ``` diff --git a/site/docs/quickstart.md b/site/docs/quickstart.md index 4e3eba29..6bfe5da7 100644 --- a/site/docs/quickstart.md +++ b/site/docs/quickstart.md @@ -92,7 +92,7 @@ docker compose up ### 1. Clone the main repo ```shell -playbooks-repository clone https://github.com/SquirrelCorporation/SquirrelServersManager +git clone https://github.com/SquirrelCorporation/SquirrelServersManager ``` ### 2. CD to the directory: ```shell From 8f66a8a2c14942da99cae9757f10200326008f5c Mon Sep 17 00:00:00 2001 From: manu Date: Sun, 30 Jun 2024 16:54:59 +0200 Subject: [PATCH 04/11] Refactor server code and create new tests The server integrations and use-cases have been updated for improved efficiency and clarity. The `syncToRepository` method was removed from multiple components for redundancy. The handling of asynchronous calls and data manipulation methods was modified as well. New test files were added to verify tree-utils functionality and the `PlaybookRepositoryComponent`. --- client/src/components/TerminalModal/index.tsx | 2 +- .../git-repository/GitRepositoryComponent.ts | 7 - .../LocalRepositoryComponent.ts | 4 - .../PlaybooksRepositoryComponent.ts | 3 +- .../{utils.ts => tree-utils.ts} | 0 .../PlaybookRepositoryComponent.test.ts | 30 ++ .../playbooks-repository/tree-utils.test.ts | 316 ++++++++++++++++++ .../src/use-cases/LocalRepositoryUseCases.ts | 4 +- .../use-cases/PlaybooksRepositoryUseCases.ts | 2 +- 9 files changed, 350 insertions(+), 18 deletions(-) rename server/src/integrations/playbooks-repository/{utils.ts => tree-utils.ts} (100%) create mode 100644 server/src/tests/integrations/playbooks-repository/PlaybookRepositoryComponent.test.ts create mode 100644 server/src/tests/integrations/playbooks-repository/tree-utils.test.ts diff --git a/client/src/components/TerminalModal/index.tsx b/client/src/components/TerminalModal/index.tsx index d6a7c659..b021c4fd 100644 --- a/client/src/components/TerminalModal/index.tsx +++ b/client/src/components/TerminalModal/index.tsx @@ -111,7 +111,7 @@ const TerminalModal = (props: TerminalModalProps) => { try { const res = !props.terminalProps.quickRef ? await executePlaybook( - props.terminalProps.command, + props.terminalProps.command as string, props.terminalProps.target?.map((e) => e.uuid), props.terminalProps.extraVars, ) diff --git a/server/src/integrations/git-repository/GitRepositoryComponent.ts b/server/src/integrations/git-repository/GitRepositoryComponent.ts index 5edce628..fc4e8b61 100644 --- a/server/src/integrations/git-repository/GitRepositoryComponent.ts +++ b/server/src/integrations/git-repository/GitRepositoryComponent.ts @@ -121,13 +121,6 @@ class GitRepositoryComponent extends PlaybooksRepositoryComponent implements Abs await this.clone(); } - async syncToRepository() { - const files = await findFilesInDirectory(this.directory, FILE_PATTERN); - for (const file of files) { - this.childLogger.info(`syncToDatabase --> ${file}`); - } - } - async syncFromRepository() { await this.forcePull(); } diff --git a/server/src/integrations/local-repository/LocalRepositoryComponent.ts b/server/src/integrations/local-repository/LocalRepositoryComponent.ts index a02b423f..328a9f39 100644 --- a/server/src/integrations/local-repository/LocalRepositoryComponent.ts +++ b/server/src/integrations/local-repository/LocalRepositoryComponent.ts @@ -19,10 +19,6 @@ class LocalRepositoryComponent extends PlaybooksRepositoryComponent implements A async syncFromRepository(): Promise { await this.syncToDatabase(); } - - async syncToRepository(): Promise { - await this.syncToDatabase(); - } } export default LocalRepositoryComponent; diff --git a/server/src/integrations/playbooks-repository/PlaybooksRepositoryComponent.ts b/server/src/integrations/playbooks-repository/PlaybooksRepositoryComponent.ts index c432cf80..5cf5b28b 100644 --- a/server/src/integrations/playbooks-repository/PlaybooksRepositoryComponent.ts +++ b/server/src/integrations/playbooks-repository/PlaybooksRepositoryComponent.ts @@ -10,7 +10,7 @@ import logger from '../../logger'; import { Playbooks } from '../../types/typings'; import Shell from '../shell'; import { deleteFilesAndDirectory } from '../shell/utils'; -import { recursivelyFlattenTree } from './utils'; +import { recursivelyFlattenTree } from './tree-utils'; export const DIRECTORY_ROOT = '/playbooks'; export const FILE_PATTERN = /\.yml$/; @@ -160,7 +160,6 @@ export interface AbstractComponent extends PlaybooksRepositoryComponent { save(playbookUuid: string, content: string): Promise; init(): Promise; delete(): Promise; - syncToRepository(): Promise; syncFromRepository(): Promise; } diff --git a/server/src/integrations/playbooks-repository/utils.ts b/server/src/integrations/playbooks-repository/tree-utils.ts similarity index 100% rename from server/src/integrations/playbooks-repository/utils.ts rename to server/src/integrations/playbooks-repository/tree-utils.ts diff --git a/server/src/tests/integrations/playbooks-repository/PlaybookRepositoryComponent.test.ts b/server/src/tests/integrations/playbooks-repository/PlaybookRepositoryComponent.test.ts new file mode 100644 index 00000000..646e8935 --- /dev/null +++ b/server/src/tests/integrations/playbooks-repository/PlaybookRepositoryComponent.test.ts @@ -0,0 +1,30 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import LocalRepositoryComponent from '../../../integrations/local-repository/LocalRepositoryComponent'; +import PlaybooksRepositoryComponent from '../../../integrations/playbooks-repository/PlaybooksRepositoryComponent'; + +describe('PlaybooksRepositoryComponent', () => { + let playbooksRepositoryComponent: PlaybooksRepositoryComponent; + + beforeEach(() => { + const logger = { child: vi.fn() }; + playbooksRepositoryComponent = new LocalRepositoryComponent('uuid', logger, 'name', 'path'); + }); + + describe('fileBelongToRepository method', () => { + test('returns true if the root of the file path matches the root of the repository directory', () => { + const filePath = 'repository/file.ts'; + playbooksRepositoryComponent.directory = 'repository'; + const result = playbooksRepositoryComponent.fileBelongToRepository(filePath); + + expect(result).toBe(true); + }); + + test('returns false if the root of the file path does not match the root of the repository directory', () => { + const filePath = 'another_repository/file.ts'; + playbooksRepositoryComponent.directory = 'repository'; + const result = playbooksRepositoryComponent.fileBelongToRepository(filePath); + + expect(result).toBe(false); + }); + }); +}); diff --git a/server/src/tests/integrations/playbooks-repository/tree-utils.test.ts b/server/src/tests/integrations/playbooks-repository/tree-utils.test.ts new file mode 100644 index 00000000..a7fc03bd --- /dev/null +++ b/server/src/tests/integrations/playbooks-repository/tree-utils.test.ts @@ -0,0 +1,316 @@ +import { describe, expect, test, vi } from 'vitest'; +import { DirectoryTree } from 'ssm-shared-lib'; +import { + recursiveTreeCompletion, + recursivelyFlattenTree, +} from '../../../integrations/playbooks-repository/tree-utils'; + +const mockTree: DirectoryTree.TreeNode = { + path: '/root', + name: 'root', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [ + { + path: '/root/folder1', + name: 'folder1', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [ + { + path: '/root/folder1/file1', + name: 'file1', + extension: '.yml', + type: DirectoryTree.CONSTANTS.FILE, + }, + ], + }, + { + path: '/root/folder2', + name: 'folder2', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [ + { + path: '/root/folder2/file2', + name: 'file2', + extension: '.yml', + type: DirectoryTree.CONSTANTS.FILE, + }, + ], + }, + ], +}; + +const complexTree1: DirectoryTree.TreeNode = { + path: '/root', + name: 'root', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [ + { + path: '/root/folder1', + name: 'folder1', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [ + { + path: '/root/folder1/subfolder1', + name: 'subfolder1', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [ + { + path: '/root/folder1/subfolder1/file1', + name: 'file1', + extension: '.yml', + type: DirectoryTree.CONSTANTS.FILE, + }, + { + path: '/root/folder1/subfolder1/file2', + name: 'file2', + extension: '.txt', + type: DirectoryTree.CONSTANTS.FILE, + }, + ], + }, + ], + }, + { + path: '/root/folder2', + name: 'folder2', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [ + { + path: '/root/folder2/file1', + name: 'file1', + extension: '.yml', + type: DirectoryTree.CONSTANTS.FILE, + }, + ], + }, + ], +}; + +const complexTree2: DirectoryTree.TreeNode = { + path: '/home', + name: 'home', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [ + { + path: '/home/documents', + name: 'documents', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [ + { + path: '/home/documents/file1', + name: 'file1', + extension: '.docx', + type: DirectoryTree.CONSTANTS.FILE, + }, + { + path: '/home/documents/file2', + name: 'image', + extension: '.jpeg', + type: DirectoryTree.CONSTANTS.FILE, + }, + ], + }, + { + path: '/home/pictures', + name: 'pictures', + type: DirectoryTree.CONSTANTS.DIRECTORY, + }, + ], +}; + +describe('recursiveTreeCompletion', () => { + vi.mock('../../../data/database/repository/PlaybookRepo', async (importOriginal) => { + return { + default: { + ...(await importOriginal< + typeof import('../../../data/database/repository/PlaybookRepo') + >()), + findOneByPath: async () => { + return { uuid: 'uuid' }; + }, + }, + }; + }); + + vi.mock('../../../integrations/ansible/utils/ExtraVars', async (importOriginal) => { + return { + default: { + ...(await importOriginal()), + findValueOfExtraVars: async () => { + return undefined; + }, + }, + }; + }); + + test('should recursively process a tree and return new tree with completed nodes', async () => { + const newTree = await recursiveTreeCompletion(mockTree); + expect(newTree).not.toBeNull(); + expect(newTree).not.toBeUndefined(); + expect(newTree.length).toBeGreaterThan(0); + }); + + test('should throws an Error when the depth is greater than 20', async () => { + let error: Error | undefined; + try { + await recursiveTreeCompletion(mockTree, 21); + } catch (err) { + error = err as Error; + } + expect(error).toBeDefined(); + expect(error?.message).toEqual( + 'Depth is too high, to prevent any infinite loop, directories depth is limited to 20', + ); + }); + + test('should handle an empty tree', async () => { + const emptyTree: DirectoryTree.TreeNode = { + path: '/', + name: '', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [], + }; + + const result = await recursiveTreeCompletion(emptyTree); + + expect(result).not.toBeNull(); + expect(result[0]?.children).toBeUndefined(); + }); + + test('should not modify original tree', async () => { + const originalTree = { ...mockTree }; + + await recursiveTreeCompletion(mockTree); + + expect(JSON.stringify(originalTree)).toEqual(JSON.stringify(mockTree)); + }); + + test('should keep the original structure of tree', async () => { + const newTree = await recursiveTreeCompletion(mockTree); + + expect(JSON.stringify(newTree)).not.toEqual(JSON.stringify(mockTree)); + }); + + test('should correctly process complexTree1 and return a completed tree', async () => { + const newTree = await recursiveTreeCompletion(complexTree1); + expect(newTree).not.toBeNull(); + expect(newTree).not.toBeUndefined(); + expect(newTree[0].children?.length).toBe(1); + expect(newTree[0].children?.[0]?.children?.length).toBe(2); + expect(newTree[0].children?.[0]?.children?.[0]?.children?.length).toBeUndefined(); + expect(newTree[0].children?.[1]?.children?.length).toBeUndefined(); + }); + + test('should process complexTree2 correctly when it has a node with no children', async () => { + const newTree = await recursiveTreeCompletion(complexTree2); + expect(newTree[0]?.children?.[1]?.children?.length).toBe(2); + }); + + test('should add correct UUID to each node in the tree', async () => { + const newTree = await recursiveTreeCompletion(mockTree); + const assertUUID = (node: DirectoryTree.TreeNode) => { + if (node.type === DirectoryTree.CONSTANTS.FILE) { + expect((node as DirectoryTree.ExtendedTreeNode).uuid).toBe('uuid'); // make sure UUID is added correctly + } + if (Array.isArray(node.children)) { + (node.children as DirectoryTree.TreeNode[]).forEach(assertUUID); + } + }; + newTree.forEach(assertUUID); + }); +}); + +describe('recursivelyFlattenTree', () => { + const fileNode: DirectoryTree.TreeNode = { + path: '/file', + name: 'file', + extension: '.yml', + type: DirectoryTree.CONSTANTS.FILE, + }; + + const directoryNode: DirectoryTree.TreeNode = { + path: '/dir', + name: 'dir', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [fileNode], + }; + + test('should correctly flatten a tree with one node', () => { + const result = recursivelyFlattenTree(fileNode); + expect(result).toEqual([fileNode]); + }); + + test('should correctly flatten a tree with a file and a directory', () => { + const result = recursivelyFlattenTree(directoryNode); + expect(result).toEqual([fileNode]); + }); + + test('should throw an error when depth is greater than 20', () => { + const deepTree: DirectoryTree.TreeNode = directoryNode; + let node = deepTree; + for (let i = 0; i < 20; i++) { + node.children = [{ ...directoryNode, path: `/dir${i}` }]; + node = node.children[0] as DirectoryTree.TreeNode; + } + expect(() => recursivelyFlattenTree(deepTree)).toThrowError( + 'Depth is too high, to prevent any infinite loop, directories depth is limited to 20', + ); + }); + + test('should correctly handle a tree with more than one level of depth', () => { + const result = recursivelyFlattenTree({ + path: '/root', + name: 'root', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [ + { + path: '/root/dir1', + name: 'dir1', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [ + { ...fileNode, path: '/root/dir1/file1', name: 'file1' }, + { ...fileNode, path: '/root/dir1/file2', name: 'file2' }, + ], + }, + { + path: '/root/dir2', + name: 'dir2', + type: DirectoryTree.CONSTANTS.DIRECTORY, + children: [ + { ...fileNode, path: '/root/dir2/file1', name: 'file1' }, + { ...fileNode, path: '/root/dir2/file2', name: 'file2' }, + ], + }, + ], + }); + expect(result).toHaveLength(4); + expect(result).toContainEqual( + expect.objectContaining({ + path: '/root/dir1/file1', + name: 'file1', + type: DirectoryTree.CONSTANTS.FILE, + }), + ); + expect(result).toContainEqual( + expect.objectContaining({ + path: '/root/dir1/file2', + name: 'file2', + type: DirectoryTree.CONSTANTS.FILE, + }), + ); + expect(result).toContainEqual( + expect.objectContaining({ + path: '/root/dir2/file1', + name: 'file1', + type: DirectoryTree.CONSTANTS.FILE, + }), + ); + expect(result).toContainEqual( + expect.objectContaining({ + path: '/root/dir2/file2', + name: 'file2', + type: DirectoryTree.CONSTANTS.FILE, + }), + ); + }); +}); diff --git a/server/src/use-cases/LocalRepositoryUseCases.ts b/server/src/use-cases/LocalRepositoryUseCases.ts index e95f6b95..fe0d2f12 100644 --- a/server/src/use-cases/LocalRepositoryUseCases.ts +++ b/server/src/use-cases/LocalRepositoryUseCases.ts @@ -4,7 +4,6 @@ import { NotFoundError } from '../core/api/ApiError'; import PlaybooksRepositoryRepo from '../data/database/repository/PlaybooksRepositoryRepo'; import { DIRECTORY_ROOT } from '../integrations/playbooks-repository/PlaybooksRepositoryComponent'; import PlaybooksRepositoryEngine from '../integrations/playbooks-repository/PlaybooksRepositoryEngine'; -import { createDirectoryWithFullPath } from '../integrations/shell/utils'; import logger from '../logger'; async function addLocalRepository(name: string) { @@ -16,7 +15,6 @@ async function addLocalRepository(name: string) { enabled: true, directory: DIRECTORY_ROOT, }); - logger.info(localRepository.uuid); await PlaybooksRepositoryRepo.create({ uuid, type: Playbooks.PlaybooksRepositoryType.LOCAL, @@ -25,7 +23,7 @@ async function addLocalRepository(name: string) { enabled: true, }); try { - await createDirectoryWithFullPath(localRepository.getDirectory()); + await localRepository.init(); void localRepository.syncToDatabase(); } catch (error: any) { logger.warn(error); diff --git a/server/src/use-cases/PlaybooksRepositoryUseCases.ts b/server/src/use-cases/PlaybooksRepositoryUseCases.ts index 445faee7..f7d8bb5d 100644 --- a/server/src/use-cases/PlaybooksRepositoryUseCases.ts +++ b/server/src/use-cases/PlaybooksRepositoryUseCases.ts @@ -5,7 +5,7 @@ import PlaybooksRepository from '../data/database/model/PlaybooksRepository'; import PlaybooksRepositoryRepo from '../data/database/repository/PlaybooksRepositoryRepo'; import PlaybooksRepositoryComponent from '../integrations/playbooks-repository/PlaybooksRepositoryComponent'; import PlaybooksRepositoryEngine from '../integrations/playbooks-repository/PlaybooksRepositoryEngine'; -import { recursiveTreeCompletion } from '../integrations/playbooks-repository/utils'; +import { recursiveTreeCompletion } from '../integrations/playbooks-repository/tree-utils'; import shell from '../integrations/shell'; import { createDirectoryWithFullPath, deleteFilesAndDirectory } from '../integrations/shell/utils'; import logger from '../logger'; From 4d6f063ff7e2dfa1f1725ac6f967b47ad2e0aae7 Mon Sep 17 00:00:00 2001 From: manu Date: Sun, 30 Jun 2024 17:06:13 +0200 Subject: [PATCH 05/11] Update references from 'playbooks' to 'ansible' The term 'playbooks' has been replaced with 'ansible' in multiple locations across the codebase - both in commands and in actual code. This clarifies the usage and the relevance of Ansible in areas such as the AnsibleGalaxyCommand, in shell commands, and in cache interactions. Even in the technical documentation and guide, references have been updated to reflect this change. --- .../DeviceComponents/OSSoftwaresVersions/SoftwareIcon.tsx | 2 +- server/src/core/system/version.ts | 6 +++--- server/src/integrations/ansible/AnsibleGalaxyCmd.ts | 2 +- server/src/integrations/shell/index.ts | 6 +++--- site/.vitepress/config.ts | 2 +- site/contribute/index.md | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/client/src/components/DeviceComponents/OSSoftwaresVersions/SoftwareIcon.tsx b/client/src/components/DeviceComponents/OSSoftwaresVersions/SoftwareIcon.tsx index 70af322f..97a15f56 100644 --- a/client/src/components/DeviceComponents/OSSoftwaresVersions/SoftwareIcon.tsx +++ b/client/src/components/DeviceComponents/OSSoftwaresVersions/SoftwareIcon.tsx @@ -115,7 +115,7 @@ const SoftwareIcon: React.FC = (props) => { /* "systemOpenssl": "3.2.0", - "playbooks-repository": "2.23.0", + "git": "2.23.0", "tsc": "5.3.3", "mysql": "", "cache": "", diff --git a/server/src/core/system/version.ts b/server/src/core/system/version.ts index b4aa64d7..85015ce8 100644 --- a/server/src/core/system/version.ts +++ b/server/src/core/system/version.ts @@ -2,13 +2,13 @@ import { getFromCache, setToCache } from '../../data/cache'; import Shell from '../../integrations/shell'; export async function getAnsibleVersion() { - const ansibleVersion = await getFromCache('playbooks-version'); + const ansibleVersion = await getFromCache('ansible-version'); if (ansibleVersion) { return ansibleVersion; } else { const retrievedAnsibleVersion = await Shell.getAnsibleVersion(); if (retrievedAnsibleVersion) { - await setToCache('playbooks-version', retrievedAnsibleVersion); + await setToCache('ansible-version', retrievedAnsibleVersion); } return retrievedAnsibleVersion; } @@ -17,6 +17,6 @@ export async function getAnsibleVersion() { export async function setAnsibleVersion() { const retrievedAnsibleVersion = await Shell.getAnsibleVersion(); if (retrievedAnsibleVersion) { - await setToCache('playbooks-version', retrievedAnsibleVersion); + await setToCache('ansible-version', retrievedAnsibleVersion); } } diff --git a/server/src/integrations/ansible/AnsibleGalaxyCmd.ts b/server/src/integrations/ansible/AnsibleGalaxyCmd.ts index efabe9ba..a0491f84 100644 --- a/server/src/integrations/ansible/AnsibleGalaxyCmd.ts +++ b/server/src/integrations/ansible/AnsibleGalaxyCmd.ts @@ -1,5 +1,5 @@ class AnsibleGalaxyCommandBuilder { - static readonly ansibleGalaxy = 'playbooks-galaxy'; + static readonly ansibleGalaxy = 'ansible-galaxy'; static readonly collection = 'collection'; getInstallCollectionCmd(name: string, namespace: string) { diff --git a/server/src/integrations/shell/index.ts b/server/src/integrations/shell/index.ts index a910f115..91dc1e00 100644 --- a/server/src/integrations/shell/index.ts +++ b/server/src/integrations/shell/index.ts @@ -151,7 +151,7 @@ async function deletePlaybook(playbookPath: string) { async function getAnsibleVersion() { try { logger.info('[SHELL] - getAnsibleVersion - Starting...'); - return shell.exec('playbooks --version').toString(); + return shell.exec('ansible --version').toString(); } catch (error) { logger.error('[SHELL]- - getAnsibleVersion'); } @@ -185,10 +185,10 @@ async function installAnsibleGalaxyCollection(name: string, namespace: string) { await timeout(2000); shell.exec( AnsibleGalaxyCmd.getListCollectionsCmd(name, namespace) + - ' > /tmp/playbooks-collection-output.tmp.txt', + ' > /tmp/ansible-collection-output.tmp.txt', ); await timeout(2000); - collectionList = shell.cat('/tmp/playbooks-collection-output.tmp.txt').toString(); + collectionList = shell.cat('/tmp/ansible-collection-output.tmp.txt').toString(); } if (!collectionList.includes(`${namespace}.${name}`)) { throw new Error('[SHELL] - installAnsibleGalaxyCollection has failed'); diff --git a/site/.vitepress/config.ts b/site/.vitepress/config.ts index 689c30bc..e88edf66 100644 --- a/site/.vitepress/config.ts +++ b/site/.vitepress/config.ts @@ -114,7 +114,7 @@ export default defineConfig({ }, { text: 'Technical Guide', link: '/docs/technical-guide.md', items: [ - { text: 'Ansible', link: '/docs/playbooks.md' }, + { text: 'Ansible', link: '/docs/ansible.md' }, { text: 'Docker', link: '/docs/docker.md' }, { text: 'Manually installing the agent', link: '/docs/manual-install-agent.md' diff --git a/site/contribute/index.md b/site/contribute/index.md index 23b118b4..8f094c60 100644 --- a/site/contribute/index.md +++ b/site/contribute/index.md @@ -27,7 +27,7 @@ git clone https://github.com/SquirrelCorporation/SquirrelServersManager.git cd SquirrelCorporation # Ensure you are in the 'master' branch -playbooks-repository checkout master +git checkout master # Pull the latest updates playbooks-repository pull From 576ec80358bd3270b825a93c3946db46862525ad Mon Sep 17 00:00:00 2001 From: manu Date: Sun, 30 Jun 2024 17:08:26 +0200 Subject: [PATCH 06/11] Update documentation with correct Git commands The documentation has been updated to replace incorrect references of 'playbooks-repository' with the correct 'git' command. Additionally, an icon source in the main site page was replaced to better reflect the compatibility with Ansible. These changes ensure that users are provided with accurate instructions and consistent visual cues. --- site/contribute/index.md | 2 +- site/contribute/release.md | 6 +++--- site/docs/quickstart.md | 2 +- site/index.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/site/contribute/index.md b/site/contribute/index.md index 8f094c60..6322fdac 100644 --- a/site/contribute/index.md +++ b/site/contribute/index.md @@ -30,7 +30,7 @@ cd SquirrelCorporation git checkout master # Pull the latest updates -playbooks-repository pull +git pull # Create a new branch for your changes playbooks-repository checkout -b diff --git a/site/contribute/release.md b/site/contribute/release.md index f432349e..e9c6bdf3 100644 --- a/site/contribute/release.md +++ b/site/contribute/release.md @@ -8,7 +8,7 @@ Before initiating a new release, ensure you have pulled the latest changes from ```sh # Pull the latest changes from the remote repository -playbooks-repository pull origin master +git pull origin master # Run your tests to ensure everything works as expected npm test @@ -21,11 +21,11 @@ Once your master branch is up to date and tests are passing, create a new Git ta ```sh # The format is v.. # For example: -playbooks-repository tag v1.0.0 +git tag v1.0.0 ``` After creating the new tag, it needs to be pushed to the GitHub repository. ```sh -playbooks-repository push origin v1.0.0 +git push origin v1.0.0 ``` diff --git a/site/docs/quickstart.md b/site/docs/quickstart.md index ac77a75d..8723c898 100644 --- a/site/docs/quickstart.md +++ b/site/docs/quickstart.md @@ -137,7 +137,7 @@ Docker will create a volume directory *.data.prod* in the directory for persiste In SSM cloned directory: ```shell -playbooks-repository pull +git pull ``` ```shell diff --git a/site/index.md b/site/index.md index b6e8789b..eed9eb97 100644 --- a/site/index.md +++ b/site/index.md @@ -27,7 +27,7 @@ features: - title: Ansible & Docker Compatible details: Thanks to the power of Ansible and Docker, you can totally manage your servers, services and configuration from SSM icon: - src: /playbooks.svg + src: /ansible.svg - title: Simple to use, yet powerful details: Even is SSM is focused on easiness of use, it flexibility enables you to make powerful and complex setups icon: From 7bc9dd526ec23aa200928fc2fbae038ecfeaea15 Mon Sep 17 00:00:00 2001 From: manu Date: Sun, 30 Jun 2024 17:09:29 +0200 Subject: [PATCH 07/11] Update documentation with correct Git commands The documentation has been updated to replace incorrect references of 'playbooks-repository' with the correct 'git' command. Additionally, an icon source in the main site page was replaced to better reflect the compatibility with Ansible. These changes ensure that users are provided with accurate instructions and consistent visual cues. --- site/contribute/index.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/site/contribute/index.md b/site/contribute/index.md index 6322fdac..ae3de365 100644 --- a/site/contribute/index.md +++ b/site/contribute/index.md @@ -32,8 +32,7 @@ git checkout master # Pull the latest updates git pull -# Create a new branch for your changes -playbooks-repository checkout -b +git checkout -b ``` ## Making Changes @@ -51,13 +50,13 @@ After your changes, commit and push them to your GitHub repository. ```sh # Add changes -playbooks-repository add . +git add . # Commit the changes -playbooks-repository commit -m "" +git commit -m "" # Push your changes to your repository -playbooks-repository push origin +git push origin ``` ## Submitting a Pull Request From fb2d51254fda88ce44e53b38e17099836892767e Mon Sep 17 00:00:00 2001 From: manu Date: Sun, 30 Jun 2024 17:10:40 +0200 Subject: [PATCH 08/11] Add instruction to create new branch in contribute guide The contribute guide has been updated to include an explicit step for creating a new branch for changes. This clarification ensures contributors understand the importance of not making changes directly on the main branch. --- site/contribute/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/site/contribute/index.md b/site/contribute/index.md index ae3de365..33a7e291 100644 --- a/site/contribute/index.md +++ b/site/contribute/index.md @@ -32,6 +32,7 @@ git checkout master # Pull the latest updates git pull +# Create a new branch for your changes git checkout -b ``` From e355e2fb5898a9265d875f0f13028c1ebdbe616a Mon Sep 17 00:00:00 2001 From: manu Date: Mon, 1 Jul 2024 10:17:53 +0200 Subject: [PATCH 09/11] Refactor playbook file and repository management Codebase has been refactored to break down shell operations pertaining to playbooks, files, and repositories. Export structure has been improved and segment separation helps identify which components are responsible for each task. While enhancing code readability, this also facilitates future code maintenance and improvements. --- .../components/DirectoryTreeView.tsx | 7 +- .../components/NewFileDrawerForm.tsx | 5 + .../components/PlaybookDropdownMenu.tsx | 22 +- .../Playbooks/components/TreeComponent.tsx | 10 +- .../data/database/repository/PlaybookRepo.ts | 20 +- .../git-repository/GitRepositoryComponent.ts | 5 +- .../integrations/git-repository/lib/clone.ts | 13 +- .../integrations/git-repository/lib/errors.ts | 11 +- .../git-repository/lib/forcePull.ts | 2 +- .../git-repository/lib/initGit.ts | 4 +- .../git-repository/lib/inspect.ts | 56 ++--- .../git-repository/lib/interface.ts | 2 +- .../LocalRepositoryComponent.ts | 4 +- .../PlaybooksRepositoryComponent.ts | 13 +- .../PlaybooksRepositoryEngine.ts | 2 +- server/src/integrations/shell/ansible.ts | 116 ++++++++++ .../src/integrations/shell/authentication.ts | 19 ++ server/src/integrations/shell/index.ts | 217 +----------------- .../src/integrations/shell/playbook-file.ts | 64 ++++++ server/src/integrations/shell/utils.ts | 40 +++- .../services/playbooks-repository/local.ts | 1 - .../platbooks-repository.validator.ts | 6 +- server/src/use-cases/DeviceAuthUseCases.ts | 2 +- server/src/use-cases/DeviceUseCases.ts | 2 +- server/src/use-cases/GitRepositoryUseCases.ts | 6 +- server/src/use-cases/PlaybookUseCases.ts | 11 +- .../use-cases/PlaybooksRepositoryUseCases.ts | 15 +- 27 files changed, 368 insertions(+), 307 deletions(-) create mode 100644 server/src/integrations/shell/ansible.ts create mode 100644 server/src/integrations/shell/authentication.ts create mode 100644 server/src/integrations/shell/playbook-file.ts diff --git a/client/src/pages/Playbooks/components/DirectoryTreeView.tsx b/client/src/pages/Playbooks/components/DirectoryTreeView.tsx index 0417f0c7..a50a747f 100644 --- a/client/src/pages/Playbooks/components/DirectoryTreeView.tsx +++ b/client/src/pages/Playbooks/components/DirectoryTreeView.tsx @@ -41,6 +41,7 @@ type DirectoryTreeViewProps = { const DirectoryTreeView: React.FC = (props) => { const [storeModal, setStoreModal] = React.useState(false); + const [selectedPath, setSelectedPath] = React.useState(''); const { onSelect, selectedFile, @@ -71,8 +72,12 @@ const DirectoryTreeView: React.FC = (props) => { onSelect={onSelect} treeData={playbookRepositories} selectedKeys={[selectedFile?.path as React.Key]} + expandedKeys={[selectedPath as React.Key]} + /> + - Promise; + setSelectedNode: any; }; const NewFileDrawerForm: React.FC = (props) => { @@ -64,11 +65,15 @@ const NewFileDrawerForm: React.FC = (props) => { .finally(() => { setLoading(false); }); + return true; }} > { + props.setSelectedNode(repositories?.find((f) => f.uuid === e)?.name); + }} request={async () => { return await getPlaybooksRepositories().then((e) => { setRepositories(e?.data); diff --git a/client/src/pages/Playbooks/components/PlaybookDropdownMenu.tsx b/client/src/pages/Playbooks/components/PlaybookDropdownMenu.tsx index 4a5d97a1..b45b64ff 100644 --- a/client/src/pages/Playbooks/components/PlaybookDropdownMenu.tsx +++ b/client/src/pages/Playbooks/components/PlaybookDropdownMenu.tsx @@ -1,8 +1,10 @@ import { Callbacks } from '@/pages/Playbooks/components/TreeComponent'; import { + ArrowDownOutlined, DeleteOutlined, FileOutlined, FolderOpenOutlined, + SyncOutlined, } from '@ant-design/icons'; import { Dropdown, MenuProps, Popconfirm } from 'antd'; import React from 'react'; @@ -15,6 +17,7 @@ type PlaybookDropdownMenuType = { children: React.ReactNode; callbacks: Callbacks; cannotDelete?: boolean; + remoteRootNode?: boolean; }; type PlaybookDrownMenuItemType = { @@ -46,6 +49,21 @@ const menuItems: PlaybookDrownMenuItemType[] = [ }, ]; +const menuItemsGitRootNode: PlaybookDrownMenuItemType[] = [ + { + label: 'Commit & Sync', + icon: , + key: '4', + fileType: 'any', + }, + { + label: 'Force pull', + icon: , + key: '5', + fileType: 'any', + }, +]; + const PlaybookDropdownMenu: React.FC = (props) => { const [open, setOpen] = React.useState(false); const items = menuItems @@ -61,7 +79,9 @@ const PlaybookDropdownMenu: React.FC = (props) => { icon: e.icon, }; }) as MenuProps['items']; - + if (props.remoteRootNode) { + items?.push(...menuItemsGitRootNode); + } const onClick: MenuProps['onClick'] = async ({ key, domEvent }) => { domEvent.stopPropagation(); switch (key) { diff --git a/client/src/pages/Playbooks/components/TreeComponent.tsx b/client/src/pages/Playbooks/components/TreeComponent.tsx index 8e95514d..93f1deaa 100644 --- a/client/src/pages/Playbooks/components/TreeComponent.tsx +++ b/client/src/pages/Playbooks/components/TreeComponent.tsx @@ -5,10 +5,10 @@ import { import { Typography } from 'antd'; import React, { ReactNode } from 'react'; import { - DirectoryTree, API, - Playbooks, + DirectoryTree, DirectoryTree as DT, + Playbooks, } from 'ssm-shared-lib'; import PlaybookDropdownMenu from './PlaybookDropdownMenu'; @@ -40,8 +40,6 @@ export type Callbacks = { callbackDeleteFile: (path: string, playbookRepositoryUuid: string) => void; }; -export type RootNode = {}; - export function buildTree( rootNode: API.PlaybooksRepository, callbacks: Callbacks, @@ -59,16 +57,18 @@ export function buildTree( }} callbacks={callbacks} cannotDelete={true} + remoteRootNode={rootNode.type === Playbooks.PlaybooksRepositoryType.GIT} > {rootNode.name} +           ), key: rootNode.name, icon: rootNode.type === Playbooks.PlaybooksRepositoryType.LOCAL ? ( - + ) : ( ), diff --git a/server/src/data/database/repository/PlaybookRepo.ts b/server/src/data/database/repository/PlaybookRepo.ts index ed364765..d5bd052d 100644 --- a/server/src/data/database/repository/PlaybookRepo.ts +++ b/server/src/data/database/repository/PlaybookRepo.ts @@ -25,11 +25,17 @@ async function findAllWithActiveRepositories(): Promise { } async function findOneByName(name: string): Promise { - return await PlaybookModel.findOne({ name: name }).lean().exec(); + return await PlaybookModel.findOne({ name: name }) + .populate({ path: 'playbooksRepository' }) + .lean() + .exec(); } async function findOneByUuid(uuid: string): Promise { - return await PlaybookModel.findOne({ uuid: uuid }).lean().exec(); + return await PlaybookModel.findOne({ uuid: uuid }) + .populate({ path: 'playbooksRepository' }) + .lean() + .exec(); } async function listAllByRepository( @@ -43,11 +49,17 @@ async function deleteByUuid(uuid: string): Promise { } async function findOneByPath(path: string): Promise { - return await PlaybookModel.findOne({ path: path }).lean().exec(); + return await PlaybookModel.findOne({ path: path }) + .populate({ path: 'playbooksRepository' }) + .lean() + .exec(); } async function findOneByUniqueQuickReference(quickRef: string): Promise { - return await PlaybookModel.findOne({ uniqueQuickRef: quickRef }).lean().exec(); + return await PlaybookModel.findOne({ uniqueQuickRef: quickRef }) + .populate({ path: 'playbooksRepository' }) + .lean() + .exec(); } async function deleteByRepository(playbooksRepository: PlaybooksRepository): Promise { diff --git a/server/src/integrations/git-repository/GitRepositoryComponent.ts b/server/src/integrations/git-repository/GitRepositoryComponent.ts index fc4e8b61..20b5d695 100644 --- a/server/src/integrations/git-repository/GitRepositoryComponent.ts +++ b/server/src/integrations/git-repository/GitRepositoryComponent.ts @@ -1,9 +1,8 @@ import PlaybooksRepositoryComponent, { AbstractComponent, DIRECTORY_ROOT, - FILE_PATTERN, } from '../playbooks-repository/PlaybooksRepositoryComponent'; -import { createDirectoryWithFullPath, findFilesInDirectory } from '../shell/utils'; +import { createDirectoryWithFullPath } from '../shell/utils'; import { GitStep, IGitUserInfos, @@ -52,7 +51,7 @@ class GitRepositoryComponent extends PlaybooksRepositoryComponent implements Abs async clone() { this.childLogger.info('Clone starting...'); try { - void createDirectoryWithFullPath(this.directory); + void createDirectoryWithFullPath(this.directory, DIRECTORY_ROOT); await clone({ ...this.options, logger: { diff --git a/server/src/integrations/git-repository/lib/clone.ts b/server/src/integrations/git-repository/lib/clone.ts index f12fcbf4..9ddaba37 100644 --- a/server/src/integrations/git-repository/lib/clone.ts +++ b/server/src/integrations/git-repository/lib/clone.ts @@ -44,7 +44,7 @@ export async function clone(options: { remoteUrl, }); - logProgress(GitStep.PrepareCloneOnlineWiki); + logProgress(GitStep.PrepareClone); logDebug( JSON.stringify({ @@ -54,17 +54,20 @@ export async function clone(options: { length: 24, }), }), - GitStep.PrepareCloneOnlineWiki, + GitStep.PrepareClone, ); - logDebug(`Running git init for clone in dir ${dir}`, GitStep.PrepareCloneOnlineWiki); + logDebug(`Running git init for clone in dir ${dir}`, GitStep.PrepareClone); await initGitWithBranch(dir, branch, { initialCommit: false }); const remoteName = await getRemoteName(dir, branch); - logDebug(`Successfully Running git init for clone in dir ${dir}`, GitStep.PrepareCloneOnlineWiki); + logDebug(`Successfully Running git init for clone in dir ${dir}`, GitStep.PrepareClone); logProgress(GitStep.StartConfiguringGithubRemoteRepository); await credentialOn(dir, remoteUrl, gitUserName, accessToken, remoteName); try { logProgress(GitStep.StartFetchingFromGithubRemote); - const { stderr: pullStdError, exitCode } = await GitProcess.exec(['pull', remoteName, `${branch}:${branch}`], dir); + const { stderr: pullStdError, exitCode } = await GitProcess.exec( + ['pull', remoteName, `${branch}:${branch}`], + dir, + ); if (exitCode === 0) { logProgress(GitStep.SynchronizationFinish); } else { diff --git a/server/src/integrations/git-repository/lib/errors.ts b/server/src/integrations/git-repository/lib/errors.ts index b46a02e1..0444bb9d 100644 --- a/server/src/integrations/git-repository/lib/errors.ts +++ b/server/src/integrations/git-repository/lib/errors.ts @@ -44,8 +44,8 @@ export class GitPullPushError extends Error { super(extraMessages); Object.setPrototypeOf(this, GitPullPushError.prototype); this.name = 'GitPullPushError'; - this.message = `E-3 failed to config git to successfully pull from or push to remote with configuration ${ - JSON.stringify({ + this.message = `E-3 failed to config git to successfully pull from or push to remote with configuration ${JSON.stringify( + { ...configuration, userInfo: { ...configuration.userInfo, @@ -53,8 +53,8 @@ export class GitPullPushError extends Error { length: 24, }), }, - }) - }.\nerrorMessages: ${extraMessages}`; + }, + )}.\nerrorMessages: ${extraMessages}`; } } @@ -86,8 +86,7 @@ export class CantSyncInSpecialGitStateAutoFixFailed extends Error { Object.setPrototypeOf(this, CantSyncInSpecialGitStateAutoFixFailed.prototype); this.stateMessage = stateMessage; this.name = 'CantSyncInSpecialGitStateAutoFixFailed'; - this.message = - `E-6 Unable to Sync, this folder is in special condition, thus can't Sync directly. An auto-fix has been tried, but error still remains. Please resolve all the conflict manually (For example, use VSCode to open the wiki folder), if this still don't work out, please use professional Git tools (Source Tree, GitKraken) to solve this. This is caused by procedural bug in the git-sync-js.\n${stateMessage}`; + this.message = `E-6 Unable to Sync, this folder is in special condition, thus can't Sync directly. An auto-fix has been tried, but error still remains. Please resolve all the conflict manually , if this still don't work out, please use professional Git tools (Source Tree, GitKraken) to solve this. This is caused by procedural bug in the git-sync-js.\n${stateMessage}`; } } diff --git a/server/src/integrations/git-repository/lib/forcePull.ts b/server/src/integrations/git-repository/lib/forcePull.ts index e3550f5e..6575235d 100644 --- a/server/src/integrations/git-repository/lib/forcePull.ts +++ b/server/src/integrations/git-repository/lib/forcePull.ts @@ -10,7 +10,7 @@ import { fetchRemote } from './sync'; export interface IForcePullOptions { /** Optional fallback of userInfo. If some info is missing in userInfo, will use defaultGitInfo instead. */ defaultGitInfo?: typeof defaultDefaultGitInfo; - /** wiki folder path, can be relative */ + /** folder path, can be relative */ dir: string; logger?: ILogger; /** the storage service url we are sync to, for example your github repo url diff --git a/server/src/integrations/git-repository/lib/initGit.ts b/server/src/integrations/git-repository/lib/initGit.ts index 609ffdfa..47ce243f 100644 --- a/server/src/integrations/git-repository/lib/initGit.ts +++ b/server/src/integrations/git-repository/lib/initGit.ts @@ -9,7 +9,7 @@ import { commitFiles } from './sync'; export interface IInitGitOptionsSyncImmediately { /** Optional fallback of userInfo. If some info is missing in userInfo, will use defaultGitInfo instead. */ defaultGitInfo?: typeof defaultDefaultGitInfo; - /** wiki folder path, can be relative */ + /** folder path, can be relative */ dir: string; logger?: ILogger; /** only required if syncImmediately is true, the storage service url we are sync to, for example your github repo url */ @@ -21,7 +21,7 @@ export interface IInitGitOptionsSyncImmediately { } export interface IInitGitOptionsNotSync { defaultGitInfo?: typeof defaultDefaultGitInfo; - /** wiki folder path, can be relative */ + /** folder path, can be relative */ dir: string; logger?: ILogger; /** should we sync after playbooks-repository init? */ diff --git a/server/src/integrations/git-repository/lib/inspect.ts b/server/src/integrations/git-repository/lib/inspect.ts index 19067ef5..b172eccd 100644 --- a/server/src/integrations/git-repository/lib/inspect.ts +++ b/server/src/integrations/git-repository/lib/inspect.ts @@ -25,10 +25,10 @@ export interface ModifiedFileList { } /** * Get modified files and modify type in a folder - * @param {string} wikiFolderPath location to scan playbooks-repository modify state + * @param {string} folderPath location to scan playbooks-repository modify state */ -export async function getModifiedFileList(wikiFolderPath: string): Promise { - const { stdout } = await GitProcess.exec(['status', '--porcelain'], wikiFolderPath); +export async function getModifiedFileList(folderPath: string): Promise { + const { stdout } = await GitProcess.exec(['status', '--porcelain'], folderPath); const stdoutLines = stdout.split('\n'); const nonEmptyLines = compact(stdoutLines); const statusMatrixLines = compact( @@ -66,7 +66,7 @@ export async function getModifiedFileList(wikiFolderPath: string): Promise item.fileRelativePath.localeCompare(item2.fileRelativePath, 'zh')); @@ -74,7 +74,7 @@ export async function getModifiedFileList(wikiFolderPath: string): Promise 0) { - return wikiRepoName; + if (repoName.length > 0) { + return repoName; } return undefined; } /** * See if there is any file not being committed - * @param {string} wikiFolderPath repo path to test + * @param {string} folderPath repo path to test * @example ```ts if (await haveLocalChanges(dir)) { // ... do commit and push ``` */ -export async function haveLocalChanges(wikiFolderPath: string): Promise { - const { stdout } = await GitProcess.exec(['status', '--porcelain'], wikiFolderPath); +export async function haveLocalChanges(folderPath: string): Promise { + const { stdout } = await GitProcess.exec(['status', '--porcelain'], folderPath); const matchResult = stdout.match(/^(\?\?|[ACMR] |[ ACMR][DM])*/gm); // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions return !!matchResult?.some?.(Boolean); @@ -128,11 +128,11 @@ export async function haveLocalChanges(wikiFolderPath: string): Promise * Get "master" or "main" from playbooks-repository repo * * https://github.com/simonthum/git-sync/blob/31cc140df2751e09fae2941054d5b61c34e8b649/git-sync#L228-L232 - * @param wikiFolderPath + * @param folderPath */ -export async function getDefaultBranchName(wikiFolderPath: string): Promise { +export async function getDefaultBranchName(folderPath: string): Promise { try { - const { stdout } = await GitProcess.exec(['rev-parse', '--abbrev-ref', 'HEAD'], wikiFolderPath); + const { stdout } = await GitProcess.exec(['rev-parse', '--abbrev-ref', 'HEAD'], folderPath); const [branchName] = stdout.split('\n'); // don't return empty string, so we can use ?? syntax if (branchName === '') { @@ -218,12 +218,12 @@ export async function getSyncState( } export async function assumeSync( - wikiFolderPath: string, + folderPath: string, defaultBranchName: string, remoteName: string, logger?: ILogger, ): Promise { - const syncState = await getSyncState(wikiFolderPath, defaultBranchName, remoteName, logger); + const syncState = await getSyncState(folderPath, defaultBranchName, remoteName, logger); if (syncState === 'equal') { return; } @@ -232,19 +232,16 @@ export async function assumeSync( /** * get various repo state in string format - * @param wikiFolderPath repo path to check + * @param folderPath repo path to check * @param logger * @returns gitState * // TODO: use template literal type to get exact type of playbooks-repository state */ -export async function getGitRepositoryState( - wikiFolderPath: string, - logger?: ILogger, -): Promise { - if (!(await hasGit(wikiFolderPath))) { +export async function getGitRepositoryState(folderPath: string, logger?: ILogger): Promise { + if (!(await hasGit(folderPath))) { return 'NOGIT'; } - const gitDirectory = await getGitDirectory(wikiFolderPath, logger); + const gitDirectory = await getGitDirectory(folderPath, logger); const [isRebaseI, isRebaseM, isAMRebase, isMerging, isCherryPicking, isBisecting] = await Promise.all([ // isRebaseI @@ -295,7 +292,7 @@ export async function getGitRepositoryState( } } result += ( - await GitProcess.exec(['rev-parse', '--is-bare-repository', wikiFolderPath], wikiFolderPath) + await GitProcess.exec(['rev-parse', '--is-bare-repository', folderPath], folderPath) ).stdout.startsWith('true') ? '|BARE' : ''; @@ -308,7 +305,7 @@ export async function getGitRepositoryState( } } */ // previous above `playbooks-repository diff --no-ext-diff --quiet --exit-code` logic from playbooks-repository-sync script can only detect if an existed file changed, can't detect newly added file, so we use `haveLocalChanges` instead - if (await haveLocalChanges(wikiFolderPath)) { + if (await haveLocalChanges(folderPath)) { result += '|DIRTY'; } @@ -340,7 +337,10 @@ export async function getGitDirectory(dir: string, logger?: ILogger): Promise; + public rootPath: string; - protected constructor(uuid: string, name: string, path: string) { - const dir = `${path}/${uuid}`; + protected constructor(uuid: string, name: string, rootPath: string) { + this.rootPath = rootPath; + const dir = `${rootPath}/${uuid}`; this.uuid = uuid; this.directory = dir; this.name = name; @@ -118,9 +120,10 @@ abstract class PlaybooksRepositoryComponent { foundPlaybook: Playbook, playbooksRepository: PlaybooksRepository, ): Promise { - const configurationFileContent = await Shell.readPlaybookConfigurationFileIfExists( - foundPlaybook.path.replace('.yml', '.json'), - ); + const configurationFileContent = + await Shell.PlaybookFileShell.readPlaybookConfigurationFileIfExists( + foundPlaybook.path.replace('.yml', '.json'), + ); const isCustomPlaybook = !foundPlaybook.name.startsWith('_'); const playbookFoundInDatabase = await PlaybookRepo.findOneByPath(foundPlaybook.path); const playbookData: Playbook = { diff --git a/server/src/integrations/playbooks-repository/PlaybooksRepositoryEngine.ts b/server/src/integrations/playbooks-repository/PlaybooksRepositoryEngine.ts index 75bddfba..4faad033 100644 --- a/server/src/integrations/playbooks-repository/PlaybooksRepositoryEngine.ts +++ b/server/src/integrations/playbooks-repository/PlaybooksRepositoryEngine.ts @@ -72,7 +72,7 @@ async function registerRepository(playbookRepository: PlaybooksRepository) { await registerLocalRepository(playbookRepository); break; default: - throw new Error('Unknown playbook type'); + throw new Error('Unknown playbook repository type'); } return state.playbooksRepository[playbookRepository.uuid]; } diff --git a/server/src/integrations/shell/ansible.ts b/server/src/integrations/shell/ansible.ts new file mode 100644 index 00000000..0ed099a7 --- /dev/null +++ b/server/src/integrations/shell/ansible.ts @@ -0,0 +1,116 @@ +import shell from 'shelljs'; +import { v4 as uuidv4 } from 'uuid'; +import { API } from 'ssm-shared-lib'; +import User from '../../data/database/model/User'; +import AnsibleTaskRepo from '../../data/database/repository/AnsibleTaskRepo'; +import DeviceAuthRepo from '../../data/database/repository/DeviceAuthRepo'; +import logger from '../../logger'; +import { Playbooks } from '../../types/typings'; +import ansibleCmd from '../ansible/AnsibleCmd'; +import AnsibleGalaxyCmd from '../ansible/AnsibleGalaxyCmd'; +import Inventory from '../ansible/utils/InventoryTransformer'; +import { timeout } from './utils'; + +export const ANSIBLE_PATH = '/server/src/ansible/'; + +async function executePlaybook( + playbookPath: string, + user: User, + target?: string[], + extraVars?: API.ExtraVars, +) { + logger.info('[SHELL] - executePlaybook - Starting...'); + + let inventoryTargets: (Playbooks.All & Playbooks.HostGroups) | undefined; + if (target) { + logger.info(`[SHELL] - executePlaybook - called with target: ${target}`); + const devicesAuth = await DeviceAuthRepo.findManyByDevicesUuid(target); + if (!devicesAuth || devicesAuth.length === 0) { + logger.error(`[SHELL] - executePlaybook - Target not found (Authentication not found)`); + throw new Error('Exec failed, no matching target (Authentication not found)'); + } + inventoryTargets = Inventory.inventoryBuilderForTarget(devicesAuth); + } + return await executePlaybookOnInventory(playbookPath, user, inventoryTargets, extraVars); +} + +async function executePlaybookOnInventory( + playbookPath: string, + user: User, + inventoryTargets?: Playbooks.All & Playbooks.HostGroups, + extraVars?: API.ExtraVars, +) { + shell.cd(ANSIBLE_PATH); + shell.rm('/server/src/playbooks/inventory/hosts'); + shell.rm('/server/src/playbooks/env/_extravars'); + const uuid = uuidv4(); + const result = await new Promise((resolve) => { + const cmd = ansibleCmd.buildAnsibleCmd(playbookPath, uuid, inventoryTargets, user, extraVars); + logger.info(`[SHELL] - executePlaybook - Executing ${cmd}`); + const child = shell.exec(cmd, { + async: true, + }); + child.stdout?.on('data', function (data) { + resolve(data); + }); + child.on('exit', function () { + resolve(null); + }); + }); + logger.info('[SHELL] - executePlaybook - launched'); + if (result) { + logger.info(`[SHELL] - executePlaybook - ExecId is ${uuid}`); + await AnsibleTaskRepo.create({ + ident: uuid, + status: 'created', + cmd: `playbook ${playbookPath}`, + }); + return result; + } else { + logger.error('[SHELL] - executePlaybook - Result was not properly set'); + throw new Error('Exec failed'); + } +} + +async function getAnsibleVersion() { + try { + logger.info('[SHELL] - getAnsibleVersion - Starting...'); + return shell.exec('ansible --version').toString(); + } catch (error) { + logger.error('[SHELL]- - getAnsibleVersion'); + } +} + +async function installAnsibleGalaxyCollection(name: string, namespace: string) { + try { + logger.info('[SHELL] - installAnsibleGalaxyCollection Starting...'); + const result = shell.exec(AnsibleGalaxyCmd.getInstallCollectionCmd(name, namespace)); + if (result.code !== 0) { + throw new Error('[SHELL] - installAnsibleGalaxyCollection has failed'); + } + let collectionList = ''; + let i = 0; + while (!collectionList.includes(`${namespace}.${name}`) && i++ < 60) { + await timeout(2000); + shell.exec( + AnsibleGalaxyCmd.getListCollectionsCmd(name, namespace) + + ' > /tmp/ansible-collection-output.tmp.txt', + ); + await timeout(2000); + collectionList = shell.cat('/tmp/ansible-collection-output.tmp.txt').toString(); + } + if (!collectionList.includes(`${namespace}.${name}`)) { + throw new Error('[SHELL] - installAnsibleGalaxyCollection has failed'); + } + } catch (error) { + logger.error('[SHELL] - installAnsibleGalaxyCollection'); + throw error; + } +} + +export { + executePlaybook, + getAnsibleVersion, + executePlaybookOnInventory, + installAnsibleGalaxyCollection, +}; diff --git a/server/src/integrations/shell/authentication.ts b/server/src/integrations/shell/authentication.ts new file mode 100644 index 00000000..7b6723d0 --- /dev/null +++ b/server/src/integrations/shell/authentication.ts @@ -0,0 +1,19 @@ +import shell from 'shelljs'; +import logger from '../../logger'; + +async function saveSshKey(key: string, uuid: string) { + try { + logger.info('[SHELL] - vaultSshKey Starting...'); + if (shell.exec(`echo '${key}' > /tmp/${uuid}.key`).code !== 0) { + throw new Error('[SHELL] - vaultSshKey - Error creating tmp file'); + } + if (shell.chmod(600, `/tmp/${uuid}.key`).code !== 0) { + throw new Error('[SHELL] - vaultSshKey - Error chmoding file'); + } + } catch (error) { + logger.error('[SHELL] - vaultSshKey'); + throw error; + } +} + +export { saveSshKey }; diff --git a/server/src/integrations/shell/index.ts b/server/src/integrations/shell/index.ts index 91dc1e00..a16de1ba 100644 --- a/server/src/integrations/shell/index.ts +++ b/server/src/integrations/shell/index.ts @@ -1,214 +1,9 @@ -import shell from 'shelljs'; -import { API } from 'ssm-shared-lib'; -import { v4 as uuidv4 } from 'uuid'; -import User from '../../data/database/model/User'; -import AnsibleTaskRepo from '../../data/database/repository/AnsibleTaskRepo'; -import DeviceAuthRepo from '../../data/database/repository/DeviceAuthRepo'; -import logger from '../../logger'; -import AnsibleGalaxyCmd from '../ansible/AnsibleGalaxyCmd'; -import Inventory from '../ansible/utils/InventoryTransformer'; -import { Playbooks } from '../../types/typings'; -import ansibleCmd from '../ansible/AnsibleCmd'; - -export const ANSIBLE_PATH = '/server/src/ansible/'; - -function timeout(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function executePlaybook( - playbookPath: string, - user: User, - target?: string[], - extraVars?: API.ExtraVars, -) { - logger.info('[SHELL]-[ANSIBLE] - executePlaybook - Starting...'); - - let inventoryTargets: (Playbooks.All & Playbooks.HostGroups) | undefined; - if (target) { - logger.info(`[SHELL]-[ANSIBLE] - executePlaybook - called with target: ${target}`); - const devicesAuth = await DeviceAuthRepo.findManyByDevicesUuid(target); - if (!devicesAuth || devicesAuth.length === 0) { - logger.error( - `[SHELL]-[ANSIBLE] - executePlaybook - Target not found (Authentication not found)`, - ); - throw new Error('Exec failed, no matching target (Authentication not found)'); - } - inventoryTargets = Inventory.inventoryBuilderForTarget(devicesAuth); - } - return await executePlaybookOnInventory(playbookPath, user, inventoryTargets, extraVars); -} - -async function executePlaybookOnInventory( - playbookPath: string, - user: User, - inventoryTargets?: Playbooks.All & Playbooks.HostGroups, - extraVars?: API.ExtraVars, -) { - shell.cd(ANSIBLE_PATH); - shell.rm('/server/src/playbooks/inventory/hosts'); - shell.rm('/server/src/playbooks/env/_extravars'); - const uuid = uuidv4(); - const result = await new Promise((resolve) => { - const cmd = ansibleCmd.buildAnsibleCmd(playbookPath, uuid, inventoryTargets, user, extraVars); - logger.info(`[SHELL]-[ANSIBLE] - executePlaybook - Executing ${cmd}`); - const child = shell.exec(cmd, { - async: true, - }); - child.stdout?.on('data', function (data) { - resolve(data); - }); - child.on('exit', function () { - resolve(null); - }); - }); - logger.info('[SHELL]-[ANSIBLE] - executePlaybook - launched'); - if (result) { - logger.info(`[SHELL]-[ANSIBLE] - executePlaybook - ExecId is ${uuid}`); - await AnsibleTaskRepo.create({ - ident: uuid, - status: 'created', - cmd: `playbook ${playbookPath}`, - }); - return result; - } else { - logger.error('[SHELL]-[ANSIBLE] - executePlaybook - Result was not properly set'); - throw new Error('Exec failed'); - } -} - -async function listPlaybooks() { - try { - logger.info('[SHELL]-[ANSIBLE] - listPlaybook - Starting...'); - shell.cd(ANSIBLE_PATH); - const listOfPlaybooks: string[] = []; - shell.ls('*.yml').forEach(function (file) { - listOfPlaybooks.push(file); - }); - logger.info('[SHELL]-[ANSIBLE] - listPlaybook - ended'); - return listOfPlaybooks; - } catch (error) { - logger.error('[SHELL]-[ANSIBLE] - listPlaybook'); - throw new Error('listPlaybooks failed'); - } -} - -async function readPlaybook(playbook: string) { - try { - logger.info(`[SHELL]-[ANSIBLE] - readPlaybook - ${playbook} - Starting...`); - return shell.cat(playbook).toString(); - } catch (error) { - logger.error('[SHELL]-[ANSIBLE] - readPlaybook'); - throw new Error('readPlaybook failed'); - } -} - -async function readPlaybookConfigurationFileIfExists(path: string) { - try { - logger.info(`[SHELL]-[ANSIBLE] - readPlaybookConfiguration - ${path} - Starting...`); - if (!shell.test('-f', `${path}`)) { - logger.info(`[SHELL]-[ANSIBLE] - readPlaybookConfiguration - not found`); - return undefined; - } - return shell.cat(`${path}`).toString(); - } catch (error) { - logger.error('[SHELL]-[ANSIBLE] - readPlaybookConfiguration'); - throw new Error('readPlaybookConfiguration failed'); - } -} - -async function editPlaybook(playbookPath: string, content: string) { - try { - logger.info('[SHELL]-[ANSIBLE] - editPlaybook - Starting...'); - shell.ShellString(content).to(playbookPath); - } catch (error) { - logger.error('[SHELL]-[ANSIBLE] - editPlaybook'); - throw new Error('editPlaybook failed'); - } -} - -async function newPlaybook(playbook: string) { - try { - logger.info('[SHELL]-[ANSIBLE] - newPlaybook - Starting...'); - shell.cd(ANSIBLE_PATH); - shell.touch(playbook + '.yml'); - } catch (error) { - logger.error('[SHELL]-[ANSIBLE] - newPlaybook'); - throw new Error('newPlaybook failed'); - } -} - -async function deletePlaybook(playbookPath: string) { - try { - logger.info('[SHELL]-[ANSIBLE] - deletePlaybook - Starting...'); - shell.rm(playbookPath); - } catch (error) { - logger.error('[SHELL]-[ANSIBLE] - deletePlaybook'); - throw new Error('deletePlaybook failed'); - } -} - -async function getAnsibleVersion() { - try { - logger.info('[SHELL] - getAnsibleVersion - Starting...'); - return shell.exec('ansible --version').toString(); - } catch (error) { - logger.error('[SHELL]- - getAnsibleVersion'); - } -} - -async function saveSshKey(key: string, uuid: string) { - try { - logger.info('[SHELL] - vaultSshKey Starting...'); - if (shell.exec(`echo '${key}' > /tmp/${uuid}.key`).code !== 0) { - throw new Error('[SHELL] - vaultSshKey - Error creating tmp file'); - } - if (shell.chmod(600, `/tmp/${uuid}.key`).code !== 0) { - throw new Error('[SHELL] - vaultSshKey - Error chmoding file'); - } - } catch (error) { - logger.error('[SHELL] - vaultSshKey'); - throw error; - } -} - -async function installAnsibleGalaxyCollection(name: string, namespace: string) { - try { - logger.info('[SHELL] - installAnsibleGalaxyCollection Starting...'); - const result = shell.exec(AnsibleGalaxyCmd.getInstallCollectionCmd(name, namespace)); - if (result.code !== 0) { - throw new Error('[SHELL] - installAnsibleGalaxyCollection has failed'); - } - let collectionList = ''; - let i = 0; - while (!collectionList.includes(`${namespace}.${name}`) && i++ < 60) { - await timeout(2000); - shell.exec( - AnsibleGalaxyCmd.getListCollectionsCmd(name, namespace) + - ' > /tmp/ansible-collection-output.tmp.txt', - ); - await timeout(2000); - collectionList = shell.cat('/tmp/ansible-collection-output.tmp.txt').toString(); - } - if (!collectionList.includes(`${namespace}.${name}`)) { - throw new Error('[SHELL] - installAnsibleGalaxyCollection has failed'); - } - } catch (error) { - logger.error('[SHELL] - installAnsibleGalaxyCollection'); - throw error; - } -} +import * as AnsibleShell from './ansible'; +import * as AuthenticationShell from './authentication'; +import * as PlaybookFileShell from './playbook-file'; export default { - executePlaybook, - listPlaybooks, - readPlaybook, - editPlaybook, - newPlaybook, - deletePlaybook, - readPlaybookConfigurationFileIfExists, - getAnsibleVersion, - saveSshKey, - executePlaybookOnInventory, - installAnsibleGalaxyCollection, + AnsibleShell, + AuthenticationShell, + PlaybookFileShell, }; diff --git a/server/src/integrations/shell/playbook-file.ts b/server/src/integrations/shell/playbook-file.ts new file mode 100644 index 00000000..b2b1138c --- /dev/null +++ b/server/src/integrations/shell/playbook-file.ts @@ -0,0 +1,64 @@ +import shell from 'shelljs'; +import logger from '../../logger'; + +async function readPlaybook(playbook: string) { + try { + logger.info(`[SHELL] - readPlaybook - ${playbook} - Starting...`); + return shell.cat(playbook).toString(); + } catch (error) { + logger.error('[SHELL] - readPlaybook'); + throw new Error('readPlaybook failed'); + } +} + +async function readPlaybookConfigurationFileIfExists(path: string) { + try { + logger.info(`[SHELL] - readPlaybookConfiguration - ${path} - Starting...`); + if (!shell.test('-f', `${path}`)) { + logger.info(`[SHELL] - readPlaybookConfiguration - not found`); + return undefined; + } + return shell.cat(`${path}`).toString(); + } catch (error) { + logger.error('[SHELL] - readPlaybookConfiguration'); + throw new Error('readPlaybookConfiguration failed'); + } +} + +async function editPlaybook(playbookPath: string, content: string) { + try { + logger.info('[SHELL] - editPlaybook - Starting...'); + shell.ShellString(content).to(playbookPath); + } catch (error) { + logger.error('[SHELL] - editPlaybook'); + throw new Error('editPlaybook failed'); + } +} + +async function newPlaybook(fullFilePath: string) { + try { + logger.info('[SHELL] - newPlaybook - Starting...'); + shell.touch(fullFilePath); + } catch (error) { + logger.error('[SHELL] - newPlaybook'); + throw new Error('newPlaybook failed'); + } +} + +async function deletePlaybook(playbookPath: string) { + try { + logger.info('[SHELL] - deletePlaybook - Starting...'); + shell.rm(playbookPath); + } catch (error) { + logger.error('[SHELL] - deletePlaybook'); + throw new Error('deletePlaybook failed'); + } +} + +export { + readPlaybookConfigurationFileIfExists, + readPlaybook, + editPlaybook, + newPlaybook, + deletePlaybook, +}; diff --git a/server/src/integrations/shell/utils.ts b/server/src/integrations/shell/utils.ts index f053d572..9aaf53af 100644 --- a/server/src/integrations/shell/utils.ts +++ b/server/src/integrations/shell/utils.ts @@ -1,17 +1,37 @@ -import shell from 'shelljs'; -import logger from '../../logger'; +import path from 'path'; +import fs from 'fs-extra'; +import shell, { ShellString } from 'shelljs'; -export async function createDirectoryWithFullPath(fullPath: string) { +export async function createDirectoryWithFullPath( + fullPath: string, + rootPath?: string, +): Promise { + if (rootPath) { + const filePath = path.resolve(rootPath, fullPath); + if (!filePath.startsWith(rootPath)) { + throw new Error('Attempt to create a file or directory outside the root directory'); + } + } + let fileExist: string | undefined; + try { + fileExist = fs.realpathSync(path.resolve(fullPath)); + } catch {} + if (fileExist) { + throw new Error('Directory or file already exists'); + } return shell.mkdir('-p', fullPath); } -export async function findFilesInDirectory(directory: string, pattern: RegExp) { - return shell.find(directory).filter(function (file) { - logger.debug(`[SHELL][UTILS] - findFilesInDirectory - checking ${file} for pattern ${pattern}`); - return file.match(pattern); - }); +export async function deleteFilesAndDirectory(directory: string, rootPath?: string) { + if (rootPath) { + const filePath = fs.realpathSync(path.resolve(rootPath, directory)); + if (!filePath.startsWith(rootPath)) { + throw new Error('Attempt to delete a file or directory outside the root directory'); + } + } + shell.rm('-rf', directory); } -export async function deleteFilesAndDirectory(directory: string) { - shell.rm('-rf', directory); +export function timeout(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/server/src/services/playbooks-repository/local.ts b/server/src/services/playbooks-repository/local.ts index a6f6d222..ef208ea4 100644 --- a/server/src/services/playbooks-repository/local.ts +++ b/server/src/services/playbooks-repository/local.ts @@ -3,7 +3,6 @@ import { NotFoundError } from '../../core/api/ApiError'; import { SuccessResponse } from '../../core/api/ApiResponse'; import PlaybooksRepositoryRepo from '../../data/database/repository/PlaybooksRepositoryRepo'; import asyncHandler from '../../helpers/AsyncHandler'; -import GitRepositoryComponent from '../../integrations/git-repository/GitRepositoryComponent'; import LocalRepositoryComponent from '../../integrations/local-repository/LocalRepositoryComponent'; import PlaybooksRepositoryEngine from '../../integrations/playbooks-repository/PlaybooksRepositoryEngine'; import logger from '../../logger'; diff --git a/server/src/services/playbooks-repository/platbooks-repository.validator.ts b/server/src/services/playbooks-repository/platbooks-repository.validator.ts index 49654488..194561dd 100644 --- a/server/src/services/playbooks-repository/platbooks-repository.validator.ts +++ b/server/src/services/playbooks-repository/platbooks-repository.validator.ts @@ -4,19 +4,19 @@ import validator from '../../middlewares/validator'; export const addDirectoryToPlaybookRepositoryValidator = [ param('uuid').exists().isString().isUUID().withMessage('Uuid is incorrect'), param('directoryName').exists().isString().withMessage('Name is incorrect'), - body('fullPath').exists().isString().withMessage('path is incorrect'), + body('fullPath').exists().isString().not().contains('..').withMessage('path is incorrect'), validator, ]; export const addPlaybookToRepositoryValidator = [ param('uuid').exists().isString().isUUID().withMessage('Uuid is incorrect'), param('playbookName').exists().isString().withMessage('Name is incorrect'), - body('fullPath').exists().isString().withMessage('path is incorrect'), + body('fullPath').exists().isString().not().contains('..').withMessage('path is incorrect'), validator, ]; export const deleteAnyFromRepositoryValidator = [ param('uuid').exists().isString().isUUID().withMessage('Uuid is incorrect'), - body('fullPath').exists().isString().withMessage('path is incorrect'), + body('fullPath').exists().isString().not().contains('..').withMessage('path is incorrect'), validator, ]; diff --git a/server/src/use-cases/DeviceAuthUseCases.ts b/server/src/use-cases/DeviceAuthUseCases.ts index 97739e3c..30e0bd60 100644 --- a/server/src/use-cases/DeviceAuthUseCases.ts +++ b/server/src/use-cases/DeviceAuthUseCases.ts @@ -4,7 +4,7 @@ import Shell from '../integrations/shell'; async function saveAllDeviceAuthSshKeys() { const devicesAuth = (await DeviceAuthRepo.findAllPopWithSshKey()) || []; for (const deviceAuth of devicesAuth) { - await Shell.saveSshKey(deviceAuth.sshKey as string, deviceAuth.device.uuid); + await Shell.AuthenticationShell.saveSshKey(deviceAuth.sshKey as string, deviceAuth.device.uuid); } } diff --git a/server/src/use-cases/DeviceUseCases.ts b/server/src/use-cases/DeviceUseCases.ts index 9eb39af9..58b414e7 100644 --- a/server/src/use-cases/DeviceUseCases.ts +++ b/server/src/use-cases/DeviceUseCases.ts @@ -141,7 +141,7 @@ async function checkAnsibleConnection( await setToCache(SettingsKeys.AnsibleReservedExtraVarsKeys.MASTER_NODE_URL, masterNodeUrl); } if (sshKey) { - await Shell.saveSshKey(sshKey, 'tmp'); + await Shell.AuthenticationShell.saveSshKey(sshKey, 'tmp'); } const mockedInventoryTarget = Inventory.inventoryBuilderForTarget([ { diff --git a/server/src/use-cases/GitRepositoryUseCases.ts b/server/src/use-cases/GitRepositoryUseCases.ts index e22dca50..d8752ec1 100644 --- a/server/src/use-cases/GitRepositoryUseCases.ts +++ b/server/src/use-cases/GitRepositoryUseCases.ts @@ -1,11 +1,7 @@ -import { Error } from 'mongoose'; -import { v4 as uuidv4 } from 'uuid'; import { Playbooks } from 'ssm-shared-lib'; -import PlaybooksRepository from '../data/database/model/PlaybooksRepository'; -import PlaybookRepo from '../data/database/repository/PlaybookRepo'; +import { v4 as uuidv4 } from 'uuid'; import PlaybooksRepositoryRepo from '../data/database/repository/PlaybooksRepositoryRepo'; import PlaybooksRepositoryEngine from '../integrations/playbooks-repository/PlaybooksRepositoryEngine'; -import playbooksRepository from '../routes/playbooks-repository'; async function addGitRepository( name: string, diff --git a/server/src/use-cases/PlaybookUseCases.ts b/server/src/use-cases/PlaybookUseCases.ts index d1772607..328d7386 100644 --- a/server/src/use-cases/PlaybookUseCases.ts +++ b/server/src/use-cases/PlaybookUseCases.ts @@ -3,7 +3,7 @@ import { setToCache } from '../data/cache'; import Playbook, { PlaybookModel } from '../data/database/model/Playbook'; import User from '../data/database/model/User'; import ExtraVars from '../integrations/ansible/utils/ExtraVars'; -import shell from '../integrations/shell'; +import Shell from '../integrations/shell'; import { Playbooks } from '../types/typings'; async function completeExtraVar( @@ -33,7 +33,12 @@ async function executePlaybook( target, extraVarsForcedValues, ); - return await shell.executePlaybook(playbook.path, user, target, substitutedExtraVars); + return await Shell.AnsibleShell.executePlaybook( + playbook.path, + user, + target, + substitutedExtraVars, + ); } async function executePlaybookOnInventory( @@ -47,7 +52,7 @@ async function executePlaybookOnInventory( undefined, extraVarsForcedValues, ); - return await shell.executePlaybookOnInventory( + return await Shell.AnsibleShell.executePlaybookOnInventory( playbook.path, user, inventoryTargets, diff --git a/server/src/use-cases/PlaybooksRepositoryUseCases.ts b/server/src/use-cases/PlaybooksRepositoryUseCases.ts index f7d8bb5d..57e63bdb 100644 --- a/server/src/use-cases/PlaybooksRepositoryUseCases.ts +++ b/server/src/use-cases/PlaybooksRepositoryUseCases.ts @@ -6,7 +6,7 @@ import PlaybooksRepositoryRepo from '../data/database/repository/PlaybooksReposi import PlaybooksRepositoryComponent from '../integrations/playbooks-repository/PlaybooksRepositoryComponent'; import PlaybooksRepositoryEngine from '../integrations/playbooks-repository/PlaybooksRepositoryEngine'; import { recursiveTreeCompletion } from '../integrations/playbooks-repository/tree-utils'; -import shell from '../integrations/shell'; +import Shell from '../integrations/shell'; import { createDirectoryWithFullPath, deleteFilesAndDirectory } from '../integrations/shell/utils'; import logger from '../logger'; @@ -53,9 +53,9 @@ async function createDirectoryInPlaybookRepository( throw new InternalError('Repository is not registered, try restarting or force sync'); } if (!playbooksRepositoryComponent.fileBelongToRepository(path)) { - throw new ForbiddenError('The selected path doesnt seems to belong to the repository'); + throw new ForbiddenError("The selected path doesn't seems to belong to the repository"); } - await createDirectoryWithFullPath(path); + await createDirectoryWithFullPath(path, playbooksRepositoryComponent.rootPath); await playbooksRepositoryComponent.updateDirectoriesTree(); } @@ -80,7 +80,7 @@ async function createPlaybookInRepository( playbooksRepository: playbooksRepository, playableInBatch: true, }); - await shell.newPlaybook(fullPath); + await Shell.PlaybookFileShell.newPlaybook(fullPath + '.yml'); await playbooksRepositoryComponent.syncToDatabase(); return playbook; } @@ -93,7 +93,7 @@ async function deletePlaybooksInRepository(playbook: Playbook) { throw new InternalError(`PlaybookRepository doesnt seem registered`); } await PlaybookModel.deleteOne({ uuid: playbook.uuid }); - await shell.deletePlaybook(playbook.path); + await Shell.PlaybookFileShell.deletePlaybook(playbook.path); await playbooksRepositoryComponent.syncToDatabase(); } @@ -110,7 +110,7 @@ async function deleteAnyInPlaybooksRepository( if (!playbooksRepositoryComponent.fileBelongToRepository(fullPath)) { throw new ForbiddenError('The selected path doesnt seems to belong to the repository'); } - await deleteFilesAndDirectory(fullPath); + await deleteFilesAndDirectory(fullPath, playbooksRepositoryComponent.rootPath); await playbooksRepositoryComponent.syncToDatabase(); } @@ -122,9 +122,10 @@ async function deleteRepository(repository: PlaybooksRepository): Promise throw new InternalError(`PlaybookRepository doesnt seem registered`); } const directory = playbooksRepositoryComponent.getDirectory(); + const rootPath = playbooksRepositoryComponent.rootPath; await PlaybooksRepositoryEngine.deregisterRepository(repository.uuid); await PlaybooksRepositoryRepo.deleteByUuid(repository.uuid); - await deleteFilesAndDirectory(directory); + await deleteFilesAndDirectory(directory, rootPath); } export default { From 2a32f56f57bdf6ff417b114503225327d2a9e29e Mon Sep 17 00:00:00 2001 From: manu Date: Mon, 1 Jul 2024 13:46:20 +0200 Subject: [PATCH 10/11] Update codebase for increased functionality and efficiency This commit introduces substantial updates across several different files. Changes include aligning directory paths with updated UUIDs, integrating migration instructions with Scheme version tracking, improving Shell methods imports, standardising tree building across several components, ensuring unique name for Playbook model, and modifying Git commit messages to reflect updates through SSM. The updates aim to improve functionality, increase efficiency, and fix previous issues. --- .../CreateFileInRepositoryModalForm.tsx | 1 + .../components/DirectoryTreeView.tsx | 73 +++++++++- .../components/NewFileDrawerForm.tsx | 1 + .../components/PlaybookDropdownMenu.tsx | 23 +++- .../Playbooks/components/TreeComponent.tsx | 125 ++++++------------ client/src/pages/Playbooks/index.tsx | 11 +- server/src/core/startup/index.ts | 18 ++- server/src/core/system/version.ts | 4 +- server/src/data/database/model/Playbook.ts | 1 + .../git-repository/lib/commitAndSync.ts | 6 +- server/src/services/devices/device.ts | 4 +- server/src/services/devices/deviceauth.ts | 2 +- .../src/services/playbooks-repository/git.ts | 2 + server/src/services/playbooks/playbook.ts | 6 +- server/src/use-cases/GitRepositoryUseCases.ts | 1 + .../src/use-cases/LocalRepositoryUseCases.ts | 2 +- .../use-cases/PlaybooksRepositoryUseCases.ts | 2 +- 17 files changed, 169 insertions(+), 113 deletions(-) diff --git a/client/src/pages/Playbooks/components/CreateFileInRepositoryModalForm.tsx b/client/src/pages/Playbooks/components/CreateFileInRepositoryModalForm.tsx index 1606fb08..1171087b 100644 --- a/client/src/pages/Playbooks/components/CreateFileInRepositoryModalForm.tsx +++ b/client/src/pages/Playbooks/components/CreateFileInRepositoryModalForm.tsx @@ -20,6 +20,7 @@ export type CreateFileInRepositoryModalFormProps = { const CreateFileInRepositoryModalForm: React.FC< CreateFileInRepositoryModalFormProps > = (props) => { + console.log(`path: ${props.path}, basedPath: ${props.basedPath}`); return ( title={`Create a new ${props.mode}`} diff --git a/client/src/pages/Playbooks/components/DirectoryTreeView.tsx b/client/src/pages/Playbooks/components/DirectoryTreeView.tsx index a50a747f..f78a2528 100644 --- a/client/src/pages/Playbooks/components/DirectoryTreeView.tsx +++ b/client/src/pages/Playbooks/components/DirectoryTreeView.tsx @@ -1,14 +1,31 @@ import GalaxyStoreModal from '@/pages/Playbooks/components/GalaxyStoreModal'; import CreateFileInRepositoryModalForm from '@/pages/Playbooks/components/CreateFileInRepositoryModalForm'; import NewFileDrawerForm from '@/pages/Playbooks/components/NewFileDrawerForm'; +import PlaybookDropdownMenu from '@/pages/Playbooks/components/PlaybookDropdownMenu'; import { ClientPlaybooksTrees } from '@/pages/Playbooks/components/TreeComponent'; import { AppstoreOutlined } from '@ant-design/icons'; -import { Button, Card, Tree } from 'antd'; +import { Button, Card, Tree, Typography } from 'antd'; import React, { Key } from 'react'; -import { API } from 'ssm-shared-lib'; +import { API, DirectoryTree as SSMDirectoryTree } from 'ssm-shared-lib'; const { DirectoryTree } = Tree; +export type Callbacks = { + callbackCreateDirectory: ( + path: string, + playbookRepositoryUuid: string, + playbookRepositoryName: string, + playbookRepositoryBasePath: string, + ) => void; + callbackCreatePlaybook: ( + path: string, + playbookRepositoryUuid: string, + playbookRepositoryName: string, + playbookRepositoryBasePath: string, + ) => void; + callbackDeleteFile: (path: string, playbookRepositoryUuid: string) => void; +}; + type DirectoryTreeViewProps = { onSelect: ( selectedKeys: Key[], @@ -37,11 +54,12 @@ type DirectoryTreeViewProps = { mode: 'directory' | 'playbook', ) => Promise; selectedFile?: API.PlaybookFile; + callbacks: Callbacks; }; const DirectoryTreeView: React.FC = (props) => { const [storeModal, setStoreModal] = React.useState(false); - const [selectedPath, setSelectedPath] = React.useState(''); + const [selectedPath, setSelectedPath] = React.useState([]); const { onSelect, selectedFile, @@ -51,6 +69,7 @@ const DirectoryTreeView: React.FC = (props) => { setNewRepositoryFileModal, } = props; // @ts-ignore + return ( = (props) => { onSelect={onSelect} treeData={playbookRepositories} selectedKeys={[selectedFile?.path as React.Key]} - expandedKeys={[selectedPath as React.Key]} + titleRender={(node) => { + if (node.nodeType === SSMDirectoryTree.CONSTANTS.DIRECTORY) { + return ( + + + {node._name} + + + ); + } else { + return ( + + + {node._name} + + + ); + } + }} /> = (props) => { } drawerProps={{ destroyOnClose: true, + onClose: () => props.setSelectedNode([]), }} onFinish={async (values) => { setLoading(true); diff --git a/client/src/pages/Playbooks/components/PlaybookDropdownMenu.tsx b/client/src/pages/Playbooks/components/PlaybookDropdownMenu.tsx index b45b64ff..0a2b1f4d 100644 --- a/client/src/pages/Playbooks/components/PlaybookDropdownMenu.tsx +++ b/client/src/pages/Playbooks/components/PlaybookDropdownMenu.tsx @@ -1,4 +1,8 @@ -import { Callbacks } from '@/pages/Playbooks/components/TreeComponent'; +import { Callbacks } from '@/pages/Playbooks/components/DirectoryTreeView'; +import { + commitAndSyncGitRepository, + forcePullGitRepository, +} from '@/services/rest/playbooks-repositories'; import { ArrowDownOutlined, DeleteOutlined, @@ -6,7 +10,7 @@ import { FolderOpenOutlined, SyncOutlined, } from '@ant-design/icons'; -import { Dropdown, MenuProps, Popconfirm } from 'antd'; +import { Dropdown, MenuProps, message, Popconfirm } from 'antd'; import React from 'react'; import { DirectoryTree } from 'ssm-shared-lib'; @@ -104,6 +108,21 @@ const PlaybookDropdownMenu: React.FC = (props) => { case '3': setOpen(true); break; + case '4': + await commitAndSyncGitRepository(props.playbookRepository.uuid).then( + () => { + message.info({ + content: 'Commit & sync command sent', + duration: 6, + }); + }, + ); + break; + case '5': + await forcePullGitRepository(props.playbookRepository.uuid).then(() => { + message.info({ content: 'Force pull command sent', duration: 6 }); + }); + break; } }; diff --git a/client/src/pages/Playbooks/components/TreeComponent.tsx b/client/src/pages/Playbooks/components/TreeComponent.tsx index 93f1deaa..7c41511c 100644 --- a/client/src/pages/Playbooks/components/TreeComponent.tsx +++ b/client/src/pages/Playbooks/components/TreeComponent.tsx @@ -15,57 +15,35 @@ import PlaybookDropdownMenu from './PlaybookDropdownMenu'; export type ClientPlaybooksTrees = { isLeaf?: boolean; _name: string; - title: ReactNode; + nodeType: DirectoryTree.CONSTANTS; children?: ClientPlaybooksTrees[]; + rootNode?: boolean; + remoteRootNode?: boolean; + playbookRepository: { uuid: string; name: string; basePath: string }; + depth: number; key: string; uuid?: string; extraVars?: API.ExtraVar[]; custom?: boolean; selectable?: boolean; -}; - -export type Callbacks = { - callbackCreateDirectory: ( - path: string, - playbookRepositoryUuid: string, - playbookRepositoryName: string, - playbookRepositoryBasePath: string, - ) => void; - callbackCreatePlaybook: ( - path: string, - playbookRepositoryUuid: string, - playbookRepositoryName: string, - playbookRepositoryBasePath: string, - ) => void; - callbackDeleteFile: (path: string, playbookRepositoryUuid: string) => void; + icon?: ReactNode; }; export function buildTree( rootNode: API.PlaybooksRepository, - callbacks: Callbacks, -) { +): ClientPlaybooksTrees { return { _name: rootNode.name, - title: ( - - - {rootNode.name} -           - - - ), + remoteRootNode: rootNode.type === Playbooks.PlaybooksRepositoryType.GIT, + depth: 0, + rootNode: true, + playbookRepository: { + basePath: rootNode.path, + name: rootNode.name, + uuid: rootNode.uuid, + }, key: rootNode.name, + nodeType: DT.CONSTANTS.DIRECTORY, icon: rootNode.type === Playbooks.PlaybooksRepositoryType.LOCAL ? ( @@ -81,7 +59,6 @@ export function buildTree( path: rootNode.name, }, { uuid: rootNode.uuid, name: rootNode.name, basePath: rootNode.path }, - callbacks, ) : undefined, }; @@ -90,7 +67,6 @@ export function buildTree( export function recursiveTreeTransform( tree: DirectoryTree.ExtendedTreeNode, playbookRepository: { uuid: string; name: string; basePath: string }, - callbacks: Callbacks, depth = 0, ): ClientPlaybooksTrees[] { const node = tree; @@ -106,26 +82,17 @@ export function recursiveTreeTransform( newTree.push({ key: child.path, _name: child.name, - title: ( - - - {child.name} - - - ), + nodeType: child.type, + playbookRepository: { + basePath: playbookRepository.basePath, + name: playbookRepository.name, + uuid: playbookRepository.uuid, + }, + depth: depth, selectable: false, children: recursiveTreeTransform( child, playbookRepository, - callbacks, depth + 1, ), }); @@ -134,22 +101,13 @@ export function recursiveTreeTransform( newTree.push({ key: child.path, _name: child.name, - title: ( - - - {child.name} - - - ), + nodeType: DirectoryTree.CONSTANTS.FILE, + playbookRepository: { + basePath: playbookRepository.basePath, + name: playbookRepository.name, + uuid: playbookRepository.uuid, + }, + depth: depth, uuid: (child as DirectoryTree.ExtendedTreeNode).uuid, extraVars: (child as DirectoryTree.ExtendedTreeNode).extraVars, custom: (child as DirectoryTree.ExtendedTreeNode).custom, @@ -162,22 +120,13 @@ export function recursiveTreeTransform( newTree.push({ key: node.path, _name: node.name, - title: ( - - - {node.name} - - - ), + nodeType: DirectoryTree.CONSTANTS.FILE, + playbookRepository: { + basePath: playbookRepository.basePath, + name: playbookRepository.name, + uuid: playbookRepository.uuid, + }, + depth: depth, uuid: node.uuid, extraVars: node.extraVars, custom: node.custom, diff --git a/client/src/pages/Playbooks/index.tsx b/client/src/pages/Playbooks/index.tsx index ab5ca4e5..704c7ae3 100644 --- a/client/src/pages/Playbooks/index.tsx +++ b/client/src/pages/Playbooks/index.tsx @@ -142,11 +142,7 @@ const Index: React.FC = () => { if (list?.data) { setPlaybookRepositories( list.data.map((e: API.PlaybooksRepository) => { - return buildTree(e, { - callbackCreatePlaybook: handleShouldCreatePlaybook, - callbackCreateDirectory: handleShouldCreateRepository, - callbackDeleteFile: handleShouldDeleteFile, - }); + return buildTree(e); }), ); if (createdPlaybook) { @@ -313,6 +309,11 @@ const Index: React.FC = () => { setNewRepositoryFileModal={setNewRepositoryFileModal} createNewFile={createNewFile} selectedFile={selectedFile} + callbacks={{ + callbackCreatePlaybook: handleShouldCreatePlaybook, + callbackCreateDirectory: handleShouldCreateRepository, + callbackDeleteFile: handleShouldDeleteFile, + }} /> diff --git a/server/src/core/startup/index.ts b/server/src/core/startup/index.ts index 33da2282..5102824c 100644 --- a/server/src/core/startup/index.ts +++ b/server/src/core/startup/index.ts @@ -1,6 +1,9 @@ +import { Model } from 'mongoose'; import { Playbooks, SettingsKeys } from 'ssm-shared-lib'; import { getFromCache } from '../../data/cache'; import initRedisValues from '../../data/cache/defaults'; +import { connection } from '../../data/database'; +import { PlaybookModel } from '../../data/database/model/Playbook'; import PlaybooksRepositoryRepo from '../../data/database/repository/PlaybooksRepositoryRepo'; import Crons from '../../integrations/crons'; import WatcherEngine from '../../integrations/docker/core/WatcherEngine'; @@ -16,7 +19,7 @@ const corePlaybooksRepository = { uuid: '00000000-0000-0000-0000-000000000000', enabled: true, type: Playbooks.PlaybooksRepositoryType.LOCAL, - directory: '/server/src/ansible', + directory: '/server/src/ansible/00000000-0000-0000-0000-000000000000', default: true, }; @@ -25,7 +28,7 @@ const toolsPlaybooksRepository = { uuid: '00000000-0000-0000-0000-000000000001', enabled: true, type: Playbooks.PlaybooksRepositoryType.LOCAL, - directory: '/server/src/ansible', + directory: '/server/src/ansible/00000000-0000-0000-0000-000000000001', default: true, }; @@ -41,7 +44,8 @@ async function init() { void Crons.initScheduledJobs(); void WatcherEngine.init(); - if (version !== SettingsKeys.DefaultValue.SCHEME_VERSION) { + if (version !== SettingsKeys.DefaultValue.SCHEME_VERSION + 1) { + await migrate(); logger.warn(`[CONFIGURATION] - Scheme version differed, starting writing updates`); await initRedisValues(); void setAnsibleVersion(); @@ -54,6 +58,14 @@ async function init() { } } +async function migrate() { + try { + await PlaybookModel.syncIndexes(); + } catch (error: any) { + logger.error(error); + } +} + export default { init, }; diff --git a/server/src/core/system/version.ts b/server/src/core/system/version.ts index 85015ce8..84cf1f44 100644 --- a/server/src/core/system/version.ts +++ b/server/src/core/system/version.ts @@ -6,7 +6,7 @@ export async function getAnsibleVersion() { if (ansibleVersion) { return ansibleVersion; } else { - const retrievedAnsibleVersion = await Shell.getAnsibleVersion(); + const retrievedAnsibleVersion = await Shell.AnsibleShell.getAnsibleVersion(); if (retrievedAnsibleVersion) { await setToCache('ansible-version', retrievedAnsibleVersion); } @@ -15,7 +15,7 @@ export async function getAnsibleVersion() { } export async function setAnsibleVersion() { - const retrievedAnsibleVersion = await Shell.getAnsibleVersion(); + const retrievedAnsibleVersion = await Shell.AnsibleShell.getAnsibleVersion(); if (retrievedAnsibleVersion) { await setToCache('ansible-version', retrievedAnsibleVersion); } diff --git a/server/src/data/database/model/Playbook.ts b/server/src/data/database/model/Playbook.ts index 333e63e8..34246141 100644 --- a/server/src/data/database/model/Playbook.ts +++ b/server/src/data/database/model/Playbook.ts @@ -21,6 +21,7 @@ const schema = new Schema( name: { type: Schema.Types.String, required: true, + unique: true, }, uuid: { type: Schema.Types.String, diff --git a/server/src/integrations/git-repository/lib/commitAndSync.ts b/server/src/integrations/git-repository/lib/commitAndSync.ts index 9bb7f29c..f13dcf15 100644 --- a/server/src/integrations/git-repository/lib/commitAndSync.ts +++ b/server/src/integrations/git-repository/lib/commitAndSync.ts @@ -1,4 +1,5 @@ import { GitProcess } from 'dugite'; +import myLogger from '../../../logger'; import { credentialOff, credentialOn } from './credential'; import { defaultGitInfo as defaultDefaultGitInfo } from './defaultGitInfo'; import { @@ -44,7 +45,7 @@ export async function commitAndSync(options: ICommitAndSyncOptions): Promise { becomePass: becomePass ? await vaultEncrypt(becomePass, DEFAULT_VAULT_ID) : undefined, } as DeviceAuth); if (sshKey) { - await Shell.saveSshKey(sshKey, createdDevice.uuid); + await Shell.AuthenticationShell.saveSshKey(sshKey, createdDevice.uuid); } logger.info(`[CONTROLLER] Device - Created device with uuid: ${createdDevice.uuid}`); new SuccessResponse('Add device successful', { device: createdDevice as API.DeviceItem }).send( diff --git a/server/src/services/devices/deviceauth.ts b/server/src/services/devices/deviceauth.ts index 0c7c00eb..73d71232 100644 --- a/server/src/services/devices/deviceauth.ts +++ b/server/src/services/devices/deviceauth.ts @@ -113,7 +113,7 @@ export const addOrUpdateDeviceAuth = asyncHandler(async (req, res) => { becomeUser: becomeUser, } as DeviceAuth); if (sshKey) { - await Shell.saveSshKey(sshKey, device.uuid); + await Shell.AuthenticationShell.saveSshKey(sshKey, device.uuid); } logger.info( `[CONTROLLER] - POST - Device Auth - Updated or Created device with uuid: ${device.uuid}`, diff --git a/server/src/services/playbooks-repository/git.ts b/server/src/services/playbooks-repository/git.ts index da0da251..f61e1e6f 100644 --- a/server/src/services/playbooks-repository/git.ts +++ b/server/src/services/playbooks-repository/git.ts @@ -101,6 +101,7 @@ export const forcePullRepository = asyncHandler(async (req, res) => { throw new NotFoundError(); } await repository.forcePull(); + await repository.syncToDatabase(); return new SuccessResponse('Forced pull playbooks git repository').send(res); }); @@ -114,6 +115,7 @@ export const forceCloneRepository = asyncHandler(async (req, res) => { throw new NotFoundError(); } await repository.clone(); + await repository.syncToDatabase(); return new SuccessResponse('Forced cloned playbooks git repository').send(res); }); diff --git a/server/src/services/playbooks/playbook.ts b/server/src/services/playbooks/playbook.ts index 34d1f929..05ea5d07 100644 --- a/server/src/services/playbooks/playbook.ts +++ b/server/src/services/playbooks/playbook.ts @@ -3,7 +3,7 @@ import { SuccessResponse } from '../../core/api/ApiResponse'; import PlaybookRepo from '../../data/database/repository/PlaybookRepo'; import asyncHandler from '../../helpers/AsyncHandler'; import logger from '../../logger'; -import shell from '../../integrations/shell'; +import Shell from '../../integrations/shell'; import PlaybooksRepositoryUseCases from '../../use-cases/PlaybooksRepositoryUseCases'; import PlaybookUseCases from '../../use-cases/PlaybookUseCases'; @@ -24,7 +24,7 @@ export const getPlaybook = asyncHandler(async (req, res) => { throw new NotFoundError(`Playbook ${uuid} not found`); } try { - const content = await shell.readPlaybook(playbook.path); + const content = await Shell.PlaybookFileShell.readPlaybook(playbook.path); new SuccessResponse('Get Playbook successful', content).send(res); } catch (error: any) { throw new InternalError(error.message); @@ -39,7 +39,7 @@ export const editPlaybook = asyncHandler(async (req, res) => { throw new NotFoundError(`Playbook ${uuid} not found`); } try { - await shell.editPlaybook(playbook.path, req.body.content); + await Shell.PlaybookFileShell.editPlaybook(playbook.path, req.body.content); new SuccessResponse('Edit playbook successful').send(res); } catch (error: any) { throw new InternalError(error.message); diff --git a/server/src/use-cases/GitRepositoryUseCases.ts b/server/src/use-cases/GitRepositoryUseCases.ts index d8752ec1..c26ded9c 100644 --- a/server/src/use-cases/GitRepositoryUseCases.ts +++ b/server/src/use-cases/GitRepositoryUseCases.ts @@ -1,6 +1,7 @@ import { Playbooks } from 'ssm-shared-lib'; import { v4 as uuidv4 } from 'uuid'; import PlaybooksRepositoryRepo from '../data/database/repository/PlaybooksRepositoryRepo'; +import GitRepositoryComponent from '../integrations/git-repository/GitRepositoryComponent'; import PlaybooksRepositoryEngine from '../integrations/playbooks-repository/PlaybooksRepositoryEngine'; async function addGitRepository( diff --git a/server/src/use-cases/LocalRepositoryUseCases.ts b/server/src/use-cases/LocalRepositoryUseCases.ts index fe0d2f12..0a2b40ed 100644 --- a/server/src/use-cases/LocalRepositoryUseCases.ts +++ b/server/src/use-cases/LocalRepositoryUseCases.ts @@ -19,7 +19,7 @@ async function addLocalRepository(name: string) { uuid, type: Playbooks.PlaybooksRepositoryType.LOCAL, name, - directory: DIRECTORY_ROOT, + directory: localRepository.getDirectory(), enabled: true, }); try { diff --git a/server/src/use-cases/PlaybooksRepositoryUseCases.ts b/server/src/use-cases/PlaybooksRepositoryUseCases.ts index 57e63bdb..c8e5276e 100644 --- a/server/src/use-cases/PlaybooksRepositoryUseCases.ts +++ b/server/src/use-cases/PlaybooksRepositoryUseCases.ts @@ -27,7 +27,7 @@ async function getAllPlaybooksRepositories() { children: await recursiveTreeCompletion(playbookRepository.tree), type: playbookRepository.type, uuid: playbookRepository.uuid, - path: playbookRepository.directory + '/' + playbookRepository.uuid, + path: playbookRepository.directory, default: playbookRepository.default, }; }, From c3424194ae3b5e255ce1757e5642c1e00b947cc7 Mon Sep 17 00:00:00 2001 From: manu Date: Mon, 1 Jul 2024 13:48:34 +0200 Subject: [PATCH 11/11] Refactor ansible shell call in galaxy.ts Adjusted the function call to install an Ansible Galaxy Collection in the galaxy.ts file. The updated version adds a necessary layer to the function call (`AnsibleShell`) ensuring the correct execution of the installAnsibleGalaxyCollection method. --- server/src/services/playbooks/galaxy.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/services/playbooks/galaxy.ts b/server/src/services/playbooks/galaxy.ts index 31f75ee3..056b26b7 100644 --- a/server/src/services/playbooks/galaxy.ts +++ b/server/src/services/playbooks/galaxy.ts @@ -1,9 +1,9 @@ -import axios from 'axios'; import { parse } from 'url'; +import axios from 'axios'; +import { API } from 'ssm-shared-lib'; import { InternalError } from '../../core/api/ApiError'; import { SuccessResponse } from '../../core/api/ApiResponse'; import asyncHandler from '../../helpers/AsyncHandler'; -import { API } from 'ssm-shared-lib'; import Shell from '../../integrations/shell'; import logger from '../../logger'; @@ -50,7 +50,7 @@ export const getAnsibleGalaxyCollection = asyncHandler(async (req, res) => { export const postInstallAnsibleGalaxyCollection = asyncHandler(async (req, res) => { const { name, namespace } = req.body; try { - await Shell.installAnsibleGalaxyCollection(name, namespace); + await Shell.AnsibleShell.installAnsibleGalaxyCollection(name, namespace); } catch (error: any) { throw new InternalError(error.message); }