diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index 219014fa9..78f4c5274 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -36,13 +36,13 @@ jobs: cd client/ yarn install yarn build - yarn configapp dev - name: Run client unit and integration tests run: | cd client/ yarn install - yarn test -u - yarn test -i + yarn config:test + yarn test:int + yarn test:unit - name: Upload unit and integration test coverage to Codecov uses: codecov/codecov-action@v3 diff --git a/.github/workflows/lib-ms.yml b/.github/workflows/lib-ms.yml index 7bc6cdf2d..f56d2965d 100644 --- a/.github/workflows/lib-ms.yml +++ b/.github/workflows/lib-ms.yml @@ -39,12 +39,27 @@ jobs: yarn install yarn build - - name: Run unit and integration tests + - name: Run unit tests run: | cd servers/lib yarn install - yarn test -u - yarn test -i + yarn test:unit + env: + PORT: 4001 + LOCAL_PATH: ${{ github.workspace }}/files + GITLAB_GROUP: dtaas + GITLAB_URL: https://gitlab.com/api/graphql + TOKEN: token + MODE: gitlab + LOG_LEVEL: debug + APOLLO_PATH: /lib + + - name: Run integration tests + if: ${{ always() }} + run: | + cd servers/lib + yarn install + yarn test:int env: PORT: 4001 LOCAL_PATH: ${{ github.workspace }}/files diff --git a/client/README.md b/client/README.md index 07d34ffe6..d79e1597f 100644 --- a/client/README.md +++ b/client/README.md @@ -21,22 +21,24 @@ yarn graph # generate dependency graphs in the code yarn build #build the react app into build/ directory yarn develop #start the development server without building. Great for live edits. +#Required: Choose one config for application (mandatory) +yarn config:dev +yarn config:prod +yarn config:test -#Required: Specify the environment; specify only one -yarn configapp [prod | dev] #If not specified, the app wont run. - -yarn start #start the application -yarn clean #clean the directory of temporary files +yarn start #start the application +yarn clean #clean the directory of temporary files ``` It is also possible to run different types of tests using the yarn -test command by passing different flags: +test commands: ```bash -yarn test -a #run all tests -yarn test -u #run unit tests -yarn test -i #run integration tests -yarn test -e #run end-to-end tests +#Tests require installation of devDependencies +yarn test:unit #run unit tests +yarn test:int #run integration tests +yarn test:e2e #run end-to-end tests +yarn test:all #run all tests ``` --- @@ -74,8 +76,8 @@ yarn configapp `yarn build`, will be used in the build. It is therefore reccommend to keep the configurations in the -`client/config/` directory and use the `yarn configapp` command to -switch between them. +`client/config/` directory and use one of the `yarn config` +sub-commands to switch between them. ## Example configuration for developers @@ -99,13 +101,14 @@ window.env = { REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', - REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', + REACT_APP_AUTH_AUTHORITY: 'https://gitlab.com/', REACT_APP_REDIRECT_URI: 'http://localhost:4000/Library', REACT_APP_LOGOUT_REDIRECT_URI: 'http://localhost:4000/', REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', ``` -Here the gitlab URL is `https://gitlab.foo.com/` and the client ID is `934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0`. +Here the gitlab URL is `https://gitlab.com/` and the client ID is +`934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0`. If a basename is needed for the client application, then the configuration file needs to be updated with the basename. For example, with a basename of @@ -125,25 +128,31 @@ window.env = { REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', - REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', + REACT_APP_AUTH_AUTHORITY: 'https://gitlab.com/', REACT_APP_REDIRECT_URI: 'http://localhost:4000/au/Library', REACT_APP_LOGOUT_REDIRECT_URI: 'http://localhost:4000/au', REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', ``` +It is sufficient to have +[user-owned oauth](https://docs.gitlab.com/ee/integration/oauth_provider.html#create-a-user-owned-application) +application. You can create this application in your gitlab account. Do remember that the oauth application on gitlab needs to have the redirect (callback) URL is correctly registered. -## Caveat - -The client website relies on the background services to provide most of the -functionality. These services would not be running on the developer computer. -The complete application setup exists either on the integration server or as an -installation instance. During development, there will be +**NOTE**: The development environment does not have user workspaces and +traefik gateway running in the background. As a consequence, the iframe +links pointing to user workspace will not work correctly. Instead, you +will see the following error. ```txt Unexpected Application Error! 404 Not Found ``` -error on the **Library** and **Digital Twins** pages. This is expected. +This error can be seen on the **Library** and **Digital Twins** pages. +This error is expected. + +If you want to do _client/_ development with a valid user workspace, +please perform [localhost](../docs/admin/localhost.md) installation, +and then perform client development within the installed environment. diff --git a/client/config/dev.js b/client/config/dev.js index 3c73cc2bf..a6f2ccc8d 100644 --- a/client/config/dev.js +++ b/client/config/dev.js @@ -1,18 +1,20 @@ -window.env = { - REACT_APP_ENVIRONMENT: 'dev', - REACT_APP_URL: 'https://foo.com/', - REACT_APP_URL_BASENAME: 'dtaas', - REACT_APP_URL_DTLINK: '/lab', - REACT_APP_URL_LIBLINK: '', - REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', - REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', - REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', - REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', - REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', +if (typeof window !== 'undefined') { + window.env = { + REACT_APP_ENVIRONMENT: 'dev', + REACT_APP_URL: 'https://foo.com/', + REACT_APP_URL_BASENAME: 'dtaas', + REACT_APP_URL_DTLINK: '/lab', + REACT_APP_URL_LIBLINK: '', + REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', + REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', + REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', + REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', + REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', - REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', - REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', - REACT_APP_REDIRECT_URI: 'https://foo.com/Library', - REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/', - REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', -}; \ No newline at end of file + REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', + REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', + REACT_APP_REDIRECT_URI: 'https://foo.com/Library', + REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/', + REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + }; +}; diff --git a/client/config/prod.js b/client/config/prod.js index 86eabbaec..8b9957232 100644 --- a/client/config/prod.js +++ b/client/config/prod.js @@ -1,18 +1,20 @@ -window.env = { - REACT_APP_ENVIRONMENT: 'prod', - REACT_APP_URL: 'https://foo.com/', - REACT_APP_URL_BASENAME: 'dtaas', - REACT_APP_URL_DTLINK: '/lab', - REACT_APP_URL_LIBLINK: '', - REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', - REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', - REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', - REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', - REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', +if (typeof window !== 'undefined') { + window.env = { + REACT_APP_ENVIRONMENT: 'prod', + REACT_APP_URL: 'https://foo.com/', + REACT_APP_URL_BASENAME: 'dtaas', + REACT_APP_URL_DTLINK: '/lab', + REACT_APP_URL_LIBLINK: '', + REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', + REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', + REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', + REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', + REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', - REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', - REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', - REACT_APP_REDIRECT_URI: 'https://foo.com/Library', - REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/', - REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', + REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', + REACT_APP_REDIRECT_URI: 'https://foo.com/Library', + REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/', + REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + }; }; diff --git a/client/config/test.js b/client/config/test.js index c6db9e06e..d241e082a 100644 --- a/client/config/test.js +++ b/client/config/test.js @@ -1,18 +1,20 @@ -window.env = { - REACT_APP_ENVIRONMENT: 'dev', - REACT_APP_URL: 'http://localhost:4000/', - REACT_APP_URL_BASENAME: '', - REACT_APP_URL_DTLINK: '/lab', - REACT_APP_URL_LIBLINK: '', - REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', - REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', - REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', - REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', - REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', +if (typeof window !== 'undefined') { + window.env = { + REACT_APP_ENVIRONMENT: 'test', + REACT_APP_URL: 'http://localhost:4000/', + REACT_APP_URL_BASENAME: '', + REACT_APP_URL_DTLINK: '/lab', + REACT_APP_URL_LIBLINK: '', + REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', + REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', + REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', + REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', + REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', - REACT_APP_CLIENT_ID: '1be55736756190b3ace4c2c4fb19bde386d1dcc748d20b47ea8cfb5935b8446c', - REACT_APP_AUTH_AUTHORITY: 'https://gitlab.com/', - REACT_APP_REDIRECT_URI: 'http://localhost:4000/Library', - REACT_APP_LOGOUT_REDIRECT_URI: 'http://localhost:4000/', - REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', -}; \ No newline at end of file + REACT_APP_CLIENT_ID: '1be55736756190b3ace4c2c4fb19bde386d1dcc748d20b47ea8cfb5935b8446c', + REACT_APP_AUTH_AUTHORITY: 'https://gitlab.com/', + REACT_APP_REDIRECT_URI: 'http://localhost:4000/Library', + REACT_APP_LOGOUT_REDIRECT_URI: 'http://localhost:4000/', + REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + }; +}; diff --git a/client/package.json b/client/package.json index c51375924..2852c2a15 100644 --- a/client/package.json +++ b/client/package.json @@ -15,15 +15,21 @@ "private": false, "type": "module", "scripts": { - "build": "react-scripts build", + "build": "npx react-scripts build", "clean": "npx rimraf build/ node_modules/ coverage/ playwright-report/ *.svg", - "configapp": "script/config.bash", + "config:dev": "npx shx cp config/dev.js public/env.js && npx shx cp config/dev.js build/env.js", + "config:prod": "npx shx cp config/prod.js public/env.js && npx shx cp config/prod.js build/env.js", + "config:test": "npx shx cp config/test.js public/env.js && npx shx cp config/test.js build/env.js", "develop": "npx react-scripts start", "format": "prettier --ignore-path ../.gitignore --write \"**/*.{ts,tsx,css,scss}\"", "graph": "npx madge --image src.svg src && npx madge --image test.svg test", "start": "serve -s build -l 4000", + "stop": "npx kill-port 4000", "syntax": "npx eslint . --fix", - "test": "script/test.bash" + "test:all": "yarn test:unit && yarn test:int && yarn test:e2e", + "test:e2e": "yarn build && yarn configapp:test && npx kill-port 4000 && yarn start >/dev/null & playwright test && npx kill-port 4000", + "test:int": "jest -c ./jest.integration.config.json ../test/integration --coverage", + "test:unit": "jest -c ./jest.config.json ../test/unitTests --coverage" }, "eslintConfig": { "extends": [ @@ -40,33 +46,33 @@ "@mui/icons-material": "^5.14.8", "@mui/material": "^5.14.8", "@reduxjs/toolkit": "^1.9.7", + "@types/styled-components": "^5.1.32", + "@typescript-eslint/eslint-plugin": "^6.12.0", + "@typescript-eslint/parser": "^6.12.0", "dotenv": "^16.1.4", + "eslint": "^8.54.0", + "eslint-config-airbnb-base": "^15.0.0", + "eslint-config-prettier": "^9.0.0", + "eslint-plugin-import": "^2.29.0", + "eslint-plugin-jest": "^27.6.0", + "eslint-plugin-jsx-a11y": "^6.8.0", + "eslint-plugin-react": "^7.33.2", "oidc-client-ts": "^2.2.2", "prop-types": "^15.8.1", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-iframe": "^1.8.5", "react-is": "^18.2.0", "react-oidc-context": "^2.2.2", "react-redux": "^8.1.3", "react-router-dom": "^6.20.0", + "react-scripts": "^5.0.1", "react-tabs": "^6.0.2", "redux": "^4.2.1", "resize-observer-polyfill": "^1.5.1", "serve": "^14.2.1", "styled-components": "^6.1.1", - "typescript": "^4.9.5", - "react-iframe": "^1.8.5", - "eslint-config-airbnb-base": "^15.0.0", - "eslint-config-prettier": "^9.0.0", - "eslint-plugin-jest": "^27.6.0", - "@types/styled-components": "^5.1.32", - "eslint": "^8.54.0", - "eslint-plugin-import": "^2.29.0", - "eslint-plugin-jsx-a11y": "^6.8.0", - "eslint-plugin-react": "^7.33.2", - "@typescript-eslint/eslint-plugin": "^6.12.0", - "@typescript-eslint/parser": "^6.12.0", - "react-scripts": "^5.0.1" + "typescript": "^4.9.5" }, "devDependencies": { "@babel/core": "^7.23.3", @@ -84,6 +90,7 @@ "jest-environment-jsdom": "^29.7.0", "playwright": "^1.32.1", "prettier": "3.1.0", + "shx": "^0.3.4", "ts-jest": "^29.1.1" }, "browserslist": { diff --git a/client/script/config.bash b/client/script/config.bash deleted file mode 100755 index 769416f28..000000000 --- a/client/script/config.bash +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# copy the correct environment variables file to react SPA -# https://stackoverflow.com/questions/51653931/react-configuration-file-for-post-deployment-settings -# https://dev.to/akdevcraft/react-runtime-variables-49dc -mode=$1 -if [ -z "$mode" ]; then - printf "Use yarn configapp with either dev, prod or test:" - printf "i.e. \"yarn configapp dev\" " - exit 1 -fi - -set_env() { - printf "Setting env to %s" "$1" - cp "config/$1.js" "public/env.js" # Configure dev environment in public for next build - if [ -d build ]; then - cp "public/env.js" "build/env.js" # Hot swap dev to build - fi -} - -case "$mode" in - dev) - set_env dev - ;; - prod) - set_env prod - ;; - test) - set_env test - ;; - *) - echo "Invalid mode $mode - use 'dev', 'prod' or 'test'" - exit 1 - ;; -esac - -exit 0 diff --git a/client/script/install.bash b/client/script/install.bash deleted file mode 100755 index 60a2ee173..000000000 --- a/client/script/install.bash +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -yarn install --frozen-lockfile -npx playwright install-deps \ No newline at end of file diff --git a/client/script/test.bash b/client/script/test.bash deleted file mode 100755 index 51c8fd9a0..000000000 --- a/client/script/test.bash +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -set -eux -printf "\n\n################ Testing in progress ################" -printf "\n################ Running Jest ################\n" -PATH="$(yarn bin):$PATH" -export PATH - -if [ -z "${1+x}" ]; then - mode="unit-tests" # Default mode if no argument passed -else - mode=$1 -fi - -if [ "$mode" == "-a" ]; then - printf "\n################ Running all tests ################\n" - printf "Running unit tests...\n" - jest -c ./jest.config.json ../test/unitTests --coverage - printf "Running integration tests ...\n" - jest -c ./jest.integration.config.json ../test/integration - yarn configapp test # Switch to testing env - yarn start >/dev/null & # Start server in background. Suppress stderr. - printf "\n\n################ Running Playwright ################" - playwright test - printf "Closing server on port 4000..." - npx kill-port 4000 - yarn configapp dev -elif [ "$mode" == "-u" ]; then - printf "Running unit tests only...\n" - jest -c ./jest.config.json ../test/unitTests --coverage -elif [ "$mode" == "-i" ]; then - printf "Running integration tests only...\n" - jest -c ./jest.integration.config.json ../test/integration -elif [ "$mode" == "-e" ]; then - printf "\n################ Running e2e tests ################\n" - yarn configapp test # Switch to testing env - yarn start >/dev/null & # Start server in background. Suppress stderr. - printf "\n\n################ Running Playwright ################" - playwright test - printf "Closing server on port 4000..." - npx kill-port 4000 - yarn configapp dev -else - printf "Running unit tests only...\n" - printf "Use -a for all tests, -u for unit tests, -i for integration tests or -e for e2e tests" - jest -c ./jest.config.json ../test/unitTests --coverage -fi \ No newline at end of file diff --git a/client/yarn.lock b/client/yarn.lock index d4908c049..7f2c20586 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -5718,7 +5718,7 @@ glob@7.1.6: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: +glob@^7.0.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: version "7.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== @@ -6123,6 +6123,11 @@ internal-slot@^1.0.4, internal-slot@^1.0.5: hasown "^2.0.0" side-channel "^1.0.4" +interpret@^1.0.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e" + integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA== + ipaddr.js@1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" @@ -7832,7 +7837,7 @@ minimatch@^5.0.1: dependencies: brace-expansion "^2.0.1" -minimist@^1.2.0, minimist@^1.2.6: +minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.6: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== @@ -9293,6 +9298,13 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +rechoir@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" + integrity sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw== + dependencies: + resolve "^1.1.6" + recursive-readdir@^2.2.2: version "2.2.3" resolved "https://registry.yarnpkg.com/recursive-readdir/-/recursive-readdir-2.2.3.tgz#e726f328c0d69153bcabd5c322d3195252379372" @@ -9488,7 +9500,7 @@ resolve.exports@^2.0.0: resolved "https://registry.yarnpkg.com/resolve.exports/-/resolve.exports-2.0.2.tgz#f8c934b8e6a13f539e38b7098e2e36134f01e800" integrity sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg== -resolve@^1.1.7, resolve@^1.14.2, resolve@^1.19.0, resolve@^1.20.0, resolve@^1.22.2, resolve@^1.22.4: +resolve@^1.1.6, resolve@^1.1.7, resolve@^1.14.2, resolve@^1.19.0, resolve@^1.20.0, resolve@^1.22.2, resolve@^1.22.4: version "1.22.8" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== @@ -9820,6 +9832,23 @@ shell-quote@^1.7.3, shell-quote@^1.8.1: resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.8.1.tgz#6dbf4db75515ad5bac63b4f1894c3a154c766680" integrity sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA== +shelljs@^0.8.5: + version "0.8.5" + resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.5.tgz#de055408d8361bed66c669d2f000538ced8ee20c" + integrity sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow== + dependencies: + glob "^7.0.0" + interpret "^1.0.0" + rechoir "^0.6.2" + +shx@^0.3.4: + version "0.3.4" + resolved "https://registry.yarnpkg.com/shx/-/shx-0.3.4.tgz#74289230b4b663979167f94e1935901406e40f02" + integrity sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g== + dependencies: + minimist "^1.2.3" + shelljs "^0.8.5" + side-channel@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" diff --git a/deploy/README.md b/deploy/README.md index f66e21561..6e01aec7b 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -77,8 +77,6 @@ The first step in this configuration is to prepare the a filesystem for users. An example file system in `files/` directory. You can rename the top-level user1/user2 to the usernames chosen by you. -Add an environment file named `.env` in lib for the library microservice. -An example `.env` file is given below. The simplest possibility is to use `local` mode with the following example. The filepath is the absolute filepath to `files/` directory. You can copy this configuration into _[config/lib](config/lib)_ file to get started. diff --git a/deploy/config/client/env.js b/deploy/config/client/env.js index ea8dc9b9f..cd60bf7a3 100644 --- a/deploy/config/client/env.js +++ b/deploy/config/client/env.js @@ -1,18 +1,20 @@ -window.env = { - REACT_APP_ENVIRONMENT: 'prod', - REACT_APP_URL: 'https://foo.com/', - REACT_APP_URL_BASENAME: '', - REACT_APP_URL_DTLINK: '/lab', - REACT_APP_URL_LIBLINK: '', - REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', - REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', - REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', - REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', - REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', +if (typeof window !== 'undefined') { + window.env = { + REACT_APP_ENVIRONMENT: 'prod', + REACT_APP_URL: 'https://foo.com/', + REACT_APP_URL_BASENAME: '', + REACT_APP_URL_DTLINK: '/lab', + REACT_APP_URL_LIBLINK: '', + REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', + REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', + REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', + REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', + REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', - REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', - REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', - REACT_APP_REDIRECT_URI: 'https://foo.com/Library', - REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/', - REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', + REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', + REACT_APP_REDIRECT_URI: 'https://foo.com/Library', + REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/', + REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + }; }; diff --git a/deploy/config/client/env.local.js b/deploy/config/client/env.local.js index 330364169..70c324760 100644 --- a/deploy/config/client/env.local.js +++ b/deploy/config/client/env.local.js @@ -1,19 +1,20 @@ -window.env = { - REACT_APP_ENVIRONMENT: "local", - REACT_APP_URL: "http://localhost/", - REACT_APP_URL_BASENAME: "", - REACT_APP_URL_DTLINK: "/lab", - REACT_APP_URL_LIBLINK: "", - REACT_APP_WORKBENCHLINK_TERMINAL: "/terminals/main", - REACT_APP_WORKBENCHLINK_VNCDESKTOP: "/tools/vnc/?password=vncpassword", - REACT_APP_WORKBENCHLINK_VSCODE: "/tools/vscode/", - REACT_APP_WORKBENCHLINK_JUPYTERLAB: "/lab", - REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: "", +if (typeof window !== 'undefined') { + window.env = { + REACT_APP_ENVIRONMENT: 'local', + REACT_APP_URL: 'http://localhost/', + REACT_APP_URL_BASENAME: '', + REACT_APP_URL_DTLINK: '/lab', + REACT_APP_URL_LIBLINK: '', + REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', + REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', + REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', + REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', + REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', - REACT_APP_CLIENT_ID: - "1be55736756190b3ace4c2c4fb19bde386d1dcc748d20b47ea8cfb5935b8446c", - REACT_APP_AUTH_AUTHORITY: "https://gitlab.com/", - REACT_APP_REDIRECT_URI: "http://localhost/Library", - REACT_APP_LOGOUT_REDIRECT_URI: "http://localhost/", - REACT_APP_GITLAB_SCOPES: "openid profile read_user read_repository api", + REACT_APP_CLIENT_ID: '1be55736756190b3ace4c2c4fb19bde386d1dcc748d20b47ea8cfb5935b8446c', + REACT_APP_AUTH_AUTHORITY: 'https://gitlab.com/', + REACT_APP_REDIRECT_URI: 'http://localhost/Library', + REACT_APP_LOGOUT_REDIRECT_URI: 'http://localhost/', + REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + }; }; diff --git a/deploy/config/client/env.trial.js b/deploy/config/client/env.trial.js index 14c3ee101..a359a63d7 100644 --- a/deploy/config/client/env.trial.js +++ b/deploy/config/client/env.trial.js @@ -1,18 +1,20 @@ -window.env = { - REACT_APP_ENVIRONMENT: 'trial', - REACT_APP_URL: 'https://foo.com/', - REACT_APP_URL_BASENAME: '', - REACT_APP_URL_DTLINK: '/lab', - REACT_APP_URL_LIBLINK: '', - REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', - REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', - REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', - REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', - REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', +if (typeof window !== 'undefined') { + window.env = { + REACT_APP_ENVIRONMENT: 'trial', + REACT_APP_URL: 'https://foo.com/', + REACT_APP_URL_BASENAME: '', + REACT_APP_URL_DTLINK: '/lab', + REACT_APP_URL_LIBLINK: '', + REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', + REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', + REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', + REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', + REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', - REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', - REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', - REACT_APP_REDIRECT_URI: 'https://foo.com/Library', - REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/', - REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', + REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', + REACT_APP_REDIRECT_URI: 'https://foo.com/Library', + REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/', + REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + }; }; diff --git a/deploy/config/gateway/fileConfig.local.yml b/deploy/config/gateway/fileConfig.local.yml index b8a85c62b..b07b2b5a2 100644 --- a/deploy/config/gateway/fileConfig.local.yml +++ b/deploy/config/gateway/fileConfig.local.yml @@ -3,19 +3,19 @@ http: dtaas: entryPoints: - http - rule: "Host(`localhost`)" + rule: 'Host(`localhost`)' service: dtaas user1: entryPoints: - http - rule: "Host(`localhost`) && PathPrefix(`/user1`)" + rule: 'Host(`localhost`) && PathPrefix(`/user1`)' service: user1 libms: entryPoints: - http - rule: "Host(localhost`) && PathPrefix(`/lib`)" + rule: 'Host(`localhost`) && PathPrefix(`/lib`)' service: libms services: diff --git a/deploy/cron.sh b/deploy/cron.sh index e48fd2858..287d267fb 100755 --- a/deploy/cron.sh +++ b/deploy/cron.sh @@ -18,8 +18,7 @@ nc -z localhost 4001 PORT_STATUS=$? if (( PORT_STATUS == 1 )) then - cd "${PROJECT_PATH}/servers/lib" || exit - nohup yarn start & disown + nohup libms -c "${PROJECT_PATH}/deploy/config/lib" & disown fi docker start traefik-gateway \ No newline at end of file diff --git a/deploy/install.sh b/deploy/install.sh index 18088b12c..98134963b 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -34,18 +34,15 @@ yarn install --production yarn build #one of the environments; specify only one; "dev" used the REACT_APP_ENV is not set -yarn configapp prod +yarn config:prod cp "${TOP_DIR}/deploy/config/client/env.js" build/env.js nohup serve -s build -l 4000 & disown #------------- -printf "\n\nStart the lib microservice\n " +printf "\n\n Install and run the lib microservice\n " printf "...........\n " -cd "${TOP_DIR}/servers/lib" || exit -yarn install -yarn build -cp "${TOP_DIR}/deploy/config/lib" .env -nohup yarn start & disown +sudo npm install -g @into-cps-association/libms +nohup libms --config "${TOP_DIR/deploy/config}/lib" & disown #------------- printf "\n \n Start the user workspaces\n " diff --git a/deploy/services/gitlab.js b/deploy/services/gitlab.js new file mode 100755 index 000000000..e19c3c333 --- /dev/null +++ b/deploy/services/gitlab.js @@ -0,0 +1,51 @@ +#!/usr/bin/node +/* Install gitlab for DTaaS */ +import {$} from 'execa'; +import chalk from 'chalk'; +import fs from 'fs'; +import yaml from 'js-yaml'; + +const $$ = $({stdio: 'inherit'}); +const log = console.log; +let config; + +const sleep = (ms) => + new Promise((r) => { + setTimeout(r, ms); + }); + + try { + log(chalk.blue("Load services configuration")); + config = await yaml.load(fs.readFileSync('gitlab.yml', 'utf8')); + log(chalk.green("configuration loading is successful and config is a valid yaml file")); +} catch (e) { + log(chalk.red("configuration is invalid. Please rectify services.yml file")); + process.exit(1); +} + +//--------------- +log(chalk.blue("Start Gitlab server")); +const gitlabConfig = config.services.gitlab; + +try { + log(chalk.green("Attempt to delete any existing Gitlab server docker container")); + await $$`docker stop gitlab`; + await $$`docker rm gitlab`; +} catch (e) { +} + +log(chalk.green("Start new Gitlab server docker container")); +await $$`docker run -d \ + --hostname ${gitlabConfig.hostname} \ + --env EXTERNAL_URL=${gitlabConfig.url} \ + --publish ${gitlabConfig.port}:80 \ + --name gitlab \ + --restart always \ + --volume ${gitlabConfig.homedir}/config:/etc/gitlab:Z \ + --volume ${gitlabConfig.homedir}/logs:/var/log/gitlab:Z \ + --volume ${gitlabConfig.homedir}/data:/var/opt/gitlab:Z \ + --shm-size 256m \ + gitlab/gitlab-ce:16.4.1-ce.0`; + +log(chalk.green("Gitlab server docker container started successfully")); + diff --git a/deploy/services/gitlab.yml b/deploy/services/gitlab.yml new file mode 100644 index 000000000..ce9d95f21 --- /dev/null +++ b/deploy/services/gitlab.yml @@ -0,0 +1,7 @@ +kind: deployment +services: + gitlab: + hostname: "foo.com" + url: "foo.com/gitlab" + port: 8969 + homedir: "/home/vagrant/DTaaS/deploy/services/gitlab" #no spaces in the path diff --git a/deploy/services/services.yml b/deploy/services/services.yml index 20fee7eb2..6ea101ae0 100644 --- a/deploy/services/services.yml +++ b/deploy/services/services.yml @@ -15,7 +15,7 @@ services: influxdb: username: "dtaas" password: "dtaas1357" # need to have letters and numbers - datapath: "/home/prasad/git/prasadtalasila/DTaaS/deploy/vagrant/two-machine/influx" #no spaces in the path + datapath: "/home/vagrant/DTaaS/deploy/services/influx" #no spaces in the path port: 80 mqtt: # usernames and passwords are in deploy/config/services/mqtt/config/password username: "dtaas" @@ -24,4 +24,4 @@ services: username: "dtaas" password: "dtaas" port: 27017 - datapath: "/home/prasad/git/prasadtalasila/DTaaS/deploy/vagrant/two-machine/mongodb" #no spaces in the path + datapath: "/home/vagrant/DTaaS/deploy/services/mongodb" #no spaces in the path diff --git a/deploy/single-script-install.sh b/deploy/single-script-install.sh index 776c5688b..225a93c90 100755 --- a/deploy/single-script-install.sh +++ b/deploy/single-script-install.sh @@ -90,7 +90,7 @@ then curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | \ sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg fi -NODE_MAJOR=18 +NODE_MAJOR=20 printf "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] \ https://deb.nodesource.com/node_%s.x nodistro main" "$NODE_MAJOR" | \ sudo tee /etc/apt/sources.list.d/nodesource.list @@ -150,7 +150,7 @@ cd "${TOP_DIR}/client" || exit yarn install --production yarn build -yarn configapp dev +yarn config:dev if [ -n "$env_variable" ] ; then cp "${TOP_DIR}/deploy/config/client/env.${env_variable}.js" build/env.js else @@ -159,11 +159,9 @@ fi nohup serve -s build -l 4000 & disown #------------- -printf "\n\n Build, configure and run the lib microservice\n " +printf "\n\n Install and run the lib microservice\n " printf "...........\n " -cd "${TOP_DIR}/servers/lib" || exit -yarn install -yarn build +sudo npm install -g @into-cps-association/libms { printf "PORT='4001'\n " @@ -175,8 +173,8 @@ yarn build printf "LOG_LEVEL='debug'\n " printf "APOLLO_PATH='lib'\n " printf "GRAPHQL_PLAYGROUND='true'\n " -} > .env -nohup yarn start & disown +} > "${TOP_DIR}/deploy/config/lib" +nohup libms --config "${TOP_DIR}/deploy/config/lib" & disown #------------- @@ -209,7 +207,6 @@ else fi - #------------- printf "\n\n Start the traefik gateway server\n " printf "...........\n " diff --git a/deploy/vagrant/make_boxes/dtaas/README.md b/deploy/vagrant/make_boxes/dtaas/README.md index 3d0841578..99690dc42 100644 --- a/deploy/vagrant/make_boxes/dtaas/README.md +++ b/deploy/vagrant/make_boxes/dtaas/README.md @@ -34,7 +34,7 @@ are not a developer, no changes are required to the `Vagrantfile`. This vagrant box installed for users will have the following items: * docker v24.0 -* nodejs v18.8 +* nodejs v20.10 * yarn v1.22 * npm v10.2 * containers @@ -62,9 +62,7 @@ base vagrant box. ```bash #create a key pair -ssh-keygen -b 4096 -t rsa -f key -q -N "" -mv key vagrant -mv key.pub vagrant.pub +ssh-keygen -b 4096 -t rsa -f vagrant -q -N "" vagrant up @@ -79,9 +77,9 @@ git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-m # inside ~/.zshrc, modify the following line plugins=(git zsh-autosuggestions history cp tmux) -# to replace the vagrant ssh key-pair with personal one -# remove the vagrant default public key - first line of -# /home/vagrant/.ssh/authorized_keys +# to replace the default vagrant ssh key-pair with +# the generated private key into authorized keys +cp /vagrant/vagrant.pub /home/vagrant/.ssh/authorized_keys # exit vagrant guest machine and then # copy own private key to vagrant private key location diff --git a/deploy/vagrant/make_boxes/dtaas/Vagrantfile b/deploy/vagrant/make_boxes/dtaas/Vagrantfile index e37807bb5..b3fd6f18b 100644 --- a/deploy/vagrant/make_boxes/dtaas/Vagrantfile +++ b/deploy/vagrant/make_boxes/dtaas/Vagrantfile @@ -4,15 +4,17 @@ # Creates the base environment for running the DTaaS application Vagrant.configure("2") do |config| - config.vm.box = "ubuntu/jammy64" - - config.vm.provider "virtualbox" do |vb| - vb.name = "dtaas" - vb.memory = 16384 - vb.cpus = 8 - end - - config.vm.provision "shell", path: "user.sh" - #config.vm.provision "shell", path: "developer.sh" -end + config.vm.box = "ubuntu/jammy64" + config.disksize.size = "200GB" + config.vm.hostname = "foo.com" + config.vm.network "public_network", :mac => "02163ee0bf12" + + config.vm.provider "virtualbox" do |vb| + vb.name = "dtaas" + vb.memory = 16384 + vb.cpus = 8 + end + config.vm.provision "shell", path: "user.sh" + #config.vm.provision "shell", path: "developer.sh" +end diff --git a/deploy/vagrant/make_boxes/dtaas/developer.sh b/deploy/vagrant/make_boxes/dtaas/developer.sh index 70d387dac..382de7fbd 100755 --- a/deploy/vagrant/make_boxes/dtaas/developer.sh +++ b/deploy/vagrant/make_boxes/dtaas/developer.sh @@ -6,7 +6,7 @@ apt-get update -y apt-get upgrade -y #install docker-compose from https://docs.docker.com/compose/install/other/ -curl -SL "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-linux-x86_64" \ +curl -SL "https://github.com/docker/compose/releases/download/v2.23.3/docker-compose-linux-x86_64" \ -o /usr/local/bin/docker-compose ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose chmod 755 /usr/local/bin/docker-compose /usr/bin/docker-compose diff --git a/deploy/vagrant/make_boxes/dtaas/user.sh b/deploy/vagrant/make_boxes/dtaas/user.sh index 36c0f4f52..ebbf88997 100755 --- a/deploy/vagrant/make_boxes/dtaas/user.sh +++ b/deploy/vagrant/make_boxes/dtaas/user.sh @@ -47,7 +47,7 @@ then curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | \ gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg fi -NODE_MAJOR=18 +NODE_MAJOR=20 printf "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] \ https://deb.nodesource.com/node_%s.x nodistro main" "$NODE_MAJOR" | \ tee /etc/apt/sources.list.d/nodesource.list @@ -79,3 +79,6 @@ docker pull rabbitmq:3-management docker pull eclipse-mosquitto:2 docker pull mongo:7.0.3 docker pull gitlab/gitlab-ce:16.4.1-ce.0 + +# remove default route inserted by vagrant +printf "* * * * * ip route del default via 10.0.2.2 dev enp0s3\n" | crontab - \ No newline at end of file diff --git a/deploy/vagrant/single-machine/Vagrantfile b/deploy/vagrant/single-machine/Vagrantfile index 16ae2352b..d40898600 100644 --- a/deploy/vagrant/single-machine/Vagrantfile +++ b/deploy/vagrant/single-machine/Vagrantfile @@ -6,14 +6,12 @@ Vagrant.configure(2) do |config| config.vm.box = "dtaas" - # gateway node - hosts traefik, dns and static client website config.vm.define "dtaas" do |node| - node.vm.hostname = "foo.com" - node.vm.network "public_network", :bridge => 'ens10f3', :mac => "02163ef4f4f4" + node.vm.hostname = "foo.com" + node.vm.network "public_network", :mac => "02163ef4f4f4" - - node.vm.provider "virtualbox" do |vb| - vb.name = "DTaaSOne" + node.vm.provider "virtualbox" do |vb| + vb.name = "dtaas-one" vb.memory = 16384 vb.cpus = 8 end diff --git a/deploy/vagrant/two-machine/Vagrantfile b/deploy/vagrant/two-machine/Vagrantfile index 24eaeb3e1..bc7653526 100644 --- a/deploy/vagrant/two-machine/Vagrantfile +++ b/deploy/vagrant/two-machine/Vagrantfile @@ -11,7 +11,7 @@ Vagrant.configure(2) do |config| config.vm.define opts["name"] do |node| node.vm.box = "dtaas" node.vm.hostname = opts["hostname"] - node.vm.network "public_network", :bridge => 'ens10f3', :mac => opts["mac"] + node.vm.network "public_network", :mac => opts["mac"] node.vm.provider "virtualbox" do |vb| vb.memory = 6096 @@ -19,7 +19,6 @@ Vagrant.configure(2) do |config| vb.name = opts["name"] end - node.ssh.private_key_path = "vagrant" end end diff --git a/deploy/vagrant/two-machine/boxes.json b/deploy/vagrant/two-machine/boxes.json index e47a5dc42..4f4fe00d5 100644 --- a/deploy/vagrant/two-machine/boxes.json +++ b/deploy/vagrant/two-machine/boxes.json @@ -1,6 +1,6 @@ [ { - "name" : "dtaas", + "name" : "dtaas-two", "hostname" : "foo.com", "mac" : "02163e0d3c28", "ssh_port" : 11022, diff --git a/docs/FAQ.md b/docs/FAQ.md index 94c45cc87..90a67b950 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -18,6 +18,31 @@ Please see the [features](user/features.md) page to get a sense of the things you can do in DaaS. +??? Question "What is the scope and current capabilities of DTaaS?" + + 1. DTaaS is a web based interface to allow you to invoke various tools + related to work you want to perform with one or more DTs. + 2. DTaaS as such doesn't help you create DTs; it loads and runs DTs + that you create using tools and services that are available via DTaaS. + 3. DTaaS can help you create DTs only if your DT or DT asset authoring tools + can work in user workspace containing + Linux terminal / Desktop / Jupyter / VSCode services. + 4. DTaaS as such won't help you to install DTs that you get from elsewhere. + 5. DTs are just executables, as far as DTaaS is concerned. + Users are not constrained to work with DTs in a certain way. + The DTaaS suggests creation of DTs from reusable assets and provides + a suggestive structure for DTs. The [examples](./user/examples/index.md) + provide more insight into DTaaS way of working with DTs. + But this suggested workflow is not binding on the users. + 6. DTaaS doesn't help you to autorun run DTs, other than by providing + user workspaces. These user workspaces are based on Ubuntu 20.04 + Operating system. + 7. The current user interface of DTaaS web application is heavily reliant + on the use of Jupyter lab and notebook. + The **Digital Twins** page has Create / Execute / Analyze sections but + all point to Jupyter lab. Web interface. The functionality of these pages + is still under development. + ??? Question "Are there any Key Performance / Capability Indicators for DTaaS?" | Key Performance Indicator | Value | diff --git a/docs/admin/client/CLIENT.md b/docs/admin/client/CLIENT.md index ee2dc0647..efb224024 100644 --- a/docs/admin/client/CLIENT.md +++ b/docs/admin/client/CLIENT.md @@ -1,5 +1,14 @@ # Host the DTaaS Client Website + + +!!! tip + The [Localhost](../localhost.md) installation setup is a faster + way to use the DTaaS software. You might want to consider the localhost + option before using only the **DTaaS-client.zip** package. + + + To host DTaaS client website on your server, follow these steps: - Download the **DTaaS-client.zip** from the @@ -17,62 +26,69 @@ To host DTaaS client website on your server, follow these steps: See the definitions below: ```js - window.env = { - REACT_APP_ENVIRONMENT: "prod | dev", - REACT_APP_URL: "URL for the gateway", - REACT_APP_URL_BASENAME: "Base URL for the client website"(optional), - REACT_APP_URL_DTLINK: "Endpoint for the Digital Twin", - REACT_APP_URL_LIBLINK: "Endpoint for the Library Assets", - REACT_APP_WORKBENCHLINK_TERMINAL: "Endpoint for the terminal link", - REACT_APP_WORKBENCHLINK_VNCDESKTOP: "Endpoint for the VNC Desktop link", - REACT_APP_WORKBENCHLINK_VSCODE: "Endpoint for the VS Code link", - REACT_APP_WORKBENCHLINK_JUPYTERLAB: "Endpoint for the Jupyter Lab link", - REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: - "Endpoint for the Jupyter Notebook link", - REACT_APP_CLIENT_ID: 'AppID genereated by the gitlab OAuth provider', - REACT_APP_AUTH_AUTHORITY: 'URL of the private gitlab instance', - REACT_APP_REDIRECT_URI: 'URL of the homepage for the logged in users of the website', - REACT_APP_LOGOUT_REDIRECT_URI: 'URL of the homepage for the anonymous users of the website', - REACT_APP_GITLAB_SCOPES: 'OAuth scopes. These should match with the scopes set in gitlab OAuth provider', + if (typeof window !== 'undefined') { + window.env = { + REACT_APP_ENVIRONMENT: "prod | dev", + REACT_APP_URL: "URL for the gateway", + REACT_APP_URL_BASENAME: "Base URL for the client website"(optional), + REACT_APP_URL_DTLINK: "Endpoint for the Digital Twin", + REACT_APP_URL_LIBLINK: "Endpoint for the Library Assets", + REACT_APP_WORKBENCHLINK_TERMINAL: "Endpoint for the terminal link", + REACT_APP_WORKBENCHLINK_VNCDESKTOP: "Endpoint for the VNC Desktop link", + REACT_APP_WORKBENCHLINK_VSCODE: "Endpoint for the VS Code link", + REACT_APP_WORKBENCHLINK_JUPYTERLAB: "Endpoint for the Jupyter Lab link", + REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: + "Endpoint for the Jupyter Notebook link", + REACT_APP_CLIENT_ID: 'AppID genereated by the gitlab OAuth provider', + REACT_APP_AUTH_AUTHORITY: 'URL of the private gitlab instance', + REACT_APP_REDIRECT_URI: 'URL of the homepage for the logged in users of the website', + REACT_APP_LOGOUT_REDIRECT_URI: 'URL of the homepage for the anonymous users of the website', + REACT_APP_GITLAB_SCOPES: 'OAuth scopes. These should match with the scopes set in gitlab OAuth provider', + }; }; // Example values with no base URL. Trailing and ending slashes are optional. - window.env = { - REACT_APP_ENVIRONMENT: 'prod', - REACT_APP_URL: 'https://foo.com/', - REACT_APP_URL_BASENAME: '', - REACT_APP_URL_DTLINK: '/lab', - REACT_APP_URL_LIBLINK: '', - REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', - REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', - REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', - REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', - REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', - REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', - REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', - REACT_APP_REDIRECT_URI: 'https://foo.com/Library', - REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/', - REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + if (typeof window !== 'undefined') { + window.env = { + REACT_APP_ENVIRONMENT: 'prod', + REACT_APP_URL: 'https://foo.com/', + REACT_APP_URL_BASENAME: '', + REACT_APP_URL_DTLINK: '/lab', + REACT_APP_URL_LIBLINK: '', + REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', + REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', + REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', + REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', + REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', + REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', + REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', + REACT_APP_REDIRECT_URI: 'https://foo.com/Library', + REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/', + REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + }; }; + // Example values with "bar" as basename URL. //Trailing and ending slashes are optional. - window.env = { - REACT_APP_ENVIRONMENT: "dev", - REACT_APP_URL: 'https://foo.com/', - REACT_APP_URL_BASENAME: 'bar', - REACT_APP_URL_DTLINK: '/lab', - REACT_APP_URL_LIBLINK: '', - REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', - REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', - REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', - REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', - REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', - REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', - REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', - REACT_APP_REDIRECT_URI: 'https://foo.com/bar/Library', - REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/bar', - REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + if (typeof window !== 'undefined') { + window.env = { + REACT_APP_ENVIRONMENT: "dev", + REACT_APP_URL: 'https://foo.com/', + REACT_APP_URL_BASENAME: 'bar', + REACT_APP_URL_DTLINK: '/lab', + REACT_APP_URL_LIBLINK: '', + REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', + REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', + REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', + REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', + REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', + REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', + REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', + REACT_APP_REDIRECT_URI: 'https://foo.com/bar/Library', + REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/bar', + REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + }; }; ``` diff --git a/docs/admin/guides/gitlab.md b/docs/admin/guides/gitlab.md new file mode 100644 index 000000000..2e1142faa --- /dev/null +++ b/docs/admin/guides/gitlab.md @@ -0,0 +1,83 @@ +# Install Gitlab + +This guide helps with installation of a dedicated +[Gitlab](https://gitlab.com) service. This Gitlab installation can be used +as OAuth2 authentication provider to the DTaaS software. + +There are two possible ways you can install Gitlab: + +* At dedicated domain name (ex: _gitlab.foo.com_) +* At a URL path on existing WWW server (ex: foo.com/gitlab) + +This guide illustrates the installation of Gitlab at: +foo.com/gitlab. But the instructions can be +adapted to install Gitlab at a dedicated domain name. + +## Configure and Install + +If you have not cloned the DTaaS git repository, cloning would be +the first step. +In case you already have the codebase, you can skip the cloning step. +To clone, do: + +```bash +git clone https://github.com/into-cps-association/DTaaS.git +cd DTaaS/deploy/services +``` + +The next step in installation is to specify the config for the Gitlab. +The __gitlab.yml__ contains the required configuration settings. +Update the configuration file before proceeding with the installation. + +Now continue with the installation of services. + +```bash +yarn install +node gitlab.js +``` + +### Post-install Configuration + +Gitlab also requires post-installation configuration. + +!!! Information + + This configuration needs to be done from within the running container. + +```bash +docker exec -it gitlab bash +``` + +The configuration file to change is _/etc/gitlab/gitlab.rb_. +The variables to change are: + +```ini +external_url 'http(s)://foo.com/gitlab' +nginx['listen_port'] = 80 +nginx['enable'] = true + +nginx['listen_https'] = false +nginx['redirect_http_to_https'] = false +letsencrypt['enable'] = false +``` + +If you want to use _gitlab.foo.com'_ for Gitlab installation, +you can make one change: +`external_url http(s)://foo.com/gitlab`. Other settings remain the same. + +```bash +gitlab-ctl reconfigure +``` + +The administrator username for Gitlab is: `root`. The password for +this user account is available in: `/etc/gitlab/initial_root_password`. + +### Traefik Gateway Configuration + +It is likely that you are going to put the Gitlab behind Traefik gateway. +If so, do remove the middleware authentication from gitlab PathPrefix. + +## Use + +After the installation is complete, the required user accounts and OAuth2 +applications can be setup on this Gitlab instance. \ No newline at end of file diff --git a/docs/admin/host.md b/docs/admin/host.md index f6209081d..e4b323d9b 100644 --- a/docs/admin/host.md +++ b/docs/admin/host.md @@ -87,12 +87,10 @@ The library microservice requires configuration. A template of this configuration file is given in _deploy/config/lib_ file. Please modify this file as per your needs. -The first step in this configuration is to prepare the a filesystem for users. -An example file system in `files/` directory. +The first step in this configuration is to prepare a filesystem for users. +An example file system is in `files/` directory. You can rename the top-level user1/user2 to the usernames chosen by you. -Add an environment file named `.env` in lib for the library microservice. -An example `.env` file is given below. The simplest possibility is to use `local` mode with the following example. The filepath is the absolute filepath to `files/` directory. You can copy this configuration into _deploy/config/lib_ file to get started. @@ -133,24 +131,25 @@ Remember to Create gitlab accounts for usernames chosen by you. Change the React website configuration in _deploy/config/client/env.js_. ```js -window.env = { - REACT_APP_ENVIRONMENT: "prod", - REACT_APP_URL: "https://foo.com/", - REACT_APP_URL_BASENAME: "dtaas", - REACT_APP_URL_DTLINK: "/lab", - REACT_APP_URL_LIBLINK: "", - REACT_APP_WORKBENCHLINK_TERMINAL: "/terminals/main", - REACT_APP_WORKBENCHLINK_VNCDESKTOP: "/tools/vnc/?password=vncpassword", - REACT_APP_WORKBENCHLINK_VSCODE: "/tools/vscode/", - REACT_APP_WORKBENCHLINK_JUPYTERLAB: "/lab", - REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: "", - - REACT_APP_CLIENT_ID: - "934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0", - REACT_APP_AUTH_AUTHORITY: "https://gitlab.foo.com/", - REACT_APP_REDIRECT_URI: "https://foo.com/Library", - REACT_APP_LOGOUT_REDIRECT_URI: "https://foo.com/", - REACT_APP_GITLAB_SCOPES: "openid profile read_user read_repository api", +if (typeof window !== 'undefined') { + window.env = { + REACT_APP_ENVIRONMENT: 'prod', + REACT_APP_URL: 'https://foo.com/', + REACT_APP_URL_BASENAME: 'dtaas', + REACT_APP_URL_DTLINK: '/lab', + REACT_APP_URL_LIBLINK: '', + REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', + REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', + REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', + REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', + REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', + + REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', + REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', + REACT_APP_REDIRECT_URI: 'https://foo.com/Library', + REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/', + REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + }; }; ``` diff --git a/docs/admin/overview.md b/docs/admin/overview.md index a182711cf..8cf7bb8c4 100644 --- a/docs/admin/overview.md +++ b/docs/admin/overview.md @@ -1,25 +1,27 @@ # Overview -## What is the goal? +## Goal The goal is to set up the DTaaS infrastructure in order to enable your users to use the DTaaS. As an admin you will administrate the users and the servers of the system. -## What are the requirements? +## Requirements -### OAuth Provider +### OAuth Provider (Mandatory) You need to have an OAuth Provider running, which the DTaaS can use for authentication. This is described further in the [authentication section](./client/auth.md). -### Domain name +### Domain name (Optional) -The DTaaS software can only be hosted on a server with a domain name -like _foo.com_. +The DTaaS software is a web application and is preferably hosted + on a server with a domain name like _foo.com_. +However, it is possible to install the software on your computer +and use access it at _localhost_. -### Reverse Proxy +### Reverse Proxy (Optional) The installation setup assumes that the foo.com server is behind a reverse proxy / load balancer that provides https termination. You can still use @@ -30,15 +32,17 @@ with in [OAuth registration](./client/auth.md). Other installation configuration remains the same. -## What to install? - -The DTaaS can be installed in different ways. Each version is for different purposes: - -- [Trial installation on single host](./trial.md) -- [Production installation on single host](./host.md) -- On [one](vagrant/single-machine.md) or [two](vagrant/two-machines.md) - Vagrant virtual machines -- Seperater Packages: [client website](client/CLIENT.md) and - [lib microservice](servers/lib/LIB-MS.md) +## Install +The DTaaS can be installed in different ways. Each version serves a different purpose. Follow the installation that fits your usecase. + +| Installation Setup | Purpose | +|:-----|:-----| +| [Trial installation on localhost](./trial.md) | Install DTaaS on your computer for a single user; does not need a web server. _This setup also does not require reverse proxy and domain name._ | +| [Trial installation on single host](./trial.md) | Install DTaaS on server for a single user. | +| [Production installation on single host](./host.md) | Install DTaaS on server for multiple users. | +| [One vagrant machine](vagrant/single-machine.md) | Install DTaaS on a virtual machine; can be used for single or multiple users. | +| [Two vagrant machines](vagrant/two-machines.md) | Install DTaaS on two virtual machines; can be used for single or multiple users. | +| | The core DTaaS application is installed on the first virtual machine and all the services (RabbitMQ, MQTT, InfluxDB and Grafana) are installed on second virtual machine. | +| Seperater Packages: [client website](client/CLIENT.md) and [lib microservice](servers/lib/LIB-MS.md) | Can be used independently; do not need full installation of DTaaS. | diff --git a/docs/admin/vagrant/base-box.md b/docs/admin/vagrant/base-box.md index 1cf32e90e..f5feb494f 100644 --- a/docs/admin/vagrant/base-box.md +++ b/docs/admin/vagrant/base-box.md @@ -34,7 +34,7 @@ are not a developer, no changes are required to the `Vagrantfile`. This vagrant box installed for users will have the following items: 1. docker v24.0 -1. nodejs v18.8 +1. nodejs v20.10 1. yarn v1.22 1. npm v10.2 1. containers - ml-workspace-minimal v0.13, traefik v2.10, gitlab-ce v16.4, @@ -61,9 +61,7 @@ base vagrant box. ```bash #create a key pair -ssh-keygen -b 4096 -t rsa -f key -q -N "" -mv key vagrant -mv key.pub vagrant.pub +ssh-keygen -b 4096 -t rsa -f vagrant -q -N "" vagrant up @@ -79,8 +77,9 @@ git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-m # inside ~/.zshrc, modify the following line plugins=(git zsh-autosuggestions history cp tmux) -# remove the vagrant default public key - first line of -# /home/vagrant/.ssh/authorized_keys +# to replace the default vagrant ssh key-pair with +# the generated private key into authorized keys +cp /vagrant/vagrant.pub /home/vagrant/.ssh/authorized_keys # exit vagrant guest machine and then # copy own private key to vagrant private key location @@ -89,6 +88,7 @@ cp vagrant .vagrant/machines/default/virtualbox/private_key # check vagrant ssh #should work +# exit vagrant guest machine and then vagrant halt vagrant package --base dtaas \ diff --git a/docs/admin/vagrant/basebox.png b/docs/admin/vagrant/basebox.png index 78d5f4f9b..ffb13ae3e 100644 Binary files a/docs/admin/vagrant/basebox.png and b/docs/admin/vagrant/basebox.png differ diff --git a/docs/admin/vagrant/single-machine.png b/docs/admin/vagrant/single-machine.png index 469cfa63b..1c1f3ba94 100644 Binary files a/docs/admin/vagrant/single-machine.png and b/docs/admin/vagrant/single-machine.png differ diff --git a/docs/admin/vagrant/two-machine-use-legend.png b/docs/admin/vagrant/two-machine-use-legend.png index 06482ec0a..315c90424 100644 Binary files a/docs/admin/vagrant/two-machine-use-legend.png and b/docs/admin/vagrant/two-machine-use-legend.png differ diff --git a/docs/admin/vagrant/two-machines.md b/docs/admin/vagrant/two-machines.md index 10f1e8fec..fd23825be 100644 --- a/docs/admin/vagrant/two-machines.md +++ b/docs/admin/vagrant/two-machines.md @@ -68,7 +68,7 @@ Please follow these steps to make this work in your local environment. Update the **boxes.json**. There are entries one for each server. The fields to update are: - 1. `name` - name of server1 (`"name" = "dtaas"`) + 1. `name` - name of server1 (`"name" = "dtaas-two"`) 1. `hostname` - hostname of server1 (`"name" = "foo.com"`) 1. MAC address (`:mac => "xxxxxxxx"`). This change is required if you have a DHCP server assigning domain names diff --git a/docs/developer/npm-packages.md b/docs/developer/npm-packages.md index e0e7ba8b2..cf21f153e 100644 --- a/docs/developer/npm-packages.md +++ b/docs/developer/npm-packages.md @@ -1,14 +1,41 @@ # Publish NPM packages The DTaaS software is developed as a monorepo with multiple npm packages. + +## Default npm registry + +The default registry for npm packages is [npmjs](https://registry.npmjs.org). +The freely-accessible public packages are published to the **npmjs** registry. +The publication step is manual. + +```bash +npm login --registry="https://registry.npmjs.org" +cat ~/.npmrc #shows the auth token for the registry +//registry.npmjs.org/:_authToken=xxxxxxxxxx +yarn publish --registry="https://registry.npmjs.org" \ + --no-git-tag-version --access public +``` + +At least one package is published to this registry for each release of DTaaS. +This published package is used in the release scripts. + +## Github npm registry + +The Github actions of the project publish +[packages](https://github.com/orgs/INTO-CPS-Association/packages?repo_name=DTaaS). +The only limitation is that the users need an +[access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) +from Github. + +## Private Registry + +### Setup private npm registry + Since publishing to [npmjs](https://www.npmjs.com/) is irrevocable and public, developers are encouraged to setup their own private npm registry for local development. - A private npm registry will help with local publish and unpublish steps. -## Setup private npm registry - We recommend using [verdaccio](https://verdaccio.org) for this task. The following commands help you create a working private npm registry for development. @@ -25,7 +52,7 @@ npm login #login with the credentials for npm utility You can open `http://localhost:4873` in your browser, login with the user credentials to see the packages published. -### Publish to private npm registry +#### Publish to private npm registry To publish a package to your local registry, do: @@ -56,7 +83,7 @@ sudo npm list -g # should list @dtaas/runner in the packages sudo npm remove --global @dtaas/runner ``` -## :rocket: Use the packages +### :rocket: Use the packages The packages available in private npm registry can be used like the regular npm packages installed from [npmjs](https://www.npmjs.com/). diff --git a/docs/developer/system/DTaaS.drawio b/docs/developer/system/DTaaS.drawio index 47566f8dc..b67af62a2 100644 --- a/docs/developer/system/DTaaS.drawio +++ b/docs/developer/system/DTaaS.drawio @@ -1,6 +1,6 @@ - + - + @@ -2741,7 +2741,7 @@ - + @@ -2883,7 +2883,7 @@ - + @@ -3254,7 +3254,7 @@ - + @@ -3317,7 +3317,7 @@ - + @@ -3356,90 +3356,90 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -3447,7 +3447,7 @@ - + @@ -3456,13 +3456,13 @@ - + - + - + diff --git a/docs/developer/testing/intro.md b/docs/developer/testing/intro.md index b8ed9f267..e152fcb8a 100644 --- a/docs/developer/testing/intro.md +++ b/docs/developer/testing/intro.md @@ -102,8 +102,7 @@ This task has been postponed to future. Testing any code changes through the end user interface of your software is essential to verify if your code has the desired effect for the user. [End-to-End tests in DTaaS](https://github.com/INTO-CPS-Association/DTaaS/blob/feature/distributed-demo/client/test/README.md) -a functional setup. For more information -[visit here](https://github.com/INTO-CPS-Association/DTaaS/blob/feature/distributed-demo/client/test/README.md). +a functional setup. There are end-to-end tests in the DTaaS. This task has been postponed to future. diff --git a/docs/user/digital-twins/lifecycle.md b/docs/user/digital-twins/lifecycle.md index 3bc5f2719..b6776378e 100644 --- a/docs/user/digital-twins/lifecycle.md +++ b/docs/user/digital-twins/lifecycle.md @@ -76,6 +76,10 @@ A dedicated program exists for each phase of DT lifecycle. Each program can be as simple as a script that launches other programs or sends messages to a live digital twin. +:fontawesome-solid-circle-info: **The recommended way to implement lifecycle** +**phases within DTaaS is to create scripts. These scripts can be as simple** +**as shell scripts.** + ## Example Lifecycle Scripts Here are the example programs / scripts to manage three phases in diff --git a/docs/user/examples/drobotti-rmqfmu/README.md b/docs/user/examples/drobotti-rmqfmu/README.md index 014b9dd31..af90e952f 100644 --- a/docs/user/examples/drobotti-rmqfmu/README.md +++ b/docs/user/examples/drobotti-rmqfmu/README.md @@ -114,6 +114,20 @@ lifecycle/execute The results can be found in the _/workspace/examples/digital\_twins/drobotti\_rmqfmu directory_. +Executing the DT will generate and launch a co-simulation +(RMQFMU and distance FMU), and two python scripts. +One to publish data that is read from a file. +And one to consume what is sent by the distance FMU. + +In this examples the DT will run for 10 seconds, with a stepsize of 100ms. +Thereafter it is possible to examine the logs produce in +```/workspace/examples/digital_twins/drobotti_rmqfmu/target```. +The outputs for each FMU, xpos and ypos for the RMQFMU, and the distance for +the distance FMU are recorded in the ```outputs.csv``` file. +Other logs can be examined for each FMU and the publisher scripts. +Note that, the RMQFMU only sends data, if the current input +is different to the previous one. + ### Terminate phase Terminate to clean up the debug files and co-simulation output files. diff --git a/docs/user/examples/examples.drawio b/docs/user/examples/examples.drawio index fe8e1b4f7..25224746b 100755 --- a/docs/user/examples/examples.drawio +++ b/docs/user/examples/examples.drawio @@ -1,6 +1,6 @@ - + - + @@ -691,25 +691,25 @@ - + - + - + - + - + - + - + @@ -718,7 +718,7 @@ - + @@ -727,13 +727,13 @@ - + - + - + @@ -742,16 +742,16 @@ - + - + - + - + @@ -761,28 +761,28 @@ - + - + - + - + - + - + - + @@ -792,27 +792,143 @@ - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + - + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/user/examples/opc-ua-waterplant/README.md b/docs/user/examples/opc-ua-waterplant/README.md index eec7b2451..17a6de9d8 100644 --- a/docs/user/examples/opc-ua-waterplant/README.md +++ b/docs/user/examples/opc-ua-waterplant/README.md @@ -13,12 +13,12 @@ adjusting setpoints could significantly reduce energy consumption. ## Physical Twin Overview -This example demonstrates the communication between a physical waste -water plant in the tertiary treatment and a digital twin of -the ultaviolet (UV) disinfection process based on Computational Fluid -Dynamics (CFD) and compartment models. The aim of this digital twin is to -develop "virtual sensors" that provide continuous information that -facilitates the decision making process for the plant operator. +This example demonstrates the communication between a physical +ultraviolet (UV) disinfection process (the tertiary treatment of a WWT plant) +and its digital twin, which is based on Computational Fluid Dynamics (CFD) +and compartment models. The aim of this digital twin is to develop +"virtual sensors" that provide continuous information that facilitates +the decision making process for the plant operator. ![Waste water treatment plant](images/water-treatment-plant.png) @@ -37,7 +37,7 @@ the UV intensity (measured by the intensity sensor). The information of the volumetric flow and power (in the three parts of the channel) is transmitted to the PLC of the plant. Furthermore, the PLC is working as OPC UA Server to send and receive -data to and from an OPC UA client. +data to and from an OPC UA Client. Additionally, some sizing parameters and initial values are read from a spreadsheet filled in by the plant operator. In this case, the spreadsheet is an Open Office file (.ods) due to the software installed in the SCADA PC. @@ -55,9 +55,9 @@ conservation laws (mass, energy and momentum), but simplifies details (geometry, mainly) to ensure real-time calculations and accurate results. The results are compared to the ones obtained by the CFD. C solver developed is used by the OpenModelica model. OpenModelica converts it into -the FMI standard, to be integrated in the main script execution (_main.py_). +the FMI standard, to be integrated in the OPC UA Client (_client-opcua.py_). -![Structure of Python Script](images/dt-structure.png) +![Structure of Python Script](images/dt-structure-uv_channel.png) ## Digital Twin Configuration diff --git a/mkdocs-github.yml b/mkdocs-github.yml index 6b0088b80..aaaef7556 100644 --- a/mkdocs-github.yml +++ b/mkdocs-github.yml @@ -28,12 +28,13 @@ nav: - Single Machine: admin/vagrant/single-machine.md - Two Machines: admin/vagrant/two-machines.md - Separate Packages: - - website: admin/client/CLIENT.md - - library microservice: admin/servers/lib/LIB-MS.md + - Website: admin/client/CLIENT.md + - Library microservice: admin/servers/lib/LIB-MS.md - Services: admin/services.md - - Guides: + - Guides: - New user: admin/guides/add_user.md - New Service: admin/guides/add_service.md + - Install Gitlab: admin/guides/gitlab.md - Link services: admin/guides/link_service.md - Common Assets: admin/guides/common_workspace_readonly.md - HTTP Mode: admin/guides/hosting_site_without_https.md @@ -75,7 +76,7 @@ nav: use_directory_urls: false extra_css: - - docs/assets/css/extra.css + - assets/css/extra.css extra_javascript: - https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS-MML_HTMLorMML @@ -85,8 +86,8 @@ extra_javascript: markdown_extensions: - attr_list - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.highlight: linenums: true use_pygments: true diff --git a/mkdocs.yml b/mkdocs.yml index 14875273f..fb5590333 100755 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,12 +28,13 @@ nav: - Single Machine: admin/vagrant/single-machine.md - Two Machines: admin/vagrant/two-machines.md - Separate Packages: - - website: admin/client/CLIENT.md - - library microservice: admin/servers/lib/LIB-MS.md + - Website: admin/client/CLIENT.md + - Library microservice: admin/servers/lib/LIB-MS.md - Services: admin/services.md - - Guides: + - Guides: - New user: admin/guides/add_user.md - New Service: admin/guides/add_service.md + - Install Gitlab: admin/guides/gitlab.md - Link services: admin/guides/link_service.md - Common Assets: admin/guides/common_workspace_readonly.md - HTTP Mode: admin/guides/hosting_site_without_https.md @@ -75,7 +76,7 @@ nav: use_directory_urls: false extra_css: - - docs/assets/css/extra.css + - assets/css/extra.css extra_javascript: - https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS-MML_HTMLorMML @@ -85,8 +86,8 @@ extra_javascript: markdown_extensions: - attr_list - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.highlight: linenums: true use_pygments: true diff --git a/script/base.sh b/script/base.sh index 8be4fb37d..06da3c331 100755 --- a/script/base.sh +++ b/script/base.sh @@ -54,7 +54,7 @@ sudo systemctl enable containerd.service #install docker-compose from https://docs.docker.com/compose/install/other/ -sudo curl -SL "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-linux-x86_64" \ +sudo curl -SL "https://github.com/docker/compose/releases/download/v2.23.3/docker-compose-linux-x86_64" \ -o /usr/local/bin/docker-compose sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose sudo chmod 755 /usr/local/bin/docker-compose /usr/bin/docker-compose @@ -69,7 +69,7 @@ then curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | \ sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg fi -NODE_MAJOR=18 +NODE_MAJOR=20 printf "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] \ https://deb.nodesource.com/node_%s.x nodistro main" "$NODE_MAJOR" | \ sudo tee /etc/apt/sources.list.d/nodesource.list diff --git a/servers/execution/runner/README.md b/servers/execution/runner/README.md index c0cb8434a..497f7cda8 100644 --- a/servers/execution/runner/README.md +++ b/servers/execution/runner/README.md @@ -38,7 +38,8 @@ yarn clean # Deletes directories "build", "coverage", and "dist" The Github actions workflow of [lib microservice](../../../.github/workflows/runner.yml) publishes the **runner** -into [public packages](https://github.com/orgs/INTO-CPS-Association/packages). +into +[packages](https://github.com/orgs/INTO-CPS-Association/packages?repo_name=DTaaS). ### Verdaccio - Local Package Registry diff --git a/servers/lib/DEVELOPER.md b/servers/lib/DEVELOPER.md index 186e72dbb..8b08460aa 100644 --- a/servers/lib/DEVELOPER.md +++ b/servers/lib/DEVELOPER.md @@ -12,31 +12,34 @@ yarn install # Install dependencies for the microservice yarn syntax # Analyze code for errors and style issues yarn graph # Generate dependency graphs in the code yarn build # Compile ES6 to ES5 and copy JS files to build/ directory -yarn test -a # Run all tests -yarn test -e # Run end-to-end tests -yarn test -i # Run integration tests -yarn test -u # Run unit tests -yarn start # Start the application +yarn test:unit # Run all tests +yarn test:e2e # Run end-to-end tests +yarn test:int # Run integration tests +yarn test:all # Run unit tests yarn start -h # List of all the CLI commands yarn start # Start the application -yarn clean # Deletes directories "build", "coverage", and "dist" +yarn clean # Deletes "build", "coverage", "dist" and other temp files ``` +**NOTE**: All tests require a valid `.env` file placed in the same directory +as `package.json`. + ## :package: :ship: NPM package ### Github Package Registry The Github actions workflow of -[lib microservice](../../.github/workflows/lib-ms.yml) publishes the __libms__ -into [public packages](https://github.com/orgs/INTO-CPS-Association/packages). +[lib microservice](../../.github/workflows/lib-ms.yml) publishes the **libms** +into +[packages](https://github.com/orgs/INTO-CPS-Association/packages?repo_name=DTaaS). ### Verdaccio - Local Package Registry Use the instructions in [publish npm package](../../docs/developer/npm-packages.md) for help -with publishing __libms npm package__ in local computer. +with publishing **libms npm package** in local computer. -Application of the advice given on that page for __libms__ will require +Application of the advice given on that page for **libms** will require running the following commands. ### Publish diff --git a/servers/lib/package.json b/servers/lib/package.json index 6c856f270..7d0aabff3 100644 --- a/servers/lib/package.json +++ b/servers/lib/package.json @@ -11,13 +11,17 @@ "private": false, "license": "SEE LICENSE IN ", "scripts": { - "build": "script/build.bash", - "clean": "script/clean.bash", + "build": "npx tsc", + "clean": "npx rimraf build node_modules coverage dist src.svg test.svg", "format": "prettier --ignore-path ../.gitignore --write \"**/*.{ts,tsx,css,scss}\"", - "start": "script/start.bash", - "syntax": "script/syntax.bash", - "test": "script/test.bash", - "graph": "script/graph.bash" + "graph": "npx madge --image src.svg src && npx madge --image test.svg test", + "start": "node dist/src/main.js", + "syntax": "npx eslint . --fix", + "pretest": "npx shx cp test/data/user2/tools/README.md ../../files/user2/tools/README.md", + "test:all": "jest --coverage", + "test:e2e": "npx shx cp test/data/user2/tools/README.md ../../files/user2/tools/ && npx cross-env TEST_PATH=test/data jest --config ./test/jest-e2e.json --coverage && npx rimraf ../../files/user2/tools/README.md", + "test:int": "npx shx cp test/data/user2/tools/README.md ../../files/user2/tools/ && npx cross-env TEST_PATH=test/data jest ../test/integration --coverage && npx rimraf ../../files/user2/tools/README.md", + "test:unit": "npx shx cp test/data/user2/tools/README.md ../../files/user2/tools/ && npx cross-env TEST_PATH=test/data jest ../test/unit --coverage && npx rimraf ../../files/user2/tools/README.md" }, "bin": "./dist/src/main.js", "dependencies": { diff --git a/servers/lib/script/build.bash b/servers/lib/script/build.bash deleted file mode 100755 index 2ccc7f10a..000000000 --- a/servers/lib/script/build.bash +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -new_path="$(yarn bin):$PATH" -export PATH="$new_path" -tsc \ No newline at end of file diff --git a/servers/lib/script/clean.bash b/servers/lib/script/clean.bash deleted file mode 100755 index a11e0e0f6..000000000 --- a/servers/lib/script/clean.bash +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -rm -rf build node_modules coverage dist "*.svg" \ No newline at end of file diff --git a/servers/lib/script/graph.bash b/servers/lib/script/graph.bash deleted file mode 100755 index 4546da092..000000000 --- a/servers/lib/script/graph.bash +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -PATH="$(yarn bin):$PATH" -export PATH -printf "Generate dependency graph for code" - -madge --image src.svg src -madge --image test.svg test \ No newline at end of file diff --git a/servers/lib/script/install.bash b/servers/lib/script/install.bash deleted file mode 100755 index dcf0491ec..000000000 --- a/servers/lib/script/install.bash +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -yarn install \ No newline at end of file diff --git a/servers/lib/script/start.bash b/servers/lib/script/start.bash deleted file mode 100755 index ed879816f..000000000 --- a/servers/lib/script/start.bash +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -printf "starting the node application\n" -node dist/src/main.js "$@" diff --git a/servers/lib/script/syntax.bash b/servers/lib/script/syntax.bash deleted file mode 100755 index 22370179d..000000000 --- a/servers/lib/script/syntax.bash +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -new_path="$(yarn bin):$PATH" -export PATH="$new_path" -printf "Running eslint" -eslint . --fix diff --git a/servers/lib/script/test.bash b/servers/lib/script/test.bash deleted file mode 100755 index 864bdae49..000000000 --- a/servers/lib/script/test.bash +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -echo "content123" > ../../files/user2/tools/README.md -printf "README.md file created with 'content123'\n" - -printf "testing in progress" -new_path="$(yarn bin):$PATH" -export PATH="$new_path" -test_path=$(pwd)/test/data -export TEST_PATH="$test_path" -# Get the first argument passed to the script -mode=$1 - -if [ "$mode" == "-a" ]; then - printf "making all tests ... and retrieving coverage report" - jest --coverage -elif [ "$mode" == "-u" ]; then - printf "making unit tests" - jest ../test/unit -elif [ "$mode" == "-i" ]; then - printf "making integration tests" - jest ../test/integration -elif [ "$mode" == "-e" ]; then - printf "making e2e tests" - jest --config ./test/jest-e2e.json -else - printf "Invalid option. Please use -a for all tests, -u for unit tests, -i for integration tests, or -e for e2e tests." -fi - -rm ../../files/user2/tools/README.md -printf "README.md file removed\n" \ No newline at end of file