Adds http mode to library microservice (INTO-CPS-Association#480 and I…

…NTO-CPS-Association#345)

  - Adds http file service to library microservice
    using cloudcmd package
  - Adds yarn commands to libms using pm2
    npm package
  - Updates diagrams in developer documentation
  - Adds libms tests to pre-commit configuration

---------
Co-authored-by: Mads Kelberg <[email protected]>
Co-authored-by: Mads Kelberg <[email protected]>

.github/workflows/lib-ms.yml

@@ -23,10 +23,14 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
           cache: "yarn"
           cache-dependency-path: "**/yarn.lock"
 
+      # - name: Install pm2
+      #   run: |
+      #     npm install -g pm2
+
       - name: Run the linting checks
         run: |
           cd servers/lib
@@ -43,6 +47,7 @@ jobs:
         run: |
           cd servers/lib
           yarn install
+          yarn build
           yarn test:all
         env:
           PORT: 4001
@@ -51,6 +56,19 @@ jobs:
           LOG_LEVEL: debug
           APOLLO_PATH: /lib
 
+      # - name: Run http mode tests
+      #   run: |
+      #     cd servers/lib
+      #     yarn install
+      #     yarn build
+      #     yarn test:http-github
+      #   env:
+      #     PORT: 4002
+      #     LOCAL_PATH: ${{ github.workspace }}/files
+      #     MODE: local
+      #     LOG_LEVEL: debug
+      #     APOLLO_PATH: /lib
+
       - name: Upload test coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

.pre-commit-config.yaml

@@ -36,13 +36,13 @@ repos:
         files: "^servers/execution/runner/.*"
         args: ["-c", "cd servers/execution/runner && yarn test:nocov"]
         stages: [pre-push]
-      # - id: yarn-test-lib
-      #   name: yarn test lib
-      #   entry: bash
-      #   language: system
-      #   files: "^servers/lib/.*"
-      #   args: ["-c", "cd servers/lib && yarn jest . --coverage=false"]
-      #   stages: [pre-push]
+      - id: yarn-test-lib
+        name: yarn test lib
+        entry: bash
+        language: system
+        files: "^servers/lib/.*"
+        args: ["-c", "cd servers/lib && yarn test:nocov"]
+        stages: [pre-push]
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.1.0

script/base.sh

@@ -90,3 +90,4 @@ fi
 sudo apt-get update -y
 sudo apt-get install -y yarn
 sudo npm install -g serve
+sudo npm install -g pm2

servers/lib/DEVELOPER.md

@@ -13,16 +13,19 @@ yarn syntax     # Analyze code for errors and style issues
 yarn format     #format .ts[x] and .js[x] files with prettier
 yarn graph      # Generate dependency graphs in the code
 yarn build      # Compile ES6 to ES5 and copy JS files to build/ directory
-yarn test:unit  # Run all tests
-yarn test:e2e   # Run end-to-end tests
+yarn test:unit  # Run unit tests
 yarn test:int   # Run integration tests
-yarn test:all   # Run unit tests
+yarn test:e2e   # Run end-to-end tests
+yarn test:http  # Run tests for HTTP file service provided by cloudcmd
+yarn test:all   # Run all tests except http tests
 yarn start -h   # List of all the CLI commands
 yarn start      # Start the application
+yarn start:pm2  # Start the application with process manager 2
+yarn stop:pm2   # Stop the application managed by process manager 2
 yarn clean      # Deletes "build", "coverage", "dist" and other temp files
 ```
 
-**NOTE:** The integration and end-to-end tests require a valid
+**NOTE:** The integration, end-to-end and http tests require a valid
 `.env` file. Here is a sample file.
 
 ```ini

servers/lib/README.md

@@ -40,7 +40,8 @@ GRAPHQL_PLAYGROUND='false' or 'true'
 
 The `LOCAL_PATH` variable is the absolute filepath to the
 location of the local directory which will be served to users
-by the Library microservice.
+by the Library microservice. This
+[sample configuration file](./config/.env.default) can be used.
 
 Replace the default values the appropriate values for your setup.
 
@@ -49,10 +50,24 @@ Replace the default values the appropriate values for your setup.
 Display help.
 
 ```bash
-libms -h
+$libms -h
+Usage: libms [options]
+
+The lib microservice is a file server. It supports file transfer
+over GraphQL and HTTP protocols.
+
+Options:
+  -c, --config <file>  provide the config file (default .env)
+  -H, --http <file>    enable the HTTP server with the specified config
+  -h, --help           display help for libms
 ```
 
-The config is saved `.env` file by convention. The **libms** looks for
+Both the options are not mandatory.
+
+### Configuration file
+
+The config is saved `.env` file by convention. If `-c` is not specified
+The **libms** looks for
 `.env` file in the working directory from which it is run.
 If you want to run **libms** without explicitly specifying the configuration
 file, run
@@ -69,15 +84,35 @@ libms --config FILE-PATH
 ```
 
 If the environment file is named something other than `.env`,
-for example as `.env.development`, you can run
+for example as `config/.env.default`, you can run
 
 ```sh
-libms -c ".env.development"
+libms -c "config/.env.default"
 ```
 
 You can press `Ctl+C` to halt the application.
 
-The microservice is available at: 'localhost:PORT/lib'
+### Protocol Support
+
+The **libms** supports GraphQL protocol by default.
+It is possible to enable the HTTP protocol by setting
+the `-H` option.
+
+To run **libms** with a custom config for HTTP protocol, use
+
+```bash
+libms -H FILE-PATH
+libms --http FILE-PATH
+```
+
+A sample configuration is [available](./config/http.json).
+
+### Accessible URLs
+
+The microservice is available at:
+
+**GraphQL protocol**: 'localhost:PORT/lib'
+**HTTP protocol**: 'localhost:PORT/lib/files'
 
 <!-- markdownlint-disable-next-line MD013 -->
 The [API](https://into-cps-association.github.io/DTaaS/development/user/servers/lib/LIB-MS.html) page shows sample queries and responses.

servers/lib/config/.env.default

@@ -0,0 +1,6 @@
+PORT='4001'
+MODE='local' or 'gitlab'
+LOCAL_PATH ='/Users/<Username>/DTaaS/files'
+LOG_LEVEL='debug'
+APOLLO_PATH='/lib' or ''
+GRAPHQL_PLAYGROUND='false' or 'true'

servers/lib/config/http.json

@@ -0,0 +1,30 @@
+{
+  "name": "DTaaS Fileserver",
+  "auth": false,
+  "editor": "edward",
+  "packer": "zip",
+  "diff": true,
+  "zip": true,
+  "buffer": true,
+  "dirStorage": true,
+  "online": false,
+  "open": false,
+  "oneFilePanel": true,
+  "keysPanel": false,
+  "prefix": "/lib/files",
+  "confirmCopy": true,
+  "confirmMove": true,
+  "showConfig": false,
+  "showFileName": true,
+  "contact": false,
+  "configDialog": false,
+  "console": false,
+  "terminal": false,
+  "vim": false,
+  "columns": "name-size-date-owner-mode",
+  "export": false,
+  "import": false,
+  "dropbox": false,
+  "dropboxToken": "",
+  "log": true
+}

servers/lib/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@into-cps-association/libms",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "microservices that handles request by fetching and returning the file-names and folders of given directory",
   "author": "[email protected]",
   "contributors": [
@@ -16,12 +16,18 @@
     "format": "prettier --ignore-path ../.gitignore --write \"**/*.{ts,tsx,css,scss}\"",
     "graph": "npx madge --image src.svg src && npx madge --image test.svg test",
     "start": "node dist/src/main.js",
+    "start:pm2": "pm2 start pm2.config.js",
+    "stop:pm2": "pm2 delete libms",
     "syntax": "npx eslint . --fix",
     "pretest": "npx shx cp test/data/user2/tools/README.md ../../files/user2/tools/README.md",
     "posttest": "npx rimraf ../../files/user2/tools/README.md",
-    "test:all": "npx cross-env LOCAL_PATH=test/data jest --coverage",
+    "test:all": "npx cross-env LOCAL_PATH=test/data jest --testPathIgnorePatterns=cloudcmd --coverage",
     "test:e2e": "npx cross-env LOCAL_PATH=test/data jest --config ./test/jest-e2e.json --coverage",
+    "test:http": "yarn build && pm2 start -f --name libms-test dist/src/main.js -- -c .env -H ./config/http.json && jest test/cloudcmd --coverage --coverageThreshold=\"{}\" && pm2 delete libms-test",
+    "test:http-nocov": "yarn build && pm2 start -f --name libms-test dist/src/main.js -- -c .env -H ./config/http.json && jest test/cloudcmd  --coverage=false && pm2 delete libms-test",
+    "test:http-github": "yarn build && yarn start:pm2 && jest test/cloudcmd --coverage --coverageThreshold=\"{}\" && pm2 delete libms",
     "test:int": "npx cross-env LOCAL_PATH=test/data jest ../test/integration --coverage",
+    "test:nocov": "yarn test:http-nocov && npx cross-env LOCAL_PATH=test/data jest --testPathIgnorePatterns=cloudcmd --coverage=false",
     "test:unit": "npx cross-env LOCAL_PATH=test/data jest ../test/unit --coverage"
   },
   "bin": "./dist/src/main.js",
@@ -34,12 +40,15 @@
     "@nestjs/core": "^10.3.0",
     "@nestjs/graphql": "^12.0.11",
     "@nestjs/platform-express": "^10.3.0",
+    "axios": "^1.5.1",
+    "cloudcmd": "^16.17.7",
     "commander": "^11.1.0",
     "dotenv": "^16.3.1",
     "graphql": "^16.8.1",
     "mock-fs": "^5.2.0",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1",
+    "socket.io": "^4.7.2",
     "type-graphql": "^2.0.0-beta.3"
   },
   "devDependencies": {
@@ -52,6 +61,7 @@
     "@types/supertest": "^2.0.13",
     "@typescript-eslint/eslint-plugin": "^6.19.0",
     "@typescript-eslint/parser": "^6.19.0",
+    "cross-fetch": "^4.0.0",
     "eslint": "^8.56.0",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-config-prettier": "^9.1.0",

servers/lib/pm2.config.js

@@ -0,0 +1,10 @@
+module.exports = {
+  apps : [
+      {
+        name: "libms",
+        script: "./dist/src/main.js",
+        watch: false,
+        args: "-H ./config/http.json"
+      }
+  ]
+}

servers/lib/src/bootstrap.ts

@@ -2,9 +2,11 @@ import { NestFactory } from '@nestjs/core';
 import { ConfigService } from '@nestjs/config';
 import * as dotenv from 'dotenv';
 import AppModule from './app.module';
+import cloudCMD from './cloudcmd/cloudcmd';
 
 type BootstrapOptions = {
   config?: string;
+  httpServer?: string;
   runHelp?: CallableFunction;
 };
 
@@ -22,8 +24,14 @@ export default async function bootstrap(options?: BootstrapOptions) {
       process.exit(1);
     }
   }
+
   const app = await NestFactory.create(AppModule);
   const configService = app.get(ConfigService);
   const port = configService.get<number>('PORT');
+
+  if (options.httpServer) {
+    cloudCMD(app, options.httpServer, configService.get<string>('LOCAL_PATH'));
+  }
+
   await app.listen(port);
 }

servers/lib/src/cloudcmd/cloudcmd.ts

@@ -0,0 +1,32 @@
+import { INestApplication } from '@nestjs/common';
+import { Server } from 'socket.io';
+import * as cloudcmd from 'cloudcmd';
+import { join } from 'path';
+
+const runCloudCMD = (
+  app: INestApplication,
+  optionsPath: string,
+  filesPath: string,
+) => {
+  const { createConfigManager } = cloudcmd;
+  const configManager = createConfigManager({
+    configPath: join(process.cwd(), optionsPath),
+  });
+
+  configManager('root', filesPath);
+
+  const server = app.getHttpServer();
+
+  const socket = new Server(server, {
+    path: `${configManager('prefix')}/socket.io`,
+  });
+  app.use(
+    configManager('prefix'),
+    cloudcmd({
+      configManager,
+      socket,
+    }),
+  );
+};
+
+export default runCloudCMD;

servers/lib/src/main.ts

@@ -4,24 +4,29 @@ import bootstrap from './bootstrap';
 
 type ProgramOptions = {
   config?: string;
+  http?: string;
 };
 
 const program = new Command();
 
 program
   .description(
-    'The lib microservice is responsible for handling and serving the contents of library assets of the DTaaS platform. It provides API endpoints for clients to query, and fetch these assets.',
+    'The lib microservice is a file server. It supports file transfer over GraphQL and HTTP protocols.',
+  )
+  .option('-c, --config <file>', 'provide the config file (default .env)')
+  .option(
+    '-H, --http <file>',
+    'enable the HTTP server with the specified config',
   )
-  .option('-c, --config <path>', 'set the config path (default .env)')
   .helpOption('-h, --help', 'display help for libms')
   .showHelpAfterError();
 
 program.parse(process.argv);
 
 const options: ProgramOptions = program.opts();
 
-if (options.config) {
-  bootstrap({ config: options.config, runHelp: () => program.help() });
-} else {
-  bootstrap({ runHelp: () => program.help() });
-}
+bootstrap({
+  config: options.config,
+  httpServer: options.http,
+  runHelp: () => program.help(),
+});