refact!: dockerization + federation/multi-node portal

.github/workflows/build.yml

@@ -0,0 +1,158 @@
+on:
+  release:
+    types: [ published ]
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+jobs:
+  build-server:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log into the container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up edge/PR server image metadata
+        id: meta-edge
+        uses: docker/metadata-action@v5
+        if: ${{ github.event_name != 'release' }}
+        with:
+          images: |
+            ghcr.io/c3g/epivar-server
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=edge,enable={{is_default_branch}}
+            type=ref,event=pr
+            type=sha,prefix=sha-
+
+      - name: Build and push edge/PR server image
+        uses: docker/build-push-action@v5
+        if: ${{ github.event_name != 'release' }}
+        with:
+          context: .
+          file: spec/server.Dockerfile
+          push: true
+          tags: ${{ steps.meta-edge.outputs.tags }}
+          labels: ${{ steps.meta-edge.outputs.labels }}
+          platforms: linux/amd64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Set up server release image metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        if: ${{ github.event_name == 'release' }}
+        with:
+          images: |
+            ghcr.io/c3g/epivar-server
+          flavor: |
+            latest=true
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build and push server release image
+        uses: docker/build-push-action@v5
+        if: ${{ github.event_name == 'release' }}
+        with:
+          context: .
+          file: spec/server.Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  build-portal:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log into the container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up edge/PR portal image metadata
+        id: meta-edge
+        uses: docker/metadata-action@v5
+        if: ${{ github.event_name != 'release' }}
+        with:
+          images: |
+            ghcr.io/c3g/epivar-portal
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=edge,enable={{is_default_branch}}
+            type=ref,event=pr
+            type=sha,prefix=sha-
+
+      - name: Build and push edge/PR portal image
+        uses: docker/build-push-action@v5
+        if: ${{ github.event_name != 'release' }}
+        with:
+          context: .
+          file: spec/portal.Dockerfile
+          push: true
+          tags: ${{ steps.meta-edge.outputs.tags }}
+          labels: ${{ steps.meta-edge.outputs.labels }}
+          platforms: linux/amd64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Set up portal release image metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        if: ${{ github.event_name == 'release' }}
+        with:
+          images: |
+            ghcr.io/c3g/epivar-portal
+          flavor: |
+            latest=true
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build and push portal release image
+        uses: docker/build-push-action@v5
+        if: ${{ github.event_name == 'release' }}
+        with:
+          context: .
+          file: spec/portal.Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.gitignore

@@ -1,11 +1,10 @@
+~$*
 /.idea
-/data/*
-!/data/.gitkeep
-!/data/otherData
-!/data/ucsc.other-tracks.txt
 /input-files/*.csv
 !/input-files/flu-infection-gene-peaks.csv
 /config.js
 /public/*
 node_modules
 /.env
+/*.gff3
+/*.gff3.gz

app.mjs

@@ -1,4 +1,5 @@
-const path = await import('path');
+const path = await import('node:path');
+import cors from "cors";
 import express from 'express';
 // const favicon = await import('serve-favicon');
 import logger from 'morgan';
@@ -8,23 +9,29 @@ import passport from "passport";
 import { fileURLToPath } from "url";
 import { createClient } from "redis";
 
-(await import('dotenv')).config();
+import { REDIS_CONNECTION, MERGED_TRACKS_DIR, OTHER_DATA_PATH, SESSION_SECRET, PORTAL_ORIGIN } from "./envConfig.js";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const app = express();
 
 // view engine setup
-app.set('views', path.join(__dirname, 'views'))
-app.set('view engine', 'pug')
-
+app.set('views', path.join(__dirname, 'views'));
+app.set('view engine', 'pug');
+
+app.use(cors({
+  origin: PORTAL_ORIGIN,
+  methods: ["GET", "POST"],
+  credentials: true,
+  preflightContinue: true,
+}));
 // uncomment after placing your favicon in /public
 //app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')))
-app.use(logger(':req[x-real-ip] [:date[clf]] :method :url :status :response-time ms - :res[content-length]'))
-app.use(bodyParser.json())
-app.use(bodyParser.urlencoded({ extended: false }))
-app.use(cookieParser())
-app.use(express.static(path.join(__dirname, 'public')))
+app.use(logger(':req[x-real-ip] [:date[clf]] :method :url :status :response-time ms - :res[content-length]'));
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ extended: false }));
+app.use(cookieParser());
+app.use(express.static(path.join(__dirname, 'public')));
 
 // Set up auth if it's enabled
 if (!process.env.VARWIG_DISABLE_AUTH) {
@@ -36,11 +43,11 @@ if (!process.env.VARWIG_DISABLE_AUTH) {
   const RedisStore = connectRedis(session);
 
   // Legacy mode redis client for connect-redis
-  const redisClient = createClient({legacyMode: true});
+  const redisClient = createClient({ url: REDIS_CONNECTION, legacyMode: true });
   redisClient.connect().catch(console.error);
 
   app.use(session({
-    secret: process.env.VARWIG_SESSION_SECRET,
+    secret: SESSION_SECRET,
     httpOnly: false,
     resave: false,
     saveUninitialized: false,
@@ -52,21 +59,28 @@ if (!process.env.VARWIG_DISABLE_AUTH) {
   app.use(passport.authenticate("session"));
   app.use("/api/auth",       (await import("./routes/auth.mjs")).default);
 }
+//  - utility routers
+app.use('/api/messages',     (await import('./routes/messages.mjs')).default);
 
+//  - common data
 app.use('/api/assays',       (await import('./routes/assays.mjs')).default);
+//  - dataset title, dataset, conditions, ethnicities:
+app.use('/api/dataset',      (await import('./routes/dataset.mjs')).default);
+//  - data access routers
 app.use('/api/autocomplete', (await import('./routes/autocomplete.mjs')).default);
-app.use('/api/messages',     (await import('./routes/messages.mjs')).default);
-app.use('/api/assembly',     (await import('./routes/assembly.mjs')).default);
 app.use('/api/overview',     (await import('./routes/overview.mjs')).default);
-app.use('/api/conditions',   (await import('./routes/conditions.mjs')).default);
-app.use('/api/ethnicities',  (await import('./routes/ethnicities.mjs')).default);
 app.use('/api/peaks',        (await import('./routes/peaks.mjs')).default);
 app.use('/api/sessions',     (await import('./routes/sessions.mjs')).default);
 app.use('/api/tracks',       (await import('./routes/tracks.mjs')).default);
 //  - routers for genome browsers
 app.use('/api/igvjs',        (await import('./routes/igvjs.mjs')).default);
 app.use('/api/ucsc',         (await import('./routes/ucsc.mjs')).default);
 
+// Data static file hosting
+// - merged tracks
+app.use('/api/merged',    express.static(MERGED_TRACKS_DIR));
+// - other data
+app.use('/api/otherData', express.static(OTHER_DATA_PATH));
 
 // catch 404 and forward to error handler
 app.use((req, res, next) => {

client/.babelrc.js

@@ -0,0 +1,18 @@
+module.exports = {
+  "presets": [
+    ["@babel/preset-env", {
+      "modules": false
+    }],
+    ["@babel/preset-react", {
+      runtime: "automatic",
+      development: process.env.NODE_ENV === "development"
+    }],
+  ],
+  "plugins": [
+    "@babel/syntax-dynamic-import",
+    ["import", {
+      "libraryName": "antd",
+      "style": true,
+    }],
+  ],
+};

client/create_config_prod.js

@@ -0,0 +1,11 @@
+const siteConfig = {
+  NODES: (process.env.EPIVAR_NODES ?? "").split(/\s*;\s*/).filter((v) => !!v),
+};
+
+if (typeof require !== "undefined" && require.main === module) {
+  process.stdout.write(`EPIVAR_CONFIG = ${JSON.stringify(siteConfig, null, 2)};\n`);
+}
+
+module.exports = {
+  siteConfig,
+};