A new and better Docker setup

.env

@@ -1,6 +1,10 @@
 KEYDB_PASSWORD=jgsay7f2fgfgda6acCa7g()jaba51!
+KEYDB_DOCKER_VERSION=latest
 POSTGRES_USER=sitespeedio
 POSTGRES_PASSWORD=jhde7gFArda33Saa1!4dd2dagJ
 POSTGRES_DB=sitespeedio
+POSTGRESQL_DOCKER_VERSION=16.2
 MINIO_ROOT_USER=sitespeedio
 MINIO_ROOT_PASSWORD=tracksofmytears
+MINIO_DOCKER_VERSION=RELEASE.2024-05-10T01-41-38Z
+MINIO_MC_DOCKER_VERSION=RELEASE.2024-10-08T09-37-26Z

.github/workflows/build-docker-server-release.yml

@@ -0,0 +1,51 @@
+name: Build Docker server container on new tag
+on:
+  push:
+    tags:
+      - 'server-v*.*.*'
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v4
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Extract version
+        id: extract_version
+        run: |
+            VERSION_TAG=${GITHUB_REF#refs/tags/}
+            echo "Git tag: $VERSION_TAG"
+
+            VERSION=${VERSION_TAG#v}
+            echo "Full version without 'v': $VERSION"
+
+            MAJOR_VERSION=${VERSION%%.*}
+            echo "Major version: $MAJOR_VERSION"
+
+            echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+            echo "MAJOR_VERSION=$MAJOR_VERSION" >> $GITHUB_OUTPUT
+      -
+        name: Build and push sitespeed.io
+        uses: docker/build-push-action@v6
+        with:
+          context: ./server/
+          file: ./server/Dockerile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          provenance: false
+          tags: |
+            sitespeedio/server:${{steps.extract_version.outputs.VERSION}}
+            sitespeedio/server:${{steps.extract_version.outputs.MAJOR_VERSION}}
+            sitespeedio/server:latest

.github/workflows/build-docker-testrunner-release.yml

@@ -0,0 +1,51 @@
+name: Build Docker testrunner container on new tag
+on:
+  push:
+    tags:
+      - 'testrunner-v*.*.*'
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v4
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Extract version
+        id: extract_version
+        run: |
+            VERSION_TAG=${GITHUB_REF#refs/tags/}
+            echo "Git tag: $VERSION_TAG"
+
+            VERSION=${VERSION_TAG#v}
+            echo "Full version without 'v': $VERSION"
+
+            MAJOR_VERSION=${VERSION%%.*}
+            echo "Major version: $MAJOR_VERSION"
+
+            echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+            echo "MAJOR_VERSION=$MAJOR_VERSION" >> $GITHUB_OUTPUT
+      -
+        name: Build and push sitespeed.io
+        uses: docker/build-push-action@v6
+        with:
+          context: ./testrunner/
+          file: ./testrunner/Dockerile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          provenance: false
+          tags: |
+            sitespeedio/testrunner:${{steps.extract_version.outputs.VERSION}}
+            sitespeedio/testrunner:${{steps.extract_version.outputs.MAJOR_VERSION}}
+            sitespeedio/testrunner:latest

.github/workflows/docker-local-linux.yml

@@ -0,0 +1,38 @@
+name: Test running everything in Docker using Linux
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Use Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20.x'
+    - uses: actions/checkout@v4
+    - name: Start dependencies
+      run: docker compose -f docker-compose.yml -f docker-compose-local.yml up -d
+    - name: Show versions
+      run: |
+        docker --version
+        docker network ls
+        docker network inspect skynet
+    - name: Get host IP
+      run: echo "HOST_IP=$(hostname -I | awk '{print $1}')" >> $GITHUB_ENV
+    - name: Run a test
+      run: |
+        git clone https://github.com/sitespeedio/sitespeed.io.git
+        cd sitespeed.io
+        npm install
+        bin/sitespeed.js https://www.wikipedia.org -n 1 --api.hostname 127.0.0.1  --api.location docker --headless --api.json --s3.endpoint "http://${{ env.HOST_IP }}:9000"
+    - name: Display Server log
+      if: failure() || success()
+      run: docker logs onlinetest-sitespeed.io-server-1
+    - name: Display testrunner log
+      if: failure() || success()
+      run: docker logs onlinetest-sitespeed.io-testrunner-1

docker-compose-local.yml

@@ -0,0 +1,46 @@
+services:
+  sitespeed.io-server:
+    image: sitespeedio/server:latest
+    environment:
+      redis_host: keydb
+      redis_port: 6379
+      redis_password: ${KEYDB_PASSWORD}
+      database_user: ${POSTGRES_USER}
+      database_server: postgresql
+      database_name: sitespeedio
+      database_password: ${POSTGRES_PASSWORD}
+      database_port: 5432
+    ports:
+      - "3000:3000"
+    networks:
+      - skynet
+    depends_on:
+      - keydb
+      - postgresql
+      - minio
+  sitespeed.io-testrunner:
+    image: sitespeedio/testrunner:latest
+    environment:
+      redis_host: keydb
+      redis_port: 6379
+      redis_password: ${KEYDB_PASSWORD}
+      docker_extraparameters: '--cap-add=NET_ADMIN --network=skynet'
+      sitespeed.io_s3_endpoint: "http://minio:9000" 
+      sitespeed.io_s3_bucketname: 'sitespeedio'
+      sitespeed.io_s3_key: 'sitespeedio'
+      sitespeed.io_s3_secret:  "tracksofmytears"
+      sitespeed.io_s3_region: "motown"
+      sitespeed.io_s3_options_forcePathStyle: true
+      sitespeed.io_s3_removeLocalResult: true
+      sitespeed.io_resultBaseURL: "http://127.0.0.1:9000/sitespeedio"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /tmp:/tmp
+    cap_add:
+      - NET_ADMIN
+    depends_on:
+      - keydb
+      - minio
+      - sitespeed.io-server
+    networks:
+      - skynet

docker-compose.yml

@@ -1,53 +1,61 @@
 services:
-    keydb:
-      image: eqalpha/keydb:latest
-      hostname: keydb
-      ports:
-        - "6379:6379"
-      command: ["keydb-server", "/etc/keydb/keydb.conf", "--server-threads" , "2"]
-      restart: always
-      volumes:
-        - keydb_data:/data
-        - ./keydb.conf:/etc/keydb/keydb.conf 
-    postgresql:
-      image: postgres:16.2
-      hostname: postgresql
-      ports:
-        - "5432:5432"
-      environment:
-        - POSTGRES_USER=${POSTGRES_USER}
-        - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-        - POSTGRES_DB=sitespeedio
-      volumes:
-        - ./server/database/setup/:/docker-entrypoint-initdb.d
-        - postgres-data:/var/lib/postgresql/data
-    minio:
-      image: minio/minio:RELEASE.2024-05-10T01-41-38Z
-      volumes:
-        - minio-data:/data
-      ports:
-        - "9000:9000"
-        - "9001:9001"
-      environment:
-        - MINIO_ROOT_USER=${MINIO_ROOT_USER}
-        - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}
-      command: server /data --console-address ":9001"
-    mc:
-      image: minio/mc:RELEASE.2024-05-09T17-04-24Z
-      depends_on:
-        - minio
-      entrypoint: >
-        /bin/sh -c "
-        /usr/bin/mc alias set sitespeedio http://minio:9000 sitespeedio tracksofmytears &&
-        (/usr/bin/mc mb --region --region=motown sitespeedio/sitespeedio || echo 'Bucket already exists') &&
-        /usr/bin/mc ilm rule add --expire-days 30 sitespeedio/sitespeedio &&
-        /usr/bin/mc anonymous set public myminio/sitespeedio &&
-        /usr/bin/mc admin config set sitespeedio compression extensions=".html,.css,.js" &&
-        echo 'MinIO and lifecycle policy setup complete.'"
-      environment:
-        MC_HOST_myminio: http://sitespeedio:tracksofmytears@minio:9000
+  keydb:
+    image: eqalpha/keydb:${KEYDB_DOCKER_VERSION}
+    hostname: keydb
+    ports:
+      - "6379:6379"
+    command: ["keydb-server", "/etc/keydb/keydb.conf", "--server-threads" , "2"]
+    restart: always
+    networks:
+      - skynet
+    volumes:
+      - keydb_data:/data
+      - ./keydb.conf:/etc/keydb/keydb.conf 
+  postgresql:
+    image: postgres:${POSTGRESQL_DOCKER_VERSION}
+    hostname: postgresql
+    ports:
+      - "5432:5432"
+    networks:
+      - skynet
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=sitespeedio
+    volumes:
+      - ./server/database/setup/:/docker-entrypoint-initdb.d
+      - postgres-data:/var/lib/postgresql/data
+  minio:
+    image: minio/minio:${MINIO_DOCKER_VERSION}
+    volumes:
+      - minio-data:/data
+    ports:
+      - "9000:9000"
+    environment:
+      - MINIO_ROOT_USER=${MINIO_ROOT_USER}
+      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}
+    command: server /data --console-address ":9001"
+    networks:
+      - skynet
+  mc:
+    image: minio/mc:${MINIO_MC_DOCKER_VERSION}
+    depends_on:
+      - minio
+    networks:
+      - skynet
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set sitespeedio http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} &&
+      (/usr/bin/mc mb --region=motown sitespeedio/sitespeedio || echo 'Bucket already exists') &&
+      /usr/bin/mc ilm rule add --expire-days 30 sitespeedio/sitespeedio &&
+      /usr/bin/mc anonymous set public sitespeedio/sitespeedio &&
+      /usr/bin/mc admin config set sitespeedio compression extensions=\".html,.css,.js\" &&
+      echo 'MinIO and lifecycle policy setup complete.'"
 volumes:
   postgres-data:
   minio-data:
   keydb_data:
-    driver: local
+    driver: local
+networks:
+  skynet:
+    name: skynet

server/Dockerfile

@@ -4,9 +4,10 @@ WORKDIR /usr/src/app
 
 COPY package.json npm-shrinkwrap.json ./
 
-RUN npm ci --omit=dev
+RUN npm ci --omit=dev && npm cache clean --force
 
 COPY . .
+COPY config/default.yaml /usr/src/app/config/default.yaml
 
 RUN useradd --user-group --create-home --shell /bin/false sitespeed && \
     chown -R sitespeed:sitespeed /usr/src/app
@@ -16,4 +17,4 @@ USER sitespeed
 EXPOSE 3000
 
 ENTRYPOINT ["node", "app.js"]
-CMD []
+CMD []

server/app.js

@@ -41,6 +41,8 @@ const configFile = nconf.get('config')
 const fileExtension = extname(configFile).toLowerCase();
 let configFromFile;
 
+nconf.env({ parseValues: true, separator: '_' });
+
 try {
   const fileContent = fs.readFileSync(configFile, 'utf8');
   if (fileExtension === '.json') {

server/config/default.yaml

@@ -15,7 +15,7 @@ redis:
 # Configure the postgresql database
 database:
   user: sitespeedio
-  server: localhost
+  server: 127.0.0.1
   name: sitespeedio
   password: jhde7gFArda33Saa1!4dd2dagJ
   port: 5432
@@ -87,9 +87,6 @@ admin:
   basicAuth:
     login: appojke
     password: monkeyboy 
-
-# The default sitspeed.io configuration file that will be merged to all tests.
-# defaultSitespeedioConfigFile: 
 
 # The number of jobs to keep in the queue
 # Depending on how many jobs you run and how much memory 
@@ -98,12 +95,10 @@ queue:
   removeOnComplete: 50
   removeOnFail: 50
 
-sitespeed.io:
-  s3:
-    endpoint: "http://127.0.0.1:9000"
-    bucketname: "sitespeedio"
-    key: "sitespeedio"
-    secret: "tracksofmytears"
-    region: "motown"
-
-  resultBaseURL: "http://127.0.0.1:9000/sitespeedio"
+
+# The default sitspeed.io configuration file that will be merged to all tests.
+# defaultSitespeedioConfigFile: 
+
+# If you don't setup a default sitespeed.io configuration, you can add 
+# your sitespeed.io configuration directly in this block
+sitespeed.io: