publish docker images/image updates for major versions (#1372)

* feat: orioledb-17 Docker image and publish workflow

* feat: refactor to build docker images from major versions
skip if no Dockerfile exists

* feat: gen common vars hcl file

* chore: cleanup

* chore: adj trigger for run in PR`

* chore: trigger release all major vers

* chore: trigger on any path

* chore: cleanup version handling

* chore: simplify name matching

* chore: matrix on version and arch in build_release_image

* chore: try to get json formatting correct

* chore: more fixes to json parse

* feat: collect all versions into file, then array

* chore: try to account for gh action handling

* chore: output single matrix for each version

* chore: more fixes to matrix handling

* chore : fix output versions

* chore: matrix in all tasks

* chore: refactor matrix handling

* chore: fix syntax

* chore: look in the right dir

* chore: need to pass full tag

* fix: formatting

* chore: strip the prefix when passing to mirror.yml

* chore: get version formatted to pass to mirror.yml

* fix: typo

* chore: fix vars

* fix: tryo to pass the version

* chore: matrix on version and arch

* chore: try to account for orioledb version issue

* chore: remove version

* chore: reading the version info directly from vars.yml

* chore: force string context

* chore: fixing logic

* chore: fix string handling logic

* chore: use sed and grep instead

* chore: revert this to working version

* chore: version extraction

* chore: different logic on version processing if stmnt

* chore: try to avoid jq involvement in this section

* chore: back to this version

* chore: refactor using nushell where scripting needed

* chore: use data from previous step

* chore: install with snap

* chore: raw output on json

* chore: should be able to run on ephemeral runners

* chore: incrementing changes to matrix

* chore: just use special key name if present

* chore: account for postgres namespace in matrix

* chore: pg_version should be key/value

* chrore: ubuntu latest

* chore: needs arm-runner after all

* chore: source nushell from nix

* chore: cannot have uses and run keys on same id

* chore: detect runner and adjust the way command is run

* chore: formatting

* chore: make sure env var can be appended in all contexts

* chore: outputs instead of env

* chore: fix get_publish_version

* chore: handle oriole

* chore: remove base64 encoding

* ore: raw json

* chore: ensure proper variable naming

* chore: use the matrix config we already have access to

* chore: handling architecture

* chore directly use matrix

* chore: process each version

* chore: matrix_json variable

* fix: mirroring the merged manifest instead of per arch image

* chore: using tags from merge_manifest

* chore: no longer need get_publish_version at all

* chore: combine tags in outputs at the end of merge_manifest

* chore: cleanup steps to unbreak yaml

* chore: no from json

* chore: try to build up json over iterations

* chore: use the github outputs array

* chore: first collect all versions into array then output json array

* chore: debug mainfest output

* chore: store the results data in artifacts

* chore: unique upload, then download and combine

* fix: re-add actions and login

* chore: try quotes for name matching

* chore: utilise versions from prepare for download of artifact

* chore: format correctly for nushell

* chore: parse matrix config directly

* chore: convert table to list of strings

* chore:rm redundant

* chore: add debug

* chore: from json

* chore: download with pattern

* chore: use list operations

* chore: include and debug

* chore: extract version

* chore: set up for merge to develop

* chore: add permission to proper dir

* chore: remove old files

* chore: newline

.github/workflows/dockerhub-release-matrix.yml

@@ -0,0 +1,240 @@
+name: Release all major versions on Dockerhub
+
+on:
+  push:
+    branches:
+      - develop
+      - release/*
+    paths:
+      - ".github/workflows/dockerhub-release-matrix.yml"
+  workflow_dispatch:
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix_config: ${{ steps.set-matrix.outputs.matrix_config }}
+    steps:
+      - uses: DeterminateSystems/nix-installer-action@main
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Generate build matrix
+        id: set-matrix
+        run: |
+          nix run nixpkgs#nushell -- -c 'let versions = (open ansible/vars.yml | get postgres_major)
+          let matrix = ($versions | each { |ver|
+            let version = ($ver | str trim)
+            let dockerfile = $"Dockerfile-($version)"
+            if ($dockerfile | path exists) {
+              {
+                version: $version,
+                dockerfile: $dockerfile
+              }
+            } else {
+              null
+            }
+          } | compact)
+
+          let matrix_config = {
+            include: $matrix
+          }
+
+          $"matrix_config=($matrix_config | to json -r)" | save --append $env.GITHUB_OUTPUT'
+  build:
+    needs: prepare
+    strategy:
+      matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }}
+    runs-on: ubuntu-latest
+    outputs:
+      build_args: ${{ steps.args.outputs.result }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: DeterminateSystems/nix-installer-action@main
+      - name: Set PostgreSQL version environment variable
+        run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV
+
+      - id: args
+        run: |
+          nix run nixpkgs#nushell -- -c '
+            open ansible/vars.yml 
+            | items { |key value| {name: $key, item: $value} } 
+            | where { |it| ($it.item | describe) == "string" } 
+            | each { |it| $"($it.name)=($it.item)" } 
+            | str join "\n" 
+            | save --append $env.GITHUB_OUTPUT
+          '    
+  build_release_image:
+    needs: [prepare, build]
+    strategy:
+      matrix:
+        postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }}
+        arch: [amd64, arm64]
+    runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'arm-runner' }}
+    timeout-minutes: 180
+    steps:
+      - uses: actions/checkout@v3
+      - uses: DeterminateSystems/nix-installer-action@main
+      - run: docker context create builders
+      - uses: docker/setup-buildx-action@v3
+        with:
+          endpoint: builders
+      - uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Get image tag
+        id: image
+        run: |
+          if [[ "${{ matrix.arch }}" == "arm64" ]]; then
+            pg_version=$(sudo nix run nixpkgs#nushell -- -c '
+              let version = "${{ matrix.postgres.version }}"
+              let release_key = if ($version | str contains "orioledb") {
+                $"postgresorioledb-17"
+              } else {
+                $"postgres($version)"
+              }
+              open ansible/vars.yml | get postgres_release | get $release_key | str trim
+            ')
+            echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT
+          else
+            pg_version=$(nix run nixpkgs#nushell -- -c '
+              let version = "${{ matrix.postgres.version }}"
+              let release_key = if ($version | str contains "orioledb") {
+                $"postgresorioledb-17"
+              } else {
+                $"postgres($version)"
+              }
+              open ansible/vars.yml | get postgres_release | get $release_key | str trim
+            ')
+            echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT
+          fi
+      - id: build
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          build-args: |
+            ${{ needs.build.outputs.build_args }}
+          target: production
+          tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }}
+          platforms: linux/${{ matrix.arch }}
+          cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
+          cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
+          file: ${{ matrix.postgres.dockerfile }}
+  merge_manifest:
+    needs: [prepare, build, build_release_image]
+    strategy:
+      matrix:
+        include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Get image tag
+        id: get_version
+        run: |
+          nix run nixpkgs#nushell -- -c '
+            let version = "${{ matrix.version }}"
+            let release_key = if ($version | str contains "orioledb") {
+              $"postgresorioledb-17"
+            } else {
+              $"postgres($version)"
+            }
+            let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim)
+            $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT
+          '
+      - name: Output version
+        id: output_version
+        run: |
+          echo "result=${{ steps.get_version.outputs.pg_version }}" >> $GITHUB_OUTPUT
+      - name: Collect versions
+        id: collect_versions
+        run: |
+          echo "${{ steps.output_version.outputs.result }}" >> results.txt  # Append results
+      - name: Upload Results Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: merge_results-${{ matrix.version }}
+          path: results.txt
+          if-no-files-found: warn
+      - name: Merge multi-arch manifests
+        run: |
+          docker buildx imagetools create -t ${{ steps.get_version.outputs.pg_version }} \
+          ${{ steps.get_version.outputs.pg_version }}_amd64 \
+          ${{ steps.get_version.outputs.pg_version }}_arm64
+  combine_results:
+    needs: [prepare, merge_manifest]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: DeterminateSystems/nix-installer-action@main
+
+      - name: Debug Input from Prepare
+        run: |
+          echo "Raw matrix_config output:"
+          echo "${{ needs.prepare.outputs.matrix_config }}"
+      - name: Get Versions from Matrix Config
+        id: get_versions
+        run: |
+          nix run nixpkgs#nushell -- -c '
+            # Parse the matrix configuration directly
+            let matrix_config = (${{ toJson(needs.prepare.outputs.matrix_config) }} | from json)
+      
+            # Get versions directly from include array
+            let versions = ($matrix_config.include | get version)
+
+            echo "Versions: $versions"
+
+            # Convert the versions to a comma-separated string
+            let versions_str = ($versions | str join ",")
+            $"versions=$versions_str" | save --append $env.GITHUB_ENV
+          '
+      - name: Download Results Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          pattern: merge_results-*
+      - name: Combine Results
+        id: combine
+        run: |
+          nix run nixpkgs#nushell -- -c '
+            # Get all results files and process them in one go
+            let files = (ls **/results.txt | get name)
+            echo $"Found files: ($files)"
+
+            let matrix = {
+              include: (
+                $files
+                | each { |file| open $file }          # Open each file
+                | each { |content| $content | lines }  # Split into lines
+                | flatten                             # Flatten the nested lists
+                | where { |line| $line != "" }        # Filter empty lines
+                | each { |line| 
+                    # Extract just the version part after the last colon
+                    let version = ($line | parse "supabase/postgres:{version}" | get version.0)
+                    {version: $version}
+                }
+              )
+            }
+
+            let json_output = ($matrix | to json -r)  # -r for raw output
+            echo $"Debug output: ($json_output)"
+
+            $"matrix=($json_output)" | save --append $env.GITHUB_OUTPUT
+          '
+      - name: Debug Combined Results
+        run: |
+          echo "Combined Results: '${{ steps.combine.outputs.matrix }}'"
+    outputs:
+      matrix: ${{ steps.combine.outputs.matrix }}
+  publish:
+      needs: combine_results
+      strategy:
+        matrix: ${{ fromJson(needs.combine_results.outputs.matrix) }}
+      uses: ./.github/workflows/mirror.yml
+      with:
+        version: ${{ matrix.version }}
+      secrets: inherit

Dockerfile-15

@@ -41,26 +41,14 @@ ARG wal_g_release=2.0.1
 
 FROM ubuntu:focal as base
 
-
-ENV DEBIAN_FRONTEND=noninteractive \
-    DEBCONF_NONINTERACTIVE_SEEN=true \
-    TZ=Etc/UTC
-
-# Pre-configure tzdata before any installations
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
-    echo $TZ > /etc/timezone && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends tzdata && \
-    apt-get install -y \
+RUN apt update -y && apt install -y \
     curl \
     gnupg \
     lsb-release \
     software-properties-common \
     wget \
     sudo \
-    git \
-    && apt clean && \
-    rm -rf /var/lib/apt/lists/*
+    && apt clean
 
 
 RUN adduser --system  --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres
@@ -87,11 +75,9 @@ WORKDIR /
 RUN mkdir -p /usr/lib/postgresql/bin \
     /usr/lib/postgresql/share/postgresql \
     /usr/share/postgresql \
-    # /usr/lib/postgresql/share/postgresql/contrib \
-    #/usr/lib/postgresql/share/postgresql/timezonesets \
-    #/usr/lib/postgresql/share/postgresql/tsearch_data \
-    # /usr/lib/postgresql/share/postgresql/extension \
+    /var/lib/postgresql \
     && chown -R postgres:postgres /usr/lib/postgresql \
+    && chown -R postgres:postgres /var/lib/postgresql \
     && chown -R postgres:postgres /usr/share/postgresql
 
 # Create symbolic links
@@ -114,6 +100,12 @@ RUN chown -R postgres:postgres /usr/lib/postgresql
 RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets
 
 
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends tzdata
+
+RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
+    dpkg-reconfigure --frontend noninteractive tzdata
+
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     build-essential \