diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml new file mode 100644 index 000000000..c7af95e67 --- /dev/null +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -0,0 +1,240 @@ +name: Release all major versions on Dockerhub + +on: + push: + branches: + - develop + - release/* + paths: + - ".github/workflows/dockerhub-release-matrix.yml" + workflow_dispatch: + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + matrix_config: ${{ steps.set-matrix.outputs.matrix_config }} + steps: + - uses: DeterminateSystems/nix-installer-action@main + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Generate build matrix + id: set-matrix + run: | + nix run nixpkgs#nushell -- -c 'let versions = (open ansible/vars.yml | get postgres_major) + let matrix = ($versions | each { |ver| + let version = ($ver | str trim) + let dockerfile = $"Dockerfile-($version)" + if ($dockerfile | path exists) { + { + version: $version, + dockerfile: $dockerfile + } + } else { + null + } + } | compact) + + let matrix_config = { + include: $matrix + } + + $"matrix_config=($matrix_config | to json -r)" | save --append $env.GITHUB_OUTPUT' + build: + needs: prepare + strategy: + matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} + runs-on: ubuntu-latest + outputs: + build_args: ${{ steps.args.outputs.result }} + steps: + - uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV + + - id: args + run: | + nix run nixpkgs#nushell -- -c ' + open ansible/vars.yml + | items { |key value| {name: $key, item: $value} } + | where { |it| ($it.item | describe) == "string" } + | each { |it| $"($it.name)=($it.item)" } + | str join "\n" + | save --append $env.GITHUB_OUTPUT + ' + build_release_image: + needs: [prepare, build] + strategy: + matrix: + postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} + arch: [amd64, arm64] + runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'arm-runner' }} + timeout-minutes: 180 + steps: + - uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - run: docker context create builders + - uses: docker/setup-buildx-action@v3 + with: + endpoint: builders + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Get image tag + id: image + run: | + if [[ "${{ matrix.arch }}" == "arm64" ]]; then + pg_version=$(sudo nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.postgres.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($version)" + } + open ansible/vars.yml | get postgres_release | get $release_key | str trim + ') + echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT + else + pg_version=$(nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.postgres.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($version)" + } + open ansible/vars.yml | get postgres_release | get $release_key | str trim + ') + echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT + fi + - id: build + uses: docker/build-push-action@v5 + with: + push: true + build-args: | + ${{ needs.build.outputs.build_args }} + target: production + tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + file: ${{ matrix.postgres.dockerfile }} + merge_manifest: + needs: [prepare, build, build_release_image] + strategy: + matrix: + include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Get image tag + id: get_version + run: | + nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($version)" + } + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT + ' + - name: Output version + id: output_version + run: | + echo "result=${{ steps.get_version.outputs.pg_version }}" >> $GITHUB_OUTPUT + - name: Collect versions + id: collect_versions + run: | + echo "${{ steps.output_version.outputs.result }}" >> results.txt # Append results + - name: Upload Results Artifact + uses: actions/upload-artifact@v3 + with: + name: merge_results-${{ matrix.version }} + path: results.txt + if-no-files-found: warn + - name: Merge multi-arch manifests + run: | + docker buildx imagetools create -t ${{ steps.get_version.outputs.pg_version }} \ + ${{ steps.get_version.outputs.pg_version }}_amd64 \ + ${{ steps.get_version.outputs.pg_version }}_arm64 + combine_results: + needs: [prepare, merge_manifest] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + + - name: Debug Input from Prepare + run: | + echo "Raw matrix_config output:" + echo "${{ needs.prepare.outputs.matrix_config }}" + - name: Get Versions from Matrix Config + id: get_versions + run: | + nix run nixpkgs#nushell -- -c ' + # Parse the matrix configuration directly + let matrix_config = (${{ toJson(needs.prepare.outputs.matrix_config) }} | from json) + + # Get versions directly from include array + let versions = ($matrix_config.include | get version) + + echo "Versions: $versions" + + # Convert the versions to a comma-separated string + let versions_str = ($versions | str join ",") + $"versions=$versions_str" | save --append $env.GITHUB_ENV + ' + - name: Download Results Artifacts + uses: actions/download-artifact@v3 + with: + pattern: merge_results-* + - name: Combine Results + id: combine + run: | + nix run nixpkgs#nushell -- -c ' + # Get all results files and process them in one go + let files = (ls **/results.txt | get name) + echo $"Found files: ($files)" + + let matrix = { + include: ( + $files + | each { |file| open $file } # Open each file + | each { |content| $content | lines } # Split into lines + | flatten # Flatten the nested lists + | where { |line| $line != "" } # Filter empty lines + | each { |line| + # Extract just the version part after the last colon + let version = ($line | parse "supabase/postgres:{version}" | get version.0) + {version: $version} + } + ) + } + + let json_output = ($matrix | to json -r) # -r for raw output + echo $"Debug output: ($json_output)" + + $"matrix=($json_output)" | save --append $env.GITHUB_OUTPUT + ' + - name: Debug Combined Results + run: | + echo "Combined Results: '${{ steps.combine.outputs.matrix }}'" + outputs: + matrix: ${{ steps.combine.outputs.matrix }} + publish: + needs: combine_results + strategy: + matrix: ${{ fromJson(needs.combine_results.outputs.matrix) }} + uses: ./.github/workflows/mirror.yml + with: + version: ${{ matrix.version }} + secrets: inherit diff --git a/Dockerfile-15 b/Dockerfile-15 index 01895db39..52c4e5af3 100644 --- a/Dockerfile-15 +++ b/Dockerfile-15 @@ -41,26 +41,14 @@ ARG wal_g_release=2.0.1 FROM ubuntu:focal as base - -ENV DEBIAN_FRONTEND=noninteractive \ - DEBCONF_NONINTERACTIVE_SEEN=true \ - TZ=Etc/UTC - -# Pre-configure tzdata before any installations -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - echo $TZ > /etc/timezone && \ - apt-get update && \ - apt-get install -y --no-install-recommends tzdata && \ - apt-get install -y \ +RUN apt update -y && apt install -y \ curl \ gnupg \ lsb-release \ software-properties-common \ wget \ sudo \ - git \ - && apt clean && \ - rm -rf /var/lib/apt/lists/* + && apt clean RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres @@ -87,11 +75,9 @@ WORKDIR / RUN mkdir -p /usr/lib/postgresql/bin \ /usr/lib/postgresql/share/postgresql \ /usr/share/postgresql \ - # /usr/lib/postgresql/share/postgresql/contrib \ - #/usr/lib/postgresql/share/postgresql/timezonesets \ - #/usr/lib/postgresql/share/postgresql/tsearch_data \ - # /usr/lib/postgresql/share/postgresql/extension \ + /var/lib/postgresql \ && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /var/lib/postgresql \ && chown -R postgres:postgres /usr/share/postgresql # Create symbolic links @@ -114,6 +100,12 @@ RUN chown -R postgres:postgres /usr/lib/postgresql RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets +RUN apt-get update && \ + apt-get install -y --no-install-recommends tzdata + +RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + dpkg-reconfigure --frontend noninteractive tzdata + RUN apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ diff --git a/Dockerfile-16 b/Dockerfile-orioledb-17 similarity index 84% rename from Dockerfile-16 rename to Dockerfile-orioledb-17 index c281ccd7f..e75694c16 100644 --- a/Dockerfile-16 +++ b/Dockerfile-orioledb-17 @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1.6 -ARG postgresql_major=15 +ARG postgresql_major=17-orioledb ARG postgresql_release=${postgresql_major}.1 # Bump default build arg to build a package from source @@ -41,25 +41,15 @@ ARG wal_g_release=2.0.1 FROM ubuntu:focal as base -ENV DEBIAN_FRONTEND=noninteractive \ - DEBCONF_NONINTERACTIVE_SEEN=true \ - TZ=Etc/UTC - -# Pre-configure tzdata before any installations -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - echo $TZ > /etc/timezone && \ - apt-get update && \ - apt-get install -y --no-install-recommends tzdata && \ - apt-get install -y \ +RUN apt update -y && apt install -y \ curl \ gnupg \ lsb-release \ software-properties-common \ wget \ sudo \ - git \ - && apt clean && \ - rm -rf /var/lib/apt/lists/* + tree \ + && apt clean RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres @@ -76,7 +66,7 @@ COPY . /nixpg WORKDIR /nixpg -RUN nix profile install .#psql_16/bin +RUN nix profile install .#psql_orioledb-17/bin @@ -86,11 +76,9 @@ WORKDIR / RUN mkdir -p /usr/lib/postgresql/bin \ /usr/lib/postgresql/share/postgresql \ /usr/share/postgresql \ - # /usr/lib/postgresql/share/postgresql/contrib \ - #/usr/lib/postgresql/share/postgresql/timezonesets \ - #/usr/lib/postgresql/share/postgresql/tsearch_data \ - # /usr/lib/postgresql/share/postgresql/extension \ + /var/lib/postgresql \ && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /var/lib/postgresql \ && chown -R postgres:postgres /usr/share/postgresql # Create symbolic links @@ -104,15 +92,19 @@ RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql RUN chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/ RUN chown -R postgres:postgres /usr/share/postgresql/ # Create symbolic links for contrib directory -RUN mkdir -p /usr/lib/postgresql/share/postgresql/contrib \ - && find /nix/var/nix/profiles/default/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ - && chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/contrib/ +RUN tree /nix > /tmp/tree.txt && cat /tmp/tree.txt && cat /tmp/tree.txt >&2 RUN chown -R postgres:postgres /usr/lib/postgresql RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets - + +RUN apt-get update && \ + apt-get install -y --no-install-recommends tzdata + +RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + dpkg-reconfigure --frontend noninteractive tzdata + RUN apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ @@ -191,15 +183,28 @@ RUN sed -i \ mkdir -p /etc/postgresql-custom && \ chown postgres:postgres /etc/postgresql-custom + # Remove items from postgresql.conf +RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf" + #as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present +RUN sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "/etc/postgresql/postgresql.conf" +RUN sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "/etc/postgresql-custom/supautils.conf" +RUN sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "/etc/postgresql/postgresql.conf" +RUN echo "default_table_access_method = 'orioledb'" >> "/etc/postgresql/postgresql.conf" + + + # # Include schema migrations COPY migrations/db /docker-entrypoint-initdb.d/ COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql +# Enable orioledb extension first +RUN echo "CREATE EXTENSION orioledb;" > /docker-entrypoint-initdb.d/init-scripts/00-pre-init.sql && \ + chown postgres:postgres /docker-entrypoint-initdb.d/init-scripts/00-pre-init.sql # # Add upstream entrypoint script COPY --from=gosu /usr/local/bin/gosu /usr/local/bin/gosu ADD --chmod=0755 \ - https://github.com/docker-library/postgres/raw/master/15/bullseye/docker-entrypoint.sh \ + https://github.com/docker-library/postgres/raw/master/17/bullseye/docker-entrypoint.sh \ /usr/local/bin/ RUN mkdir -p /var/run/postgresql && chown postgres:postgres /var/run/postgresql @@ -213,6 +218,7 @@ EXPOSE 5432 ENV POSTGRES_HOST=/var/run/postgresql ENV POSTGRES_USER=supabase_admin ENV POSTGRES_DB=postgres +ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8" RUN apt-get update && apt-get install -y --no-install-recommends \ locales \ && rm -rf /var/lib/apt/lists/* && \