From 4153af82024fb748210250b39fca212a879ba6e2 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Thu, 14 Sep 2023 23:12:18 -0300 Subject: [PATCH] feat: add start-postgres-replica --- README.md | 1 + docs/start-client-server.md | 54 ++++++++++++++++++++++++++++++++++++- flake.nix | 11 ++++++++ tests/postgresql.conf.in | 4 +-- tools/run-replica.sh.in | 39 +++++++++++++++++++++++++++ tools/run-server.sh.in | 2 +- 6 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 tools/run-replica.sh.in diff --git a/README.md b/README.md index 952611b..032bf87 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ The binary distributions have many extensions enabled; these include: - pg_tle - wrappers - supautils +- citus You can just use `CREATE EXTENSION` to enable most of these. Some may require tweaks to [postgresql.conf](./tests/postgresql.conf) to enable. diff --git a/docs/start-client-server.md b/docs/start-client-server.md index 8031a1b..2bde533 100644 --- a/docs/start-client-server.md +++ b/docs/start-client-server.md @@ -1,3 +1,5 @@ +## Running the server + If you want to run a postgres server, just do this from the root of the repository: @@ -18,7 +20,7 @@ valid "flake reference": nix run github:supabase/nix-postgres#start-server 14 ``` -## Arbitrary versions at arbitrary git revisions +### Arbitrary versions at arbitrary git revisions Let's say you want to use a PostgreSQL build from a specific version of the repository. You can change the syntax of the above to use _any_ version of the @@ -39,3 +41,53 @@ nix run github:supabase/nix-postgres#start-server 14 & sleep 5 nix run github:supabase/nix-postgres#start-client 15 ``` + +## Running a server replica + +To start a replica you can use the `start-postgres-replica` command. + +- first argument: the master version +- second argument: the master port +- third argument: the replica server port + +First start a server and a couple of replicas: + +``` +$ start-postgres-server 15 5435 + +$ start-postgres-replica 15 5439 + +$ start-postgres-replica 15 5440 +``` + +Now check the master server: + +``` +$ start-postgres-client 15 5435 +``` + +```sql +SELECT client_addr, state +FROM pg_stat_replication; + client_addr | state +-------------+----------- + ::1 | streaming + ::1 | streaming +(2 rows) + +create table items as select x::int from generate_series(1,100) x; +``` + +And a replica: + +``` +$ start-postgres-client 15 5439 +``` + +```sql +select count(*) from items; + count +------- + 100 +(1 row) +``` diff --git a/flake.nix b/flake.nix index 6dc1c09..540373d 100644 --- a/flake.nix +++ b/flake.nix @@ -301,6 +301,15 @@ chmod +x $out/bin/migrate-postgres ''; + + start-replica = pkgs.runCommand "start-postgres-replica" {} '' + mkdir -p $out/bin + substitute ${./tools/run-replica.sh.in} $out/bin/start-postgres-replica \ + --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ + --subst-var-by 'PSQL14_BINDIR' '${basePackages.psql_14.bin}' \ + --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' + chmod +x $out/bin/start-postgres-replica + ''; }; # Create a testing harness for a PostgreSQL package. This is used for @@ -361,6 +370,7 @@ in { start-server = mkApp "start-server" "start-postgres-server"; start-client = mkApp "start-client" "start-postgres-client"; + start-replica = mkApp "start-replica" "start-postgres-replica"; migration-test = mkApp "migrate-tool" "migrate-postgres"; }; @@ -375,6 +385,7 @@ basePackages.start-server basePackages.start-client + basePackages.start-replica basePackages.migrate-tool ]; shellHook = '' diff --git a/tests/postgresql.conf.in b/tests/postgresql.conf.in index 60256a4..5930cca 100644 --- a/tests/postgresql.conf.in +++ b/tests/postgresql.conf.in @@ -199,7 +199,7 @@ dynamic_shared_memory_type = posix # the default is the first option # - Settings - -#wal_level = replica # minimal, replica, or logical +wal_level = logical # minimal, replica, or logical # (change requires restart) #fsync = on # flush data to disk for crash safety # (turning this off can cause @@ -214,7 +214,7 @@ dynamic_shared_memory_type = posix # the default is the first option # fsync_writethrough # open_sync #full_page_writes = on # recover from partial page writes -#wal_log_hints = off # also do full page writes of non-critical updates +wal_log_hints = on # also do full page writes of non-critical updates # (change requires restart) #wal_compression = off # enable compression of full-page writes #wal_init_zero = on # zero-fill new WAL files diff --git a/tools/run-replica.sh.in b/tools/run-replica.sh.in new file mode 100644 index 0000000..473c430 --- /dev/null +++ b/tools/run-replica.sh.in @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +[ ! -z "$DEBUG" ] && set -x + +# first argument should be '14' or '15' for the version +if [ "$1" == "14" ]; then + echo "Starting server for PSQL 14" + PSQL14=@PSQL14_BINDIR@ + BINDIR="$PSQL14" +elif [ "$1" == "15" ]; then + echo "Starting server for PSQL 15" + PSQL15=@PSQL15_BINDIR@ + BINDIR="$PSQL15" +else + echo "Please provide a valid Postgres version (14 or 15)" + exit 1 +fi + +export PATH=$BINDIR/bin:$PATH + +PGSQL_SUPERUSER=@PGSQL_SUPERUSER@ +MASTER_PORTNO="$2" +REPLICA_PORTNO="$3" +REPLICA_SLOT="replica_$RANDOM" +DATDIR=$(mktemp -d) +mkdir -p "$DATDIR" + +echo "NOTE: runing pg_basebackup for server on port $MASTER_PORTNO" +echo "NOTE: using replica slot $REPLICA_SLOT" + +pg_basebackup -p "$MASTER_PORTNO" -h localhost -U "${PGSQL_SUPERUSER}" -X stream -C -S "$REPLICA_SLOT" -v -R -D "$DATDIR" + +echo "NOTE: using port $REPLICA_PORTNO for replica" +echo "NOTE: using temporary directory $DATDIR for data, which will not be removed" +echo "NOTE: you are free to re-use this data directory at will" +echo + +exec postgres -p "$REPLICA_PORTNO" -D "$DATDIR" -k /tmp diff --git a/tools/run-server.sh.in b/tools/run-server.sh.in index 84df19e..955dce0 100644 --- a/tools/run-server.sh.in +++ b/tools/run-server.sh.in @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +l!/usr/bin/env bash # shellcheck shell=bash [ ! -z "$DEBUG" ] && set -x