Enable team members to replace changelog_dev with a prod db dump (#533)

* Enable team members to replace changelog_dev with a prod db dump

There is a single command to run:

    just restore-dev-db-from-prod

This implies that:
- direnv.net is hooked into the shell
-`just envrc-secrets` has already run & `.envrc.secrets` exists
- Posgres 16 has been installed using `just postgres`

Part of this change we exposed a few more private recipes and made it
clear who they are meant for:
- `contributor` means anyone that wants to play with this repo locally
- `team` means Adam, Jerod & Gerhard

We also added `just add-oban-pro-repo` so that we make this integration
easier for team members - I always forget about it!

Signed-off-by: Gerhard Lazu <[email protected]>

* Go back to managing Postgres with asdf

Single tool to rule them all ✊ icu4c seems to be working on macOS 14.7
ARM now. Checked that the new command works as expected. Also confirmed
that CI works locally.

Also run:

    npx browserslist@latest --update-db

Signed-off-by: Gerhard Lazu <[email protected]>

---------

Signed-off-by: Gerhard Lazu <[email protected]>

.envrc

@@ -0,0 +1,9 @@
+# Required for image publishing to work locally
+export GITHUB_REPOSITORY="thechangelog/changelog.com"
+export GITHUB_REF_NAME="master"
+export IMAGE_OWNER=thechangelog
+export GHCR_USERNAME=$USER
+
+# Load secrets if configured - only available to changelog.com team members:
+# https://github.com/orgs/thechangelog/people
+source_env_if_exists .envrc.secrets

.gitignore

@@ -40,9 +40,10 @@ npm-debug.log*
 /import/cookies.yml
 /cover
 
-# User-specific environment variables, maybe even secrets
-.envrc
+# User-specific environment variables
 .env.user
+# Secret variables
+.envrc.secrets
 
 # Local config files, host-specific
 .config

.tool-versions

@@ -7,3 +7,4 @@ flyctl 0.2.65
 golang 1.22.4
 nodejs 20.14.0
 yarn 1.22.22
+postgres 16.5

CONTRIBUTING.md

@@ -91,14 +91,21 @@ We are using [`just`](https://github.com/casey/just) to manage `brew` & `asdf` w
 Once you have [`just` installed](https://github.com/casey/just?tab=readme-ov-file#installation), running `just` in the root of this repository will produce the following output:
 
 ```console
+just --list
 Available recipes:
-    contribute    # Setup everything needed for your first contribution
-    deps          # Get app dependencies
-    dev           # Run app in dev mode
-    install       # Install all system dependencies
-    postgres-down # Stop Postgres server
-    postgres-up   # Start Postgres server
-    test          # Run app tests
+    [contributor]
+    contribute                                   # Setup everything needed for your first contribution
+    deps                                         # Get app dependencies
+    dev                                          # Run app in dev mode
+    install                                      # Install all system dependencies
+    postgres-down                                # Stop Postgres server
+    postgres-up                                  # Start Postgres server
+    test                                         # Run app tests
+
+    [team]
+    envrc-secrets                                # Create .envrc.secrets with credentials from 1Password
+    restore-dev-db-from-prod format="c"          # Delete & replace changelog_dev with a prod db dump
+    tag-kaizen version episode discussion commit # Tag Kaizen $version with $episode & $discussion at $commit (recording date)
 ```
 
 The only command that you need to run is `just contribute`.

assets/yarn.lock

@@ -1479,15 +1479,10 @@ caniuse-api@^3.0.0:
     lodash.memoize "^4.1.2"
     lodash.uniq "^4.5.0"
 
-caniuse-lite@^1.0.0:
-  version "1.0.30001338"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001338.tgz#b5dd7a7941a51a16480bdf6ff82bded1628eec0d"
-  integrity sha512-1gLHWyfVoRDsHieO+CaeYe7jSo/MT7D7lhaXUiwwbuR5BwQxORs0f1tAwUSQr3YbxRXJvxHM/PA5FfPQRnsPeQ==
-
-caniuse-lite@^1.0.30001332, caniuse-lite@^1.0.30001426, caniuse-lite@^1.0.30001449:
-  version "1.0.30001458"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001458.tgz#871e35866b4654a7d25eccca86864f411825540c"
-  integrity sha512-lQ1VlUUq5q9ro9X+5gOEyH7i3vm+AYVT1WDCVB69XOZ17KZRhnZ9J0Sqz7wTHQaLBJccNCHq8/Ww5LlOIZbB0w==
+caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001332, caniuse-lite@^1.0.30001426, caniuse-lite@^1.0.30001449:
+  version "1.0.30001684"
+  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001684.tgz"
+  integrity sha512-G1LRwLIQjBQoyq0ZJGqGIJUXzJ8irpbjHLpVRXDvBEScFJ9b17sgK6vlx0GAJFE21okD7zXl08rRRUfq6HdoEQ==
 
 chalk@^2.0.0:
   version "2.4.2"
@@ -2862,13 +2857,13 @@ pegjs@^0.10.0:
   integrity sha512-qI5+oFNEGi3L5HAxDwN2LA4Gg7irF70Zs25edhjld9QemOgp0CbvMtbFcMvFtEo1OityPrcCzkQFB8JP/hxgow==
 
 "phoenix@file:../deps/phoenix":
-  version "1.7.11"
+  version "1.7.14"
 
 "phoenix_html@file:../deps/phoenix_html":
-  version "3.3.3"
+  version "3.3.4"
 
 "phoenix_live_view@file:../deps/phoenix_live_view":
-  version "0.20.9"
+  version "0.20.17"
 
 picocolors@^1.0.0:
   version "1.0.0"

envrc.secrets.op

@@ -0,0 +1,13 @@
+export OBAN_KEY_FINGERPRINT="op://changelog/oban/key_fingerprint"
+export OBAN_LICENSE_KEY="op://changelog/oban/license_key"
+
+# Required for deploys to work locally
+export FLY_API_TOKEN="$(flyctl auth token)"
+
+# Required for image publishing to work locally
+export GHCR_PASSWORD="op://changelog/ghcr/credential"
+
+# Required for db sync to work locally
+export DB_PROD_HOST="op://changelog/neon/server"
+export DB_PROD_USERNAME="op://changelog/neon/username"
+export DB_PROD_DBNAME="op://changelog/neon/database"

justfile

@@ -33,20 +33,23 @@ imagemagick: brew
     @[ -d $(brew --prefix)/opt/imagemagick ] \
     || brew install imagemagick
 
-PGPATH := "PATH=$(brew --prefix)/opt/postgresql@16/bin:" + env_var("PATH")
-PGDATA := "PGDATA=$(brew --prefix)/var/postgresql@16"
-PG := PGPATH + " " + PGDATA
+export OP_ACCOUNT := "changelog.1password.com"
 
-[private]
-postgres: brew
-    @[ -d $(brew --prefix)/opt/postgresql@16 ] \
-    || brew install postgresql@16
+# Create .envrc.secrets with credentials from 1Password
+[group('team')]
+envrc-secrets:
+    op inject --in-file envrc.secrets.op --out-file .envrc.secrets
 
 [private]
 gpg: brew
     @[ -d $(brew --prefix)/opt/gpg ] \
     || brew install gpg
 
+[private]
+icu4c: brew
+    @brew --prefix icu4c 1> /dev/null 2>&1 \
+    || brew install icu4c pkg-config
+
 # https://tldp.org/LDP/abs/html/exitcodes.html
 [private]
 asdf:
@@ -59,63 +62,103 @@ asdf-shell: brew
     @echo "source $(brew --prefix)/opt/asdf/libexec/asdf.sh"
 
 # Install all system dependencies
-install: asdf brew imagemagick postgres gpg
+[group('contributor')]
+install: asdf brew imagemagick gpg icu4c
     @awk '{ system("asdf plugin-add " $1) }' < .tool-versions
-    @asdf install
+    @PKG_CONFIG_PATH="$(brew --prefix icu4c)/lib/pkgconfig:$PKG_CONFIG_PATH" asdf install
 
 export ELIXIR_ERL_OPTIONS := if os() == "linux" { "+fnu" } else { "" }
 
+# Add Oban Pro repository
+[group('team')]
+add-oban-pro-repo:
+    [ -n "$OBAN_LICENSE_KEY" ] \
+    && [ -n "$OBAN_KEY_FINGERPRINT" ] \
+    && mix hex.repo add oban https://getoban.pro/repo --fetch-public-key $OBAN_KEY_FINGERPRINT --auth-key $OBAN_LICENSE_KEY
+
 # Get app dependencies
-deps:
+[group('contributor')]
+deps: add-oban-pro-repo
     mix local.hex --force
     mix deps.get --only dev
     mix deps.get --only test
 
 [private]
 pg_ctl:
-    @{{ PG }} which pg_ctl >/dev/null \
-    || (echo "Please install Postgres using: {{ BOLD }}just install{{ RESET }}" && exit 127)
+    @which pg_ctl >/dev/null \
+    || (echo "{{ REDB }}{{ WHITE }}Postgres is not installed.{{ RESET }} To fix this, run: {{ GREENB }}{{ WHITE }}just install{{ RESET }}" && exit 127)
 
 # Start Postgres server
+[group('contributor')]
 postgres-up: pg_ctl
-    @({{ PG }} pg_ctl status | grep -q "is running") || {{ PG }} pg_ctl start
+    @(pg_ctl status | grep -q "is running") || pg_ctl start
 
 # Stop Postgres server
+[group('contributor')]
 postgres-down: pg_ctl
-    @({{ PG }} pg_ctl status | grep -q "no server running") || {{ PG }} pg_ctl stop
+    @(pg_ctl status | grep -q "no server running") || pg_ctl stop
 
 [private]
 postgres-db db:
-    @({{ PG }} psql --list --quiet --tuples-only | grep -q {{ db }}) \
-    || {{ PG }} createdb {{ db }}
+    @(psql --list --quiet --tuples-only | grep -q {{ db }}) \
+    || createdb {{ db }}
+
+export PGUSER := "postgres"
+export DB_USER := PGUSER
+
+# Delete & replace changelog_dev with a prod db dump
+[confirm("This DELETEs and REPLACEs changelog_dev with the prod db dump. Are you sure that you want to continue?")]
+[group('team')]
+restore-dev-db-from-prod format="c": changelog_dev
+    @echo "\n{{ GREEN }}🛬 Dumping prod db...{{ RESET }}"
+    [ -f $DB_PROD_DBNAME.{{ format }}.sql ] \
+    || time PGSSLMODE=require PGPASSWORD=$(op read op://changelog/neon/password) pg_dump \
+        --format={{ format }} --verbose \
+        --host=$DB_PROD_HOST \
+        --username=$DB_PROD_USERNAME \
+        --dbname=$DB_PROD_DBNAME > $DB_PROD_DBNAME.{{ format }}.sql
+    @echo "\n{{ GREEN }}🛫 Recreating {{ CHANGELOG_DEV_DB }} from prod dump{{ RESET }}..."
+    dropdb {{ CHANGELOG_DEV_DB }}
+    createdb {{ CHANGELOG_DEV_DB }}
+    time pg_restore \
+        --format=c --verbose \
+        --dbname={{ CHANGELOG_DEV_DB }} \
+        --exit-on-error \
+        --no-owner \
+        --no-privileges < $DB_PROD_DBNAME.{{ format }}.sql
+    @echo "\n{{ GREEN }}⚡️ Warm up the query planner...{{ RESET }} https://www.postgresql.org/docs/current/sql-analyze.html..."
+    time psql --dbname={{ CHANGELOG_DEV_DB }} --command "ANALYZE VERBOSE;"
 
-export DB_USER := `whoami`
 
 [private]
 changelog_test: postgres-up (postgres-db "changelog_test")
 
 # Run app tests
+[group('contributor')]
 test: changelog_test
     mix test
 
+CHANGELOG_DEV_DB := "changelog_dev"
 [private]
-changelog_dev: postgres-up (postgres-db "changelog_dev")
+changelog_dev: postgres-up (postgres-db CHANGELOG_DEV_DB)
     mix ecto.setup
 
 [private]
 yarn:
     @which yarn >/dev/null \
-    || (echo "Please install Node.js & Yarn using: {{ BOLD }}just install{{ RESET }}" && exit 127)
+    || (echo "{{ REDB }}{{ WHITE }}Yarn is not installed.{{ RESET }} To fix this, run: {{ GREENB }}{{ WHITE }}just install{{ RESET }}" && exit 127)
 
 [private]
 assets: yarn
     cd assets && yarn install
 
 # Run app in dev mode
+[group('contributor')]
 dev: changelog_dev assets
     mix phx.server
 
 # Setup everything needed for your first contribution
+[group('contributor')]
 contribute: install
     #!/usr/bin/env bash
     eval "$(just asdf-shell)"
@@ -145,7 +188,7 @@ do-it:
     just contribute
 
 # Tag Kaizen $version with $episode & $discussion at $commit (recording date)
-[private]
+[group('team')]
 tag-kaizen version episode discussion commit:
     git tag --force --sign \
         --message="Recorded as 🎧 <https://changelog.com/friends/{{ episode }}>" \

magefiles/tools/main.go

@@ -49,9 +49,9 @@ func (v *Versions) Nodejs() string {
 }
 
 // https://www.postgresql.org/docs/release
-// Hard-coding & installing via brew (keep hitting icu4c & root permissions with asdf)
+// asdf list all postgres
 func (v *Versions) Postgres() string {
-	return "16"
+	return v.toolVersions["postgres"]
 }
 
 // https://github.com/yarnpkg/yarn/releases