diff --git a/.github/workflows/dagger_on_fly.yml b/.github/workflows/dagger_on_fly.yml index fa09118c9a..f906a82657 100644 --- a/.github/workflows/dagger_on_fly.yml +++ b/.github/workflows/dagger_on_fly.yml @@ -51,6 +51,7 @@ jobs: R2_API_HOST: "${{ secrets.R2_API_HOST }}" R2_ACCESS_KEY_ID: "${{ secrets.R2_ACCESS_KEY_ID }}" R2_SECRET_ACCESS_KEY: "${{ secrets.R2_SECRET_ACCESS_KEY }}" + R2_ASSETS_BUCKET: "${{ env.R2_ASSETS_BUCKET }}" OBAN_KEY_FINGERPRINT: "${{ secrets.OBAN_KEY_FINGERPRINT }}" OBAN_LICENSE_KEY: "${{ secrets.OBAN_LICENSE_KEY }}" run: | diff --git a/.github/workflows/dagger_on_github.yml b/.github/workflows/dagger_on_github.yml index c1d8d74084..46f4f533d9 100644 --- a/.github/workflows/dagger_on_github.yml +++ b/.github/workflows/dagger_on_github.yml @@ -24,6 +24,7 @@ jobs: R2_API_HOST: "${{ secrets.R2_API_HOST }}" R2_ACCESS_KEY_ID: "${{ secrets.R2_ACCESS_KEY_ID }}" R2_SECRET_ACCESS_KEY: "${{ secrets.R2_SECRET_ACCESS_KEY }}" + R2_ASSETS_BUCKET: "${{ env.R2_ASSETS_BUCKET }}" OBAN_KEY_FINGERPRINT: "${{ secrets.OBAN_KEY_FINGERPRINT }}" OBAN_LICENSE_KEY: "${{ secrets.OBAN_LICENSE_KEY }}" run: | diff --git a/.github/workflows/dagger_on_k8s.yml b/.github/workflows/dagger_on_k8s.yml index 87c30b6d16..f7498615cb 100644 --- a/.github/workflows/dagger_on_k8s.yml +++ b/.github/workflows/dagger_on_k8s.yml @@ -25,6 +25,7 @@ jobs: R2_API_HOST: "${{ secrets.R2_API_HOST }}" R2_ACCESS_KEY_ID: "${{ secrets.R2_ACCESS_KEY_ID }}" R2_SECRET_ACCESS_KEY: "${{ secrets.R2_SECRET_ACCESS_KEY }}" + R2_ASSETS_BUCKET: "${{ env.R2_ASSETS_BUCKET }}" OBAN_KEY_FINGERPRINT: "${{ secrets.OBAN_KEY_FINGERPRINT }}" OBAN_LICENSE_KEY: "${{ secrets.OBAN_LICENSE_KEY }}" run: | diff --git a/.tool-versions b/.tool-versions index 86ffd113a4..4facf871b5 100644 --- a/.tool-versions +++ b/.tool-versions @@ -4,5 +4,7 @@ erlang 26.2 golang 1.20.12 nodejs 20.10.0 yarn 1.22.19 -postgres 15.3 -flyctl 0.1.134 +postgres 16.1 +flyctl 0.1.135 +1password-cli 2.24.0 +dagger 0.6.4 diff --git a/2022.fly/lazu.ch/fastly-lazu-v11.vcl b/2022.fly/lazu.ch/fastly-lazu-v11.vcl deleted file mode 100644 index 1697dca3ba..0000000000 --- a/2022.fly/lazu.ch/fastly-lazu-v11.vcl +++ /dev/null @@ -1,570 +0,0 @@ -# vim: set ft=config -pragma optional_param geoip_opt_in true; -pragma optional_param max_object_size 2147483648; -pragma optional_param smiss_max_object_size 5368709120; -pragma optional_param fetchless_purge_all 1; -pragma optional_param default_ssl_check_cert 1; -pragma optional_param max_backends 5; -pragma optional_param customer_id "3ftsRHRYALfBKJ8uiRDUFJ"; -C! -W! -# Backends - -backend F_caddy { - .between_bytes_timeout = 10s; - .connect_timeout = 1s; - .dynamic = true; - .first_byte_timeout = 15s; - .host = "22.lazu.ch"; - .max_connections = 200; - .port = "443"; - .share_key = "1AxxzRW3YiHB4eR1GRz59a"; - - .ssl = true; - .ssl_cert_hostname = "22.lazu.ch"; - .ssl_check_cert = always; - .ssl_sni_hostname = "22.lazu.ch"; - - .probe = { - .expected_response = 200; - .initial = 1; - .interval = 60s; - .request = "HEAD / HTTP/1.1" "Host: 22.lazu.ch" "Connection: close" "User-Agent: Varnish/fastly (healthcheck)"; - .threshold = 1; - .timeout = 5s; - .window = 2; - } -} -backend F_fly { - .between_bytes_timeout = 10s; - .connect_timeout = 1s; - .dynamic = true; - .first_byte_timeout = 15s; - .host = "old-flower-9005.fly.dev"; - .max_connections = 200; - .port = "443"; - .share_key = "1AxxzRW3YiHB4eR1GRz59a"; - - .ssl = true; - .ssl_cert_hostname = "old-flower-9005.fly.dev"; - .ssl_check_cert = always; - .ssl_sni_hostname = "old-flower-9005.fly.dev"; - - .probe = { - .expected_response = 200; - .initial = 1; - .interval = 60s; - .request = "HEAD / HTTP/1.1" "Host: old-flower-9005.fly.dev" "Connection: close" "User-Agent: Varnish/fastly (healthcheck)"; - .threshold = 1; - .timeout = 5s; - .window = 2; - } -} - - - - - - - - - - - -sub vcl_recv { -#--FASTLY RECV BEGIN - if (req.restarts == 0) { - if (!req.http.X-Timer) { - set req.http.X-Timer = "S" time.start.sec "." time.start.usec_frac; - } - set req.http.X-Timer = req.http.X-Timer ",VS0"; - } - - - - declare local var.fastly_req_do_shield BOOL; - set var.fastly_req_do_shield = (req.restarts == 0); - -# Snippet http3-quic-v1-4IhpR1yeOjc64fIsnmafRy-recv-recv : 100 -# Begin http3-quic recv - h3.alt_svc(); -# End http3-quic recv - - - # default conditions - - if (!req.http.Fastly-SSL) { - error 801 "Force SSL"; - } - - - - - # end default conditions - - - - # Request Condition: www.lazu.ch host Prio: 10 - if( req.http.host == "www.lazu.ch" ) { - - - - - # ResponseObject: 301 redirect - error 900 "Fastly Internal"; - } - #end condition - # Request Condition: lazu.ch host Prio: 10 - if( req.http.host == "lazu.ch" ) { - - set req.backend = F_fly; - - - } - #end condition - # Request Condition: cdn.lazu.ch host Prio: 10 - if( req.http.host == "cdn.lazu.ch" ) { - - set req.backend = F_caddy; - - - } - #end condition - - - - -#--FASTLY RECV END - - - - if (req.request != "HEAD" && req.request != "GET" && req.request != "FASTLYPURGE") { - return(pass); - } - - - return(lookup); -} - - -sub vcl_fetch { - - -#--FASTLY FETCH BEGIN - - if (beresp.status >= 500 && beresp.status < 600) { - - if (stale.exists) { - return(deliver_stale); - } - - if (req.restarts < 1 && (req.request == "GET" || req.request == "HEAD")) { - restart; - } - - } - set beresp.stale_if_error = 43200s; - - -# record which cache ran vcl_fetch for this object and when - set beresp.http.Fastly-Debug-Path = "(F " server.identity " " now.sec ") " if(beresp.http.Fastly-Debug-Path, beresp.http.Fastly-Debug-Path, ""); - -# generic mechanism to vary on something - if (req.http.Fastly-Vary-String) { - if (beresp.http.Vary) { - set beresp.http.Vary = "Fastly-Vary-String, " beresp.http.Vary; - } else { - set beresp.http.Vary = "Fastly-Vary-String, "; - } - } - - - - # priority: 10 - if ( req.http.host == "cdn.lazu.ch" ) { - - - # cache for 30s, 1h for stale - set beresp.ttl = 30s; - set beresp.grace = 3600s; - - - - - - } - - # priority: 10 - if ( req.http.host == "lazu.ch" ) { - - - # do not cache - set beresp.ttl = 0s; - set beresp.grace = 0s; - return(pass); - - - - - } - -#--FASTLY FETCH END - - - - if ((beresp.status == 500 || beresp.status == 503) && req.restarts < 1 && (req.request == "GET" || req.request == "HEAD")) { - restart; - } - - if(req.restarts > 0 ) { - set beresp.http.Fastly-Restarts = req.restarts; - } - - if (beresp.http.Set-Cookie) { - set req.http.Fastly-Cachetype = "SETCOOKIE"; - return (pass); - } - - if (beresp.http.Cache-Control ~ "private") { - set req.http.Fastly-Cachetype = "PRIVATE"; - return (pass); - } - - if (beresp.status == 500 || beresp.status == 503) { - set req.http.Fastly-Cachetype = "ERROR"; - set beresp.ttl = 1s; - set beresp.grace = 5s; - return (deliver); - } - - - if (beresp.http.Expires || beresp.http.Surrogate-Control ~ "max-age" || beresp.http.Cache-Control ~"(s-maxage|max-age)") { - # keep the ttl here - } else { - # apply the default ttl - set beresp.ttl = 3600s; - - } - - return(deliver); -} - -sub vcl_hit { -#--FASTLY HIT BEGIN - -# we cannot reach obj.ttl and obj.grace in deliver, save them when we can in vcl_hit - set req.http.Fastly-Tmp-Obj-TTL = obj.ttl; - set req.http.Fastly-Tmp-Obj-Grace = obj.grace; - - { - set req.http.Fastly-Cachetype = "HIT"; - } - -#--FASTLY HIT END - if (!obj.cacheable) { - return(pass); - } - return(deliver); -} - -sub vcl_miss { -#--FASTLY MISS BEGIN - - -# this is not a hit after all, clean up these set in vcl_hit - unset req.http.Fastly-Tmp-Obj-TTL; - unset req.http.Fastly-Tmp-Obj-Grace; - - { - if (req.http.Fastly-Check-SHA1) { - error 550 "Doesnt exist"; - } - -#--FASTLY BEREQ BEGIN - { - { - if (req.http.Fastly-FF) { - set bereq.http.Fastly-Client = "1"; - } - } - { - # do not send this to the backend - unset bereq.http.Fastly-Original-Cookie; - unset bereq.http.Fastly-Original-URL; - unset bereq.http.Fastly-Vary-String; - unset bereq.http.X-Varnish-Client; - } - if (req.http.Fastly-Temp-XFF) { - if (req.http.Fastly-Temp-XFF == "") { - unset bereq.http.X-Forwarded-For; - } else { - set bereq.http.X-Forwarded-For = req.http.Fastly-Temp-XFF; - } - # unset bereq.http.Fastly-Temp-XFF; - } - } -#--FASTLY BEREQ END - - - #; - - set req.http.Fastly-Cachetype = "MISS"; - - } - -#--FASTLY MISS END - return(fetch); -} - -sub vcl_deliver { - - -#--FASTLY DELIVER BEGIN - -# record the journey of the object, expose it only if req.http.Fastly-Debug. - if (req.http.Fastly-Debug || req.http.Fastly-FF) { - set resp.http.Fastly-Debug-Path = "(D " server.identity " " now.sec ") " - if(resp.http.Fastly-Debug-Path, resp.http.Fastly-Debug-Path, ""); - - set resp.http.Fastly-Debug-TTL = if(obj.hits > 0, "(H ", "(M ") - server.identity - if(req.http.Fastly-Tmp-Obj-TTL && req.http.Fastly-Tmp-Obj-Grace, " " req.http.Fastly-Tmp-Obj-TTL " " req.http.Fastly-Tmp-Obj-Grace " ", " - - ") - if(resp.http.Age, resp.http.Age, "-") - ") " - if(resp.http.Fastly-Debug-TTL, resp.http.Fastly-Debug-TTL, ""); - - set resp.http.Fastly-Debug-Digest = digest.hash_sha256(req.digest); - } else { - unset resp.http.Fastly-Debug-Path; - unset resp.http.Fastly-Debug-TTL; - unset resp.http.Fastly-Debug-Digest; - } - - # add or append X-Served-By/X-Cache(-Hits) - { - - if(!resp.http.X-Served-By) { - set resp.http.X-Served-By = server.identity; - } else { - set resp.http.X-Served-By = resp.http.X-Served-By ", " server.identity; - } - - set resp.http.X-Cache = if(resp.http.X-Cache, resp.http.X-Cache ", ","") if(fastly_info.state ~ "HIT($|-)", "HIT", "MISS"); - - if(!resp.http.X-Cache-Hits) { - set resp.http.X-Cache-Hits = obj.hits; - } else { - set resp.http.X-Cache-Hits = resp.http.X-Cache-Hits ", " obj.hits; - } - - } - - if (req.http.X-Timer) { - set resp.http.X-Timer = req.http.X-Timer ",VE" time.elapsed.msec; - } - - # VARY FIXUP - { - # remove before sending to client - set resp.http.Vary = regsub(resp.http.Vary, "Fastly-Vary-String, ", ""); - if (resp.http.Vary ~ "^\s*$") { - unset resp.http.Vary; - } - } - unset resp.http.X-Varnish; - - - # Pop the surrogate headers into the request object so we can reference them later - set req.http.Surrogate-Key = resp.http.Surrogate-Key; - set req.http.Surrogate-Control = resp.http.Surrogate-Control; - - # If we are not forwarding or debugging unset the surrogate headers so they are not present in the response - if (!req.http.Fastly-FF && !req.http.Fastly-Debug) { - unset resp.http.Surrogate-Key; - unset resp.http.Surrogate-Control; - } - - if(resp.status == 550) { - return(deliver); - } - #default response conditions - - -# Header rewrite lazu.ch Location : 10 - - - set resp.http.Location = "https://lazu.ch" req.url; - - - -# Header rewrite Generated by force TLS and enable HSTS : 100 - - - set resp.http.Strict-Transport-Security = "max-age=300"; - - - - # Request Condition: www.lazu.ch host Prio: 10 - if (resp.status == 900 ) { - set resp.status = 301; - set resp.response = "Moved Permanently"; - } - - - -#--FASTLY DELIVER END - return(deliver); -} - -sub vcl_error { -#--FASTLY ERROR BEGIN - - if (obj.status >= 500 && obj.status < 600) { - if (stale.exists) { - return(deliver_stale); - } - } - - if (obj.status == 801) { - set obj.status = 301; - set obj.response = "Moved Permanently"; - set obj.http.Location = "https://" req.http.host req.url; - synthetic {""}; - return (deliver); - } - - - # Response Condition: www.lazu.ch host Prio: 10 - -if (obj.status == 900 ) { - return(deliver); -} - - - if (req.http.Fastly-Restart-On-Error) { - if (obj.status == 503 && req.restarts == 0) { - restart; - } - } - - { - if (obj.status == 550) { - return(deliver); - } - } -#--FASTLY ERROR END - - - -} - -sub vcl_pipe { -#--FASTLY PIPE BEGIN - { - - -#--FASTLY BEREQ BEGIN - { - { - if (req.http.Fastly-FF) { - set bereq.http.Fastly-Client = "1"; - } - } - { - # do not send this to the backend - unset bereq.http.Fastly-Original-Cookie; - unset bereq.http.Fastly-Original-URL; - unset bereq.http.Fastly-Vary-String; - unset bereq.http.X-Varnish-Client; - } - if (req.http.Fastly-Temp-XFF) { - if (req.http.Fastly-Temp-XFF == "") { - unset bereq.http.X-Forwarded-For; - } else { - set bereq.http.X-Forwarded-For = req.http.Fastly-Temp-XFF; - } - # unset bereq.http.Fastly-Temp-XFF; - } - } -#--FASTLY BEREQ END - - - #; - set req.http.Fastly-Cachetype = "PIPE"; - set bereq.http.connection = "close"; - } -#--FASTLY PIPE END - -} - -sub vcl_pass { -#--FASTLY PASS BEGIN - - - { - -#--FASTLY BEREQ BEGIN - { - { - if (req.http.Fastly-FF) { - set bereq.http.Fastly-Client = "1"; - } - } - { - # do not send this to the backend - unset bereq.http.Fastly-Original-Cookie; - unset bereq.http.Fastly-Original-URL; - unset bereq.http.Fastly-Vary-String; - unset bereq.http.X-Varnish-Client; - } - if (req.http.Fastly-Temp-XFF) { - if (req.http.Fastly-Temp-XFF == "") { - unset bereq.http.X-Forwarded-For; - } else { - set bereq.http.X-Forwarded-For = req.http.Fastly-Temp-XFF; - } - # unset bereq.http.Fastly-Temp-XFF; - } - } -#--FASTLY BEREQ END - - - #; - set req.http.Fastly-Cachetype = "PASS"; - } - -#--FASTLY PASS END - -} - -sub vcl_log { -#--FASTLY LOG BEGIN - - # default response conditions - - - -#--FASTLY LOG END - -} - -sub vcl_hash { - -#--FASTLY HASH BEGIN - - - - #if unspecified fall back to normal - { - - - set req.hash += req.url; - set req.hash += req.http.host; - set req.hash += req.vcl.generation; - return (hash); - } -#--FASTLY HASH END - - -} - diff --git a/2022.fly/lazu.ch/fly.toml b/2022.fly/lazu.ch/fly.toml deleted file mode 100644 index 0548dbb7ae..0000000000 --- a/2022.fly/lazu.ch/fly.toml +++ /dev/null @@ -1,27 +0,0 @@ -# fly.toml file generated for old-flower-9005 on 2022-04-03T16:58:35+01:00 - -app = "old-flower-9005" - -[build] - builder = "paketobuildpacks/builder:base" - buildpacks = ["gcr.io/paketo-buildpacks/go"] - -[[services]] - internal_port = 8080 - protocol = "tcp" - - [services.concurrency] - hard_limit = 25 - soft_limit = 20 - - [[services.ports]] - handlers = ["http"] - port = "80" - - [[services.ports]] - handlers = ["tls", "http"] - port = "443" - - [[services.tcp_checks]] - interval = 10000 - timeout = 2000 diff --git a/2022.fly/lazu.ch/main.go b/2022.fly/lazu.ch/main.go deleted file mode 100644 index b2679437a4..0000000000 --- a/2022.fly/lazu.ch/main.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import ( - "fmt" - "embed" - "html/template" - "log" - "net/http" - "os" - "strings" -) - -//go:embed templates/* -var resources embed.FS - -var t = template.Must(template.ParseFS(resources, "templates/*")) - -func main() { - port := os.Getenv("PORT") - if port == "" { - port = "8080" - } - - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - data := map[string]string{ - "Header": fmt.Sprintf("%#v", r.Header), - "Host": r.Host, - "Method": r.Method, - "PublicIp": os.Getenv("FLY_PUBLIC_IP"), - "Region": os.Getenv("FLY_REGION"), - "RemoteAddr": r.RemoteAddr, - "URL": r.URL.Redacted(), - } - for header, values := range r.Header { - data[strings.Replace(header, "-", "", -1)] = strings.Join(values, ", ") - } - - t.ExecuteTemplate(w, "index.html.tmpl", data) - }) - - log.Println("listening on", port) - log.Fatal(http.ListenAndServe(":"+port, nil)) -} diff --git a/2022.fly/lazu.ch/templates/index.html.tmpl b/2022.fly/lazu.ch/templates/index.html.tmpl deleted file mode 100644 index 0031e3b71f..0000000000 --- a/2022.fly/lazu.ch/templates/index.html.tmpl +++ /dev/null @@ -1,73 +0,0 @@ - - - - - - -

- {{.Method}} {{ if .XForwardedHost }}{{.XForwardedHost}}{{ else }}{{.Host}}{{ end }}{{.URL}} - {{ if .FastlyFf }}โ†’ fastly:{{.XForwardedServer}}{{ end }} - {{ if .FlyClientIp }}โ†’ fly-proxy:{{.FlyRegion}}{{ end }} - {{ if .Region }}โ†’ fly-app:{{.Region}}{{ else }}โ†’ app{{ end }} -

- -
- - {{ if .FastlyClientIp }} - - - - - {{ end }} - {{ if .FastlyFf }} - - - - - {{ end }} - {{ if .FlyClientIp }} - - - - - {{ end }} - - - - - {{ if .XForwardedFor }} - - - - - {{ end }} - {{ if .PublicIp }} - - - - - {{ end }} -
Fastly client IP{{.FastlyClientIp}}
Fastly POP(s){{.FastlyFf}}
Fly proxy client & protocol{{.FlyClientIp}} via http {{ .Via }}
{{ if .FlyClientIp }}Fly proxy{{ else }}Client{{ end }} TCP socket{{.RemoteAddr}}
Fly app client IP{{.XForwardedFor}}
Fly app public IP{{.PublicIp}}
- -
- -{{.Header}} - - - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 01ee6d3685..f63ff07b84 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,7 +79,7 @@ Create a new pull request via https://github.com/thechangelog/changelog.com ## How do I run the application locally? You will need to have the following dependencies installed: -- [PostgreSQL](https://www.postgresql.org/download/) v15 +- [PostgreSQL](https://www.postgresql.org/download/) v16 - [Elixir](https://elixir-lang.org/install.html) v1.14 - [Erlang/OTP](https://www.erlang.org/downloads) v26 - usually installed as an Elixir dependency - [Node.js](https://nodejs.org/en/download/) v20 LTS - [latest-v20.x](https://nodejs.org/download/release/latest-v20.x/) @@ -93,9 +93,11 @@ This is what that looks like on macOS 12, our usual development environment: ```console + # ๐Ÿ›  INSTALL DEPENDENCIES ๐Ÿ›  awk '{ system("asdf plugin-add " $1) }' < .tool-versions -asdf install +# icu4c required by https://github.com/smashedtoatoms/asdf-postgres +PKG_CONFIG_PATH="$(brew --prefix)/opt/icu4c/lib/pkgconfig" asdf install #๐Ÿ‘‡ installed on a MacBook Pro 16" (2021) running macOS 12.7.1 in ~4mins on Dec 16, 2023 by @gerhard # - Elixir v1.14.5 @@ -103,7 +105,7 @@ asdf install # - Golang 1.20.12 # - Node.js v20.10.0 # - Yarn v1.22.19 -# - PostgreSQL v15.3 +# - PostgreSQL v16.1 #๐Ÿ‘† installed on a MacBook Pro 16" (2021) running macOS 12.7.1 in ~4mins on Dec 16, 2023 by @gerhard # You will also need to install imagemagick via Homebrew. diff --git a/INFRASTRUCTURE.md b/INFRASTRUCTURE.md index 0d6ef2a97e..7eb7498c4e 100644 --- a/INFRASTRUCTURE.md +++ b/INFRASTRUCTURE.md @@ -1,4 +1,4 @@ -[![shields.io](https://img.shields.io/badge/Last%20updated%20on-Aug.%202%2C%202023-success?style=for-the-badge)](https://shipit.show/80) +[![shields.io](https://img.shields.io/badge/Last%20updated%20on-Jan.%2012%2C%202024-success?style=for-the-badge)](https://shipit.show/80) This diagram shows the current changelog.com setup: @@ -30,7 +30,7 @@ graph TD cicd --> |success #dev| chat end - repo -.- |2022.fly| app + repo -.- |fly.io/changelog-2024-01-12| app registry --> |ghcr.io/changelog/changelog-prod| app container --> |flyctl deploy| app @@ -39,25 +39,15 @@ graph TD %% PaaS - https://fly.io/dashboard/changelog subgraph Fly.io - proxy{fa:fa-globe Proxy} proxy ==> |https| app - container([ fa:fa-project-diagram Dagger Engine 2023-05-20 ]):::link click container "https://fly.io/apps/dagger-engine-2023-05-20" - app(( fab:fa-phoenix-framework App changelog-2022-03-13.fly.dev )):::link + app(( fab:fa-phoenix-framework App changelog-2024-01-12.fly.dev )):::link style app fill:#488969; - click app "https://fly.io/apps/changelog-2022-03-13" - - dbw([ fa:fa-database PostgreSQL Leader 2023-07-31 ]):::link - click dbw "https://fly.io/apps/changelog-postgres-2023-07-31" - - dbr1([ fa:fa-database PostgreSQL Replica 2023-07-31 ]) - - app <==> |pgsql| dbw - dbw -.-> |replication| dbr1 + click app "https://fly.io/apps/changelog-2024-01-12" automation --> |wireguard| container container --> |ghcr.io/changelog/changelog-runtime| registry @@ -68,10 +58,19 @@ graph TD click metrics "https://fly-metrics.net" metrics --- |promql| metricsdb metricsdb -.- |metrics| app - metricsdb -.- |metrics| dbw metricsdb -.- |metrics| container end + app <==> |Postgres| dbrw + + subgraph Neon.tech + dbrw([ fa:fa-database main branch primary ]):::link + click dbrw "https://console.neon.tech/app/projects/orange-sound-86604986/branches/br-wandering-smoke-78468159" + + dbro1([ fa:fa-database main branch replica ]) + dbrw -.-> |replicate| dbro1 + end + %% Secrets secrets(( fa:fa-key 1Password )):::link click secrets "https://changelog.1password.com/" @@ -130,7 +129,7 @@ graph TD ``` > **Note** -> [Continue live editing this Mermaid diagram](https://mermaid.live/edit#pako:eNqdWHlv2zYU_yqEihYtEEm2FceJum7I4i4t0m5d3W7A4qKgJFrmLIkqSdVx43z3PVIX5Ug95j9sHe9-v3fQt1bIImL51sOHaC1lLnzXXbFM4i0RLCVOyFJXEMzD9TKLOc7X6N18mSH4hAkWYk5WKKHZBgnJ2Yb4D6ZnweR0dlTe2lsaybXv5TdPS57yG1RdgFL0CIEEIkX5VBRBqeCSyhdFUD5UH05ydnuLVjjwV9iOqVwXAZJrEq5xFpOExW5zpcxFd3e-7yujWhFhQsONFoSWVu1mKUl7OCxtaS0zQw4No2sXTFGWhJSHCbFD4N1URqPzUFKWIRst1jRHLyVaLj8MWKNk_ag1LtbyhbtlfLNK2Fa4AhR9pNLZpYmytdbRXuFCshQrtuvlsokiS0AsmuM4JhxdMrSYXyF3yNRWxA8bHCQscFMsJOHwE5MVTYhor5yYdSPMSUwBPLvHjw8SHq9D7lCGnjwZzG7J2W8i47Ho2pnjcANmiIMMr7FsdYsEiNBCfw8qViyG0tZ_zV2mDeqHfgbHL0be4vz0r-eTQ68BmbZj_4z2Tml0f4b3GjVdPCJbsfXHdm-krg8bLb8owpAIgR5E5PNeu1RSkSwyS7c2FO0no8nEWSU7UJHnXZIqD1pulTa3Bx92zllksIeq7dAM8Kg5QXYoExSRPGGmlj5jgFYpiTScbZLFIMYGCz17NIXffSu7jjr0oDcYL6BSm65XyxDrgGEetYYe9KffNKHpsvqAMze727IzxAB6gt6oJ3cHFOjZM_BN66x86qCvNvPxddVkgOlfEko7ohi0p3XFPtcuotZF9GEQnk1YW4xWvoJ-MRg0s5scdJQ8b0skXzOS0Rt7BdYRBVp0nueoTbLCiT3y7LGn0OIAvPoqSchdQpRgBNBN_AfHp6dnJ2dP7zUiIOj3ok_hsAdRsG0iHGGJAywgX0zImJPFn6_QK4IjCFgZjJntjYfDC6K-aVJeShZ2K_BrtvHxV417CxVBQ9y1zkSRitJPGmd5LD5B0wAbO75XrYaXglRz2Gu1HSFtz9fluKVQ2AXURaeY7gH321XPi0zSlOybRvG_pJS9oxXRCkmJ5DQUUdAEEbi4tBNVMFCT8H5NCqFjdsDTMLCkSDOBLjle4QwjyKpdkTgZkWhoTlY0XTyYnGbWa2LbhgYG7qQqU43xPf6Ura66PWiIQ0SdzA8RHeRTt_ymSy5IyNsdrbzR1a8CtSE7NH4DWxwUfnRY12VMKpbeATnOK1Y9JOvg1BwlSKs7w9-B92oadOwul9aSQ103VsOEzOhqR7MYOjVYgN7tciJIBmX2pAKFKiHbcSoNinuP6jW4UfH8JiS53seqwDX3jaoQczvkMFLAnkwNxd4YtYxGmIRmUPiH3QVn9AsuV7-2BlICU8GlQhQw9n-pJsWz6ckJ9M5JHU3Tk1bP3tBpeHQx_92YiClgPybOCrY3aGsqRYCUFY0LTsA6_pnCxuDO4qsgvFpcXc53n97PXhZzfuyRy8ORqUWYHYrcXKPu2j64fAJtL3qq7fx-B23UnsMoL3jW7bRhlF2rL-dAfUvV7DxGJbQy_144C68lBn7dN_QwxFthjD71qpbbyNTuPFIGlPhVRHstxcjDH4GKLw5oQmUVNmY-avAVFDF6wTKyAxeCfnB1GI1AFtRZ15zdLquGjjqXtY8-shzQSBKiesbOXUMTPcSXdkZyDJjYd3WWdDoYpss9NIb_CsqwFTAOCDQCZ25Pep86yM5FwopolWBOnLcTA2_6nNmkKWyojGyVNAYoc1jfyb38QeK6O1x1hG1sf80yCnZDdzmw7Vci4RS0kHAoMBcfLItm8CR4B6tYzFmRV2-c7yuSktjsHj3c3TNH2tjZoAl2DTjQSpJlVKD3uRrTw8eelt8ElWZyAu2qkPX5RxKcumPP80YTt-ITnVnYymqwVNPtdeYHaLuEkJvvojOQ8w3KMowGBKwjKyU8xTSyfOtWvVhasE-oevDhMsJ8oxy7Azq1QS12WWj5khfkyCpyKCwyLxd5C-KdCHia4-wfxjr3ln9r3Vi-PfNOnJPZaHR2ejyZzqbj0yNrZ_lnY2d8PJ15k9kIHk-96d2R9UVLGDszb-qdHY9G49FoNgJyGBDgx-vyHx79R8_df1dCyJo) +> [Continue live editing this Mermaid diagram](https://mermaid.live/edit#pako:eNqdWHlv2zYU_yqEihYNEEk-4rPrhizu0iJoF8ztBiwuBkqiZS4SqZJUHTfOd98jdVGO3GP-w9bx7vd7B33vhDwiztx5-hRtlMrk3PfXnCm8JZKnxAt56kuCRbhZsVjgbIPeL1YMwSdMsJQLskYJZbdIKsFvyfzJaBYMppPT4tbd0kht5sPs7kXBU3yDqgtQip4hkECULJ7KPCgUXFL1Og-Kh_ojSMbv79EaB_M1dmOqNnmA1IaEG8xikvDYr6-0uejhYT6fa6MaEWFCw1sjCK2cys1CkvHwuLSVs2KWHBpGNz6Yoi0JqQgT4obAe1sajc5DRTlDLlpuaIbeKLRafTxijZb1o9b42MiX_paL23XCt9KXoOgfqrxdmmhbKx3NFc4VT7Fmu1mt6ijyBMSiBY5jItAlR8vFFfKPmdqI-GGDg4QHfoqlIgJ-YrKmCZHNlRfzdoQFiSmAZ_f8-UHC400oPMrRycnR7Bac3SZyEcu2nRkOb8EMeZDhDVaNbpkAEVqa76OKNYultPHfcBdpg_qhn8Hxi95weT7989Xg0GtApuu5P6O9VxjdneG9QU0bj8jVbN2x3Vup68JGwy_zMCRSoicR-bw3LhVUhEV26VaGov062UE6Gm_dQW8wdPsDtz8BrVnW5ipTY1SVmfQ7IONmgkcWe6g7EWUAUcMJOkOVoIhkCd9ZZF-xLzIIdwmLQUxhY28Ev_tGdpUIaEvXGC-heOtGWMmQm4BjETWGHrSs3wxhYwy4cbe7L9pEDBVA0LV-8nBAgV6-BK-MttIbKzWVec9vyn4DLP-SULkRxaA1rYr3lXENNa6hj0eRWoezgWvpI2iXR4NlN5aD5pJlTbVkG04YvXPXYB3R-EXnWYa6EOJprYC0rqKSapcQLRgBipP5k7PpdDaevXjUk4Cg24suhe1ys_qZwdWWAkJzSHALFY8y8W34ipwpmpJ9jfj_JaUogkZEIyQlStBQRkENCuASyk00AgBi8H5DcgkIeMRTM_AkT5lElwKvMcMIIueWJB4jCh2bASVNO-Y2pw2Sith1oRLBnfQTtK7a-A5_ipotbw8q-xjRQapMpyouNTZ-MrV1zaWKBQHqKBDb6n1dt-8IZ54iermptGm6OroRVjjAkqAUFKFAYBZuoHRpiqGZHS0zLcIeCDCweUIgRqUyDVO_rGYJk0nn3ZU8Z5E7HY97Z7Pp2C90QTsPhLvFLCKCstiVqd6qJtOz8bQ_mrVBDVp5_-uWQ3cEA3ELHsbYYvSUrxXZF8IexRU65JKEolnZihvTAbTSW7JD_WtY6qD4o8PaLmJTsnTOy35WspqZWeGp4ihsLO8siBx5rydBy-5ihy049HVtNQxMRtc7Hd9Y77To_S4jkjAI3slJAyjX80oNmnuPqq24VvHqLiSZWc_KwNX3taoQCzcUME7AHqYHYmeMGkYrTNIw6JYBqwxm9AsuNsGmbaQEJoNPpcwBNr-U-Ho5Go-hfw6qaNqeNHr2lk7Lo4vFO2saptAuYuKtYZmDbqtTBMhe0zgXBKwTnyksEP4kvgrCq-XV5WL36cPkTb4QZ0NyeTgujQh7ipC7G9Te4o_uokDbiZ5yWX88o2q15zDGc8HasyyM2I3-8g7UN1T1CmRVQiPzr6W3HDbEwG9arRmIeCut8adfVXJrmcadZ9qAAr-aaG-kWHn4PdDxxQFNqCrDxu1HNb6CPEavOSM7cCHoBleL0QpkTr1NxdkeTLqV6GNa8-gfngEaSUJ0L975G5g7h_gyziiBQ917-WP7TTBslztoLP81lGEzgF4ZEytw9v5kNqqD7FwkPI_WCRbE-2Ng4c0cO-s0hTWVla2CxgJlBts8eZQ_SFx7iytPtLXtbzmjYDd0lwPbfiUKDkVLBWcEe_nBKq9ndYJ3sI7FgudZ-cb7viIpiO3u0cHdHh9pbWeNJpggcL5VhDEq0YdMbzbHT0ENvw0qw-QFxlWpquOQIjj1-8PhsDfwSz7ZWh8aWTWWKrq9yfwR2jYh5Oa76CzkfIOyCKMFAefUSYmAGRs5c-dev1g5sILpepjDZYTFrXbsAej00rncsdCZK5GTUyfPoLDIoljmHYh3IuFphtnfnLfunfm9c-fM3clw7I0nvd5sejYYTUb96amzc-azvtc_G02Gg0kPHo-Go4dT54uR0Pcmw9Fwdtbr9Xu9SQ_IYUCAH2-LP3zM_z4P_wG9QNng) Let's dig into how all the above pieces fit together. @@ -144,8 +143,9 @@ TL;DR: - Cloudflare R2 - **Application** - Elixir / Phoenix + - Typesense search - **Database** - - PostgreSQL + - PostgreSQL (Neon.tech) [changelog.com](https://changelog.com) is a monolithic [Elixir](http://elixir-lang.org) application built with the @@ -171,18 +171,20 @@ Fastly (changelog.com) โ†“ Fly.io Proxy โ†“ -Application (changelog-2022-03-13.fly.dev) +Application (changelog-2024-01-12.fly.dev) ``` -The production database - PostgreSQL - is running on Fly.io too. It is a -replicated setup, with one leader & one replica. +The production database - PostgreSQL - is running on Neon.tech. It is a +replicated setup, with one leader (RW) & one replica (RO). We are currently not +using the replica, and since Neon.tech scales down to 0, this isn't costing +anything. ``` -Application (changelog-2022-03-13.fly.dev) +Application (changelog-2024-01-12.fly.dev) โ†“ -PostgreSQL Leader +PostgreSQL Leader (RW) โ†“ -PostgreSQL Replica +PostgreSQL Replica (RO) ``` @@ -210,9 +212,13 @@ workflow jobs perspective, it is fairly standard: ## Secrets All our secrets are stored in [1Password](https://changelog.1password.com/), in -the **Shared** Vault. Currently, they are manually declared in Fly.io via -`flyctl`. They are pasted manually in [GitHub Actions -secrets](https://github.com/thechangelog/changelog.com/settings/secrets/actions). +the **changelog** vault. We are declaring a single secret in Fly.io, +`OP_SERVICE_ACCOUNT_TOKEN`, and then loading all other secrets into memory part +of app boot via `op` & `env.op`. + +In [GitHub Actions +secrets](https://github.com/thechangelog/changelog.com/settings/secrets/actions), +we are still pasting them manually. That is something that should use `op` too. ## Metrics & observability @@ -240,7 +246,7 @@ We use Typesense for search. It's near-instant & it just works. The above is what we have so far. While we like to keep things simple, our setup is a constant work in progress. We keep making small improvements all the -time, and we talk about them every 10 weeks in the context of our [Ship It! +time, and we talk about them every few months in the context of our [Ship It! Kaizen episodes](https://changelog.com/topic/kaizen). For example, this diagram and document were created in the context of [๐ŸŽง Kaizen 8: 24 improvements & a lot more](https://shipit.show/80). If you would prefer to stay in reading mode, @@ -253,43 +259,59 @@ you very much! --- -## How to upgrade our PostgreSQL instance running on Fly.io? +## How to create a new app instance? -1. Provision a new PostgreSQL instance -```console -flyctl postgres create \ - --org changelog --region iad \ - --name changelog-postgres-2023-07-31 \ - --initial-cluster-size 2 \ - --vm-size performance-2x \ - --volume-size 10 -``` +1. Start by creating a new app, e.g. `flyctl apps create changelog-2024-01-12 --org changelog` +2. Copy the existing app instance config, e.g. `cp -r fly.io/changelog-{2023-12-17,2024-01-12}` +3. Run all following commands in the app directory, e.g. `cd fly.io/changelog-2024-01-12` +4. Update the app name in e.g. `fly.toml` to match the newly created app +5. From within the app directory, set a few secrets required by the app to work correctly while testing -2. Connect to newly created instance (we want to use the new `pd_dump`, with the latest improvements) + flyctl secrets set --stage \ + OP_SERVICE_ACCOUNT_TOKEN="$(op read op://changelog/op/credential --account changelog.1password.com --cache)" \ + R2_FEEDS_BUCKET=changelog-feeds-dev \ + URL_HOST=changelog-2024-01-12.fly.dev + +6. Deploy the latest app image from + + flyctl deploy --vm-size performance-4x --image + + +## How to load data into a Neon.tech from a Fly.io Postgres instance? + +The assumption is that a Neon.tech instance has already been provisioned. + +### Pre-requisites + +- Credentials for the Fly.io Postgres: ```console -flyctl ssh console --app changelog-postgres-2023-07-31 +op read op://changelog/changelog-postgres-2023-07-31/url --account changelog.1password.com --cache ``` - -3. Create new db +- Credentials for the Neon.tech Postgres: ```console -createdb changelog --host localhost --username postgres +op read op://changelog/neon/url --account changelog.1password.com --cache ``` +- ensure there are no app instances connected to the db being restored +- ensure the db is clean before restore + +### Step-by-step guide -4. Dump database to local file +1. Connect to the **replica** instance: ```console -pg_dump --host postgres-2022-03-12.internal --username postgres changelog > changelog.sql +flyctl ssh console --select --app changelog-postgres-2023-07-31 ``` -5. Restore database from local file +2. Backup db to local file, then restore to remote host: ```console -psql --host localhost --username postgres --single-transaction changelog < changelog.sql -psql --host localhost --command 'ANALYZE VERBOSE;' changelog postgres -``` +time pg_dump --dbname="" --format=c --verbose > /data/changelog.sql +# Expected to take ~40s -> **Note** -> If a previous restore failed, run `dropdb --force --host localhost --username postgres changelog`, then `createdb ...` again. +time pg_restore --dbname="" --format=c --exit-on-error --no-owner --no-privileges < /data/changelog.sql +# Expected to take ~1m +``` -6. Configure app to use new PostgreSQL instance +3. [Warm-up the query planner](https://www.postgresql.org/docs/current/sql-analyze.html): ```console -flyctl secrets set DB_HOST=changelog-postgres-2023-07-31.flycast DB_PASS= --app changelog-2022-03-13 +time psql "" --command "ANALYZE VERBOSE;" +# Expected to take ~3s ``` diff --git a/config/prod.exs b/config/prod.exs index a753575da0..e7d907b9ef 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -23,17 +23,6 @@ config :changelog, ChangelogWeb.Endpoint, config :waffle, asset_host: "https://#{static_url_host}" -if System.get_env("HTTPS") do - config :changelog, ChangelogWeb.Endpoint, - https: [ - port: System.get_env("HTTPS_PORT", "443"), - cipher_suite: :strong, - otp_app: :changelog, - certfile: System.get_env("HTTPS_CERTFILE"), - keyfile: System.get_env("HTTPS_KEYFILE") - ] -end - config :logger, level: :info, backends: [:console, Sentry.LoggerBackend] @@ -42,15 +31,22 @@ config :changelog, Changelog.Repo, adapter: Ecto.Adapters.Postgres, database: System.get_env("DB_NAME", "changelog"), hostname: System.get_env("DB_HOST", "db"), + username: System.get_env("DB_USER", "postgres"), + ssl: true, + + ssl_opts: [ + # The cacertfile values implies Ubuntu, + # which is what we use when building the prod container image + cacertfile: "/etc/ssl/certs/ca-certificates.crt", + verify: :verify_peer, + server_name_indication: String.to_charlist(System.get_env("DB_HOST", "db")), + customize_hostname_check: [ + match_fun: :public_key.pkix_verify_hostname_match_fun(:https) + ] + ], password: SecretOrEnv.get("DB_PASS"), - pool_size: 40, timeout: 60000, - username: System.get_env("DB_USER", "postgres") - -if System.get_env("FLY_APP_NAME") do - config :changelog, Changelog.Repo, - socket_options: [:inet6] -end + pool_size: 40 config :changelog, Changelog.Mailer, adapter: Bamboo.SMTPAdapter, diff --git a/env.op b/env.op new file mode 100644 index 0000000000..9dec520ef3 --- /dev/null +++ b/env.op @@ -0,0 +1,48 @@ +export AWS_ACCESS_KEY_ID="op://changelog/aws.stats/access_key_id" +export AWS_SECRET_ACCESS_KEY="op://changelog/aws.stats/secret_access_key" +export AWS_REGION="op://changelog/aws.stats/region" +export AWS_API_HOST="op://changelog/aws.stats/api_host" + +export CM_API_TOKEN="op://changelog/campaign.monitor/api_token" +export CM_SMTP_TOKEN="op://changelog/campaign.monitor/smtp_token" + +export DB_HOST="op://changelog/neon/server" +export DB_USER="op://changelog/neon/username" +export DB_PASS="op://changelog/neon/password" +export DB_ENDPOINT_AND_PASS="op://changelog/neon/endpoint_and_password" +export DB_NAME="op://changelog/neon/database" + +export FASTLY_TOKEN="op://changelog/fastly/credential" + +export GITHUB_API_TOKEN="op://changelog/github/api_token" +export GITHUB_CLIENT_ID="op://changelog/github/client_id" +export GITHUB_CLIENT_SECRET="op://changelog/github/client_secret" + +export HONEYCOMB_API_KEY="op://changelog/honeycomb/credential" + +export MASTODON_API_TOKEN="op://changelog/mastodon/api_token" +export MASTODON_CLIENT_ID="op://changelog/mastodon/client_id" +export MASTODON_CLIENT_SECRET="op://changelog/mastodon/client_secret" + +export PLUSPLUS_SLUG="op://changelog/plusplus/credential" + +export R2_API_HOST="op://changelog/r2/api_host" +export R2_ACCESS_KEY_ID="op://changelog/r2/access_key_id" +export R2_SECRET_ACCESS_KEY="op://changelog/r2/secret_access_key" +export R2_ASSETS_BUCKET="op://changelog/r2/assets_bucket" +export R2_FEEDS_BUCKET="op://changelog/r2/feeds_bucket" + +export SECRET_KEY_BASE="op://changelog/phoenix/secret_key_base" + +export SHOPIFY_API_KEY="op://changelog/shopify/api_key" +export SHOPIFY_API_PASSWORD="op://changelog/shopify/api_password" + +export SIGNING_SALT="op://changelog/phoenix/signing_salt" + +export SLACK_APP_API_TOKEN="op://changelog/slack/app_api_token" +export SLACK_INVITE_API_TOKEN="op://changelog/slack/invite_api_token" + +export TURNSTILE_SECRET_KEY="op://changelog/turnstile/credential" + +export TYPESENSE_API_KEY="op://changelog/typesense/credential" +export TYPESENSE_URL="op://changelog/typesense/url" diff --git a/envrc.op b/envrc.op new file mode 100644 index 0000000000..98171f9ed0 --- /dev/null +++ b/envrc.op @@ -0,0 +1,24 @@ +# Requires OP_SERVICE_ACCOUNT_TOKEN to be set in the environment. +# See the `op` item in the `changelog` vault for details. +# +# Once the above is set, run: +# +# op inject -i envrc.op -o .envrc +# +# And then run: +# +# direnv allow +# # OR source envrc + +export OBAN_KEY_FINGERPRINT="op://changelog/oban/key_fingerprint" +export OBAN_LICENSE_KEY="op://changelog/oban/license_key" + +# Required for deploys to work locally +export FLY_API_TOKEN="$(flyctl auth token)" + +# Required for image publishing to work locally +export GITHUB_REPOSITORY="thechangelog/changelog.com" +export GITHUB_REF_NAME="master" +export IMAGE_OWNER=thechangelog +export GHCR_USERNAME=$USER +export GHCR_PASSWORD="op://changelog/ghcr/credential" diff --git a/2022.fly/fly.toml b/fly.io/changelog-2024-01-12/fly.toml similarity index 54% rename from 2022.fly/fly.toml rename to fly.io/changelog-2024-01-12/fly.toml index 015fee6ae7..f09476db19 100644 --- a/2022.fly/fly.toml +++ b/fly.io/changelog-2024-01-12/fly.toml @@ -1,22 +1,14 @@ # Inspired by https://github.com/fly-apps/live_beats/blob/master/fly.toml # Full app config reference: https://fly.io/docs/reference/configuration/ -app = "changelog-2022-03-13" +app = "changelog-2024-01-12" +primary_region = "iad" kill_signal = "SIGTERM" kill_timeout = 30 [deploy] strategy = "bluegreen" -release_command = "mix ecto.migrate" - -[env] -AWS_REGION = "us-east-1" -R2_ASSETS_BUCKET = "changelog-assets" -R2_FEEDS_BUCKET = "changelog-feeds" -PORT = "4000" -STATIC_URL_HOST = "cdn.changelog.com" -URL_HOST = "changelog.com" -DB_NAME = "changelog" +release_command = "db.migrate" [[services]] internal_port = 4000 @@ -40,13 +32,10 @@ handlers = ["http"] port = "80" force_https = true -# Increase the limit of connections that a single app instance can handle -# Otherwise Fly proxy will start sending the requests to instances that we don't have... yet -# Related to https://github.com/thechangelog/changelog.com/issues/424 [services.concurrency] hard_limit = 2500 soft_limit = 2000 type = "connections" [experimental] -cmd = ["mix", "phx.server"] +cmd = ["app.start"] diff --git a/magefiles/image/app.go b/magefiles/image/app.go index c4de937fe2..bfc024cd13 100644 --- a/magefiles/image/app.go +++ b/magefiles/image/app.go @@ -16,6 +16,7 @@ func (image *Image) WithAppSrc() *Image { "lib", "priv/repo", "test", + "env.op", "mix.exs", "mix.lock", }, @@ -126,6 +127,30 @@ func (image *Image) WithAppLegacyAssets() *Image { return image } +func (image *Image) WithDbMigrate() *Image { + image.container = image.container. + WithNewFile("/usr/local/bin/db.migrate", dagger.ContainerWithNewFileOpts{ + Contents: `#!/bin/bash +OP_SERVICE_ACCOUNT_TOKEN="${OP_SERVICE_ACCOUNT_TOKEN:?must be set}" +op run --env-file="./env.op" --no-masking -- mix ecto.migrate`, + Permissions: 555, + }) + + return image +} + +func (image *Image) WithAppStart() *Image { + image.container = image.container. + WithNewFile("/usr/local/bin/app.start", dagger.ContainerWithNewFileOpts{ + Contents: `#!/bin/bash +OP_SERVICE_ACCOUNT_TOKEN="${OP_SERVICE_ACCOUNT_TOKEN:?must be set}" +op run --env-file="./env.op" --no-masking -- mix phx.server`, + Permissions: 555, + }) + + return image +} + func (image *Image) WithAppRelease() *Image { image.container = image.container. WithExec([]string{ diff --git a/magefiles/image/fly.go b/magefiles/image/fly.go index aa5ce1c939..bb2a4e7389 100644 --- a/magefiles/image/fly.go +++ b/magefiles/image/fly.go @@ -25,12 +25,14 @@ func (image *Image) Deploy() *Image { } image.container = image.container. + WithEnvVariable("CACHE_BUSTED_AT", time.Now().String()). WithExec([]string{ "status", }). WithExec([]string{ "deploy", "--image", image.ProductionImageRef(), + "--vm-size", "performance-4x", }) return image.OK() @@ -107,7 +109,7 @@ func (image *Image) flyctl() *Image { func (image *Image) app() *Image { image.container = image.container. - WithMountedFile("fly.toml", image.dag.Host().Directory("2022.fly").File("fly.toml")) + WithMountedFile("fly.toml", image.dag.Host().Directory("fly.io/changelog-2024-01-12").File("fly.toml")) return image } diff --git a/magefiles/image/main.go b/magefiles/image/main.go index f664d10e83..0c899d3ddc 100644 --- a/magefiles/image/main.go +++ b/magefiles/image/main.go @@ -56,6 +56,16 @@ func (image *Image) Env(key string) *env.Env { } func (image *Image) Publish(reference string) *Image { + registryUsername := image.Env("GHCR_USERNAME") + if registryUsername.Value() == "" { + fmt.Printf( + "\n๐Ÿ‘ฎ Skip publishing %s\n"+ + "๐Ÿ‘ฎ GHCR_USERNAME env var is required to publish this image\n", + reference, + ) + return image + } + registryPassword := image.Env("GHCR_PASSWORD") if registryPassword.Value() == "" { fmt.Printf( diff --git a/magefiles/image/production.go b/magefiles/image/production.go index f339890dce..c93d5fde3f 100644 --- a/magefiles/image/production.go +++ b/magefiles/image/production.go @@ -39,11 +39,14 @@ func (image *Image) Production() *Image { // I donโ€™t think this is sufferable much longer // https://github.com/thechangelog/changelog.com/actions/runs/4430462525 func (image *Image) ProductionClean() *Image { - obanLicenseKey := os.Getenv("OBAN_LICENSE_KEY") - if obanLicenseKey == "" { + if os.Getenv("OBAN_LICENSE_KEY") == "" { fmt.Printf("\n๐Ÿ‘ฎ Building the production image requires an OBAN_LICENSE_KEY\n") return image } + if os.Getenv("OBAN_KEY_FINGERPRINT") == "" { + fmt.Printf("\n๐Ÿ‘ฎ Building the production image requires an OBAN_KEY_FINGERPRINT\n") + return image + } app := image.Production().container. WithExec([]string{ "cp", "--recursive", "--preserve=mode,ownership,timestamps", "/app", "/app.prod", @@ -65,9 +68,12 @@ func (image *Image) ProductionClean() *Image { WithAptPackages(). WithGit(). WithImagemagick(). + WithOnePassword(). + WithDbMigrate(). + WithAppStart(). WithProdEnv() - if os.Getenv("R2_ACCESS_KEY_ID") != "" { + if os.Getenv("R2_ACCESS_KEY_ID") != "" && os.Getenv("R2_SECRET_ACCESS_KEY") != "" { fmt.Printf("โšก๏ธ Uploading static assets...") image = image.UploadStaticAssets() } @@ -131,6 +137,7 @@ func (image *Image) UploadStaticAssets() *Image { R2_API_HOST := env.Get(image.ctx, image.dag.Host(), "R2_API_HOST").Secret() R2_ACCESS_KEY_ID := env.Get(image.ctx, image.dag.Host(), "R2_ACCESS_KEY_ID").Secret() R2_SECRET_ACCESS_KEY := env.Get(image.ctx, image.dag.Host(), "R2_SECRET_ACCESS_KEY").Secret() + R2_ASSETS_BUCKET := env.Get(image.ctx, image.dag.Host(), "R2_ASSETS_BUCKET").Value() _, err := image.Production(). // ๐Ÿค” Why do we need to start the app - and therefore require the DB - to upload static assets? @@ -144,7 +151,7 @@ func (image *Image) UploadStaticAssets() *Image { "mix", "ecto.migrate", }). WithSecretVariable("R2_API_HOST", R2_API_HOST). - WithEnvVariable("R2_ASSETS_BUCKET", "changelog-assets"). + WithEnvVariable("R2_ASSETS_BUCKET", R2_ASSETS_BUCKET). WithSecretVariable("R2_ACCESS_KEY_ID", R2_ACCESS_KEY_ID). WithSecretVariable("R2_SECRET_ACCESS_KEY", R2_SECRET_ACCESS_KEY). WithExec([]string{ diff --git a/magefiles/image/runtime.go b/magefiles/image/runtime.go index d8bf8d54c9..6f936eb93b 100644 --- a/magefiles/image/runtime.go +++ b/magefiles/image/runtime.go @@ -170,6 +170,20 @@ func (image *Image) WithInotifyTools() *Image { return image } +func (image *Image) WithOnePassword() *Image { + op := image.NewContainer(). + From(fmt.Sprintf("1password/op:%s", image.versions.OnePassword())). + File("/usr/local/bin/op") + + image.container = image.container. + WithFile("/usr/local/bin/op", op). + WithExec([]string{ + "op", "--version", + }) + + return image +} + func (image *Image) WithPostgreSQLClient() *Image { image.container = image.container. WithExec([]string{ diff --git a/magefiles/magefiles.go b/magefiles/magefiles.go index 5143fb554d..1cb8eec90b 100644 --- a/magefiles/magefiles.go +++ b/magefiles/magefiles.go @@ -103,6 +103,7 @@ func (Fly) Deploy(ctx context.Context) { image.New(ctx, dag.Pipeline("๐Ÿ DEPLOY")).Deploy() } +// Start Dagger Engine on Fly.io func (Fly) DaggerStart(ctx context.Context) { defer sysexit.Handle() @@ -118,6 +119,7 @@ func (Fly) DaggerStart(ctx context.Context) { image.New(ctx, dag.Pipeline("๐Ÿš™ START DAGGER ENGINE")).DaggerStart() } +// Stop Dagger Engine on Fly.io func (Fly) DaggerStop(ctx context.Context) { defer sysexit.Handle() diff --git a/magefiles/tools/main.go b/magefiles/tools/main.go index 16884f5b38..fe12175350 100644 --- a/magefiles/tools/main.go +++ b/magefiles/tools/main.go @@ -60,6 +60,11 @@ func (v *Versions) Flyctl() string { return v.toolVersions["flyctl"] } +// https://app-updates.agilebits.com/product_history/CLI2 || asdf list all 1password-cli +func (v *Versions) OnePassword() string { + return v.toolVersions["1password-cli"] +} + func toolVersions() map[string]string { wd, err := os.Getwd() if err != nil {