merge from main; resolve conflicts

block · Oct 8, 2024 · ea0108d · ea0108d
2 parents 319997c + 28ac330
commit ea0108d
Show file tree

Hide file tree

Showing 514 changed files with 16,504 additions and 12,407 deletions.
diff --git a/.github/workflows/cc.yml b/.github/workflows/cc.yml
@@ -9,7 +9,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.ref }}
-      - uses: cashapp/activate-hermit@v1
+      - uses: cashapp/activate-hermit@v1.1.3
       - name: Update PR title
         run: |
           gh pr view --json title -q .title | grep -qE '^(\w+)[:(]' && exit 0

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,7 +13,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Docker Compose
@@ -29,7 +29,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Docker Compose
@@ -45,7 +45,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Test Scripts
         run: just test-scripts
   sql:
@@ -55,7 +55,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Docker Compose
@@ -74,7 +74,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Freeze Migrations
         run: just ensure-frozen-migrations
   lint:
@@ -86,7 +86,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: golangci-lint
@@ -116,7 +116,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Proto Breaking Change Check
         shell: bash
         run: |
@@ -129,7 +129,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Console pnpm install
@@ -147,7 +147,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: VSCode extension pnpm install
@@ -165,7 +165,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
         with:
           cache: true
       - name: Build Cache
@@ -182,7 +182,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Docker Compose
@@ -207,22 +207,22 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
-      - uses: cashapp/activate-hermit@v1
+      - uses: cashapp/activate-hermit@v1.1.3
       - run: cd docs && zola build
   arch-lint:
     name: Lint Architecture
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: cashapp/activate-hermit@v1
+      - uses: cashapp/activate-hermit@v1.1.3
       - run: go-arch-lint check || true
   docker-build-controller:
     name: Build Controller Docker Image
     # if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: cashapp/activate-hermit@v1
+      - uses: cashapp/activate-hermit@v1.1.3
       - uses: ./.github/actions/build-cache
       - run: just build-docker controller
   docker-build-runner:
@@ -231,7 +231,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: cashapp/activate-hermit@v1
+      - uses: cashapp/activate-hermit@v1.1.3
       - uses: ./.github/actions/build-cache
       - run: just build-docker runner
   docker-build-box:
@@ -240,7 +240,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: cashapp/activate-hermit@v1
+      - uses: cashapp/activate-hermit@v1.1.3
       - uses: ./.github/actions/build-cache
       - run: just build-docker box
   console-e2e:
@@ -251,7 +251,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Console pnpm install
@@ -268,7 +268,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Extract test cases
         id: extract-tests
         run: |
@@ -287,7 +287,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
         with:
           cache: true
       - name: Build Cache
@@ -313,7 +313,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Extract test cases
         id: extract-tests
         run: |
@@ -332,7 +332,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
         with:
           cache: true
       - name: Build Cache
@@ -348,6 +348,19 @@ jobs:
           set -euo pipefail
           # shellcheck disable=SC2046
           go test -v -race -tags infrastructure -run '^${{ matrix.test }}$' $(git grep -l '^//go:build infrastructure' | xargs grep -l '^func ${{ matrix.test }}' | xargs -I {} dirname ./{})
+      - name: Archive Report
+        uses: actions/upload-artifact@v4
+        if: always() # Always upload the report even on failure
+        with:
+          name: kube-report-${{ matrix.test }}
+          path: /tmp/ftl-kube-report/
+      - name: Teardown Cluster
+        working-directory: deployment
+        if: always() # Always cleanup the cluster even on failure
+        run: just teardown-cluster
+      - name: Delete Images
+        if: always() # We don't want to cache these images, delete them before the cache action
+        run: docker image rm localhost:5000/ftl-runner:latest ftl-runner:latest ftl-controller:latest localhost:5000/ftl-controller:latest
   integration-success:
     name: Integration Success
     needs: [integration-run]

diff --git a/.github/workflows/createnewrelease.yml b/.github/workflows/createnewrelease.yml
@@ -11,7 +11,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Auto-version
         id: autoversion
         run: scripts/autoversion

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -11,7 +11,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Build
         run: |
           just build-docker runner
@@ -30,7 +30,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Build
         run: |
           just build-docker controller
@@ -49,7 +49,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Build
         run: |
           just build-docker box
@@ -74,7 +74,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Retrieve Runner Docker image
         uses: actions/download-artifact@v4
         with:
@@ -122,7 +122,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Build Console
@@ -144,7 +144,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
 
       - uses: actions/cache@v4
         with:
@@ -184,7 +184,7 @@ jobs:
           ref: "main"
           token: ${{ secrets.FTL_HERMIT_AUTOVERSION }}
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Setup Git Config
         run: |
           git config --global user.email "github-actions[bot]@users.noreply.github.com"
@@ -206,7 +206,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Publish
         run: |
           set -euo pipefail

diff --git a/.github/workflows/workflow-roadmap.yml b/.github/workflows/workflow-roadmap.yml
@@ -33,7 +33,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - uses: cashapp/activate-hermit@v1
+      - uses: cashapp/activate-hermit@v1.1.3
       - run: update-dashboard-issue
         env:
           GH_TOKEN: ${{ secrets.FTL_WORKFLOW_TOKEN }}
@@ -49,7 +49,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - uses: cashapp/activate-hermit@v1
+      - uses: cashapp/activate-hermit@v1.1.3
       - run: |
           declare -A label_to_issue_map=(
             ["codebase-health"]=2443

diff --git a/.github/workflows/writecache.yml b/.github/workflows/writecache.yml
@@ -14,7 +14,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Init Hermit
-        uses: cashapp/activate-hermit@v1
+        uses: cashapp/activate-hermit@v1.1.3
       - name: Docker Compose
         run: docker compose up -d --wait
       - name: Init DB

diff --git a/.go-arch-lint.yml b/.go-arch-lint.yml
@@ -8,7 +8,7 @@ components:
   internal: { in: internal/** }
   dal: { in: backend/dal/** }
   protos: { in: backend/protos/** }
-  schema: { in: backend/schema/** }
+  schema: { in: internal/schema/** }
   ftl: { in: . }
   ftl-cmd: { in: frontend/cli/ftl/** }
   go-runtime: { in: go-runtime/** }
@@ -19,7 +19,7 @@ components:
   frontend: { in: frontend/console/** }
   ftl-gen-lsp-cmd: { in: cmd/ftl-gen-lsp/** }
   ftl-initdb-cmd: { in: cmd/ftl-initdb/** }
-  ftl-schema-cmd: { in: cmd/ftl-schema/** }
+  go2proto: { in: cmd/go2proto/** }
   lint-commit-or-rollback-cmd: { in: cmd/lint-commit-or-rollback/** }
   databasetesting: { in: backend/controller/sql/databasetesting/** }
   sql: { in: backend/controller/sql/** }

diff --git a/.golangci.yml b/.golangci.yml
@@ -101,7 +101,6 @@ linters-settings:
   #   ignorePackageGlobs:
   #     - github.com/TBD54566975/ftl/*
   spancheck:
-
     extra-start-span-signatures:
       - "github.com/TBD54566975/ftl/backend/controller/observability.BeginSpan:opentelemetry"
 issues:
@@ -143,3 +142,4 @@ issues:
     - "strings.Title has been deprecated"
     - "error returned from external package is unwrapped.*TranslatePGError"
     - "struct literal uses unkeyed fields"
+    - "exported: comment on exported type"
diff --git a/Justfile b/Justfile
@@ -47,7 +47,7 @@ generate-kube-migrations:
 
 # Run "go generate" on all packages
 build-generate:
-  @mk backend/schema/aliaskind_enumer.go : backend/schema/metadataalias.go -- go generate -x ./backend/schema
+  @mk internal/schema/aliaskind_enumer.go : internal/schema/metadataalias.go -- go generate -x ./internal/schema
   @mk internal/log/log_level_string.go : internal/log/api.go -- go generate -x ./internal/log
 
 # Build command-line tools
@@ -129,12 +129,12 @@ pnpm-install:
 
 # Regenerate protos
 build-protos: pnpm-install
-  @mk {{SCHEMA_OUT}} : backend/schema -- "ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint"
+  @mk {{SCHEMA_OUT}} : internal/schema -- "go2proto -o "{{SCHEMA_OUT}}" -g 'github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema;schemapb' xyz.block.ftl.v1.schema ./internal/schema.Schema && buf format -w && buf lint"
   @mk {{PROTOS_OUT}} : {{PROTOS_IN}} -- "cd backend/protos && buf generate"
 
 # Unconditionally rebuild protos
 build-protos-unconditionally: pnpm-install
-  ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint
+  go2proto -o "{{SCHEMA_OUT}}" -g 'github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema;schemapb' xyz.block.ftl.v1.schema ./internal/schema.Schema && buf format -w && buf lint
   cd backend/protos && buf generate
 
 # Run integration test(s)
@@ -190,7 +190,9 @@ lint-backend:
   @lint-commit-or-rollback ./backend/...
 
 lint-scripts:
-	@shellcheck -f gcc -e SC2016 $(find scripts -type f -not -path scripts/tests) | to-annotation
+  #!/bin/bash
+  set -euo pipefail
+  shellcheck -f gcc -e SC2016 $(find scripts -type f -not -path scripts/tests) | to-annotation
 
 # Run live docs server
 docs: