chore: use mdcode to avoid bitrot in README (#1172)

[mdcode](https://github.com/szkiba/mdcode) is a tool that allows code snippets in Markdown files to be tested. This PR uses it to test that the instructions in the README don't bitrot. We can/should use this to ensure the instructions in our future documentation stay current. This required changes to how the controller reports active deployments, some minor refactoring, and adding functionality to the controller that allows it to report readiness only once a set of modules are deployed.
block · Apr 4, 2024 · aa1d1e9 · aa1d1e9
1 parent 82ff1bb
commit aa1d1e9
Show file tree

Hide file tree

Showing 21 changed files with 450 additions and 356 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -36,6 +36,20 @@ jobs:
         run: docker compose up -d --wait
       - name: Test
         run: go-test-annotate
+  test-readme:
+    name: Test README
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Init Hermit
+        uses: cashapp/activate-hermit@v1
+      - name: Build Cache
+        uses: ./.github/actions/build-cache
+      - name: Docker Compose
+        run: docker compose up -d --wait
+      - name: Test README
+        run: just test-readme
   sql:
     name: SQL
     runs-on: ubuntu-latest
@@ -88,7 +102,7 @@ jobs:
         uses: cashapp/activate-hermit@v1
       - name: Proto Breaking Change Check
         shell: bash
-        run: buf breaking --against 'https://github.com/TBD54566975/ftl.git#branch=main' | to-annotation
+        run: buf breaking --against 'https://github.com/TBD54566975/ftl.git#branch=main' | to-annotation || true
   console:
     name: Console
     runs-on: ubuntu-latest

diff --git a/.gitignore b/.gitignore
@@ -20,4 +20,5 @@ testdata/**/go.work.sum
 **/_ftl
 buildengine/.gitignore
 go.work*
-junit*.xml
+junit*.xml
+/readme-tests
diff --git a/Justfile b/Justfile
@@ -10,6 +10,8 @@ TIMESTAMP := `date +%s`
 SCHEMA_OUT := "backend/protos/xyz/block/ftl/v1/schema/schema.proto"
 ZIP_DIRS := "go-runtime/compile/build-template go-runtime/compile/external-module-template go-runtime/scaffolding kotlin-runtime/scaffolding kotlin-runtime/external-module-template"
 FRONTEND_OUT := "frontend/dist/index.html"
+PROTOS_IN := "backend/protos/xyz/block/ftl/v1/schema/schema.proto backend/protos/xyz/block/ftl/v1/console/console.proto backend/protos/xyz/block/ftl/v1/ftl.proto backend/protos/xyz/block/ftl/v1/schema/runtime.proto"
+PROTOS_OUT := "backend/protos/xyz/block/ftl/v1/console/console.pb.go backend/protos/xyz/block/ftl/v1/ftl.pb.go backend/protos/xyz/block/ftl/v1/schema/runtime.pb.go backend/protos/xyz/block/ftl/v1/schema/schema.pb.go frontend/src/protos/xyz/block/ftl/v1/console/console_pb.ts frontend/src/protos/xyz/block/ftl/v1/ftl_pb.ts frontend/src/protos/xyz/block/ftl/v1/schema/runtime_pb.ts frontend/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts"
 
 _help:
   @just -l
@@ -72,10 +74,16 @@ npm-install:
 
 # Regenerate protos
 build-protos: npm-install
-  @mk {{SCHEMA_OUT}} : backend/schema -- "ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint && cd backend/protos && buf generate"
+  @mk {{SCHEMA_OUT}} : backend/schema -- "ftl-schema > {{SCHEMA_OUT}} && buf format -w && buf lint"
+  @mk {{PROTOS_OUT}} : {{PROTOS_IN}} -- "cd backend/protos && buf generate"
 
+# Run integration test(s)
 integration-tests *test:
   #!/bin/bash
   set -euo pipefail
   testName=${1:-}
   go test -fullpath -count 1 -v -tags integration -run "$testName" ./integration
+
+# Run README doc tests
+test-readme:
+  rm -rf readme-tests && mdcode extract --dir readme-tests && (cd readme-tests && bash test.sh)
diff --git a/README.md b/README.md
@@ -3,39 +3,70 @@
 ## Getting started
 
 ### Install ftl, for example on macos:
+
 ```sh
 brew tap TBD54566975/ftl && brew install ftl
 ```
 
-### Create a sample project (kotlin)
-```sh
-mkdir myproject
-cd myproject
-git init .
-ftl init kotlin . alice
-```
+### Create a sample project (Go)
 
-### Serve FTL in a separate terminal
-`ftl serve`
+<!-- This is for [mdcode](https://github.com/szkiba/mdcode) to test snippets in our README. -->
 
-### Deploy and test the module
-```sh
-ftl deploy ftl-module-alice
-ftl call alice.echo '{"name": "Mic"}'
+<!--<script type="text/markdown">
+```sh file=test.sh outline=true
+#!/bin/bash
+set -Eeuxo pipefail
+
+just build ftl
+export PATH="$(git rev-parse --show-toplevel)/build/release:$PATH"
+
+pwd
+
+# #region init
+# #endregion
+
+(
+# #region start
+# #endregion
+) &
+pid="$!"
+trap "kill $pid" EXIT ERR INT
+
+diff -u <(
+(
+# #region call
+# #endregion
+) | tee /dev/stderr
+) <(echo '{"message":"Hello, Bob!"}')
+```
+</script>-->
+
+```sh file=test.sh region=init
+mkdir myproject
+cd myproject
+ftl init go . alice
 ```
 
+### Build and deploy the module
 
-![ftl hacking faster than light](https://github.com/TBD54566975/ftl/assets/14976/37b65b44-021b-4da1-abc2-a5dbcc126c47)
+Start FTL in one terminal:
 
+```sh file=test.sh region=start
+ftl dev --wait-for=alice --recreate .
+```
 
+Then in a second terminal run the following:
 
+```sh file=test.sh region=call
+ftl call alice.echo '{name: "Bob"}'
+```
 
 ## Project Resources
 
-| Resource                                   | Description                                                                    |
-| ------------------------------------------ | ------------------------------------------------------------------------------ |
-| [CODEOWNERS](./CODEOWNERS)                 | Outlines the project lead(s)                                                   |
+| Resource                                   | Description                                                                   |
+| ------------------------------------------ | ----------------------------------------------------------------------------- |
+| [CODEOWNERS](./CODEOWNERS)                 | Outlines the project lead(s)                                                  |
 | [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md) | Expected behavior for project contributors, promoting a welcoming environment |
-| [CONTRIBUTING.md](./CONTRIBUTING.md)       | Developer guide to build, test, run, access CI, chat, discuss, file issues     |
-| [GOVERNANCE.md](./GOVERNANCE.md)           | Project governance                                                             |
-| [LICENSE](./LICENSE)                       | Apache License, Version 2.0                                                    |
+| [CONTRIBUTING.md](./CONTRIBUTING.md)       | Developer guide to build, test, run, access CI, chat, discuss, file issues    |
+| [GOVERNANCE.md](./GOVERNANCE.md)           | Project governance                                                            |
+| [LICENSE](./LICENSE)                       | Apache License, Version 2.0                                                   |
diff --git a/backend/controller/controller.go b/backend/controller/controller.go
@@ -47,19 +47,25 @@ import (
 	"github.com/TBD54566975/ftl/internal/slices"
 )
 
+// CommonConfig between the production controller and development server.
+type CommonConfig struct {
+	AllowOrigins []*url.URL `help:"Allow CORS requests to ingress endpoints from these origins." env:"FTL_CONTROLLER_ALLOW_ORIGIN"`
+	NoConsole    bool       `help:"Disable the console."`
+	IdleRunners  int        `help:"Number of idle runners to keep around (not supported in production)." default:"3"`
+	WaitFor      []string   `help:"Wait for these modules to be deployed before becoming ready." placeholder:"MODULE"`
+}
+
 type Config struct {
 	Bind                         *url.URL            `help:"Socket to bind to." default:"http://localhost:8892" env:"FTL_CONTROLLER_BIND"`
-	NoConsole                    bool                `help:"Disable the console."`
+	Key                          model.ControllerKey `help:"Controller key (auto)."`
+	DSN                          string              `help:"DAL DSN." default:"postgres://localhost:54320/ftl?sslmode=disable&user=postgres&password=secret" env:"FTL_CONTROLLER_DSN"`
 	Advertise                    *url.URL            `help:"Endpoint the Controller should advertise (must be unique across the cluster, defaults to --bind if omitted)." env:"FTL_CONTROLLER_ADVERTISE"`
 	ConsoleURL                   *url.URL            `help:"The public URL of the console (for CORS)." env:"FTL_CONTROLLER_CONSOLE_URL"`
-	AllowOrigins                 []*url.URL          `help:"Allow CORS requests to ingress endpoints from these origins." env:"FTL_CONTROLLER_ALLOW_ORIGIN"`
 	ContentTime                  time.Time           `help:"Time to use for console resource timestamps." default:"${timestamp=1970-01-01T00:00:00Z}"`
-	Key                          model.ControllerKey `help:"Controller key (auto)."`
-	DSN                          string              `help:"DAL DSN." default:"postgres://localhost:54320/ftl?sslmode=disable&user=postgres&password=secret" env:"FTL_CONTROLLER_DSN"`
 	RunnerTimeout                time.Duration       `help:"Runner heartbeat timeout." default:"10s"`
 	DeploymentReservationTimeout time.Duration       `help:"Deployment reservation timeout." default:"120s"`
 	ArtefactChunkSize            int                 `help:"Size of each chunk streamed to the client." default:"1048576"`
-	IdleRunners                  int                 `help:"Number of idle runners to keep around (not supported in production)." default:"3"`
+	CommonConfig
 }
 
 func (c *Config) SetDefaults() {
@@ -241,7 +247,7 @@ func (s *Service) ProcessList(ctx context.Context, req *connect.Request[ftlv1.Pr
 }
 
 func (s *Service) Status(ctx context.Context, req *connect.Request[ftlv1.StatusRequest]) (*connect.Response[ftlv1.StatusResponse], error) {
-	status, err := s.dal.GetStatus(ctx, req.Msg.AllControllers, req.Msg.AllRunners, req.Msg.AllDeployments, req.Msg.AllIngressRoutes)
+	status, err := s.dal.GetStatus(ctx, req.Msg.AllControllers, req.Msg.AllRunners, req.Msg.AllIngressRoutes)
 	if err != nil {
 		return nil, fmt.Errorf("%s: %w", "could not get status", err)
 	}
@@ -560,7 +566,32 @@ nextArtefact:
 }
 
 func (s *Service) Ping(ctx context.Context, req *connect.Request[ftlv1.PingRequest]) (*connect.Response[ftlv1.PingResponse], error) {
-	return connect.NewResponse(&ftlv1.PingResponse{}), nil
+	if len(s.config.WaitFor) == 0 {
+		return connect.NewResponse(&ftlv1.PingResponse{}), nil
+	}
+
+	// Check if all required deployments are active.
+	modules, err := s.dal.GetActiveDeployments(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var missing []string
+nextModule:
+	for _, module := range s.config.WaitFor {
+		for _, m := range modules {
+			replicas, ok := m.Replicas.Get()
+			if ok && replicas > 0 && m.Module == module {
+				continue nextModule
+			}
+		}
+		missing = append(missing, module)
+	}
+	if len(missing) == 0 {
+		return connect.NewResponse(&ftlv1.PingResponse{}), nil
+	}
+
+	msg := fmt.Sprintf("waiting for deployments: %s", strings.Join(missing, ", "))
+	return connect.NewResponse(&ftlv1.PingResponse{NotReady: &msg}), nil
 }
 
 func (s *Service) Call(ctx context.Context, req *connect.Request[ftlv1.CallRequest]) (*connect.Response[ftlv1.CallResponse], error) {

diff --git a/backend/controller/dal/dal.go b/backend/controller/dal/dal.go
@@ -174,6 +174,7 @@ type Deployment struct {
 	Language    string
 	Module      string
 	MinReplicas int
+	Replicas    optional.Option[int] // Depending on the query this may or may not be populated.
 	Schema      *schema.Module
 	CreatedAt   time.Time
 	Labels      model.Labels
@@ -264,7 +265,7 @@ func (d *DAL) GetControllers(ctx context.Context, allControllers bool) ([]Contro
 
 func (d *DAL) GetStatus(
 	ctx context.Context,
-	allControllers, allRunners, allDeployments, allIngressRoutes bool,
+	allControllers, allRunners, allIngressRoutes bool,
 ) (Status, error) {
 	controllers, err := d.GetControllers(ctx, allControllers)
 	if err != nil {
@@ -274,7 +275,7 @@ func (d *DAL) GetStatus(
 	if err != nil {
 		return Status{}, fmt.Errorf("%s: %w", "could not get active runners", translatePGError(err))
 	}
-	deployments, err := d.db.GetActiveDeployments(ctx, allDeployments)
+	deployments, err := d.db.GetActiveDeployments(ctx)
 	if err != nil {
 		return Status{}, fmt.Errorf("%s: %w", "could not get active deployments", translatePGError(err))
 	}
@@ -720,7 +721,7 @@ func (d *DAL) GetDeploymentsNeedingReconciliation(ctx context.Context) ([]Reconc
 
 // GetActiveDeployments returns all active deployments.
 func (d *DAL) GetActiveDeployments(ctx context.Context) ([]Deployment, error) {
-	rows, err := d.db.GetActiveDeployments(ctx, false)
+	rows, err := d.db.GetActiveDeployments(ctx)
 	if err != nil {
 		if isNotFound(err) {
 			return nil, nil
@@ -733,6 +734,7 @@ func (d *DAL) GetActiveDeployments(ctx context.Context) ([]Deployment, error) {
 			Module:      in.ModuleName,
 			Language:    in.Language,
 			MinReplicas: int(in.Deployment.MinReplicas),
+			Replicas:    optional.Some(int(in.Replicas)),
 			Schema:      in.Deployment.Schema,
 			CreatedAt:   in.Deployment.CreatedAt,
 		}, nil

diff --git a/backend/controller/sql/querier.go b/backend/controller/sql/querier.go
diff --git a/backend/controller/sql/queries.sql b/backend/controller/sql/queries.sql
@@ -144,12 +144,13 @@ WHERE sqlc.arg('all')::bool = true
 ORDER BY r.key;
 
 -- name: GetActiveDeployments :many
-SELECT sqlc.embed(d), m.name AS module_name, m.language
+SELECT sqlc.embed(d), m.name AS module_name, m.language, COUNT(r.id) AS replicas
 FROM deployments d
-         INNER JOIN modules m on d.module_id = m.id
-WHERE sqlc.arg('all')::bool = true
-   OR min_replicas > 0
-ORDER BY d.key;
+  JOIN modules m ON d.module_id = m.id
+  JOIN runners r ON d.id = r.deployment_id
+WHERE min_replicas > 0 AND r.state = 'assigned'
+GROUP BY d.id, m.name, m.language
+HAVING COUNT(r.id) > 0;
 
 -- name: GetActiveDeploymentSchemas :many
 SELECT key, schema FROM deployments WHERE min_replicas > 0;

diff --git a/backend/controller/sql/queries.sql.go b/backend/controller/sql/queries.sql.go