From be6997ac383c185ecb67e1873f94bf907abd5c68 Mon Sep 17 00:00:00 2001 From: Elizabeth Worstell Date: Thu, 5 Dec 2024 14:15:29 -0500 Subject: [PATCH] add kube configs for timeline service --- Justfile | 3 +- backend/controller/admin/local_client_test.go | 2 + .../controller/encryption/integration_test.go | 1 + .../sql/database_integration_test.go | 1 + charts/ftl/templates/_helpers.tpl | 7 +- charts/ftl/templates/timeline-deployment.yaml | 83 +++++++++++++++++++ charts/ftl/templates/timeline-role.yaml | 8 ++ charts/ftl/templates/timeline-services.yaml | 24 ++++++ charts/ftl/values.yaml | 60 ++++++++++++++ deployment/Justfile | 4 +- frontend/cli/integration_test.go | 5 +- .../ftl/ftltest/ftltest_integration_test.go | 1 + .../languageplugin/plugin_integration_test.go | 4 + internal/integration/harness.go | 23 ++++- internal/projectconfig/integration_test.go | 2 + 15 files changed, 222 insertions(+), 6 deletions(-) create mode 100644 charts/ftl/templates/timeline-deployment.yaml create mode 100644 charts/ftl/templates/timeline-role.yaml create mode 100644 charts/ftl/templates/timeline-services.yaml diff --git a/Justfile b/Justfile index fb4d9d9c89..c84c58b30e 100644 --- a/Justfile +++ b/Justfile @@ -46,7 +46,8 @@ DOCKER_IMAGES := ''' "cron": {}, "http-ingress": {}, "runner": {}, - "runner-jvm": {} + "runner-jvm": {}, + "timeline": {} } ''' diff --git a/backend/controller/admin/local_client_test.go b/backend/controller/admin/local_client_test.go index db95b4ac15..46a20ede60 100644 --- a/backend/controller/admin/local_client_test.go +++ b/backend/controller/admin/local_client_test.go @@ -29,6 +29,7 @@ func TestDiskSchemaRetrieverWithBuildArtefact(t *testing.T) { in.WithFTLConfig("ftl-project-dr.toml"), in.WithoutController(), in.WithoutProvisioner(), + in.WithoutTimeline(), in.CopyModule("dischema"), in.Build("dischema"), func(t testing.TB, ic in.TestContext) { @@ -47,6 +48,7 @@ func TestDiskSchemaRetrieverWithNoSchema(t *testing.T) { in.WithFTLConfig("ftl-project-dr.toml"), in.WithoutController(), in.WithoutProvisioner(), + in.WithoutTimeline(), in.CopyModule("dischema"), func(t testing.TB, ic in.TestContext) { _, err := getDiskSchema(t, ic.Context) diff --git a/backend/controller/encryption/integration_test.go b/backend/controller/encryption/integration_test.go index 79898ea739..df9960a14a 100644 --- a/backend/controller/encryption/integration_test.go +++ b/backend/controller/encryption/integration_test.go @@ -62,6 +62,7 @@ func TestKMSEncryptorLocalstack(ts *testing.T) { in.WithLocalstack(), in.WithoutController(), in.WithoutProvisioner(), + in.WithoutTimeline(), in.Action(func(t testing.TB, ic in.TestContext) { endpoint := "http://localhost:4566" diff --git a/backend/controller/sql/database_integration_test.go b/backend/controller/sql/database_integration_test.go index d6f1fb7c86..2e475b0537 100644 --- a/backend/controller/sql/database_integration_test.go +++ b/backend/controller/sql/database_integration_test.go @@ -54,6 +54,7 @@ func TestMigrate(t *testing.T) { in.Run(t, in.WithoutController(), in.WithoutProvisioner(), + in.WithoutTimeline(), in.DropDBAction(t, dbName), in.Fail(q(), "Should fail because the database does not exist."), in.Exec("ftl", "migrate", "--dsn", dbUri), diff --git a/charts/ftl/templates/_helpers.tpl b/charts/ftl/templates/_helpers.tpl index c773eabaca..6581edbd3c 100644 --- a/charts/ftl/templates/_helpers.tpl +++ b/charts/ftl/templates/_helpers.tpl @@ -56,4 +56,9 @@ app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/name: {{ include "ftl.fullname" . }} app.kubernetes.io/component: http-ingress app.kubernetes.io/instance: {{ .Release.Name }} -{{- end -}} \ No newline at end of file +{{- end -}} +{{- define "ftl-timeline.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ftl.fullname" . }} +app.kubernetes.io/component: timeline +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/charts/ftl/templates/timeline-deployment.yaml b/charts/ftl/templates/timeline-deployment.yaml new file mode 100644 index 0000000000..a6c826ce94 --- /dev/null +++ b/charts/ftl/templates/timeline-deployment.yaml @@ -0,0 +1,83 @@ +{{- if .Values.timeline.enabled }} +{{ $version := printf "v%s" .Chart.Version -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ftl.fullname" . }}-timeline + labels: + {{- include "ftl.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.timeline.replicas }} + revisionHistoryLimit: {{ .Values.timeline.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "ftl-timeline.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "ftl-timeline.selectorLabels" . | nindent 8 }} + {{- if .Values.timeline.podAnnotations }} + annotations: + {{- toYaml .Values.timeline.podAnnotations | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.timeline.serviceAccountName }} + containers: + - name: app + image: "{{ .Values.timeline.image.repository }}:{{ .Values.timeline.image.tag | default $version }}" + imagePullPolicy: {{ .Values.timeline.image.pullPolicy }} + resources: + limits: + cpu: "{{ .Values.timeline.resources.limits.cpu }}" + memory: "{{ .Values.timeline.resources.limits.memory }}" + requests: + cpu: "{{ .Values.timeline.resources.requests.cpu }}" + memory: "{{ .Values.timeline.resources.requests.memory }}" + {{- if .Values.timeline.envFrom }} + envFrom: + {{- if .Values.timeline.envFrom }} + {{- toYaml .Values.timeline.envFrom | nindent 12 }} + {{- end }} + {{- end }} + env: + {{- if .Values.timeline.env }} + {{- toYaml .Values.timeline.env | nindent 12 }} + {{- end }} + + ports: + {{- range .Values.timeline.ports }} + - name: {{ .name }} + containerPort: {{ .containerPort }} + protocol: {{ .protocol | default "TCP" }} + {{- end }} + readinessProbe: + {{- if .Values.timeline.readinessProbe }} + {{- toYaml .Values.timeline.readinessProbe | nindent 12 }} + {{- else }} + httpGet: + path: /healthz + port: 8892 + initialDelaySeconds: 1 + periodSeconds: 2 + timeoutSeconds: 2 + successThreshold: 1 + failureThreshold: 15 + {{- end }} + {{- if .Values.timeline.nodeSelector }} + nodeSelector: + {{- toYaml .Values.timeline.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.timeline.affinity }} + affinity: + {{- toYaml .Values.timeline.affinity | nindent 8 }} + {{- end }} + {{- if .Values.timeline.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml .Values.timeline.topologySpreadConstraints | nindent 8 }} + {{- end }} + {{- if .Values.timeline.tolerations }} + tolerations: + {{- toYaml .Values.timeline.tolerations | nindent 8 }} + {{- end }} + + {{- end }} diff --git a/charts/ftl/templates/timeline-role.yaml b/charts/ftl/templates/timeline-role.yaml new file mode 100644 index 0000000000..bf6f92aa3e --- /dev/null +++ b/charts/ftl/templates/timeline-role.yaml @@ -0,0 +1,8 @@ +{{- if .Values.timeline.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.timeline.serviceAccountName }} + namespace: {{ .Release.Namespace }} + +{{- end }} diff --git a/charts/ftl/templates/timeline-services.yaml b/charts/ftl/templates/timeline-services.yaml new file mode 100644 index 0000000000..edb8473aea --- /dev/null +++ b/charts/ftl/templates/timeline-services.yaml @@ -0,0 +1,24 @@ +{{- if .Values.timeline.enabled }} +apiVersion: v1 +kind: Service +metadata: + labels: + {{- include "ftl.labels" . | nindent 4 }} + name: {{ include "ftl.fullname" . }}-timeline + {{- if .Values.timeline.service.annotations }} + annotations: + {{- toYaml .Values.timeline.service.annotations | nindent 4 }} + {{- end }} +spec: + ports: + {{- range .Values.timeline.service.ports }} + - name: {{ .name }} + port: {{ .port }} + protocol: {{ .protocol | default "TCP" }} + targetPort: {{ .targetPort }} + {{- end }} + selector: + {{- include "ftl-timeline.selectorLabels" . | nindent 4 }} + type: {{ .Values.timeline.service.type | default "ClusterIP" }} + +{{- end }} diff --git a/charts/ftl/values.yaml b/charts/ftl/values.yaml index aba69b132c..7bca2fb883 100644 --- a/charts/ftl/values.yaml +++ b/charts/ftl/values.yaml @@ -305,3 +305,63 @@ ingress: affinity: null topologySpreadConstraints: null tolerations: null + +timeline: + enabled: true + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: FTL_ENDPOINT + value: "http://ftl-controller:8892" + - name: FTL_BIND + value: "http://$(MY_POD_IP):8892" + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + ports: + - name: http + containerPort: 8892 + protocol: TCP + port: 8892 + + ingressAnnotations: + kubernetes.io/ingress.class: nginx + replicas: 2 + revisionHistoryLimit: 0 + + image: + repository: "ftl0/ftl-timeline" + pullPolicy: IfNotPresent + + resources: + requests: + memory: 512Mi + cpu: 10m + limits: + memory: 512Mi + cpu: 2 + + envFrom: null + serviceAccountName: ftl-timeline + + readinessProbe: null + + service: + type: ClusterIP + annotations: null + ports: + - name: "http-8892" + port: 80 + protocol: TCP + targetPort: 8892 + + podAnnotations: + proxy.istio.io/config: | + holdApplicationUntilProxyStarts: true + nodeSelector: null + affinity: null + topologySpreadConstraints: null + tolerations: null diff --git a/deployment/Justfile b/deployment/Justfile index 1212519636..21c1a01e5e 100644 --- a/deployment/Justfile +++ b/deployment/Justfile @@ -27,6 +27,7 @@ full-deploy: build-all-images setup-istio-cluster kubectl rollout restart deployment ftl-provisioner || true # if this exists already restart it to get the latest image kubectl rollout restart deployment ftl-cron || true # if this exists already restart it to get the latest image kubectl rollout restart deployment ftl-http-ingress || true # if this exists already restart it to get the latest image + kubectl rollout restart deployment ftl-timeline || true # if this exists already restart it to get the latest image just apply || sleep 5 # wait for CRDs to be created, the initial apply will usually fail just apply @@ -35,6 +36,7 @@ wait-for-kube: #!/bin/bash while [ -z "$(kubectl get pod ftl-postgresql-0)" ]; do sleep 1; done kubectl wait --for=condition=ready pod/ftl-postgresql-0 --timeout=5m + kubectl wait --for=condition=available deployment/ftl-timeline --timeout=5m kubectl wait --for=condition=available deployment/ftl-controller --timeout=5m kubectl wait --for=condition=available deployment/ftl-http-ingress --timeout=5m kubectl wait --for=condition=available deployment/registry --timeout=5m @@ -59,7 +61,7 @@ setup-cluster: setup-registry #!/bin/bash bash -c "cd ../ && just chart dep-update" if [ -z "$(k3d cluster list | grep ftl)" ]; then - k3d cluster create ftl --api-port 6550 -p "8892:80@loadbalancer" -p "8891:80@loadbalancer" -p "8893:80@loadbalancer" --agents 2 \ + k3d cluster create ftl --api-port 6550 -p "8892:80@loadbalancer" -p "8891:80@loadbalancer" -p "8893:80@loadbalancer" --agents 2 \ --registry-use {{registry_full}} \ --registry-config '{{mirrors}}'\ --k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%@agent:*' \ diff --git a/frontend/cli/integration_test.go b/frontend/cli/integration_test.go index 06456d0be8..80fd18d5ab 100644 --- a/frontend/cli/integration_test.go +++ b/frontend/cli/integration_test.go @@ -17,7 +17,7 @@ func TestConfigsWithController(t *testing.T) { } func TestConfigsWithoutController(t *testing.T) { - Run(t, configActions(t, WithoutController(), WithoutProvisioner())...) + Run(t, configActions(t, WithoutController(), WithoutProvisioner(), WithoutTimeline())...) } func configActions(t *testing.T, prepend ...ActionOrOption) []ActionOrOption { @@ -44,7 +44,7 @@ func TestSecretsWithController(t *testing.T) { } func TestSecretsWithoutController(t *testing.T) { - Run(t, secretActions(t, WithoutController(), WithoutProvisioner())...) + Run(t, secretActions(t, WithoutController(), WithoutProvisioner(), WithoutTimeline())...) } func secretActions(t *testing.T, prepend ...ActionOrOption) []ActionOrOption { @@ -97,6 +97,7 @@ func testImportExport(t *testing.T, object string) { Run(t, WithoutController(), WithoutProvisioner(), + WithoutTimeline(), // duplicate project file in the temp directory Exec("cp", firstProjFile, secondProjFile), // import into first project file diff --git a/go-runtime/ftl/ftltest/ftltest_integration_test.go b/go-runtime/ftl/ftltest/ftltest_integration_test.go index ece545c14e..2e7cfd5437 100644 --- a/go-runtime/ftl/ftltest/ftltest_integration_test.go +++ b/go-runtime/ftl/ftltest/ftltest_integration_test.go @@ -13,6 +13,7 @@ func TestModuleUnitTests(t *testing.T) { in.WithFTLConfig("wrapped/ftl-project.toml"), in.WithoutController(), in.WithoutProvisioner(), + in.WithoutTimeline(), in.GitInit(), in.CopyModule("time"), in.CopyModule("wrapped"), diff --git a/internal/buildengine/languageplugin/plugin_integration_test.go b/internal/buildengine/languageplugin/plugin_integration_test.go index 2e6af93566..8c7456ce78 100644 --- a/internal/buildengine/languageplugin/plugin_integration_test.go +++ b/internal/buildengine/languageplugin/plugin_integration_test.go @@ -69,6 +69,7 @@ func TestBuilds(t *testing.T) { in.WithLanguages("go"), in.WithoutController(), in.WithoutProvisioner(), + in.WithoutTimeline(), in.CopyModule(MODULE_NAME), startPlugin(), setUpModuleConfig(MODULE_NAME), @@ -130,6 +131,7 @@ func TestDependenciesUpdate(t *testing.T) { in.WithLanguages("go"), //no java support yet, as it relies on writeGenericSchemaFiles in.WithoutController(), in.WithoutProvisioner(), + in.WithoutTimeline(), in.CopyModule(MODULE_NAME), startPlugin(), setUpModuleConfig(MODULE_NAME), @@ -165,6 +167,7 @@ func TestBuildLock(t *testing.T) { in.WithLanguages("go"), in.WithoutController(), in.WithoutProvisioner(), + in.WithoutTimeline(), in.CopyModule(MODULE_NAME), startPlugin(), setUpModuleConfig(MODULE_NAME), @@ -194,6 +197,7 @@ func TestBuildsWhenAlreadyLocked(t *testing.T) { in.WithLanguages("go"), in.WithoutController(), in.WithoutProvisioner(), + in.WithoutTimeline(), in.CopyModule(MODULE_NAME), startPlugin(), setUpModuleConfig(MODULE_NAME), diff --git a/internal/integration/harness.go b/internal/integration/harness.go index e2a8cbf632..e326c79351 100644 --- a/internal/integration/harness.go +++ b/internal/integration/harness.go @@ -153,6 +153,13 @@ func WithoutProvisioner() Option { } } +// WithoutTimeline is a Run* option that disables starting the timeline service. +func WithoutTimeline() Option { + return func(o *options) { + o.startTimeline = false + } +} + // WithProvisionerConfig is a Run* option that specifies the provisioner config to use. func WithProvisionerConfig(config string) Option { return func(o *options) { @@ -174,6 +181,7 @@ type options struct { startController bool devMode bool startProvisioner bool + startTimeline bool provisionerConfig string requireJava bool envars map[string]string @@ -193,6 +201,7 @@ func run(t *testing.T, actionsOrOptions ...ActionOrOption) { opts := options{ startController: true, startProvisioner: true, + startTimeline: true, languages: []string{"go"}, envars: map[string]string{}, } @@ -337,6 +346,9 @@ func run(t *testing.T, actionsOrOptions ...ActionOrOption) { if opts.startProvisioner { provisioner = rpc.Dial(provisionerconnect.NewProvisionerServiceClient, "http://localhost:8893", log.Debug) } + if opts.startTimeline { + timeline = rpc.Dial(timelinev1connect.NewTimelineServiceClient, "http://localhost:8894", log.Debug) + } testData := filepath.Join(cwd, "testdata", language) if opts.testDataDir != "" { @@ -372,7 +384,6 @@ func run(t *testing.T, actionsOrOptions ...ActionOrOption) { ic.Controller = controller ic.Schema = schema ic.Console = console - ic.Timeline = timeline Infof("Waiting for controller to be ready") ic.AssertWithRetry(t, func(t testing.TB, ic TestContext) { @@ -391,6 +402,16 @@ func run(t *testing.T, actionsOrOptions ...ActionOrOption) { }) } + if opts.startTimeline { + ic.Timeline = timeline + + Infof("Waiting for timeline to be ready") + ic.AssertWithRetry(t, func(t testing.TB, ic TestContext) { + _, err := ic.Timeline.Ping(ic, connect.NewRequest(&ftlv1.PingRequest{})) + assert.NoError(t, err) + }) + } + Infof("Starting test") for _, action := range actions { diff --git a/internal/projectconfig/integration_test.go b/internal/projectconfig/integration_test.go index 889dbe7497..5f7b67f538 100644 --- a/internal/projectconfig/integration_test.go +++ b/internal/projectconfig/integration_test.go @@ -30,6 +30,7 @@ func TestConfigCmdWithoutController(t *testing.T) { in.WithFTLConfig("configs-ftl-project.toml"), in.WithoutController(), in.WithoutProvisioner(), + in.WithoutTimeline(), in.ExecWithExpectedOutput("\"value\"\n", "ftl", "config", "get", "key"), ) } @@ -56,6 +57,7 @@ func TestFindConfig(t *testing.T) { in.Run(t, in.WithoutController(), in.WithoutProvisioner(), + in.WithoutTimeline(), in.CopyModule("findconfig"), checkConfig("findconfig"), checkConfig("findconfig/subdir"),