From 81241162e7f57b6567a65c0669221cbdcf36f06b Mon Sep 17 00:00:00 2001 From: ChrisMcKenzie Date: Tue, 19 Nov 2024 14:20:37 -0600 Subject: [PATCH 1/7] add: live-query to big-peer chart --- charts/big-peer/Chart.lock | 7 +++++-- charts/big-peer/Chart.yaml | 6 ++++++ charts/big-peer/values.yaml | 3 +++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/charts/big-peer/Chart.lock b/charts/big-peer/Chart.lock index 92ccb5a..f7bbb74 100644 --- a/charts/big-peer/Chart.lock +++ b/charts/big-peer/Chart.lock @@ -26,5 +26,8 @@ dependencies: - name: dummy-auth-webhook repository: oci://quay.io/ditto-external version: 0.1.1 -digest: sha256:15b4bb1cc780a48d43a97b5aa1ec9a92ae929ea46f1d9ef566b85cfc6080995f -generated: "2024-11-12T16:35:51.590322745-05:00" +- name: live-query + repository: oci://quay.io/ditto-external + version: 1.39.2 +digest: sha256:2906f8672d73e99ce7898dda8b3d18ada8e49b0ea0f79114ffe88e5c7875ad29 +generated: "2024-11-19T14:18:33.906943369-06:00" diff --git a/charts/big-peer/Chart.yaml b/charts/big-peer/Chart.yaml index 17cd8a8..2a990d3 100644 --- a/charts/big-peer/Chart.yaml +++ b/charts/big-peer/Chart.yaml @@ -87,3 +87,9 @@ dependencies: version: "0.1.1" tags: - dummy-auth-webhook + - name: live-query + repository: "oci://quay.io/ditto-external" + condition: live-query.enabled + version: "1.39.2" + tags: + - live-query diff --git a/charts/big-peer/values.yaml b/charts/big-peer/values.yaml index 73dc301..7b379f5 100644 --- a/charts/big-peer/values.yaml +++ b/charts/big-peer/values.yaml @@ -768,3 +768,6 @@ dummy-auth-webhook: image: name: quay.io/ditto-external/big-peer-auth-webhook:1.39.2 pullPolicy: IfNotPresent + +live-query: + enabled: false From ee3ce39f5bc79c12928ff011f2b4d77869d22d7a Mon Sep 17 00:00:00 2001 From: ChrisMcKenzie Date: Tue, 19 Nov 2024 15:29:02 -0600 Subject: [PATCH 2/7] add: live-query resource controller to helm chart --- charts/big-peer/Chart.lock | 8 +++---- charts/big-peer/Chart.yaml | 5 +++-- charts/big-peer/values.yaml | 45 +++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/charts/big-peer/Chart.lock b/charts/big-peer/Chart.lock index f7bbb74..a26194f 100644 --- a/charts/big-peer/Chart.lock +++ b/charts/big-peer/Chart.lock @@ -26,8 +26,8 @@ dependencies: - name: dummy-auth-webhook repository: oci://quay.io/ditto-external version: 0.1.1 -- name: live-query +- name: live-query-resource-controller repository: oci://quay.io/ditto-external - version: 1.39.2 -digest: sha256:2906f8672d73e99ce7898dda8b3d18ada8e49b0ea0f79114ffe88e5c7875ad29 -generated: "2024-11-19T14:18:33.906943369-06:00" + version: 0.1.0-002c59 +digest: sha256:241c4da54c53eaf4b53397243ca2fc133f304b2320e674e806dd8d74e5db7248 +generated: "2024-11-19T15:20:57.960082134-06:00" diff --git a/charts/big-peer/Chart.yaml b/charts/big-peer/Chart.yaml index 2a990d3..9d5ad26 100644 --- a/charts/big-peer/Chart.yaml +++ b/charts/big-peer/Chart.yaml @@ -87,9 +87,10 @@ dependencies: version: "0.1.1" tags: - dummy-auth-webhook - - name: live-query + - name: live-query-resource-controller + alias: live-query repository: "oci://quay.io/ditto-external" condition: live-query.enabled - version: "1.39.2" + version: "0.1.0-002c59" tags: - live-query diff --git a/charts/big-peer/values.yaml b/charts/big-peer/values.yaml index 7b379f5..39e3f65 100644 --- a/charts/big-peer/values.yaml +++ b/charts/big-peer/values.yaml @@ -771,3 +771,48 @@ dummy-auth-webhook: live-query: enabled: false + + image: + repository: quay.io/ditto-external/big-peer-live-query-resource-controller + tag: latest + + env: + # Main config + INSTANCE_NAME: "live-query-resource-controller" + LIVE_QUERY_WATCHER_NAMESPACE: "" + TARGET_NAMESPACE_TEMPLATE: "bp" + APP_LABEL_SELECTOR: "cloud.app.ditto.live/cluster-ns=bp" + + # Kafka user ingress config + KAFKA_INGRESS_DOMAIN: "" + KAFKA_INGRESS_CLASS: "" + + # CDC workload config + CDC_RESOURCE_REQUEST_CPU: 40m + CDC_RESOURCE_REQUEST_MEMORY: 1G + CDC_RESOURCE_LIMIT_MEMORY: 4G + CDC_IMAGE: quay.io/ditto-external/hydra-cdc:1.39.2 + CDC_TOKEN_AUDIENCE: cloud.ditto.live + CDC_KAFKA_REPLICATION_FACTOR: 3 + + # StreamSplitter workload config + STREAM_SPLITTER_RESOURCE_REQUEST_CPU: 20m + STREAM_SPLITTER_RESOURCE_REQUEST_MEMORY: 256Mi + STREAM_SPLITTER_RESOURCE_LIMIT_MEMORY: 1G + STREAM_SPLITTER_IMAGE: quay.io/ditto-external/hydra-stream-splitter:latest-46073-c0fe1e + STREAM_SPLITTER_TOKEN_AUDIENCE: cloud.ditto.live + + # WebhookDispatcher workload config + WEBHOOK_DISPATCHER_RESOURCE_REQUEST_CPU: 20m + WEBHOOK_DISPATCHER_RESOURCE_REQUEST_MEMORY: 24Mi + WEBHOOK_DISPATCHER_RESOURCE_LIMIT_MEMORY: 128Mi + WEBHOOK_DISPATCHER_IMAGE: quay.io/ditto-external/hydra-webhook-dispatcher:latest-46073-c0fe1e + WEBHOOK_DISPATCHER_TOKEN_AUDIENCE: cloud.ditto.live + + # CDC Hearbeat workload config + CDC_HEARTBEAT_RESOURCE_REQUEST_CPU: 20m + CDC_HEARTBEAT_RESOURCE_REQUEST_MEMORY: 50Mi + CDC_HEARTBEAT_RESOURCE_LIMIT_MEMORY: 150Mi + CDC_HEARTBEAT_IMAGE: quay.io/ditto-external/cdc-heartbeat:latest-46073-c0fe1e + CDC_HEARTBEAT_TOKEN_AUDIENCE: cloud.ditto.live + CDC_HEARTBEAT_KAFKA_REPLICATION_FACTOR: 3 From 970b49898125ce040cbf2b24443c042071e9a3a8 Mon Sep 17 00:00:00 2001 From: ChrisMcKenzie Date: Tue, 19 Nov 2024 17:21:19 -0600 Subject: [PATCH 3/7] add: crds and app creation templs --- .../crds/ditto_v1alpha2_livequerycores.yaml | 154 +++++++++ .../crds/ditto_v1alpha2_livequerysinks.yaml | 293 ++++++++++++++++++ .../crds/ditto_v1alpha2_livequerysources.yaml | 159 ++++++++++ .../crds/ditto_v1alpha2_registryapp.yaml | 175 +++++++++++ charts/big-peer/templates/_app.tpl | 13 + charts/big-peer/templates/app.yaml | 1 + .../templates/classes/_registry_app.tpl | 45 +++ charts/big-peer/values.yaml | 7 +- 8 files changed, 846 insertions(+), 1 deletion(-) create mode 100755 charts/big-peer/crds/ditto_v1alpha2_livequerycores.yaml create mode 100755 charts/big-peer/crds/ditto_v1alpha2_livequerysinks.yaml create mode 100755 charts/big-peer/crds/ditto_v1alpha2_livequerysources.yaml create mode 100755 charts/big-peer/crds/ditto_v1alpha2_registryapp.yaml create mode 100644 charts/big-peer/templates/_app.tpl create mode 100644 charts/big-peer/templates/app.yaml create mode 100644 charts/big-peer/templates/classes/_registry_app.tpl diff --git a/charts/big-peer/crds/ditto_v1alpha2_livequerycores.yaml b/charts/big-peer/crds/ditto_v1alpha2_livequerycores.yaml new file mode 100755 index 0000000..88c004f --- /dev/null +++ b/charts/big-peer/crds/ditto_v1alpha2_livequerycores.yaml @@ -0,0 +1,154 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: livequerycores.cloud.app.ditto.live +spec: + group: cloud.app.ditto.live + names: + categories: + - livequery + kind: LiveQueryCore + plural: livequerycores + shortNames: + - lqc + singular: livequerycore + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The name of the RegistryApp + jsonPath: ".spec.appId" + name: RegistryApp + type: string + - description: The name of the HydraCluster + jsonPath: ".spec.hydraClusterRef.name" + name: HydraCluster + type: string + - description: The namespace of the HydraCluster + jsonPath: ".spec.hydraClusterRef.namespace" + name: HydraClusterNS + type: string + - description: The name of the CDCKafka cluster + jsonPath: ".spec.cdcKafkaClusterRef.name" + name: CDCKafka + type: string + - description: The namespace of the CDCKafka cluster + jsonPath: ".spec.cdcKafkaClusterRef.namespace" + name: CDCKafkaNS + type: string + name: v1alpha2 + schema: + openAPIV3Schema: + description: "Live Query Core represents the core workloads for the Change Data Capture (CDC) subsystem, including the CDC transformer, the stream splitter, and the CDC heartbeat. The CDC heartbeat pushes heartbeat data through the internal database's transaction log and consumes it to ensure the database is healthy. The CDC transformer consumes transaction data off of the internal database's transaction log directly, computing a DocumentDiff and sending it downstream to the stream splitter. The stream splitter splits the input stream into the target format and fans the data out to downstream LiveQuerySource Kafka Topics. LiveQuerySource and LiveQuerySink objects require a deployed LiveQueryCore to function." + properties: + spec: + properties: + appId: + description: "Stable, unique ID of the Ditto app this object belongs to" + type: string + cdcKafkaClusterRef: + description: "Kubernetes ref to the `Kafka` CR where the CDC pipeline is hosted" + properties: + name: + description: "Name of the `Kafka` CR object" + type: string + namespace: + description: "Namespace where the `Kafka` CR is located" + type: string + required: + - name + - namespace + type: object + description: + description: Optional description of this object + nullable: true + type: string + httpApiServerPoolRef: + description: "Kubernetes ref to an `HttpApiServerPool` available to this app" + nullable: true + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + hydraClusterRef: + description: "Kubernetes ref to the `HydraCluster` this app belongs to" + properties: + name: + description: "Name of the `HydraCluster` CR object" + type: string + namespace: + description: "Namespace where the `HydraCluster` CR is located" + type: string + required: + - name + - namespace + type: object + subscriptionPoolRef: + description: "Kubernetes ref to the `SubscriptionPool` this app uses" + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - appId + - cdcKafkaClusterRef + - hydraClusterRef + - subscriptionPoolRef + type: object + status: + nullable: true + properties: + conditions: + properties: + kafka_resources_created: + properties: + message: + nullable: true + type: string + status: + nullable: true + type: boolean + type: object + ready: + properties: + message: + nullable: true + type: string + status: + nullable: true + type: boolean + type: object + workloads_deployed: + properties: + message: + nullable: true + type: string + status: + nullable: true + type: boolean + type: object + required: + - kafka_resources_created + - ready + - workloads_deployed + type: object + required: + - conditions + type: object + required: + - spec + title: LiveQueryCore + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/big-peer/crds/ditto_v1alpha2_livequerysinks.yaml b/charts/big-peer/crds/ditto_v1alpha2_livequerysinks.yaml new file mode 100755 index 0000000..e18c450 --- /dev/null +++ b/charts/big-peer/crds/ditto_v1alpha2_livequerysinks.yaml @@ -0,0 +1,293 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: livequerysinks.cloud.app.ditto.live +spec: + group: cloud.app.ditto.live + names: + categories: + - cdc + kind: LiveQuerySink + plural: livequerysinks + shortNames: + - lqsink + singular: livequerysink + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: "The id of the app, equivalent to RegistryApp name" + jsonPath: ".spec.appId" + name: App + type: string + - description: The output webhook for the sink + jsonPath: ".spec.destination.webhook" + name: Webhook + type: string + - description: The output kafka topic for this sink + jsonPath: ".spec.destination.kafka" + name: Topic + type: string + - jsonPath: ".metadata.creationTimestamp" + name: Age + type: date + name: v1alpha2 + schema: + openAPIV3Schema: + description: "Live Query Sinks are dedicated data stream \"sinks\" of event data for the Change Data Capture (CDC) subsystem. Each sink is a destination for data queried out of the document store in the database. Typically, sinks use a corresponding LiveQuerySource's KafkaTopic as data input, connecting it to the requested customer interface (e.g. a Kafka user and ingress to the CDC broker bootstrap host, a connector component that consumes from the source and forwards to some destination like a webhook, et cetera)" + properties: + spec: + description: "Kubernetes CRD for Live Query \"Sink\" connectors\n\nA Sink is an application that reads query data from a Source's data stream and does something with it. Sinks will likely be connectors to other data platforms, implemented as a Kafka consumer along with a data client such as an S3 library.\n\nNew Sinks may be implemented freely whenever one has a need to read or consume the data that was written to a [`LiveQuerySource`] stream." + properties: + appId: + description: "AppId of the [`RegistryApp`] this Sink belongs to" + format: uuid + type: string + description: + description: Optional description of this Sink + nullable: true + type: string + destination: + description: "Type of Sink, per the destination where the sink sends data" + oneOf: + - required: + - kafkaConsumer + - required: + - webhook + properties: + kafkaConsumer: + properties: + cluster: + description: "Ref to a `Kafka` CRD hosting the CDC topics" + nullable: true + properties: + name: + description: "Name of the `Kafka` CR object" + type: string + namespace: + description: "Namespace where the `Kafka` CR is located" + type: string + required: + - name + - namespace + type: object + consumerGroup: + type: string + topicName: + type: string + required: + - consumerGroup + - topicName + type: object + webhook: + properties: + sourceTopicName: + type: string + url: + format: uri + type: string + required: + - sourceTopicName + - url + type: object + type: object + liveQueryCoreRef: + description: "Ref to the [`LiveQueryCore`] object associated with this Sink's [`RegistryApp`]" + nullable: true + properties: + name: + description: "The `.metadata.name` of the `LiveQueryCore` being referenced" + type: string + namespace: + description: "The `.metadata.namespace` of the `LiveQueryCore` being referenced" + type: string + required: + - name + - namespace + type: object + liveQuerySourceRef: + description: "Ref to the [`LiveQuerySource`] object associated with this Sink's [`RegistryApp`]" + properties: + name: + description: "The `.metadata.name` of the `LiveQuerySource` being referenced" + type: string + namespace: + description: "The `.metadata.namespace` of the `LiveQuerySource` being referenced" + nullable: true + type: string + required: + - name + type: object + required: + - appId + - destination + - liveQuerySourceRef + type: object + status: + nullable: true + properties: + clientConfig: + nullable: true + oneOf: + - required: + - kafka + - required: + - webhook + properties: + kafka: + properties: + authConfig: + properties: + clusterCaCert: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names" + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + clusterCaCertPassword: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names" + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + secretNamespace: + type: string + userCaCert: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names" + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + userCaCertPassword: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names" + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + required: + - clusterCaCert + - clusterCaCertPassword + - secretNamespace + - userCaCert + - userCaCertPassword + type: object + consumerGroup: + type: string + endpoint: + type: string + host: + format: uri + type: string + topicName: + type: string + required: + - authConfig + - consumerGroup + - endpoint + - host + - topicName + type: object + webhook: + properties: + hmacSeed: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names" + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + required: + - hmacSeed + type: object + type: object + conditions: + nullable: true + properties: + ready: + properties: + condition_type: + enum: + - requested + - ready + type: string + message: + nullable: true + type: string + status: + nullable: true + type: boolean + required: + - condition_type + type: object + requested: + properties: + condition_type: + enum: + - requested + - ready + type: string + message: + nullable: true + type: string + status: + nullable: true + type: boolean + required: + - condition_type + type: object + required: + - ready + - requested + type: object + type: object + required: + - spec + title: LiveQuerySink + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/big-peer/crds/ditto_v1alpha2_livequerysources.yaml b/charts/big-peer/crds/ditto_v1alpha2_livequerysources.yaml new file mode 100755 index 0000000..f4eabbf --- /dev/null +++ b/charts/big-peer/crds/ditto_v1alpha2_livequerysources.yaml @@ -0,0 +1,159 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: livequerysources.cloud.app.ditto.live +spec: + group: cloud.app.ditto.live + names: + categories: + - livequery + kind: LiveQuerySource + plural: livequerysources + shortNames: + - lqsrc + singular: livequerysource + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The name of the RegistryApp + jsonPath: ".spec.appId" + name: RegistryApp + type: string + - description: The CDC Core object this source is tied to. + jsonPath: ".spec.liveQueryCoreRef" + name: LQ Core + type: string + - description: The message schema of the output (typed or untyped) + jsonPath: ".spec.schema" + name: Schema + type: string + - jsonPath: ".metadata.creationTimestamp" + name: Age + type: date + name: v1alpha2 + schema: + openAPIV3Schema: + description: "Live Query Sources are dedicated data stream \"sources\" of event data for the Change Data Capture (CDC) subsystem. Each source typically represents a query against a set of documents in the Hydra Store mirrored out into a Kafka topic. Note that multiple live query sources may share the same query if they specify a different output schema." + properties: + spec: + description: "Kubernetes CRD for Live Query \"Source\" streams\n\nA Source stream is the result of taking the full CDC pipeline of change events and creating a filtered subset of that stream by applying a `DQL` query to it. Defining a [`LiveQuerySource`] leads to the provisioning of a new Kafka topic, and instructs `stream-splitter` to funnel data for our query to the stream.\n\nBeyond this, see [`LiveQuerySink`] for a mechanism for reading data from a Source stream and exporting it to various other data platforms.\n\n[`LiveQuerySink`]: crate::crds::v1alpha2::live_query_sink::LiveQuerySink" + properties: + appId: + description: "The AppId of the [`RegistryApp`] this source belongs to" + format: uuid + type: string + collection: + description: "Ditto collection the Live Query is subscribed for; here `None` refers to all collections which means the \"firehose\" of data" + nullable: true + type: string + description: + description: Optional description of this source object + nullable: true + type: string + liveQueryCoreRef: + description: "Reference to the `LiveQueryCore` object of this source's [`RegistryApp`]" + nullable: true + properties: + name: + description: "The `.metadata.name` of the `LiveQueryCore` being referenced" + type: string + namespace: + description: "The `.metadata.namespace` of the `LiveQueryCore` being referenced" + type: string + required: + - name + - namespace + type: object + output: + description: "The type of data stream to write our [`LiveQuerySource`] data to" + oneOf: + - required: + - kafka + properties: + kafka: + description: "Send [`LiveQuerySource`] data to a Kafka topic" + properties: + cluster: + description: "Ref to the `Kafka` CRD where CDC is hosted" + properties: + name: + description: "Name of the `Kafka` CR object" + type: string + namespace: + description: "Namespace where the `Kafka` CR is located" + type: string + required: + - name + - namespace + type: object + topicName: + description: The topic name to write to + type: string + required: + - cluster + - topicName + type: object + type: object + queryFilterExpression: + description: Query filter expression to scope the data matching for the Live Query + type: string + schema: + description: "Schema of CDC messages to be written in this source's data stream" + enum: + - untyped + - typed + type: string + required: + - appId + - output + - queryFilterExpression + - schema + type: object + status: + nullable: true + properties: + conditions: + properties: + kafka_topic_created: + properties: + message: + nullable: true + type: string + status: + nullable: true + type: boolean + type: object + ready: + properties: + message: + nullable: true + type: string + status: + nullable: true + type: boolean + type: object + requested: + properties: + message: + nullable: true + type: string + status: + nullable: true + type: boolean + type: object + required: + - kafka_topic_created + - ready + - requested + type: object + required: + - conditions + type: object + required: + - spec + title: LiveQuerySource + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/big-peer/crds/ditto_v1alpha2_registryapp.yaml b/charts/big-peer/crds/ditto_v1alpha2_registryapp.yaml new file mode 100755 index 0000000..6f264d3 --- /dev/null +++ b/charts/big-peer/crds/ditto_v1alpha2_registryapp.yaml @@ -0,0 +1,175 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: registryapps.cloud.app.ditto.live +spec: + group: cloud.app.ditto.live + names: + kind: RegistryApp + plural: registryapps + singular: registryapp + scope: Cluster + versions: + - additionalPrinterColumns: + - description: The name of the HydraCluster + jsonPath: ".spec.hydraClusterRef.name" + name: HydraCluster + type: string + - description: The namespace of the HydraCluster + jsonPath: ".spec.hydraClusterRef.namespace" + name: HydraClusterNS + type: string + - description: The name of the CDCKafka cluster + jsonPath: ".spec.cdcKafkaClusterRef.name" + name: CDCKafka + type: string + - description: The namespace of the CDCKafka cluster + jsonPath: ".spec.cdcKafkaClusterRef.namespace" + name: CDCKafkaNS + type: string + - jsonPath: ".metadata.creationTimestamp" + name: Age + type: date + name: v1alpha2 + schema: + openAPIV3Schema: + description: "Auto-generated derived type for RegistryAppSpec via `CustomResource`" + properties: + spec: + properties: + appId: + type: string + cdcKafkaClusterRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + description: + nullable: true + type: string + httpApiServerPoolRef: + nullable: true + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + hydraClusterRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + ingressDomains: + additionalProperties: + properties: + httpEnabled: + type: boolean + wsEnabled: + type: boolean + required: + - httpEnabled + - wsEnabled + type: object + default: {} + type: object + name: + type: string + organizationUrl: + type: string + slug: + type: string + subscriptionPoolRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + url: + type: string + required: + - appId + - cdcKafkaClusterRef + - hydraClusterRef + - name + - organizationUrl + - slug + - subscriptionPoolRef + - url + type: object + status: + nullable: true + properties: + changeDataCaptureStatus: + default: {} + properties: + actualCount: + format: int16 + nullable: true + type: integer + desiredCount: + format: int16 + nullable: true + type: integer + state: + enum: + - ERROR + - REQUESTED + - READY + nullable: true + type: string + type: object + ingressDomainStatus: + additionalProperties: + properties: + http: + enum: + - ERROR + - CREATED + - READY + nullable: true + type: string + observedStatus: + enum: + - ERROR + - CREATED + - READY + nullable: true + type: string + webSocket: + enum: + - ERROR + - CREATED + - READY + nullable: true + type: string + type: object + default: {} + type: object + type: object + required: + - spec + title: RegistryApp + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/big-peer/templates/_app.tpl b/charts/big-peer/templates/_app.tpl new file mode 100644 index 0000000..63b54b5 --- /dev/null +++ b/charts/big-peer/templates/_app.tpl @@ -0,0 +1,13 @@ +{{- define "common.app" -}} + {{- /* Generate named ingresses as required */ -}} + {{- range $name, $app := .Values.apps }} + {{- if $app.enabled -}} + {{- $appValues := $app -}} + + {{- $_ := set $ "ObjectValues" (dict "app" $appValues) -}} + {{- include "common.classes.registryapp" $ }} + {{- end }} + {{- end }} +{{- end }} + + diff --git a/charts/big-peer/templates/app.yaml b/charts/big-peer/templates/app.yaml new file mode 100644 index 0000000..ff52645 --- /dev/null +++ b/charts/big-peer/templates/app.yaml @@ -0,0 +1 @@ +{{ include "common.app" . }} diff --git a/charts/big-peer/templates/classes/_registry_app.tpl b/charts/big-peer/templates/classes/_registry_app.tpl new file mode 100644 index 0000000..5bca2c3 --- /dev/null +++ b/charts/big-peer/templates/classes/_registry_app.tpl @@ -0,0 +1,45 @@ +{{- define "common.classes.registryapp" -}} + {{- $fullname := include "common.names.fullname" . -}} + {{- $id := .Values.id -}} + {{- $appValues := .Values.app -}} + {{- $values := .Values.app -}} + {{- if hasKey . "ObjectValues" -}} + {{- with .ObjectValues.app -}} + {{- $appValues = . -}} + {{- end -}} + {{ end -}} + {{- if and (hasKey $appValues "nameOverride") $appValues.nameOverride -}} + {{- $app = $appValues.nameOverride -}} + {{- end -}} +--- +apiVersion: cloud.app.ditto.live/v1alpha2 +kind: RegistryApp +metadata: + name: {{ $appValues.id }} + {{- with (merge ($values.labels | default dict) (include "common.labels" $ | fromYaml)) }} + labels: {{- toYaml . | nindent 4 }} + {{- end }} + {{- with (merge ($values.annotations | default dict) (include "common.annotations" $ | fromYaml)) }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + appId: {{ $appValues.id }} + cdcKafkaClusterRef: + name: {{ .Release.Name }}-kafka + namespace: {{ .Release.Namespace }} + description: null + httpApiServerPoolRef: + name: {{ .Release.Name }}hydra-subscription-api + namespace: {{.Release.Namespace}} + hydraClusterRef: + name: {{ .Release.Name }}-hydra-store + namespace: {{ .Release.Namespace }} + ingressDomains: {} + name: {{ $fullname }} + slug: {{ $fullname }} + organizationUrl: "" + url: "" + subscriptionPoolRef: + name: {{ .Release.Name }}-hydra-subscription + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/charts/big-peer/values.yaml b/charts/big-peer/values.yaml index 39e3f65..176eb9b 100644 --- a/charts/big-peer/values.yaml +++ b/charts/big-peer/values.yaml @@ -47,6 +47,11 @@ global: tolerations: [] affinity: {} +apps: + my-app: + enabled: true + id: 891c1bdb-71c6-40d5-9374-128e467c367e + # global kafka cluster options kafka: @@ -770,7 +775,7 @@ dummy-auth-webhook: pullPolicy: IfNotPresent live-query: - enabled: false + enabled: true image: repository: quay.io/ditto-external/big-peer-live-query-resource-controller From 178154d6bb3130e824ab32dca6bbbab88b91ed42 Mon Sep 17 00:00:00 2001 From: ChrisMcKenzie Date: Thu, 21 Nov 2024 10:57:06 -0600 Subject: [PATCH 4/7] add: all live query components --- charts/big-peer/Chart.yaml | 8 +++ charts/big-peer/templates/_app.tpl | 8 ++- .../templates/classes/_live_query_core.tpl | 42 ++++++++++++++++ .../templates/classes/_live_query_sink.tpl | 41 ++++++++++++++++ .../templates/classes/_live_query_source.tpl | 42 ++++++++++++++++ .../templates/classes/_registry_app.tpl | 3 +- .../big-peer/templates/lib/chart/_labels.tpl | 12 +++++ charts/big-peer/values.yaml | 49 ++++++++++++++++--- 8 files changed, 195 insertions(+), 10 deletions(-) create mode 100644 charts/big-peer/templates/classes/_live_query_core.tpl create mode 100644 charts/big-peer/templates/classes/_live_query_sink.tpl create mode 100644 charts/big-peer/templates/classes/_live_query_source.tpl diff --git a/charts/big-peer/Chart.yaml b/charts/big-peer/Chart.yaml index 9d5ad26..5f97f4f 100644 --- a/charts/big-peer/Chart.yaml +++ b/charts/big-peer/Chart.yaml @@ -94,3 +94,11 @@ dependencies: version: "0.1.0-002c59" tags: - live-query + - name: kafka + alias: live-query-kafka + version: "0.2.2" + repository: "oci://quay.io/ditto-external" + condition: live-query.enabled + tags: + - kafka + diff --git a/charts/big-peer/templates/_app.tpl b/charts/big-peer/templates/_app.tpl index 63b54b5..6da7b85 100644 --- a/charts/big-peer/templates/_app.tpl +++ b/charts/big-peer/templates/_app.tpl @@ -5,7 +5,13 @@ {{- $appValues := $app -}} {{- $_ := set $ "ObjectValues" (dict "app" $appValues) -}} - {{- include "common.classes.registryapp" $ }} +{{ include "common.classes.registryapp" $ }} + {{- if $app.liveQuery.enabled -}} + {{- $_ := set $ "ObjectValues" (dict "app" $appValues) }} +{{ include "common.classes.live_query_core" $ }} +{{ include "common.classes.live_query_source" $ }} +{{ include "common.classes.live_query_sink" $ }} + {{- end -}} {{- end }} {{- end }} {{- end }} diff --git a/charts/big-peer/templates/classes/_live_query_core.tpl b/charts/big-peer/templates/classes/_live_query_core.tpl new file mode 100644 index 0000000..bacaf99 --- /dev/null +++ b/charts/big-peer/templates/classes/_live_query_core.tpl @@ -0,0 +1,42 @@ +{{- define "common.classes.live_query_core" -}} + {{- $fullname := include "common.names.fullname" . -}} + {{- $appValues := .Values.app -}} + {{- if hasKey . "ObjectValues" -}} + {{- with .ObjectValues.app -}} + {{- $appValues = . -}} + {{- end -}} + {{ end -}} + + {{- $values := $appValues.liveQuery -}} + {{- if and (hasKey $appValues "nameOverride") $appValues.nameOverride -}} + {{- $app = $appValues.nameOverride -}} + {{- end -}} + + {{- $id := .Values.id -}} +--- +apiVersion: cloud.app.ditto.live/v1alpha2 +kind: LiveQueryCore +metadata: + name: {{ $appValues.id }} + {{- with (merge ($values.labels | default dict) (include "common.labels" $ | fromYaml)) }} + labels: {{- toYaml . | nindent 4 }} + {{- end }} + {{- with (merge ($values.annotations | default dict) (include "common.annotations" $ | fromYaml)) }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + appId: {{ $appValues.id }} + cdcKafkaClusterRef: + name: {{ .Release.Name }}-live-query-kafka + namespace: {{ .Release.Namespace }} + description: null + httpApiServerPoolRef: + name: {{ .Release.Name }}hydra-subscription-api + namespace: {{.Release.Namespace}} + hydraClusterRef: + name: {{ .Release.Name }}-hydra-store + namespace: {{ .Release.Namespace }} + subscriptionPoolRef: + name: {{ .Release.Name }}-hydra-subscription + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/big-peer/templates/classes/_live_query_sink.tpl b/charts/big-peer/templates/classes/_live_query_sink.tpl new file mode 100644 index 0000000..aa31bf1 --- /dev/null +++ b/charts/big-peer/templates/classes/_live_query_sink.tpl @@ -0,0 +1,41 @@ +{{- define "common.classes.live_query_sink" -}} + {{- $fullname := include "common.names.fullname" . -}} + {{- $id := .Values.id -}} + {{- $appValues := .Values.app -}} + {{- $values := .Values.app -}} + {{- if hasKey . "ObjectValues" -}} + {{- with .ObjectValues.app -}} + {{- $appValues = . -}} + {{- end -}} + {{ end -}} + {{- if and (hasKey $appValues "nameOverride") $appValues.nameOverride -}} + {{- $app = $appValues.nameOverride -}} + {{- end -}} +--- +apiVersion: cloud.app.ditto.live/v1alpha2 +kind: LiveQuerySink +metadata: + name: {{ $appValues.id }} + {{- with (merge ($values.labels | default dict) (include "common.labels" $ | fromYaml)) }} + labels: {{- toYaml . | nindent 4 }} + {{- end }} + {{- with (merge ($values.annotations | default dict) (include "common.annotations" $ | fromYaml)) }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + appId: {{ $appValues.id }} + description: null + destination: + kafkaConsumer: + cluster: + name: {{ .Release.Name }}-live-query-kafka + namespace: {{ .Release.Namespace }} + consumerGroup: user-consumable-{{ $appValues.id }} + topicName: user-consumable-{{ $appValues.id }} + liveQueryCoreRef: + name: {{ $appValues.id }} + namespace: {{ .Release.Namespace }} + liveQuerySourceRef: + name: {{ $appValues.id }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/big-peer/templates/classes/_live_query_source.tpl b/charts/big-peer/templates/classes/_live_query_source.tpl new file mode 100644 index 0000000..3698c50 --- /dev/null +++ b/charts/big-peer/templates/classes/_live_query_source.tpl @@ -0,0 +1,42 @@ +{{- define "common.classes.live_query_source" -}} + {{- $fullname := include "common.names.fullname" . -}} + {{- $id := .Values.id -}} + {{- $appValues := .Values.app -}} + {{- $values := .Values.app -}} + {{- if hasKey . "ObjectValues" -}} + {{- with .ObjectValues.app -}} + {{- $appValues = . -}} + {{- end -}} + {{ end -}} + + {{- $values := $appValues.liveQuery -}} + {{- if and (hasKey $appValues "nameOverride") $appValues.nameOverride -}} + {{- $app = $appValues.nameOverride -}} + {{- end -}} +--- +apiVersion: cloud.app.ditto.live/v1alpha2 +kind: LiveQuerySource +metadata: + name: {{ $appValues.id }} + {{- with (merge ($values.labels | default dict) (include "common.labels" $ | fromYaml)) }} + labels: {{- toYaml . | nindent 4 }} + {{- end }} + {{- with (merge ($values.annotations | default dict) (include "common.annotations" $ | fromYaml)) }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + appId: {{ $appValues.id }} + description: null + collection: null + liveQueryCoreRef: + name: {{ $appValues.id }} + namespace: {{ .Release.Namespace }} + output: + kafka: + cluster: + name: {{ .Release.Name }}-live-query-kafka + namespace: {{ .Release.Namespace }} + topicName: user-consumable-{{ $appValues.id }} + queryFilterExpression: {{ $values.queryFilterExpression | quote }} + schema: {{ $values.schema }} +{{- end }} diff --git a/charts/big-peer/templates/classes/_registry_app.tpl b/charts/big-peer/templates/classes/_registry_app.tpl index 5bca2c3..d15e912 100644 --- a/charts/big-peer/templates/classes/_registry_app.tpl +++ b/charts/big-peer/templates/classes/_registry_app.tpl @@ -11,6 +11,7 @@ {{- if and (hasKey $appValues "nameOverride") $appValues.nameOverride -}} {{- $app = $appValues.nameOverride -}} {{- end -}} + --- apiVersion: cloud.app.ditto.live/v1alpha2 kind: RegistryApp @@ -42,4 +43,4 @@ spec: subscriptionPoolRef: name: {{ .Release.Name }}-hydra-subscription namespace: {{ .Release.Namespace }} -{{- end -}} +{{- end }} diff --git a/charts/big-peer/templates/lib/chart/_labels.tpl b/charts/big-peer/templates/lib/chart/_labels.tpl index 73b5068..3b8e4b2 100644 --- a/charts/big-peer/templates/lib/chart/_labels.tpl +++ b/charts/big-peer/templates/lib/chart/_labels.tpl @@ -2,6 +2,11 @@ {{- define "common.labels" -}} helm.sh/chart: {{ include "common.names.chart" . }} {{ include "common.labels.selectorLabels" . }} + {{- if hasKey . "ObjectValues" }} + {{- with .ObjectValues.app }} +{{ include "common.labels.registryApp" . }} + {{ end }} + {{- end }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -20,3 +25,10 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/name: {{ include "common.names.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} + + +{{- define "common.labels.registryApp" -}} +cloud.app.ditto.live/app-id: {{ .id | quote }} +cloud.app.ditto.live/org-url: {{ .organizationUrl | default "not_used" | quote }} +cloud.app.ditto.live/app-slug: {{ .slug | default .id | quote }} +{{- end }} diff --git a/charts/big-peer/values.yaml b/charts/big-peer/values.yaml index 176eb9b..88bd543 100644 --- a/charts/big-peer/values.yaml +++ b/charts/big-peer/values.yaml @@ -48,10 +48,15 @@ global: affinity: {} apps: - my-app: + myApp: enabled: true id: 891c1bdb-71c6-40d5-9374-128e467c367e + liveQuery: + enabled: true + queryFilterExpression: "true" + schema: untyped + # global kafka cluster options kafka: @@ -774,6 +779,10 @@ dummy-auth-webhook: name: quay.io/ditto-external/big-peer-auth-webhook:1.39.2 pullPolicy: IfNotPresent + +###################################### +# Live Query +##################################### live-query: enabled: true @@ -784,13 +793,16 @@ live-query: env: # Main config INSTANCE_NAME: "live-query-resource-controller" - LIVE_QUERY_WATCHER_NAMESPACE: "" + LIVE_QUERY_WATCHER_NAMESPACE: "bp" TARGET_NAMESPACE_TEMPLATE: "bp" - APP_LABEL_SELECTOR: "cloud.app.ditto.live/cluster-ns=bp" + APP_LABEL_SELECTOR: "app.kubernetes.io/instance=ditto-bp" + DEPLOY_NETWORK_POLICIES: false + DEPLOY_SERVICE_MONITORS: false # Kafka user ingress config KAFKA_INGRESS_DOMAIN: "" KAFKA_INGRESS_CLASS: "" + KAFKA_REPLICATION_FACTOR: "1" # CDC workload config CDC_RESOURCE_REQUEST_CPU: 40m @@ -798,26 +810,47 @@ live-query: CDC_RESOURCE_LIMIT_MEMORY: 4G CDC_IMAGE: quay.io/ditto-external/hydra-cdc:1.39.2 CDC_TOKEN_AUDIENCE: cloud.ditto.live - CDC_KAFKA_REPLICATION_FACTOR: 3 + CDC_KAFKA_REPLICATION_FACTOR: 1 # StreamSplitter workload config STREAM_SPLITTER_RESOURCE_REQUEST_CPU: 20m STREAM_SPLITTER_RESOURCE_REQUEST_MEMORY: 256Mi STREAM_SPLITTER_RESOURCE_LIMIT_MEMORY: 1G - STREAM_SPLITTER_IMAGE: quay.io/ditto-external/hydra-stream-splitter:latest-46073-c0fe1e + STREAM_SPLITTER_IMAGE: quay.io/ditto-external/hydra-stream-splitter:1.39.2 STREAM_SPLITTER_TOKEN_AUDIENCE: cloud.ditto.live # WebhookDispatcher workload config WEBHOOK_DISPATCHER_RESOURCE_REQUEST_CPU: 20m WEBHOOK_DISPATCHER_RESOURCE_REQUEST_MEMORY: 24Mi WEBHOOK_DISPATCHER_RESOURCE_LIMIT_MEMORY: 128Mi - WEBHOOK_DISPATCHER_IMAGE: quay.io/ditto-external/hydra-webhook-dispatcher:latest-46073-c0fe1e + WEBHOOK_DISPATCHER_IMAGE: quay.io/ditto-external/hydra-webhook-dispatcher:1.39.2 WEBHOOK_DISPATCHER_TOKEN_AUDIENCE: cloud.ditto.live # CDC Hearbeat workload config CDC_HEARTBEAT_RESOURCE_REQUEST_CPU: 20m CDC_HEARTBEAT_RESOURCE_REQUEST_MEMORY: 50Mi CDC_HEARTBEAT_RESOURCE_LIMIT_MEMORY: 150Mi - CDC_HEARTBEAT_IMAGE: quay.io/ditto-external/cdc-heartbeat:latest-46073-c0fe1e + CDC_HEARTBEAT_IMAGE: quay.io/ditto-external/cdc-heartbeat:1.39.2 CDC_HEARTBEAT_TOKEN_AUDIENCE: cloud.ditto.live - CDC_HEARTBEAT_KAFKA_REPLICATION_FACTOR: 3 + CDC_HEARTBEAT_KAFKA_REPLICATION_FACTOR: 1 + +live-query-kafka: + # enable kraft mode for dual use Kafka nodes in a common node pool + kraft: + enabled: false + nodePool: + dualRoles: true + replicas: 1 + + # disable kafka supporting services that are not required + strimzi: + replicas: 3 + authorization: simple + external: true + + #interBrokerProtocolVersion: 3.8.0 + kafkaVersion: 3.8.0 + kafkaExporter: + enabled: false + cruiseControl: + enabled: false From 8c18d40f5dddc25510b47d0c1675ff70b8d99ebf Mon Sep 17 00:00:00 2001 From: ChrisMcKenzie Date: Thu, 21 Nov 2024 12:22:13 -0600 Subject: [PATCH 5/7] add: support for multiple livquery sinks --- charts/big-peer/templates/_app.tpl | 3 +- .../templates/classes/_live_query_sink.tpl | 36 +++++++++++++++++++ .../big-peer/templates/lib/chart/_labels.tpl | 1 - charts/big-peer/values.yaml | 8 ++++- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/charts/big-peer/templates/_app.tpl b/charts/big-peer/templates/_app.tpl index 6da7b85..d8fa542 100644 --- a/charts/big-peer/templates/_app.tpl +++ b/charts/big-peer/templates/_app.tpl @@ -7,10 +7,9 @@ {{- $_ := set $ "ObjectValues" (dict "app" $appValues) -}} {{ include "common.classes.registryapp" $ }} {{- if $app.liveQuery.enabled -}} - {{- $_ := set $ "ObjectValues" (dict "app" $appValues) }} {{ include "common.classes.live_query_core" $ }} {{ include "common.classes.live_query_source" $ }} -{{ include "common.classes.live_query_sink" $ }} +{{ include "common.classes.live_query_sinks" $ }} {{- end -}} {{- end }} {{- end }} diff --git a/charts/big-peer/templates/classes/_live_query_sink.tpl b/charts/big-peer/templates/classes/_live_query_sink.tpl index aa31bf1..1611931 100644 --- a/charts/big-peer/templates/classes/_live_query_sink.tpl +++ b/charts/big-peer/templates/classes/_live_query_sink.tpl @@ -3,11 +3,21 @@ {{- $id := .Values.id -}} {{- $appValues := .Values.app -}} {{- $values := .Values.app -}} + {{- $dest := dict "enabled" false -}} + {{- $dest_type := dict "destination_type" "" -}} {{- if hasKey . "ObjectValues" -}} {{- with .ObjectValues.app -}} {{- $appValues = . -}} {{- end -}} + + {{- with .ObjectValues.sink -}} + {{- $dest = . -}} + {{- end -}} + {{- with .ObjectValues.sink_type -}} + {{- $dest_type = . -}} + {{- end -}} {{ end -}} + {{- if and (hasKey $appValues "nameOverride") $appValues.nameOverride -}} {{- $app = $appValues.nameOverride -}} {{- end -}} @@ -26,12 +36,19 @@ spec: appId: {{ $appValues.id }} description: null destination: + {{- if eq $dest_type "kafka" }} kafkaConsumer: cluster: name: {{ .Release.Name }}-live-query-kafka namespace: {{ .Release.Namespace }} consumerGroup: user-consumable-{{ $appValues.id }} topicName: user-consumable-{{ $appValues.id }} + {{- end }} + {{- if eq $dest_type "webhook" }} + webhook: + sourceTopicName: {{ $appValues.id }} + url: {{ $dest.url | quote }} + {{- end }} liveQueryCoreRef: name: {{ $appValues.id }} namespace: {{ .Release.Namespace }} @@ -39,3 +56,22 @@ spec: name: {{ $appValues.id }} namespace: {{ .Release.Namespace }} {{- end }} + +{{- define "common.classes.live_query_sinks"}} + {{- $appValues := .Values.app -}} + {{- if hasKey . "ObjectValues" }} + {{- with .ObjectValues.app }} + {{- $appValues = . }} + {{- end }} + {{- end }} + + + {{- range $type, $sink := $appValues.liveQuery.sinks }} + {{- $ObjectValues := dict "app" $appValues "sink" $sink "sink_type" $type }} + + {{- $_ := set $ "ObjectValues" $ObjectValues }} + {{- if and $sink.enabled }} +{{ include "common.classes.live_query_sink" $ }} + {{- end }} + {{- end }} +{{- end}} diff --git a/charts/big-peer/templates/lib/chart/_labels.tpl b/charts/big-peer/templates/lib/chart/_labels.tpl index 3b8e4b2..7c74e8e 100644 --- a/charts/big-peer/templates/lib/chart/_labels.tpl +++ b/charts/big-peer/templates/lib/chart/_labels.tpl @@ -26,7 +26,6 @@ app.kubernetes.io/name: {{ include "common.names.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} - {{- define "common.labels.registryApp" -}} cloud.app.ditto.live/app-id: {{ .id | quote }} cloud.app.ditto.live/org-url: {{ .organizationUrl | default "not_used" | quote }} diff --git a/charts/big-peer/values.yaml b/charts/big-peer/values.yaml index 88bd543..07b00ea 100644 --- a/charts/big-peer/values.yaml +++ b/charts/big-peer/values.yaml @@ -50,12 +50,18 @@ global: apps: myApp: enabled: true - id: 891c1bdb-71c6-40d5-9374-128e467c367e + id: 891c1bdb-71c6-40d5-9374-128e467c367e # uuid generated via uuidgen liveQuery: enabled: true queryFilterExpression: "true" schema: untyped + sinks: + webhook: + enabled: true + url: "http://myapp.com/rimshot" + kafka: + enabled: true # global kafka cluster options From 809246d8f7ccfa13b4309bb270b5793aa09f7a4c Mon Sep 17 00:00:00 2001 From: ChrisMcKenzie Date: Thu, 21 Nov 2024 15:39:47 -0600 Subject: [PATCH 6/7] add: final rendition of app layout for live-query --- charts/big-peer/templates/_app.tpl | 5 ++++ .../templates/classes/_live_query_core.tpl | 4 +-- .../templates/classes/_live_query_sink.tpl | 22 +++++++++++---- .../templates/classes/_live_query_source.tpl | 19 +++++++++---- .../big-peer/templates/lib/chart/_topics.tpl | 5 ++++ charts/big-peer/values.yaml | 28 +++++++++++++------ 6 files changed, 62 insertions(+), 21 deletions(-) create mode 100644 charts/big-peer/templates/lib/chart/_topics.tpl diff --git a/charts/big-peer/templates/_app.tpl b/charts/big-peer/templates/_app.tpl index d8fa542..15021f4 100644 --- a/charts/big-peer/templates/_app.tpl +++ b/charts/big-peer/templates/_app.tpl @@ -8,8 +8,13 @@ {{ include "common.classes.registryapp" $ }} {{- if $app.liveQuery.enabled -}} {{ include "common.classes.live_query_core" $ }} +{{- range $query := $app.liveQuery.queries }} + {{- if $query.enabled }} + {{- $_ := set $ "ObjectValues" (dict "app" $appValues "query" $query "source_name" (include "common.topics.name" (dict "id" $appValues.id "queryName" $query.name))) -}} {{ include "common.classes.live_query_source" $ }} {{ include "common.classes.live_query_sinks" $ }} + {{- end }} +{{- end }} {{- end -}} {{- end }} {{- end }} diff --git a/charts/big-peer/templates/classes/_live_query_core.tpl b/charts/big-peer/templates/classes/_live_query_core.tpl index bacaf99..f4230be 100644 --- a/charts/big-peer/templates/classes/_live_query_core.tpl +++ b/charts/big-peer/templates/classes/_live_query_core.tpl @@ -10,9 +10,9 @@ {{- $values := $appValues.liveQuery -}} {{- if and (hasKey $appValues "nameOverride") $appValues.nameOverride -}} {{- $app = $appValues.nameOverride -}} - {{- end -}} + {{ end -}} - {{- $id := .Values.id -}} + {{ $id := .Values.id }} --- apiVersion: cloud.app.ditto.live/v1alpha2 kind: LiveQueryCore diff --git a/charts/big-peer/templates/classes/_live_query_sink.tpl b/charts/big-peer/templates/classes/_live_query_sink.tpl index 1611931..4fbc027 100644 --- a/charts/big-peer/templates/classes/_live_query_sink.tpl +++ b/charts/big-peer/templates/classes/_live_query_sink.tpl @@ -3,13 +3,19 @@ {{- $id := .Values.id -}} {{- $appValues := .Values.app -}} {{- $values := .Values.app -}} + {{- $queryValues := .Values.query -}} {{- $dest := dict "enabled" false -}} {{- $dest_type := dict "destination_type" "" -}} + {{- $source_name := .ObjectValues.source_name -}} {{- if hasKey . "ObjectValues" -}} {{- with .ObjectValues.app -}} {{- $appValues = . -}} {{- end -}} + {{- with .ObjectValues.query -}} + {{- $queryValues = . -}} + {{- end -}} + {{- with .ObjectValues.sink -}} {{- $dest = . -}} {{- end -}} @@ -20,12 +26,12 @@ {{- if and (hasKey $appValues "nameOverride") $appValues.nameOverride -}} {{- $app = $appValues.nameOverride -}} - {{- end -}} + {{ end -}} --- apiVersion: cloud.app.ditto.live/v1alpha2 kind: LiveQuerySink metadata: - name: {{ $appValues.id }} + name: {{ uuidv4 }} {{- with (merge ($values.labels | default dict) (include "common.labels" $ | fromYaml)) }} labels: {{- toYaml . | nindent 4 }} {{- end }} @@ -46,28 +52,32 @@ spec: {{- end }} {{- if eq $dest_type "webhook" }} webhook: - sourceTopicName: {{ $appValues.id }} + sourceTopicName: {{ $source_name }} url: {{ $dest.url | quote }} {{- end }} liveQueryCoreRef: name: {{ $appValues.id }} namespace: {{ .Release.Namespace }} liveQuerySourceRef: - name: {{ $appValues.id }} + name: {{ $source_name }} namespace: {{ .Release.Namespace }} {{- end }} {{- define "common.classes.live_query_sinks"}} {{- $appValues := .Values.app -}} + {{- $queryValues := .Values.query -}} {{- if hasKey . "ObjectValues" }} {{- with .ObjectValues.app }} {{- $appValues = . }} {{- end }} + {{- with .ObjectValues.query }} + {{- $queryValues = . }} + {{- end }} {{- end }} - {{- range $type, $sink := $appValues.liveQuery.sinks }} - {{- $ObjectValues := dict "app" $appValues "sink" $sink "sink_type" $type }} + {{- range $type, $sink := $queryValues.sinks }} + {{- $ObjectValues := dict "app" $appValues "sink" $sink "sink_type" $type "query" $queryValues "source_name" $.ObjectValues.source_name }} {{- $_ := set $ "ObjectValues" $ObjectValues }} {{- if and $sink.enabled }} diff --git a/charts/big-peer/templates/classes/_live_query_source.tpl b/charts/big-peer/templates/classes/_live_query_source.tpl index 3698c50..06e065c 100644 --- a/charts/big-peer/templates/classes/_live_query_source.tpl +++ b/charts/big-peer/templates/classes/_live_query_source.tpl @@ -3,21 +3,30 @@ {{- $id := .Values.id -}} {{- $appValues := .Values.app -}} {{- $values := .Values.app -}} + {{- $source_name := .ObjectValues.source_name -}} {{- if hasKey . "ObjectValues" -}} {{- with .ObjectValues.app -}} {{- $appValues = . -}} {{- end -}} {{ end -}} +{{- $queryValues := .Values.query -}} +{{- if hasKey . "ObjectValues" -}} + {{- with .ObjectValues.query -}} + {{- $queryValues = . -}} + {{- end -}} +{{ end -}} + + {{- $values := $appValues.liveQuery -}} {{- if and (hasKey $appValues "nameOverride") $appValues.nameOverride -}} {{- $app = $appValues.nameOverride -}} - {{- end -}} + {{ end }} --- apiVersion: cloud.app.ditto.live/v1alpha2 kind: LiveQuerySource metadata: - name: {{ $appValues.id }} + name: {{ $source_name }} {{- with (merge ($values.labels | default dict) (include "common.labels" $ | fromYaml)) }} labels: {{- toYaml . | nindent 4 }} {{- end }} @@ -36,7 +45,7 @@ spec: cluster: name: {{ .Release.Name }}-live-query-kafka namespace: {{ .Release.Namespace }} - topicName: user-consumable-{{ $appValues.id }} - queryFilterExpression: {{ $values.queryFilterExpression | quote }} - schema: {{ $values.schema }} + topicName: {{ include "common.topics.name" (dict "id" $appValues.id "queryName" $queryValues.name) }} + queryFilterExpression: {{ $queryValues.queryFilterExpression | quote }} + schema: {{ $queryValues.schema }} {{- end }} diff --git a/charts/big-peer/templates/lib/chart/_topics.tpl b/charts/big-peer/templates/lib/chart/_topics.tpl new file mode 100644 index 0000000..e103454 --- /dev/null +++ b/charts/big-peer/templates/lib/chart/_topics.tpl @@ -0,0 +1,5 @@ +{{- define "common.topics.name" -}} + {{- printf "%s-%s" .id .queryName | trunc 63 | replace "_" "-" | trimSuffix "-" -}} +{{- end -}} + + diff --git a/charts/big-peer/values.yaml b/charts/big-peer/values.yaml index 07b00ea..94d19b0 100644 --- a/charts/big-peer/values.yaml +++ b/charts/big-peer/values.yaml @@ -49,19 +49,31 @@ global: apps: myApp: - enabled: true + enabled: false id: 891c1bdb-71c6-40d5-9374-128e467c367e # uuid generated via uuidgen liveQuery: - enabled: true - queryFilterExpression: "true" - schema: untyped - sinks: - webhook: + enabled: false + queries: + - name: "all_the_things" enabled: true - url: "http://myapp.com/rimshot" - kafka: + queryFilterExpression: "true" + schema: untyped + sinks: + kafka: + enabled: true + webhook: + enabled: true + url: "http://myapp.com/rimshot" + - name: "send_to_hook" enabled: true + queryFilterExpression: "true" + schema: untyped + sinks: + webhook: + enabled: true + url: "http://myapp.com/rimshot" + # global kafka cluster options From caf65af91d2855f04a6491f6b96f4c81c4b877be Mon Sep 17 00:00:00 2001 From: ChrisMcKenzie Date: Thu, 21 Nov 2024 15:50:31 -0600 Subject: [PATCH 7/7] chore(version): bump big-peer to 0.2.9 --- charts/big-peer/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/big-peer/Chart.yaml b/charts/big-peer/Chart.yaml index 5f97f4f..cb4c335 100644 --- a/charts/big-peer/Chart.yaml +++ b/charts/big-peer/Chart.yaml @@ -15,7 +15,7 @@ home: https://docs.ditto.live/ # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.9 +version: 0.2.9 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to