diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 11d1fbb6f6..699da3a78a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,6 +54,16 @@ jobs: run: | bazel test --test_tag_filters=-lint //... + - name: Cleanup space + run: | + df -h + sudo apt-get autoremove -y + sudo apt-get clean + docker images prune -a + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /opt/hostedtoolcache + df -h + - name: Build all artifacts run: | bazel build //... @@ -64,11 +74,6 @@ jobs: echo ${{ secrets.PAT }} | docker login ghcr.io -u airydevci --password-stdin ./scripts/push-images.sh - - name: Install aws cli - uses: chrislennon/action-aws-cli@v1.1 - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' - - name: Upload airy binary to S3 if: ${{ github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release') || github.ref == 'refs/heads/main' }} run: | @@ -76,6 +81,7 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} GITHUB_BRANCH: ${{ github.ref }} - name: Publish helm charts diff --git a/README.md b/README.md index 92d5b4b5b6..2d80c67605 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ --- -![Airy_Explainer_Highlevel_Readme](https://airy.co/docs/core/img/getting-started/introduction-light.png) +![Airy_Explainer_Highlevel_Readme](https://airy.co/docs/core/img/getting-started/introduction.png) Airy Core is an is an open-source streaming app framework to train ML models and supply them with historical and real-time data. With Airy you can process data from a variety of sources: diff --git a/VERSION b/VERSION index 524456c776..316ba4bd9e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.54.0 +0.55.0 diff --git a/backend/components/chat-plugin/helm/templates/backend/deployment.yaml b/backend/components/chat-plugin/helm/templates/backend/deployment.yaml index 1f8995b817..a82f25fd2f 100644 --- a/backend/components/chat-plugin/helm/templates/backend/deployment.yaml +++ b/backend/components/chat-plugin/helm/templates/backend/deployment.yaml @@ -51,6 +51,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.backend.resources | indent 10 }} initContainers: @@ -68,3 +73,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/contacts/helm/templates/deployment.yaml b/backend/components/contacts/helm/templates/deployment.yaml index 5e4c6846a8..b0d2e68d9b 100644 --- a/backend/components/contacts/helm/templates/deployment.yaml +++ b/backend/components/contacts/helm/templates/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.resources | indent 10 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/facebook/helm/templates/deployments.yaml b/backend/components/facebook/helm/templates/deployments.yaml index 4d72ef5174..a530a8122f 100644 --- a/backend/components/facebook/helm/templates/deployments.yaml +++ b/backend/components/facebook/helm/templates/deployments.yaml @@ -59,6 +59,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.connector.resources | indent 12 }} initContainers: @@ -76,6 +81,11 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} --- apiVersion: apps/v1 kind: Deployment @@ -124,6 +134,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.eventsRouter.resources | indent 10 }} initContainers: @@ -141,3 +156,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/flink-connector/Dockerfile.result-sender b/backend/components/flink-connector/Dockerfile.result-sender new file mode 100644 index 0000000000..6580476fad --- /dev/null +++ b/backend/components/flink-connector/Dockerfile.result-sender @@ -0,0 +1,16 @@ +FROM golang:1.17 + +WORKDIR /app + +COPY ./src/types.go ./src/tools.go ./src/result-sender.go ./ + +RUN go mod init main && \ + go get github.com/confluentinc/confluent-kafka-go/v2/kafka && \ + go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry && \ + go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde && \ + go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/avro && \ + go get golang.org/x/net + +RUN go build -o app + +CMD ["./app"] diff --git a/backend/components/flink-connector/Dockerfile.statements-executor b/backend/components/flink-connector/Dockerfile.statements-executor new file mode 100644 index 0000000000..3280b144f7 --- /dev/null +++ b/backend/components/flink-connector/Dockerfile.statements-executor @@ -0,0 +1,16 @@ +FROM golang:1.17 + +WORKDIR /app + +COPY ./src/types.go ./src/tools.go ./src/statements-executor.go ./ + +RUN go mod init main && \ + go get github.com/confluentinc/confluent-kafka-go/v2/kafka && \ + go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry && \ + go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde && \ + go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/avro && \ + go get golang.org/x/net + +RUN go build -o app + +CMD ["./app"] diff --git a/backend/components/flink-connector/Makefile b/backend/components/flink-connector/Makefile new file mode 100644 index 0000000000..1e7dcec8d0 --- /dev/null +++ b/backend/components/flink-connector/Makefile @@ -0,0 +1,13 @@ +build-statements-executor: + docker build -t flink-connector/statements-executor -f Dockerfile.statements-executor . + +release-statements-executor: build-statements-executor + docker tag flink-connector/statements-executor ghcr.io/airyhq/connectors/flink/statements-executor:release + docker push ghcr.io/airyhq/connectors/flink/statements-executor:release + +build-result-sender: + docker build -t flink-connector/result-sender -f Dockerfile.result-sender . + +release-result-sender: build-result-sender + docker tag flink-connector/result-sender ghcr.io/airyhq/connectors/flink/result-sender:release + docker push ghcr.io/airyhq/connectors/flink/result-sender:release diff --git a/backend/components/flink-connector/helm/BUILD b/backend/components/flink-connector/helm/BUILD new file mode 100644 index 0000000000..45805d5f1c --- /dev/null +++ b/backend/components/flink-connector/helm/BUILD @@ -0,0 +1,3 @@ +load("//tools/build:helm.bzl", "helm_ruleset_core_version") + +helm_ruleset_core_version() diff --git a/backend/components/flink-connector/helm/Chart.yaml b/backend/components/flink-connector/helm/Chart.yaml new file mode 100644 index 0000000000..4eb993fb25 --- /dev/null +++ b/backend/components/flink-connector/helm/Chart.yaml @@ -0,0 +1,6 @@ + +apiVersion: v2 +appVersion: "1.0" +description: Flink connector +name: flink-connector +version: 1.0 \ No newline at end of file diff --git a/backend/components/flink-connector/helm/templates/configmap.yaml b/backend/components/flink-connector/helm/templates/configmap.yaml new file mode 100644 index 0000000000..d8301a65d1 --- /dev/null +++ b/backend/components/flink-connector/helm/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.component }} + labels: + core.airy.co/managed: "true" + core.airy.co/mandatory: "{{ .Values.mandatory }}" + core.airy.co/component: "{{ .Values.component }}" + annotations: + core.airy.co/enabled: "{{ .Values.enabled }}" \ No newline at end of file diff --git a/backend/components/flink-connector/helm/templates/result-sender/deployment.yaml b/backend/components/flink-connector/helm/templates/result-sender/deployment.yaml new file mode 100644 index 0000000000..0f50fc5540 --- /dev/null +++ b/backend/components/flink-connector/helm/templates/result-sender/deployment.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.component }}-{{ .Values.resultSender.name }} + labels: + app: {{ .Values.component }} + core.airy.co/managed: "true" + core.airy.co/mandatory: "{{ .Values.mandatory }}" + core.airy.co/component: {{ .Values.component }} +spec: + replicas: {{ if .Values.enabled }} 1 {{ else }} 0 {{ end }} + selector: + matchLabels: + app: {{ .Values.component }}-{{ .Values.resultSender.name }} + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: {{ .Values.component }}-{{ .Values.resultSender.name }} + spec: + containers: + - name: app + image: "ghcr.io/airyhq/{{ .Values.resultSender.image }}:release" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: security + - configMapRef: + name: kafka-config + - configMapRef: + name: {{ .Values.component }} + env: + - name: KAFKA_TOPIC_NAME + value: {{ .Values.resultSender.topic }} + - name: API_COMMUNICATION_URL + value: {{ .Values.apiCommunicationUrl }} + livenessProbe: + httpGet: + path: /actuator/health + port: {{ .Values.port }} + httpHeaders: + - name: Health-Check + value: health-check + initialDelaySeconds: 43200 + periodSeconds: 10 + failureThreshold: 3 \ No newline at end of file diff --git a/backend/components/flink-connector/helm/templates/result-sender/service.yaml b/backend/components/flink-connector/helm/templates/result-sender/service.yaml new file mode 100644 index 0000000000..cdf73d72ba --- /dev/null +++ b/backend/components/flink-connector/helm/templates/result-sender/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.component }}-{{ .Values.resultSender.name }} + labels: + app: {{ .Values.component }}-{{ .Values.resultSender.name }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: {{ .Values.component }}-{{ .Values.resultSender.name }} + port: 80 + targetPort: {{ .Values.port }} + selector: + app: {{ .Values.component }}-{{ .Values.resultSender.name }} \ No newline at end of file diff --git a/backend/components/flink-connector/helm/templates/statements-executor/deployment.yaml b/backend/components/flink-connector/helm/templates/statements-executor/deployment.yaml new file mode 100644 index 0000000000..44c7b6e59c --- /dev/null +++ b/backend/components/flink-connector/helm/templates/statements-executor/deployment.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.component }}-{{ .Values.executor.name }} + labels: + app: {{ .Values.component }} + core.airy.co/managed: "true" + core.airy.co/mandatory: "{{ .Values.mandatory }}" + core.airy.co/component: {{ .Values.component }} +spec: + replicas: {{ if .Values.enabled }} 1 {{ else }} 0 {{ end }} + selector: + matchLabels: + app: {{ .Values.component }}-{{ .Values.executor.name }} + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: {{ .Values.component }}-{{ .Values.executor.name }} + spec: + containers: + - name: app + image: "ghcr.io/airyhq/{{ .Values.executor.image }}:release" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: security + - configMapRef: + name: kafka-config + - configMapRef: + name: {{ .Values.component }} + env: + - name: KAFKA_TOPIC_NAME + value: {{ .Values.executor.topic }} + - name: FLINK_GATEWAY_URL + value: {{ .Values.gatewayUrl }} + livenessProbe: + httpGet: + path: /actuator/health + port: {{ .Values.port }} + httpHeaders: + - name: Health-Check + value: health-check + initialDelaySeconds: 43200 + periodSeconds: 10 + failureThreshold: 3 \ No newline at end of file diff --git a/backend/components/flink-connector/helm/templates/statements-executor/service.yaml b/backend/components/flink-connector/helm/templates/statements-executor/service.yaml new file mode 100644 index 0000000000..3e5fbfc30c --- /dev/null +++ b/backend/components/flink-connector/helm/templates/statements-executor/service.yaml @@ -0,0 +1,16 @@ + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.component }}-{{ .Values.executor.name }} + labels: + app: {{ .Values.component }}-{{ .Values.executor.name }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: {{ .Values.component }}-{{ .Values.executor.name }} + port: 80 + targetPort: {{ .Values.port }} + selector: + app: {{ .Values.component }}-{{ .Values.executor.name }} \ No newline at end of file diff --git a/backend/components/flink-connector/helm/values.yaml b/backend/components/flink-connector/helm/values.yaml new file mode 100644 index 0000000000..71f4475a38 --- /dev/null +++ b/backend/components/flink-connector/helm/values.yaml @@ -0,0 +1,16 @@ + +component: flink-connector +mandatory: false +enabled: false +port: 8080 +resources: +gatewayUrl: "http://flink-jobmanager:8083" +apiCommunicationUrl: "http://api-communication/messages.send" +executor: + name: statements-executor + image: connectors/flink/statements-executor + topic: flink.statements +resultSender: + name: result-sender + image: connectors/flink/result-sender + topic: flink.output \ No newline at end of file diff --git a/backend/components/flink-connector/src/result-sender.go b/backend/components/flink-connector/src/result-sender.go new file mode 100644 index 0000000000..90bfff7381 --- /dev/null +++ b/backend/components/flink-connector/src/result-sender.go @@ -0,0 +1,158 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/confluentinc/confluent-kafka-go/v2/kafka" +) + +func main() { + + // Create Kafka consumer to read the statements + kafkaURL := os.Getenv("KAFKA_BROKERS") + schemaRegistryURL := os.Getenv("KAFKA_SCHEMA_REGISTRY_URL") + topicName := os.Getenv("KAFKA_TOPIC_NAME") + systemToken := os.Getenv("systemToken") + authUsername := os.Getenv("AUTH_JAAS_USERNAME") + authPassword := os.Getenv("AUTH_JAAS_PASSWORD") + flinkProvider := os.Getenv("provider") + groupID := "result-sender" + msgNormal := false + msgDebug := true + + if kafkaURL == "" || schemaRegistryURL == "" || topicName == "" { + fmt.Println("KAFKA_BROKERS, KAFKA_SCHEMA_REGISTRY_URL, and KAFKA_TOPIC_NAME environment variables must be set") + return + } + + var confluentConnection ConfluentConnection + confluentConnection.Token = os.Getenv("confluentToken") + confluentConnection.ComputePoolID = os.Getenv("confluentComputePoolID") + confluentConnection.Principal = os.Getenv("confluentPrincipal") + confluentConnection.SQLCurrentCatalog = os.Getenv("confluentSQLCurrentCatalog") + confluentConnection.SQLCurrentDatabase = os.Getenv("confluentSQLCurrentDatabase") + + // Healthcheck + http.HandleFunc("/actuator/health", func(w http.ResponseWriter, r *http.Request) { + response := map[string]string{"status": "UP"} + jsonResponse, err := json.Marshal(response) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write(jsonResponse) + }) + + go func() { + if err := http.ListenAndServe(":80", nil); err != nil { + panic(err) + } + }() + + fmt.Println("Health-check started") + + // Create Kafka consumer configuration + fmt.Println("Creating Kafka consumer for topic: ", topicName) + + c, err := kafka.NewConsumer(&kafka.ConfigMap{ + "bootstrap.servers": kafkaURL, + "group.id": groupID, + "auto.offset.reset": "earliest", + "security.protocol": "SASL_SSL", + "sasl.mechanisms": "PLAIN", + "sasl.username": authUsername, + "sasl.password": authPassword, + }) + if err != nil { + fmt.Printf("Error creating consumer: %v\n", err) + return + } + c.SubscribeTopics([]string{topicName}, nil) + // Channel for signals + signals := make(chan os.Signal, 1) + done := make(chan bool, 1) + + signal.Notify(signals, os.Interrupt, syscall.SIGTERM) + + go func() { + for { + select { + case sig := <-signals: + // If an interrupt signal is received, break the loop + fmt.Printf("Caught signal %v: terminating\n", sig) + done <- true + return + default: + msg, err := c.ReadMessage(-1) + if err == nil { + var flinkOutput FlinkOutput + if err := json.Unmarshal(msg.Value, &flinkOutput); err != nil { + fmt.Printf("Error unmarshalling message: %v\n", err) + continue + } else { + fmt.Printf("Received message: %+v\n", flinkOutput) + + flinkGatewayURL := os.Getenv("FLINK_GATEWAY_URL") + confluentGatewayURL := os.Getenv("CONFLUENT_GATEWAY_URL") + + var result FlinkResult + var headerConfluent []string + var resultConfluent string + + if flinkProvider == "flink" { + fmt.Println("Flink gateway: ", flinkGatewayURL) + result, err = getFlinkResult(flinkGatewayURL, flinkOutput.SessionID) + headerConfluent = []string{} + } else { + fmt.Println("Flink gateway: ", confluentGatewayURL) + fmt.Println("Waiting 20 seconds...") + time.Sleep(20 * time.Second) + headerConfluent, resultConfluent, err = getFlinkResultConfluent(confluentGatewayURL, flinkOutput.SessionID, confluentConnection) + } + if err != nil { + fmt.Println("Unable to get Flink result:", err) + sendMessage("Error: "+err.Error(), flinkOutput.ConversationID, systemToken, msgDebug) + return + } + if flinkProvider == "flink" { + sendMessage("Result retrieved from Flink: "+fmt.Sprintf("%#v", result), flinkOutput.ConversationID, systemToken, msgDebug) + sendMessage("Now converting the result to Markdown", flinkOutput.ConversationID, systemToken, msgDebug) + response, err := convertResultToMarkdown(result) + if err != nil { + fmt.Println("Unable to generate Markdown from result:", err) + sendMessage("Error: "+err.Error(), flinkOutput.ConversationID, systemToken, msgDebug) + sendMessage("I'm sorry, I am unable to fetch the results from the Flink table.", flinkOutput.ConversationID, systemToken, msgNormal) + return + } + sendMessage(response, flinkOutput.ConversationID, systemToken, msgNormal) + } else { + sendMessage("Result retrieved from Flink: "+fmt.Sprintf("%#v", resultConfluent), flinkOutput.ConversationID, systemToken, msgDebug) + sendMessage("Now converting the result to Markdown", flinkOutput.ConversationID, systemToken, msgDebug) + response, err := convertConfluentResultToMarkdown(headerConfluent, resultConfluent) + if err != nil { + fmt.Println("Unable to generate Markdown from result:", err) + sendMessage("Error: "+err.Error(), flinkOutput.ConversationID, systemToken, msgDebug) + sendMessage("I'm sorry, I am unable to fetch the results from the Flink table.", flinkOutput.ConversationID, systemToken, msgNormal) + return + } + sendMessage(response, flinkOutput.ConversationID, systemToken, msgNormal) + } + } + } else { + fmt.Printf("Consumer error: %v\n", err) + } + } + } + }() + <-done + c.Close() + fmt.Println("Consumer closed") +} diff --git a/backend/components/flink-connector/src/statements-executor.go b/backend/components/flink-connector/src/statements-executor.go new file mode 100644 index 0000000000..6ccf2e91be --- /dev/null +++ b/backend/components/flink-connector/src/statements-executor.go @@ -0,0 +1,140 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + + "github.com/confluentinc/confluent-kafka-go/v2/kafka" +) + +func main() { + + kafkaURL := os.Getenv("KAFKA_BROKERS") + schemaRegistryURL := os.Getenv("KAFKA_SCHEMA_REGISTRY_URL") + topicName := os.Getenv("KAFKA_TOPIC_NAME") + systemToken := os.Getenv("systemToken") + authUsername := os.Getenv("AUTH_JAAS_USERNAME") + authPassword := os.Getenv("AUTH_JAAS_PASSWORD") + flinkProvider := os.Getenv("provider") + groupID := "statement-executor-" + msgNormal := false + msgDebug := true + + if kafkaURL == "" || schemaRegistryURL == "" || topicName == "" { + fmt.Println("KAFKA_BROKERS, KAFKA_SCHEMA_REGISTRY_URL, and KAFKA_TOPIC_NAME environment variables must be set") + return + } + + var confluentConnection ConfluentConnection + confluentConnection.Token = os.Getenv("confluentToken") + confluentConnection.ComputePoolID = os.Getenv("confluentComputePoolID") + confluentConnection.Principal = os.Getenv("confluentPrincipal") + confluentConnection.SQLCurrentCatalog = os.Getenv("confluentSQLCurrentCatalog") + confluentConnection.SQLCurrentDatabase = os.Getenv("confluentSQLCurrentDatabase") + + // Healthcheck + http.HandleFunc("/actuator/health", func(w http.ResponseWriter, r *http.Request) { + response := map[string]string{"status": "UP"} + jsonResponse, err := json.Marshal(response) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write(jsonResponse) + }) + + go func() { + if err := http.ListenAndServe(":80", nil); err != nil { + panic(err) + } + }() + + fmt.Println("Health-check started") + + // Create Kafka consumer configuration + fmt.Println("Creating Kafka consumer for topic: ", topicName) + + c, err := kafka.NewConsumer(&kafka.ConfigMap{ + "bootstrap.servers": kafkaURL, + "group.id": groupID, + "auto.offset.reset": "earliest", + "security.protocol": "SASL_SSL", + "sasl.mechanisms": "PLAIN", + "sasl.username": authUsername, + "sasl.password": authPassword, + }) + if err != nil { + fmt.Printf("Error creating consumer: %v\n", err) + return + } + c.SubscribeTopics([]string{topicName}, nil) + // Channel for signals + signals := make(chan os.Signal, 1) + done := make(chan bool, 1) + + signal.Notify(signals, os.Interrupt, syscall.SIGTERM) + + go func() { + for { + select { + case sig := <-signals: + fmt.Printf("Caught signal %v: terminating\n", sig) + done <- true + return + default: + msg, err := c.ReadMessage(-1) + if err == nil { + var statementSet FlinkStatementSet + if err := json.Unmarshal(msg.Value, &statementSet); err != nil { + fmt.Printf("Error unmarshalling message: %v\n", err) + continue + } else { + fmt.Printf("Received message: %+v\n", statementSet) + + flinkGatewayURL := os.Getenv("FLINK_GATEWAY_URL") + confluentGatewayURL := os.Getenv("CONFLUENT_GATEWAY_URL") + var sessionID string + if flinkProvider == "flink" { + sessionID, err = sendFlinkSQL(flinkGatewayURL, statementSet) + } else { + sessionID, err = sendFlinkSQLConfluent(confluentGatewayURL, statementSet, confluentConnection) + } + + if err != nil { + fmt.Println("Error running Flink statement:", err) + sendMessage("Error: "+err.Error(), statementSet.ConversationID, systemToken, msgDebug) + sendMessage("I am sorry, I am unable to answer that question.", statementSet.ConversationID, systemToken, msgNormal) + return + } + fmt.Println("Successfully executed the Flink statement.") + sendMessage("FlinkSessionID: "+sessionID, statementSet.ConversationID, systemToken, msgDebug) + var flinkOutput FlinkOutput + flinkOutput.SessionID = sessionID + flinkOutput.Question = statementSet.Question + flinkOutput.MessageID = statementSet.MessageID + flinkOutput.ConversationID = statementSet.ConversationID + err = produceFlinkOutput(flinkOutput, kafkaURL, "flink-producer-"+groupID, authUsername, authPassword) + if err != nil { + + fmt.Printf("error producing message to Kafka: %v\n", err) + sendMessage("Error: "+err.Error(), statementSet.ConversationID, systemToken, msgDebug) + sendMessage("I am sorry, I am unable to answer that question.", statementSet.ConversationID, systemToken, msgNormal) + } + sendMessage("Message produced to topic: flink.outputs", statementSet.ConversationID, systemToken, msgDebug) + } + } else { + fmt.Printf("Consumer error: %v\n", err) + } + } + } + }() + <-done + c.Close() + fmt.Println("Consumer closed") +} diff --git a/backend/components/flink-connector/src/tools.go b/backend/components/flink-connector/src/tools.go new file mode 100644 index 0000000000..be68a63687 --- /dev/null +++ b/backend/components/flink-connector/src/tools.go @@ -0,0 +1,503 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "strings" + "time" + + "github.com/confluentinc/confluent-kafka-go/v2/kafka" +) + +func sendFlinkSQL(url string, statementSet FlinkStatementSet) (string, error) { + timestamp := time.Now().Unix() + strTimestamp := fmt.Sprintf("%d", timestamp) + replacements := map[string]string{ + "{PROPERTIES_GROUP_ID}": "flink-" + strTimestamp, + "{PROPERTIES_BOOTSTRAP_SERVERS}": os.Getenv("KAFKA_BROKERS"), + "{PROPERTIES_SASL_JAAS_CONFIG}": fmt.Sprintf("org.apache.flink.kafka.shaded.org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";", os.Getenv("AUTH_JAAS_USERNAME"), os.Getenv("AUTH_JAAS_PASSWORD")), + } + for i, stmt := range statementSet.Statements { + for placeholder, value := range replacements { + stmt = strings.Replace(stmt, placeholder, value, -1) + } + statementSet.Statements[i] = stmt + } + fmt.Println("Updated StatementSet: %+v\n", statementSet.Statements) + + req, err := http.NewRequest("POST", url+"/v1/sessions/", bytes.NewReader([]byte(""))) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return "", err + } + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body from the API: %v", err) + } + fmt.Println("Response: ", string(body)) + var sessionResponse FlinkSessionResponse + if err := json.Unmarshal(body, &sessionResponse); err != nil { + fmt.Printf("Error unmarshaling message: %v\n", err) + return "", err + } + defer resp.Body.Close() + + fmt.Println("The Flink session is: ", sessionResponse.SessionHandle) + for _, statement := range statementSet.Statements { + payload := FlinkSQLRequest{ + Statement: statement, + } + payloadBytes, err := json.Marshal(payload) + if err != nil { + return "", err + } + + req, err = http.NewRequest("POST", url+"/v1/sessions/"+sessionResponse.SessionHandle+"/statements/", bytes.NewReader(payloadBytes)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + + client = &http.Client{} + resp, err = client.Do(req) + if err != nil { + return "", err + } + body, err = io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body from the API: %v", err) + } + fmt.Println("Statement submitted. Response: ", string(body)) + var statementResponse FlinkStatementResponse + if err := json.Unmarshal(body, &statementResponse); err != nil { + fmt.Printf("Error unmarshaling message: %v\n", err) + return "", err + } + fmt.Printf("Check status on: %s/v1/sessions/%s/operations/%s/result/0\n", url, sessionResponse.SessionHandle, statementResponse.OperationHandle) + defer resp.Body.Close() + } + + return sessionResponse.SessionHandle, nil +} + +func produceFlinkOutput(flinkOutput FlinkOutput, kafkaURL, groupID, authUsername, authPassword string) error { + + kafkaTopic := "flink.outputs" + + flinkOutputJSON, err := json.Marshal(flinkOutput) + if err != nil { + return fmt.Errorf("error marshaling query to JSON: %w", err) + } + + configMap := kafka.ConfigMap{ + "bootstrap.servers": kafkaURL, + } + if authUsername != "" && authPassword != "" { + configMap.SetKey("security.protocol", "SASL_SSL") + configMap.SetKey("sasl.mechanisms", "PLAIN") + configMap.SetKey("sasl.username", authUsername) + configMap.SetKey("sasl.password", authPassword) + } + + producer, err := kafka.NewProducer(&configMap) + if err != nil { + return fmt.Errorf("failed to create producer: %w", err) + } + defer producer.Close() + + // Produce the message + message := kafka.Message{ + TopicPartition: kafka.TopicPartition{Topic: &kafkaTopic, Partition: kafka.PartitionAny}, + Key: []byte(flinkOutput.SessionID), + Value: flinkOutputJSON, + } + + err = producer.Produce(&message, nil) + if err != nil { + return fmt.Errorf("failed to produce message: %w", err) + } + fmt.Println("message scheduled for production") + producer.Flush(15 * 1000) + fmt.Println("message flushed") + return nil +} + +func sendMessage(message string, conversationId string, systemToken string, debug bool) (int, string, error) { + messageContent := messageContent{ + Text: message, + Debug: debug, + } + messageToSend := ApplicationCommunicationSendMessage{ + ConversationID: conversationId, + Message: messageContent, + } + messageJSON, err := json.Marshal(messageToSend) + if err != nil { + fmt.Printf("Error encoding response to JSON: %v\n", err) + return 0, "", errors.New("The message could not be encoded to JSON for sending.") + } + + req, err := http.NewRequest("POST", "http://api-communication/messages.send", bytes.NewReader(messageJSON)) + if err != nil { + fmt.Printf("Error creating request: %v\n", err) + return 0, "", errors.New("The message could not be sent.") + } + req.Header.Add("Authorization", "Bearer "+systemToken) + req.Header.Add("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + fmt.Printf("Error sending POST request: %v\n", err) + return 0, "", errors.New("Error sending POST request.") + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body:", err) + return 0, "", errors.New("Error reading response body.") + } + + var response SendMessageResponse + err = json.Unmarshal(body, &response) + if err != nil { + fmt.Println("Error unmarshaling response:", err) + return 0, "", errors.New("Response couldn't be unmarshaled.") + } + + fmt.Printf("Message sent with status code: %d\n", resp.StatusCode) + return resp.StatusCode, response.ID, nil +} + +func sendFlinkSQLConfluent(url string, statementSet FlinkStatementSet, connection ConfluentConnection) (string, error) { + timestamp := time.Now().Unix() + strTimestamp := fmt.Sprintf("%d", timestamp) + statementName := "airy-" + strTimestamp + payload := ConfluentFlink{ + Name: statementName, + Spec: ConfluentFlinkSpec{ + Statement: statementSet.Statements[0], + ComputePoolID: connection.ComputePoolID, + Principal: connection.Principal, + Properties: FlinkSpecProperties{ + SQLCurrentCatalog: connection.SQLCurrentCatalog, + SQLCurrentDatabase: connection.SQLCurrentDatabase, + }, + Stopped: false, + }, + } + payloadBytes, err := json.Marshal(payload) + if err != nil { + return "", err + } + + req, err := http.NewRequest("POST", url, bytes.NewReader(payloadBytes)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Add("Authorization", "Basic "+connection.Token) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return "", err + } + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body from the API: %v", err) + } + fmt.Println("Statement submitted. Response: ", string(body)) + var statementResponse ConfluentFlinkStatementResponse + if err := json.Unmarshal(body, &statementResponse); err != nil { + fmt.Printf("Error unmarshaling message: %v\n", err) + return "", err + } + fmt.Printf("Check status on: %s/%s\n", url, statementName) + defer resp.Body.Close() + + return statementName, nil +} + +func getFlinkResult(url, sessionID string) (FlinkResult, error) { + fmt.Println("The Flink session is: ", sessionID) + payload := FlinkSQLRequest{ + Statement: "select * from output;", + } + payloadBytes, err := json.Marshal(payload) + if err != nil { + return FlinkResult{}, err + } + + req, err := http.NewRequest("POST", url+"/v1/sessions/"+sessionID+"/statements/", bytes.NewReader(payloadBytes)) + if err != nil { + return FlinkResult{}, err + } + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return FlinkResult{}, err + } + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body from the API: %v", err) + } + fmt.Println("Statement submitted. Response: ", string(body)) + var statementResponse FlinkStatementResponse + if err := json.Unmarshal(body, &statementResponse); err != nil { + fmt.Printf("Error unmarshaling message: %v\n", err) + return FlinkResult{}, err + } + + fmt.Printf("Fetching result from: %s/v1/sessions/%s/operations/%s/result/0\n", url, sessionID, statementResponse.OperationHandle) + time.Sleep(20 * time.Second) + req, err = http.NewRequest("GET", url+"/v1/sessions/"+sessionID+"/operations/"+statementResponse.OperationHandle+"/result/0", nil) + if err != nil { + return FlinkResult{}, err + } + req.Header.Set("Content-Type", "application/json") + + client = &http.Client{} + resp, err = client.Do(req) + if err != nil { + return FlinkResult{}, err + } + body, err = io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body from the API: %v", err) + } + fmt.Println("Statement submitted. Response: ", string(body)) + var flinkResultResponse FlinkResultResponse + if err := json.Unmarshal(body, &flinkResultResponse); err != nil { + fmt.Printf("Error unmarshaling message: %v\n", err) + return FlinkResult{}, err + } + defer resp.Body.Close() + + if flinkResultResponse.Errors != nil { + statementError := errors.New(strings.Join(flinkResultResponse.Errors, ",")) + return FlinkResult{}, statementError + } + return flinkResultResponse.Results, nil +} + +func markdown(message string) (string, error) { + return message, nil +} + +func convertResultToMarkdown(result FlinkResult) (string, error) { + var builder strings.Builder + + if len(result.Columns) == 0 { + return "", errors.New("No columns found for generating the Markdown table.") + } + for _, col := range result.Columns { + builder.WriteString("| " + col.Name + " ") + } + builder.WriteString("|\n") + + for range result.Columns { + builder.WriteString("|---") + } + builder.WriteString("|\n") + + for _, d := range result.Data { + for _, field := range d.Fields { + builder.WriteString(fmt.Sprintf("| %v ", field)) + } + builder.WriteString("|\n") + } + + return builder.String(), nil +} + +func getFlinkResultConfluent(url, sessionID string, connection ConfluentConnection) ([]string, string, error) { + req, err := http.NewRequest("GET", url+"/"+sessionID, bytes.NewReader([]byte(""))) + if err != nil { + return []string{}, "", err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Add("Authorization", "Basic "+connection.Token) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return []string{}, "", err + } + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body from the API: %v", err) + } + fmt.Println("Statement submitted. Response: ", string(body)) + var statementResponse ConfluentFlinkStatementResponse + if err := json.Unmarshal(body, &statementResponse); err != nil { + fmt.Printf("Error unmarshaling message: %v\n", err) + return []string{}, "", err + } + fmt.Printf("Received result for statement: %s\n", sessionID) + fmt.Println("Phase: ", statementResponse.Status.Phase, " Detail: ", statementResponse.Status.Detail) + defer resp.Body.Close() + + if statementResponse.Status.Phase == "RUNNING" || statementResponse.Status.Phase == "COMPLETED" { + columns, err := getColumnNames(statementResponse.Status.ResultSchema) + if err != nil { + fmt.Println("Extracting of the column names failed.") + return []string{}, "", err + } + req, err := http.NewRequest("GET", url+"/"+sessionID+"/results", bytes.NewReader([]byte(""))) + if err != nil { + return []string{}, "", err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Add("Authorization", "Basic "+connection.Token) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return []string{}, "", err + } + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body from the API: %v", err) + } + fmt.Println("Statement submitted. Response: ", string(body)) + var result ConfluentFlinkResultsResponse + if err := json.Unmarshal(body, &result); err != nil { + fmt.Printf("Error unmarshaling message: %v\n", err) + return []string{}, "", err + } + nextResult := result.Metadata.Next + fmt.Println("Next result: ", nextResult) + fmt.Println("Result: ", result.Results.Data) + data, err := dataToString(result.Results.Data) + if data != "" { + return columns, data, nil + } else { + req, err := http.NewRequest("GET", nextResult, bytes.NewReader([]byte(""))) + if err != nil { + return []string{}, "", err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Add("Authorization", "Basic "+connection.Token) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return []string{}, "", err + } + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body from the API: %v", err) + } + fmt.Println("Statement submitted. Response: ", string(body)) + var result ConfluentFlinkResultsResponse + if err := json.Unmarshal(body, &result); err != nil { + fmt.Printf("Error unmarshaling message: %v\n", err) + return []string{}, "", err + } + data, err := dataToString(result.Results.Data) + return columns, data, err + } + } else { + err := errors.New("Flink statement failed. Status: " + statementResponse.Status.Phase) + return []string{}, "", err + } +} + +func dataToString(data interface{}) (string, error) { + if slice, ok := data.([]interface{}); ok && len(slice) > 0 { + dataBytes, err := json.Marshal(data) + if err != nil { + return "", err + } + return string(dataBytes), nil + } + return "", nil +} + +func convertConfluentResultToMarkdown(headerNames []string, jsonStr string) (string, error) { + var dataRows []ConfluentDataRow + err := json.Unmarshal([]byte(jsonStr), &dataRows) + if err != nil { + return "", err + } + + var sb strings.Builder + + header := generateMarkdownHeader(headerNames) + sb.WriteString(header) + sb.WriteString("\n") + + separator := strings.Repeat("| --- ", strings.Count(header, "|")-1) + "|" + sb.WriteString(separator) + sb.WriteString("\n") + + for _, dataRow := range dataRows { + sb.WriteString("|") + for _, cell := range dataRow.Row { + sb.WriteString(" ") + sb.WriteString(cell) + sb.WriteString(" |") + } + sb.WriteString("\n") + } + + return sb.String(), nil +} + +func extractColumnNames(jsonStr string) ([]string, error) { + var schema ConfluentResultSchema + err := json.Unmarshal([]byte(jsonStr), &schema) + if err != nil { + return nil, err + } + + var columnNames []string + for _, column := range schema.Columns { + columnNames = append(columnNames, column.Name) + } + + return columnNames, nil +} + +func generateMarkdownHeader(columnNames []string) string { + var header string + + for _, name := range columnNames { + header += "| " + name + " " + } + header += "|" + + return header +} + +func ResultsToString(rs ConfluentResultSchema) string { + var columnNames []string + for _, column := range rs.Columns { + columnNames = append(columnNames, column.Name) + } + return strings.Join(columnNames, ", ") +} + +func getColumnNames(schema ConfluentResultSchema) ([]string, error) { + var columnNames []string + for _, column := range schema.Columns { + columnNames = append(columnNames, column.Name) + } + return columnNames, nil +} diff --git a/backend/components/flink-connector/src/types.go b/backend/components/flink-connector/src/types.go new file mode 100644 index 0000000000..c67ee3944f --- /dev/null +++ b/backend/components/flink-connector/src/types.go @@ -0,0 +1,137 @@ +package main + +type ApplicationCommunicationSendMessage struct { + ConversationID string `json:"conversation_id"` + Message messageContent `json:"message"` + Metadata map[string]string `json:"metadata"` +} + +type messageContent struct { + Text string `json:"text"` + Debug bool `json:"debug"` +} + +type SendMessageResponse struct { + ID string `json:"id"` + State string `json:"state"` +} + +type FlinkOutput struct { + SessionID string `json:"session_id"` + Question string `json:"question"` + MessageID string `json:"message_id"` + ConversationID string `json:"conversation_id"` +} + +type FlinkSQLRequest struct { + Statement string `json:"statement"` +} + +type FlinkSessionResponse struct { + SessionHandle string `json:"sessionHandle"` +} + +type FlinkStatementResponse struct { + OperationHandle string `json:"operationHandle"` +} + +type Column struct { + Name string `json:"name"` + LogicalType struct { + Type string `json:"type"` + Nullable bool `json:"nullable"` + Length int `json:"length,omitempty"` + } `json:"logicalType"` + Comment interface{} `json:"comment"` +} + +type Data struct { + Kind string `json:"kind"` + Fields []interface{} `json:"fields"` +} + +type FlinkResult struct { + Columns []Column `json:"columns"` + RowFormat string `json:"rowFormat"` + Data []Data `json:"data"` +} + +type FlinkResultResponse struct { + ResultType string `json:"resultType"` + IsQueryResult bool `json:"isQueryResult"` + JobID string `json:"jobID"` + ResultKind string `json:"resultKind"` + Results FlinkResult `json:"results"` + NextResultUri string `json:"nextResultUri"` + Errors []string `json:"errors"` +} + +type ConfluentFlink struct { + Name string `json:"name"` + Spec ConfluentFlinkSpec `json:"spec"` +} + +type ConfluentFlinkSpec struct { + Statement string `json:"statement"` + ComputePoolID string `json:"compute_pool_id"` + Principal string `json:"principal"` + Properties FlinkSpecProperties `json:"properties"` + Stopped bool `json:"stopped"` +} + +type FlinkSpecProperties struct { + SQLCurrentCatalog string `json:"sql.current-catalog"` + SQLCurrentDatabase string `json:"sql.current-database"` +} + +type ConfluentFlinkStatementResponse struct { + Name string `json:"name"` + Status ConfluentFlinkStatementStatus `json:"status"` +} + +type ConfluentFlinkStatementStatus struct { + Detail string `json:"detail"` + Phase string `json:"phase"` + ResultSchema ConfluentResultSchema `json:"result_schema"` +} + +type ConfluentResultSchema struct { + Columns []struct { + Name string `json:"name"` + } `json:"columns"` +} + +type ConfluentFlinkResultsResponse struct { + Metadata ResultResponseMetadata `json:"metadata"` + Results ResultResponseResults `json:"results"` +} + +type ResultResponseMetadata struct { + CreatedAt string `json:"created_at"` + Next string `json:"next"` + Self string `json:"self"` +} + +type ResultResponseResults struct { + Data interface{} `json:"data"` +} + +type ConfluentDataRow struct { + Op int `json:"op"` + Row []string `json:"row"` +} + +type FlinkStatementSet struct { + Statements []string `json:"statements"` + Question string `json:"question"` + MessageID string `json:"message_id"` + ConversationID string `json:"conversation_id"` +} + +type ConfluentConnection struct { + Token string + ComputePoolID string + Principal string + SQLCurrentCatalog string + SQLCurrentDatabase string +} diff --git a/backend/components/google/helm/templates/deployments.yaml b/backend/components/google/helm/templates/deployments.yaml index 7af611787e..80c2af2376 100644 --- a/backend/components/google/helm/templates/deployments.yaml +++ b/backend/components/google/helm/templates/deployments.yaml @@ -54,6 +54,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.connector.resources | indent 12 }} initContainers: @@ -71,6 +76,11 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} --- apiVersion: apps/v1 kind: Deployment @@ -122,6 +132,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.eventsRouter.resources | indent 10 }} initContainers: @@ -139,3 +154,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/media-resolver/helm/templates/deployment.yaml b/backend/components/media-resolver/helm/templates/deployment.yaml index b31af97ffd..3484e0b71f 100644 --- a/backend/components/media-resolver/helm/templates/deployment.yaml +++ b/backend/components/media-resolver/helm/templates/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.resources | indent 12 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/schema-registry-manager/Dockerfile b/backend/components/schema-registry-manager/Dockerfile new file mode 100644 index 0000000000..e069954862 --- /dev/null +++ b/backend/components/schema-registry-manager/Dockerfile @@ -0,0 +1,19 @@ +FROM node:18 + +WORKDIR /app + +COPY ./src/package*.json ./ +COPY ./src/tsconfig*.json ./ + +RUN npm install +RUN npm install typescript -g + +COPY ./src/app.ts ./ +COPY ./src/types.ts ./ +COPY ./src/providers/karapace.ts ./providers/ + +RUN tsc + +EXPOSE 3000 + +CMD [ "node", "app.js" ] diff --git a/backend/components/schema-registry-manager/Makefile b/backend/components/schema-registry-manager/Makefile new file mode 100644 index 0000000000..3aa78401e3 --- /dev/null +++ b/backend/components/schema-registry-manager/Makefile @@ -0,0 +1,6 @@ +build: + docker build -t schema-registry-manager . + +release: build + docker tag schema-registry-manager ghcr.io/airyhq/backend/schema-registry-manager:release + docker push ghcr.io/airyhq/backend/schema-registry-manager:release diff --git a/backend/components/schema-registry-manager/helm/BUILD b/backend/components/schema-registry-manager/helm/BUILD new file mode 100644 index 0000000000..45805d5f1c --- /dev/null +++ b/backend/components/schema-registry-manager/helm/BUILD @@ -0,0 +1,3 @@ +load("//tools/build:helm.bzl", "helm_ruleset_core_version") + +helm_ruleset_core_version() diff --git a/backend/components/schema-registry-manager/helm/Chart.yaml b/backend/components/schema-registry-manager/helm/Chart.yaml new file mode 100644 index 0000000000..7205b553ce --- /dev/null +++ b/backend/components/schema-registry-manager/helm/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: "1.0" +description: Schema registry component to manage different providers +name: schema-registry-manager +version: 1.0 diff --git a/backend/components/schema-registry-manager/helm/templates/configmap.yaml b/backend/components/schema-registry-manager/helm/templates/configmap.yaml new file mode 100644 index 0000000000..05de4d5898 --- /dev/null +++ b/backend/components/schema-registry-manager/helm/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.component }} + labels: + core.airy.co/managed: "true" + core.airy.co/mandatory: "{{ .Values.mandatory }}" + core.airy.co/component: "{{ .Values.component }}" + annotations: + core.airy.co/enabled: "{{ .Values.enabled }}" diff --git a/backend/components/schema-registry-manager/helm/templates/deployment.yaml b/backend/components/schema-registry-manager/helm/templates/deployment.yaml new file mode 100644 index 0000000000..b902145179 --- /dev/null +++ b/backend/components/schema-registry-manager/helm/templates/deployment.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.component }} + labels: + app: {{ .Values.component }} + core.airy.co/managed: "true" + core.airy.co/mandatory: "{{ .Values.mandatory }}" + core.airy.co/component: {{ .Values.component }} +spec: + replicas: {{ if .Values.enabled }} 1 {{ else }} 0 {{ end }} + selector: + matchLabels: + app: {{ .Values.component }} + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: {{ .Values.component }} + spec: + containers: + - name: app + image: "ghcr.io/airyhq/{{ .Values.image }}:{{ .Values.imageTag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: security + - configMapRef: + name: kafka-config + - configMapRef: + name: {{ .Values.component }} + env: + - name: KAFKA_TOPIC_NAME + value: {{ .Values.kafka.topic }} + livenessProbe: + httpGet: + path: /actuator/health + port: {{ .Values.port }} + httpHeaders: + - name: Health-Check + value: health-check + initialDelaySeconds: 43200 + periodSeconds: 10 + failureThreshold: 3 diff --git a/backend/components/schema-registry-manager/helm/templates/service.yaml b/backend/components/schema-registry-manager/helm/templates/service.yaml new file mode 100644 index 0000000000..4d636e8b26 --- /dev/null +++ b/backend/components/schema-registry-manager/helm/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.component }} + labels: + app: {{ .Values.component }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: {{ .Values.component }} + port: 80 + targetPort: {{ .Values.port }} + selector: + app: {{ .Values.component }} diff --git a/backend/components/schema-registry-manager/helm/values.yaml b/backend/components/schema-registry-manager/helm/values.yaml new file mode 100644 index 0000000000..200ef3ca36 --- /dev/null +++ b/backend/components/schema-registry-manager/helm/values.yaml @@ -0,0 +1,9 @@ +component: schema-registry-manager +mandatory: false +enabled: false +image: backend/schema-registry-manager +imageTag: release +port: 3000 +resources: +kafka: + topic: application.communication.messages \ No newline at end of file diff --git a/backend/components/schema-registry-manager/src/app.ts b/backend/components/schema-registry-manager/src/app.ts new file mode 100644 index 0000000000..fb7f384e3e --- /dev/null +++ b/backend/components/schema-registry-manager/src/app.ts @@ -0,0 +1,263 @@ +import dotenv from 'dotenv'; +import express, {Express, Request as ExpressRequest, Response as ExpressResponse} from 'express'; +import http from 'http'; +import cors from 'cors'; + +import {SchemaProvider} from './types'; +import { + checkCompatibilityOfNewSchema, + createSchema, + deleteSchema, + getLastMessage, + getSchemaInfo, + getSchemaVersions, + getSchemas, + updateSchema, +} from './providers/karapace'; + +dotenv.config(); + +const app: Express = express(); +const port = process.env.PORT || 3000; +const bodyParser = require('body-parser'); +const currentProvider: SchemaProvider = SchemaProvider.karapace; + +// Middleware +app.use(bodyParser.json()); + +// CORS options +const corsOptions = { + origin: 'http://localhost:8080', +}; + +// Use cors middleware with the specified options +app.use(cors(corsOptions)); + +app.get('/schemas.provider', (req: ExpressRequest, res: ExpressResponse) => { + res.status(200).send(currentProvider); +}); + +app.get('/schemas.list', (req: ExpressRequest, res: ExpressResponse) => { + switch (currentProvider) { + case SchemaProvider.karapace: + getSchemas(req.get('host') as string) + .then((response: string[]) => { + res.send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.get('/schemas.versions', (req: ExpressRequest, res: ExpressResponse) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + getSchemaVersions(req.get('host') as string, req.query.topicName as string) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.get('/schemas.info', (req: ExpressRequest, res: ExpressResponse) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + + let version = 'latest'; + if (req.query.version) { + version = req.query.version as string; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + getSchemaInfo(req.get('host') as string, req.query.topicName as string, version) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.post('/schemas.update', (req: ExpressRequest, res: ExpressResponse) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + if (!req.body.schema) { + res.status(400).send('Missing schema'); + return; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + updateSchema(req.get('host') as string, req.query.topicName as string, req.body.schema as string) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.post('/schemas.create', (req: ExpressRequest, res: ExpressResponse) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + if (!req.body.schema) { + res.status(400).send('Missing schema'); + return; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + createSchema(req.get('host') as string, req.query.topicName as string, req.body.schema as string) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.post('/schemas.compatibility', (req: ExpressRequest, res: ExpressResponse) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + if (!req.query.version) { + res.status(400).send('Missing version'); + return; + } + if (!req.body.schema) { + res.status(400).send('Missing schema'); + return; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + checkCompatibilityOfNewSchema( + req.get('host') as string, + req.query.topicName as string, + req.body.schema as string, + req.query.version as string + ) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.post('/schemas.delete', (req: ExpressRequest, res: ExpressResponse) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + deleteSchema(req.get('host') as string, req.query.topicName as string) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.get('/schemas.lastMessage', (req: ExpressRequest, res: ExpressResponse) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + getLastMessage(req.get('host') as string, req.query.topicName as string) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +async function startHealthcheck() { + const server = http.createServer((req: any, res: any) => { + if (req.url === '/actuator/health' && req.method === 'GET') { + const response = {status: 'UP'}; + const jsonResponse = JSON.stringify(response); + + res.writeHead(200, {'Content-Type': 'application/json'}); + res.end(jsonResponse); + } else { + res.writeHead(404, {'Content-Type': 'text/plain'}); + res.end('Not Found'); + } + }); + + server.listen(80, () => { + console.log('Health-check started'); + }); +} + +async function main() { + startHealthcheck(); + app.listen(port, () => { + console.log(`Server is running on http://localhost:${port}`); + }); +} + +main().catch(console.error); diff --git a/backend/components/schema-registry-manager/src/package.json b/backend/components/schema-registry-manager/src/package.json new file mode 100644 index 0000000000..970c2ea5f3 --- /dev/null +++ b/backend/components/schema-registry-manager/src/package.json @@ -0,0 +1,17 @@ +{ + "dependencies": { + "@types/express": "^4.17.21", + "@types/node": "^20.10.3", + "cors": "^2.8.5", + "dotenv": "^16.4.2", + "express": "^4.18.2", + "node-fetch": "^2.6.1" + }, + "devDependencies": { + "@types/cors": "^2.8.17", + "@types/express": "^4.17.21", + "@types/node": "^20.10.3", + "ts-node": "^10.9.2", + "typescript": "^5.3.3" + } +} diff --git a/backend/components/schema-registry-manager/src/providers/karapace.ts b/backend/components/schema-registry-manager/src/providers/karapace.ts new file mode 100644 index 0000000000..487aa31a8a --- /dev/null +++ b/backend/components/schema-registry-manager/src/providers/karapace.ts @@ -0,0 +1,121 @@ +export async function getSchemas(host: string) { + return getData(host, 'subjects').then(response => { + return response; + }); +} + +export async function getSchemaVersions(host: string, topicName: string) { + return getData(host, `subjects/${topicName}/versions`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + return response; + }); +} + +export async function getSchemaInfo(host: string, topicName: string, version: string) { + return getData(host, `subjects/${topicName}/versions/${version}`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + return response; + }); +} + +export async function updateSchema(host: string, topicName: string, schema: string) { + const body = { + schema: JSON.stringify({...JSON.parse(schema)}), + }; + return postData(host, `subjects/${topicName}/versions`, body).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + if (response.id) return response; + if (response.message) return Promise.reject(response.message); + return Promise.reject('Unknown Error'); + }); +} + +export async function createSchema(host: string, topicName: string, schema: string) { + const body = { + schema: JSON.stringify({...JSON.parse(schema)}), + }; + return postData(host, `subjects/${topicName}/versions`, body) + .then(response => { + if (response.id) return response; + if (response.message) return Promise.reject(response.message); + return Promise.reject('Unknown Error'); + }) + .catch(e => { + return Promise.reject(e); + }); +} + +export async function checkCompatibilityOfNewSchema(host: string, topicName: string, schema: string, version: string) { + const body = { + schema: JSON.stringify({...JSON.parse(schema)}), + }; + + return postData(host, `compatibility/subjects/${topicName}/versions/${version}`, body) + .then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + if (response.is_compatible !== undefined) { + if (response.is_compatible === true) { + return response; + } + return Promise.reject('Schema Not Compatible'); + } + if (response.message) return Promise.reject(response.message); + return Promise.reject('Unknown Error'); + }) + .catch(e => { + return Promise.reject(e); + }); +} + +export async function deleteSchema(host: string, topicName: string) { + return deleteData(host, `subjects/${topicName}`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + return response; + }); +} + +export async function getLastMessage(host: string, topicName: string) { + const body = { + ksql: `PRINT '${topicName}' FROM BEGINNING LIMIT 1;`, + streamsProperties: {}, + }; + return postData(host, 'query', body).then(response => { + return response; + }); +} + +async function getData(host: string, url: string) { + const response = await fetch('https://' + host + '/' + url, { + method: 'GET', + }); + return response.json(); +} + +async function deleteData(host: string, url: string) { + const response = await fetch('https://' + host + '/' + url, { + method: 'DELETE', + }); + return response.json(); +} + +async function postData(host: string, url: string, body: any) { + const response = await fetch('https://' + host + '/' + url, { + method: 'POST', + headers: { + 'Content-Type': 'application/vnd.schemaregistry.v1+json', + }, + body: JSON.stringify(body), + }); + + return response.json(); +} diff --git a/backend/components/schema-registry-manager/src/tsconfig.json b/backend/components/schema-registry-manager/src/tsconfig.json new file mode 100644 index 0000000000..8975f66043 --- /dev/null +++ b/backend/components/schema-registry-manager/src/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "target": "ES2016", + "module": "commonjs", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + } +} \ No newline at end of file diff --git a/backend/components/schema-registry-manager/src/types.ts b/backend/components/schema-registry-manager/src/types.ts new file mode 100644 index 0000000000..53776abb6d --- /dev/null +++ b/backend/components/schema-registry-manager/src/types.ts @@ -0,0 +1,4 @@ +export enum SchemaProvider { + karapace = 'karapace', + confluentCloud = 'confluent-cloud', +} diff --git a/backend/components/sources-api/helm/templates/deployment.yaml b/backend/components/sources-api/helm/templates/deployment.yaml index 9ce7cda665..4d23b16dea 100644 --- a/backend/components/sources-api/helm/templates/deployment.yaml +++ b/backend/components/sources-api/helm/templates/deployment.yaml @@ -50,6 +50,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.resources | indent 10 }} initContainers: @@ -67,3 +72,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/streams/helm/templates/deployment.yaml b/backend/components/streams/helm/templates/deployment.yaml index 067e9bf6c9..b0cd1dfb3f 100644 --- a/backend/components/streams/helm/templates/deployment.yaml +++ b/backend/components/streams/helm/templates/deployment.yaml @@ -48,6 +48,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.resources | indent 10 }} initContainers: @@ -67,3 +72,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/twilio/helm/templates/deployments.yaml b/backend/components/twilio/helm/templates/deployments.yaml index 0e3a6a5706..461dd6c30d 100644 --- a/backend/components/twilio/helm/templates/deployments.yaml +++ b/backend/components/twilio/helm/templates/deployments.yaml @@ -54,6 +54,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.connector.resources | indent 12 }} initContainers: @@ -141,3 +146,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/viber/helm/templates/deployments.yaml b/backend/components/viber/helm/templates/deployments.yaml index ebb0d5254b..55fbb10d72 100644 --- a/backend/components/viber/helm/templates/deployments.yaml +++ b/backend/components/viber/helm/templates/deployments.yaml @@ -49,6 +49,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.connector.resources | indent 12 }} initContainers: @@ -66,3 +71,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/webhook/helm/templates/deployments.yaml b/backend/components/webhook/helm/templates/deployments.yaml index 814a41c9ae..6c6cb5b1b1 100644 --- a/backend/components/webhook/helm/templates/deployments.yaml +++ b/backend/components/webhook/helm/templates/deployments.yaml @@ -55,6 +55,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.consumer.resources | indent 10 }} initContainers: @@ -157,3 +162,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/whatsapp/helm/templates/deployments.yaml b/backend/components/whatsapp/helm/templates/deployments.yaml index 1fcbff4b11..4c5e04b48a 100644 --- a/backend/components/whatsapp/helm/templates/deployments.yaml +++ b/backend/components/whatsapp/helm/templates/deployments.yaml @@ -113,6 +113,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.eventsRouter.resources | indent 10 }} initContainers: @@ -130,3 +135,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/docs/docs/changelog.md b/docs/docs/changelog.md index 0b0cb793ab..9d4aa94e5c 100644 --- a/docs/docs/changelog.md +++ b/docs/docs/changelog.md @@ -3,6 +3,64 @@ title: Changelog sidebar_label: 📝 Changelog --- +## 0.55.0 + +#### Changes + +- [[#4167](https://github.com/airyhq/airy/issues/4167)] Kafka certificate authentication [[#4168](https://github.com/airyhq/airy/pull/4168)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Change name for Pinecone connector [[#4127](https://github.com/airyhq/airy/pull/4127)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Add description for components [[#4126](https://github.com/airyhq/airy/pull/4126)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Add LLM filter in Catalog [[#4125](https://github.com/airyhq/airy/pull/4125)] + +#### 🚀 Features + +- [[#4139](https://github.com/airyhq/airy/issues/4139)] Add Flink connector [[#4147](https://github.com/airyhq/airy/pull/4147)] +- [[#3945](https://github.com/airyhq/airy/issues/3945)] Slack connector [[#4146](https://github.com/airyhq/airy/pull/4146)] +- [[#4144](https://github.com/airyhq/airy/issues/4144)] Adapt Frontend with Schema Manager [[#4145](https://github.com/airyhq/airy/pull/4145)] +- [[#4141](https://github.com/airyhq/airy/issues/4141)] Schema Registry Manager [[#4143](https://github.com/airyhq/airy/pull/4143)] +- [[#4137](https://github.com/airyhq/airy/issues/4137)] Improve Kafka Sections [[#4138](https://github.com/airyhq/airy/pull/4138)] +- [[#4132](https://github.com/airyhq/airy/issues/4132)] Added Copilot source to libs and apps [[#4133](https://github.com/airyhq/airy/pull/4133)] + +#### 🐛 Bug Fixes + +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Fix selecting fields in streams [[#4123](https://github.com/airyhq/airy/pull/4123)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Fix symbol [[#4122](https://github.com/airyhq/airy/pull/4122)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Update icons for LLM components [[#4120](https://github.com/airyhq/airy/pull/4120)] +- [[#4108](https://github.com/airyhq/airy/issues/4108)] Fix schemas and topics screens [[#4109](https://github.com/airyhq/airy/pull/4109)] + +#### 📚 Documentation + +- [[#4134](https://github.com/airyhq/airy/issues/4134)] Update the diagram in the README file [[#4136](https://github.com/airyhq/airy/pull/4136)] +- [[#4134](https://github.com/airyhq/airy/issues/4134)] Update main diagram [[#4135](https://github.com/airyhq/airy/pull/4135)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Improve docs [[#4131](https://github.com/airyhq/airy/pull/4131)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Improve docs [[#4128](https://github.com/airyhq/airy/pull/4128)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Docs - Remove Solutions link [[#4115](https://github.com/airyhq/airy/pull/4115)] + +#### 🧰 Maintenance + +- Bump google.golang.org/protobuf from 1.28.0 to 1.33.0 [[#4155](https://github.com/airyhq/airy/pull/4155)] +- Bump webpack-dev-middleware from 5.3.3 to 5.3.4 in /docs [[#4154](https://github.com/airyhq/airy/pull/4154)] +- Bump google.golang.org/protobuf from 1.28.0 to 1.33.0 in /infrastructure/lib/go/k8s/handler [[#4152](https://github.com/airyhq/airy/pull/4152)] +- Bump webpack-dev-middleware from 5.3.3 to 5.3.4 [[#4153](https://github.com/airyhq/airy/pull/4153)] +- Bump express from 4.18.2 to 4.19.2 [[#4151](https://github.com/airyhq/airy/pull/4151)] +- Bump express from 4.18.2 to 4.19.2 in /docs [[#4150](https://github.com/airyhq/airy/pull/4150)] +- Bump follow-redirects from 1.15.2 to 1.15.6 [[#4149](https://github.com/airyhq/airy/pull/4149)] +- Bump follow-redirects from 1.15.2 to 1.15.6 in /docs [[#4148](https://github.com/airyhq/airy/pull/4148)] +- Bump github.com/cyphar/filepath-securejoin from 0.2.3 to 0.2.4 [[#4118](https://github.com/airyhq/airy/pull/4118)] +- Bump @adobe/css-tools from 4.0.1 to 4.3.1 [[#4116](https://github.com/airyhq/airy/pull/4116)] +- Bump @svgr/plugin-svgo from 6.5.1 to 8.1.0 [[#4114](https://github.com/airyhq/airy/pull/4114)] +- Bump semver from 5.7.1 to 5.7.2 in /docs [[#4110](https://github.com/airyhq/airy/pull/4110)] +- Bump sass-loader from 13.1.0 to 13.3.2 [[#4103](https://github.com/airyhq/airy/pull/4103)] +- Bump word-wrap from 1.2.3 to 1.2.4 [[#4112](https://github.com/airyhq/airy/pull/4112)] + +#### Airy CLI + +You can download the Airy CLI for your operating system from the following links: + +[MacOS](https://airy-core-binaries.s3.amazonaws.com/0.55.0/darwin/amd64/airy) +[Linux](https://airy-core-binaries.s3.amazonaws.com/0.55.0/linux/amd64/airy) +[Windows](https://airy-core-binaries.s3.amazonaws.com/0.55.0/windows/amd64/airy.exe) + ## 0.54.0 #### 🚀 Features @@ -1348,38 +1406,3 @@ You can download the Airy CLI for your operating system from the following links [Linux](https://airy-core-binaries.s3.amazonaws.com/0.35.0/linux/amd64/airy) [Windows](https://airy-core-binaries.s3.amazonaws.com/0.35.0/windows/amd64/airy.exe) -## 0.34.0 - -#### Changes - -#### 🚀 Features - -- [[#2518](https://github.com/airyhq/airy/issues/2518)] Add fargate annotation [[#2540](https://github.com/airyhq/airy/pull/2540)] -- [[#2305](https://github.com/airyhq/airy/issues/2305)] Add CLI outdated version warning [[#2529](https://github.com/airyhq/airy/pull/2529)] - -#### 🐛 Bug Fixes - -- [[#2434](https://github.com/airyhq/airy/issues/2434)] Fix broken instagram Facebook inbox ingestion [[#2535](https://github.com/airyhq/airy/pull/2535)] -- [[#2457](https://github.com/airyhq/airy/issues/2457)] Fix upgrade to same version [[#2538](https://github.com/airyhq/airy/pull/2538)] -- [[#2510](https://github.com/airyhq/airy/issues/2510)] Improve error logging for helm install [[#2522](https://github.com/airyhq/airy/pull/2522)] -- [[#2255](https://github.com/airyhq/airy/issues/2255)] Fix helm chart url [[#2525](https://github.com/airyhq/airy/pull/2525)] -- [[#2523](https://github.com/airyhq/airy/issues/2523)] Fix VERSION and add changelog [[#2524](https://github.com/airyhq/airy/pull/2524)] -- [[#2473](https://github.com/airyhq/airy/issues/2473)] fix failing cypress test [[#2507](https://github.com/airyhq/airy/pull/2507)] - -#### 🧰 Maintenance - -- Bump react-redux from 7.2.5 to 7.2.6 [[#2539](https://github.com/airyhq/airy/pull/2539)] -- Bump reselect from 4.0.0 to 4.1.1 [[#2533](https://github.com/airyhq/airy/pull/2533)] -- Bump sass-loader from 12.1.0 to 12.3.0 [[#2534](https://github.com/airyhq/airy/pull/2534)] -- Bump @types/react-dom from 17.0.9 to 17.0.10 [[#2526](https://github.com/airyhq/airy/pull/2526)] -- Bump react-markdown from 7.0.1 to 7.1.0 [[#2527](https://github.com/airyhq/airy/pull/2527)] -- Bump webpack from 5.54.0 to 5.59.1 [[#2517](https://github.com/airyhq/airy/pull/2517)] - -#### Airy CLI - -You can download the Airy CLI for your operating system from the following links: - -[MacOS](https://airy-core-binaries.s3.amazonaws.com/0.34.0/darwin/amd64/airy) -[Linux](https://airy-core-binaries.s3.amazonaws.com/0.34.0/linux/amd64/airy) -[Windows](https://airy-core-binaries.s3.amazonaws.com/0.34.0/windows/amd64/airy.exe) - diff --git a/docs/docs/connectors/conversational-ai/introduction.md b/docs/docs/connectors/conversational-ai/introduction.md index df111a622c..151d33b063 100644 --- a/docs/docs/connectors/conversational-ai/introduction.md +++ b/docs/docs/connectors/conversational-ai/introduction.md @@ -9,7 +9,7 @@ import ButtonBox from "@site/src/components/ButtonBox"; Level up your channels' communication with Airy Core's conversational AI [connectors](/concepts/architecture#components). -Airy Core features conversational AI [connectors](/concepts/architecture#components) that you can easily install and configure on your instance. +Airy Core features conversational AI [connectors](/concepts/architecture#components) that you can easily install and configure on your instance. For all of the LLM connectors that Airy supports, please refer to our [Enterprise Docs](https://airy.co/docs/enterprise/). } iconInvertible={true} title='WebSockets to power real-time applications' - description="A WebSocket server that allows clients to receive near real-time updates about data flowing through the system." + description="A WebSocket server that allows clients to receive near real-time updates about data flowing through the system. Particularly useful in combination with our LLM connectors and apps, that can send real-time data to enrich the interaction your customers." link='/api/websocket' /> } iconInvertible={true} title='UI: From a control center to dashboards' - description="No-code interfaces to manage and control Airy, your connectors and your streams." + description="No-code interfaces to manage and control Airy, your connectors, your LLM integrations and your streams. " link='/ui/inbox/introduction' /> @@ -96,5 +96,6 @@ Here is a list of the open source components which can be added to `Airy Core`: - sources-twilio - sources-viber - sources-whatsapp +- flink-connector More information about the components API can be found [here](/api/endpoints/components). diff --git a/docs/docs/getting-started/glossary.md b/docs/docs/getting-started/glossary.md index d6060e5d34..f995120149 100644 --- a/docs/docs/getting-started/glossary.md +++ b/docs/docs/getting-started/glossary.md @@ -118,21 +118,39 @@ A tag is a special use case of metadata, which is used to tag common, Airy Core provides specialized endpoints and filters for tagging conversations. +## AI & ML + +## Large language model + +A type of artificial intelligence model designed to understand and generate human-like text based on vast amounts of data. It's trained on diverse internet text to predict the next word in a sequence, enabling it to answer questions, generate content, and assist with various tasks. Airy allows a plug-able interface into different LLMs. + +## Vector database + +A high-dimensional database store which is suitable for persistent storage for natural language processing or images. The data is represented as vectors and retrieval is based on similarity, allowing for efficient similarity searches and context creation. Vector databases are very convenient for storing vector representations of streaming data that can be queried and add context to questions that are sent to LLMs, in real time. + +## Automation + +The ability of a an Airy component to react and simulate human-like conversations and automate specific tasks, in real time. It aims to provide users with immediate, consistent responses, reducing the need for human intervention in customer support, inquiries, and other conversational scenarios. + ## Source A source represents a system that generates messaging data that a user wants to process with Airy Core. -## Stream - -The whole Airy platform is based on Kafka and real-time streaming of messages. In the context of `streams` feature that Airy supports, a `stream` is the process of joining two or multiple Kafka topics, combining the data and creating an outout topic where the result of the streaming operation will be stored. It is based on KSQL. - ### Provider Source providers are API platforms that allow Airy Core to connect to one or more of their sources typically via a webhook. E.g. Twilio is a source provider for the Twilio SMS and WhatsApp sources. +## App + +Third party open-source packages that can be installed alongside Airy, in the same Kubernetes cluster, to provide a more robust and powerful application development environment. These `Apps` can vary from databases (ex. PostgreSQL or Redis) to LLM implementations and vector databases (ex. Llama2 or FAISS). + +## Stream + +The whole Airy platform is based on Kafka and real-time streaming of messages. In the context of `streams` feature that Airy supports, a `stream` is the process of joining two or multiple Kafka topics, combining the data and creating an outout topic where the result of the streaming operation will be stored. It is based on KSQL. + ## User A user represents one authorized agent in Airy Core, which is different from a Contact diff --git a/docs/docs/getting-started/installation/helm.md b/docs/docs/getting-started/installation/helm.md index 03dcc7038f..4d3738cc0f 100644 --- a/docs/docs/getting-started/installation/helm.md +++ b/docs/docs/getting-started/installation/helm.md @@ -290,6 +290,49 @@ Run the following command to create the `Airy` platform without the bundled inst helm install airy airy/airy --timeout 10m --set prerequisites.kafka.enabled=false --values ./airy.yaml ``` +#### Confluent + +To connect to a Kafka instance in Confluent cloud, settings the `config.kafka.brokers` and `config.kafka.aurhJaas` is enough, prior to deploying the Helm chart. + +#### Aiven + +Aiven cloud uses a keystore and truststore certificates that need to be loaded on the workloads that are connecting to Kafka. Get the necessary certificates and connection files from Aiven using the `avn` CLI and place them in a separate directory. + +``` +avn service user-kafka-java-creds {KAFKA_INSTANCE} --username {USERNAME} -d ./aiven/ --password {PASSWORD} +``` + +Create a Kubernetes ConfigMap that contains the contents of the created directory: + +``` +kubectl create configmap kafka-config-certs --from-file aiven/ +``` + +Set the connection appropriate parameters in your `airy.yaml` file: + +```yaml +config: + kafka: + brokers: "the-aiven-kafka-broker-url" + keyTrustSecret: "the-key-trust-secret" +``` + +Then install Airy with the following command: + +```sh +helm install airy airy/airy --timeout 10m --set prerequisites.kafka.enabled=false --set global.kafkaCertAuth=true --values ./airy.yaml +``` + +### Kafka partitions per topic + +Currently all the default topics in the Airy instance are created with 10 partitions. To create these topics with a different number of partitions, add the following to your `airy.yaml` file before running `helm install` (before the initial creation of the topics): + +``` +provisioning: + kafka: + partitions: 2 +``` + ### Beanstalkd The default installation creates its own [Beanstalkd](https://beanstalkd.github.io/) deployment, as it is a prerequisite for using the `integration/webhook` component. diff --git a/docs/docs/getting-started/installation/minikube.md b/docs/docs/getting-started/installation/minikube.md index bf9a7912dc..9feddc0fc1 100644 --- a/docs/docs/getting-started/installation/minikube.md +++ b/docs/docs/getting-started/installation/minikube.md @@ -14,6 +14,11 @@ Run Airy on minikube with one command. The goal of this document is to provide an overview of how to run Airy Core on your local machine using [minikube](https://minikube.sigs.k8s.io/). +## Requirements + +- [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) v1.2.0+ +- [Kubectl](https://kubernetes.io/docs/tasks/tools/) + ## Install :::note diff --git a/docs/docs/getting-started/introduction.md b/docs/docs/getting-started/introduction.md index ea4ecebc36..52e28db5f8 100644 --- a/docs/docs/getting-started/introduction.md +++ b/docs/docs/getting-started/introduction.md @@ -23,7 +23,7 @@ Airy Core is an is an **open-source** **streaming** **app framework** to train M - +

Get Airy up and running with one command

diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index 1f1fbae64a..84987c3900 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -66,12 +66,6 @@ module.exports = { position: 'left', to: 'https://airy.co/docs/enterprise/', }, - { - target: '_self', - label: 'Solutions', - position: 'left', - href: 'https://airy.co/solutions', - }, { target: '_self', label: 'Customer Stories', diff --git a/docs/static/img/getting-started/introduction.png b/docs/static/img/getting-started/introduction.png new file mode 100644 index 0000000000..2391349407 Binary files /dev/null and b/docs/static/img/getting-started/introduction.png differ diff --git a/docs/yarn.lock b/docs/yarn.lock index b57c514a24..89bc9054dc 100644 --- a/docs/yarn.lock +++ b/docs/yarn.lock @@ -2620,13 +2620,13 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== -body-parser@1.20.1: - version "1.20.1" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.1.tgz#b1812a8912c195cd371a3ee5e66faa2338a5c668" - integrity sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw== +body-parser@1.20.2: + version "1.20.2" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" + integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== dependencies: bytes "3.1.2" - content-type "~1.0.4" + content-type "~1.0.5" debug "2.6.9" depd "2.0.0" destroy "1.2.0" @@ -2634,7 +2634,7 @@ body-parser@1.20.1: iconv-lite "0.4.24" on-finished "2.4.1" qs "6.11.0" - raw-body "2.5.1" + raw-body "2.5.2" type-is "~1.6.18" unpipe "1.0.0" @@ -3057,7 +3057,7 @@ content-disposition@0.5.4: dependencies: safe-buffer "5.2.1" -content-type@~1.0.4: +content-type@~1.0.4, content-type@~1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.5.tgz#8b773162656d1d1086784c8f23a54ce6d73d7918" integrity sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA== @@ -3072,10 +3072,10 @@ cookie-signature@1.0.6: resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== -cookie@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" - integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== +cookie@0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" + integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== copy-text-to-clipboard@^3.0.1: version "3.1.0" @@ -3713,16 +3713,16 @@ execa@^5.0.0: strip-final-newline "^2.0.0" express@^4.17.3: - version "4.18.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.18.2.tgz#3fabe08296e930c796c19e3c516979386ba9fd59" - integrity sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ== + version "4.19.2" + resolved "https://registry.yarnpkg.com/express/-/express-4.19.2.tgz#e25437827a3aa7f2a827bc8171bbbb664a356465" + integrity sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q== dependencies: accepts "~1.3.8" array-flatten "1.1.1" - body-parser "1.20.1" + body-parser "1.20.2" content-disposition "0.5.4" content-type "~1.0.4" - cookie "0.5.0" + cookie "0.6.0" cookie-signature "1.0.6" debug "2.6.9" depd "2.0.0" @@ -3909,9 +3909,9 @@ flux@^4.0.1: fbjs "^3.0.1" follow-redirects@^1.0.0, follow-redirects@^1.14.7: - version "1.15.2" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" - integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== + version "1.15.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" + integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== fork-ts-checker-webpack-plugin@^6.5.0: version "6.5.3" @@ -5990,10 +5990,10 @@ range-parser@^1.2.1, range-parser@~1.2.1: resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -raw-body@2.5.1: - version "2.5.1" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857" - integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== +raw-body@2.5.2: + version "2.5.2" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.2.tgz#99febd83b90e08975087e8f1f9419a149366b68a" + integrity sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA== dependencies: bytes "3.1.2" http-errors "2.0.0" @@ -6522,19 +6522,19 @@ semver-diff@^3.1.1: semver "^6.3.0" semver@^5.4.1: - version "5.7.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" - integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== + version "5.7.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" - integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== semver@^7.3.2, semver@^7.3.4, semver@^7.3.7, semver@^7.3.8: - version "7.3.8" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.8.tgz#07a78feafb3f7b32347d725e33de7e2a2df67798" - integrity sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A== + version "7.5.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== dependencies: lru-cache "^6.0.0" @@ -7378,9 +7378,9 @@ webpack-bundle-analyzer@^4.5.0: ws "^7.3.1" webpack-dev-middleware@^5.3.1: - version "5.3.3" - resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-5.3.3.tgz#efae67c2793908e7311f1d9b06f2a08dcc97e51f" - integrity sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA== + version "5.3.4" + resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz#eb7b39281cbce10e104eb2b8bf2b63fce49a3517" + integrity sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q== dependencies: colorette "^2.0.10" memfs "^3.4.3" diff --git a/frontend/control-center/src/App.tsx b/frontend/control-center/src/App.tsx index 294aabaae9..4788b33dbb 100644 --- a/frontend/control-center/src/App.tsx +++ b/frontend/control-center/src/App.tsx @@ -17,6 +17,8 @@ import { STREAMS_ROUTE, TOPICS_ROUTE, SCHEMAS_ROUTE, + LLMS_ROUTE, + LLM_CONSUMERS_ROUTE, } from './routes/routes'; import NotFound from './pages/NotFound'; import ConnectorsOutlet from './pages/Connectors/ConnectorsOutlet'; @@ -36,6 +38,8 @@ import {getAppExternalURL} from './services/getAppExternalURL'; import Streams from './pages/Streams'; import Topics from './pages/Topics'; import Schemas from './pages/Schemas'; +import LLMs from './pages/LLMs'; +import LLMConsumers from './pages/LLMConsumers'; const mapDispatchToProps = { getClientConfig, @@ -111,6 +115,8 @@ const App = (props: ConnectedProps) => { } /> } /> + } /> + } /> } /> diff --git a/frontend/control-center/src/actions/streams/index.ts b/frontend/control-center/src/actions/streams/index.ts index 3d47b61e03..2813ef3e8d 100644 --- a/frontend/control-center/src/actions/streams/index.ts +++ b/frontend/control-center/src/actions/streams/index.ts @@ -10,6 +10,7 @@ const SET_TOPIC_INFO = '@@metadata/SET_TOPIC_INFO'; const SET_TOPIC_SCHEMAS = '@@metadata/SET_TOPIC_SCHEMAS'; const SET_STREAMS = '@@metadata/SET_STREAMS'; const SET_SCHEMAS_INFO = '@@metadata/SET_SCHEMAS_INFO'; +const SET_SCHEMAS_VERSIONS = '@@metadata/SET_SCHEMAS_VERSIONS'; const SET_STREAM_INFO = '@@metadata/SET_STREAM_INFO'; const SET_LAST_MESSAGE = '@@metadata/SET_LAST_MESSAGRE'; @@ -68,14 +69,30 @@ export const getTopicInfo = (topicName: string) => async (dispatch: Dispatch async (dispatch: Dispatch) => { - return getData('subjects').then(response => { + return getData('schemas.list').then(response => { + console.log(response); dispatch(setTopicSchemasAction(response)); return Promise.resolve(true); }); }; -export const getSchemaInfo = (topicName: string) => async (dispatch: Dispatch) => { - return getData(`subjects/${topicName}/versions/latest`).then(response => { +export const getSchemaVersions = (topicName: string) => async (dispatch: Dispatch) => { + return getData(`schemas.versions?topicName=${topicName}`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } else { + dispatch(setCurrentSchemaVersionsAction({name: topicName, versions: response})); + } + return Promise.resolve(true); + }); +}; + +export const getSchemaInfo = (topicName: string, version?: string) => async (dispatch: Dispatch) => { + let v = 'latest'; + if (version) { + v = version; + } + return getData(`schemas.info?topicName=${topicName}&version=${v}`).then(response => { if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { return Promise.reject('404 Not Found'); } else { @@ -89,7 +106,7 @@ export const setSchemaSchema = (topicName: string, schema: string) => async () = const body = { schema: JSON.stringify({...JSON.parse(schema)}), }; - return postData(`subjects/${topicName}/versions`, body).then(response => { + return postData(`schemas.update?topicName=${topicName}`, body).then(response => { if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { return Promise.reject('404 Not Found'); } @@ -103,7 +120,7 @@ export const createSchema = (topicName: string, schema: string) => async () => { const body = { schema: JSON.stringify({...JSON.parse(schema)}), }; - return postData(`subjects/${topicName}/versions`, body) + return postData(`schemas.create?topicName=${topicName}`, body) .then(response => { if (response.id) return Promise.resolve(true); if (response.message) return Promise.reject(response.message); @@ -118,7 +135,7 @@ export const checkCompatibilityOfNewSchema = (topicName: string, schema: string, const body = { schema: JSON.stringify({...JSON.parse(schema)}), }; - return postData(`compatibility/subjects/${topicName}/versions/${version}`, body) + return postData(`schemas.compatibility?topicName=${topicName}&version=${version}`, body) .then(response => { if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { return Promise.reject('404 Not Found'); @@ -138,7 +155,7 @@ export const checkCompatibilityOfNewSchema = (topicName: string, schema: string, }; export const deleteSchema = (topicName: string) => async () => { - return deleteData(`subjects/${topicName}`).then(response => { + return deleteData(`schemas.delete?topicName=${topicName}`).then(response => { if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { return Promise.reject('404 Not Found'); } @@ -147,11 +164,7 @@ export const deleteSchema = (topicName: string) => async () => { }; export const getLastMessage = (topicName: string) => async (dispatch: Dispatch) => { - const body = { - ksql: `PRINT '${topicName}' FROM BEGINNING LIMIT 1;`, - streamsProperties: {}, - }; - return postData('query', body).then(response => { + return getData(`schemas.lastMessage?topicName=${topicName}`).then(response => { dispatch(setLastMessage(response)); return Promise.resolve(true); }); @@ -177,11 +190,10 @@ async function postData(url: string, body: any) { const response = await fetch(apiHostUrl + '/' + url, { method: 'POST', headers: { - 'Content-Type': 'application/vnd.schemaregistry.v1+json', + 'Content-Type': 'application/json', }, body: JSON.stringify(body), }); - return response.json(); } @@ -197,6 +209,11 @@ export const setStreamsAction = createAction(SET_STREAMS, (streams: Stream[]) => export const setCurrentSchemaInfoAction = createAction(SET_SCHEMAS_INFO, (topicInfo: Schema) => topicInfo)(); +export const setCurrentSchemaVersionsAction = createAction( + SET_SCHEMAS_VERSIONS, + (topicInfo: {name: string; versions: []}) => topicInfo +)<{name: string; versions: []}>(); + export const setCurrentStreamInfoAction = createAction( SET_STREAM_INFO, (streamInfo: StreamInfo) => streamInfo diff --git a/frontend/control-center/src/components/ChannelAvatar/index.tsx b/frontend/control-center/src/components/ChannelAvatar/index.tsx index 7a26ed24fc..6894e2c241 100644 --- a/frontend/control-center/src/components/ChannelAvatar/index.tsx +++ b/frontend/control-center/src/components/ChannelAvatar/index.tsx @@ -19,6 +19,15 @@ import {ReactComponent as IbmWatsonAssistantAvatar} from 'assets/images/icons/ib import {ReactComponent as RedisAvatar} from 'assets/images/icons/redisLogo.svg'; import {ReactComponent as PostgresAvatar} from 'assets/images/icons/postgresLogo.svg'; import {ReactComponent as FeastAvatar} from 'assets/images/icons/feastLogo.svg'; +import {ReactComponent as MetaAvatar} from 'assets/images/icons/meta.svg'; +import {ReactComponent as OpenaiAvatar} from 'assets/images/icons/openai.svg'; +import {ReactComponent as PineconeAvatar} from 'assets/images/icons/pinecone.svg'; +import {ReactComponent as ChromaAvatar} from 'assets/images/icons/chroma.svg'; +import {ReactComponent as MosaicAvatar} from 'assets/images/icons/mosaic.svg'; +import {ReactComponent as WeaviateAvatar} from 'assets/images/icons/weaviate.svg'; +import {ReactComponent as GmailAvatar} from 'assets/images/icons/gmail.svg'; +import {ReactComponent as SlackAvatar} from 'assets/images/icons/slack.svg'; +import {ReactComponent as FlinkAvatar} from 'assets/images/icons/flink.svg'; import {Channel, Source} from 'model'; import styles from './index.module.scss'; @@ -98,6 +107,37 @@ export const getChannelAvatar = (source: string) => { case Source.feast: case 'Feast': return ; + case Source.faiss: + case 'FAISS': + return ; + case Source.faissConnector: + case 'FAISS connector': + return ; + case Source.llama2: + case 'LLama2': + return ; + case Source.openaiConnector: + case 'OpenAI connector': + return ; + case Source.pineconeConnector: + case 'Pinecone connector': + return ; + case Source.chroma: + case 'Chroma': + return ; + case Source.mosaic: + case 'Mosaic': + return ; + case Source.weaviate: + case 'Weaviate': + return ; + case Source.gmail: + case 'GMail connector': + return ; + case 'Slack connector': + return ; + case 'Flink connector': + return ; default: return ; diff --git a/frontend/control-center/src/components/Sidebar/index.tsx b/frontend/control-center/src/components/Sidebar/index.tsx index 043513f45f..95285c91e9 100644 --- a/frontend/control-center/src/components/Sidebar/index.tsx +++ b/frontend/control-center/src/components/Sidebar/index.tsx @@ -15,6 +15,8 @@ import { STREAMS_ROUTE, TOPICS_ROUTE, SCHEMAS_ROUTE, + LLMS_ROUTE, + LLM_CONSUMERS_ROUTE, } from '../../routes/routes'; import {ReactComponent as ConnectorsIcon} from 'assets/images/icons/gitMerge.svg'; @@ -31,18 +33,18 @@ type SideBarProps = {} & ConnectedProps; const mapStateToProps = (state: StateModel) => ({ version: state.data.config.clusterVersion, components: state.data.config.components, + connectors: state.data.catalog, }); const connector = connect(mapStateToProps); const Sidebar = (props: SideBarProps) => { - const {version, components} = props; + const {version, components, connectors} = props; const componentInfo = useCurrentComponentForSource(Source.airyWebhooks); - const webhooksEnabled = componentInfo.installationStatus === InstallationStatus.installed; const inboxEnabled = components[Source.frontendInbox]?.enabled || false; - const showLine = inboxEnabled || webhooksEnabled; - + const llmsEnabled = connectors['llm-controller']?.installationStatus === 'installed' || false; + const showLine = inboxEnabled || webhooksEnabled || llmsEnabled; const isActive = (route: string) => { return useMatch(`${route}/*`); }; @@ -51,6 +53,9 @@ const Sidebar = (props: SideBarProps) => { const [kafkaSectionOpen, setKafkaSectionOpen] = useState( href.includes(TOPICS_ROUTE) || href.includes(STREAMS_ROUTE) ); + const [llmSectionOpen, setLlmSectionOpen] = useState( + href.includes(LLMS_ROUTE) || href.includes(LLM_CONSUMERS_ROUTE) + ); return ( diff --git a/frontend/control-center/src/pages/Connectors/ConnectorWrapper/index.module.scss b/frontend/control-center/src/pages/Connectors/ConnectorWrapper/index.module.scss index ae262da930..91129c4325 100644 --- a/frontend/control-center/src/pages/Connectors/ConnectorWrapper/index.module.scss +++ b/frontend/control-center/src/pages/Connectors/ConnectorWrapper/index.module.scss @@ -114,9 +114,12 @@ .connectorIcon { display: flex; width: 75px; + height: 75px; margin-right: 15px; svg { + width: 75px; + height: 75px; fill: var(--color-text-contrast); } } diff --git a/frontend/control-center/src/pages/LLMConsumers/EmptyState/index.module.scss b/frontend/control-center/src/pages/LLMConsumers/EmptyState/index.module.scss new file mode 100644 index 0000000000..39684b4f2b --- /dev/null +++ b/frontend/control-center/src/pages/LLMConsumers/EmptyState/index.module.scss @@ -0,0 +1,52 @@ +@import 'assets/scss/colors.scss'; +@import 'assets/scss/fonts.scss'; + +.container { + display: flex; + justify-content: center; + align-items: center; + width: 100%; + height: calc(100% - 88px); +} + +.contentContainer { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + + h1 { + @include font-m; + font-weight: 800; + color: var(--color-text-contrast); + margin: 31px 0; + } + + span { + @include font-base; + color: var(--color-text-gray); + } + + .subscribeButton { + color: var(--color-airy-blue); + &:hover { + cursor: pointer; + text-decoration: underline; + } + } +} + +.iconContainer { + display: flex; + justify-content: center; + align-items: center; + background: var(--color-background-gray); + height: 95px; + width: 105px; +} + +.searchIcon { + height: 45px; + width: 45px; + color: var(--color-airy-blue); +} diff --git a/frontend/control-center/src/pages/LLMConsumers/EmptyState/index.tsx b/frontend/control-center/src/pages/LLMConsumers/EmptyState/index.tsx new file mode 100644 index 0000000000..1a542500b8 --- /dev/null +++ b/frontend/control-center/src/pages/LLMConsumers/EmptyState/index.tsx @@ -0,0 +1,30 @@ +import React, {Dispatch, SetStateAction} from 'react'; +import styles from './index.module.scss'; +import {ReactComponent as SearchIcon} from 'assets/images/icons/search.svg'; +import {useTranslation} from 'react-i18next'; + +type EmptyStateProps = { + createNewLLM: Dispatch>; +}; + +export const EmptyState = (props: EmptyStateProps) => { + const {createNewLLM} = props; + const {t} = useTranslation(); + + return ( +
+
+
+ +
+

{t('noLLMConsumers')}

+ + {t('noLLMConsumersText')} + createNewLLM(true)} className={styles.subscribeButton}> + {t('create') + ' one'} + + +
+
+ ); +}; diff --git a/frontend/control-center/src/pages/LLMConsumers/LLMConsumerItem/index.module.scss b/frontend/control-center/src/pages/LLMConsumers/LLMConsumerItem/index.module.scss new file mode 100644 index 0000000000..b4f9b0d97a --- /dev/null +++ b/frontend/control-center/src/pages/LLMConsumers/LLMConsumerItem/index.module.scss @@ -0,0 +1,47 @@ +@import 'assets/scss/fonts.scss'; +@import 'assets/scss/colors.scss'; + +.container { + display: flex; + flex-direction: row; + height: 50px; + align-items: center; + justify-content: flex-start; + + p { + @include font-base; + color: var(--color-text-contrast); + font-weight: bold; + width: 25%; + } + + p:first-child { + width: 30%; + } + + p:nth-child(4) { + width: 15%; + } +} + +.actionButton { + width: 2%; + outline: none; + cursor: pointer; + border: none; + background: none; + padding: 0; +} + +.actionSVG { + width: 16px; + height: 18px; + path { + fill: var(--color-dark-elements-gray); + } + &:hover { + path { + fill: var(--color-airy-blue); + } + } +} diff --git a/frontend/control-center/src/pages/LLMConsumers/LLMConsumerItem/index.tsx b/frontend/control-center/src/pages/LLMConsumers/LLMConsumerItem/index.tsx new file mode 100644 index 0000000000..b59ce205bc --- /dev/null +++ b/frontend/control-center/src/pages/LLMConsumers/LLMConsumerItem/index.tsx @@ -0,0 +1,38 @@ +import React from 'react'; +import {ReactComponent as TrashIcon} from 'assets/images/icons/trash.svg'; +import {useTranslation} from 'react-i18next'; +import {HttpClientInstance} from '../../../httpClient'; +import styles from './index.module.scss'; +import {NotificationModel} from 'model'; + +type EmptyStateProps = { + item: {name: string; topic: string; status: string; lag: number}; + setNotification: (object: NotificationModel) => void; +}; + +export const LLMConsumerItem = (props: EmptyStateProps) => { + const {item, setNotification} = props; + const {t} = useTranslation(); + + const deleteConsumer = () => { + HttpClientInstance.deleteLLMConsumer({name: item.name}) + .then(() => { + setNotification({show: true, successful: true, text: 'Consumer Deleted'}); + }) + .catch(() => { + setNotification({show: true, successful: false, text: t('errorOccurred')}); + }); + }; + + return ( +
+

{item.name}

+

{item.topic}

+

{item.status}

+

{item.lag}

+ +
+ ); +}; diff --git a/frontend/control-center/src/pages/LLMConsumers/index.module.scss b/frontend/control-center/src/pages/LLMConsumers/index.module.scss new file mode 100644 index 0000000000..8eb971f0db --- /dev/null +++ b/frontend/control-center/src/pages/LLMConsumers/index.module.scss @@ -0,0 +1,136 @@ +@import 'assets/scss/fonts.scss'; +@import 'assets/scss/colors.scss'; +@import 'assets/scss/animations.scss'; + +.llmsWrapper { + background: var(--color-background-white); + border-top-right-radius: 10px; + border-top-left-radius: 10px; + padding: 32px; + margin: 88px 1.5em 0 191px; + height: calc(100vh - 88px); + overflow-y: scroll; + overflow-x: hidden; + width: 100%; +} + +.headlineContainer { + display: flex; + flex-direction: row; + justify-content: space-between; + width: 100%; +} + +.llmsHeadline { + @include font-xl; + font-weight: 900; + letter-spacing: 0; + display: flex; + justify-content: space-between; + color: var(--color-text-contrast); + margin-bottom: 14px; +} + +.llmsHeadlineText { + @include font-xl; + font-weight: 900; +} + +.wrapper { + display: flex; + flex-direction: row; + flex-wrap: wrap; +} + +.listHeader { + display: flex; + flex-direction: row; + height: 50px; + align-items: center; + justify-content: flex-start; + + h2 { + @include font-base; + color: var(--color-text-gray); + font-weight: bold; + width: 25%; + } + + h2:first-child { + width: 30%; + } + + h2:last-child { + width: 10%; + } +} + +.successfullySubscribed { + @include font-base; + color: white; +} + +@keyframes translateYIn { + 0% { + transform: translateY(-50px); + opacity: 0; + } + + 50% { + transform: translateY(16px); + opacity: 1; + } + + 100% { + transform: translateY(-50px); + opacity: 0; + } +} + +.translateYAnimIn { + animation: translateYIn 4s ease-in-out; +} + +.animateIn { + animation: fadeInTranslateXLeft 3000ms ease; +} + +.animateOut { + animation: fadeInTranslateXLeft 3000ms ease; +} + +.llmCreateContainer { + display: flex; + flex-direction: column; + justify-content: space-between; + width: 100%; + margin-top: 32px; + button { + @include font-base; + font-weight: bold; + align-self: center; + color: white; + border-radius: 5px; + border: none; + padding: 8px 16px; + margin-top: 16px; + width: 60%; + cursor: pointer; + transition: all 0.2s ease-in-out; + &:hover { + background: var(--color-background-blue); + color: var(--color-text-contrast); + } + } + label { + margin-bottom: 12px; + } +} + +.dropdownContainer { + button { + border: 1px solid gray; + width: 100%; + color: black; + } +} diff --git a/frontend/control-center/src/pages/LLMConsumers/index.tsx b/frontend/control-center/src/pages/LLMConsumers/index.tsx new file mode 100644 index 0000000000..d4d86eed5d --- /dev/null +++ b/frontend/control-center/src/pages/LLMConsumers/index.tsx @@ -0,0 +1,216 @@ +import React, {useEffect, useState} from 'react'; +import {Dropdown, Input, NotificationComponent} from 'components'; +import {SettingsModal} from 'components/alerts/SettingsModal'; +import {Button} from 'components/cta/Button'; +import {useTranslation} from 'react-i18next'; +import {connect, ConnectedProps} from 'react-redux'; +import {setPageTitle} from '../../services/pageTitle'; +import {NotificationModel} from 'model'; +import {AiryLoader} from 'components/loaders/AiryLoader'; +import {EmptyState} from './EmptyState'; +import {HttpClientInstance} from '../../httpClient'; +import {LLMConsumerItem} from './LLMConsumerItem'; +import {getValidTopics} from '../../selectors'; +import {StateModel} from '../../reducers'; +import styles from './index.module.scss'; +import {getSchemaInfo, getSchemas} from '../../actions'; + +type LLMConsumersProps = {} & ConnectedProps; + +const mapDispatchToProps = { + getSchemas, + getSchemaInfo, +}; + +const mapStateToProps = (state: StateModel) => { + return { + topics: getValidTopics(state), + schemas: state.data.streams.schemas, + }; +}; + +const connector = connect(mapStateToProps, mapDispatchToProps); + +const LLMConsumers = (props: LLMConsumersProps) => { + const {topics, getSchemas} = props; + + const [consumers, setConsumers] = useState([]); + const [notification, setNotification] = useState(null); + const [dataFetched, setDataFetched] = useState(false); + const [showSettingsModal, setShowSettingsModal] = useState(false); + const [name, setName] = useState(''); + const [topic, setTopic] = useState(''); + const [type, setType] = useState(''); + const [textfield, setTextfield] = useState(''); + const [metadataFields, setMetadataFields] = useState(''); + const {t} = useTranslation(); + + useEffect(() => { + setPageTitle('LLM Consumers'); + getSchemas(); + }, []); + + useEffect(() => { + HttpClientInstance.listLLMConsumers() + .then((response: any) => { + setConsumers(response); + setDataFetched(true); + }) + .catch(() => { + handleNotification(true); + }); + }, []); + + const handleNotification = (show: boolean) => { + setNotification({show: show, successful: false, text: t('errorOccurred')}); + }; + + const toggleCreateView = () => { + setShowSettingsModal(!showSettingsModal); + }; + + const createNewLLM = () => { + const metadataFieldsArray = metadataFields.replace(' ', '').split(','); + HttpClientInstance.createLLMConsumer({ + name: name.trim(), + topic: topic.trim(), + textField: textfield.trim(), + metadataFields: metadataFieldsArray, + }) + .then(() => { + setNotification({show: true, successful: true, text: t('llmConsumerCreatedSuccessfully')}); + toggleCreateView(); + setName(''); + setTopic(''); + setTextfield(''); + setMetadataFields(''); + setType(''); + }) + .catch(() => { + handleNotification(true); + }); + }; + + return ( + <> + {' '} + {showSettingsModal && ( + +
+ ) => setName(event.target.value)} + minLength={6} + required={true} + height={32} + fontClass="font-base" + /> +
+ { + setTopic(topic); + // getSchemaInfo(topic).catch(() => { + // getSchemaInfo(topic + '-value'); + // }); + }} + /> +
+ ) => setType(event.target.value)} + minLength={2} + required={true} + height={32} + fontClass="font-base" + /> + ) => setTextfield(event.target.value)} + minLength={2} + required={true} + height={32} + fontClass="font-base" + /> + ) => setMetadataFields(event.target.value)} + minLength={6} + required={true} + height={32} + fontClass="font-base" + /> + +
+ + )} +
+
+
+

LLM Consumers

+
+
+ +
+
+ {consumers?.length === 0 && dataFetched ? ( + + ) : consumers?.length === 0 ? ( + toggleCreateView()} /> + ) : ( + <> +
+

Name

+

Topic

+

Status

+

Lag

+
+
+ {consumers && + consumers.map((consumer: any) => ( + + ))} +
+ {notification?.show && ( + + )} + + )} +
+ + ); +}; + +export default connector(LLMConsumers); diff --git a/frontend/control-center/src/pages/LLMs/EmptyState/index.module.scss b/frontend/control-center/src/pages/LLMs/EmptyState/index.module.scss new file mode 100644 index 0000000000..39684b4f2b --- /dev/null +++ b/frontend/control-center/src/pages/LLMs/EmptyState/index.module.scss @@ -0,0 +1,52 @@ +@import 'assets/scss/colors.scss'; +@import 'assets/scss/fonts.scss'; + +.container { + display: flex; + justify-content: center; + align-items: center; + width: 100%; + height: calc(100% - 88px); +} + +.contentContainer { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + + h1 { + @include font-m; + font-weight: 800; + color: var(--color-text-contrast); + margin: 31px 0; + } + + span { + @include font-base; + color: var(--color-text-gray); + } + + .subscribeButton { + color: var(--color-airy-blue); + &:hover { + cursor: pointer; + text-decoration: underline; + } + } +} + +.iconContainer { + display: flex; + justify-content: center; + align-items: center; + background: var(--color-background-gray); + height: 95px; + width: 105px; +} + +.searchIcon { + height: 45px; + width: 45px; + color: var(--color-airy-blue); +} diff --git a/frontend/control-center/src/pages/LLMs/EmptyState/index.tsx b/frontend/control-center/src/pages/LLMs/EmptyState/index.tsx new file mode 100644 index 0000000000..2f1f7d7373 --- /dev/null +++ b/frontend/control-center/src/pages/LLMs/EmptyState/index.tsx @@ -0,0 +1,30 @@ +import React, {Dispatch, SetStateAction} from 'react'; +import styles from './index.module.scss'; +import {ReactComponent as SearchIcon} from 'assets/images/icons/search.svg'; +import {useTranslation} from 'react-i18next'; + +type EmptyStateProps = { + createNewLLM: Dispatch>; +}; + +export const EmptyState = (props: EmptyStateProps) => { + const {createNewLLM} = props; + const {t} = useTranslation(); + + return ( +
+
+
+ +
+

{t('noLLMs')}

+ + {t('noLLMsText')} + createNewLLM(true)} className={styles.subscribeButton}> + {t('create') + ' one'} + + +
+
+ ); +}; diff --git a/frontend/control-center/src/pages/LLMs/LLMInfoItem/index.module.scss b/frontend/control-center/src/pages/LLMs/LLMInfoItem/index.module.scss new file mode 100644 index 0000000000..331d24a20e --- /dev/null +++ b/frontend/control-center/src/pages/LLMs/LLMInfoItem/index.module.scss @@ -0,0 +1,21 @@ +@import 'assets/scss/fonts.scss'; +@import 'assets/scss/colors.scss'; + +.container { + display: flex; + flex-direction: row; + height: 50px; + align-items: center; + justify-content: flex-start; + + p { + @include font-base; + color: var(--color-text-contrast); + font-weight: bold; + width: 25%; + } + + p:first-child { + width: 30%; + } +} diff --git a/frontend/control-center/src/pages/LLMs/LLMInfoItem/index.tsx b/frontend/control-center/src/pages/LLMs/LLMInfoItem/index.tsx new file mode 100644 index 0000000000..bcea927a86 --- /dev/null +++ b/frontend/control-center/src/pages/LLMs/LLMInfoItem/index.tsx @@ -0,0 +1,18 @@ +import React from 'react'; +import styles from './index.module.scss'; + +type EmptyStateProps = { + item: {llm: string; vectorDatabase: string; llmModel: string}; +}; + +export const LLMInfoItem = (props: EmptyStateProps) => { + const {item} = props; + + return ( +
+

{item.llm}

+

{item.vectorDatabase}

+

{item.llmModel}

+
+ ); +}; diff --git a/frontend/control-center/src/pages/LLMs/index.module.scss b/frontend/control-center/src/pages/LLMs/index.module.scss new file mode 100644 index 0000000000..31195db2da --- /dev/null +++ b/frontend/control-center/src/pages/LLMs/index.module.scss @@ -0,0 +1,106 @@ +@import 'assets/scss/fonts.scss'; +@import 'assets/scss/colors.scss'; +@import 'assets/scss/animations.scss'; + +.webhooksWrapper { + background: var(--color-background-white); + border-top-right-radius: 10px; + border-top-left-radius: 10px; + padding: 32px; + margin: 88px 1.5em 0 191px; + height: calc(100vh - 88px); + overflow-y: scroll; + overflow-x: hidden; + width: 100%; +} + +.headlineContainer { + display: flex; + flex-direction: row; + justify-content: space-between; + width: 100%; +} + +.webhooksHeadline { + @include font-xl; + font-weight: 900; + letter-spacing: 0; + display: flex; + justify-content: space-between; + color: var(--color-text-contrast); + margin-bottom: 14px; +} + +.webhooksHeadlineText { + @include font-xl; + font-weight: 900; +} + +.wrapper { + display: flex; + flex-direction: row; + flex-wrap: wrap; +} + +.listHeader { + display: flex; + flex-direction: row; + height: 50px; + align-items: center; + justify-content: flex-start; + + h2 { + @include font-base; + color: var(--color-text-gray); + font-weight: bold; + width: 25%; + } + + h2:first-child { + width: 30%; + } + + h2:last-child { + width: 10%; + } +} + +.successfullySubscribed { + @include font-base; + color: white; +} + +@keyframes translateYIn { + 0% { + transform: translateY(-50px); + opacity: 0; + } + + 50% { + transform: translateY(16px); + opacity: 1; + } + + 100% { + transform: translateY(-50px); + opacity: 0; + } +} + +.translateYAnimIn { + animation: translateYIn 4s ease-in-out; +} + +.animateIn { + animation: fadeInTranslateXLeft 3000ms ease; +} + +.animateOut { + animation: fadeInTranslateXLeft 3000ms ease; +} + +.embeddingsSection { + color: var(--color-text-contrast); + font-weight: bold; + margin-top: 64px; +} diff --git a/frontend/control-center/src/pages/LLMs/index.tsx b/frontend/control-center/src/pages/LLMs/index.tsx new file mode 100644 index 0000000000..385cdd1cb4 --- /dev/null +++ b/frontend/control-center/src/pages/LLMs/index.tsx @@ -0,0 +1,92 @@ +import React, {useEffect, useState} from 'react'; +import {NotificationComponent} from 'components'; +import {useTranslation} from 'react-i18next'; +import {connect} from 'react-redux'; +import {setPageTitle} from '../../services/pageTitle'; +import {NotificationModel} from 'model'; +import {AiryLoader} from 'components/loaders/AiryLoader'; +import styles from './index.module.scss'; +import {EmptyState} from './EmptyState'; +import {HttpClientInstance} from '../../httpClient'; +import {LLMSStatsPayload} from 'httpclient/src'; +import {LLMInfoItem} from './LLMInfoItem'; + +const mapDispatchToProps = {}; + +const connector = connect(null, mapDispatchToProps); + +const LLMs = () => { + const [llms, setLlms] = useState([]); + const [embeddings, setEmbeddings] = useState(0); + const [notification, setNotification] = useState(null); + const [dataFetched, setDataFetched] = useState(false); + const {t} = useTranslation(); + + useEffect(() => { + setPageTitle('LLMs'); + }, []); + + useEffect(() => { + HttpClientInstance.getLLMInfo() + .then((response: any) => { + setLlms(response); + setDataFetched(true); + }) + .catch(() => { + handleNotification(true); + }); + HttpClientInstance.getLLMStats() + .then((response: LLMSStatsPayload) => { + setEmbeddings(response.embeddings); + }) + .catch(() => { + handleNotification(true); + }); + }, []); + + const handleNotification = (show: boolean) => { + setNotification({show: show, successful: false, text: t('errorOccurred')}); + }; + + const createNewLLM = () => { + console.log('create new LLM'); + }; + + return ( + <> +
+
+
+

LLM Controller

+
+
+ {llms?.length === 0 && dataFetched ? ( + + ) : llms?.length === 0 ? ( + createNewLLM()} /> + ) : ( + <> +
+

LLM Provider

+

Vector Database

+

Model

+
+
{llms && llms.map((llm: any) => )}
+
Embeddings: {embeddings}
+ {notification?.show && ( + + )} + + )} +
+ + ); +}; + +export default connector(LLMs); diff --git a/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/EnrichedSchemaSection/index.tsx b/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/EnrichedSchemaSection/index.tsx new file mode 100644 index 0000000000..7811f24b5c --- /dev/null +++ b/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/EnrichedSchemaSection/index.tsx @@ -0,0 +1,296 @@ +import React, {MutableRefObject, useEffect, useRef, useState} from 'react'; +import MonacoEditor from '@uiw/react-monacoeditor'; +import {calculateHeightOfCodeString, isJSON} from '../../../../../services'; +import {HttpClientInstance} from '../../../../../httpClient'; +import styles from '../index.module.scss'; +import {Button} from 'components'; +import {ConnectedProps, connect} from 'react-redux'; +import {checkCompatibilityOfNewSchema, setSchemaSchema} from '../../../../../actions'; +import {useTranslation} from 'react-i18next'; + +type EnrichedSchemaSectionProps = { + schemaName: string; + code: string; + setCode: (code: string) => void; + setFirstTabSelected: (flag: boolean) => void; + editorMode: string; + wrapperSection: MutableRefObject; + isEditMode: boolean; + setIsEditMode: (flag: boolean) => void; + setErrorMessage: (error: string) => void; + setShowErrorPopUp: (flag: boolean) => void; + version: number; +} & ConnectedProps; + +const mapDispatchToProps = { + setSchemaSchema, + checkCompatibilityOfNewSchema, +}; + +const connector = connect(null, mapDispatchToProps); + +const EnrichedSchemaSection = (props: EnrichedSchemaSectionProps) => { + const { + schemaName, + code, + setCode, + setFirstTabSelected, + editorMode, + wrapperSection, + setSchemaSchema, + isEditMode, + setIsEditMode, + checkCompatibilityOfNewSchema, + setErrorMessage, + setShowErrorPopUp, + version, + } = props; + + const [localCode, setLocalCode] = useState(undefined); + const [hasBeenChanged, setHasBeenChanged] = useState(false); + const codeRef = useRef(null); + const {t} = useTranslation(); + + useEffect(() => { + if (isEnrichmentAvailable(code)) { + setTimeout(() => { + const enriched = localStorage.getItem(schemaName); + if (enriched) { + setLocalCode(enriched); + recalculateContainerHeight(enriched); + } + }, 100); + enrichCode(code); + } else { + wrapperSection.current.style.height = '156px'; + } + }, [code]); + + const resetCodeAndEndEdition = () => { + setIsEditMode(false); + setFirstTabSelected(true); + }; + + const recalculateContainerHeight = (code: string) => { + const basicHeight = 220; + if (wrapperSection && wrapperSection.current) { + wrapperSection.current.style.height = `${calculateHeightOfCodeString(code) + basicHeight}px`; + } else { + wrapperSection.current.style.height = `${basicHeight}px`; + } + if ((wrapperSection.current.style.height.replace('px', '') as number) > 700) { + wrapperSection.current.style.height = '700px'; + } + }; + + const recalculateCodeHeight = (code: string) => { + const codeHeight = calculateHeightOfCodeString(code); + if (codeHeight > 478) { + return 478; + } + return codeHeight; + }; + + const isEnrichmentAvailable = (code: string): boolean => { + let needsEnrichment = false; + const parsedCode = JSON.parse(code); + (parsedCode.fields || []).map(field => { + if (typeof field.type === 'object' && !Array.isArray(field.type)) { + if (!field.type.doc) { + needsEnrichment = true; + } + } else if (!field.doc) { + needsEnrichment = true; + } + }); + return needsEnrichment; + }; + + const enrichCode = async (code: string) => { + let enrichedSchema = localStorage.getItem(schemaName); + + if (!enrichedSchema) { + const enrichedCode = JSON.parse(code); + + // Use map to create an array of promises + const promises = (enrichedCode.fields || []).map(async field => { + console.log(typeof field.type); + if (typeof field.type === 'object' && !Array.isArray(field.type)) { + if (!field.type.doc) { + const doc = await generateDocForField(field); + field.type.doc = doc; + } + } else if (!field.doc) { + const doc = await generateDocForField(field); + field.doc = doc; + } + }); + + // Wait for all promises to resolve + await Promise.all(promises); + + enrichedSchema = JSON.stringify(enrichedCode, null, 2); + localStorage.setItem(schemaName, enrichedSchema); + } + + setLocalCode(enrichedSchema); + recalculateContainerHeight(enrichedSchema); + }; + + const saveEnrichedSchema = () => { + setSchemaSchema(schemaName, JSON.stringify(localCode, null, 2)); + }; + + const checkCompatibility = (_schemaName: string, _code: string, _version: number) => { + checkCompatibilityOfNewSchema(_schemaName, _code, _version) + .then(() => { + setSchemaSchema(_schemaName, _code) + .then(() => { + setCode(localCode); + setHasBeenChanged(false); + }) + .catch((e: string) => { + setIsEditMode(true); + setErrorMessage(e); + setShowErrorPopUp(true); + setTimeout(() => setShowErrorPopUp(false), 5000); + }); + }) + .catch((e: string) => { + if (e.includes('404')) { + checkCompatibility(_schemaName + '-value', _code, _version); + } else { + setIsEditMode(true); + setErrorMessage(e); + setShowErrorPopUp(true); + setTimeout(() => setShowErrorPopUp(false), 5000); + } + }); + }; + + const generateDocForField = async (field: any): Promise => { + try { + const response = await HttpClientInstance.llmQuery({ + query: `This is the payload of a metadata field of a Kafka Schema ${JSON.stringify( + field + )}. A the name of the schema is ${schemaName}. This is the whole schema: ${code}. Give an accurante description of the field, so the users can understand what it is and what it is used for.`, + }); + return response.answer.result; + } catch (error) { + console.error('Error in generateDocForField:', error); + return ''; + } + }; + + return ( + <> +
+
+ + +
+
+ {isEnrichmentAvailable(code) ? ( + <> +
+
+
+ This schema can be automatically enriched with documentation and saved as a new version as follows. +
+ +
+
+
New schema:
+
+ + {hasBeenChanged && ( + + )} +
+
+
+ {localCode && localCode !== '{}' && ( +
+ { + if (value !== code) { + setHasBeenChanged(true); + } else { + setHasBeenChanged(false); + } + }} + onBlur={() => { + setLocalCode(codeRef.current.editor.getModel().getValue()); + }} + options={{ + scrollBeyondLastLine: isEditMode, + readOnly: !isEditMode, + theme: editorMode, + }} + /> +
+ )} + + ) : ( +
+
+
This schema has been enriched already with documentation.
+
+
+ )} + + ); +}; + +export default connector(EnrichedSchemaSection); diff --git a/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/SchemaDescription.tsx b/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/SchemaDescription.tsx index 673d8dda08..36ff6cf9ad 100644 --- a/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/SchemaDescription.tsx +++ b/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/SchemaDescription.tsx @@ -1,14 +1,14 @@ import React, {MutableRefObject, useEffect, useState} from 'react'; -import {getSchemaInfo} from '../../../../actions'; +import {getSchemaInfo, getSchemaVersions} from '../../../../actions'; import {connect, ConnectedProps} from 'react-redux'; import {ErrorPopUp} from 'components'; -import {calculateHeightOfCodeString} from '../../../../services'; import SchemaSection from './SchemaSection'; import styles from './index.module.scss'; -import {MessageSection, lastMessageMock} from './MessageSection'; +import EnrichedSchemaSection from './EnrichedSchemaSection'; const mapDispatchToProps = { getSchemaInfo, + getSchemaVersions, }; const connector = connect(null, mapDispatchToProps); @@ -19,15 +19,17 @@ type SchemaDescriptionProps = { setCode: (code: string) => void; wrapperSection: MutableRefObject; version: number; + versions: string[]; } & ConnectedProps; const SchemaDescription = (props: SchemaDescriptionProps) => { - const {schemaName, code, setCode, getSchemaInfo, wrapperSection, version} = props; + const {schemaName, code, setCode, getSchemaInfo, getSchemaVersions, wrapperSection, version, versions} = props; useEffect(() => { getSchemaInfo(schemaName).catch(() => { getSchemaInfo(schemaName + '-value'); }); + getSchemaVersions(schemaName); }, []); useEffect(() => { @@ -43,26 +45,14 @@ const SchemaDescription = (props: SchemaDescriptionProps) => { const [errorMessage, setErrorMessage] = useState(''); const [editorMode, setEditorMode] = useState(localStorage.getItem('theme') === 'dark' ? 'vs-dark' : 'vs'); - useEffect(() => { - if (firstTabSelected) { - recalculateContainerHeight(code); - } else { - recalculateContainerHeight(lastMessageMock); - } - }, [firstTabSelected, code]); - const setNewSchemaCode = (text: string) => { setCode(text); }; - const recalculateContainerHeight = (code: string) => { - const basicHeight = 50; - const headerHeight = 32; - if (wrapperSection && wrapperSection.current) { - wrapperSection.current.style.height = `${calculateHeightOfCodeString(code) + headerHeight + basicHeight}px`; - } else { - wrapperSection.current.style.height = `${basicHeight}px`; - } + const loadSchemaVersion = (version: string) => { + getSchemaInfo(schemaName, version).catch(() => { + getSchemaInfo(schemaName + '-value', version); + }); }; return ( @@ -76,17 +66,26 @@ const SchemaDescription = (props: SchemaDescriptionProps) => { setIsEditMode={setIsEditMode} setFirstTabSelected={setFirstTabSelected} editorMode={editorMode} - recalculateContainerHeight={recalculateContainerHeight} + wrapperSection={wrapperSection} setErrorMessage={setErrorMessage} setShowErrorPopUp={setShowErrorPopUp} version={version} + versions={versions} + loadSchemaVersion={loadSchemaVersion} /> ) : ( - )} {showErrorPopUp && setShowErrorPopUp(false)} />} diff --git a/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/SchemaSection/index.tsx b/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/SchemaSection/index.tsx index 701b6f136c..fad1bef22d 100644 --- a/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/SchemaSection/index.tsx +++ b/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/SchemaSection/index.tsx @@ -1,11 +1,11 @@ -import React, {useEffect, useRef, useState} from 'react'; +import React, {MutableRefObject, useEffect, useRef, useState} from 'react'; import MonacoEditor from '@uiw/react-monacoeditor'; import {calculateHeightOfCodeString, isJSON} from '../../../../../services'; import {useTranslation} from 'react-i18next'; -import {Button} from 'components'; -import styles from '../index.module.scss'; +import {Button, Dropdown} from 'components'; import {checkCompatibilityOfNewSchema, setSchemaSchema} from '../../../../../actions'; import {ConnectedProps, connect} from 'react-redux'; +import styles from '../index.module.scss'; const mapDispatchToProps = { setSchemaSchema, @@ -22,10 +22,12 @@ type SchemaSectionProps = { setIsEditMode: (flag: boolean) => void; setFirstTabSelected: (flag: boolean) => void; editorMode: string; - recalculateContainerHeight: (text: string) => void; + wrapperSection: MutableRefObject; setErrorMessage: (error: string) => void; setShowErrorPopUp: (flag: boolean) => void; version: number; + loadSchemaVersion: (version: string) => void; + versions: string[]; } & ConnectedProps; const SchemaSection = (props: SchemaSectionProps) => { @@ -37,12 +39,14 @@ const SchemaSection = (props: SchemaSectionProps) => { setIsEditMode, setFirstTabSelected, editorMode, - recalculateContainerHeight, + wrapperSection, checkCompatibilityOfNewSchema, setSchemaSchema, setErrorMessage, setShowErrorPopUp, version, + loadSchemaVersion, + versions, } = props; const [localCode, setLocalCode] = useState(code); @@ -60,6 +64,51 @@ const SchemaSection = (props: SchemaSectionProps) => { setIsEditMode(!isEditMode); }; + const recalculateContainerHeight = (code: string) => { + const basicHeight = 220; + if (wrapperSection && wrapperSection.current) { + wrapperSection.current.style.height = `${calculateHeightOfCodeString(code) + basicHeight}px`; + } else { + wrapperSection.current.style.height = `${basicHeight}px`; + } + if (!isEnrichmentAvailable(code)) { + if ((wrapperSection.current.style.height.replace('px', '') as number) > 600) { + wrapperSection.current.style.height = '600px'; + } + } else { + if ((wrapperSection.current.style.height.replace('px', '') as number) > 700) { + wrapperSection.current.style.height = '700px'; + } + } + }; + + const recalculateCodeHeight = (code: string) => { + const codeHeight = calculateHeightOfCodeString(code); + let height = 478; + if (!isEnrichmentAvailable(code)) { + height = 510; + } + if (codeHeight > height) { + return height; + } + return codeHeight; + }; + + const isEnrichmentAvailable = (code: string): boolean => { + let needsEnrichment = false; + const parsedCode = JSON.parse(code); + (parsedCode.fields || []).map(field => { + if (typeof field.type === 'object' && !Array.isArray(field.type)) { + if (!field.type.doc) { + needsEnrichment = true; + } + } else if (!field.doc) { + needsEnrichment = true; + } + }); + return needsEnrichment; + }; + const checkCompatibility = (_schemaName: string, _code: string, _version: number) => { checkCompatibilityOfNewSchema(_schemaName, _code, _version) .then(() => { @@ -99,14 +148,17 @@ const SchemaSection = (props: SchemaSectionProps) => { > Schema - {/* */} + />
+ {isEnrichmentAvailable(code) && ( + <> +
+
+
+ This schema can be automatically enriched with documentation and saved as a new version. +
+ +
+
Current schema:
+
+ + )} {code && code !== '{}' && ( { diff --git a/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/index.module.scss b/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/index.module.scss index 6edad57e01..27566454c2 100644 --- a/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/index.module.scss +++ b/frontend/control-center/src/pages/Schemas/SchemaItem/SchemaDescription/index.module.scss @@ -59,3 +59,70 @@ flex: auto; padding-bottom: 8px; } + +.enrichmentContainer { + display: flex; + flex-direction: column; + align-items: center; + margin-top: 16px; +} + +.enrichmentText { + padding: 8px; +} + +.enrichmentButton { + @include font-s; + border: none; + background-color: var(--color-code-no-edit); + color: var(--color-text-contrast); + font-weight: 600; + cursor: pointer; +} + +.enrichmentSchemaText { + padding: 8px; + @include font-s; + font-size: 12; + font-weight: 800; +} + +.codeContainer { + height: 100%; + width: 100%; + overflow: auto; + padding: 8px; +} + +.version { + display: flex; + p { + @include font-s; + font-size: 13px; + font-weight: 800; + padding: 8px 0; + margin-left: 16px; + color: var(--color-text-contrast); + } + button { + margin: 4px; + height: 24px; + width: 44px; + font-size: 12px; + align-items: center; + justify-content: center; + } + svg { + margin: 0 0 0 6px; + width: 11px; + height: 11px; + } + div { + font-size: 12px; + } +} + +.infoContainer { + display: flex; + background: transparent; +} diff --git a/frontend/control-center/src/pages/Schemas/SchemaItem/index.tsx b/frontend/control-center/src/pages/Schemas/SchemaItem/index.tsx index 071f3fcc8a..54367e9989 100644 --- a/frontend/control-center/src/pages/Schemas/SchemaItem/index.tsx +++ b/frontend/control-center/src/pages/Schemas/SchemaItem/index.tsx @@ -9,6 +9,7 @@ import SchemaDescription from './SchemaDescription/SchemaDescription'; const mapStateToProps = (state: StateModel) => { return { schemas: state.data.streams.schemas, + schemasVersions: state.data.streams.schemasVersions, }; }; @@ -32,6 +33,7 @@ const SchemaItem = (props: SchemaItemProps) => { addSchemasToSelection, itemSelected, setItemSelected, + schemasVersions, } = props; const [code, setCode] = useState(formatJSON(schemas[schemaName] ? schemas[schemaName].schema : '{}')); @@ -59,6 +61,10 @@ const SchemaItem = (props: SchemaItemProps) => { return 1; }; + const getVersions = (): string[] => { + return schemasVersions[schemaName] || []; + }; + return (
{ setCode={setCode} wrapperSection={wrapperSection} version={getVersion()} + versions={getVersions()} /> )}
diff --git a/frontend/control-center/src/pages/Streams/Creation/index.tsx b/frontend/control-center/src/pages/Streams/Creation/index.tsx index d929c8f36d..378a0bc8da 100644 --- a/frontend/control-center/src/pages/Streams/Creation/index.tsx +++ b/frontend/control-center/src/pages/Streams/Creation/index.tsx @@ -112,7 +112,7 @@ const Creation = (props: ListModeProps) => { fields.forEach(field => { formatedFields.push({ name: field['name'], - newName: field['name'] + '-' + topic, + newName: field['name'] + '_' + topic, }); }); return formatedFields; @@ -203,6 +203,7 @@ const Creation = (props: ListModeProps) => { backgroundColor: 'transparent', border: '1px solid gray', borderRadius: '10px', + pointerEvents: 'none', }} /> @@ -232,6 +233,7 @@ const Creation = (props: ListModeProps) => { backgroundColor: 'transparent', border: '1px solid gray', borderRadius: '10px', + pointerEvents: 'none', }} /> @@ -281,6 +283,7 @@ const Creation = (props: ListModeProps) => { backgroundColor: 'transparent', border: '1px solid gray', borderRadius: '10px', + pointerEvents: 'none', }} /> @@ -312,6 +315,7 @@ const Creation = (props: ListModeProps) => { backgroundColor: 'transparent', border: '1px solid gray', borderRadius: '10px', + pointerEvents: 'none', }} /> @@ -394,6 +398,7 @@ const Creation = (props: ListModeProps) => { backgroundColor: 'transparent', border: '1px solid gray', borderRadius: '10px', + pointerEvents: 'none', }} /> @@ -437,6 +442,7 @@ const Creation = (props: ListModeProps) => { backgroundColor: 'transparent', border: '1px solid gray', borderRadius: '10px', + pointerEvents: 'none', }} /> diff --git a/frontend/control-center/src/pages/Topics/TopicItem/TopicDescription/TopicDescription.tsx b/frontend/control-center/src/pages/Topics/TopicItem/TopicDescription/TopicDescription.tsx index eeec77ad27..1a7d8f8dc9 100644 --- a/frontend/control-center/src/pages/Topics/TopicItem/TopicDescription/TopicDescription.tsx +++ b/frontend/control-center/src/pages/Topics/TopicItem/TopicDescription/TopicDescription.tsx @@ -35,14 +35,23 @@ const TopicDescription = (props: TopicDescriptionProps) => { if (!_expanded) { wrapperSection.current.style.height = `${basicHeight}px`; } else { - console.log('in'); if (wrapperSection && wrapperSection.current) { - console.log('in in'); wrapperSection.current.style.height = `${calculateHeightOfCodeString(code) + 100 + headerHeight}px`; } else { wrapperSection.current.style.height = `${basicHeight}px`; } + if ((wrapperSection.current.style.height.replace('px', '') as number) > 700) { + wrapperSection.current.style.height = '640px'; + } + } + }; + + const recalculateCodeHeight = (code: string) => { + const codeHeight = calculateHeightOfCodeString(code); + if (codeHeight > 478) { + return 478; } + return codeHeight; }; const calculateRetentionTime = (_jsonCode): string => { @@ -89,7 +98,7 @@ const TopicDescription = (props: TopicDescriptionProps) => { {expanded && ( { export const formatJSON = (jsonString: string): string => { if (jsonString) { - return JSON.stringify(JSON.parse(jsonString), null, 4); + return JSON.stringify(JSON.parse(jsonString), null, 2); } return ''; }; diff --git a/frontend/inbox/src/pages/Inbox/MessageInput/index.tsx b/frontend/inbox/src/pages/Inbox/MessageInput/index.tsx index f17606e7bd..4dba3375c5 100644 --- a/frontend/inbox/src/pages/Inbox/MessageInput/index.tsx +++ b/frontend/inbox/src/pages/Inbox/MessageInput/index.tsx @@ -286,6 +286,9 @@ const MessageInput = (props: Props) => { case Source.whatsapp: message.message = outboundMapper.getTextPayload(input); break; + case Source.airyCopilot: + message.message = outboundMapper.getTextPayload(input); + break; } sendMessages(message).then(() => { diff --git a/go.mod b/go.mod index 37b2cb1748..5e80f333dd 100644 --- a/go.mod +++ b/go.mod @@ -6,26 +6,11 @@ go 1.18 // Automatically generated by running //tools/update-deps require ( - github.com/Masterminds/sprig v2.22.0+incompatible - github.com/TwinProduction/go-color v1.0.0 github.com/airyhq/airy/lib/go/httpclient v0.0.0-20230426122014-b3add0488aa1 - github.com/aws/aws-sdk-go v1.44.21 - github.com/aws/aws-sdk-go-v2/config v1.15.7 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.44.0 - github.com/aws/aws-sdk-go-v2/service/eks v1.21.1 - github.com/aws/aws-sdk-go-v2/service/iam v1.18.5 + github.com/airyhq/airy/lib/go/payloads v0.0.0-20230426122014-b3add0488aa1 + github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/gorilla/mux v1.8.0 - github.com/kr/pretty v0.3.0 - github.com/mitchellh/go-homedir v1.1.0 - github.com/spf13/cobra v1.5.0 - github.com/spf13/viper v1.11.0 - github.com/thanhpk/randstr v1.0.4 - github.com/txn2/txeh v1.3.0 - goji.io v2.0.2+incompatible - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 - gopkg.in/segmentio/analytics-go.v3 v3.1.0 - gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.24.2 k8s.io/apimachinery v0.24.2 k8s.io/client-go v0.24.2 @@ -33,38 +18,6 @@ require ( ) require ( - cloud.google.com/go v0.102.0 // indirect - cloud.google.com/go/compute v1.7.0 // indirect - cloud.google.com/go/iam v0.3.0 // indirect - cloud.google.com/go/storage v1.22.1 // indirect - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/BurntSushi/toml v1.1.0 // indirect - github.com/MakeNowJust/heredoc v1.0.0 // indirect - github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver v1.5.0 // indirect - github.com/Masterminds/semver/v3 v3.1.1 // indirect - github.com/Masterminds/sprig/v3 v3.2.2 // indirect - github.com/Masterminds/squirrel v1.5.3 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/airyhq/airy/lib/go/payloads v0.0.0-20230426122014-b3add0488aa1 // indirect - github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/aws/aws-sdk-go-v2 v1.16.4 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.2 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.11 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.12 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.16.6 // indirect - github.com/aws/smithy-go v1.11.2 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect - github.com/containerd/containerd v1.6.6 // indirect - github.com/cyphar/filepath-securejoin v0.2.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/go-logr/logr v1.2.3 // indirect @@ -72,9 +25,10 @@ require ( github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.21.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/iancoleman/strcase v0.2.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kr/pretty v0.3.0 // indirect @@ -83,25 +37,18 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/rogpeppe/go-internal v1.8.1 // indirect - github.com/stretchr/testify v1.8.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - helm.sh/helm/v3 v3.9.0 // indirect - k8s.io/apiextensions-apiserver v0.24.2 // indirect - k8s.io/apiserver v0.24.2 // indirect - k8s.io/cli-runtime v0.24.2 // indirect - k8s.io/component-base v0.24.2 // indirect - k8s.io/helm v2.17.0+incompatible // indirect k8s.io/klog/v2 v2.70.0 // indirect k8s.io/kube-openapi v0.0.0-20220627174259-011e075b9cb8 // indirect k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect diff --git a/go.sum b/go.sum index e5914c5737..a071abe33f 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,7 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= @@ -45,24 +46,21 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/hcsshim v0.9.4 h1:mnUj0ivWy6UzbB1uLFqKR6F+ZyiDc7j4iGgHTpO+5+I= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/TwinProduction/go-color v1.0.0/go.mod h1:5hWpSyT+mmKPjCwPNEruBW5Dkbs/2PwOuU468ntEXNQ= github.com/airyhq/airy/lib/go/httpclient v0.0.0-20230426122014-b3add0488aa1 h1:oNPI7IFTuBW+EO2XLgYUye3wXx3Jf7MekUlN6lFq3xg= github.com/airyhq/airy/lib/go/httpclient v0.0.0-20230426122014-b3add0488aa1/go.mod h1:DmqTLti3ZCZ3PZcNNNL0C9oNzNSnAbrigBDwN66Ogaw= github.com/airyhq/airy/lib/go/payloads v0.0.0-20230426122014-b3add0488aa1 h1:nAKH/lr1NPPGQIPB7A69TINQCBQQH8pkYzeGgpseD4M= github.com/airyhq/airy/lib/go/payloads v0.0.0-20230426122014-b3add0488aa1/go.mod h1:QokQEnLolj8aluiKF11Sv52K88M6fO6XVxPtvdG4dR8= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -73,10 +71,18 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 h1:icCHutJouWlQREayFwCc7lxDAhws08td+W3/gdqgZts= +github.com/confluentinc/confluent-kafka-go/v2 v2.3.0/go.mod h1:/VTy8iEpe6mD9pkCH5BhijlUl8ulUXymKv1Qig5Rgb8= +github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= +github.com/containerd/containerd v1.6.8 h1:h4dOFDwzHmqFEP754PgfgTeVXFnLiRc6kiqC7tplDJs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/docker v20.10.17+incompatible h1:JYCuMrWaVNophQTOrMMoSwudOVEfcegoZZrleKc1xwE= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -121,7 +127,11 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -147,8 +157,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= @@ -187,18 +198,9 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -207,8 +209,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -232,6 +232,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -239,12 +240,16 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs= +github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -261,8 +266,12 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= +github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -272,21 +281,20 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/testcontainers/testcontainers-go v0.14.0 h1:h0D5GaYG9mhOWr2qHdEKDXpkce/VlvaYOCzTRi6UBi8= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -301,6 +309,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -384,8 +393,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -457,12 +466,12 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -472,8 +481,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -605,6 +614,7 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 h1:0BOZf6qNozI3pkN3fJLwNubheHJYHhMh91GRFOWWK08= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -625,6 +635,7 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -638,8 +649,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -678,8 +689,6 @@ k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2U k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA= k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/helm v2.17.0+incompatible/go.mod h1:LZzlS4LQBHfciFOurYBFkCMTaZ0D1l+p0teMg7TSULI= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= diff --git a/infrastructure/controller/pkg/endpoints/BUILD b/infrastructure/controller/pkg/endpoints/BUILD index fdcbf72037..c8705df73c 100644 --- a/infrastructure/controller/pkg/endpoints/BUILD +++ b/infrastructure/controller/pkg/endpoints/BUILD @@ -12,6 +12,7 @@ go_library( "components_list.go", "components_update.go", "cors.go", + "proxy.go", "server.go", "services.go", ], diff --git a/infrastructure/controller/pkg/endpoints/proxy.go b/infrastructure/controller/pkg/endpoints/proxy.go new file mode 100644 index 0000000000..8dc0f90afc --- /dev/null +++ b/infrastructure/controller/pkg/endpoints/proxy.go @@ -0,0 +1,58 @@ +package endpoints + +import ( + "log" + "net/http" + "net/http/httputil" + "net/url" + "strings" + + "k8s.io/client-go/kubernetes" + "k8s.io/helm/cmd/helm/search" +) + +type proxyTarget struct { + name string + url string + stripUri string +} + +type KafkaSubjects struct { + ClientSet *kubernetes.Clientset + Namespace string + Index *search.Index +} + +type KafkaTopics struct { + ClientSet *kubernetes.Clientset + Namespace string + Index *search.Index +} + +func (s *KafkaSubjects) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var proxyUpstream = proxyTarget{"subjects", "http://schema-registry:8081/subjects", "/kafka/subjects"} + proxyRequest(w, r, proxyUpstream) +} + +func (s *KafkaTopics) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var proxyUpstream = proxyTarget{"subjects", "http://schema-registry:8082/topics", "/kafka/topics"} + proxyRequest(w, r, proxyUpstream) +} + +func proxyRequest(w http.ResponseWriter, r *http.Request, proxyUpstream proxyTarget) { + target, err := url.Parse(proxyUpstream.url) + if err != nil { + log.Printf("Error parsing target URL: %v\n", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + + proxy := httputil.NewSingleHostReverseProxy(target) + r.URL.Host = target.Host + r.URL.Scheme = target.Scheme + r.Header.Set("X-Forwarded-Host", r.Header.Get("Host")) + r.Host = target.Host + r.URL.Path = strings.TrimPrefix(r.URL.Path, proxyUpstream.stripUri) + + proxy.ServeHTTP(w, r) +} diff --git a/infrastructure/controller/pkg/endpoints/server.go b/infrastructure/controller/pkg/endpoints/server.go index 27e24c426c..b3c68443fe 100644 --- a/infrastructure/controller/pkg/endpoints/server.go +++ b/infrastructure/controller/pkg/endpoints/server.go @@ -70,6 +70,12 @@ func Serve(clientSet *kubernetes.Clientset, namespace string, kubeConfig *rest.C componentsList := ComponentsList{ClientSet: clientSet, Namespace: namespace, Index: helmIndex} r.Handle("/components.list", &componentsList) + kafkaSubjects := KafkaSubjects{ClientSet: clientSet, Namespace: namespace, Index: helmIndex} + r.Handle("/kafka/subjects", &kafkaSubjects) + + kafkaTopics := KafkaTopics{ClientSet: clientSet, Namespace: namespace, Index: helmIndex} + r.Handle("/kafka/topics", &kafkaTopics) + log.Fatal(http.ListenAndServe(":8080", r)) } diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/templates/service.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/templates/service.yaml index 7d17a5bd92..52c14795e7 100644 --- a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/templates/service.yaml +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/templates/service.yaml @@ -9,6 +9,10 @@ spec: ports: - name: schema-registry port: {{ .Values.servicePort }} +{{- if .Values.restEnabled }} + - name: rest-api + port: {{ .Values.restPort }} +{{- end}} selector: app: schema-registry release: {{ .Release.Name }} diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/values.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/values.yaml index 3f89f02c9a..d95dfcad3f 100644 --- a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/values.yaml +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/values.yaml @@ -9,4 +9,4 @@ kafka: bootstrapServers: kafka-headless:9092 minBrokers: 1 resources: {} -restEnabled: false \ No newline at end of file +restEnabled: true \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml b/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml index 75cab9b8dc..3af2e13445 100644 --- a/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml @@ -60,6 +60,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.admin.resources | indent 10 }} initContainers: @@ -77,3 +82,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml b/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml index 362428ee8b..a461864d1d 100644 --- a/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.communication.resources | indent 10 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml b/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml index fd865b890b..363863fafc 100644 --- a/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml @@ -81,6 +81,11 @@ spec: initialDelaySeconds: 60 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.components.installer.resources | indent 10 }} initContainers: @@ -102,4 +107,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts - +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/api-websocket/deployment.yaml b/infrastructure/helm-chart/templates/components/api-websocket/deployment.yaml index 6739cfe3c0..c4d95e0c57 100644 --- a/infrastructure/helm-chart/templates/components/api-websocket/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/api-websocket/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.websocket.resources | indent 10 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/ingress.yaml b/infrastructure/helm-chart/templates/components/ingress.yaml index b4ddcdb3db..5a65593cd7 100644 --- a/infrastructure/helm-chart/templates/components/ingress.yaml +++ b/infrastructure/helm-chart/templates/components/ingress.yaml @@ -82,7 +82,7 @@ spec: pathType: Prefix backend: service: - name: api-admin + name: airy-controller port: number: 80 - path: /kafka diff --git a/infrastructure/helm-chart/templates/components/unread-counter/deployment.yaml b/infrastructure/helm-chart/templates/components/unread-counter/deployment.yaml index c8650099f1..4b44212560 100644 --- a/infrastructure/helm-chart/templates/components/unread-counter/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/unread-counter/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.unread_counter.resources | indent 10 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/config/kafka.yaml b/infrastructure/helm-chart/templates/config/kafka.yaml index a00933d800..23897e7214 100644 --- a/infrastructure/helm-chart/templates/config/kafka.yaml +++ b/infrastructure/helm-chart/templates/config/kafka.yaml @@ -13,3 +13,4 @@ data: {{- end }} KAFKA_SCHEMA_REGISTRY_URL: {{ .Values.config.kafka.schemaRegistryUrl }} KAFKA_COMMIT_INTERVAL_MS: "{{ .Values.config.kafka.commitInterval }}" + KAFKA_KEY_TRUST_SECRET: {{ .Values.config.kafka.keyTrustSecret }} diff --git a/infrastructure/helm-chart/values.yaml b/infrastructure/helm-chart/values.yaml index e4f6afa182..982a736b54 100644 --- a/infrastructure/helm-chart/values.yaml +++ b/infrastructure/helm-chart/values.yaml @@ -13,6 +13,7 @@ config: brokers: "kafka-headless:9092" zookeepers: "zookeeper:2181" authJaas: "" + keyTrustSecret: "" minimumReplicas: 1 schemaRegistryUrl: "http://schema-registry:8081" commitInterval: 1000 diff --git a/infrastructure/lib/go/k8s/handler/go.mod b/infrastructure/lib/go/k8s/handler/go.mod index 5621294cbf..065623955d 100644 --- a/infrastructure/lib/go/k8s/handler/go.mod +++ b/infrastructure/lib/go/k8s/handler/go.mod @@ -34,7 +34,7 @@ require ( golang.org/x/text v0.7.0 // indirect golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/infrastructure/lib/go/k8s/handler/go.sum b/infrastructure/lib/go/k8s/handler/go.sum index a7e95d05a5..29fb9918cd 100644 --- a/infrastructure/lib/go/k8s/handler/go.sum +++ b/infrastructure/lib/go/k8s/handler/go.sum @@ -605,8 +605,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/lib/java/kafka/core/src/main/java/co/airy/kafka/core/KafkaConsumerWrapper.java b/lib/java/kafka/core/src/main/java/co/airy/kafka/core/KafkaConsumerWrapper.java index b5739d9503..ca75dc902b 100644 --- a/lib/java/kafka/core/src/main/java/co/airy/kafka/core/KafkaConsumerWrapper.java +++ b/lib/java/kafka/core/src/main/java/co/airy/kafka/core/KafkaConsumerWrapper.java @@ -12,6 +12,10 @@ import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.Properties; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SslConfigs; +import java.util.HashMap; + public class KafkaConsumerWrapper { @@ -22,6 +26,7 @@ public class KafkaConsumerWrapper { private KafkaConsumer consumer; private String jaasConfig; + private String kafkaKeyTrustSecret; public KafkaConsumerWrapper(final String brokers, final String schemaRegistryUrl) { props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers); @@ -33,13 +38,22 @@ public KafkaConsumerWrapper(final String brokers, final String schemaRegistryUrl props.put(KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG, true); } - public KafkaConsumerWrapper withAuthJaas(String jaasConfig) { + public KafkaConsumerWrapper withAuthJaas(String jaasConfig, String kafkaKeyTrustSecret) { this.jaasConfig = jaasConfig; if(jaasConfig != null) { props.put("security.protocol", "SASL_SSL"); props.put("sasl.mechanism", "PLAIN"); props.put("sasl.jaas.config", jaasConfig); } + if (kafkaKeyTrustSecret != null) { + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); + props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.truststore.jks"); + props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "PKCS12"); + props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.keystore.p12"); + props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, kafkaKeyTrustSecret); + } return this; } diff --git a/lib/java/kafka/streams/src/main/java/co/airy/kafka/streams/KafkaStreamsWrapper.java b/lib/java/kafka/streams/src/main/java/co/airy/kafka/streams/KafkaStreamsWrapper.java index f44f5075aa..18ca499cab 100644 --- a/lib/java/kafka/streams/src/main/java/co/airy/kafka/streams/KafkaStreamsWrapper.java +++ b/lib/java/kafka/streams/src/main/java/co/airy/kafka/streams/KafkaStreamsWrapper.java @@ -30,6 +30,10 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SslConfigs; +import java.util.HashMap; + public class KafkaStreamsWrapper { private static final Logger log = AiryLoggerFactory.getLogger(KafkaStreamsWrapper.class); @@ -37,6 +41,7 @@ public class KafkaStreamsWrapper { private final String brokers; private final String schemaRegistryUrl; private String jaasConfig; + private String kafkaKeyTrustSecret; private long commitIntervalInMs; private long suppressIntervalInMs; private int threadCount; @@ -70,8 +75,9 @@ public KafkaStreamsWrapper(final String brokers, final String schemaRegistryUrl) healthCheckRunnerThread = new HealthCheckRunner(testMode); } - public KafkaStreamsWrapper withJaasConfig(String jaasConfig) { + public KafkaStreamsWrapper withJaasConfig(String jaasConfig, String kafkaKeyTrustSecret) { this.jaasConfig = jaasConfig; + this.kafkaKeyTrustSecret = kafkaKeyTrustSecret; return this; } @@ -227,6 +233,17 @@ public synchronized void start(final Topology topology, final String appId) thro props.put("sasl.jaas.config", jaasConfig); } + if (this.kafkaKeyTrustSecret != null) { + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); + props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.truststore.jks"); + props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "PKCS12"); + props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.keystore.p12"); + props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, kafkaKeyTrustSecret); + } + + props.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, this.maxRequestSize); props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, this.fetchMaxBytes); diff --git a/lib/java/spring/kafka/core/src/main/java/co/airy/spring/kafka/core/KafkaCoreConfig.java b/lib/java/spring/kafka/core/src/main/java/co/airy/spring/kafka/core/KafkaCoreConfig.java index 0cd6c28122..f8488e2f6f 100644 --- a/lib/java/spring/kafka/core/src/main/java/co/airy/spring/kafka/core/KafkaCoreConfig.java +++ b/lib/java/spring/kafka/core/src/main/java/co/airy/spring/kafka/core/KafkaCoreConfig.java @@ -13,6 +13,9 @@ import org.springframework.context.annotation.Scope; import java.util.Properties; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SslConfigs; +import java.util.HashMap; @Configuration @PropertySource("classpath:kafka-core.properties") @@ -21,7 +24,8 @@ public class KafkaCoreConfig { @Lazy @Scope("prototype") public KafkaProducer kafkaProducer(@Value("${kafka.brokers}") final String brokers, @Value("${kafka.schema-registry-url}") final String schemaRegistryUrl, - @Value("${AUTH_JAAS:#{null}}") final String jaasConfig) { + @Value("${AUTH_JAAS:#{null}}") final String jaasConfig, + @Value("${KAFKA_KEY_TRUST_SECRET:#{null}}") final String kafkaKeyTrustSecret) { final Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers); @@ -37,6 +41,16 @@ public KafkaProducer kafkaProducer(@Value("${kafka.brokers}") final props.put("sasl.jaas.config", jaasConfig); } + if (kafkaKeyTrustSecret != null) { + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); + props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.truststore.jks"); + props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "PKCS12"); + props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.keystore.p12"); + props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, kafkaKeyTrustSecret); + } + return new KafkaProducer<>(props); } @@ -44,8 +58,9 @@ public KafkaProducer kafkaProducer(@Value("${kafka.brokers}") final @Lazy @Scope("prototype") public KafkaConsumerWrapper kafkaConsumer(@Value("${kafka.brokers}") final String brokers, @Value("${kafka.schema-registry-url}") final String schemaRegistryUrl, - @Value("${kafka.sasl.jaas.config:#{null}}") final String jaasConfig) { + @Value("${kafka.sasl.jaas.config:#{null}}") final String jaasConfig, + @Value("${KAFKA_KEY_TRUST_SECRET:#{null}}") final String kafkaKeyTrustSecret) { return new KafkaConsumerWrapper(brokers, schemaRegistryUrl) - .withAuthJaas(jaasConfig); + .withAuthJaas(jaasConfig, kafkaKeyTrustSecret); } } diff --git a/lib/java/spring/kafka/streams/src/main/java/co/airy/spring/kafka/streams/KafkaStreamsConfig.java b/lib/java/spring/kafka/streams/src/main/java/co/airy/spring/kafka/streams/KafkaStreamsConfig.java index f9995abfbd..217f2f9a6a 100644 --- a/lib/java/spring/kafka/streams/src/main/java/co/airy/spring/kafka/streams/KafkaStreamsConfig.java +++ b/lib/java/spring/kafka/streams/src/main/java/co/airy/spring/kafka/streams/KafkaStreamsConfig.java @@ -30,6 +30,9 @@ public class KafkaStreamsConfig { @Value("${AUTH_JAAS:#{null}}") private String jaasConfig; + @Value("${KAFKA_KEY_TRUST_SECRET:#{null}}") + private String kafkaKeyTrustSecret; + @Value("${kafka.rpc-port:0}") private int rpcPort; @@ -68,7 +71,7 @@ public class KafkaStreamsConfig { public KafkaStreamsWrapper airyKafkaStreams(@Value("${kafka.brokers}") final String brokers, @Value("${kafka.schema-registry-url}") final String schemaRegistryUrl) { return new KafkaStreamsWrapper(brokers, schemaRegistryUrl) .withCommitIntervalInMs(commitIntervalMs) - .withJaasConfig(jaasConfig) + .withJaasConfig(jaasConfig, kafkaKeyTrustSecret) .withSuppressIntervalInMs(suppressIntervalMs) .withThreadCount(streamsThreadCount) .withAppServerHost(rpcHost) diff --git a/lib/typescript/assets/images/icons/chroma.svg b/lib/typescript/assets/images/icons/chroma.svg new file mode 100644 index 0000000000..c35b980d0f --- /dev/null +++ b/lib/typescript/assets/images/icons/chroma.svg @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lib/typescript/assets/images/icons/flink.svg b/lib/typescript/assets/images/icons/flink.svg new file mode 100644 index 0000000000..fb63bd841d --- /dev/null +++ b/lib/typescript/assets/images/icons/flink.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/typescript/assets/images/icons/gmail.svg b/lib/typescript/assets/images/icons/gmail.svg new file mode 100644 index 0000000000..40b7175c11 --- /dev/null +++ b/lib/typescript/assets/images/icons/gmail.svg @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/lib/typescript/assets/images/icons/meta.svg b/lib/typescript/assets/images/icons/meta.svg new file mode 100644 index 0000000000..72316de7c8 --- /dev/null +++ b/lib/typescript/assets/images/icons/meta.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/typescript/assets/images/icons/mosaic.svg b/lib/typescript/assets/images/icons/mosaic.svg new file mode 100644 index 0000000000..0d1b995091 --- /dev/null +++ b/lib/typescript/assets/images/icons/mosaic.svg @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lib/typescript/assets/images/icons/openai.svg b/lib/typescript/assets/images/icons/openai.svg new file mode 100644 index 0000000000..3b4eff961f --- /dev/null +++ b/lib/typescript/assets/images/icons/openai.svg @@ -0,0 +1,2 @@ + +OpenAI icon \ No newline at end of file diff --git a/lib/typescript/assets/images/icons/pinecone.svg b/lib/typescript/assets/images/icons/pinecone.svg new file mode 100644 index 0000000000..2b61be24e2 --- /dev/null +++ b/lib/typescript/assets/images/icons/pinecone.svg @@ -0,0 +1,162 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/lib/typescript/assets/images/icons/slack.svg b/lib/typescript/assets/images/icons/slack.svg new file mode 100644 index 0000000000..1b18e66c1e --- /dev/null +++ b/lib/typescript/assets/images/icons/slack.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/typescript/assets/images/icons/weaviate.svg b/lib/typescript/assets/images/icons/weaviate.svg new file mode 100644 index 0000000000..7eac0ba99a --- /dev/null +++ b/lib/typescript/assets/images/icons/weaviate.svg @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lib/typescript/components/alerts/SettingsModal/ModalHeader.module.scss b/lib/typescript/components/alerts/SettingsModal/ModalHeader.module.scss index acb3117dfe..924c4834d1 100644 --- a/lib/typescript/components/alerts/SettingsModal/ModalHeader.module.scss +++ b/lib/typescript/components/alerts/SettingsModal/ModalHeader.module.scss @@ -30,5 +30,5 @@ } .closeIcon { - width: 18px; + width: 12px; } diff --git a/lib/typescript/components/general/FilterBar/FilterDropdown/index.tsx b/lib/typescript/components/general/FilterBar/FilterDropdown/index.tsx index c81007e33e..376fc885e1 100644 --- a/lib/typescript/components/general/FilterBar/FilterDropdown/index.tsx +++ b/lib/typescript/components/general/FilterBar/FilterDropdown/index.tsx @@ -10,6 +10,7 @@ const filterAttributes = [ 'Customer Service', 'Machine Learning', 'Conversational AI', + 'LLM', 'Databases', 'Marketing', 'Storage', diff --git a/lib/typescript/httpclient/src/client.ts b/lib/typescript/httpclient/src/client.ts index 9bc283d51c..dc67b2852b 100644 --- a/lib/typescript/httpclient/src/client.ts +++ b/lib/typescript/httpclient/src/client.ts @@ -37,6 +37,14 @@ import { GetStreamInfoPayload, DeleteStreamPayload, CreateStreamPayload, + LLMConsumersListPayload, + LLMInfoPayload, + LLMSStatsPayload, + LLMConsumersCreateRequestPayload, + LLMConsumersCreateResponsePayload, + LLMConsumersDeletePayload, + LLMQueryRequestPayload, + LLMQueryResponsePayload, } from './payload'; import { listChannelsDef, @@ -88,6 +96,12 @@ import { getStreamInfoDef, deleteStreamDef, createStreamDef, + llmConsumersListDef, + llmInfoDef, + llmStatsDef, + llmConsumersCreateDef, + llmConsumersDeleteDef, + llmQueryDef, } from './endpoints'; import 'isomorphic-fetch'; import FormData from 'form-data'; @@ -319,6 +333,20 @@ export class HttpClient { public createStream = this.getRequest(createStreamDef); + public getLLMInfo = this.getRequest(llmInfoDef); + + public getLLMStats = this.getRequest(llmStatsDef); + + public listLLMConsumers = this.getRequest(llmConsumersListDef); + + public deleteLLMConsumer = this.getRequest(llmConsumersDeleteDef); + + public createLLMConsumer = this.getRequest( + llmConsumersCreateDef + ); + + public llmQuery = this.getRequest(llmQueryDef); + private getRequest({endpoint, mapRequest, mapResponse}: EndpointDefinition): ApiRequest { return async (requestPayload: K) => { endpoint = typeof endpoint === 'string' ? endpoint : endpoint(requestPayload); diff --git a/lib/typescript/httpclient/src/endpoints/index.ts b/lib/typescript/httpclient/src/endpoints/index.ts index dae00d107f..8d847bac6d 100644 --- a/lib/typescript/httpclient/src/endpoints/index.ts +++ b/lib/typescript/httpclient/src/endpoints/index.ts @@ -47,3 +47,9 @@ export * from './getStreams'; export * from './getStreamInfo'; export * from './deleteStream'; export * from './createStream'; +export * from './llmConsumersCreate'; +export * from './llmConsumersList'; +export * from './llmStats'; +export * from './llmInfo'; +export * from './llmConsumersDelete'; +export * from './llmQuery'; diff --git a/lib/typescript/httpclient/src/endpoints/llmConsumersCreate.ts b/lib/typescript/httpclient/src/endpoints/llmConsumersCreate.ts new file mode 100644 index 0000000000..fe6fb40d57 --- /dev/null +++ b/lib/typescript/httpclient/src/endpoints/llmConsumersCreate.ts @@ -0,0 +1,11 @@ +import {LLMConsumersCreateRequestPayload} from '../payload'; + +export const llmConsumersCreateDef = { + endpoint: 'llm-consumers.create', + mapRequest: (request: LLMConsumersCreateRequestPayload) => ({ + name: request.name, + topic: request.topic, + textField: request.textField, + metadataFields: request.metadataFields, + }), +}; diff --git a/lib/typescript/httpclient/src/endpoints/llmConsumersDelete.ts b/lib/typescript/httpclient/src/endpoints/llmConsumersDelete.ts new file mode 100644 index 0000000000..c1a4add48d --- /dev/null +++ b/lib/typescript/httpclient/src/endpoints/llmConsumersDelete.ts @@ -0,0 +1,6 @@ +import camelcaseKeys from 'camelcase-keys'; + +export const llmConsumersDeleteDef = { + endpoint: 'llm-consumers.delete', + mapResponse: response => camelcaseKeys(response), +}; diff --git a/lib/typescript/httpclient/src/endpoints/llmConsumersList.ts b/lib/typescript/httpclient/src/endpoints/llmConsumersList.ts new file mode 100644 index 0000000000..bcb4349afa --- /dev/null +++ b/lib/typescript/httpclient/src/endpoints/llmConsumersList.ts @@ -0,0 +1,6 @@ +import camelcaseKeys from 'camelcase-keys'; + +export const llmConsumersListDef = { + endpoint: 'llm-consumers.list', + mapResponse: response => camelcaseKeys(response), +}; diff --git a/lib/typescript/httpclient/src/endpoints/llmInfo.ts b/lib/typescript/httpclient/src/endpoints/llmInfo.ts new file mode 100644 index 0000000000..4e21de899e --- /dev/null +++ b/lib/typescript/httpclient/src/endpoints/llmInfo.ts @@ -0,0 +1,6 @@ +import camelcaseKeys from 'camelcase-keys'; + +export const llmInfoDef = { + endpoint: 'llm.info', + mapResponse: response => camelcaseKeys(response), +}; diff --git a/lib/typescript/httpclient/src/endpoints/llmQuery.ts b/lib/typescript/httpclient/src/endpoints/llmQuery.ts new file mode 100644 index 0000000000..37f2bba2d9 --- /dev/null +++ b/lib/typescript/httpclient/src/endpoints/llmQuery.ts @@ -0,0 +1,6 @@ +import camelcaseKeys from 'camelcase-keys'; + +export const llmQueryDef = { + endpoint: 'llm.query', + mapResponse: response => camelcaseKeys(response), +}; diff --git a/lib/typescript/httpclient/src/endpoints/llmStats.ts b/lib/typescript/httpclient/src/endpoints/llmStats.ts new file mode 100644 index 0000000000..245a8526d1 --- /dev/null +++ b/lib/typescript/httpclient/src/endpoints/llmStats.ts @@ -0,0 +1,6 @@ +import camelcaseKeys from 'camelcase-keys'; + +export const llmStatsDef = { + endpoint: 'llm.stats', + mapResponse: response => camelcaseKeys(response), +}; diff --git a/lib/typescript/httpclient/src/payload/LLMConsumerCreateRequestPayload.ts b/lib/typescript/httpclient/src/payload/LLMConsumerCreateRequestPayload.ts new file mode 100644 index 0000000000..acc1327d33 --- /dev/null +++ b/lib/typescript/httpclient/src/payload/LLMConsumerCreateRequestPayload.ts @@ -0,0 +1,6 @@ +export interface LLMConsumersCreateRequestPayload { + name: string; + topic: string; + textField: string; + metadataFields: string[]; +} diff --git a/lib/typescript/httpclient/src/payload/LLMConsumerCreateResponsePayload.ts b/lib/typescript/httpclient/src/payload/LLMConsumerCreateResponsePayload.ts new file mode 100644 index 0000000000..5afcaf4bf4 --- /dev/null +++ b/lib/typescript/httpclient/src/payload/LLMConsumerCreateResponsePayload.ts @@ -0,0 +1,3 @@ +export interface LLMConsumersCreateResponsePayload { + status: string; +} diff --git a/lib/typescript/httpclient/src/payload/LLMConsumerListPayload.ts b/lib/typescript/httpclient/src/payload/LLMConsumerListPayload.ts new file mode 100644 index 0000000000..f96c6697dc --- /dev/null +++ b/lib/typescript/httpclient/src/payload/LLMConsumerListPayload.ts @@ -0,0 +1,10 @@ +interface LLMConsumer { + lag: number; + name: string; + status: string; + topic: string; +} + +export interface LLMConsumersListPayload { + data: LLMConsumer[]; +} diff --git a/lib/typescript/httpclient/src/payload/LLMConsumersDeletePayload.ts b/lib/typescript/httpclient/src/payload/LLMConsumersDeletePayload.ts new file mode 100644 index 0000000000..0c937869ae --- /dev/null +++ b/lib/typescript/httpclient/src/payload/LLMConsumersDeletePayload.ts @@ -0,0 +1,3 @@ +export interface LLMConsumersDeletePayload { + name: string; +} diff --git a/lib/typescript/httpclient/src/payload/LLMInfoPayload.ts b/lib/typescript/httpclient/src/payload/LLMInfoPayload.ts new file mode 100644 index 0000000000..b053b5096d --- /dev/null +++ b/lib/typescript/httpclient/src/payload/LLMInfoPayload.ts @@ -0,0 +1,8 @@ +interface LLMInfo { + llm: string; + llm_model: string; + vectorDatabase: string; +} +export interface LLMInfoPayload { + data: LLMInfo[]; +} diff --git a/lib/typescript/httpclient/src/payload/LLMQueryRequestPayload.ts b/lib/typescript/httpclient/src/payload/LLMQueryRequestPayload.ts new file mode 100644 index 0000000000..cb7b9ec7c4 --- /dev/null +++ b/lib/typescript/httpclient/src/payload/LLMQueryRequestPayload.ts @@ -0,0 +1,3 @@ +export interface LLMQueryRequestPayload { + query: string; +} diff --git a/lib/typescript/httpclient/src/payload/LLMQueryResponsePayload.ts b/lib/typescript/httpclient/src/payload/LLMQueryResponsePayload.ts new file mode 100644 index 0000000000..6780fd96a0 --- /dev/null +++ b/lib/typescript/httpclient/src/payload/LLMQueryResponsePayload.ts @@ -0,0 +1,6 @@ +export interface LLMQueryResponsePayload { + answer: { + query: string; + result: string; + }; +} diff --git a/lib/typescript/httpclient/src/payload/LLMStatsPayload.ts b/lib/typescript/httpclient/src/payload/LLMStatsPayload.ts new file mode 100644 index 0000000000..c0dda67a57 --- /dev/null +++ b/lib/typescript/httpclient/src/payload/LLMStatsPayload.ts @@ -0,0 +1,3 @@ +export interface LLMSStatsPayload { + embeddings: number; +} diff --git a/lib/typescript/httpclient/src/payload/index.ts b/lib/typescript/httpclient/src/payload/index.ts index e1696f2dd1..94aa9a34aa 100644 --- a/lib/typescript/httpclient/src/payload/index.ts +++ b/lib/typescript/httpclient/src/payload/index.ts @@ -37,3 +37,11 @@ export * from './CreateTopicPayload'; export * from './GetStreamInfoPayload'; export * from './DeleteStreamPayload'; export * from './CreateStreamPayload'; +export * from './LLMConsumerCreateRequestPayload'; +export * from './LLMConsumerCreateResponsePayload'; +export * from './LLMConsumerListPayload'; +export * from './LLMInfoPayload'; +export * from './LLMStatsPayload'; +export * from './LLMConsumersDeletePayload'; +export * from './LLMQueryResponsePayload'; +export * from './LLMQueryRequestPayload'; diff --git a/lib/typescript/model/Connectors.ts b/lib/typescript/model/Connectors.ts index 68bb055a5d..1c9b3d8e96 100644 --- a/lib/typescript/model/Connectors.ts +++ b/lib/typescript/model/Connectors.ts @@ -24,6 +24,7 @@ export enum ConnectorName { sourcesViber = 'sources-viber', rasaConnector = 'rasa-connector', zendenkConnector = 'zendesk-connector', + faissConnector = 'faiss-connector', } export enum InstallationStatus { diff --git a/lib/typescript/model/Source.ts b/lib/typescript/model/Source.ts index 2945297ff8..fffb775a01 100644 --- a/lib/typescript/model/Source.ts +++ b/lib/typescript/model/Source.ts @@ -22,22 +22,32 @@ export enum Source { redis = 'redis', postgresql = 'postgresql', feast = 'feast', + faiss = 'faiss', + faissConnector = 'faissConnector', + llama2 = 'llama2', + openaiConnector = 'openaiConnector', + pineconeConnector = 'pineconeConnector', + chroma = 'chroma', + mosaic = 'mosaic', + weaviate = 'weaviate', + gmail = 'gmail', amazons3 = 'amazons3', amazonLexV2 = 'amazonLexV2', integrationSourceApi = 'integrationSourceApi', + airyCopilot = 'copilot', } export enum SourceApps { redis = 'redis', postgresql = 'postgresql', - feast = 'feast', + faiss = 'faiss', } export const isApp = (source: string): boolean => { switch (source) { case SourceApps.postgresql: case SourceApps.redis: - case SourceApps.feast: + case SourceApps.faiss: return true; } return false; @@ -48,6 +58,7 @@ export const isAiryComponent = (source: string): boolean => { case Source.airyContacts: case Source.airyMobile: case Source.airyWebhooks: + case Source.airyCopilot: case Source.integrationSourceApi: return true; } diff --git a/lib/typescript/model/Streams.ts b/lib/typescript/model/Streams.ts index aee9ed5127..749f887f95 100644 --- a/lib/typescript/model/Streams.ts +++ b/lib/typescript/model/Streams.ts @@ -8,6 +8,9 @@ export interface Streams { schemas: { [topicName: string]: Schema; }; + schemasVersions: { + [topicName: string]: string[]; + }; streamsInfo: { [streamName: string]: StreamInfo; }; diff --git a/lib/typescript/render/outbound/index.ts b/lib/typescript/render/outbound/index.ts index d23c11016b..97d87bd220 100644 --- a/lib/typescript/render/outbound/index.ts +++ b/lib/typescript/render/outbound/index.ts @@ -13,6 +13,7 @@ export const getOutboundMapper = (source: string) => { case 'google': return new GoogleMapper(); case 'chatplugin': + case 'copilot': return new ChatpluginMapper(); case 'twilio.sms': case 'twilio.whatsapp': diff --git a/lib/typescript/translations/translations.ts b/lib/typescript/translations/translations.ts index b3d5271e50..938f8780d1 100644 --- a/lib/typescript/translations/translations.ts +++ b/lib/typescript/translations/translations.ts @@ -4,6 +4,22 @@ import {initReactI18next} from 'react-i18next'; const resources = { en: { translation: { + //Components + chromaDescription: 'Chroma is a vector database that helps LLM applications to have long term memory.', + faissDescription: + 'FAISS is a vector database that allows developers to quickly group and search embeddings of documents that are similar to each other.', + llama2Description: + 'LLama2 is a large language model that can be downloaded and deployed in a self-hosted environment.', + mosaicDescription: + 'Mosaic is an AI tool to easily train and deploy generative AI models on your data, in your secure environment.', + pineconeDescription: + 'Pinecone is a fully managed vector database solution, used to store and search vector embeddings.', + weaviateDescription: + 'Weaviate is an open-source vector database that allows storing object and vector embeddings from various ML-models.', + openaiDescription: 'The OpenAI connector is a bidirectional connector that connects Airy to the OpenAI LLM.', + gmailDescription: + 'The GMail connector is a bidirectional connector for sending and receiving e-mail messages through the Google Mail API.', + //Input Component fieldCannotBeEmpty: 'This field cannot be empty.', invalidURL: 'The URL is invalid', @@ -592,10 +608,38 @@ const resources = { noWebhooksText: `You don't have any Webhooks installed, please `, customHeader: 'Customer Header', signKey: 'Sign key', + noLLMs: 'No LLMs Found', + noLLMsText: `You don't have any LLMs installed, please `, + noLLMConsumers: 'No LLM Consumers Found', + noLLMConsumersText: `You don't have any LLM Consumers installed, please `, + llmConsumerNameExplanation: 'The name of the LLM Consumer', + llmConsumerTopicNameExplanation: 'The name of the topic to which the LLM Consumer is subscribed', + llmConsumerTextFieldExplanation: 'The text field of the LLM Consumer', + llmConsumerMetadataFieldsExplanation: 'The metadata fields of the LLM Consumer', + llmConsumerCreatedSuccessfully: 'LLM Consumer created successfully', + llmConsumerTypeExplanation: 'The type serialization of the LLM Consumer', }, }, de: { translation: { + //Components + chromaDescription: + 'Chroma ist eine Vektordatenbank, die LLM-Anwendungen dabei hilft, über ein Langzeitgedächtnis zu verfügen.', + faissDescription: + 'FAISS ist eine Vektordatenbank, die es Entwicklern ermöglicht, Einbettungen von einander ähnlichen Dokumenten schnell zu gruppieren und zu durchsuchen.', + llama2Description: + 'LLama2 ist ein großes Sprachmodell, das heruntergeladen und in einer selbst gehosteten Umgebung bereitgestellt werden kann.', + mosaicDescription: + 'Mosaik ist ein KI-Tool zum einfachen Trainieren und Bereitstellen generativer KI-Modelle auf Ihren Daten in Ihrer sicheren Umgebung.', + pineconeDescription: + 'Pinecone ist eine vollständig verwaltete Vektordatenbanklösung, die zum Speichern und Durchsuchen von Vektoreinbettungen verwendet wird.', + weaviateDescription: + 'Weaviate ist eine Open-Source-Vektordatenbank, die das Speichern von Objekt- und Vektoreinbettungen aus verschiedenen ML-Modellen ermöglicht.', + openaiDescription: + 'Der OpenAI-Connector ist ein bidirektionaler Connector, der Airy mit dem OpenAI LLM verbindet.', + gmailDescription: + 'Der GMail-Connector ist ein bidirektionaler Connector zum Senden und Empfangen von E-Mail-Nachrichten über die Google Mail-API.', + //Input Component fieldCannotBeEmpty: 'Dieses Feld kann nicht leer sein.', invalidURL: 'Die URL ist ungültig', @@ -1195,10 +1239,37 @@ const resources = { noWebhooksText: 'Sie haben keine Webhooks installiert, bitte ', customHeader: 'Kundenkopfzeile', signKey: 'Signierschlüssel', + noLLMs: 'Keine LLMs gefunden', + noLLMsText: `Sie haben keine LLMs installiert, bitte `, + noLLMConsumers: 'Keine LLM Consumers gefunden', + noLLMConsumersText: `Sie haben keine LLM Consumers installiert, bitte `, + llmConsumerNameExplanation: 'Der Name des LLM Consumer', + llmConsumerTopicNameExplanation: 'Der Name des LLM Consumer Topics', + llmConsumerTextFieldExplanation: 'Das Textfeld des LLM Consumer', + llmConsumerMetadataFieldsExplanation: 'Die Metadatenfelder des LLM Consumer', + llmConsumerCreatedSuccessfully: 'LLM Consumer erfolgreich erstellt', + llmConsumerTypeExplanation: 'Der Typ des LLM Consumer', }, }, fr: { translation: { + //Components + chromaDescription: + "Chroma est une base de données vectorielle qui aide les applications LLM à disposer d'une mémoire à long terme.", + faissDescription: + 'FAISS est une base de données vectorielle qui permet aux développeurs de regrouper et de rechercher rapidement des intégrations de documents similaires les uns aux autres.', + llama2Description: + 'LLama2 est un grand modèle de langage qui peut être téléchargé et déployé dans un environnement auto-hébergé.', + mosaicDescription: + "Mosaic est un outil d'IA permettant de former et de déployer facilement des modèles d'IA génératifs sur vos données, dans votre environnement sécurisé.", + pineconeDescription: + 'Pinecone est une solution de base de données vectorielle entièrement gérée, utilisée pour stocker et rechercher des intégrations vectorielles.', + weaviateDescription: + "Weaviate est une base de données vectorielles open source qui permet de stocker des intégrations d'objets et de vecteurs à partir de divers modèles ML.", + openaiDescription: 'Le connecteur OpenAI est un connecteur bidirectionnel qui connecte Airy au OpenAI LLM.', + gmailDescription: + "Le connecteur GMail est un connecteur bidirectionnel permettant d'envoyer et de recevoir des messages électroniques via l'API Google Mail.", + //Input Component fieldCannotBeEmpty: 'Ce champ ne peut pas être vide.', invalidURL: 'URL non valide', @@ -1792,10 +1863,37 @@ const resources = { noWebhooksText: `Vous n'avez pas de Webhooks installé, veuillez vous `, customHeader: 'En-tête du client', signKey: 'Touche de signature', + noLLMs: 'Pas de LLMs trouvés', + noLLMsText: `Vous n'avez pas de LLMs installé, veuillez vous `, + noLLMConsumers: 'Pas de LLM Consumers trouvés', + noLLMConsumersText: `Vous n'avez pas de LLM Consumers installé, veuillez vous `, + llmConsumerNameExplanation: 'Le nom du LLM Consumer', + llmConsumerTopicNameExplanation: 'Le nom du topic du LLM Consumer', + llmConsumerTextFieldExplanation: 'Le champ de texte du LLM Consumer', + llmConsumerMetadataFieldsExplanation: 'Les champs de métadonnées du LLM Consumer', + llmConsumerCreatedSuccessfully: 'LLM Consumer créé avec succès', + llmConsumerTypeExplanation: 'Le type du LLM Consumer', }, }, es: { translation: { + //Components + chromaDescription: + 'Chroma es una base de datos vectorial que ayuda a las aplicaciones LLM a tener memoria a largo plazo.', + faissDescription: + 'FAISS es una base de datos vectorial que permite a los desarrolladores agrupar y buscar rápidamente incrustaciones de documentos que son similares entre sí.', + llama2Description: + 'LLama2 es un modelo de lenguaje grande que se puede descargar e implementar en un entorno autohospedado.', + mosaicDescription: + 'Mosaic es una herramienta de IA para entrenar e implementar fácilmente modelos de IA generativos en sus datos, en su entorno seguro.', + pineconeDescription: + 'Pinecone es una solución de base de datos de vectores totalmente administrada, que se utiliza para almacenar y buscar incrustaciones de vectores.', + weaviateDescription: + 'Weaviate es una base de datos vectorial de código abierto que permite almacenar incrustaciones de objetos y vectores de varios modelos ML.', + openaiDescription: 'El conector OpenAI es un conector bidireccional que conecta Airy con OpenAI LLM.', + gmailDescription: + 'El conector GMail es un conector bidireccional para enviar y recibir mensajes de correo electrónico a través de la API de Google Mail.', + //Input Component fieldCannotBeEmpty: 'El campo de texto no puede estar vacío.', invalidURL: 'La URL no es válida', @@ -2392,6 +2490,16 @@ const resources = { noWebhooksText: 'No tiene instalado ningún Webhooks, por favor, ', customHeader: 'Cabecera del cliente', signKey: 'Clave de la firma', + noLLMs: 'No se han encontrado LLMs', + noLLMsText: `No tiene instalado ningún LLMs, por favor, `, + noLLMConsumers: 'No se han encontrado LLM Consumers', + noLLMConsumersText: `No tiene instalado ningún LLM Consumer, por favor, `, + llmConsumerNameExplanation: 'El nombre del LLM Consumer', + llmConsumerTopicNameExplanation: 'El nombre del topic del LLM Consumer', + llmConsumerTextFieldExplanation: 'El campo de texto del LLM Consumer', + llmConsumerMetadataFieldsExplanation: 'Los campos de metadatos del LLM Consumer', + llmConsumerCreatedSuccessfully: 'LLM Consumer creado con éxito', + llmConsumerTypeExplanation: 'El tipo de serialización del LLM Consumer', }, }, }; diff --git a/package.json b/package.json index 346b78c5e7..8a76eb9747 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@babel/preset-typescript": "^7.18.6", "@bazel/typescript": "5.6.0", "@hot-loader/react-dom": "^17.0.2", - "@svgr/plugin-svgo": "^6.5.0", + "@svgr/plugin-svgo": "^8.1.0", "@svgr/webpack": "^6.5.0", "@testing-library/dom": "^8.19.0", "@testing-library/react": "13.4.0", @@ -86,7 +86,7 @@ "react-hot-loader": "^4.13.0", "react-test-renderer": "^18.2.0", "sass": "^1.55.0", - "sass-loader": "^13.0.2", + "sass-loader": "^13.3.2", "style-loader": "^3.3.1", "terser-webpack-plugin": "^5.3.6", "typescript": "^4.8.4", diff --git a/yarn.lock b/yarn.lock index 8818dbae48..54c2331402 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3,9 +3,9 @@ "@adobe/css-tools@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.0.1.tgz#b38b444ad3aa5fedbb15f2f746dcd934226a12dd" - integrity sha512-+u76oB43nOHrF4DDWRLWDCtci7f3QJoEBigemIdIeTi1ODqjx6Tad9NCVnPRwewWlKkVab5PlK8DCtPTyX7S8g== + version "4.3.1" + resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.3.1.tgz#abfccb8ca78075a2b6187345c26243c1a0842f28" + integrity sha512-/62yikz7NLScCGAAST5SHdnjaDJQBDq0M2muyRTpf2VQhw6StBg2ALiu73zSJQ4fMVLA+0uBhBHAle7Wg+2kSg== "@ampproject/remapping@^2.2.0": version "2.2.0" @@ -1672,7 +1672,7 @@ "@svgr/hast-util-to-babel-ast" "^6.5.1" svg-parser "^2.0.4" -"@svgr/plugin-svgo@^6.5.0", "@svgr/plugin-svgo@^6.5.1": +"@svgr/plugin-svgo@^6.5.1": version "6.5.1" resolved "https://registry.yarnpkg.com/@svgr/plugin-svgo/-/plugin-svgo-6.5.1.tgz#0f91910e988fc0b842f88e0960c2862e022abe84" integrity sha512-omvZKf8ixP9z6GWgwbtmP9qQMPX4ODXi+wzbVZgomNFsUIlHA1sf4fThdwTWSsZGgvGAG6yE+b/F5gWUkcZ/iQ== @@ -1681,6 +1681,15 @@ deepmerge "^4.2.2" svgo "^2.8.0" +"@svgr/plugin-svgo@^8.1.0": + version "8.1.0" + resolved "https://registry.yarnpkg.com/@svgr/plugin-svgo/-/plugin-svgo-8.1.0.tgz#b115b7b967b564f89ac58feae89b88c3decd0f00" + integrity sha512-Ywtl837OGO9pTLIN/onoWLmDQ4zFUycI1g76vuKGEz6evR/ZTJlJuz3G/fIkb6OVBJ2g0o6CGJzaEjfmEo3AHA== + dependencies: + cosmiconfig "^8.1.3" + deepmerge "^4.3.1" + svgo "^3.0.2" + "@svgr/webpack@^6.5.0": version "6.5.1" resolved "https://registry.yarnpkg.com/@svgr/webpack/-/webpack-6.5.1.tgz#ecf027814fc1cb2decc29dc92f39c3cf691e40e8" @@ -2633,6 +2642,11 @@ argparse@^1.0.7: dependencies: sprintf-js "~1.0.2" +argparse@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" + integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== + aria-query@^5.0.0: version "5.1.3" resolved "https://registry.yarnpkg.com/aria-query/-/aria-query-5.1.3.tgz#19db27cd101152773631396f7a95a3b58c22c35e" @@ -2913,13 +2927,13 @@ bluebird@^3.7.2: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== -body-parser@1.20.1: - version "1.20.1" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.1.tgz#b1812a8912c195cd371a3ee5e66faa2338a5c668" - integrity sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw== +body-parser@1.20.2: + version "1.20.2" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" + integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== dependencies: bytes "3.1.2" - content-type "~1.0.4" + content-type "~1.0.5" debug "2.6.9" depd "2.0.0" destroy "1.2.0" @@ -2927,7 +2941,7 @@ body-parser@1.20.1: iconv-lite "0.4.24" on-finished "2.4.1" qs "6.11.0" - raw-body "2.5.1" + raw-body "2.5.2" type-is "~1.6.18" unpipe "1.0.0" @@ -3351,6 +3365,11 @@ content-type@~1.0.4: resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA== +content-type@~1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.5.tgz#8b773162656d1d1086784c8f23a54ce6d73d7918" + integrity sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA== + convert-source-map@^1.4.0, convert-source-map@^1.6.0, convert-source-map@^1.7.0: version "1.9.0" resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.9.0.tgz#7faae62353fb4213366d0ca98358d22e8368b05f" @@ -3361,10 +3380,10 @@ cookie-signature@1.0.6: resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== -cookie@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" - integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== +cookie@0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" + integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== copy-webpack-plugin@^11.0.0: version "11.0.0" @@ -3421,6 +3440,16 @@ cosmiconfig@^7.0.1: path-type "^4.0.0" yaml "^1.10.0" +cosmiconfig@^8.1.3: + version "8.2.0" + resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-8.2.0.tgz#f7d17c56a590856cd1e7cee98734dca272b0d8fd" + integrity sha512-3rTMnFJA1tCOPwRxtgF4wd7Ab2qvDbL8jX+3smjIbS4HlZBagTlpERbdN7iAbWlrfxE3M8c27kTwTawQ7st+OQ== + dependencies: + import-fresh "^3.2.1" + js-yaml "^4.1.0" + parse-json "^5.0.0" + path-type "^4.0.0" + create-require@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" @@ -3460,6 +3489,17 @@ css-select@^4.1.3: domutils "^2.8.0" nth-check "^2.0.1" +css-select@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/css-select/-/css-select-5.1.0.tgz#b8ebd6554c3637ccc76688804ad3f6a6fdaea8a6" + integrity sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg== + dependencies: + boolbase "^1.0.0" + css-what "^6.1.0" + domhandler "^5.0.2" + domutils "^3.0.1" + nth-check "^2.0.1" + css-tree@^1.1.2, css-tree@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.1.3.tgz#eb4870fb6fd7707327ec95c2ff2ab09b5e8db91d" @@ -3468,7 +3508,23 @@ css-tree@^1.1.2, css-tree@^1.1.3: mdn-data "2.0.14" source-map "^0.6.1" -css-what@^6.0.1: +css-tree@^2.2.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-2.3.1.tgz#10264ce1e5442e8572fc82fbe490644ff54b5c20" + integrity sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw== + dependencies: + mdn-data "2.0.30" + source-map-js "^1.0.1" + +css-tree@~2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-2.2.1.tgz#36115d382d60afd271e377f9c5f67d02bd48c032" + integrity sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA== + dependencies: + mdn-data "2.0.28" + source-map-js "^1.0.1" + +css-what@^6.0.1, css-what@^6.1.0: version "6.1.0" resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4" integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw== @@ -3490,6 +3546,13 @@ csso@^4.2.0: dependencies: css-tree "^1.1.2" +csso@^5.0.5: + version "5.0.5" + resolved "https://registry.yarnpkg.com/csso/-/csso-5.0.5.tgz#f9b7fe6cc6ac0b7d90781bb16d5e9874303e2ca6" + integrity sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ== + dependencies: + css-tree "~2.2.0" + cssom@^0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.5.0.tgz#d254fa92cd8b6fbd83811b9fbaed34663cc17c36" @@ -3645,10 +3708,10 @@ deep-is@^0.1.3, deep-is@~0.1.3: resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== -deepmerge@^4.2.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955" - integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg== +deepmerge@^4.2.2, deepmerge@^4.3.1: + version "4.3.1" + resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" + integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== default-gateway@^6.0.3: version "6.0.3" @@ -3779,12 +3842,21 @@ dom-serializer@^1.0.1: domhandler "^4.2.0" entities "^2.0.0" +dom-serializer@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-2.0.0.tgz#e41b802e1eedf9f6cae183ce5e622d789d7d8e53" + integrity sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.2" + entities "^4.2.0" + dom-walk@^0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/dom-walk/-/dom-walk-0.1.2.tgz#0c548bef048f4d1f2a97249002236060daa3fd84" integrity sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w== -domelementtype@^2.0.1, domelementtype@^2.2.0: +domelementtype@^2.0.1, domelementtype@^2.2.0, domelementtype@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d" integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== @@ -3803,6 +3875,13 @@ domhandler@^4.0.0, domhandler@^4.2.0, domhandler@^4.3.1: dependencies: domelementtype "^2.2.0" +domhandler@^5.0.2, domhandler@^5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31" + integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w== + dependencies: + domelementtype "^2.3.0" + domutils@^2.5.2, domutils@^2.8.0: version "2.8.0" resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135" @@ -3812,6 +3891,15 @@ domutils@^2.5.2, domutils@^2.8.0: domelementtype "^2.2.0" domhandler "^4.2.0" +domutils@^3.0.1: + version "3.1.0" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.1.0.tgz#c47f551278d3dc4b0b1ab8cbb42d751a6f0d824e" + integrity sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA== + dependencies: + dom-serializer "^2.0.0" + domelementtype "^2.3.0" + domhandler "^5.0.3" + dot-case@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/dot-case/-/dot-case-3.0.4.tgz#9b2b670d00a431667a8a75ba29cd1b98809ce751" @@ -3914,6 +4002,11 @@ entities@^2.0.0: resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== +entities@^4.2.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48" + integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw== + entities@^4.4.0: version "4.4.0" resolved "https://registry.yarnpkg.com/entities/-/entities-4.4.0.tgz#97bdaba170339446495e653cfd2db78962900174" @@ -4271,16 +4364,16 @@ expect@^29.0.0: jest-util "^29.2.1" express@^4.17.3: - version "4.18.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.18.2.tgz#3fabe08296e930c796c19e3c516979386ba9fd59" - integrity sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ== + version "4.19.2" + resolved "https://registry.yarnpkg.com/express/-/express-4.19.2.tgz#e25437827a3aa7f2a827bc8171bbbb664a356465" + integrity sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q== dependencies: accepts "~1.3.8" array-flatten "1.1.1" - body-parser "1.20.1" + body-parser "1.20.2" content-disposition "0.5.4" content-type "~1.0.4" - cookie "0.5.0" + cookie "0.6.0" cookie-signature "1.0.6" debug "2.6.9" depd "2.0.0" @@ -4473,9 +4566,9 @@ flatted@^3.1.0: integrity sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ== follow-redirects@^1.0.0: - version "1.15.2" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" - integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== + version "1.15.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" + integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== for-each@^0.3.3: version "0.3.3" @@ -5995,6 +6088,13 @@ js-yaml@^3.13.1: argparse "^1.0.7" esprima "^4.0.0" +js-yaml@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== + dependencies: + argparse "^2.0.1" + jsbn@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" @@ -6126,11 +6226,6 @@ kleur@^4.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-4.1.5.tgz#95106101795f7050c6c650f350c683febddb1780" integrity sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ== -klona@^2.0.4: - version "2.0.5" - resolved "https://registry.yarnpkg.com/klona/-/klona-2.0.5.tgz#d166574d90076395d9963aa7a928fabb8d76afbc" - integrity sha512-pJiBpiXMbt7dkzXe8Ghj/u4FfXOOa98fPW+bihOJ4SjnoijweJrNThJfd3ifXpXhREjpoF2mZVH1GfS9LV3kHQ== - lazy-ass@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513" @@ -6402,6 +6497,16 @@ mdn-data@2.0.14: resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.14.tgz#7113fc4281917d63ce29b43446f701e68c25ba50" integrity sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow== +mdn-data@2.0.28: + version "2.0.28" + resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.28.tgz#5ec48e7bef120654539069e1ae4ddc81ca490eba" + integrity sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g== + +mdn-data@2.0.30: + version "2.0.30" + resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.30.tgz#ce4df6f80af6cfbe218ecd5c552ba13c4dfa08cc" + integrity sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA== + media-typer@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" @@ -7344,10 +7449,10 @@ range-parser@^1.2.1, range-parser@~1.2.1: resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -raw-body@2.5.1: - version "2.5.1" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857" - integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== +raw-body@2.5.2: + version "2.5.2" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.2.tgz#99febd83b90e08975087e8f1f9419a149366b68a" + integrity sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA== dependencies: bytes "3.1.2" http-errors "2.0.0" @@ -7884,12 +7989,11 @@ safe-regex-test@^1.0.0: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -sass-loader@^13.0.2: - version "13.1.0" - resolved "https://registry.yarnpkg.com/sass-loader/-/sass-loader-13.1.0.tgz#e5b9acf14199a9bc6eaed7a0b8b23951c2cebf6f" - integrity sha512-tZS1RJQ2n2+QNyf3CCAo1H562WjL/5AM6Gi8YcPVVoNxQX8d19mx8E+8fRrMWsyc93ZL6Q8vZDSM0FHVTJaVnQ== +sass-loader@^13.3.2: + version "13.3.2" + resolved "https://registry.yarnpkg.com/sass-loader/-/sass-loader-13.3.2.tgz#460022de27aec772480f03de17f5ba88fa7e18c6" + integrity sha512-CQbKl57kdEv+KDLquhC+gE3pXt74LEAzm+tzywcA0/aHZuub8wTErbjAoNI57rPUWRYRNC5WUnNl8eGJNbDdwg== dependencies: - klona "^2.0.4" neo-async "^2.6.2" sass@^1.55.0: @@ -8128,7 +8232,7 @@ sockjs@^0.3.24: uuid "^8.3.2" websocket-driver "^0.7.4" -"source-map-js@>=0.6.2 <2.0.0", source-map-js@^1.0.2: +"source-map-js@>=0.6.2 <2.0.0", source-map-js@^1.0.1, source-map-js@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw== @@ -8406,6 +8510,18 @@ svgo@^2.8.0: picocolors "^1.0.0" stable "^0.1.8" +svgo@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/svgo/-/svgo-3.0.2.tgz#5e99eeea42c68ee0dc46aa16da093838c262fe0a" + integrity sha512-Z706C1U2pb1+JGP48fbazf3KxHrWOsLme6Rv7imFBn5EnuanDW1GPaA/P1/dvObE670JDePC3mnj0k0B7P0jjQ== + dependencies: + "@trysound/sax" "0.2.0" + commander "^7.2.0" + css-select "^5.1.0" + css-tree "^2.2.1" + csso "^5.0.5" + picocolors "^1.0.0" + symbol-tree@^3.2.4: version "3.2.4" resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" @@ -9025,9 +9141,9 @@ webpack-cli@^4.10.0: webpack-merge "^5.7.3" webpack-dev-middleware@^5.3.1: - version "5.3.3" - resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-5.3.3.tgz#efae67c2793908e7311f1d9b06f2a08dcc97e51f" - integrity sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA== + version "5.3.4" + resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz#eb7b39281cbce10e104eb2b8bf2b63fce49a3517" + integrity sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q== dependencies: colorette "^2.0.10" memfs "^3.4.3" @@ -9206,9 +9322,9 @@ wildcard@^2.0.0: integrity sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw== word-wrap@^1.2.3, word-wrap@~1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" - integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== + version "1.2.4" + resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.4.tgz#cb4b50ec9aca570abd1f52f33cd45b6c61739a9f" + integrity sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA== wrap-ansi@^6.2.0: version "6.2.0"