diff --git a/README.md b/README.md index 46355ed73..49027c0cb 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,26 @@ kubectl port-forward 9002:9002 You should be able to access the frontend via http://localhost:9002. Once you confirm that the pods are running well, you can set up ingress for datahub-frontend -to expose the 9002 port to the public. +to expose the 9002 port to the public. + +## Default Credentials + +There are a few keys and credentials created as part of the deployment using randomized values. They can be overridden using various configuration parameters. + +Also consider changing the default credentials used by any of the underlying data stores pulled in by the companion helm chart for +the prerequisites. Refer to the upstream helm charts or point to your own managed data stores for these components. + +### DataHub Login + +For controlling the default admin password, see the following [configuration](charts/datahub/values.yaml#L36). + +### Encryption Key + +Used by the Play framework and GMS to encrypt secrets at the application level, this can be configured [here](charts/datahub/values.yaml#L579). + +### Token Signing Key + +Used to sign tokens for authentication, see configuration [here](charts/datahub/values.yaml#L605). ## Contributing diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 43e2cf266..b5cefe067 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,29 +4,29 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.182 +version: 0.2.189 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 0.10.5 +appVersion: 0.11.0 dependencies: - name: datahub-gms - version: 0.2.150 + version: 0.2.151 repository: file://./subcharts/datahub-gms condition: datahub-gms.enabled - name: datahub-frontend - version: 0.2.139 + version: 0.2.141 repository: file://./subcharts/datahub-frontend condition: datahub-frontend.enabled - name: datahub-mae-consumer - version: 0.2.146 + version: 0.2.147 repository: file://./subcharts/datahub-mae-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-mce-consumer - version: 0.2.149 + version: 0.2.150 repository: file://./subcharts/datahub-mce-consumer condition: global.datahub_standalone_consumers_enabled - name: datahub-ingestion-cron - version: 0.2.134 + version: 0.2.136 repository: file://./subcharts/datahub-ingestion-cron condition: datahub-ingestion-cron.enabled - name: acryl-datahub-actions diff --git a/charts/datahub/README.md b/charts/datahub/README.md index b077dedf2..986a84bd4 100644 --- a/charts/datahub/README.md +++ b/charts/datahub/README.md @@ -22,17 +22,17 @@ helm install datahub datahub/datahub --values <> |-----|------|---------|-------------| | datahub-frontend.enabled | bool | `true` | Enable Datahub Front-end | | datahub-frontend.image.repository | string | `"linkedin/datahub-frontend-react"` | Image repository for datahub-frontend | -| datahub-frontend.image.tag | string | `"v0.10.4"` | Image tag for datahub-frontend | +| datahub-frontend.image.tag | string | `"v0.11.0"` | Image tag for datahub-frontend | | datahub-frontend.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-frontend | | datahub-gms.enabled | bool | `true` | Enable GMS | | datahub-gms.image.repository | string | `"linkedin/datahub-gms"` | Image repository for datahub-gms | -| datahub-gms.image.tag | string | `"v0.10.4"` | Image tag for datahub-gms | +| datahub-gms.image.tag | string | `"v0.11.0"` | Image tag for datahub-gms | | datahub-gms.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-gms | | datahub-mae-consumer.image.repository | string | `"linkedin/datahub-mae-consumer"` | Image repository for datahub-mae-consumer | -| datahub-mae-consumer.image.tag | string | `"v0.10.4"` | Image tag for datahub-mae-consumer | +| datahub-mae-consumer.image.tag | string | `"v0.11.0"` | Image tag for datahub-mae-consumer | | datahub-mae-consumer.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-mae-consumer | | datahub-mce-consumer.image.repository | string | `"linkedin/datahub-mce-consumer"` | Image repository for datahub-mce-consumer | -| datahub-mce-consumer.image.tag | string | `"v0.10.4"` | Image tag for datahub-mce-consumer | +| datahub-mce-consumer.image.tag | string | `"v0.11.0"` | Image tag for datahub-mce-consumer | | datahub-mce-consumer.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for datahub-mce-consumer | | datahub-ingestion-cron.enabled | bool | `false` | Enable cronjob for periodic ingestion | | datahubUpgrade.podSecurityContext | object | `{}` | Pod security context for datahubUpgrade jobs | @@ -44,7 +44,7 @@ helm install datahub datahub/datahub --values <> | datahubUpgrade.restoreIndices.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | elasticsearchSetupJob.enabled | bool | `true` | Enable setup job for elasicsearch | | elasticsearchSetupJob.image.repository | string | `"linkedin/datahub-elasticsearch-setup"` | Image repository for elasticsearchSetupJob | -| elasticsearchSetupJob.image.tag | string | `"v0.10.4"` | Image repository for elasticsearchSetupJob | +| elasticsearchSetupJob.image.tag | string | `"v0.11.0"` | Image repository for elasticsearchSetupJob | | elasticsearchSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for elasticsearchSetupJob | | elasticsearchSetupJob.resources | object | '{}' | Kube Resource definitions for elasticsearchSetupJob | | elasticsearchSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for elasticsearchSetupJob | @@ -53,7 +53,7 @@ helm install datahub datahub/datahub --values <> | elasticsearchSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | kafkaSetupJob.enabled | bool | `true` | Enable setup job for kafka | | kafkaSetupJob.image.repository | string | `"linkedin/datahub-kafka-setup"` | Image repository for kafkaSetupJob | -| kafkaSetupJob.image.tag | string | `"v0.10.4"` | Image repository for kafkaSetupJob | +| kafkaSetupJob.image.tag | string | `"v0.11.0"` | Image repository for kafkaSetupJob | | kafkaSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for kafkaSetupJob| | kafkaSetupJob.resources | object | '{}' | Kube Resource definitions for kafkaSetupJob | | kafkaSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for kafkaSetupJob | @@ -62,7 +62,7 @@ helm install datahub datahub/datahub --values <> | kafkaSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | mysqlSetupJob.enabled | bool | `false` | Enable setup job for mysql | | mysqlSetupJob.image.repository | string | `"acryldata/datahub-mysql-setup"` | Image repository for mysqlSetupJob | -| mysqlSetupJob.image.tag | string | `"v0.10.4"` | Image repository for mysqlSetupJob | +| mysqlSetupJob.image.tag | string | `"v0.11.0"` | Image repository for mysqlSetupJob | | mysqlSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for mysqlSetupJob| | mysqlSetupJob.resources | object | '{}' | Kube Resource definitions for mysqlSetupJob | | mysqlSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for mysqlSetupJob | @@ -71,7 +71,7 @@ helm install datahub datahub/datahub --values <> | mysqlSetupJob.extraSidecars | list | `[]` | Add additional sidecar containers to the job pod | | postgresqlSetupJob.enabled | bool | `false` | Enable setup job for postgresql | | postgresqlSetupJob.image.repository | string | `"acryldata/datahub-postgres-setup"` | Image repository for postgresqlSetupJob | -| postgresqlSetupJob.image.tag | string | `"v0.10.4"` | Image repository for postgresqlSetupJob | +| postgresqlSetupJob.image.tag | string | `"v0.11.0"` | Image repository for postgresqlSetupJob | | postgresqlSetupJob.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for postgresqlSetupJob| | postgresqlSetupJob.resources | object | '{}' | Kube Resource definitions for postgresqlSetupJob | | postgresqlSetupJob.podSecurityContext | object | `{"fsGroup": 1000}` | Pod security context for mysqlSetupJob | @@ -164,7 +164,7 @@ helm install datahub datahub/datahub --values <> | global.datahub.managed_ingestion.enabled | bool | `true` | Whether or not UI-based ingestion experience is enabled. | | global.datahub.encryptionKey.secretRef | string | `datahub-encryption-secrets` | The reference to a secret containing an alpha-numeric encryption key, which is used to encrypt Secrets on DataHub. If a secret reference is not provided, a random one will be generated for you in a Kubernetes secret named `datahub-encryption-secrets`. | | global.datahub.encryptionKey.secretKey | string | `encryption_key_secret` | The key of a secret containing an alpha-numeric encryption key, which is used to encrypt Secrets on DataHub. If a secret reference is not provided, a random one will be generated for you in a Kubernetes secret value named `encryption_key_secret` within a secret named `datahub-encryption-secrets`. | -| global.datahub.managed_ingestion.defaultCliVersion | string | `` |0.10.0 This is the version of the DataHub CLI to use for UI ingestion, by default. | +| global.datahub.managed_ingestion.defaultCliVersion | string | `` |0.11.0 This is the version of the DataHub CLI to use for UI ingestion, by default. | | global.datahub.encryptionKey.provisionSecret.enabled | bool | `true` | Whether an encryption key secret should be provisioned on the first deployment for you. Set this to false if you are overriding global.datahub.encryptionKey.secretRef. | | global.datahub.encryptionKey.provisionSecret.autoGenerate | bool | `true` | Whether an encryption key secret should be provisioned for you **with a random seed** on the first deployment for you. Set this to false and use `global.datahub.encryptionKey.provisionSecret.secretValues.encryptionKey` if you would like to specify the secret values directly. | | global.datahub.encryptionKey.provisionSecret.secretValues.encryptionKey | string | `` | The encryption key value to be used if specified directly. | diff --git a/charts/datahub/quickstart-values-with-neo4j.yaml b/charts/datahub/quickstart-values-with-neo4j.yaml index d616daf7a..2d96a889c 100644 --- a/charts/datahub/quickstart-values-with-neo4j.yaml +++ b/charts/datahub/quickstart-values-with-neo4j.yaml @@ -4,14 +4,14 @@ datahub-gms: enabled: true image: repository: linkedin/datahub-gms - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version datahub-frontend: enabled: true image: repository: linkedin/datahub-frontend-react - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version # Set up ingress to expose react front-end ingress: @@ -40,35 +40,35 @@ elasticsearchSetupJob: enabled: true image: repository: linkedin/datahub-elasticsearch-setup - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version kafkaSetupJob: enabled: true image: repository: linkedin/datahub-kafka-setup - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version mysqlSetupJob: enabled: true image: repository: acryldata/datahub-mysql-setup - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version datahubUpgrade: enabled: true image: repository: acryldata/datahub-upgrade - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version datahub-ingestion-cron: enabled: false image: repository: acryldata/datahub-ingestion - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version global: @@ -107,7 +107,7 @@ global: secretKey: mysql-root-password datahub: - version: v0.10.5 + version: v0.11.0 gms: port: "8080" mae_consumer: @@ -116,4 +116,4 @@ global: managed_ingestion: enabled: true - defaultCliVersion: "0.10.5.4" + defaultCliVersion: "0.11.0" diff --git a/charts/datahub/subcharts/datahub-frontend/Chart.yaml b/charts/datahub/subcharts/datahub-frontend/Chart.yaml index 7437c207c..1c07a61b5 100644 --- a/charts/datahub/subcharts/datahub-frontend/Chart.yaml +++ b/charts/datahub/subcharts/datahub-frontend/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.139 +version: 0.2.141 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v0.10.0 +appVersion: v0.11.0 diff --git a/charts/datahub/subcharts/datahub-frontend/README.md b/charts/datahub/subcharts/datahub-frontend/README.md index 7c14edbe7..3b04d39e7 100644 --- a/charts/datahub/subcharts/datahub-frontend/README.md +++ b/charts/datahub/subcharts/datahub-frontend/README.md @@ -6,66 +6,67 @@ Current chart version is `0.2.0` ## Chart Values -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | object | `{}` | | -| datahub.play.mem.buffer.size | string | `"10MB"` | | -| existingGmsSecret | object | {} | Reference to GMS secret if already exists | -| exporters.jmx.enabled | boolean | false | | -| extraLabels | object | `{}` | Extra labels for deployment configuration | -| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | -| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) | -| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | -| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | -| fullnameOverride | string | `"datahub-frontend"` | | -| global.datahub_analytics_enabled | boolean | true | | -| global.datahub.gms.port | string | `"8080"` | | -| image.pullPolicy | string | `"IfNotPresent"` | | -| image.repository | string | `"linkedin/datahub-frontend-react"` | | -| image.tag | string | `"head"` | | -| imagePullSecrets | list | `[]` | | -| ingress.annotations | object | `{}` | | -| ingress.enabled | bool | `false` | | -| ingress.extraLabels | object | `{}` | provides extra labels for ingress configuration | -| ingress.hosts[0].host | string | `"chart-example.local"` | | -| ingress.hosts[0].paths | list | `[]` | | -| ingress.hosts[0].redirectPaths | list | `[]` | | -| ingress.tls | list | `[]` | | -| livenessProbe.initialDelaySeconds | int | `60` | | -| livenessProbe.periodSeconds | int | `30` | | -| livenessProbe.failureThreshold | int | `4` | | -| nameOverride | string | `""` | | -| nodeSelector | object | `{}` | | -| oidcAuthentication.enabled | boolean | `false` | Enable [OIDC authentication](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react) | -| oidcAuthentication.provider | string | `""` | One of the supported OIDC providers: [google](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react-google), [okta](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react-okta), or [azure](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react-azure) | -| oidcAuthentication.clientId | string | `""` | A unique identifier for your application with the identity provider | -| oidcAuthentication.clientSecret | string | `""` | A shared secret to use for exchange between you and your identity provider | -| oidcAuthentication.clientSecretRef.secretRef | string | `"nil"` | Optional, this is the reference to the shared secret to use for exchange between you and your identity provider | -| oidcAuthentication.clientSecretRef.secretKey | string | `"nil"` | Optional, this is the key of the shared secret to use for exchange between you and your identity provider | -| oidcAuthentication.oktaDomain | string | `""` | Okta domain, e.g. `dev-12345.okta.com`; needed only if `provider` is set to `okta` | -| oidcAuthentication.azureTenantId | string | `""` | Azure directory (tenant) ID; neede only if `provider` is set to `azure` | -| podAnnotations | object | `{}` | | -| podSecurityContext | object | `{}` | | -| readinessProbe.initialDelaySeconds | int | `60` | | -| readinessProbe.periodSeconds | int | `30` | | -| readinessProbe.failureThreshold | int | `4` | | -| replicaCount | int | `1` | | -| revisionHistoryLimit | int | `10` | | -| lifecycle | object | `{}` | | -| resources | object | `{}` | | -| securityContext | object | `{}` | | -| service.port | int | `9001` | | -| service.nodePort | int | `""` | | -| service.type | string | `"LoadBalancer"` | | -| serviceAccount.annotations | object | `{}` | | -| serviceAccount.create | bool | `true` | | -| serviceAccount.name | string | `nil` | | -| serviceMonitor.create | bool | `false` | If set true and `global.datahub.monitoring.enablePrometheus` is set `true` it will create a ServiceMonitor resource | -| tolerations | list | `[]` | | -| global.elasticsearch.host | string | `"elasticsearch"` | | -| global.elasticsearch.port | string | `"9200"` | | -| global.kafka.bootstrap.server | string | `"broker:9092"` | | -| datahub.metadata_service_authentication.enabled | bool | `false` | Whether Metadata Service Authentication is enabled. | -| global.datahub.metadata_service_authentication.systemClientId | string | `"__datahub_system"` | The internal system id that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | -| global.datahub.metadata_service_authentication.systemClientSecret.secretRef | string | `nil` | The reference to a secret containing the internal system secret that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | -| global.datahub.metadata_service_authentication.systemClientSecret.secretKey | string | `nil` | The key of a secret containing the internal system secret that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | +| Key | Type | Default | Description | +|-----------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| affinity | object | `{}` | | +| datahub.play.mem.buffer.size | string | `"10MB"` | | +| existingGmsSecret | object | {} | Reference to GMS secret if already exists | +| exporters.jmx.enabled | boolean | false | | +| extraLabels | object | `{}` | Extra labels for deployment configuration | +| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | +| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) | +| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | +| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | +| fullnameOverride | string | `"datahub-frontend"` | | +| global.datahub_analytics_enabled | boolean | true | | +| global.datahub.gms.port | string | `"8080"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"linkedin/datahub-frontend-react"` | | +| image.tag | string | `"head"` | | +| imagePullSecrets | list | `[]` | | +| ingress.annotations | object | `{}` | | +| ingress.enabled | bool | `false` | | +| ingress.extraLabels | object | `{}` | provides extra labels for ingress configuration | +| ingress.hosts[0].host | string | `"chart-example.local"` | | +| ingress.hosts[0].paths | list | `[]` | | +| ingress.hosts[0].redirectPaths | list | `[]` | | +| ingress.tls | list | `[]` | | +| livenessProbe.initialDelaySeconds | int | `60` | | +| livenessProbe.periodSeconds | int | `30` | | +| livenessProbe.failureThreshold | int | `4` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| auth.sessionTTLHours | string | `24` | Configures the length of time a session token is valid for after creation. | +| oidcAuthentication.enabled | boolean | `false` | Enable [OIDC authentication](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react) | +| oidcAuthentication.provider | string | `""` | One of the supported OIDC providers: [google](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react-google), [okta](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react-okta), or [azure](https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react-azure) | +| oidcAuthentication.clientId | string | `""` | A unique identifier for your application with the identity provider | +| oidcAuthentication.clientSecret | string | `""` | A shared secret to use for exchange between you and your identity provider | +| oidcAuthentication.clientSecretRef.secretRef | string | `"nil"` | Optional, this is the reference to the shared secret to use for exchange between you and your identity provider | +| oidcAuthentication.clientSecretRef.secretKey | string | `"nil"` | Optional, this is the key of the shared secret to use for exchange between you and your identity provider | +| oidcAuthentication.oktaDomain | string | `""` | Okta domain, e.g. `dev-12345.okta.com`; needed only if `provider` is set to `okta` | +| oidcAuthentication.azureTenantId | string | `""` | Azure directory (tenant) ID; neede only if `provider` is set to `azure` | +| podAnnotations | object | `{}` | | +| podSecurityContext | object | `{}` | | +| readinessProbe.initialDelaySeconds | int | `60` | | +| readinessProbe.periodSeconds | int | `30` | | +| readinessProbe.failureThreshold | int | `4` | | +| replicaCount | int | `1` | | +| revisionHistoryLimit | int | `10` | | +| lifecycle | object | `{}` | | +| resources | object | `{}` | | +| securityContext | object | `{}` | | +| service.port | int | `9001` | | +| service.nodePort | int | `""` | | +| service.type | string | `"LoadBalancer"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `nil` | | +| serviceMonitor.create | bool | `false` | If set true and `global.datahub.monitoring.enablePrometheus` is set `true` it will create a ServiceMonitor resource | +| tolerations | list | `[]` | | +| global.elasticsearch.host | string | `"elasticsearch"` | | +| global.elasticsearch.port | string | `"9200"` | | +| global.kafka.bootstrap.server | string | `"broker:9092"` | | +| datahub.metadata_service_authentication.enabled | bool | `false` | Whether Metadata Service Authentication is enabled. | +| global.datahub.metadata_service_authentication.systemClientId | string | `"__datahub_system"` | The internal system id that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | +| global.datahub.metadata_service_authentication.systemClientSecret.secretRef | string | `nil` | The reference to a secret containing the internal system secret that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | +| global.datahub.metadata_service_authentication.systemClientSecret.secretKey | string | `nil` | The key of a secret containing the internal system secret that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. | diff --git a/charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml b/charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml index 914c97f81..5db08afbb 100644 --- a/charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml @@ -174,6 +174,8 @@ spec: name: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretRef }} key: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretKey }} {{- end }} + - name: AUTH_SESSION_TTL_HOURS + value: {{ .Values.auth.sessionTTLHours | quote }} {{- with .Values.oidcAuthentication }} {{- if .enabled }} - name: AUTH_OIDC_ENABLED diff --git a/charts/datahub/subcharts/datahub-frontend/values.yaml b/charts/datahub/subcharts/datahub-frontend/values.yaml index 2f4291106..520622b12 100644 --- a/charts/datahub/subcharts/datahub-frontend/values.yaml +++ b/charts/datahub/subcharts/datahub-frontend/values.yaml @@ -68,6 +68,8 @@ ingress: # hosts: # - chart-example.local +auth: + sessionTTLHours: "24" # OIDC auth based on https://datahubproject.io/docs/authentication/guides/sso/configure-oidc-react oidcAuthentication: enabled: false diff --git a/charts/datahub/subcharts/datahub-gms/Chart.yaml b/charts/datahub/subcharts/datahub-gms/Chart.yaml index 1c8eb0cbd..b9fcd685d 100644 --- a/charts/datahub/subcharts/datahub-gms/Chart.yaml +++ b/charts/datahub/subcharts/datahub-gms/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for LinkedIn DataHub's datahub-gms component type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.150 +version: 0.2.151 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v0.10.0 +appVersion: v0.11.0 diff --git a/charts/datahub/subcharts/datahub-gms/README.md b/charts/datahub/subcharts/datahub-gms/README.md index 24df024ae..e258611cd 100644 --- a/charts/datahub/subcharts/datahub-gms/README.md +++ b/charts/datahub/subcharts/datahub-gms/README.md @@ -81,5 +81,5 @@ Current chart version is `0.2.0` | global.datahub.managed_ingestion.enabled | bool | `true` | Whether or not UI-based ingestion experience is enabled. | | global.datahub.encryptionKey.secretRef | string | `nil` | The reference to a secret containing an alpha-numeric encryption key, which is used to encrypt Secrets on DataHub. Required if managed_ingestion_enabled is 'true'. | | global.datahub.encryptionKey.secretKey | string | `nil` | The key of a secret containing an alpha-numeric encryption key, which is used to encrypt Secrets on DataHub. Required if managed_ingestion_enabled is 'true'. | -| global.datahub.managed_ingestion.defaultCliVersion | string | `0.10.0` | This is the version of the DataHub CLI to use for UI ingestion, by default. You do not need to explicitly provide this. By default the underlying datahub-gms container will provide a latest version compatible with the server. | +| global.datahub.managed_ingestion.defaultCliVersion | string | `0.11.0` | This is the version of the DataHub CLI to use for UI ingestion, by default. You do not need to explicitly provide this. By default the underlying datahub-gms container will provide a latest version compatible with the server. | | global.datahub.enable_retention | bool | `false` | Whether or not to enable retention on local DB | diff --git a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml index 45524b096..d4d55367b 100644 --- a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml +++ b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml @@ -113,7 +113,7 @@ spec: - name: SEARCH_SERVICE_CACHE_IMPLEMENTATION value: "hazelcast" - name: SEARCH_SERVICE_HAZELCAST_SERVICE_NAME - value: {{ printf "%s-%s" .Release.Name "hazelcast-service" }} + value: {{ printf "%s-%s-%s" .Release.Name (regexReplaceAll "\\W+" .Values.global.datahub.version "-") "hazelcast-svc" | trunc 63 | trimSuffix "-" }} {{- end}} {{- if .Values.global.datahub.systemUpdate.enabled }} - name: DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID diff --git a/charts/datahub/subcharts/datahub-gms/templates/hazelcastService.yaml b/charts/datahub/subcharts/datahub-gms/templates/hazelcastService.yaml index 50abdc524..b4a1aa427 100644 --- a/charts/datahub/subcharts/datahub-gms/templates/hazelcastService.yaml +++ b/charts/datahub/subcharts/datahub-gms/templates/hazelcastService.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ printf "%s-%s" .Release.Name "hazelcast-service" }} + name: {{ printf "%s-%s-%s" .Release.Name (regexReplaceAll "\\W+" .Values.global.datahub.version "-") "hazelcast-svc" | trunc 63 | trimSuffix "-" }} spec: clusterIP: None ports: diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml index 95eca2002..7f70d6aad 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.134 +version: 0.2.136 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v0.10.0 +appVersion: v0.11.0 diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/README.md b/charts/datahub/subcharts/datahub-ingestion-cron/README.md index df7feddda..900b2b15a 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/README.md +++ b/charts/datahub/subcharts/datahub-ingestion-cron/README.md @@ -27,4 +27,12 @@ A Helm chart for datahub's metadata-ingestion framework with kerberos authentica | crons.extraInitContainers | object | `{}` | Init containers to add to the cronjob container | | crons.serviceAccountName | string | | Service account name used for the cronjob container | | crons.podAnnotations | object | `{}` | Annotations to add to the pods | -| extraSidecars | list | `[]` | Add additional sidecar containers to the deployment pod(s) | +| crons.restartPolicy | string | `"Always"` | Pod restart policy | +| crons.concurrencyPolicy | string | `"Allow"` | Specifies how to treat concurrent executions of a job | +| crons.failedJobsHistoryLimit | integer | `1` | Number of failed finished jobs to retain | +| crons.successfulJobsHistoryLimit | integer | `3` | Number of successful finished jobs to retain | +| crons.backoffLimit | integer | `6` | Number of retries before marking job failed | +| crons.nodeSelector | object | `{}` | Node labels for pod assignment | +| crons.affinity | object | `{}` | Affinity for pod assignment | +| crons.tolerations | list | `[]` | Tolerations for pod assignment | +| crons.extraSidecars | list | `[]` | Add sidecar containers to the pod | diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml index 5b2c417db..2f8001458 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/templates/cron.yaml @@ -64,6 +64,9 @@ spec: key: {{ $value.key | quote}} {{- end }} {{- end }} + {{- if .extraSidecars }} + {{- toYaml .extraSidecars | nindent 10 }} + {{- end }} restartPolicy: {{ default "OnFailure" .restartPolicy }} {{- if .nodeSelector }} nodeSelector: @@ -84,8 +87,5 @@ spec: {{- if .extraVolumes }} {{- toYaml .extraVolumes | nindent 12 }} {{- end }} - {{- if .extraSidecars }} - {{- toYaml .extraSidecars | nindent 10 }} - {{- end }} --- {{- end }} diff --git a/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml b/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml index f4ad84c1f..6f3825a8c 100644 --- a/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml +++ b/charts/datahub/subcharts/datahub-ingestion-cron/values.yaml @@ -100,12 +100,6 @@ crons: {} ## #tolerations: [] -# Add extra sidecar containers to deployment pod -extraSidecars: [] - # - name: my-image-name - # image: my-image - # imagePullPolicy: Always - global: datahub: - version: head \ No newline at end of file + version: head diff --git a/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml index 00939a712..e2ad31b49 100644 --- a/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml +++ b/charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.146 +version: 0.2.147 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v0.10.0 +appVersion: v0.11.0 diff --git a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml index f2985ec66..647ff4d54 100644 --- a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml +++ b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.149 +version: 0.2.150 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v0.10.0 +appVersion: v0.11.0 diff --git a/charts/datahub/templates/datahub-upgrade/_upgrade.tpl b/charts/datahub/templates/datahub-upgrade/_upgrade.tpl index dda094575..e67ea962b 100644 --- a/charts/datahub/templates/datahub-upgrade/_upgrade.tpl +++ b/charts/datahub/templates/datahub-upgrade/_upgrade.tpl @@ -39,15 +39,6 @@ Return the env variables for upgrade jobs value: "{{ .Values.global.sql.datasource.url }}" - name: EBEAN_DATASOURCE_DRIVER value: "{{ .Values.global.sql.datasource.driver }}" -{{- if .Values.global.datahub.metadata_service_authentication.enabled }} -- name: DATAHUB_SYSTEM_CLIENT_ID - value: {{ .Values.global.datahub.metadata_service_authentication.systemClientId }} -- name: DATAHUB_SYSTEM_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretRef }} - key: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretKey }} -{{- end }} - name: KAFKA_BOOTSTRAP_SERVER value: "{{ .Values.global.kafka.bootstrap.server }}" {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }} diff --git a/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml b/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml index fdd539d04..54d51b8f6 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-cleanup-job-template.yml @@ -58,6 +58,15 @@ spec: args: [ "-u", "NoCodeDataMigrationCleanup" ] env: {{- include "datahub.upgrade.env" . | nindent 16}} + {{- if .Values.global.datahub.metadata_service_authentication.enabled }} + - name: DATAHUB_SYSTEM_CLIENT_ID + value: {{ .Values.global.datahub.metadata_service_authentication.systemClientId }} + - name: DATAHUB_SYSTEM_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretRef }} + key: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretKey }} + {{- end }} {{- with .Values.datahubUpgrade.extraEnvs }} {{- toYaml . | nindent 16 }} {{- end }} diff --git a/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml index 7649584e8..311c1f3a5 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-nocode-migration-job.yml @@ -68,6 +68,15 @@ spec: - "dbType={{ .Values.datahubUpgrade.noCodeDataMigration.sqlDbType }}" env: {{- include "datahub.upgrade.env" . | nindent 12}} + {{- if .Values.global.datahub.metadata_service_authentication.enabled }} + - name: DATAHUB_SYSTEM_CLIENT_ID + value: {{ .Values.global.datahub.metadata_service_authentication.systemClientId }} + - name: DATAHUB_SYSTEM_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretRef }} + key: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretKey }} + {{- end }} {{- with .Values.datahubUpgrade.extraEnvs }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml b/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml index 7fbdde2aa..167b0d63a 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-restore-indices-job-template.yml @@ -70,6 +70,15 @@ spec: - "batchDelayMs={{ .Values.datahubUpgrade.batchDelayMs }}" env: {{- include "datahub.upgrade.env" . | nindent 16}} + {{- if .Values.global.datahub.metadata_service_authentication.enabled }} + - name: DATAHUB_SYSTEM_CLIENT_ID + value: {{ .Values.global.datahub.metadata_service_authentication.systemClientId }} + - name: DATAHUB_SYSTEM_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretRef }} + key: {{ .Values.global.datahub.metadata_service_authentication.systemClientSecret.secretKey }} + {{- end }} {{- with .Values.datahubUpgrade.extraEnvs }} {{- toYaml . | nindent 16 }} {{- end }} diff --git a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml index e03e8b38f..261f0b921 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml @@ -70,7 +70,7 @@ spec: - name: SCHEMA_REGISTRY_SYSTEM_UPDATE value: "true" - name: SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS - value: "false" + value: "true" - name: SPRING_KAFKA_PROPERTIES_USE_LATEST_VERSION value: "true" {{- end }} diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 771170328..beb28d385 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -4,13 +4,21 @@ datahub-gms: enabled: true image: repository: linkedin/datahub-gms - # tag: "v0.10.0 # defaults to .global.datahub.version + # tag: "v0.11.0 # defaults to .global.datahub.version resources: limits: memory: 2Gi requests: cpu: 100m memory: 1Gi + livenessProbe: + initialDelaySeconds: 60 + periodSeconds: 30 + failureThreshold: 8 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 30 + failureThreshold: 8 # Optionally set a GMS specific SQL login (defaults to global login) # sql: # datasource: @@ -23,7 +31,7 @@ datahub-frontend: enabled: true image: repository: linkedin/datahub-frontend-react - # tag: "v0.10.0" # # defaults to .global.datahub.version + # tag: "v0.11.0" # # defaults to .global.datahub.version resources: limits: memory: 1400Mi @@ -59,7 +67,7 @@ acryl-datahub-actions: datahub-mae-consumer: image: repository: linkedin/datahub-mae-consumer - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version resources: limits: memory: 1536Mi @@ -70,7 +78,7 @@ datahub-mae-consumer: datahub-mce-consumer: image: repository: linkedin/datahub-mce-consumer - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version resources: limits: memory: 1536Mi @@ -82,13 +90,13 @@ datahub-ingestion-cron: enabled: false image: repository: acryldata/datahub-ingestion - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version elasticsearchSetupJob: enabled: true image: repository: linkedin/datahub-elasticsearch-setup - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version resources: limits: cpu: 500m @@ -118,7 +126,7 @@ kafkaSetupJob: enabled: true image: repository: linkedin/datahub-kafka-setup - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version resources: limits: cpu: 500m @@ -148,7 +156,7 @@ mysqlSetupJob: enabled: true image: repository: acryldata/datahub-mysql-setup - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version resources: limits: cpu: 500m @@ -183,7 +191,7 @@ postgresqlSetupJob: enabled: false image: repository: acryldata/datahub-postgres-setup - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version resources: limits: cpu: 500m @@ -223,7 +231,7 @@ datahubUpgrade: enabled: true image: repository: acryldata/datahub-upgrade - # tag: "v0.10.0" # defaults to .global.datahub.version + # tag: "v0.11.0" # defaults to .global.datahub.version batchSize: 1000 batchDelayMs: 100 noCodeDataMigration: @@ -554,7 +562,7 @@ global: # value: password datahub: - version: v0.10.5 + version: v0.11.0 gms: port: "8080" nodePort: "30001" @@ -586,7 +594,7 @@ global: managed_ingestion: enabled: true - defaultCliVersion: "0.10.5.4" + defaultCliVersion: "0.11.0" metadata_service_authentication: enabled: false @@ -620,9 +628,9 @@ global: ## Values specific to the unified search and browse feature. search_and_browse: - show_search_v2: false # If on, show the new search filters experience as of v0.10.5 - show_browse_v2: false # If on, show the new browse experience as of v0.10.5 - backfill_browse_v2: false # If on, run the backfill upgrade job that generates default browse paths for relevant entities + show_search_v2: true # If on, show the new search filters experience as of v0.10.5 + show_browse_v2: true # If on, show the new browse experience as of v0.10.5 + backfill_browse_v2: true # If on, run the backfill upgrade job that generates default browse paths for relevant entities # hostAliases: # - ip: "192.168.0.104"