From 63dcb7342d11fb607ecf1d8242e2809f63b85c79 Mon Sep 17 00:00:00 2001 From: Josh Ladieu Date: Tue, 15 Aug 2023 09:30:54 -0400 Subject: [PATCH] AJ-1219: Convert application.properties to YAML (#324) Convert application.properties to YAML In keeping with DSP standardizing on YAML, this converts the two props files in WDS to YML format. It was tested by diffing the properties before/after conversion and visually inspecting the results using the following testing mechanism which was temporarily introduced to support this verification: ``` DUMP_PROPERTIES=true ./gradlew bootRun ``` That tool can be found in the deletion commit if it's lost in the squash/merge commit: 5743b05650ed0bf27887654622c33ebe5625e956 Additionally, the following properties were not referred to from anywhere else in https://github.com/DataBiosphere, so they are removed. * `sam.healthcheck.pingTTL` * `twds.clientId` Finally, an outdated comment referring to data.sql is removed from the configuration property. --- .../SentryInitializer.java | 6 +- .../src/main/resources/application.properties | 92 ------------- service/src/main/resources/application.yml | 124 ++++++++++++++++++ ...am.properties => application-mock-sam.yml} | 4 +- 4 files changed, 131 insertions(+), 95 deletions(-) delete mode 100644 service/src/main/resources/application.properties create mode 100644 service/src/main/resources/application.yml rename service/src/test/resources/{application-mock-sam.properties => application-mock-sam.yml} (79%) diff --git a/service/src/main/java/org/databiosphere/workspacedataservice/SentryInitializer.java b/service/src/main/java/org/databiosphere/workspacedataservice/SentryInitializer.java index be0768bdf..78f21e91c 100644 --- a/service/src/main/java/org/databiosphere/workspacedataservice/SentryInitializer.java +++ b/service/src/main/java/org/databiosphere/workspacedataservice/SentryInitializer.java @@ -1,6 +1,8 @@ package org.databiosphere.workspacedataservice; import io.sentry.Sentry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.SmartInitializingSingleton; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -13,8 +15,8 @@ @Configuration @PropertySource("classpath:git.properties") -@PropertySource("classpath:application.properties") public class SentryInitializer { + private static final Logger LOGGER = LoggerFactory.getLogger(SentryInitializer.class); @Value("${sentry.dsn}") String dsn; @@ -42,6 +44,7 @@ public class SentryInitializer { @Bean public SmartInitializingSingleton initialize() { String env = urlToEnv(samurl); + return () -> Sentry.init(options -> { options.setEnvironment(env); @@ -53,7 +56,6 @@ public SmartInitializingSingleton initialize() { }); } - /** * Extracts an environment (e.g. "dev" or "prod") from a Sam url. * Looks for ".dsde-${env} and returns ${env} if found. diff --git a/service/src/main/resources/application.properties b/service/src/main/resources/application.properties deleted file mode 100644 index 67cb8071b..000000000 --- a/service/src/main/resources/application.properties +++ /dev/null @@ -1,92 +0,0 @@ -# These values are used more than once, so we'll set them here with appropriate defaults -env.wds.db.name=${WDS_DB_NAME:wds} -env.wds.db.user=${WDS_DB_USER:wds} -env.wds.db.password=${WDS_DB_PASSWORD:wds} -env.wds.db.host=${WDS_DB_HOST:localhost} -env.wds.db.port=${WDS_DB_PORT:5432} -# When running on Azure in k8s with workload identity set ADDITIONAL_JDBC_URL_PARAMS to -# sslmode=require&authenticationPluginClassName=com.azure.identity.extensions.jdbc.postgresql.AzurePostgresqlAuthenticationPlugin -env.wds.db.additionalUrlParams=${ADDITIONAL_JDBC_URL_PARAMS:} - -management.endpoints.enabled-by-default=false -management.endpoint.info.enabled=true -management.endpoint.health.enabled=true -management.endpoint.health.probes.enabled=true -management.endpoints.web.exposure.include=info,health -management.endpoints.web.base-path=/ -management.endpoints.web.path-mapping.info=version -management.endpoints.web.path-mapping.health=status -management.endpoint.health.show-details=ALWAYS -management.endpoint.health.group.liveness.include=livenessState,db - -# additional keys to expose in the actuator info endpoint: -management.info.env.enabled=true -info.app.chart-version=${HELM_CHART:unknown} -info.app.image=${WDS_IMAGE:unknown} - -spring.datasource.hikari.jdbc-url=jdbc:postgresql://${env.wds.db.host}:${env.wds.db.port}/${env.wds.db.name}?reWriteBatchedInserts=true&${env.wds.db.additionalUrlParams} -spring.datasource.hikari.username=${env.wds.db.user} -spring.datasource.hikari.password=${env.wds.db.password} -spring.datasource.hikari.maximum-pool-size=7 -spring.datasource.hikari.minimum-idle=7 - -# set hikari logging to DEBUG or even TRACE to troubleshoot connection pool issues -logging.level.com.zaxxer.hikari.HikariConfig=INFO -logging.level.com.zaxxer.hikari=INFO - -# Ensure data.sql is NOT run on startup -spring.sql.init.mode=never -# Run Liquibase instead -spring.liquibase.change-log=classpath:liquibase/changelog.yaml - -server.error.include-stacktrace=never -server.error.include-message=always - -spring.servlet.multipart.max-request-size=5GB -spring.servlet.multipart.max-file-size=5GB - -spring.cache.jcache.config=classpath:ehcache.xml - -twds.write.batch.size=5000 -twds.streaming.fetch.size=5000 - -# Workspace Id for launching instance -twds.instance.workspace-id=${WORKSPACE_ID:} -twds.clientId=${CLIENT_ID:} -twds.instance.source-workspace-id=${SOURCE_WORKSPACE_ID:} -# short-lived credentials to use during cloning -twds.startup-token=${STARTUP_TOKEN:} - -twds.pg_dump.path=${PGDUMP_PATH:/usr/bin/pg_dump} -twds.pg_dump.psqlPath=${PSQL_PATH:/usr/bin/psql} -twds.pg_dump.port=${env.wds.db.port} -twds.pg_dump.user=${env.wds.db.user} -twds.pg_dump.dbName=${env.wds.db.name} -twds.pg_dump.password=${env.wds.db.password} -twds.pg_dump.host=${env.wds.db.host} -# When running on Azure in k8s with workload identity set PGDUMP_USE_AZURE_IDENTITY to true -twds.pg_dump.useAzureIdentity=${PGDUMP_USE_AZURE_IDENTITY:true} - -sentry.dsn=https://e59ecdd940784bd2922f25a0f3197ffd@o54426.ingest.sentry.io/4504299946835968 -sentry.samurl=${SAM_URL:} -sentry.mrg=${LZ_MRG:} -sentry.releasename=${RELEASE_NAME:} - -# How often WDS should ping SAM URL for a health check - 300000 milliseconds = 5 minutes -sam.healthcheck.pingTTL=300000 -sam.retry.maxAttempts=5 -sam.retry.backoff.delay=500 -sam.retry.backoff.multiplier=1.5 - -api.retry.maxAttempts=10 -api.retry.backoff.delay=150 -api.retry.backoff.multiplier=1.5 - -datarepourl=${DATA_REPO_URL:} -workspacemanagerurl=${WORKSPACE_MANAGER_URL:} -leoUrl=${LEONARDO_URL:} - -# activate the "local" profile to turn on CORS response headers, -# which may be necessary for local development. -# spring.profiles.active=local - diff --git a/service/src/main/resources/application.yml b/service/src/main/resources/application.yml new file mode 100644 index 000000000..c083df14b --- /dev/null +++ b/service/src/main/resources/application.yml @@ -0,0 +1,124 @@ +# These values are used more than once, so we'll set them here with appropriate defaults +env: + wds: + db: + host: ${WDS_DB_HOST:localhost} + port: ${WDS_DB_PORT:5432} + name: ${WDS_DB_NAME:wds} + password: ${WDS_DB_PASSWORD:wds} + user: ${WDS_DB_USER:wds} + # When running on Azure in k8s with workload identity set ADDITIONAL_JDBC_URL_PARAMS to + # sslmode=require&authenticationPluginClassName=com.azure.identity.extensions.jdbc.postgresql.AzurePostgresqlAuthenticationPlugin + additionalUrlParams: ${ADDITIONAL_JDBC_URL_PARAMS:} + +management: + endpoint: + info: + enabled: true + health: + enabled: true + show-details: ALWAYS + probes: + enabled: true + group: + liveness: + include: livenessState,db + endpoints: + enabled-by-default: false + web: + exposure.include: info,health + base-path: / + path-mapping: + info: version + health: status + # additional keys to expose in the actuator info endpoint: + info: + env: + enabled: true + +info: + app: + chart-version: ${HELM_CHART:unknown} + image: ${WDS_IMAGE:unknown} + +spring: + cache: + jcache: + config: classpath:ehcache.xml + + datasource: + hikari: + jdbc-url: jdbc:postgresql://${env.wds.db.host}:${env.wds.db.port}/${env.wds.db.name}?reWriteBatchedInserts=true&${env.wds.db.additionalUrlParams} + username: ${env.wds.db.user} + password: ${env.wds.db.password} + maximum-pool-size: 7 + minimum-idle: 7 + servlet: + multipart.max-request-size: 5GB + multipart.max-file-size: 5GB + sql: + init: + mode: never + # Run Liquibase instead + liquibase: + change-log: classpath:liquibase/changelog.yaml +# # activate the "local" profile to turn on CORS response headers, +# # which may be necessary for local development. +# profiles: +# active: local + +# set hikari logging to DEBUG or even TRACE to troubleshoot connection pool issues +logging: + level: + com.zaxxer.hikari.HikariConfig: INFO + com.zaxxer.hikari: INFO + +server: + error: + include-stacktrace: never + include-message: always + +twds: + write.batch.size: 5000 + streaming.fetch.size: 5000 + instance: + # Workspace Id for launching instance + workspace-id: ${WORKSPACE_ID:} + source-workspace-id: ${SOURCE_WORKSPACE_ID:} + # short-lived credentials to use during cloning + startup-token: ${STARTUP_TOKEN:} + + pg_dump: + path: ${PGDUMP_PATH:/usr/bin/pg_dump} + psqlPath: ${PSQL_PATH:/usr/bin/psql} + port: ${env.wds.db.port} + user: ${env.wds.db.user} + dbName: ${env.wds.db.name} + password: ${env.wds.db.password} + host: ${env.wds.db.host} + # When running on Azure in k8s with workload identity set PGDUMP_USE_AZURE_IDENTITY to true + useAzureIdentity: ${PGDUMP_USE_AZURE_IDENTITY:true} + +sentry: + dsn: https://e59ecdd940784bd2922f25a0f3197ffd@o54426.ingest.sentry.io/4504299946835968 + samurl: ${SAM_URL:} + mrg: ${LZ_MRG:} + releasename: ${RELEASE_NAME:} + +sam: + retry: + maxAttempts: 5 + backoff: + delay: 500 + multiplier: 1.5 + +api: + retry: + maxAttempts: 10 + backoff: + delay: 150 + multiplier: 1.5 + +datarepourl: ${DATA_REPO_URL:} +workspacemanagerurl: ${WORKSPACE_MANAGER_URL:} +leoUrl: ${LEONARDO_URL:} diff --git a/service/src/test/resources/application-mock-sam.properties b/service/src/test/resources/application-mock-sam.yml similarity index 79% rename from service/src/test/resources/application-mock-sam.properties rename to service/src/test/resources/application-mock-sam.yml index 1f942f90b..371f05b15 100644 --- a/service/src/test/resources/application-mock-sam.properties +++ b/service/src/test/resources/application-mock-sam.yml @@ -2,5 +2,7 @@ # Sam client underneath that dao. In order for SamConfig to create the HttpSamDao, # we need a valid workspace id. # This is the example uuid from https://en.wikipedia.org/wiki/Universally_unique_identifier -twds.instance.workspace-id=123e4567-e89b-12d3-a456-426614174000 +twds: + instance: + workspace-id: 123e4567-e89b-12d3-a456-426614174000