From 5dce0785bfc82fda03ac921b5f52847679841365 Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Fri, 15 May 2020 10:28:37 +0200 Subject: [PATCH 01/17] introduce json logging with MDC contexts --- Dockerfile | 2 + build.gradle.kts | 2 + .../wire/bots/polls/setup/KtorInstallation.kt | 40 ++++++++++++---- .../polls/setup/logging/MdcConfiguration.kt | 11 +++++ .../polls/setup/logging/ProductionLayout.kt | 48 +++++++++++++++++++ src/main/resources/logback.xml | 28 ++++++++--- 6 files changed, 115 insertions(+), 16 deletions(-) create mode 100644 src/main/kotlin/com/wire/bots/polls/setup/logging/MdcConfiguration.kt create mode 100644 src/main/kotlin/com/wire/bots/polls/setup/logging/ProductionLayout.kt diff --git a/Dockerfile b/Dockerfile index 5417e6d..c98effb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,5 +38,7 @@ ARG release_version=development ENV RELEASE_FILE_PATH=$APP_ROOT/run/release.txt RUN echo $release_version > $RELEASE_FILE_PATH +# enable json logging +ENV JSON_LOGGING=true EXPOSE 8080 ENTRYPOINT ["/bin/sh", "-c", "/app/run/bin/polls"] diff --git a/build.gradle.kts b/build.gradle.kts index 7779edc..0d24f15 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -43,6 +43,8 @@ dependencies { // logging implementation("io.github.microutils", "kotlin-logging", "1.7.9") + // if-else in logback.xml + implementation("org.codehaus.janino", "janino", "3.1.2") implementation("ch.qos.logback", "logback-classic", "1.2.3") // DI diff --git a/src/main/kotlin/com/wire/bots/polls/setup/KtorInstallation.kt b/src/main/kotlin/com/wire/bots/polls/setup/KtorInstallation.kt index 8593383..7ffb7ba 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/KtorInstallation.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/KtorInstallation.kt @@ -5,15 +5,20 @@ import com.wire.bots.polls.dao.DatabaseSetup import com.wire.bots.polls.dto.conf.DatabaseConfiguration import com.wire.bots.polls.routing.registerRoutes import com.wire.bots.polls.setup.errors.registerExceptionHandlers +import com.wire.bots.polls.setup.logging.APP_REQUEST +import com.wire.bots.polls.setup.logging.INFRA_REQUEST import com.wire.bots.polls.utils.createLogger import io.ktor.application.Application import io.ktor.application.install +import io.ktor.features.CallId import io.ktor.features.CallLogging import io.ktor.features.ContentNegotiation import io.ktor.features.DefaultHeaders +import io.ktor.features.callId import io.ktor.jackson.jackson import io.ktor.metrics.micrometer.MicrometerMetrics -import io.ktor.request.path +import io.ktor.request.header +import io.ktor.request.uri import io.ktor.routing.routing import io.micrometer.core.instrument.distribution.DistributionStatisticConfig import io.micrometer.prometheus.PrometheusMeterRegistry @@ -23,6 +28,7 @@ import org.kodein.di.generic.instance import org.kodein.di.ktor.kodein import org.slf4j.event.Level import java.text.DateFormat +import java.util.UUID private val installationLogger = createLogger("ApplicationSetup") @@ -92,21 +98,35 @@ fun Application.installFrameworks(k: LazyKodein) { } install(DefaultHeaders) + install(CallLogging) { - level = Level.TRACE - logger = createLogger("EndpointLogger") + // insert nginx id to MDC + mdc(INFRA_REQUEST) { + it.request.header("X-Request-Id") + } + + // use generated call id and insert it to MDC + mdc(APP_REQUEST) { + it.callId + } - filter { call -> call.request.path().startsWith("/messages") } + // enable logging just for /messages + // this filter does not influence MDC + filter { + it.request.uri == "/messages" + } + level = Level.DEBUG + logger = createLogger("HttpCallLogger") + } + + install(CallId) { + generate { + UUID.randomUUID().toString() + } } - configurePrometheus(k) registerExceptionHandlers(k) -} -/** - * Install prometheus. - */ -fun Application.configurePrometheus(k: LazyKodein) { val prometheusRegistry by k.instance() install(MicrometerMetrics) { registry = prometheusRegistry diff --git a/src/main/kotlin/com/wire/bots/polls/setup/logging/MdcConfiguration.kt b/src/main/kotlin/com/wire/bots/polls/setup/logging/MdcConfiguration.kt new file mode 100644 index 0000000..a500163 --- /dev/null +++ b/src/main/kotlin/com/wire/bots/polls/setup/logging/MdcConfiguration.kt @@ -0,0 +1,11 @@ +package com.wire.bots.polls.setup.logging + +/** + * ID retrieved from the header X-Request-Id coming from the infrastructure nginx. + */ +const val INFRA_REQUEST = "infra_request" + +/** + * ID generated by this application for particular request. + */ +const val APP_REQUEST = "app_request" diff --git a/src/main/kotlin/com/wire/bots/polls/setup/logging/ProductionLayout.kt b/src/main/kotlin/com/wire/bots/polls/setup/logging/ProductionLayout.kt new file mode 100644 index 0000000..bd51276 --- /dev/null +++ b/src/main/kotlin/com/wire/bots/polls/setup/logging/ProductionLayout.kt @@ -0,0 +1,48 @@ +package com.wire.bots.polls.setup.logging + + +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.CoreConstants +import ch.qos.logback.core.LayoutBase +import java.time.Instant +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter + +/** + * Layout logging into jsons. + */ +class ProductionLayout : LayoutBase() { + + private companion object { + val dateTimeFormatter: DateTimeFormatter = + DateTimeFormatter.ISO_DATE_TIME + .withZone(ZoneOffset.UTC) + } + + override fun doLayout(event: ILoggingEvent): String = + with(StringBuffer(256)) { + append("{") + appendJson("@timestamp", getTime()) + + event.mdcPropertyMap[INFRA_REQUEST]?.let { + appendJson("infra_request", it) + } + + event.mdcPropertyMap[APP_REQUEST]?.let { + appendJson("app_request", it) + } + + appendJson("logger", event.loggerName) + appendJson("message", event.formattedMessage) + appendJson("level", event.level.levelStr) + appendJson("thread_name", event.threadName, "") + append("}") + append(CoreConstants.LINE_SEPARATOR) + toString() + } + + private fun StringBuffer.appendJson(key: String, value: String, ending: String = ","): StringBuffer = + append("\"$key\":\"$value\"$ending") + + private fun getTime(): String = dateTimeFormatter.format(Instant.now()) +} diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 4eb4c30..c99564f 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -1,16 +1,32 @@ - + - %d{dd/MM/yyyy HH:mm:ss} [%level] %logger{36} - %msg%n + %d{dd/MM/yyyy HH:mm:ss} [%level] [%X{mdc-val}] %logger{36} - %msg%n - - - + + + + + + + + + + + + + + + + + + + + - From 6b5eed90b22815008e9d41f0660ebd396047e82b Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Fri, 15 May 2020 10:44:30 +0200 Subject: [PATCH 02/17] do not send requests in different coroutine scope, do it in main --- .../wire/bots/polls/services/PollService.kt | 63 ++++++++----------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/src/main/kotlin/com/wire/bots/polls/services/PollService.kt b/src/main/kotlin/com/wire/bots/polls/services/PollService.kt index 395fa6e..43507f8 100644 --- a/src/main/kotlin/com/wire/bots/polls/services/PollService.kt +++ b/src/main/kotlin/com/wire/bots/polls/services/PollService.kt @@ -8,8 +8,6 @@ import com.wire.bots.polls.dto.UsersInput import com.wire.bots.polls.dto.bot.confirmVote import com.wire.bots.polls.dto.bot.newPoll import com.wire.bots.polls.parser.PollFactory -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.launch import mu.KLogging import java.util.UUID @@ -40,21 +38,18 @@ class PollService( val pollId = repository.savePoll(poll, pollId = UUID.randomUUID().toString(), userId = usersInput.userId, botSelfId = botId) logger.info { "Poll successfully created with id: $pollId" } - // send response with async way - GlobalScope.launch { - proxySenderService.send( - token, - message = newPoll( - id = pollId, - body = poll.question.body, - buttons = poll.options, - mentions = poll.question.mentions - ) - ).whenNull { - logger.error { "It was not possible to send the poll to the Roman!" } - }?.also { (messageId) -> - logger.info { "Poll successfully created with id: $messageId" } - } + proxySenderService.send( + token, + message = newPoll( + id = pollId, + body = poll.question.body, + buttons = poll.options, + mentions = poll.question.mentions + ) + ).whenNull { + logger.error { "It was not possible to send the poll to the Roman!" } + }?.also { (messageId) -> + logger.info { "Poll successfully created with id: $messageId" } } return pollId @@ -63,10 +58,8 @@ class PollService( private suspend fun pollNotParsedFallback(token: String, usersInput: UsersInput) { usersInput.input.startsWith("/poll").whenTrue { - GlobalScope.launch { - logger.info { "Command started with /poll, sending usage to user." } - userCommunicationService.reactionToWrongCommand(token) - } + logger.info { "Command started with /poll, sending usage to user." } + userCommunicationService.reactionToWrongCommand(token) } } @@ -79,20 +72,18 @@ class PollService( repository.vote(pollAction) logger.info { "Vote registered." } - GlobalScope.launch { - proxySenderService.send( - token = token, - message = confirmVote( - pollId = pollAction.pollId, - offset = pollAction.optionId, - userId = pollAction.userId - ) - ).whenNull { - logger.error { "It was not possible to send response to vote." } - }?.also { (messageId) -> - logger.info { "Proxy received confirmation for vote under id: $messageId" } - sendStatsIfAllVoted(token, pollAction.pollId) - } + proxySenderService.send( + token = token, + message = confirmVote( + pollId = pollAction.pollId, + offset = pollAction.optionId, + userId = pollAction.userId + ) + ).whenNull { + logger.error { "It was not possible to send response to vote." } + }?.also { (messageId) -> + logger.info { "Proxy received confirmation for vote under id: $messageId" } + sendStatsIfAllVoted(token, pollAction.pollId) } } @@ -122,7 +113,7 @@ class PollService( val stats = statsFormattingService.formatStats(pollId, conversationMembersCount) .whenNull { logger.warn { "It was not possible to format stats for poll $pollId" } } ?: return - GlobalScope.launch { proxySenderService.send(token, stats) } + proxySenderService.send(token, stats) } /** From c0c1973f07d240f63eefdafafc981b99ebf1e6cc Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Fri, 15 May 2020 10:53:32 +0200 Subject: [PATCH 03/17] renaming --- .../setup/logging/{ProductionLayout.kt => JsonLoggingLayout.kt} | 2 +- src/main/resources/logback.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/main/kotlin/com/wire/bots/polls/setup/logging/{ProductionLayout.kt => JsonLoggingLayout.kt} (96%) diff --git a/src/main/kotlin/com/wire/bots/polls/setup/logging/ProductionLayout.kt b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt similarity index 96% rename from src/main/kotlin/com/wire/bots/polls/setup/logging/ProductionLayout.kt rename to src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt index bd51276..0882015 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/logging/ProductionLayout.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt @@ -11,7 +11,7 @@ import java.time.format.DateTimeFormatter /** * Layout logging into jsons. */ -class ProductionLayout : LayoutBase() { +class JsonLoggingLayout : LayoutBase() { private companion object { val dateTimeFormatter: DateTimeFormatter = diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index c99564f..51c502e 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -7,7 +7,7 @@ - + From f04df6640f2708056edae0139ad9e6144023eaf2 Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Fri, 15 May 2020 10:59:27 +0200 Subject: [PATCH 04/17] use existing timestamp instead of generating new one --- .../com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt index 0882015..592f167 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt @@ -22,7 +22,7 @@ class JsonLoggingLayout : LayoutBase() { override fun doLayout(event: ILoggingEvent): String = with(StringBuffer(256)) { append("{") - appendJson("@timestamp", getTime()) + appendJson("@timestamp", formatTime(event)) event.mdcPropertyMap[INFRA_REQUEST]?.let { appendJson("infra_request", it) @@ -44,5 +44,6 @@ class JsonLoggingLayout : LayoutBase() { private fun StringBuffer.appendJson(key: String, value: String, ending: String = ","): StringBuffer = append("\"$key\":\"$value\"$ending") - private fun getTime(): String = dateTimeFormatter.format(Instant.now()) + private fun formatTime(event: ILoggingEvent): String = + dateTimeFormatter.format(Instant.ofEpochMilli(event.timeStamp)) } From 6042a9fff514a54b3395559ea5883dd72b1caf30 Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Fri, 15 May 2020 15:14:15 +0200 Subject: [PATCH 05/17] add support for exception stacktrace --- .../polls/setup/logging/JsonLoggingLayout.kt | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt index 592f167..dc5e1d6 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt @@ -1,13 +1,17 @@ package com.wire.bots.polls.setup.logging +import ai.blindspot.ktoolz.extensions.createJson import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.classic.spi.IThrowableProxy +import ch.qos.logback.classic.spi.ThrowableProxyUtil import ch.qos.logback.core.CoreConstants import ch.qos.logback.core.LayoutBase import java.time.Instant import java.time.ZoneOffset import java.time.format.DateTimeFormatter + /** * Layout logging into jsons. */ @@ -35,12 +39,27 @@ class JsonLoggingLayout : LayoutBase() { appendJson("logger", event.loggerName) appendJson("message", event.formattedMessage) appendJson("level", event.level.levelStr) - appendJson("thread_name", event.threadName, "") + appendJson("thread_name", event.threadName) + + appendException(event.throwableProxy) + append("}") append(CoreConstants.LINE_SEPARATOR) toString() } + private fun StringBuffer.appendException(proxy: IThrowableProxy?) { + if (proxy == null) return + val json = createJson( + mapOf( + "message" to proxy.message, + "class" to proxy.className, + "stacktrace" to ThrowableProxyUtil.asString(proxy) + ) + ) + append("\"exception\":$json") + } + private fun StringBuffer.appendJson(key: String, value: String, ending: String = ","): StringBuffer = append("\"$key\":\"$value\"$ending") From eefe6bede64c134e7dfbdb3883bae4be59ac73a5 Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Fri, 15 May 2020 18:14:43 +0200 Subject: [PATCH 06/17] fix invalid json without exception --- .../polls/setup/logging/JsonLoggingLayout.kt | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt index dc5e1d6..f98c2fa 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt @@ -27,29 +27,30 @@ class JsonLoggingLayout : LayoutBase() { with(StringBuffer(256)) { append("{") appendJson("@timestamp", formatTime(event)) + appendJson("message", event.formattedMessage) + appendJson("logger", event.loggerName) + appendJson("level", event.level.levelStr) + // include nginx request id if exists event.mdcPropertyMap[INFRA_REQUEST]?.let { appendJson("infra_request", it) } - + // include app unique request id if exists event.mdcPropertyMap[APP_REQUEST]?.let { appendJson("app_request", it) } + // if this was an exception, include necessary data + if (event.throwableProxy != null) { + appendException(event.throwableProxy) + } + // json termination + appendJson("thread_name", event.threadName, ending = "}") - appendJson("logger", event.loggerName) - appendJson("message", event.formattedMessage) - appendJson("level", event.level.levelStr) - appendJson("thread_name", event.threadName) - - appendException(event.throwableProxy) - - append("}") append(CoreConstants.LINE_SEPARATOR) toString() } - private fun StringBuffer.appendException(proxy: IThrowableProxy?) { - if (proxy == null) return + private fun StringBuffer.appendException(proxy: IThrowableProxy) { val json = createJson( mapOf( "message" to proxy.message, @@ -57,7 +58,7 @@ class JsonLoggingLayout : LayoutBase() { "stacktrace" to ThrowableProxyUtil.asString(proxy) ) ) - append("\"exception\":$json") + append("\"exception\":$json,") } private fun StringBuffer.appendJson(key: String, value: String, ending: String = ","): StringBuffer = From ff8cc271aa3160543810ab243db279a490295b8b Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Mon, 18 May 2020 18:13:17 +0200 Subject: [PATCH 07/17] add wait for for execution in k8s --- Dockerfile | 3 ++ wait-for.sh | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100755 wait-for.sh diff --git a/Dockerfile b/Dockerfile index c98effb..244ec37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,9 @@ FROM adoptopenjdk/openjdk11:jre-11.0.6_10-alpine ENV APP_ROOT /app WORKDIR $APP_ROOT +# copy wait for script for running in kubernetes +COPY wait-for.sh ./wait-for.sh + # Obtain built from the base COPY --from=build /src/build/distributions/app.tar $APP_ROOT/ diff --git a/wait-for.sh b/wait-for.sh new file mode 100755 index 0000000..539a01d --- /dev/null +++ b/wait-for.sh @@ -0,0 +1,79 @@ +#!/bin/sh + +TIMEOUT=15 +QUIET=0 + +echoerr() { + if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi +} + +usage() { + exitcode="$1" + cat << USAGE >&2 +Usage: + $cmdname host:port [-t timeout] [-- command args] + -q | --quiet Do not output any status messages + -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit "$exitcode" +} + +wait_for() { + for i in `seq $TIMEOUT` ; do + nc -z "$HOST" "$PORT" > /dev/null 2>&1 + + result=$? + if [ $result -eq 0 ] ; then + if [ $# -gt 0 ] ; then + exec "$@" + fi + exit 0 + fi + sleep 1 + done + echo "Operation timed out" >&2 + exit 1 +} + +while [ $# -gt 0 ] +do + case "$1" in + *:* ) + HOST=$(printf "%s\n" "$1"| cut -d : -f 1) + PORT=$(printf "%s\n" "$1"| cut -d : -f 2) + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -t) + TIMEOUT="$2" + if [ "$TIMEOUT" = "" ]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + break + ;; + --help) + usage 0 + ;; + *) + echoerr "Unknown argument: $1" + usage 1 + ;; + esac +done + +if [ "$HOST" = "" -o "$PORT" = "" ]; then + echoerr "Error: you need to provide a host and port to test." + usage 2 +fi + +wait_for "$@" From 98f95d9c910a4bcdf56bc6ed012ff9d3d1d49c2f Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Tue, 19 May 2020 11:11:08 +0200 Subject: [PATCH 08/17] download wait-for from the repo rather than from local --- .dockerignore | 1 + Dockerfile | 16 +++++++---- wait-for.sh | 79 --------------------------------------------------- 3 files changed, 12 insertions(+), 84 deletions(-) delete mode 100755 wait-for.sh diff --git a/.dockerignore b/.dockerignore index b27bee7..17225da 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ +Makefile .gradle .idea build diff --git a/Dockerfile b/Dockerfile index 244ec37..dfabe55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,11 @@ LABEL project="wire-bots:polls" ENV PROJECT_ROOT /src WORKDIR $PROJECT_ROOT +# download wait-for script +RUN wget https://raw.githubusercontent.com/LukasForst/wait-for/master/wait-for +RUN chmod +x wait-for + +# ------------------ App specific ------------------ # Copy gradle settings COPY build.gradle.kts settings.gradle.kts gradle.properties gradlew $PROJECT_ROOT/ # Make sure gradlew is executable @@ -17,7 +22,7 @@ RUN ./gradlew --version RUN ./gradlew resolveDependencies --no-daemon # Copy project and build -COPY . $PROJECT_ROOT +COPY src $PROJECT_ROOT RUN ./gradlew distTar --no-daemon # Runtime @@ -26,9 +31,6 @@ FROM adoptopenjdk/openjdk11:jre-11.0.6_10-alpine ENV APP_ROOT /app WORKDIR $APP_ROOT -# copy wait for script for running in kubernetes -COPY wait-for.sh ./wait-for.sh - # Obtain built from the base COPY --from=build /src/build/distributions/app.tar $APP_ROOT/ @@ -36,12 +38,16 @@ COPY --from=build /src/build/distributions/app.tar $APP_ROOT/ RUN mkdir $APP_ROOT/run RUN tar -xvf app.tar --strip-components=1 -C $APP_ROOT/run +# ------------------ Wire common ------------------ +# copy wait for script for running in kubernetes +COPY --from=build /src/wait-for /wait-for # create version file ARG release_version=development ENV RELEASE_FILE_PATH=$APP_ROOT/run/release.txt RUN echo $release_version > $RELEASE_FILE_PATH - # enable json logging ENV JSON_LOGGING=true +# /------------------ Wire common ----------------- + EXPOSE 8080 ENTRYPOINT ["/bin/sh", "-c", "/app/run/bin/polls"] diff --git a/wait-for.sh b/wait-for.sh deleted file mode 100755 index 539a01d..0000000 --- a/wait-for.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/sh - -TIMEOUT=15 -QUIET=0 - -echoerr() { - if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi -} - -usage() { - exitcode="$1" - cat << USAGE >&2 -Usage: - $cmdname host:port [-t timeout] [-- command args] - -q | --quiet Do not output any status messages - -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit "$exitcode" -} - -wait_for() { - for i in `seq $TIMEOUT` ; do - nc -z "$HOST" "$PORT" > /dev/null 2>&1 - - result=$? - if [ $result -eq 0 ] ; then - if [ $# -gt 0 ] ; then - exec "$@" - fi - exit 0 - fi - sleep 1 - done - echo "Operation timed out" >&2 - exit 1 -} - -while [ $# -gt 0 ] -do - case "$1" in - *:* ) - HOST=$(printf "%s\n" "$1"| cut -d : -f 1) - PORT=$(printf "%s\n" "$1"| cut -d : -f 2) - shift 1 - ;; - -q | --quiet) - QUIET=1 - shift 1 - ;; - -t) - TIMEOUT="$2" - if [ "$TIMEOUT" = "" ]; then break; fi - shift 2 - ;; - --timeout=*) - TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - break - ;; - --help) - usage 0 - ;; - *) - echoerr "Unknown argument: $1" - usage 1 - ;; - esac -done - -if [ "$HOST" = "" -o "$PORT" = "" ]; then - echoerr "Error: you need to provide a host and port to test." - usage 2 -fi - -wait_for "$@" From 8cb36338e97087f8a0aaa2b8353eb1dacb946f40 Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Tue, 19 May 2020 15:41:25 +0200 Subject: [PATCH 09/17] serialize map instead of manual json creation --- .../polls/setup/logging/JsonLoggingLayout.kt | 60 ++++++++----------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt index f98c2fa..1631f82 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt @@ -23,46 +23,36 @@ class JsonLoggingLayout : LayoutBase() { .withZone(ZoneOffset.UTC) } - override fun doLayout(event: ILoggingEvent): String = - with(StringBuffer(256)) { - append("{") - appendJson("@timestamp", formatTime(event)) - appendJson("message", event.formattedMessage) - appendJson("logger", event.loggerName) - appendJson("level", event.level.levelStr) - - // include nginx request id if exists - event.mdcPropertyMap[INFRA_REQUEST]?.let { - appendJson("infra_request", it) - } - // include app unique request id if exists - event.mdcPropertyMap[APP_REQUEST]?.let { - appendJson("app_request", it) - } - // if this was an exception, include necessary data - if (event.throwableProxy != null) { - appendException(event.throwableProxy) - } - // json termination - appendJson("thread_name", event.threadName, ending = "}") + override fun doLayout(event: ILoggingEvent): String { + val finalMap: MutableMap = mutableMapOf( + "@timestamp" to formatTime(event), + "message" to event.formattedMessage, + "logger" to event.loggerName, + "level" to event.level.levelStr, + "thread_name" to event.threadName + ) - append(CoreConstants.LINE_SEPARATOR) - toString() + // include nginx request id if exists + event.mdcPropertyMap[INFRA_REQUEST]?.let { + finalMap["infra_request"] = it + } + // include app unique request id if exists + event.mdcPropertyMap[APP_REQUEST]?.let { + finalMap["app_request"] = it + } + // if this was an exception, include necessary data + if (event.throwableProxy != null) { + finalMap["exception"] = exception(event.throwableProxy) } - private fun StringBuffer.appendException(proxy: IThrowableProxy) { - val json = createJson( - mapOf( - "message" to proxy.message, - "class" to proxy.className, - "stacktrace" to ThrowableProxyUtil.asString(proxy) - ) - ) - append("\"exception\":$json,") + return createJson(finalMap) + CoreConstants.LINE_SEPARATOR } - private fun StringBuffer.appendJson(key: String, value: String, ending: String = ","): StringBuffer = - append("\"$key\":\"$value\"$ending") + private fun exception(proxy: IThrowableProxy) = mapOf( + "message" to proxy.message, + "class" to proxy.className, + "stacktrace" to ThrowableProxyUtil.asString(proxy) + ) private fun formatTime(event: ILoggingEvent): String = dateTimeFormatter.format(Instant.ofEpochMilli(event.timeStamp)) From 60ee7c14ed3acdaa557c5fd563fb77d213d75479 Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Tue, 19 May 2020 15:56:21 +0200 Subject: [PATCH 10/17] fix not working compose file --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index dfabe55..12feb30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ RUN ./gradlew --version RUN ./gradlew resolveDependencies --no-daemon # Copy project and build -COPY src $PROJECT_ROOT +COPY . $PROJECT_ROOT RUN ./gradlew distTar --no-daemon # Runtime From de7fe2c152b432bcef22d1306470a6e99f991037 Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Wed, 20 May 2020 11:36:53 +0200 Subject: [PATCH 11/17] include MDC logging of user sending a message --- .../com/wire/bots/polls/routing/MessagesRoute.kt | 5 +++++ .../bots/polls/services/ProxySenderService.kt | 2 +- .../polls/setup/logging/JsonLoggingLayout.kt | 16 ++++++++-------- .../bots/polls/setup/logging/MdcConfiguration.kt | 5 +++++ .../com/wire/bots/polls/utils/Extensions.kt | 16 ++++++++++++++++ 5 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/main/kotlin/com/wire/bots/polls/routing/MessagesRoute.kt b/src/main/kotlin/com/wire/bots/polls/routing/MessagesRoute.kt index 3d4404f..955b438 100644 --- a/src/main/kotlin/com/wire/bots/polls/routing/MessagesRoute.kt +++ b/src/main/kotlin/com/wire/bots/polls/routing/MessagesRoute.kt @@ -3,6 +3,8 @@ package com.wire.bots.polls.routing import com.wire.bots.polls.dto.roman.Message import com.wire.bots.polls.services.AuthService import com.wire.bots.polls.services.MessagesHandlingService +import com.wire.bots.polls.setup.logging.USER_ID +import com.wire.bots.polls.utils.mdc import io.ktor.application.call import io.ktor.http.HttpStatusCode import io.ktor.request.receive @@ -36,6 +38,9 @@ fun Routing.messages(k: LazyKodein) { call.respond(HttpStatusCode.BadRequest, "Bot did not understand the message.") }.onSuccess { routingLogger.debug { "Message parsed." } + // includes user id to current MDC + mdc(USER_ID) { it.userId } + handler.handle(it) routingLogger.debug { "Responding OK" } call.respond(HttpStatusCode.OK) diff --git a/src/main/kotlin/com/wire/bots/polls/services/ProxySenderService.kt b/src/main/kotlin/com/wire/bots/polls/services/ProxySenderService.kt index 746f0b5..f3c6821 100644 --- a/src/main/kotlin/com/wire/bots/polls/services/ProxySenderService.kt +++ b/src/main/kotlin/com/wire/bots/polls/services/ProxySenderService.kt @@ -32,7 +32,7 @@ class ProxySenderService(private val client: HttpClient, config: ProxyConfigurat * Send given message with provided token. */ suspend fun send(token: String, message: BotMessage): Response? { - logger.debug { "Sending\n:${createJson(message)}" } + logger.debug { "Sending: ${createJson(message)}" } return client.post(body = message) { url(conversationEndpoint) diff --git a/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt index 1631f82..2366331 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt @@ -32,14 +32,10 @@ class JsonLoggingLayout : LayoutBase() { "thread_name" to event.threadName ) - // include nginx request id if exists - event.mdcPropertyMap[INFRA_REQUEST]?.let { - finalMap["infra_request"] = it - } - // include app unique request id if exists - event.mdcPropertyMap[APP_REQUEST]?.let { - finalMap["app_request"] = it - } + finalMap.includeMdc(event, INFRA_REQUEST) + finalMap.includeMdc(event, APP_REQUEST) + finalMap.includeMdc(event, USER_ID) + // if this was an exception, include necessary data if (event.throwableProxy != null) { finalMap["exception"] = exception(event.throwableProxy) @@ -48,6 +44,10 @@ class JsonLoggingLayout : LayoutBase() { return createJson(finalMap) + CoreConstants.LINE_SEPARATOR } + private fun MutableMap.includeMdc(event: ILoggingEvent, mdcKey: String, mapKey: String = mdcKey) { + event.mdcPropertyMap[mdcKey]?.also { mdcValue -> this[mapKey] = mdcValue } + } + private fun exception(proxy: IThrowableProxy) = mapOf( "message" to proxy.message, "class" to proxy.className, diff --git a/src/main/kotlin/com/wire/bots/polls/setup/logging/MdcConfiguration.kt b/src/main/kotlin/com/wire/bots/polls/setup/logging/MdcConfiguration.kt index a500163..13a3251 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/logging/MdcConfiguration.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/logging/MdcConfiguration.kt @@ -9,3 +9,8 @@ const val INFRA_REQUEST = "infra_request" * ID generated by this application for particular request. */ const val APP_REQUEST = "app_request" + +/** + * ID of the user sending message. + */ +const val USER_ID = "user_id" diff --git a/src/main/kotlin/com/wire/bots/polls/utils/Extensions.kt b/src/main/kotlin/com/wire/bots/polls/utils/Extensions.kt index cd9f880..727a721 100644 --- a/src/main/kotlin/com/wire/bots/polls/utils/Extensions.kt +++ b/src/main/kotlin/com/wire/bots/polls/utils/Extensions.kt @@ -1,6 +1,7 @@ package com.wire.bots.polls.utils import mu.KLogging +import org.slf4j.MDC /** * Creates URL from [this] as base and [path] as path @@ -11,3 +12,18 @@ infix fun String.appendPath(path: String) = "${dropLastWhile { it == '/' }}/${pa * Creates logger with given name. */ fun createLogger(name: String) = KLogging().logger("com.wire.$name") + + +/** + * Includes value to current MDC under the key. + */ +inline fun mdc(key: String, value: () -> String?) = mdc(key, value()) + +/** + * Includes value to current MDC under the key if the key is not null. + */ +fun mdc(key: String, value: String?) { + if (value != null) { + MDC.put(key, value) + } +} From 501b4e89c6afdb9980bb0b35bd43187604523d61 Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Mon, 25 May 2020 14:30:44 +0200 Subject: [PATCH 12/17] use trace level --- src/main/resources/logback.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 51c502e..2adb7ba 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -27,6 +27,7 @@ - + + From 144907ac778669a87810efaadbb04cba7b6c61fd Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Tue, 26 May 2020 10:58:16 +0200 Subject: [PATCH 13/17] update entrypoint to match wire standard --- Dockerfile | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 12feb30..47b4d5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,15 +39,25 @@ RUN mkdir $APP_ROOT/run RUN tar -xvf app.tar --strip-components=1 -C $APP_ROOT/run # ------------------ Wire common ------------------ -# copy wait for script for running in kubernetes +# set APP_DIR - where is the entrypoint +ENV APP_DIR=/app/run/bin +# copy wait for script to root for running in kubernetes COPY --from=build /src/wait-for /wait-for # create version file ARG release_version=development -ENV RELEASE_FILE_PATH=$APP_ROOT/run/release.txt +ENV RELEASE_FILE_PATH=$APP_DIR/release.txt RUN echo $release_version > $RELEASE_FILE_PATH # enable json logging ENV JSON_LOGGING=true +# move to runtime directory +WORKDIR $APP_DIR # /------------------ Wire common ----------------- EXPOSE 8080 -ENTRYPOINT ["/bin/sh", "-c", "/app/run/bin/polls"] +# create entrypoint +RUN echo '\ +/bin/sh -c ./polls'\ +>> entrypoint.sh +RUN chmod +x entrypoint.sh + +ENTRYPOINT $APP_DIR/entrypoint.sh From 23b34b507d22735a18feecdae9e6311a942dfb6c Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Mon, 1 Jun 2020 10:13:55 +0200 Subject: [PATCH 14/17] disable using cloud sql proxy, use private IP instead --- Dockerfile | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 47b4d5a..786e630 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,11 +5,6 @@ LABEL project="wire-bots:polls" ENV PROJECT_ROOT /src WORKDIR $PROJECT_ROOT -# download wait-for script -RUN wget https://raw.githubusercontent.com/LukasForst/wait-for/master/wait-for -RUN chmod +x wait-for - -# ------------------ App specific ------------------ # Copy gradle settings COPY build.gradle.kts settings.gradle.kts gradle.properties gradlew $PROJECT_ROOT/ # Make sure gradlew is executable @@ -38,26 +33,14 @@ COPY --from=build /src/build/distributions/app.tar $APP_ROOT/ RUN mkdir $APP_ROOT/run RUN tar -xvf app.tar --strip-components=1 -C $APP_ROOT/run -# ------------------ Wire common ------------------ -# set APP_DIR - where is the entrypoint -ENV APP_DIR=/app/run/bin -# copy wait for script to root for running in kubernetes -COPY --from=build /src/wait-for /wait-for +# ------------------- Wire common ----------------- # create version file ARG release_version=development -ENV RELEASE_FILE_PATH=$APP_DIR/release.txt +ENV RELEASE_FILE_PATH=$APP_ROOT/run/release.txt RUN echo $release_version > $RELEASE_FILE_PATH # enable json logging ENV JSON_LOGGING=true -# move to runtime directory -WORKDIR $APP_DIR # /------------------ Wire common ----------------- EXPOSE 8080 -# create entrypoint -RUN echo '\ -/bin/sh -c ./polls'\ ->> entrypoint.sh -RUN chmod +x entrypoint.sh - -ENTRYPOINT $APP_DIR/entrypoint.sh +ENTRYPOINT ["/bin/sh", "-c", "/app/run/bin/polls"] From 97721fe472b055e6a7f56810ed89c657425a40ab Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Mon, 1 Jun 2020 14:05:54 +0200 Subject: [PATCH 15/17] disable json logging by default --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 786e630..9c81b8e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,8 @@ ARG release_version=development ENV RELEASE_FILE_PATH=$APP_ROOT/run/release.txt RUN echo $release_version > $RELEASE_FILE_PATH # enable json logging -ENV JSON_LOGGING=true +# TODO enable this once we fully migrate to JSON logging everywhere +# ENV JSON_LOGGING=true # /------------------ Wire common ----------------- EXPOSE 8080 From 84e32258acc96cd6150c00c9b3db767367fd1c77 Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Wed, 7 Oct 2020 12:46:03 +0200 Subject: [PATCH 16/17] update dependencies to the latest versions --- build.gradle.kts | 25 ++++++++++--------- gradle/wrapper/gradle-wrapper.properties | 2 +- .../integration_tests/routing/Routing.kt | 2 +- .../setup/ConfigurationDependencyInjection.kt | 4 +-- .../bots/polls/integration_tests/FlowTest.kt | 2 +- .../com/wire/bots/polls/dao/PollRepository.kt | 2 +- .../com/wire/bots/polls/parser/PollFactory.kt | 4 +-- .../wire/bots/polls/services/AuthService.kt | 2 +- .../wire/bots/polls/services/PollService.kt | 4 +-- .../bots/polls/services/ProxySenderService.kt | 2 +- .../polls/services/StatsFormattingService.kt | 6 ++--- .../setup/ConfigurationDependencyInjection.kt | 4 +-- .../com/wire/bots/polls/setup/HttpClient.kt | 11 ++++---- .../wire/bots/polls/setup/KtorInstallation.kt | 7 ++++-- .../polls/setup/logging/JsonLoggingLayout.kt | 2 +- 15 files changed, 41 insertions(+), 38 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 0d24f15..3410f77 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,17 +1,17 @@ plugins { - kotlin("jvm") version "1.3.71" + kotlin("jvm") version "1.4.10" application distribution - id("net.nemerosa.versioning") version "2.12.1" + id("net.nemerosa.versioning") version "2.14.0" } group = "com.wire.bots.polls" version = versioning.info?.tag ?: versioning.info?.lastTag ?: "development" -val mainClass = "com.wire.bots.polls.PollBotKt" +val mClass = "com.wire.bots.polls.PollBotKt" application { - mainClassName = mainClass + mainClass.set(mClass) } repositories { @@ -22,10 +22,11 @@ dependencies { // stdlib implementation(kotlin("stdlib-jdk8")) // extension functions - implementation("ai.blindspot.ktoolz", "ktoolz", "1.0.6") + implementation("pw.forst.tools", "katlib", "1.1.2") + // Ktor server dependencies - val ktorVersion = "1.3.2" + val ktorVersion = "1.4.1" implementation("io.ktor", "ktor-server-core", ktorVersion) implementation("io.ktor", "ktor-server-netty", ktorVersion) implementation("io.ktor", "ktor-jackson", ktorVersion) @@ -39,23 +40,23 @@ dependencies { // Prometheus metrics implementation("io.ktor", "ktor-metrics-micrometer", ktorVersion) - implementation("io.micrometer", "micrometer-registry-prometheus", "1.4.1") + implementation("io.micrometer", "micrometer-registry-prometheus", "1.5.5") // logging - implementation("io.github.microutils", "kotlin-logging", "1.7.9") + implementation("io.github.microutils", "kotlin-logging", "2.0.3") // if-else in logback.xml implementation("org.codehaus.janino", "janino", "3.1.2") implementation("ch.qos.logback", "logback-classic", "1.2.3") // DI - val kodeinVersion = "6.5.4" + val kodeinVersion = "6.5.5" implementation("org.kodein.di", "kodein-di-generic-jvm", kodeinVersion) implementation("org.kodein.di", "kodein-di-framework-ktor-server-jvm", kodeinVersion) // database implementation("org.postgresql", "postgresql", "42.2.2") - val exposedVersion = "0.22.1" + val exposedVersion = "0.27.1" implementation("org.jetbrains.exposed", "exposed-core", exposedVersion) implementation("org.jetbrains.exposed", "exposed-dao", exposedVersion) implementation("org.jetbrains.exposed", "exposed-jdbc", exposedVersion) @@ -63,7 +64,7 @@ dependencies { implementation("pw.forst", "exposed-upsert", "1.0") // database migrations from the code - implementation("org.flywaydb", "flyway-core", "6.3.2") + implementation("org.flywaydb", "flyway-core", "7.0.0") } tasks { @@ -84,7 +85,7 @@ tasks { register("fatJar") { manifest { - attributes["Main-Class"] = mainClass + attributes["Main-Class"] = mClass } duplicatesStrategy = DuplicatesStrategy.EXCLUDE archiveFileName.set("polls.jar") diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a4b4429..6c9a224 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/integrationTests/src/main/kotlin/com/wire/bots/polls/integration_tests/routing/Routing.kt b/integrationTests/src/main/kotlin/com/wire/bots/polls/integration_tests/routing/Routing.kt index bbf3f10..3b6c58c 100644 --- a/integrationTests/src/main/kotlin/com/wire/bots/polls/integration_tests/routing/Routing.kt +++ b/integrationTests/src/main/kotlin/com/wire/bots/polls/integration_tests/routing/Routing.kt @@ -1,6 +1,6 @@ package com.wire.bots.polls.integration_tests.routing -import ai.blindspot.ktoolz.extensions.whenNull +import pw.forst.tools.katlib.whenNull import com.wire.bots.polls.integration_tests.dto.Conversation import com.wire.bots.polls.integration_tests.dto.ProxyResponseMessage import com.wire.bots.polls.integration_tests.store.tokenStorage diff --git a/integrationTests/src/main/kotlin/com/wire/bots/polls/integration_tests/setup/ConfigurationDependencyInjection.kt b/integrationTests/src/main/kotlin/com/wire/bots/polls/integration_tests/setup/ConfigurationDependencyInjection.kt index 46fe7c7..dedea67 100644 --- a/integrationTests/src/main/kotlin/com/wire/bots/polls/integration_tests/setup/ConfigurationDependencyInjection.kt +++ b/integrationTests/src/main/kotlin/com/wire/bots/polls/integration_tests/setup/ConfigurationDependencyInjection.kt @@ -1,7 +1,7 @@ package com.wire.bots.polls.integration_tests.setup -import ai.blindspot.ktoolz.extensions.getEnv -import ai.blindspot.ktoolz.extensions.whenNull +import pw.forst.tools.katlib.getEnv +import pw.forst.tools.katlib.whenNull import com.wire.bots.polls.integration_tests.dto.BotApiConfiguration import com.wire.bots.polls.integration_tests.setup.EnvConfigVariables.BOT_API import com.wire.bots.polls.integration_tests.setup.EnvConfigVariables.SERVICE_TOKEN diff --git a/integrationTests/src/test/kotlin/com/wire/bots/polls/integration_tests/FlowTest.kt b/integrationTests/src/test/kotlin/com/wire/bots/polls/integration_tests/FlowTest.kt index c829581..eeae656 100644 --- a/integrationTests/src/test/kotlin/com/wire/bots/polls/integration_tests/FlowTest.kt +++ b/integrationTests/src/test/kotlin/com/wire/bots/polls/integration_tests/FlowTest.kt @@ -1,6 +1,6 @@ package com.wire.bots.polls.integration_tests -import ai.blindspot.ktoolz.extensions.newLine +import pw.forst.tools.katlib.newLine import com.wire.bots.polls.integration_tests.dto.botRequest import com.wire.bots.polls.integration_tests.dto.init import com.wire.bots.polls.integration_tests.dto.reaction diff --git a/src/main/kotlin/com/wire/bots/polls/dao/PollRepository.kt b/src/main/kotlin/com/wire/bots/polls/dao/PollRepository.kt index 34841b2..f7ff8c4 100644 --- a/src/main/kotlin/com/wire/bots/polls/dao/PollRepository.kt +++ b/src/main/kotlin/com/wire/bots/polls/dao/PollRepository.kt @@ -1,6 +1,5 @@ package com.wire.bots.polls.dao -import ai.blindspot.ktoolz.extensions.mapToSet import com.wire.bots.polls.dto.PollAction import com.wire.bots.polls.dto.PollDto import com.wire.bots.polls.dto.Question @@ -14,6 +13,7 @@ import org.jetbrains.exposed.sql.insert import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction import pw.forst.exposed.insertOrUpdate +import pw.forst.tools.katlib.mapToSet /** * Simple repository for handling database transactions on one place. diff --git a/src/main/kotlin/com/wire/bots/polls/parser/PollFactory.kt b/src/main/kotlin/com/wire/bots/polls/parser/PollFactory.kt index 2c7f79a..6199387 100644 --- a/src/main/kotlin/com/wire/bots/polls/parser/PollFactory.kt +++ b/src/main/kotlin/com/wire/bots/polls/parser/PollFactory.kt @@ -1,10 +1,10 @@ package com.wire.bots.polls.parser -import ai.blindspot.ktoolz.extensions.newLine -import ai.blindspot.ktoolz.extensions.whenNull import com.wire.bots.polls.dto.PollDto import com.wire.bots.polls.dto.UsersInput import mu.KLogging +import pw.forst.tools.katlib.newLine +import pw.forst.tools.katlib.whenNull /** * Class used for creating the polls from the text. Parsing and creating the poll objects. diff --git a/src/main/kotlin/com/wire/bots/polls/services/AuthService.kt b/src/main/kotlin/com/wire/bots/polls/services/AuthService.kt index 34c43ad..31197ea 100644 --- a/src/main/kotlin/com/wire/bots/polls/services/AuthService.kt +++ b/src/main/kotlin/com/wire/bots/polls/services/AuthService.kt @@ -1,8 +1,8 @@ package com.wire.bots.polls.services -import ai.blindspot.ktoolz.extensions.whenNull import io.ktor.http.Headers import mu.KLogging +import pw.forst.tools.katlib.whenNull /** * Authentication service. diff --git a/src/main/kotlin/com/wire/bots/polls/services/PollService.kt b/src/main/kotlin/com/wire/bots/polls/services/PollService.kt index 43507f8..0df5bc1 100644 --- a/src/main/kotlin/com/wire/bots/polls/services/PollService.kt +++ b/src/main/kotlin/com/wire/bots/polls/services/PollService.kt @@ -1,7 +1,5 @@ package com.wire.bots.polls.services -import ai.blindspot.ktoolz.extensions.whenNull -import ai.blindspot.ktoolz.extensions.whenTrue import com.wire.bots.polls.dao.PollRepository import com.wire.bots.polls.dto.PollAction import com.wire.bots.polls.dto.UsersInput @@ -9,6 +7,8 @@ import com.wire.bots.polls.dto.bot.confirmVote import com.wire.bots.polls.dto.bot.newPoll import com.wire.bots.polls.parser.PollFactory import mu.KLogging +import pw.forst.tools.katlib.whenNull +import pw.forst.tools.katlib.whenTrue import java.util.UUID /** diff --git a/src/main/kotlin/com/wire/bots/polls/services/ProxySenderService.kt b/src/main/kotlin/com/wire/bots/polls/services/ProxySenderService.kt index f3c6821..009b984 100644 --- a/src/main/kotlin/com/wire/bots/polls/services/ProxySenderService.kt +++ b/src/main/kotlin/com/wire/bots/polls/services/ProxySenderService.kt @@ -1,6 +1,5 @@ package com.wire.bots.polls.services -import ai.blindspot.ktoolz.extensions.createJson import com.wire.bots.polls.dto.bot.BotMessage import com.wire.bots.polls.dto.roman.Response import com.wire.bots.polls.utils.appendPath @@ -15,6 +14,7 @@ import io.ktor.http.ContentType import io.ktor.http.contentType import io.ktor.http.isSuccess import mu.KLogging +import pw.forst.tools.katlib.createJson import java.nio.charset.Charset /** diff --git a/src/main/kotlin/com/wire/bots/polls/services/StatsFormattingService.kt b/src/main/kotlin/com/wire/bots/polls/services/StatsFormattingService.kt index 88c9370..ec83ec0 100644 --- a/src/main/kotlin/com/wire/bots/polls/services/StatsFormattingService.kt +++ b/src/main/kotlin/com/wire/bots/polls/services/StatsFormattingService.kt @@ -1,11 +1,11 @@ package com.wire.bots.polls.services -import ai.blindspot.ktoolz.extensions.newLine -import ai.blindspot.ktoolz.extensions.whenNull import com.wire.bots.polls.dao.PollRepository import com.wire.bots.polls.dto.bot.BotMessage import com.wire.bots.polls.dto.bot.statsMessage import mu.KLogging +import pw.forst.tools.katlib.newLine +import pw.forst.tools.katlib.whenNull class StatsFormattingService( private val repository: PollRepository @@ -39,7 +39,7 @@ class StatsFormattingService( private fun formatVotes(stats: Map, Int>, conversationMembers: Int?): String { // we can use assert as the result size is checked - val maxVotes = stats.values.max()!! + val maxVotes = requireNotNull(stats.values.maxOrNull()) { "There were no stats!" } return stats .map { (option, votingUsers) -> VotingOption(if (votingUsers == maxVotes) "**" else "*", option.second, votingUsers) diff --git a/src/main/kotlin/com/wire/bots/polls/setup/ConfigurationDependencyInjection.kt b/src/main/kotlin/com/wire/bots/polls/setup/ConfigurationDependencyInjection.kt index a564441..2ee5036 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/ConfigurationDependencyInjection.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/ConfigurationDependencyInjection.kt @@ -1,7 +1,5 @@ package com.wire.bots.polls.setup -import ai.blindspot.ktoolz.extensions.getEnv -import ai.blindspot.ktoolz.extensions.whenNull import com.wire.bots.polls.dto.conf.DatabaseConfiguration import com.wire.bots.polls.services.ProxyConfiguration import com.wire.bots.polls.setup.EnvConfigVariables.DB_PASSWORD @@ -13,6 +11,8 @@ import com.wire.bots.polls.utils.createLogger import org.kodein.di.Kodein.MainBuilder import org.kodein.di.generic.bind import org.kodein.di.generic.singleton +import pw.forst.tools.katlib.getEnv +import pw.forst.tools.katlib.whenNull import java.io.File private val logger = createLogger("EnvironmentLoaderLogger") diff --git a/src/main/kotlin/com/wire/bots/polls/setup/HttpClient.kt b/src/main/kotlin/com/wire/bots/polls/setup/HttpClient.kt index aa459b4..4f3869a 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/HttpClient.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/HttpClient.kt @@ -10,6 +10,7 @@ import io.ktor.client.features.logging.LogLevel import io.ktor.client.features.logging.Logger import io.ktor.client.features.logging.Logging import io.ktor.client.features.observer.ResponseObserver +import io.ktor.utils.io.discard import io.micrometer.core.instrument.MeterRegistry @@ -23,12 +24,10 @@ fun createHttpClient(meterRegistry: MeterRegistry) = } // TODO check https://github.com/ktorio/ktor/issues/1813 - @Suppress("ConstantConditionIf") // temporary disabled until https://github.com/ktorio/ktor/issues/1813 is resolved - if (false) { - install(ResponseObserver) { - onResponse { - meterRegistry.httpCall(it) - } + install(ResponseObserver) { + onResponse { + meterRegistry.httpCall(it) + it.content.discard() } } diff --git a/src/main/kotlin/com/wire/bots/polls/setup/KtorInstallation.kt b/src/main/kotlin/com/wire/bots/polls/setup/KtorInstallation.kt index 7ffb7ba..4b23891 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/KtorInstallation.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/KtorInstallation.kt @@ -76,13 +76,16 @@ fun connectDatabase(k: LazyKodein) { */ fun migrateDatabase(dbConfig: DatabaseConfiguration) { installationLogger.info { "Migrating database." } - val migrationsCount = Flyway + val migrateResult = Flyway .configure() .dataSource(dbConfig.url, dbConfig.userName, dbConfig.password) .load() .migrate() - installationLogger.info { if (migrationsCount == 0) "No migrations necessary." else "Applied $migrationsCount migrations." } + installationLogger.info { + if (migrateResult.migrationsExecuted == 0) "No migrations necessary." + else "Applied ${migrateResult.migrationsExecuted} migrations." + } } /** diff --git a/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt index 2366331..61ed097 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/logging/JsonLoggingLayout.kt @@ -1,12 +1,12 @@ package com.wire.bots.polls.setup.logging -import ai.blindspot.ktoolz.extensions.createJson import ch.qos.logback.classic.spi.ILoggingEvent import ch.qos.logback.classic.spi.IThrowableProxy import ch.qos.logback.classic.spi.ThrowableProxyUtil import ch.qos.logback.core.CoreConstants import ch.qos.logback.core.LayoutBase +import pw.forst.tools.katlib.createJson import java.time.Instant import java.time.ZoneOffset import java.time.format.DateTimeFormatter From 0a3ab8d8ed066e73fb166df12f43df035da8d301 Mon Sep 17 00:00:00 2001 From: Lukas Forst Date: Wed, 7 Oct 2020 12:56:16 +0200 Subject: [PATCH 17/17] disable observer --- .../kotlin/com/wire/bots/polls/setup/HttpClient.kt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/com/wire/bots/polls/setup/HttpClient.kt b/src/main/kotlin/com/wire/bots/polls/setup/HttpClient.kt index 4f3869a..aa459b4 100644 --- a/src/main/kotlin/com/wire/bots/polls/setup/HttpClient.kt +++ b/src/main/kotlin/com/wire/bots/polls/setup/HttpClient.kt @@ -10,7 +10,6 @@ import io.ktor.client.features.logging.LogLevel import io.ktor.client.features.logging.Logger import io.ktor.client.features.logging.Logging import io.ktor.client.features.observer.ResponseObserver -import io.ktor.utils.io.discard import io.micrometer.core.instrument.MeterRegistry @@ -24,10 +23,12 @@ fun createHttpClient(meterRegistry: MeterRegistry) = } // TODO check https://github.com/ktorio/ktor/issues/1813 - install(ResponseObserver) { - onResponse { - meterRegistry.httpCall(it) - it.content.discard() + @Suppress("ConstantConditionIf") // temporary disabled until https://github.com/ktorio/ktor/issues/1813 is resolved + if (false) { + install(ResponseObserver) { + onResponse { + meterRegistry.httpCall(it) + } } }