diff --git a/backend/edsl/src/main/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/package.scala b/backend/edsl/src/main/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/package.scala index f41662dc..c03f399d 100644 --- a/backend/edsl/src/main/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/package.scala +++ b/backend/edsl/src/main/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/package.scala @@ -1,6 +1,7 @@ package ru.tinkoff.tcb.mockingbird.edsl import sttp.client4.* +import sttp.client4.DuplicateHeaderBehavior import sttp.model.Uri import sttp.model.Uri.QuerySegment @@ -16,7 +17,7 @@ package object interpreter { def buildRequest(host: Uri, m: HttpRequest): Request[String] = { val initialRequest = emptyRequest.response(asStringAlways) var req = m.body.fold(initialRequest)(initialRequest.body) - req = m.headers.foldLeft(req) { case (r, (k, v)) => r.header(k, v, replaceExisting = true) } + req = m.headers.foldLeft(req) { case (r, (k, v)) => r.header(k, v, DuplicateHeaderBehavior.Replace) } val url = makeUri(host, m) m.method match { case Delete => req.delete(url) diff --git a/backend/edsl/src/test/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/AsyncScalaTestSuiteTest.scala b/backend/edsl/src/test/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/AsyncScalaTestSuiteTest.scala index a52375e0..87e0cb4b 100644 --- a/backend/edsl/src/test/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/AsyncScalaTestSuiteTest.scala +++ b/backend/edsl/src/test/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/AsyncScalaTestSuiteTest.scala @@ -9,11 +9,13 @@ import org.scalatest.BeforeAndAfterEach import org.scalatest.Informer import org.scalatest.matchers.should.Matchers import sttp.client4.* +import sttp.client4.Response import sttp.client4.httpclient.HttpClientFutureBackend import sttp.client4.testing.WebSocketBackendStub import sttp.model.Header import sttp.model.MediaType import sttp.model.Method.* +import sttp.model.RequestMetadata import sttp.model.StatusCode import sttp.model.Uri @@ -67,7 +69,14 @@ class AsyncScalaTestSuiteTest extends AsyncScalaTestSuite with Matchers with Asy if uri == uri"http://some.domain.com:8090/api/handler?service=world" && hs.exists(h => h.name == "x-token" && h.value == "asd5453qwe") && hs.exists(h => h.name == "Content-Type" && h.value == "application/json") => - Response(body = "got request", code = StatusCode.Ok) + new Response[String]( + body = "got request", + code = StatusCode.Ok, + statusText = "", + headers = Seq.empty, + history = Nil, + request = RequestMetadata(POST, uri, hs), + ) } val example = eset.sendHttp(method, path, body.some, headers, query) @@ -81,11 +90,13 @@ class AsyncScalaTestSuiteTest extends AsyncScalaTestSuite with Matchers with Asy test("checkHttp checks code of response") { sttpbackend_ = HttpClientFutureBackend.stub().whenRequestMatches(_ => true).thenRespondOk() - val sttpResp = Response( + val sttpResp = new Response( body = "got request", code = StatusCode.InternalServerError, statusText = "", headers = Seq.empty, + history = Nil, + request = RequestMetadata(GET, uri"https://host.domain", Seq.empty) ) val example = eset.checkHttp( @@ -105,11 +116,13 @@ class AsyncScalaTestSuiteTest extends AsyncScalaTestSuite with Matchers with Asy test("checkHttp checks body of response") { sttpbackend_ = HttpClientFutureBackend.stub().whenRequestMatches(_ => true).thenRespondOk() - val sttpResp = Response( + val sttpResp = new Response( body = "got request", code = StatusCode.Ok, statusText = "", headers = Seq.empty, + history = Nil, + request = RequestMetadata(GET, uri"https://host.domain", Seq.empty) ) val example = eset.checkHttp( @@ -129,11 +142,13 @@ class AsyncScalaTestSuiteTest extends AsyncScalaTestSuite with Matchers with Asy test("checkHttp checks headers of response") { sttpbackend_ = HttpClientFutureBackend.stub().whenRequestMatches(_ => true).thenRespondOk() - val sttpResp = Response( + val sttpResp = new Response( body = "{}", code = StatusCode.Ok, statusText = "", headers = Seq(Header.contentType(MediaType.TextPlain)), + history = Nil, + request = RequestMetadata(GET, uri"https://host.domain", Seq.empty) ) val example = eset.checkHttp( diff --git a/backend/edsl/src/test/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/AsyncScalaTestSuiteWholeTest.scala b/backend/edsl/src/test/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/AsyncScalaTestSuiteWholeTest.scala index c68c140f..e61bebbb 100644 --- a/backend/edsl/src/test/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/AsyncScalaTestSuiteWholeTest.scala +++ b/backend/edsl/src/test/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/AsyncScalaTestSuiteWholeTest.scala @@ -14,6 +14,7 @@ import sttp.client4.testing.WebSocketBackendStub import sttp.model.Header import sttp.model.MediaType import sttp.model.Method.* +import sttp.model.RequestMetadata import sttp.model.StatusCode import sttp.model.Uri @@ -61,14 +62,20 @@ class AsyncScalaTestSuiteWholeTest req.headers.exists(h => h.name == "X-CSRF-TOKEN" && h.value == "unEENxJqSLS02rji2GjcKzNLc0C0ySlWih9hSxwn") } .thenRespond( - Response( + new Response( body = """{ | "fact" : "There are approximately 100 breeds of cat.", | "length" : 42.0 |}""".stripMargin, code = StatusCode.Ok, statusText = "", - headers = Seq(Header.contentType(MediaType.ApplicationJson)) + headers = Seq(Header.contentType(MediaType.ApplicationJson)), + history = Nil, + request = RequestMetadata( + GET, + uri"https://localhost.example:9977/fact", + Seq(Header("X-CSRF-TOKEN", "unEENxJqSLS02rji2GjcKzNLc0C0ySlWih9hSxwn")) + ), ) ) } diff --git a/backend/mockingbird-api/src/main/scala/ru/tinkoff/tcb/mockingbird/Mockingbird.scala b/backend/mockingbird-api/src/main/scala/ru/tinkoff/tcb/mockingbird/Mockingbird.scala index b0ab4670..4188524a 100644 --- a/backend/mockingbird-api/src/main/scala/ru/tinkoff/tcb/mockingbird/Mockingbird.scala +++ b/backend/mockingbird-api/src/main/scala/ru/tinkoff/tcb/mockingbird/Mockingbird.scala @@ -31,6 +31,7 @@ import ru.tinkoff.tcb.mockingbird.api.Tracing import ru.tinkoff.tcb.mockingbird.api.UIHttp import ru.tinkoff.tcb.mockingbird.api.WLD import ru.tinkoff.tcb.mockingbird.api.WebAPI +import ru.tinkoff.tcb.mockingbird.config.HttpVersion import ru.tinkoff.tcb.mockingbird.config.MockingbirdConfiguration import ru.tinkoff.tcb.mockingbird.config.MongoCollections import ru.tinkoff.tcb.mockingbird.config.MongoConfig @@ -133,6 +134,10 @@ object Mockingbird extends scala.App { } httpClient = HttpClient .newBuilder() + .version(pc.httpVersion match { + case HttpVersion.HTTP_1_1 => HttpClient.Version.HTTP_1_1 + case HttpVersion.HTTP_2 => HttpClient.Version.HTTP_2 + }) .connectTimeout(sttpSettings.connectionTimeout.toJava) .pipe(b => sttpSettings.proxy.fold(b)(conf => b.proxy(conf.asJavaProxySelector))) .sslContext(sslContext) diff --git a/backend/mockingbird-api/src/main/scala/ru/tinkoff/tcb/mockingbird/api/WebAPI.scala b/backend/mockingbird-api/src/main/scala/ru/tinkoff/tcb/mockingbird/api/WebAPI.scala index 1ba92122..55d5b613 100644 --- a/backend/mockingbird-api/src/main/scala/ru/tinkoff/tcb/mockingbird/api/WebAPI.scala +++ b/backend/mockingbird-api/src/main/scala/ru/tinkoff/tcb/mockingbird/api/WebAPI.scala @@ -2,6 +2,7 @@ package ru.tinkoff.tcb.mockingbird.api import scala.jdk.CollectionConverters.* +import com.typesafe.config.ConfigRenderOptions import io.vertx.core.Vertx import io.vertx.core.http.HttpMethod import io.vertx.core.http.HttpServer @@ -26,10 +27,13 @@ object WebAPI { ui <- ZIO.service[UIHttp].toManaged metrics <- ZIO.service[MetricsHttp].toManaged server <- ZManaged.acquireReleaseWith(ZIO.attempt { - val vertx = Vertx.vertx() - val serverOptions = new HttpServerOptions().setMaxFormAttributeSize(256 * 1024) - val server = vertx.createHttpServer(serverOptions) - val router = Router.router(vertx) + val vertx = Vertx.vertx() + val serverOptions = { + val vertxCfg = serverConfig.vertx.root().render(ConfigRenderOptions.concise()) + new HttpServerOptions(new io.vertx.core.json.JsonObject(vertxCfg)) + } + val server = vertx.createHttpServer(serverOptions) + val router = Router.router(vertx) router .route() .path("/api/internal/mockingbird/v*") diff --git a/backend/mockingbird/src/main/resources/application.conf b/backend/mockingbird/src/main/resources/application.conf index fa8a37ce..c145ba57 100644 --- a/backend/mockingbird/src/main/resources/application.conf +++ b/backend/mockingbird/src/main/resources/application.conf @@ -22,6 +22,10 @@ ru.tinkoff.tcb { "http://localhost:3000", "http://localhost:8228" ] + vertx { + maxFormAttributeSize = 262144 + compressionSupported = true + } } proxy { @@ -29,6 +33,8 @@ ru.tinkoff.tcb { excludedResponseHeaders = [] insecureHosts = [] logOutgoingRequests = false + disableAutoDecompressForRaw = true + httpVersion = HTTP_1_1 } event { diff --git a/backend/mockingbird/src/main/scala/ru/tinkoff/tcb/mockingbird/api/PublicApiHandler.scala b/backend/mockingbird/src/main/scala/ru/tinkoff/tcb/mockingbird/api/PublicApiHandler.scala index 35f10659..3321b973 100644 --- a/backend/mockingbird/src/main/scala/ru/tinkoff/tcb/mockingbird/api/PublicApiHandler.scala +++ b/backend/mockingbird/src/main/scala/ru/tinkoff/tcb/mockingbird/api/PublicApiHandler.scala @@ -10,9 +10,11 @@ import io.circe.Json import io.circe.parser.parse import io.circe.syntax.* import io.estatico.newtype.ops.* +import mouse.boolean.* import mouse.option.* import sttp.client4.{Backend as SttpBackend, *} import sttp.client4.circe.* +import sttp.model.HeaderNames import sttp.model.Method import zio.interop.catz.core.* @@ -155,6 +157,12 @@ final class PublicApiHandler( } yield response } + // The cases when Mockingbird implicitly changes answer: + // 1. If the answer is compressed, mockingbird decompressed it. + // 2. When proxy mock contains substitutions, they change the answer and the result have another length. + // In these cases, these headers they make the modified response is incorrect, so we need to remove them. + private def headersDependingOnContentLength = Seq(HeaderNames.ContentEncoding, HeaderNames.ContentLength) + private def proxyRequest( method: HttpMethod, headers: Map[String, String], @@ -164,6 +172,9 @@ final class PublicApiHandler( val requestUri = uri"$uri".pipe(query.foldLeft(_) { case (u, (key, value)) => u.addParam(key, value) }) for { _ <- log.debug(s"Received headers: ${headers.keys.mkString(", ")}") + // Potentially, we want to pass the request and response as is. If the client wants this + // response to be encoded, it sets `Accept-Encoding` itself. Also, by default, we don't + // unpack the response here. It's the reason here we use emptyRequest. req = basicRequest .headers(headers -- proxyConfig.excludedRequestHeaders) .method(Method(method.entryName), requestUri) @@ -184,6 +195,7 @@ final class PublicApiHandler( ) } ) + .pipe(r => proxyConfig.disableAutoDecompressForRaw.fold(r.disableAutoDecompression, r)) .response(asByteArrayAlways) _ <- log.debug("Executing request: {}", req.toCurl).when(proxyConfig.logOutgoingRequests) response <- req @@ -193,6 +205,9 @@ final class PublicApiHandler( Refined.unsafeApply[Int, HttpStatusCodeRange](response.code.code), response.headers .filterNot(h => proxyConfig.excludedResponseHeaders(h.name)) + .pipe { hs => + proxyConfig.disableAutoDecompressForRaw.fold(hs, hs.filterNot(h => headersDependingOnContentLength.exists(h.is))) + } .map(h => h.name -> h.value) .toMap, response.body.coerce[ByteArray], @@ -218,7 +233,10 @@ final class PublicApiHandler( for { _ <- log.debug(s"Received headers: ${headers.keys.mkString(", ")}") req = basicRequest - .headers(headers -- proxyConfig.excludedRequestHeaders) + .headers( + // The basicRequest adds `Accept-Encoding`, so we should remove the same incoming header + headers.filterNot(_._1.equalsIgnoreCase(HeaderNames.AcceptEncoding)) -- proxyConfig.excludedRequestHeaders + ) .method(Method(method.entryName), requestUri) .pipe(rt => body match { @@ -248,6 +266,7 @@ final class PublicApiHandler( Refined.unsafeApply[Int, HttpStatusCodeRange](response.code.code), response.headers .filterNot(h => proxyConfig.excludedResponseHeaders(h.name)) + .filterNot(h => headersDependingOnContentLength.exists(h.is)) .map(h => h.name -> h.value) .toMap, jsonResponse.patch(data, patch).use(_.noSpaces), @@ -277,7 +296,10 @@ final class PublicApiHandler( for { _ <- log.debug(s"Received headers: ${headers.keys.mkString(", ")}") req = basicRequest - .headers(headers -- proxyConfig.excludedRequestHeaders) + .headers( + // The basicRequest adds `Accept-Encoding`, so we should remove the same incoming header + headers.filterNot(_._1.equalsIgnoreCase(HeaderNames.AcceptEncoding)) -- proxyConfig.excludedRequestHeaders + ) .method(Method(method.entryName), requestUri) .pipe(rt => body match { @@ -307,6 +329,7 @@ final class PublicApiHandler( Refined.unsafeApply[Int, HttpStatusCodeRange](response.code.code), response.headers .filterNot(h => proxyConfig.excludedResponseHeaders(h.name)) + .filterNot(h => headersDependingOnContentLength.exists(h.is)) .map(h => h.name -> h.value) .toMap, xmlResponse.patchFromValues(jData, xData, patch.map { case (k, v) => k.toZoom -> v }).toString(), diff --git a/backend/mockingbird/src/main/scala/ru/tinkoff/tcb/mockingbird/config/Configuration.scala b/backend/mockingbird/src/main/scala/ru/tinkoff/tcb/mockingbird/config/Configuration.scala index 66ffeedf..9e06c4b9 100644 --- a/backend/mockingbird/src/main/scala/ru/tinkoff/tcb/mockingbird/config/Configuration.scala +++ b/backend/mockingbird/src/main/scala/ru/tinkoff/tcb/mockingbird/config/Configuration.scala @@ -3,10 +3,13 @@ package ru.tinkoff.tcb.mockingbird.config import scala.concurrent.duration.FiniteDuration import com.typesafe.config.Config +import com.typesafe.config.ConfigException.Generic import com.typesafe.config.ConfigFactory +import enumeratum.* import net.ceedubs.ficus.Ficus.* import net.ceedubs.ficus.readers.ArbitraryTypeReader.* import net.ceedubs.ficus.readers.EnumerationReader.* +import net.ceedubs.ficus.readers.ValueReader final case class JsSandboxConfig(allowedClasses: Set[String] = Set()) @@ -15,7 +18,8 @@ final case class ServerConfig( port: Int, allowedOrigins: Seq[String], healthCheckRoute: Option[String], - sandbox: JsSandboxConfig + sandbox: JsSandboxConfig, + vertx: Config ) final case class SecurityConfig(secret: String) @@ -36,12 +40,28 @@ final case class ProxyServerConfig( auth: Option[ProxyServerAuth] ) +sealed trait HttpVersion extends EnumEntry +object HttpVersion extends Enum[HttpVersion] { + val values = findValues + case object HTTP_1_1 extends HttpVersion + case object HTTP_2 extends HttpVersion + + implicit val valueReader: ValueReader[HttpVersion] = ValueReader[String].map(s => + namesToValuesMap.get(s) match { + case Some(v) => v + case None => throw new Generic(s"Cannot get instance of enum HttpVersion from the value of $s") + } + ) +} + final case class ProxyConfig( excludedRequestHeaders: Seq[String], excludedResponseHeaders: Set[String], proxyServer: Option[ProxyServerConfig], insecureHosts: Seq[String], - logOutgoingRequests: Boolean + logOutgoingRequests: Boolean, + disableAutoDecompressForRaw: Boolean, + httpVersion: HttpVersion ) final case class EventConfig(fetchInterval: FiniteDuration, reloadInterval: FiniteDuration) diff --git a/backend/mockingbird/src/main/scala/ru/tinkoff/tcb/utils/xttp/package.scala b/backend/mockingbird/src/main/scala/ru/tinkoff/tcb/utils/xttp/package.scala index f2a393cd..542bc573 100644 --- a/backend/mockingbird/src/main/scala/ru/tinkoff/tcb/utils/xttp/package.scala +++ b/backend/mockingbird/src/main/scala/ru/tinkoff/tcb/utils/xttp/package.scala @@ -1,12 +1,13 @@ package ru.tinkoff.tcb.utils +import sttp.client4.DuplicateHeaderBehavior import sttp.client4.PartialRequest package object xttp { implicit class PartialRequestTXtras[T](private val rqt: PartialRequest[T]) extends AnyVal { def headersReplacing(hs: Map[String, String]): PartialRequest[T] = hs.foldLeft(rqt) { case (request, (key, value)) => - request.header(key, value, true) + request.header(key, value, DuplicateHeaderBehavior.Replace) } } } diff --git a/backend/project/Versions.scala b/backend/project/Versions.scala index e9050807..3d67dc4a 100644 --- a/backend/project/Versions.scala +++ b/backend/project/Versions.scala @@ -6,6 +6,6 @@ object Versions { val graalvm = "22.3.0" val micrometer = "1.8.5" val glass = "0.2.1" - val sttp = "4.0.0-M9" + val sttp = "4.0.0-M13" val zio = "2.0.19" } diff --git a/configuration.md b/configuration.md index 6af73373..7ab4efbd 100644 --- a/configuration.md +++ b/configuration.md @@ -11,7 +11,11 @@ Mockingbird is configured via the secrets.conf file, which has the following str "http://localhost:3000", ... ], - "healthCheckRoute": "/ready" + "healthCheckRoute": "/ready", + "vertx": { + "maxFormAttributeSize": 262144, + "compressionSupported": true + } }, "security": { "secret": ".." @@ -24,6 +28,8 @@ Mockingbird is configured via the secrets.conf file, which has the following str "excludedResponseHeaders": [..], "insecureHosts": [..], "logOutgoingRequests": false, + "disableAutoDecompressForRaw": "true", + "httpVersion": "HTTP_1_1", "proxyServer": { "type": "http" | "socks", "type": "..", @@ -51,6 +57,8 @@ This section specifies origins for CORS. These settings affect the functionality healthCheckRoute - an optional parameter that allows configuring an endpoint always returning 200 OK, useful for health checks. +Inside the vertx section, one can set up any [HTTP server options of Vert.x](https://vertx.io/docs/apidocs/io/vertx/core/http/HttpServerOptions.html) + ### Security Section Mandatory section. Here the secret is specified - the encryption key for the configurations of source and destination. @@ -94,7 +102,9 @@ Example of a typical configuration: "insecureHosts": [ "some.host" ], - "logOutgoingRequests": false + "logOutgoingRequests": false, + "disableAutoDecompressForRaw": "true", + "httpVersion": "HTTP_1_1" } } } @@ -104,6 +114,10 @@ In the insecureHosts field, you can specify a list of hosts for which certificat The logOutgoingRequests flag allows enabling logging of requests to the remote server when the HTTP mock is operating in proxy mode. The request is logged in the form of a curl command with headers and request body. +The disableAutoDecompressForRaw flag allow disabling automatic decompression of response for proxy stub with mode `proxy`. + +The httpVersion allows to configure the desired HTTP protocol version for outgoing requests, possible values: HTTP_1_1 or HTTP_2. + Also, in this section, you can specify proxy server settings. These settings affect ALL HTTP requests made by Mockingbird, including: - requests to external servers with proxy mocks diff --git a/configuration_ru.md b/configuration_ru.md index ffb91e30..5beabf23 100644 --- a/configuration_ru.md +++ b/configuration_ru.md @@ -11,7 +11,11 @@ Mockingbird конфигурируется посредством файла sec "http://localhost:3000", ... ], - "healthCheckRoute": "/ready" + "healthCheckRoute": "/ready", + "vertx": { + "maxFormAttributeSize": 262144, + "compressionSupported": true + } }, "security": { "secret": ".." @@ -24,6 +28,8 @@ Mockingbird конфигурируется посредством файла sec "excludedResponseHeaders": [..], "insecureHosts": [..], "logOutgoingRequests": false, + "disableAutoDecompressForRaw": "true", + "httpVersion": "HTTP_1_1", "proxyServer": { "type": "http" | "socks", "type": "..", @@ -51,6 +57,8 @@ Mockingbird конфигурируется посредством файла sec healthCheckRoute - необязательный параметр, позволяет настроить эндпоинт, всегда отдающий 200 OK, полезно для healthcheck +Внутрии секции vertx можно задать [параметры для сервера Vert.x](https://vertx.io/docs/apidocs/io/vertx/core/http/HttpServerOptions.html). + ### Секция security Обязательная секция. Здесь указывается secret - ключ шифрования для конфигураций source и destination. @@ -94,7 +102,9 @@ healthCheckRoute - необязательный параметр, позволя "insecureHosts": [ "some.host" ], - "logOutgoingRequests": false + "logOutgoingRequests": false, + "disableAutoDecompressForRaw": "true", + "httpVersion": "HTTP_1_1" } } } @@ -105,6 +115,10 @@ healthCheckRoute - необязательный параметр, позволя Флаг logOutgoingRequests позволяет включить логирование запросов к удаленному серверу, когда http заглушка работет в режиме прокси. Запрос пишется в лог в виде команды curl с заголовками и телом запроса. +Флаг disableAutoDecompressForRaw отключает автоматическую декомпрессию ответа для заглушек с режимом `proxy`. + +Параметр httpVersion отвечает за версию протокола исопльзуемого для исходящих HTTP запросов, может принимать значения `HTTP_1_1` или `HTTP_2`. + Так-же в этой секции можно указать настройки прокси сервера. Эти настройки влияют на ВСЕ http запросы, которые делаем mockingbird, т.е.: - запросы к внешнему серверу с proxy моках - запросы в source и destination (включая init/shutdown)