Update options of outgoing requests, allow update server options of V…

…ert.x

backend/edsl/src/main/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/package.scala

@@ -1,6 +1,7 @@
 package ru.tinkoff.tcb.mockingbird.edsl
 
 import sttp.client4.*
+import sttp.client4.DuplicateHeaderBehavior
 import sttp.model.Uri
 import sttp.model.Uri.QuerySegment
 
@@ -16,7 +17,7 @@ package object interpreter {
   def buildRequest(host: Uri, m: HttpRequest): Request[String] = {
     val initialRequest = emptyRequest.response(asStringAlways)
     var req            = m.body.fold(initialRequest)(initialRequest.body)
-    req = m.headers.foldLeft(req) { case (r, (k, v)) => r.header(k, v, replaceExisting = true) }
+    req = m.headers.foldLeft(req) { case (r, (k, v)) => r.header(k, v, DuplicateHeaderBehavior.Replace) }
     val url = makeUri(host, m)
     m.method match {
       case Delete => req.delete(url)

backend/edsl/src/test/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/AsyncScalaTestSuiteTest.scala

@@ -9,11 +9,13 @@ import org.scalatest.BeforeAndAfterEach
 import org.scalatest.Informer
 import org.scalatest.matchers.should.Matchers
 import sttp.client4.*
+import sttp.client4.Response
 import sttp.client4.httpclient.HttpClientFutureBackend
 import sttp.client4.testing.WebSocketBackendStub
 import sttp.model.Header
 import sttp.model.MediaType
 import sttp.model.Method.*
+import sttp.model.RequestMetadata
 import sttp.model.StatusCode
 import sttp.model.Uri
 
@@ -67,7 +69,14 @@ class AsyncScalaTestSuiteTest extends AsyncScalaTestSuite with Matchers with Asy
           if uri == uri"http://some.domain.com:8090/api/handler?service=world"
             && hs.exists(h => h.name == "x-token" && h.value == "asd5453qwe")
             && hs.exists(h => h.name == "Content-Type" && h.value == "application/json") =>
-        Response(body = "got request", code = StatusCode.Ok)
+        new Response[String](
+          body = "got request",
+          code = StatusCode.Ok,
+          statusText = "",
+          headers = Seq.empty,
+          history = Nil,
+          request = RequestMetadata(POST, uri, hs),
+        )
     }
 
     val example = eset.sendHttp(method, path, body.some, headers, query)
@@ -81,11 +90,13 @@ class AsyncScalaTestSuiteTest extends AsyncScalaTestSuite with Matchers with Asy
   test("checkHttp checks code of response") {
     sttpbackend_ = HttpClientFutureBackend.stub().whenRequestMatches(_ => true).thenRespondOk()
 
-    val sttpResp = Response(
+    val sttpResp = new Response(
       body = "got request",
       code = StatusCode.InternalServerError,
       statusText = "",
       headers = Seq.empty,
+      history = Nil,
+      request = RequestMetadata(GET, uri"https://host.domain", Seq.empty)
     )
 
     val example = eset.checkHttp(
@@ -105,11 +116,13 @@ class AsyncScalaTestSuiteTest extends AsyncScalaTestSuite with Matchers with Asy
   test("checkHttp checks body of response") {
     sttpbackend_ = HttpClientFutureBackend.stub().whenRequestMatches(_ => true).thenRespondOk()
 
-    val sttpResp = Response(
+    val sttpResp = new Response(
       body = "got request",
       code = StatusCode.Ok,
       statusText = "",
       headers = Seq.empty,
+      history = Nil,
+      request = RequestMetadata(GET, uri"https://host.domain", Seq.empty)
     )
 
     val example = eset.checkHttp(
@@ -129,11 +142,13 @@ class AsyncScalaTestSuiteTest extends AsyncScalaTestSuite with Matchers with Asy
   test("checkHttp checks headers of response") {
     sttpbackend_ = HttpClientFutureBackend.stub().whenRequestMatches(_ => true).thenRespondOk()
 
-    val sttpResp = Response(
+    val sttpResp = new Response(
       body = "{}",
       code = StatusCode.Ok,
       statusText = "",
       headers = Seq(Header.contentType(MediaType.TextPlain)),
+      history = Nil,
+      request = RequestMetadata(GET, uri"https://host.domain", Seq.empty)
     )
 
     val example = eset.checkHttp(

backend/edsl/src/test/scala/ru/tinkoff/tcb/mockingbird/edsl/interpreter/AsyncScalaTestSuiteWholeTest.scala

@@ -14,6 +14,7 @@ import sttp.client4.testing.WebSocketBackendStub
 import sttp.model.Header
 import sttp.model.MediaType
 import sttp.model.Method.*
+import sttp.model.RequestMetadata
 import sttp.model.StatusCode
 import sttp.model.Uri
 
@@ -61,14 +62,20 @@ class AsyncScalaTestSuiteWholeTest
         req.headers.exists(h => h.name == "X-CSRF-TOKEN" && h.value == "unEENxJqSLS02rji2GjcKzNLc0C0ySlWih9hSxwn")
       }
       .thenRespond(
-        Response(
+        new Response(
           body = """{
                    |  "fact" : "There are approximately 100 breeds of cat.",
                    |  "length" : 42.0
                    |}""".stripMargin,
           code = StatusCode.Ok,
           statusText = "",
-          headers = Seq(Header.contentType(MediaType.ApplicationJson))
+          headers = Seq(Header.contentType(MediaType.ApplicationJson)),
+          history = Nil,
+          request = RequestMetadata(
+            GET,
+            uri"https://localhost.example:9977/fact",
+            Seq(Header("X-CSRF-TOKEN", "unEENxJqSLS02rji2GjcKzNLc0C0ySlWih9hSxwn"))
+          ),
         )
       )
   }

backend/mockingbird-api/src/main/scala/ru/tinkoff/tcb/mockingbird/Mockingbird.scala

@@ -31,6 +31,7 @@ import ru.tinkoff.tcb.mockingbird.api.Tracing
 import ru.tinkoff.tcb.mockingbird.api.UIHttp
 import ru.tinkoff.tcb.mockingbird.api.WLD
 import ru.tinkoff.tcb.mockingbird.api.WebAPI
+import ru.tinkoff.tcb.mockingbird.config.HttpVersion
 import ru.tinkoff.tcb.mockingbird.config.MockingbirdConfiguration
 import ru.tinkoff.tcb.mockingbird.config.MongoCollections
 import ru.tinkoff.tcb.mockingbird.config.MongoConfig
@@ -133,6 +134,10 @@ object Mockingbird extends scala.App {
             }
             httpClient = HttpClient
               .newBuilder()
+              .version(pc.httpVersion match {
+                case HttpVersion.HTTP_1_1 => HttpClient.Version.HTTP_1_1
+                case HttpVersion.HTTP_2   => HttpClient.Version.HTTP_2
+              })
               .connectTimeout(sttpSettings.connectionTimeout.toJava)
               .pipe(b => sttpSettings.proxy.fold(b)(conf => b.proxy(conf.asJavaProxySelector)))
               .sslContext(sslContext)

backend/mockingbird-api/src/main/scala/ru/tinkoff/tcb/mockingbird/api/WebAPI.scala

@@ -2,6 +2,7 @@ package ru.tinkoff.tcb.mockingbird.api
 
 import scala.jdk.CollectionConverters.*
 
+import com.typesafe.config.ConfigRenderOptions
 import io.vertx.core.Vertx
 import io.vertx.core.http.HttpMethod
 import io.vertx.core.http.HttpServer
@@ -26,10 +27,13 @@ object WebAPI {
     ui           <- ZIO.service[UIHttp].toManaged
     metrics      <- ZIO.service[MetricsHttp].toManaged
     server <- ZManaged.acquireReleaseWith(ZIO.attempt {
-      val vertx         = Vertx.vertx()
-      val serverOptions = new HttpServerOptions().setMaxFormAttributeSize(256 * 1024)
-      val server        = vertx.createHttpServer(serverOptions)
-      val router        = Router.router(vertx)
+      val vertx = Vertx.vertx()
+      val serverOptions = {
+        val vertxCfg = serverConfig.vertx.root().render(ConfigRenderOptions.concise())
+        new HttpServerOptions(new io.vertx.core.json.JsonObject(vertxCfg))
+      }
+      val server = vertx.createHttpServer(serverOptions)
+      val router = Router.router(vertx)
       router
         .route()
         .path("/api/internal/mockingbird/v*")

backend/mockingbird/src/main/resources/application.conf

@@ -22,13 +22,19 @@ ru.tinkoff.tcb {
       "http://localhost:3000",
       "http://localhost:8228"
     ]
+    vertx {
+      maxFormAttributeSize = 262144
+      compressionSupported = true
+    }
   }
 
   proxy {
     excludedRequestHeaders = []
     excludedResponseHeaders = []
     insecureHosts = []
     logOutgoingRequests = false
+    disableAutoDecompressForRaw = true
+    httpVersion = HTTP_1_1
   }
 
   event {

backend/mockingbird/src/main/scala/ru/tinkoff/tcb/mockingbird/api/PublicApiHandler.scala

@@ -10,9 +10,11 @@ import io.circe.Json
 import io.circe.parser.parse
 import io.circe.syntax.*
 import io.estatico.newtype.ops.*
+import mouse.boolean.*
 import mouse.option.*
 import sttp.client4.{Backend as SttpBackend, *}
 import sttp.client4.circe.*
+import sttp.model.HeaderNames
 import sttp.model.Method
 import zio.interop.catz.core.*
 
@@ -155,6 +157,12 @@ final class PublicApiHandler(
     } yield response
   }
 
+  // The cases when Mockingbird implicitly changes answer:
+  // 1. If the answer is compressed, mockingbird decompressed it.
+  // 2. When proxy mock contains substitutions, they change the answer and the result have another length.
+  // In these cases, these headers they make the modified response is incorrect, so we need to remove them.
+  private def headersDependingOnContentLength = Seq(HeaderNames.ContentEncoding, HeaderNames.ContentLength)
+
   private def proxyRequest(
       method: HttpMethod,
       headers: Map[String, String],
@@ -164,6 +172,9 @@ final class PublicApiHandler(
     val requestUri = uri"$uri".pipe(query.foldLeft(_) { case (u, (key, value)) => u.addParam(key, value) })
     for {
       _ <- log.debug(s"Received headers: ${headers.keys.mkString(", ")}")
+      // Potentially, we want to pass the request and response as is. If the client wants this
+      // response to be encoded, it sets `Accept-Encoding` itself. Also, by default, we don't
+      // unpack the response here. It's the reason here we use emptyRequest.
       req = basicRequest
         .headers(headers -- proxyConfig.excludedRequestHeaders)
         .method(Method(method.entryName), requestUri)
@@ -184,6 +195,7 @@ final class PublicApiHandler(
               )
           }
         )
+        .pipe(r => proxyConfig.disableAutoDecompressForRaw.fold(r.disableAutoDecompression, r))
         .response(asByteArrayAlways)
       _ <- log.debug("Executing request: {}", req.toCurl).when(proxyConfig.logOutgoingRequests)
       response <- req
@@ -193,6 +205,9 @@ final class PublicApiHandler(
       Refined.unsafeApply[Int, HttpStatusCodeRange](response.code.code),
       response.headers
         .filterNot(h => proxyConfig.excludedResponseHeaders(h.name))
+        .pipe { hs =>
+          proxyConfig.disableAutoDecompressForRaw.fold(hs, hs.filterNot(h => headersDependingOnContentLength.exists(h.is)))
+        }
         .map(h => h.name -> h.value)
         .toMap,
       response.body.coerce[ByteArray],
@@ -218,7 +233,10 @@ final class PublicApiHandler(
     for {
       _ <- log.debug(s"Received headers: ${headers.keys.mkString(", ")}")
       req = basicRequest
-        .headers(headers -- proxyConfig.excludedRequestHeaders)
+        .headers(
+          // The basicRequest adds `Accept-Encoding`, so we should remove the same incoming header
+          headers.filterNot(_._1.equalsIgnoreCase(HeaderNames.AcceptEncoding)) -- proxyConfig.excludedRequestHeaders
+        )
         .method(Method(method.entryName), requestUri)
         .pipe(rt =>
           body match {
@@ -248,6 +266,7 @@ final class PublicApiHandler(
           Refined.unsafeApply[Int, HttpStatusCodeRange](response.code.code),
           response.headers
             .filterNot(h => proxyConfig.excludedResponseHeaders(h.name))
+            .filterNot(h => headersDependingOnContentLength.exists(h.is))
             .map(h => h.name -> h.value)
             .toMap,
           jsonResponse.patch(data, patch).use(_.noSpaces),
@@ -277,7 +296,10 @@ final class PublicApiHandler(
     for {
       _ <- log.debug(s"Received headers: ${headers.keys.mkString(", ")}")
       req = basicRequest
-        .headers(headers -- proxyConfig.excludedRequestHeaders)
+        .headers(
+          // The basicRequest adds `Accept-Encoding`, so we should remove the same incoming header
+          headers.filterNot(_._1.equalsIgnoreCase(HeaderNames.AcceptEncoding)) -- proxyConfig.excludedRequestHeaders
+        )
         .method(Method(method.entryName), requestUri)
         .pipe(rt =>
           body match {
@@ -307,6 +329,7 @@ final class PublicApiHandler(
           Refined.unsafeApply[Int, HttpStatusCodeRange](response.code.code),
           response.headers
             .filterNot(h => proxyConfig.excludedResponseHeaders(h.name))
+            .filterNot(h => headersDependingOnContentLength.exists(h.is))
             .map(h => h.name -> h.value)
             .toMap,
           xmlResponse.patchFromValues(jData, xData, patch.map { case (k, v) => k.toZoom -> v }).toString(),

backend/mockingbird/src/main/scala/ru/tinkoff/tcb/mockingbird/config/Configuration.scala

@@ -3,10 +3,13 @@ package ru.tinkoff.tcb.mockingbird.config
 import scala.concurrent.duration.FiniteDuration
 
 import com.typesafe.config.Config
+import com.typesafe.config.ConfigException.Generic
 import com.typesafe.config.ConfigFactory
+import enumeratum.*
 import net.ceedubs.ficus.Ficus.*
 import net.ceedubs.ficus.readers.ArbitraryTypeReader.*
 import net.ceedubs.ficus.readers.EnumerationReader.*
+import net.ceedubs.ficus.readers.ValueReader
 
 final case class JsSandboxConfig(allowedClasses: Set[String] = Set())
 
@@ -15,7 +18,8 @@ final case class ServerConfig(
     port: Int,
     allowedOrigins: Seq[String],
     healthCheckRoute: Option[String],
-    sandbox: JsSandboxConfig
+    sandbox: JsSandboxConfig,
+    vertx: Config
 )
 
 final case class SecurityConfig(secret: String)
@@ -36,12 +40,28 @@ final case class ProxyServerConfig(
     auth: Option[ProxyServerAuth]
 )
 
+sealed trait HttpVersion extends EnumEntry
+object HttpVersion extends Enum[HttpVersion] {
+  val values = findValues
+  case object HTTP_1_1 extends HttpVersion
+  case object HTTP_2 extends HttpVersion
+
+  implicit val valueReader: ValueReader[HttpVersion] = ValueReader[String].map(s =>
+    namesToValuesMap.get(s) match {
+      case Some(v) => v
+      case None    => throw new Generic(s"Cannot get instance of enum HttpVersion from the value of $s")
+    }
+  )
+}
+
 final case class ProxyConfig(
     excludedRequestHeaders: Seq[String],
     excludedResponseHeaders: Set[String],
     proxyServer: Option[ProxyServerConfig],
     insecureHosts: Seq[String],
-    logOutgoingRequests: Boolean
+    logOutgoingRequests: Boolean,
+    disableAutoDecompressForRaw: Boolean,
+    httpVersion: HttpVersion
 )
 
 final case class EventConfig(fetchInterval: FiniteDuration, reloadInterval: FiniteDuration)

backend/mockingbird/src/main/scala/ru/tinkoff/tcb/utils/xttp/package.scala

@@ -1,12 +1,13 @@
 package ru.tinkoff.tcb.utils
 
+import sttp.client4.DuplicateHeaderBehavior
 import sttp.client4.PartialRequest
 
 package object xttp {
   implicit class PartialRequestTXtras[T](private val rqt: PartialRequest[T]) extends AnyVal {
     def headersReplacing(hs: Map[String, String]): PartialRequest[T] =
       hs.foldLeft(rqt) { case (request, (key, value)) =>
-        request.header(key, value, true)
+        request.header(key, value, DuplicateHeaderBehavior.Replace)
       }
   }
 }

backend/project/Versions.scala

@@ -6,6 +6,6 @@ object Versions {
   val graalvm          = "22.3.0"
   val micrometer       = "1.8.5"
   val glass            = "0.2.1"
-  val sttp             = "4.0.0-M9"
+  val sttp             = "4.0.0-M13"
   val zio              = "2.0.19"
 }