Timeout Error At Vertx Gateway

[iamakashk]

Hi I am using apiman v1.5.7,

After some time of idle time (when there are no requests at gateway) I am getting following error in my vertx gateway logs ,

[ERROR] 2021-09-11 06:56:59.405 [vert.x-eventloop-thread-2] HttpConnector - The timeout period of 120000ms has been exceeded while executing POST /processRequest for host sample.myserver.com
io.vertx.core.http.impl.HttpClientRequestBase$1: The timeout period of 120000ms has been exceeded while executing POST /processRequest for host sample.myserver.com
[ERROR] 2021-09-11 06:56:59.405 [vert.x-eventloop-thread-2] ContextImpl - Unhandled exception
java.lang.IllegalStateException: Response is closed
        at io.vertx.core.http.impl.HttpServerResponseImpl.checkValid(HttpServerResponseImpl.java:575) ~[apiman-gateway.jar:?]
        at io.vertx.core.http.impl.HttpServerResponseImpl.setChunked(HttpServerResponseImpl.java:138) ~[apiman-gateway.jar:?]
        at io.vertx.core.http.impl.HttpServerResponseImpl.setChunked(HttpServerResponseImpl.java:54) ~[apiman-gateway.jar:?]
        at io.apiman.gateway.platforms.vertx3.http.HttpPolicyAdapter.handleError(HttpPolicyAdapter.java:137) ~[apiman-gateway.jar:?]
        at io.apiman.gateway.platforms.vertx3.http.HttpPolicyAdapter.lambda$_execute$1(HttpPolicyAdapter.java:87) ~[apiman-gateway.jar:?]
        at io.apiman.gateway.platforms.vertx3.http.HttpPolicyAdapter$$Lambda$97/0x00000000c8113fc0.handle(Unknown Source) ~[?:?]
        at io.apiman.gateway.engine.impl.ApiRequestExecutorImpl.lambda$wrapResultHandler$0(ApiRequestExecutorImpl.java:190) ~[apiman-gateway.jar:?]
        at io.apiman.gateway.engine.impl.ApiRequestExecutorImpl$$Lambda$98/0x00000000c81346b0.handle(Unknown Source) ~[?:?]
        at io.apiman.gateway.engine.impl.ApiRequestExecutorImpl.lambda$createApiConnectionResponseHandler$13(ApiRequestExecutorImpl.java:733) ~[apiman-gateway.jar:?]
        at io.apiman.gateway.engine.impl.ApiRequestExecutorImpl$$Lambda$172/0x00000000c8270020.handle(Unknown Source) ~[?:?]
        at io.apiman.gateway.platforms.vertx3.connector.HttpConnector$ExceptionHandler.handle(HttpConnector.java:323) ~[apiman-gateway.jar:?]
        at io.apiman.gateway.platforms.vertx3.connector.HttpConnector$ExceptionHandler.handle(HttpConnector.java:317) ~[apiman-gateway.jar:?]
        at io.vertx.core.http.impl.HttpClientRequestBase.handleException(HttpClientRequestBase.java:139) ~[apiman-gateway.jar:?]
        at io.vertx.core.http.impl.HttpClientRequestBase.timeout(HttpClientRequestBase.java:211) ~[apiman-gateway.jar:?]
        at io.vertx.core.http.impl.HttpClientRequestBase.handleTimeout(HttpClientRequestBase.java:194) ~[apiman-gateway.jar:?]
        at io.vertx.core.http.impl.HttpClientRequestBase.lambda$setTimeout$0(HttpClientRequestBase.java:124) ~[apiman-gateway.jar:?]
        at io.vertx.core.http.impl.HttpClientRequestBase$$Lambda$174/0x00000000c8271370.handle(Unknown Source) ~[?:?]
        at io.vertx.core.impl.VertxImpl$InternalTimerHandler.handle(VertxImpl.java:907) ~[apiman-gateway.jar:?]
        at io.vertx.core.impl.VertxImpl$InternalTimerHandler.handle(VertxImpl.java:866) ~[apiman-gateway.jar:?]
        at io.vertx.core.impl.ContextImpl.executeTask(ContextImpl.java:320) ~[apiman-gateway.jar:?]
        at io.vertx.core.impl.EventLoopContext.lambda$executeAsync$0(EventLoopContext.java:38) ~[apiman-gateway.jar:?]
        at io.vertx.core.impl.EventLoopContext$$Lambda$34/0x00000000505c7f00.run(Unknown Source) ~[?:?]
        at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163) ~[apiman-gateway.jar:?]
        at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:404) ~[apiman-gateway.jar:?]
        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:462) ~[apiman-gateway.jar:?]
        at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:897) ~[apiman-gateway.jar:?]
        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[apiman-gateway.jar:?]
        at java.lang.Thread.run(Thread.java:823) [?:1.8.0_275]		

Here is my config file for your reference,

{
  "variables": {
    "apiman": {
      "es": {
        "protocol": "http",
        "host": "es_IP",
        "port": 4444,
        "username": "elastic",
        "password": "changeme",
        "timeout": 120000
      }
    }
  },

  "registry": {
    "class": "io.apiman.gateway.engine.es.PollCachingESRegistry",
    "config": {
      "client": {
        "type": "jest",
        "protocol": "${apiman.es.protocol}",
        "host": "${apiman.es.host}",
        "port": "${apiman.es.port}",
        "initialize": true,
        "username": "${apiman.es.username}",
        "password": "${apiman.es.password}",
        "timeout": "${apiman.es.timeout}" //,
        //"cache-polling-interval": 15
        //"index": ""
      }
    }
  },

  // Gateway error writer
  // A "trace" version of the error writer - comment out/remove this to suppress stack traces
  // in the JSON/XML payload returned by the gateway when an error occurs.
  "writers": {
    "error": {
      "class": "io.apiman.gateway.engine.impl.TracePolicyErrorWriter",
      "config": {}
    } //,
    // "policy-failure": {
    //   "class": "",
    //   "config": {}
    // }
  },

  "encrypter": {
    "class": "io.apiman.gateway.engine.impl.DefaultDataEncrypter",
    "config": {}
  },

  "plugin-registry": {
    "class": "io.apiman.gateway.platforms.vertx3.engine.VertxPluginRegistry",
    "config": {
      "pluginRepositories": [
        "some_url"
      ]
      //"pluginsDir": "/path/to/your/plugin_dir"
    }
  },

  "connector-factory": {
    "class": "io.apiman.gateway.platforms.vertx3.connector.ConnectorFactory",
    "config": {
      // Your usual config here
      "httpClient": {
        "maxPoolSize": 100
      },
      // -----------------------------------------------
      // SSL/TLS settings for the gateway connector(s).
      // -----------------------------------------------
      "tls": {
        // Enable devMode for HTTPS connections (gateway trusts any certificate).
        // This should *NOT* be used in production mode. *Use with great care.*
        "devMode": true

        // Whether self-signed certificates should be automatically trusted. *Use with great care.*
        // "allowSelfSigned": false,

        // Whether certificate host checks should be bypassed. *Use with great care.*
        // "allowAnyHost": false,

        // Trust store contains certificate(s) trusted by gateway.
        // "trustStore": "/path/to/your/truststore.jks",
        // "trustStorePassword": "abc123",

        // Key store contains gateway's keys (including private components: keep it safe).
        // "keyStore": "/path/to/your/keystore.jks",
        // "keyStorePassword": "abc123",

        // By default all keys can be used (will try all). If alias list provided, will only attempt to use listed keys.
        // "keyAliases": "mykey,myotherkey",

        // Allowed TLS/SSL protocols and ciphers suites as CSV. Availability will vary depending on your JVM impl.
        // Uses JVM defaults depending if not explicitly provided.
        // See: https://docs.oracle.com/javase/7/docs/technotes/guides/security/SunProviders.html
        // You may wish to consider global JVM settings by modifying java.security
        // "allowedProtocols": "TLSv1.2,TLSv1.1",
        // "allowedCiphers": "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,...",
        // "disallowedCiphers": "..."
      }
    }
  },

  "policy-factory": {
    "class": "io.apiman.gateway.engine.policy.PolicyFactoryImpl",
    "config": {}
  },

  "logger-factory": {
    "class": "io.apiman.gateway.platforms.vertx3.logging.VertxLoggerDelegate",
    "config": {}
  },

  // Elasticsearch Metrics Settings
  "metrics": {
    "class": "io.apiman.gateway.engine.es.ESMetrics",
    "config": {
      "client": {
        "type": "jest",
        "protocol": "${apiman.es.protocol}",
        "host": "${apiman.es.host}",
        "port": "${apiman.es.port}",
        "initialize": true,
        "username": "${apiman.es.username}",
        "password": "${apiman.es.password}",
        "timeout": "${apiman.es.timeout}"
      }
    }
  },

  "components": {
    // Shared State Component Settings
    "ISharedStateComponent": {
      "class": "io.apiman.gateway.engine.es.ESSharedStateComponent",
      "config": {
        "client": {
          "type": "jest",
          "protocol": "${apiman.es.protocol}",
          "host": "${apiman.es.host}",
          "port": "${apiman.es.port}",
          "initialize": true,
          "username": "${apiman.es.username}",
          "password": "${apiman.es.password}",
          "timeout": "${apiman.es.timeout}"
        }
      }
    },

    // Rate Limiter Component Settings
    "IRateLimiterComponent": {
      "class": "io.apiman.gateway.engine.es.ESRateLimiterComponent",
      "config": {
        "client": {
          "type": "jest",
          "protocol": "${apiman.es.protocol}",
          "host": "${apiman.es.host}",
          "port": "${apiman.es.port}",
          "initialize": true,
          "username": "${apiman.es.username}",
          "password": "${apiman.es.password}",
          "timeout": "${apiman.es.timeout}"
        }
      }
    },

    // Cache Store Component Settings
    "ICacheStoreComponent": {
      "class": "io.apiman.gateway.engine.es.ESCacheStoreComponent",
      "config": {
        "client": {
          "type": "jest",
          "protocol": "${apiman.es.protocol}",
          "host": "${apiman.es.host}",
          "port": "${apiman.es.port}",
          "initialize": true,
          "username": "${apiman.es.username}",
          "password": "${apiman.es.password}",
          "timeout": "${apiman.es.timeout}"
        }
      }
    },

    // Execute Blocking Component
    "IExecuteBlockingComponent": {
      "class": "io.apiman.gateway.platforms.vertx3.components.ExecuteBlockingComponentImpl",
      "config": {}
    },

    // JDBC Component Settings
    "IJdbcComponent": {
      "class": "io.apiman.gateway.platforms.vertx3.components.JdbcClientComponentImpl",
      "config": {}
    },

    // LDAP Component Settings
    "ILdapComponent": {
      "class": "io.apiman.gateway.platforms.vertx3.components.LdapClientComponentImpl",
      "config": {}
    },

    // HTTP Component Settings
    "IHttpClientComponent": {
      "class": "io.apiman.gateway.platforms.vertx3.components.HttpClientComponentImpl",
      "config": {}
    },

    // Policy Failure Factory Component
    "IPolicyFailureFactoryComponent": {
      "class": "io.apiman.gateway.platforms.vertx3.components.PolicyFailureFactoryComponent",
      "config": {}
    },

    // Buffer Factory Component
    "IBufferFactoryComponent": {
      "class": "io.apiman.gateway.platforms.vertx3.components.BufferFactoryComponentImpl",
      //"config": {}
      "config": {}
    },

    // Periodic Component
    "IPeriodicComponent": {
      "class": "io.apiman.gateway.platforms.vertx3.components.PeriodicComponentImpl",
      "config": {}
    }
  },

  // Host-name to bind to for this machine.
  "hostname": "0.0.0.0",

  "verticles": {
    "http": {
      "port": 7777,
      "count": 8
    },
    // Configure the following SSL section to enable SSL/TLS.
    "https": {
      "port": 7443,
      "count": 0
    },

    "api": {
      "port": 7778,
      "count": 1
    }
  },

  // Gateway API Authentication. See documentation for further possibilities..
  "auth": {
    "type": "keycloak",
    "config": {
      "flowType": "PASSWORD",
      "requiredRole": "realm:apipublisher",
      // Paste and overwrite your Keycloak config here.
      "realm": "apiman",
      "realm-public-key": "realm-public-key",
      "auth-server-url": "http://keycloak_host:port/auth",
      "ssl-required": "none",
      "disable-trust-manager": false,
      "allow-any-hostname": false,
      "resource": "apiman-gateway-api",
      // A limitation in the current OAuth2 implementation means a credentials section is required
      // even if your client is not set to "confidential". Leave this dummy section if you're using non-confidential.
      "credentials": {
        "secret": "secret"
      },
      "truststore": "/usr/src/apiman/apiman-distro-vertx/apiman.jks",
      "truststore-password": "secret"
      // "client-keystore": "/usr/src/apiman/apiman-distro-vertx/apiman.jks",
      // "client-keystore-password": "secret"
      // End paste here
    }
  },

  // When reporting an API's endpoint, prefer secure (i.e. https). You should probably change this after enabling HTTPS.
  "preferSecure": false
}

Is there anything missing here ?

[volkflo]

It seems that there is still an open connection on that one request.
Could you try if this occurs as well in apiman 2.1?