Fixup historical perf test shutdown (#6307)

tests/historical_query_perf.py

@@ -167,6 +167,12 @@ def test_historical_query_range(network, args):
 
     entries = {}
     node = network.find_node_by_role(role=infra.network.NodeRole.BACKUP, log_capture=[])
+
+    # During this test enclave may get very busy due to increasing amount of stores fetched in
+    # memory, so tick messages may stack up, resulting in a delayed 'stop' msg processing, therefore
+    # leading to a delayed shutdown.
+    node.remote.remote.shutdown_timeout *= 10
+
     with node.client(common_headers={"authorization": f"Bearer {jwt}"}) as c:
         # Index is currently built lazily to avoid impacting other perf tests using the same app
         # So pre-fetch to ensure index is fully constructed

tests/infra/remote.py

@@ -379,6 +379,15 @@ def __init__(
         self.name = name
         self.out = os.path.join(self.root, "out")
         self.err = os.path.join(self.root, "err")
+        self._shutdown_timeout = 10
+
+    @property
+    def shutdown_timeout(self):
+        return self._shutdown_timeout
+
+    @shutdown_timeout.setter
+    def shutdown_timeout(self, value):
+        self._shutdown_timeout = value
 
     @staticmethod
     def make_host(host):
@@ -515,12 +524,11 @@ def stop(self):
         LOG.info("[{}] closing".format(self.hostname))
         if self.proc:
             self.proc.terminate()
-            timeout = 10
             try:
-                self.proc.wait(timeout)
+                self.proc.wait(self._shutdown_timeout)
             except subprocess.TimeoutExpired:
                 LOG.exception(
-                    f"Process didn't finish within {timeout} seconds. Trying to get stack trace..."
+                    f"Process didn't finish within {self._shutdown_timeout} seconds. Trying to get stack trace..."
                 )
                 self._print_stack_trace()
                 raise