restore: Make max stale_seconds retry with a longer timeout

When timing out downloading a backup, on next retry, retry with a longer
timeout.

pghoard/restore.py

@@ -12,6 +12,7 @@
 import errno
 import io
 import logging
+import math
 import multiprocessing
 import multiprocessing.pool
 import os
@@ -657,7 +658,17 @@ def run(self, args=None):
 
 
 class BasebackupFetcher:
-    def __init__(self, *, app_config, debug, site, pgdata, tablespaces, data_files: List[FileInfo], status_output_file=None):
+    def __init__(
+        self,
+        *,
+        app_config,
+        debug,
+        site,
+        pgdata,
+        tablespaces,
+        data_files: List[FileInfo],
+        status_output_file=None,
+    ):
         self.log = logging.getLogger(self.__class__.__name__)
         self.completed_jobs: Set[str] = set()
         self.config = app_config
@@ -698,6 +709,9 @@ def fetch_all(self):
             except TimeoutError:
                 self.pending_jobs.clear()
                 self.last_progress_ts = time.monotonic()
+
+                # Increase the timeout and retry
+                self.max_stale_seconds *= 2
                 if self.errors:
                     break