Respect the Retry-After header

commcare_export/commcare_hq_client.py

@@ -9,13 +9,15 @@
 
 import copy
 import logging
+import time
 from collections import OrderedDict
+from math import ceil
 from urllib.parse import urlencode
 
+import backoff
 import requests
 from requests.auth import AuthBase, HTTPDigestAuth
 
-import backoff
 import commcare_export
 from commcare_export.repeatable_iterator import RepeatableIterator
 
@@ -30,6 +32,12 @@
 
 def on_backoff(details):
     _log_backoff(details, 'Waiting for retry.')
+    response = details["exception"].response
+    if response.status_code == 429:
+        retry_after = response.headers.get("Retry-After", 0.0)
+        retry_after = ceil(float(retry_after))
+        logger.warning(f"Sleeping for {retry_after} seconds")
+        time.sleep(retry_after)
 
 
 def on_giveup(details):
@@ -120,7 +128,9 @@ def api_url(self):
     )
     def get(self, resource, params=None):
         """
-        Gets the named resource.
+        Gets the named resource. When the server returns a 429 (too many requests), the process will sleep for
+        the amount of seconds specified in the Retry-After header from the response, after which it will raise
+        an exception to trigger the retry action.
 
         Currently a bit of a vulnerable stub that works for this
         particular use case in the hands of a trusted user; would likely

commcare_export/commcare_minilinq.py

@@ -24,6 +24,9 @@
     'ucr',
 }
 
+DEFAULT_PAGE_SIZE = 1000
+DEFAULT_UCR_PAGE_SIZE = 10000
+
 
 class PaginationMode(Enum):
     date_indexed = "date_indexed"
@@ -102,7 +105,7 @@ def __call__(self, since, until):
 
 def get_paginator(
     resource,
-    page_size=1000,
+    page_size=None,
     pagination_mode=PaginationMode.date_indexed,
 ):
     return {
@@ -134,7 +137,7 @@ class CommCareHqEnv(DictEnv):
     CommCareHq API.
     """
 
-    def __init__(self, commcare_hq_client, until=None, page_size=1000):
+    def __init__(self, commcare_hq_client, page_size=None, until=None):
         self.commcare_hq_client = commcare_hq_client
         self.until = until
         self.page_size = page_size
@@ -177,7 +180,8 @@ class SimplePaginator(object):
     Paginate based on the 'next' URL provided in the API response.
     """
 
-    def __init__(self, page_size=1000, params=None):
+    def __init__(self, page_size=None, params=None):
+        page_size = page_size if page_size else 1000
         self.page_size = page_size
         self.params = params
 
@@ -224,7 +228,8 @@ class DatePaginator(SimplePaginator):
 
     DEFAULT_PARAMS = object()
 
-    def __init__(self, since_field, page_size=1000, params=DEFAULT_PARAMS):
+    def __init__(self, since_field, page_size=None, params=DEFAULT_PARAMS):
+        page_size = page_size if page_size else DEFAULT_PAGE_SIZE
         params = DATE_PARAMS[
             since_field] if params is DatePaginator.DEFAULT_PARAMS else params
         super(DatePaginator, self).__init__(page_size, params)
@@ -268,6 +273,9 @@ def get_since_date(self, batch):
 
 
 class UCRPaginator(SimplePaginator):
+    def __init__(self, page_size=None, *args, **kwargs):
+        page_size = page_size if page_size else DEFAULT_UCR_PAGE_SIZE
+        super().__init__(page_size, *args, **kwargs)
 
     def next_page_params_from_batch(self, batch):
         params = super(UCRPaginator, self).next_page_params_from_batch(batch)