Fallback for situations where Python's fromtimestamp() raises OSError…

… or OverflowError (#2972)

parse_timestamp to fall back to timedelta method for negative and post-2038 times

.changes/next-release/bugfix-Parsers-42740.json

@@ -0,0 +1,5 @@
+{
+  "type": "bugfix",
+  "category": "Parsers",
+  "description": "Fixes datetime parse error handling for out-of-range and negative timestamps (`#2564 <https://github.com/boto/botocore/issues/2564>`__)."
+}

botocore/utils.py

@@ -904,6 +904,22 @@ def percent_encode(input_str, safe=SAFE_CHARS):
     return quote(input_str, safe=safe)
 
 
+def _epoch_seconds_to_datetime(value, tzinfo):
+    """Parse numerical epoch timestamps (seconds since 1970) into a
+    ``datetime.datetime`` in UTC using ``datetime.timedelta``. This is intended
+    as fallback when ``fromtimestamp`` raises ``OverflowError`` or ``OSError``.
+
+    :type value: float or int
+    :param value: The Unix timestamps as number.
+
+    :type tzinfo: callable
+    :param tzinfo: A ``datetime.tzinfo`` class or compatible callable.
+    """
+    epoch_zero = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc())
+    epoch_zero_localized = epoch_zero.astimezone(tzinfo())
+    return epoch_zero_localized + datetime.timedelta(seconds=value)
+
+
 def _parse_timestamp_with_tzinfo(value, tzinfo):
     """Parse timestamp with pluggable tzinfo options."""
     if isinstance(value, (int, float)):
@@ -935,15 +951,38 @@ def parse_timestamp(value):
     This will return a ``datetime.datetime`` object.
 
     """
-    for tzinfo in get_tzinfo_options():
+    tzinfo_options = get_tzinfo_options()
+    for tzinfo in tzinfo_options:
         try:
+            print(f"_parse_timestamp_with_tzinfo({value}, {tzinfo})")
             return _parse_timestamp_with_tzinfo(value, tzinfo)
-        except OSError as e:
+        except (OSError, OverflowError) as e:
             logger.debug(
                 'Unable to parse timestamp with "%s" timezone info.',
                 tzinfo.__name__,
                 exc_info=e,
             )
+    # For numeric values attempt fallback to using fromtimestamp-free method.
+    # From Python's ``datetime.datetime.fromtimestamp`` documentation: "This
+    # may raise ``OverflowError``, if the timestamp is out of the range of
+    # values supported by the platform C localtime() function, and ``OSError``
+    # on localtime() failure. It's common for this to be restricted to years
+    # from 1970 through 2038."
+    try:
+        numeric_value = float(value)
+    except (TypeError, ValueError):
+        pass
+    else:
+        try:
+            for tzinfo in tzinfo_options:
+                return _epoch_seconds_to_datetime(numeric_value, tzinfo=tzinfo)
+        except (OSError, OverflowError) as e:
+            logger.debug(
+                'Unable to parse timestamp using fallback method with "%s" '
+                'timezone info.',
+                tzinfo.__name__,
+                exc_info=e,
+            )
     raise RuntimeError(
         'Unable to calculate correct timezone offset for "%s"' % value
     )

tests/unit/test_utils.py

@@ -14,6 +14,7 @@
 import datetime
 import io
 import operator
+from contextlib import contextmanager
 from sys import getrefcount
 
 import pytest
@@ -424,6 +425,18 @@ def test_parse_epoch_zero_time(self):
             datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()),
         )
 
+    def test_parse_epoch_negative_time(self):
+        self.assertEqual(
+            parse_timestamp(-2208988800),
+            datetime.datetime(1900, 1, 1, 0, 0, 0, tzinfo=tzutc()),
+        )
+
+    def test_parse_epoch_beyond_2038(self):
+        self.assertEqual(
+            parse_timestamp(2524608000),
+            datetime.datetime(2050, 1, 1, 0, 0, 0, tzinfo=tzutc()),
+        )
+
     def test_parse_epoch_as_string(self):
         self.assertEqual(
             parse_timestamp('1222172800'),
@@ -465,6 +478,42 @@ def test_parse_timestamp_fails_with_bad_tzinfo(self):
             with self.assertRaises(RuntimeError):
                 parse_timestamp(0)
 
+    @contextmanager
+    def mocked_fromtimestamp_that_raises(self, exception_type):
+        class MockDatetime(datetime.datetime):
+            @classmethod
+            def fromtimestamp(cls, *args, **kwargs):
+                raise exception_type()
+
+        mock_fromtimestamp = mock.Mock()
+        mock_fromtimestamp.side_effect = OverflowError()
+
+        with mock.patch('datetime.datetime', MockDatetime):
+            yield
+
+    def test_parse_timestamp_succeeds_with_fromtimestamp_overflowerror(self):
+        # ``datetime.fromtimestamp()`` fails with OverflowError on some systems
+        # for timestamps beyond 2038. See
+        # https://docs.python.org/3/library/datetime.html#datetime.datetime.fromtimestamp
+        # This test mocks fromtimestamp() to always raise an OverflowError and
+        # checks that the fallback method returns the same time and timezone
+        # as fromtimestamp.
+        wout_fallback = parse_timestamp(0)
+        with self.mocked_fromtimestamp_that_raises(OverflowError):
+            with_fallback = parse_timestamp(0)
+            self.assertEqual(with_fallback, wout_fallback)
+            self.assertEqual(with_fallback.tzinfo, wout_fallback.tzinfo)
+
+    def test_parse_timestamp_succeeds_with_fromtimestamp_oserror(self):
+        # Same as test_parse_timestamp_succeeds_with_fromtimestamp_overflowerror
+        # but for systems where datetime.fromtimestamp() fails with OSerror for
+        # negative timestamps that represent times before 1970.
+        wout_fallback = parse_timestamp(0)
+        with self.mocked_fromtimestamp_that_raises(OSError):
+            with_fallback = parse_timestamp(0)
+            self.assertEqual(with_fallback, wout_fallback)
+            self.assertEqual(with_fallback.tzinfo, wout_fallback.tzinfo)
+
 
 class TestDatetime2Timestamp(unittest.TestCase):
     def test_datetime2timestamp_naive(self):