From 7e4de85fadefe90135f7ef882556be15a83658c4 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Mon, 26 Jun 2023 17:54:43 -0600 Subject: [PATCH] Fallback for situations where Python's fromtimestamp() raises OSError or OverflowError (#2972) parse_timestamp to fall back to timedelta method for negative and post-2038 times --- .../next-release/bugfix-Parsers-42740.json | 5 ++ botocore/utils.py | 43 +++++++++++++++- tests/unit/test_utils.py | 49 +++++++++++++++++++ 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 .changes/next-release/bugfix-Parsers-42740.json diff --git a/.changes/next-release/bugfix-Parsers-42740.json b/.changes/next-release/bugfix-Parsers-42740.json new file mode 100644 index 0000000000..7244ed9404 --- /dev/null +++ b/.changes/next-release/bugfix-Parsers-42740.json @@ -0,0 +1,5 @@ +{ + "type": "bugfix", + "category": "Parsers", + "description": "Fixes datetime parse error handling for out-of-range and negative timestamps (`#2564 `__)." +} diff --git a/botocore/utils.py b/botocore/utils.py index 484dd0f8f6..f40afe0c9c 100644 --- a/botocore/utils.py +++ b/botocore/utils.py @@ -904,6 +904,22 @@ def percent_encode(input_str, safe=SAFE_CHARS): return quote(input_str, safe=safe) +def _epoch_seconds_to_datetime(value, tzinfo): + """Parse numerical epoch timestamps (seconds since 1970) into a + ``datetime.datetime`` in UTC using ``datetime.timedelta``. This is intended + as fallback when ``fromtimestamp`` raises ``OverflowError`` or ``OSError``. + + :type value: float or int + :param value: The Unix timestamps as number. + + :type tzinfo: callable + :param tzinfo: A ``datetime.tzinfo`` class or compatible callable. + """ + epoch_zero = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()) + epoch_zero_localized = epoch_zero.astimezone(tzinfo()) + return epoch_zero_localized + datetime.timedelta(seconds=value) + + def _parse_timestamp_with_tzinfo(value, tzinfo): """Parse timestamp with pluggable tzinfo options.""" if isinstance(value, (int, float)): @@ -935,15 +951,38 @@ def parse_timestamp(value): This will return a ``datetime.datetime`` object. """ - for tzinfo in get_tzinfo_options(): + tzinfo_options = get_tzinfo_options() + for tzinfo in tzinfo_options: try: + print(f"_parse_timestamp_with_tzinfo({value}, {tzinfo})") return _parse_timestamp_with_tzinfo(value, tzinfo) - except OSError as e: + except (OSError, OverflowError) as e: logger.debug( 'Unable to parse timestamp with "%s" timezone info.', tzinfo.__name__, exc_info=e, ) + # For numeric values attempt fallback to using fromtimestamp-free method. + # From Python's ``datetime.datetime.fromtimestamp`` documentation: "This + # may raise ``OverflowError``, if the timestamp is out of the range of + # values supported by the platform C localtime() function, and ``OSError`` + # on localtime() failure. It's common for this to be restricted to years + # from 1970 through 2038." + try: + numeric_value = float(value) + except (TypeError, ValueError): + pass + else: + try: + for tzinfo in tzinfo_options: + return _epoch_seconds_to_datetime(numeric_value, tzinfo=tzinfo) + except (OSError, OverflowError) as e: + logger.debug( + 'Unable to parse timestamp using fallback method with "%s" ' + 'timezone info.', + tzinfo.__name__, + exc_info=e, + ) raise RuntimeError( 'Unable to calculate correct timezone offset for "%s"' % value ) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7cceaa8ec5..6b1a33ba78 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -14,6 +14,7 @@ import datetime import io import operator +from contextlib import contextmanager from sys import getrefcount import pytest @@ -424,6 +425,18 @@ def test_parse_epoch_zero_time(self): datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()), ) + def test_parse_epoch_negative_time(self): + self.assertEqual( + parse_timestamp(-2208988800), + datetime.datetime(1900, 1, 1, 0, 0, 0, tzinfo=tzutc()), + ) + + def test_parse_epoch_beyond_2038(self): + self.assertEqual( + parse_timestamp(2524608000), + datetime.datetime(2050, 1, 1, 0, 0, 0, tzinfo=tzutc()), + ) + def test_parse_epoch_as_string(self): self.assertEqual( parse_timestamp('1222172800'), @@ -465,6 +478,42 @@ def test_parse_timestamp_fails_with_bad_tzinfo(self): with self.assertRaises(RuntimeError): parse_timestamp(0) + @contextmanager + def mocked_fromtimestamp_that_raises(self, exception_type): + class MockDatetime(datetime.datetime): + @classmethod + def fromtimestamp(cls, *args, **kwargs): + raise exception_type() + + mock_fromtimestamp = mock.Mock() + mock_fromtimestamp.side_effect = OverflowError() + + with mock.patch('datetime.datetime', MockDatetime): + yield + + def test_parse_timestamp_succeeds_with_fromtimestamp_overflowerror(self): + # ``datetime.fromtimestamp()`` fails with OverflowError on some systems + # for timestamps beyond 2038. See + # https://docs.python.org/3/library/datetime.html#datetime.datetime.fromtimestamp + # This test mocks fromtimestamp() to always raise an OverflowError and + # checks that the fallback method returns the same time and timezone + # as fromtimestamp. + wout_fallback = parse_timestamp(0) + with self.mocked_fromtimestamp_that_raises(OverflowError): + with_fallback = parse_timestamp(0) + self.assertEqual(with_fallback, wout_fallback) + self.assertEqual(with_fallback.tzinfo, wout_fallback.tzinfo) + + def test_parse_timestamp_succeeds_with_fromtimestamp_oserror(self): + # Same as test_parse_timestamp_succeeds_with_fromtimestamp_overflowerror + # but for systems where datetime.fromtimestamp() fails with OSerror for + # negative timestamps that represent times before 1970. + wout_fallback = parse_timestamp(0) + with self.mocked_fromtimestamp_that_raises(OSError): + with_fallback = parse_timestamp(0) + self.assertEqual(with_fallback, wout_fallback) + self.assertEqual(with_fallback.tzinfo, wout_fallback.tzinfo) + class TestDatetime2Timestamp(unittest.TestCase): def test_datetime2timestamp_naive(self):