Merge pull request #31 from nasa/release/0.10.0

Release/0.10.0

.github/workflows/python-app.yml

@@ -15,15 +15,15 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python 3.10
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: Install Poetry
-        uses: abatilo/actions-poetry@v2
+        uses: abatilo/actions-poetry@v3
         with:
-          poetry-version: 1.5.1
+          poetry-version: 1.7.1
       - name: Install dependencies
         run: |
           poetry run python -m pip install --upgrade pip

.github/workflows/python-publish.yml

@@ -21,13 +21,13 @@ jobs:
       id-token: write
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
       - name: Install Poetry
-        uses: abatilo/actions-poetry@v2
+        uses: abatilo/actions-poetry@v3
         with:
           poetry-version: 1.5.1
       - name: Build package

.gitignore

@@ -4,4 +4,6 @@ venv/
 tags
 .venv
 *.egg-info
-dist
+dist
+.vscode/*
+.DS_Store

CHANGELOG.md

@@ -7,6 +7,15 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+## [0.10.0]
+### Changed
+- [issues/29](https://github.com/nasa/python_cmr/issues/29) - Date parsing has been improved to accept more ISO-8601 string formats as well as timezone-aware datetime objects
+### Added
+- [pull/27](https://github.com/nasa/python_cmr/pull/27) New feature to search by `readable_granlue_name` https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#g-granule-ur-or-producer-granule-id
+### Fixed
+- [pull/27](https://github.com/nasa/python_cmr/pull/27) Fixed bug with constructing the `options` sent to CMR which was causing filters to not get applied correctly.
+- [pull/28](https://github.com/nasa/python_cmr/pull/28) Fixed bug where `KeyError` was thrown if search result contained 0 hits
+
 ## [0.9.0]
 ### Added
 - [pull/17](https://github.com/nasa/python_cmr/pull/17) New feature that allows sort_keys to be passed into this Api up to the CMR. Used the valid sort_keys as of July 2023
@@ -45,7 +54,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Older]
 - Prior releases of this software originated from https://github.com/jddeal/python-cmr/releases
 
-[Unreleased]: https://github.com/nasa/python_cmr/compare/v0.8.0...HEAD
+[Unreleased]: https://github.com/nasa/python_cmr/compare/v0.10.0...HEAD
+[0.10.0]: https://github.com/nasa/python_cmr/compare/v0.9.0...v0.10.0
+[0.9.0]: https://github.com/nasa/python_cmr/compare/v0.8.0...v0.9.0
 [0.8.0]: https://github.com/nasa/python_cmr/compare/v0.7.0...v0.8.0
 [0.7.0]: https://github.com/nasa/python_cmr/compare/v0.6.0...v0.7.0
 [0.6.0]: https://github.com/nasa/python_cmr/compare/v0.5.0...v0.6.0

README.md

@@ -110,6 +110,8 @@ Granule searches support these methods (in addition to the shared methods above)
     >>> api.granule_ur("SC:AST_L1T.003:2150315169")
     # search for granules from a specific orbit
     >>> api.orbit_number(5000)
+    # search for a granule by name
+    >>> api.short_name("MOD09GA").readable_granule_name(["*h32v08*","*h30v13*"])
 
     # filter by the day/night flag
     >>> api.day_night_flag("day")

cmr/queries.py

@@ -7,11 +7,12 @@
 except ImportError:
     from urllib import pathname2url as quote
 
-from datetime import datetime
+from datetime import datetime, timezone
 from inspect import getmembers, ismethod
 from re import search
 
 from requests import get, exceptions
+from dateutil.parser import parse as dateutil_parse
 
 CMR_OPS = "https://cmr.earthdata.nasa.gov/search/"
 CMR_UAT = "https://cmr.uat.earthdata.nasa.gov/search/"
@@ -51,10 +52,15 @@ def get(self, limit=2000):
         url = self._build_url()
 
         results = []
-        page = 1
-        while len(results) < limit:
+        more_results = True
+        while more_results == True:
 
-            response = get(url, headers=self.headers, params={'page_size': page_size, 'page_num': page})
+            # Only get what we need
+            page_size = min(limit - len(results), page_size)
+            response = get(url, headers=self.headers, params={'page_size': page_size})
+            if self.headers == None:
+                self.headers = {}
+            self.headers['cmr-search-after'] = response.headers.get('cmr-search-after')
 
             try:
                 response.raise_for_status()
@@ -65,13 +71,16 @@ def get(self, limit=2000):
                 latest = response.json()['feed']['entry']
             else:
                 latest = [response.text]
-
-            if len(latest) == 0:
-                break
-
+
             results.extend(latest)
-            page += 1
-
+
+            if page_size > len(response.json()['feed']['entry']) or len(results) >= limit:
+                more_results = False
+
+        # This header is transient. We need to get rid of it before we do another different query
+        if self.headers['cmr-search-after']:
+            del self.headers['cmr-search-after']
+
         return results
 
     def hits(self):
@@ -195,7 +204,7 @@ def _build_url(self):
                 formatted_options.append("options[{}][{}]={}".format(
                     param_key,
                     option_key,
-                    val
+                    str(val).lower()
                 ))
 
         options_as_string = "&".join(formatted_options)
@@ -332,7 +341,7 @@ def temporal(self, date_from, date_to, exclude_boundary=False):
         """
         Filter by an open or closed date range.
 
-        Dates can be provided as a datetime objects or ISO 8601 formatted strings. Multiple
+        Dates can be provided as native date objects or ISO 8601 formatted strings. Multiple
         ranges can be provided by successive calls to this method before calling execute().
 
         :param date_from: earliest date of temporal range
@@ -344,29 +353,37 @@ def temporal(self, date_from, date_to, exclude_boundary=False):
         iso_8601 = "%Y-%m-%dT%H:%M:%SZ"
 
         # process each date into a datetime object
-        def convert_to_string(date):
+        def convert_to_string(date, default):
             """
             Returns the argument as an ISO 8601 or empty string.
             """
 
             if not date:
                 return ""
 
-            try:
-                # see if it's datetime-like
-                return date.strftime(iso_8601)
-            except AttributeError:
+            # handle str, date-like objects without time, and datetime objects
+            if isinstance(date, str):
+                # handle string by parsing with default
+                date = dateutil_parse(date, default=default)
+            elif not isinstance(date, datetime):
+                # handle (naive by definition) date by converting to utc datetime
                 try:
-                    # maybe it already is an ISO 8601 string
-                    datetime.strptime(date, iso_8601)
-                    return date
+                    date = datetime.combine(date, default.time())
                 except TypeError:
-                    raise ValueError(
-                        "Please provide None, datetime objects, or ISO 8601 formatted strings."
-                    )
+                    msg = f"Date must be a date object or ISO 8601 string, not {date.__class__.__name__}."
+                    raise TypeError(msg)
+                date = date.replace(tzinfo=timezone.utc)
+            else:
+                # pass aware datetime and handle naive datetime by assuming utc
+                date = date if date.tzinfo else date.replace(tzinfo=timezone.utc)
+
+            # convert aware datetime to utc datetime
+            date = date.astimezone(timezone.utc)            
 
-        date_from = convert_to_string(date_from)
-        date_to = convert_to_string(date_to)
+            return date.strftime(iso_8601)
+
+        date_from = convert_to_string(date_from, datetime(1, 1, 1, 0, 0, 0, tzinfo=timezone.utc))
+        date_to = convert_to_string(date_to, datetime(1, 12, 31, 23, 59, 59, tzinfo=timezone.utc))
 
         # if we have both dates, make sure from isn't later than to
         if date_from and date_to:
@@ -733,6 +750,27 @@ def granule_ur(self, granule_ur=""):
 
         self.params['granule_ur'] = granule_ur
         return self
+
+    def readable_granule_name(self, readable_granule_name=""):
+        """
+        Filter by the readable granule name (producer_granule_id if present, otherwise producer_granule_id).
+
+        Can use wildcards for substring matching:
+
+        asterisk (*) will match any number of characters.
+        question mark (?) will match exactly one character.
+
+        :param readable_granule_name: granule name or substring
+        :returns: Query instance
+        """
+
+        if isinstance(readable_granule_name, str):
+            readable_granule_name = [readable_granule_name]
+
+        self.params["readable_granule_name"] = readable_granule_name
+        self.options["readable_granule_name"] = {"pattern": True}
+
+        return self
 
     def _valid_state(self):