Merge pull request #106 from ifscript/main

Fix for FileNotFoundError when there is no GOES data.
blaylockbk · Oct 25, 2024 · 613c15b · 613c15b
2 parents b098d69 + 62a0417
commit 613c15b
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 7 deletions.
diff --git a/goes2go/NEW.py b/goes2go/NEW.py
@@ -257,7 +257,7 @@ def single_point_timerange(self, latitude, longitude, start=None, end=None, rece
             **kwargs,
         )
 
-    def df(self, start, end, refresh=True):
+    def df(self, start, end, refresh=True, ignore_missing=False):
         """Get list of requested GOES files as pandas.DataFrame.
 
         Parameters
@@ -267,6 +267,9 @@ def df(self, start, end, refresh=True):
         refresh : bool
             Refresh the s3fs.S3FileSystem object when files are listed.
             Default True will refresh and not use a cached list.
+        ignore_missing : bool
+            Ignore FileNotFoundError if there is missing data from
+            a satellite outage.
         """
         return _goes_file_df(
             self.satellite,
@@ -275,4 +278,5 @@ def df(self, start, end, refresh=True):
             end=end,
             bands=self.bands,
             refresh=refresh,
+            ignore_missing=ignore_missing,
         )
diff --git a/goes2go/__init__.py b/goes2go/__init__.py
@@ -71,10 +71,12 @@ def _expand(self):
 overwrite = false
 max_cpus = 1
 s3_refresh = true
+ignore_missing = false
 verbose = true
 
 ["timerange"]
 s3_refresh = false
+ignore_missing = true
 
 ["latest"]
 return_as = "xarray"

diff --git a/goes2go/data.py b/goes2go/data.py
@@ -114,7 +114,7 @@ def _check_param_inputs(**params):
     return satellite, product, domain
 
 
-def _goes_file_df(satellite, product, start, end, bands=None, refresh=True):
+def _goes_file_df(satellite, product, start, end, bands=None, refresh=True, ignore_missing=False):
     """Get list of requested GOES files as pandas.DataFrame.
 
     Parameters
@@ -140,7 +140,15 @@ def _goes_file_df(satellite, product, start, end, bands=None, refresh=True):
     # ----------------------------
     files = []
     for DATE in DATES:
-        files += fs.ls(f"{satellite}/{product}/{DATE:%Y/%j/%H/}", refresh=refresh)
+        path = f"{satellite}/{product}/{DATE:%Y/%j/%H/}"
+        if ignore_missing is True:
+            try:
+                files += fs.ls(path, refresh=refresh)
+            except FileNotFoundError:
+                print(f"Ignored missing dir: {path}")
+        else:
+            files += fs.ls(path, refresh=refresh)
+
 
     # Build a table of the files
     # --------------------------
@@ -324,6 +332,7 @@ def goes_timerange(
     max_cpus=config["timerange"].get("max_cpus"),
     bands=None,
     s3_refresh=config["timerange"].get("s3_refresh"),
+    ignore_missing=config["timerange"].get("ignore_missing"),
     verbose=config["timerange"].get("verbose", True),
 ):
     """
@@ -412,7 +421,7 @@ def goes_timerange(
         start = datetime.utcnow() - recent
         end = datetime.utcnow()
 
-    df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh)
+    df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh, ignore_missing=ignore_missing)
 
     if download:
         _download(df, save_dir=save_dir, overwrite=overwrite, verbose=verbose)
@@ -457,6 +466,7 @@ def goes_single_point_timerange(
     max_cpus=config["timerange"].get("max_cpus"),
     bands=None,
     s3_refresh=config["timerange"].get("s3_refresh"),
+    ignore_missing=config["timerange"].get("ignore_missing"),
     verbose=config["timerange"].get("verbose", True),
 ):
     """
@@ -549,7 +559,7 @@ def goes_single_point_timerange(
         start = datetime.utcnow() - recent
         end = datetime.utcnow()
 
-    df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh)
+    df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh, ignore_missing=ignore_missing)
 
     if download:
         _download(df, save_dir=save_dir, overwrite=overwrite, verbose=verbose)
@@ -577,6 +587,7 @@ def goes_latest(
     save_dir=config["latest"].get("save_dir"),
     bands=None,
     s3_refresh=config["latest"].get("s3_refresh"),
+    ignore_missing=config["latest"].get("ignore_missing"),
     verbose=config["latest"].get("verbose", True),
 ):
     """
@@ -637,7 +648,7 @@ def goes_latest(
     start = datetime.utcnow() - timedelta(hours=1)
     end = datetime.utcnow()
 
-    df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh)
+    df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh, ignore_missing=ignore_missing)
 
     # Filter for specific mesoscale domain
     if domain is not None and domain.upper() in ["M1", "M2"]:
@@ -669,6 +680,7 @@ def goes_nearesttime(
     save_dir=config["nearesttime"].get("save_dir"),
     bands=None,
     s3_refresh=config["nearesttime"].get("s3_refresh"),
+    ignore_missing=config["nearesttime"].get("ignore_missing"),
     verbose=config["nearesttime"].get("verbose", True),
 ):
     """
@@ -738,7 +750,7 @@ def goes_nearesttime(
     start = attime - within
     end = attime + within
 
-    df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh)
+    df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh, ignore_missing=ignore_missing)
 
     # return df, start, end, attime