diff --git a/goes2go/NEW.py b/goes2go/NEW.py index a1f0e8e..a5ad755 100644 --- a/goes2go/NEW.py +++ b/goes2go/NEW.py @@ -257,7 +257,7 @@ def single_point_timerange(self, latitude, longitude, start=None, end=None, rece **kwargs, ) - def df(self, start, end, refresh=True): + def df(self, start, end, refresh=True, ignore_missing=False): """Get list of requested GOES files as pandas.DataFrame. Parameters @@ -267,6 +267,9 @@ def df(self, start, end, refresh=True): refresh : bool Refresh the s3fs.S3FileSystem object when files are listed. Default True will refresh and not use a cached list. + ignore_missing : bool + Ignore FileNotFoundError if there is missing data from + a satellite outage. """ return _goes_file_df( self.satellite, @@ -275,4 +278,5 @@ def df(self, start, end, refresh=True): end=end, bands=self.bands, refresh=refresh, + ignore_missing=ignore_missing, ) diff --git a/goes2go/__init__.py b/goes2go/__init__.py index 813fe33..e74a345 100644 --- a/goes2go/__init__.py +++ b/goes2go/__init__.py @@ -71,10 +71,12 @@ def _expand(self): overwrite = false max_cpus = 1 s3_refresh = true +ignore_missing = false verbose = true ["timerange"] s3_refresh = false +ignore_missing = true ["latest"] return_as = "xarray" diff --git a/goes2go/data.py b/goes2go/data.py index 0c246ea..5660156 100644 --- a/goes2go/data.py +++ b/goes2go/data.py @@ -114,7 +114,7 @@ def _check_param_inputs(**params): return satellite, product, domain -def _goes_file_df(satellite, product, start, end, bands=None, refresh=True): +def _goes_file_df(satellite, product, start, end, bands=None, refresh=True, ignore_missing=False): """Get list of requested GOES files as pandas.DataFrame. Parameters @@ -140,7 +140,15 @@ def _goes_file_df(satellite, product, start, end, bands=None, refresh=True): # ---------------------------- files = [] for DATE in DATES: - files += fs.ls(f"{satellite}/{product}/{DATE:%Y/%j/%H/}", refresh=refresh) + path = f"{satellite}/{product}/{DATE:%Y/%j/%H/}" + if ignore_missing is True: + try: + files += fs.ls(path, refresh=refresh) + except FileNotFoundError: + print(f"Ignored missing dir: {path}") + else: + files += fs.ls(path, refresh=refresh) + # Build a table of the files # -------------------------- @@ -324,6 +332,7 @@ def goes_timerange( max_cpus=config["timerange"].get("max_cpus"), bands=None, s3_refresh=config["timerange"].get("s3_refresh"), + ignore_missing=config["timerange"].get("ignore_missing"), verbose=config["timerange"].get("verbose", True), ): """ @@ -412,7 +421,7 @@ def goes_timerange( start = datetime.utcnow() - recent end = datetime.utcnow() - df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh) + df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh, ignore_missing=ignore_missing) if download: _download(df, save_dir=save_dir, overwrite=overwrite, verbose=verbose) @@ -457,6 +466,7 @@ def goes_single_point_timerange( max_cpus=config["timerange"].get("max_cpus"), bands=None, s3_refresh=config["timerange"].get("s3_refresh"), + ignore_missing=config["timerange"].get("ignore_missing"), verbose=config["timerange"].get("verbose", True), ): """ @@ -549,7 +559,7 @@ def goes_single_point_timerange( start = datetime.utcnow() - recent end = datetime.utcnow() - df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh) + df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh, ignore_missing=ignore_missing) if download: _download(df, save_dir=save_dir, overwrite=overwrite, verbose=verbose) @@ -577,6 +587,7 @@ def goes_latest( save_dir=config["latest"].get("save_dir"), bands=None, s3_refresh=config["latest"].get("s3_refresh"), + ignore_missing=config["latest"].get("ignore_missing"), verbose=config["latest"].get("verbose", True), ): """ @@ -637,7 +648,7 @@ def goes_latest( start = datetime.utcnow() - timedelta(hours=1) end = datetime.utcnow() - df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh) + df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh, ignore_missing=ignore_missing) # Filter for specific mesoscale domain if domain is not None and domain.upper() in ["M1", "M2"]: @@ -669,6 +680,7 @@ def goes_nearesttime( save_dir=config["nearesttime"].get("save_dir"), bands=None, s3_refresh=config["nearesttime"].get("s3_refresh"), + ignore_missing=config["nearesttime"].get("ignore_missing"), verbose=config["nearesttime"].get("verbose", True), ): """ @@ -738,7 +750,7 @@ def goes_nearesttime( start = attime - within end = attime + within - df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh) + df = _goes_file_df(satellite, product, start, end, bands=bands, refresh=s3_refresh, ignore_missing=ignore_missing) # return df, start, end, attime