From 05613b6b15bef62594bde82334f0e1e488d72993 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Thu, 20 Jun 2024 15:34:10 -0400 Subject: [PATCH 1/3] fix for 915: account for different use cases of median --- spectral_cube/spectral_cube.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spectral_cube/spectral_cube.py b/spectral_cube/spectral_cube.py index e5492c0e5..cb5cd9e13 100644 --- a/spectral_cube/spectral_cube.py +++ b/spectral_cube/spectral_cube.py @@ -1181,6 +1181,17 @@ def median(self, axis=None, iterate_rays=False, **kwargs): except ImportError: bnok = False + how = kwargs.pop('how', None) + if how == 'slice' and (not isinstance(axis, (list, tuple)) or len(axis) != 2): + raise ValueError("Cannot compute median slicewise unless you're compressing over two axes.") + elif how == 'ray': + if axis not in (0, 1, 2): + raise ValueError("Cannot compute median raywise unless you're compressing over one axis.") + else: + if not iterate_rays: + iterate_rays = True + warnings.warn("how='ray' was specified in call to median; this is setting iterate_rays=True") + # slicewise median is nonsense, must force how = 'cube' # bottleneck.nanmedian does not allow axis to be a list or tuple if bnok and not iterate_rays and not isinstance(axis, (list, tuple)): @@ -1194,6 +1205,11 @@ def median(self, axis=None, iterate_rays=False, **kwargs): result = self.apply_numpy_function(np.nanmedian, axis=axis, projection=True, unit=self.unit, how='cube',**kwargs) + elif iterate_rays: + result = self.apply_numpy_function( + nanmedian if bnok else np.nanmedian if hasattr(np, 'nanmedian') else np.median, + axis=axis, projection=True, unit=self.unit, how='ray', + check_endian=True, **kwargs) else: log.debug("Using numpy median iterating over rays") result = self.apply_function(np.median, projection=True, axis=axis, From a2aca8495096e3f242ce4e36223ba10f9d6e8ae1 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Thu, 27 Jun 2024 19:39:56 -0400 Subject: [PATCH 2/3] try to fix warning about large cubes: dask cubes were warning with non-dask keywords, likely caused by wrong inheritance order in the composite (the mixin is overridden by the base SpectralCube) --- spectral_cube/dask_spectral_cube.py | 9 -------- spectral_cube/spectral_cube.py | 2 +- spectral_cube/utils.py | 33 +++++++++++++++++------------ 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/spectral_cube/dask_spectral_cube.py b/spectral_cube/dask_spectral_cube.py index 08125222a..51c3f774b 100644 --- a/spectral_cube/dask_spectral_cube.py +++ b/spectral_cube/dask_spectral_cube.py @@ -287,15 +287,6 @@ def __exit__(self, *args): def _compute(self, array): return array.compute(**self._scheduler_kwargs) - def _warn_slow(self, funcname): - if self._is_huge and not self.allow_huge_operations: - raise ValueError("This function ({0}) requires loading the entire " - "cube into memory, and the cube is large ({1} " - "pixels), so by default we disable this operation. " - "To enable the operation, set " - "`cube.allow_huge_operations=True` and try again." - .format(funcname, self.size)) - def _get_filled_data(self, view=(), fill=np.nan, check_endian=None, use_memmap=None): if check_endian: diff --git a/spectral_cube/spectral_cube.py b/spectral_cube/spectral_cube.py index cb5cd9e13..44b51561c 100644 --- a/spectral_cube/spectral_cube.py +++ b/spectral_cube/spectral_cube.py @@ -1204,7 +1204,7 @@ def median(self, axis=None, iterate_rays=False, **kwargs): log.debug("Using numpy nanmedian") result = self.apply_numpy_function(np.nanmedian, axis=axis, projection=True, unit=self.unit, - how='cube',**kwargs) + how='cube', **kwargs) elif iterate_rays: result = self.apply_numpy_function( nanmedian if bnok else np.nanmedian if hasattr(np, 'nanmedian') else np.median, diff --git a/spectral_cube/utils.py b/spectral_cube/utils.py index c2983b0d4..8b5009955 100644 --- a/spectral_cube/utils.py +++ b/spectral_cube/utils.py @@ -27,19 +27,26 @@ def warn_slow(function): @wraps(function) def wrapper(self, *args, **kwargs): # if the function accepts a 'how', the 'cube' approach requires the whole cube in memory - warn_how = (kwargs.get('how') == 'cube') or 'how' not in kwargs - if self._is_huge and not self.allow_huge_operations and warn_how: - raise ValueError("This function ({0}) requires loading the entire " - "cube into memory, and the cube is large ({1} " - "pixels), so by default we disable this operation. " - "To enable the operation, set " - "`cube.allow_huge_operations=True` and try again. " - "Alternatively, you may want to consider using an " - "approach that does not load the whole cube into " - "memory by specifying how='slice' or how='ray'. " - "See {bigdataurl} for details." - .format(str(function), self.size, - bigdataurl=bigdataurl)) + argspec = inspect.getfullargspec(function) + accepts_how_keyword = 'how' in argspec.args or argspec.varkw == 'how' + + warn_how = accepts_how_keyword and ((kwargs.get('how') == 'cube') or 'how' not in kwargs) + + if self._is_huge and not self.allow_huge_operations: + warn_message = ("This function ({0}) requires loading the entire " + "cube into memory, and the cube is large ({1} " + "pixels), so by default we disable this operation. " + "To enable the operation, set " + "`cube.allow_huge_operations=True` and try again. ").format(str(function), self.size) + + if warn_how: + warn_message += ("Alternatively, you may want to consider using an " + "approach that does not load the whole cube into " + "memory by specifying how='slice' or how='ray'. ") + + warn_message += ("See {bigdataurl} for details.".format(bigdataurl=bigdataurl)) + + raise ValueError(warn_message) elif warn_how and not self._is_huge: # TODO: add check for whether cube has been loaded into memory warnings.warn("This function ({0}) requires loading the entire cube into " From 9c3120910cc32706eb2b83cbb3df4257f22ce6a9 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Thu, 27 Jun 2024 19:54:26 -0400 Subject: [PATCH 3/3] add missing import --- spectral_cube/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spectral_cube/utils.py b/spectral_cube/utils.py index 8b5009955..1ec6c9823 100644 --- a/spectral_cube/utils.py +++ b/spectral_cube/utils.py @@ -1,4 +1,5 @@ import warnings +import inspect from functools import wraps