diff --git a/changes/8831.datamodels.rst b/changes/8831.datamodels.rst new file mode 100644 index 0000000000..22289006df --- /dev/null +++ b/changes/8831.datamodels.rst @@ -0,0 +1 @@ +Remove memory-saving options from ModelContainer \ No newline at end of file diff --git a/docs/jwst/stpipe/devel_io_design.rst b/docs/jwst/stpipe/devel_io_design.rst index af233632f2..56f106734f 100644 --- a/docs/jwst/stpipe/devel_io_design.rst +++ b/docs/jwst/stpipe/devel_io_design.rst @@ -163,8 +163,8 @@ Many of the JWST calibration steps and pipelines expect an :ref:`Association ` file as input. When opened with :meth:`~jwst.stpipe.step.Step.open_model`, a :class:`~jwst.datamodels.ModelContainer` is returned. `ModelContainer` -is, among other features, a list-like object where each element is the -`DataModel` of each member of the association. The `meta.asn_table` is +is a list-like object where each element is the +`DataModel` of each member of the association. The `asn_table` attribute is populated with the association data structure, allowing direct access to the association itself. The association file, as well as the files listed in the association file, must be in the input directory. @@ -172,6 +172,38 @@ listed in the association file, must be in the input directory. To read in a list of files, or an association file, as an association, use the `load_as_level2_asn` or `load_as_level3_asn` methods. +ModelContainer vs ModelLibrary +`````````````````````````````` + +Some steps in the pipeline, namely any steps involved in the Stage 3 Imaging pipeline, +rely on the :class:`~jwst.datamodels.ModelLibrary` class instead of the +:class:`~jwst.datamodels.ModelContainer` class to process association-type data. +The `ModelLibrary` class is purpose-built for enabling memory-saving options in the +image3 pipeline and is only recommended when working with large associations. +Additional documentation on the `ModelLibrary` class can be found in the +`stpipe ModelLibrary documentation `_. + +ModelContainer Changes in JWST 1.17 +``````````````````````````````````` + +In JWST 1.17, the `ModelContainer` class was de-scoped in light of the introduction of the +`ModelLibrary` class in JWST 1.16. The `ModelContainer` class is still the recommended class +for handling association-type data, but it is no longer a subclass of `JWSTDataModel`. The +following changes in behavior are noteworthy: + +* The `ModelContainer` class no longer has a `meta` attribute. The association data is now + stored in the top-level `asn_table` attribute, along with several other association-relevant + attributes including `asn_table_name`, `asn_pool_name`, `asn_exptypes`, `asn_n_members`, + `asn_file_path`. Note that `asn_table` is now a dictionary, not an `ObjectNode`. +* All infrastructure that attempted memory savings in the `ModelContainer` class has been removed. + Use the `ModelLibrary` class if memory-saving options are needed. +* A `ModelContainer` object can no longer hold a list of `ModelContainer` objects. +* The `ModelContainer` class is still list-like, and can be indexed and sliced like a list. +* The `ModelContainer` class is still the default class returned by stdatamodels `open()` + for association-type input data, e.g. a .json file or dict. +* The `ModelContainer` class can still be used as a context manager, such that `with open(asn_file.json)` + still works. + Input Source ------------ diff --git a/jwst/badpix_selfcal/badpix_selfcal_step.py b/jwst/badpix_selfcal/badpix_selfcal_step.py index 7d9ad62c9f..b83aaa0f63 100644 --- a/jwst/badpix_selfcal/badpix_selfcal_step.py +++ b/jwst/badpix_selfcal/badpix_selfcal_step.py @@ -35,8 +35,7 @@ class BadpixSelfcalStep(Step): """ def save_model(self, model, *args, **kwargs): - """Override save_model to suppress index 0 when save_model is True - """ + """Override save_model to suppress index 0 when save_model is True""" kwargs["idx"] = None return Step.save_model(self, model, *args, **kwargs) diff --git a/jwst/cube_build/data_types.py b/jwst/cube_build/data_types.py index d7217087a4..711f729bd1 100644 --- a/jwst/cube_build/data_types.py +++ b/jwst/cube_build/data_types.py @@ -89,7 +89,7 @@ def __init__(self, input, single, output_file, output_dir): self.output_name = 'Temp' self.input_models = input_models if not single: # find the name of the output file from the association - self.output_name = input_models.meta.asn_table.products[0].name + self.output_name = input_models.asn_table["products"][0]["name"] else: # close files opened above self.close() diff --git a/jwst/datamodels/container.py b/jwst/datamodels/container.py index 45440bcb17..196c31ca5c 100644 --- a/jwst/datamodels/container.py +++ b/jwst/datamodels/container.py @@ -1,15 +1,10 @@ -import copy from collections import OrderedDict from collections.abc import Sequence +import copy import os.path as op import re import logging - -import numpy as np - -from asdf import AsdfFile from astropy.io import fits -from stdatamodels import properties from stdatamodels.jwst.datamodels.model_base import JwstDataModel from stdatamodels.jwst.datamodels.util import open as datamodel_open @@ -19,15 +14,26 @@ __all__ = ['ModelContainer'] -_ONE_MB = 1 << 20 RECOGNIZED_MEMBER_FIELDS = ['tweakreg_catalog', 'group_id'] +EMPTY_ASN_TABLE = { + "asn_id": None, + "asn_pool": None, + "products": [ + {"name": "", + "members": [ + {"exptype": "", + "expname": ""} + ] + } + ] +} # Configure logging logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) -class ModelContainer(JwstDataModel, Sequence): +class ModelContainer(Sequence): """ A container for holding DataModels. @@ -55,10 +61,6 @@ class ModelContainer(JwstDataModel, Sequence): asn_n_members : int Open only the first N qualifying members. - iscopy : bool - Presume this model is a copy. Members will not be closed - when the model is closed/garbage-collected. - Examples -------- >>> container = ModelContainer('example_asn.json') @@ -80,19 +82,6 @@ class ModelContainer(JwstDataModel, Sequence): Notes ----- - The optional paramters ``save_open`` and ``return_open`` can be - provided to control how the `JwstDataModel` are used by the - :py:class:`ModelContainer`. If ``save_open`` is set to `False`, each input - `JwstDataModel` instance in ``init`` will be written out to disk and - closed, then only the filename for the `JwstDataModel` will be used to - initialize the :py:class:`ModelContainer` object. - Subsequent access of each member will then open the `JwstDataModel` file to - work with it. If ``return_open`` is also `False`, then the `JwstDataModel` - will be closed when access to the `JwstDataModel` is completed. The use of - these parameters can minimize the amount of memory used by this object - during processing, with these parameters being used - by :py:class:`~jwst.outlier_detection.OutlierDetectionStep`. - When ASN table's members contain attributes listed in :py:data:`RECOGNIZED_MEMBER_FIELDS`, :py:class:`ModelContainer` will read those attribute values and update the corresponding attributes @@ -146,53 +135,47 @@ class ModelContainer(JwstDataModel, Sequence): ``models_grouped`` property for more details. """ - schema_url = None - - def __init__(self, init=None, asn_exptypes=None, asn_n_members=None, - iscopy=False, **kwargs): - super().__init__(init=None, **kwargs) + def __init__(self, init=None, asn_exptypes=None, asn_n_members=None, **kwargs): self._models = [] - self._iscopy = iscopy self.asn_exptypes = asn_exptypes self.asn_n_members = asn_n_members - self.asn_table = {} + self.asn_table = copy.deepcopy(EMPTY_ASN_TABLE) self.asn_table_name = None self.asn_pool_name = None self.asn_file_path = None self._memmap = kwargs.get("memmap", False) - self._return_open = kwargs.get('return_open', True) - self._save_open = kwargs.get('save_open', True) if init is None: # Don't populate the container with models pass - elif isinstance(init, fits.HDUList): - if self._save_open: - model = [datamodel_open(init, memmap=self._memmap)] - else: - model = init._file.name - init.close() - self._models.append(model) elif isinstance(init, list): if all(isinstance(x, (str, fits.HDUList, JwstDataModel)) for x in init): - if self._save_open: - init = [datamodel_open(m, memmap=self._memmap) for m in init] + for m in init: + self._models.append(datamodel_open(m, memmap=self._memmap)) + # set asn_table_name and product name to first datamodel stem since they were not provided + fname = self._models[0].meta.filename + if fname is not None: + root = op.basename(fname).split(".")[0] + default_name = "_".join(root.split("_")[:-1]) # remove old suffix + else: + default_name = "" + self.asn_table_name = default_name + self.asn_table["products"][0]["name"] = default_name else: raise TypeError("list must contain items that can be opened " "with jwst.datamodels.open()") - self._models = init elif isinstance(init, self.__class__): - instance = copy.deepcopy(init._instance) - self._schema = init._schema - self._shape = init._shape - self._asdf = AsdfFile(instance) - self._instance = instance - self._ctx = self - self._models = init._models - self._iscopy = True + for m in init: + self._models.append(datamodel_open(m, memmap=self._memmap)) + self.asn_exptypes = init.asn_exptypes + self.asn_n_members = init.asn_n_members + self.asn_table = init.asn_table + self.asn_table_name = init.asn_table_name + self.asn_pool_name = init.asn_pool_name + self.asn_file_path = init.asn_file_path elif is_association(init): self.from_asn(init) elif isinstance(init, str): @@ -207,10 +190,7 @@ def __len__(self): return len(self._models) def __getitem__(self, index): - m = self._models[index] - if not isinstance(m, JwstDataModel) and self._return_open: - m = datamodel_open(m, memmap=self._memmap) - return m + return self._models[index] def __setitem__(self, index, model): self._models[index] = model @@ -220,8 +200,6 @@ def __delitem__(self, index): def __iter__(self): for model in self._models: - if not isinstance(model, JwstDataModel) and self._return_open: - model = datamodel_open(model, memmap=self._memmap) yield model def insert(self, index, model): @@ -236,24 +214,19 @@ def extend(self, model): def pop(self, index=-1): self._models.pop(index) + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + def copy(self, memo=None): """ Returns a deep copy of the models in this model container. """ - result = self.__class__(init=None, - pass_invalid_values=self._pass_invalid_values, - strict_validation=self._strict_validation) - instance = copy.deepcopy(self._instance, memo=memo) - result._asdf = AsdfFile(instance) - result._instance = instance - result._iscopy = self._iscopy - result._schema = self._schema - result._ctx = result + result = self.__class__(init=None) for m in self._models: - if isinstance(m, JwstDataModel): - result.append(m.copy()) - else: - result.append(m) + result.append(m.copy(memo=memo)) return result @staticmethod @@ -315,37 +288,25 @@ def from_asn(self, asn_data): try: for member in sublist: filepath = op.join(asn_dir, member['expname']) - update_model = any(attr in member for attr in RECOGNIZED_MEMBER_FIELDS) - if update_model or self._save_open: - m = datamodel_open(filepath, memmap=self._memmap) - m.meta.asn.exptype = member['exptype'] - for attr, val in member.items(): - if attr in RECOGNIZED_MEMBER_FIELDS: - if attr == 'tweakreg_catalog': - if val.strip(): - val = op.join(asn_dir, val) - else: - val = None - - setattr(m.meta, attr, val) - - if not self._save_open: - m.save(filepath, overwrite=True) - m.close() - else: - m = filepath - + m = datamodel_open(filepath, memmap=self._memmap) + m.meta.asn.exptype = member['exptype'] + for attr, val in member.items(): + if attr in RECOGNIZED_MEMBER_FIELDS: + if attr == 'tweakreg_catalog': + if val.strip(): + val = op.join(asn_dir, val) + else: + val = None + + setattr(m.meta, attr, val) self._models.append(m) except IOError: self.close() raise - # Pull the whole association table into meta.asn_table - self.meta.asn_table = {} - properties.merge_tree( - self.meta.asn_table._instance, asn_data - ) + # Pull the whole association table into the asn_table attribute + self.asn_table = copy.deepcopy(asn_data) if self.asn_file_path is not None: self.asn_table_name = op.basename(self.asn_file_path) @@ -359,7 +320,6 @@ def from_asn(self, asn_data): def save(self, path=None, - dir_path=None, save_model_func=None, **kwargs): """ @@ -367,53 +327,37 @@ def save(self, Parameters ---------- - path : str or func or None + path : str or None - If None, the `meta.filename` is used for each model. - If a string, the string is used as a root and an index is appended. - - If a function, the function takes the two arguments: - the value of model.meta.filename and the - `idx` index, returning constructed file name. - - dir_path : str - Directory to write out files. Defaults to current working dir. - If directory does not exist, it creates it. Filenames are pulled - from `.meta.filename` of each datamodel in the container. save_model_func: func or None Alternate function to save each model instead of the models `save` method. Takes one argument, the model, and keyword argument `idx` for an index. + kwargs : dict + Additional parameters to be passed to the `save` method of each + model. + Returns ------- output_paths: [str[, ...]] List of output file paths of where the models were saved. """ output_paths = [] - if path is None: - def path(filename, idx=None): - return filename - elif not callable(path): - path = make_file_with_index - for idx, model in enumerate(self): - if len(self) <= 1: - idx = None if save_model_func is None: - outpath, filename = op.split( - path(model.meta.filename, idx=idx) - ) - if dir_path: - outpath = dir_path - save_path = op.join(outpath, filename) - try: - output_paths.append( - model.save(save_path, **kwargs) + if path is None: + save_path = model.meta.filename + else: + if len(self) <= 1: + idx = None + save_path = path+str(idx)+".fits" + output_paths.append( + model.save(save_path, **kwargs) ) - except IOError as err: - raise err - else: output_paths.append(save_model_func(model, idx=idx)) return output_paths @@ -454,8 +398,6 @@ def models_grouped(self): group_dict = OrderedDict() for i, model in enumerate(self._models): params = [] - if not self._save_open: - model = datamodel_open(model, memmap=self._memmap) if (hasattr(model.meta, 'group_id') and model.meta.group_id not in [None, '']): @@ -480,10 +422,6 @@ def models_grouped(self): group_id = model.meta.group_id - if not self._save_open and not self._return_open: - model.close() - model = self._models[i] - if group_id in group_dict: group_dict[group_id].append(model) else: @@ -503,10 +441,9 @@ def group_names(self): def close(self): """Close all datamodels.""" - if not self._iscopy: - for model in self._models: - if isinstance(model, JwstDataModel): - model.close() + for model in self._models: + if isinstance(model, JwstDataModel): + model.close() @property def crds_observatory(self): @@ -518,8 +455,6 @@ def crds_observatory(self): ------- str """ - # Eventually ModelContainer will also be used for Roman, but this - # will work for now: return "jwst" def get_crds_parameters(self): @@ -530,27 +465,15 @@ def get_crds_parameters(self): Returns ------- dict - """ - with self._open_first_science_exposure() as model: - return model.get_crds_parameters() - - def _open_first_science_exposure(self): - """ - Open first model with exptype SCIENCE, or the first model - if none exists. - Returns - ------- - stdatamodels.JwstDataModel + Notes + ----- + stpipe requires ModelContainer to have a crds_observatory attribute in order + to pass through step.run(), but it is never accessed. """ - for exposure in self.meta.asn_table.products[0].members: - if exposure.exptype.upper() == "SCIENCE": - first_exposure = exposure.expname - break - else: - first_exposure = self.meta.asn_table.products[0].members[0].expname - - return datamodel_open(first_exposure) + msg = ("stpipe uses the get_crds_parameters method from the 0th model in the " + "ModelContainer. This method is currently not used.") + raise NotImplementedError(msg) def ind_asn_type(self, asn_exptype): """ @@ -571,98 +494,3 @@ def ind_asn_type(self, asn_exptype): if model.meta.asn.exptype.lower() == asn_exptype: ind.append(i) return ind - - def set_buffer(self, buffer_size, overlap=None): - """Set buffer size for scrolling section-by-section access. - - Parameters - ---------- - buffer_size : float, None - Define size of buffer in MB for each section. - If `None`, a default buffer size of 1MB will be used. - - overlap : int, optional - Define the number of rows of overlaps between sections. - If `None`, no overlap will be used. - """ - self.overlap = 0 if overlap is None else overlap - self.grow = 0 - - with datamodel_open(self._models[0]) as model: - imrows, imcols = model.data.shape - data_item_size = model.data.itemsize - data_item_type = model.data.dtype - model.close() - del model - min_buffer_size = imcols * data_item_size - - self.buffer_size = min_buffer_size if buffer_size is None else (buffer_size * _ONE_MB) - - section_nrows = min(imrows, int(self.buffer_size // min_buffer_size)) - - if section_nrows == 0: - self.buffer_size = min_buffer_size - logger.warning("WARNING: Buffer size is too small to hold a single row." - f"Increasing buffer size to {self.buffer_size / _ONE_MB}MB") - section_nrows = 1 - - nbr = section_nrows - self.overlap - nsec = (imrows - self.overlap) // nbr - if (imrows - self.overlap) % nbr > 0: - nsec += 1 - - self.n_sections = nsec - self.nbr = nbr - self.section_nrows = section_nrows - self.imrows = imrows - self.imcols = imcols - self.imtype = data_item_type - - def get_sections(self): - """Iterator to return the sections from all members of the container.""" - - for k in range(self.n_sections): - e1 = k * self.nbr - e2 = e1 + self.section_nrows - - if k == self.n_sections - 1: # last section - e2 = min(e2, self.imrows) - e1 = min(e1, e2 - self.overlap - 1) - - data_list = np.empty((len(self._models), e2 - e1, self.imcols), - dtype=self.imtype) - wht_list = np.empty((len(self._models), e2 - e1, self.imcols), - dtype=self.imtype) - for i, model in enumerate(self._models): - model = datamodel_open(model, memmap=self._memmap) - - data_list[i, :, :] = model.data[e1:e2].copy() - wht_list[i, :, :] = model.wht[e1:e2].copy() - model.close() - del model - - yield (data_list, wht_list, (e1, e2)) - - -def make_file_with_index(file_path, idx): - """Append an index to a filename - - Parameters - ---------- - file_path: str - The file to append the index to. - idx: int - An index to append - - - Returns - ------- - file_path: str - Path with index appended - """ - # Decompose path - path_head, path_tail = op.split(file_path) - base, ext = op.splitext(path_tail) - if idx is not None: - base = base + str(idx) - return op.join(path_head, base + ext) diff --git a/jwst/datamodels/tests/data/association.json b/jwst/datamodels/tests/data/association.json index 0b3d4378d3..4dfa293f57 100644 --- a/jwst/datamodels/tests/data/association.json +++ b/jwst/datamodels/tests/data/association.json @@ -13,12 +13,12 @@ "members": [ { "exptype": "SCIENCE", - "expname": "test.fits", + "expname": "test.fits" }, { "exptype": "SCIENCE", - "expname": "test.fits", - }, + "expname": "test.fits" + } ] } ], diff --git a/jwst/datamodels/tests/test_model_container.py b/jwst/datamodels/tests/test_model_container.py index 2bf3528a32..2266f7909d 100644 --- a/jwst/datamodels/tests/test_model_container.py +++ b/jwst/datamodels/tests/test_model_container.py @@ -125,3 +125,25 @@ def test_group_id(tmp_path): model_droup_ids.add(m.meta.group_id) assert asn_group_ids == model_droup_ids + + +def test_save(tmp_cwd, container): + + # container pushes us to data/ directory so need to go back to tmp_cwd + # to avoid polluting the data/ directory + with pushdir(tmp_cwd): + + # test default just saves things at model meta filename + container.save() + expected_fnames = [] + for model in container: + expected_fnames.append(model.meta.filename) + for fname in expected_fnames: + assert os.path.exists(fname) + + # test specifying path saves to custom path with indices + path = "foo" + container.save(path) + expected_fnames = [path+str(i)+".fits" for i in range(len(container))] + for fname in expected_fnames: + assert os.path.exists(fname) diff --git a/jwst/datamodels/tests/test_open_association.py b/jwst/datamodels/tests/test_open_association.py index c11c9ec8dc..5ca8c4b930 100644 --- a/jwst/datamodels/tests/test_open_association.py +++ b/jwst/datamodels/tests/test_open_association.py @@ -10,7 +10,6 @@ # Define artificial memory size MEMORY = 100 # 100 bytes - # Utilities def t_path(partial_path): """Construction the full path for test files""" diff --git a/jwst/extract_1d/extract.py b/jwst/extract_1d/extract.py index bd7cda6d8c..fac9dc799f 100644 --- a/jwst/extract_1d/extract.py +++ b/jwst/extract_1d/extract.py @@ -2790,10 +2790,7 @@ def do_extract1d( # Set "meta_source" to either the first model in a container, or the individual input model, for convenience # of retrieving meta attributes in subsequent statements if was_source_model: - if isinstance(input_model, datamodels.SlitModel): # input_model is SourceContainer with a single SlitModel - meta_source = input_model - else: - meta_source = input_model[0] + meta_source = input_model[0] else: meta_source = input_model diff --git a/jwst/extract_1d/extract_1d_step.py b/jwst/extract_1d/extract_1d_step.py index f8b43b0178..3ff620e6c5 100644 --- a/jwst/extract_1d/extract_1d_step.py +++ b/jwst/extract_1d/extract_1d_step.py @@ -197,7 +197,10 @@ def process(self, input): """ # Open the input and figure out what type of model it is - input_model = datamodels.open(input) + if isinstance(input, ModelContainer): + input_model = input + else: + input_model = datamodels.open(input) was_source_model = False # default value if isinstance(input_model, datamodels.CubeModel): diff --git a/jwst/master_background/master_background_step.py b/jwst/master_background/master_background_step.py index 1ea83855ee..727b351c47 100755 --- a/jwst/master_background/master_background_step.py +++ b/jwst/master_background/master_background_step.py @@ -86,9 +86,9 @@ def process(self, input): if self.user_background: if isinstance(input_data, ModelContainer): input_data, _ = split_container(input_data) + asn_id = input_data.asn_table["asn_id"] del _ result = ModelContainer() - result.update(input_data) background_2d_collection = ModelContainer() background_2d_collection.update(input_data) for model in input_data: @@ -100,6 +100,7 @@ def process(self, input): model.meta.background.master_background_file = basename(self.user_background) # Use user-supplied master background and subtract it else: + asn_id = None background_2d = expand_to_2d(input_data, self.user_background) background_2d_collection = background_2d result = subtract_2d_background(input_data, background_2d) @@ -109,14 +110,20 @@ def process(self, input): # Save the computed 2d background if requested by user. The user has supplied # the master background so just save the expanded 2d background if self.save_background: - asn_id = input_data.meta.asn_table.asn_id - self.save_model(background_2d_collection, suffix='masterbg2d', force=True, asn_id=asn_id) + self.save_container(background_2d_collection, suffix='masterbg2d', force=True, asn_id=asn_id) # Compute master background and subtract it else: if isinstance(input_data, ModelContainer): input_data, background_data = split_container(input_data) - asn_id = input_data.meta.asn_table.asn_id + if len(background_data) == 0: + msg = ("No background data found in input container, " + "and no user-supplied background provided. Skipping step.") + self.log.warning(msg) + result = input_data.copy() + record_step_status(result, 'master_background', success=False) + return result + asn_id = input_data.asn_table["asn_id"] for model in background_data: # Check if the background members are nodded x1d extractions @@ -143,9 +150,7 @@ def process(self, input): background_data.close() result = ModelContainer() - result.update(input_data) background_2d_collection = ModelContainer() - background_2d_collection.update(input_data) for model in input_data: background_2d = expand_to_2d(model, master_background) result.append(subtract_2d_background(model, background_2d)) @@ -166,7 +171,7 @@ def process(self, input): # Save the computed background if requested by user if self.save_background: self.save_model(master_background, suffix='masterbg1d', force=True, asn_id=asn_id) - self.save_model(background_2d_collection, suffix='masterbg2d', force=True, asn_id=asn_id) + self.save_container(background_2d_collection, suffix='masterbg2d', force=True, asn_id=asn_id) record_step_status(result, 'master_background', success=True) @@ -224,6 +229,11 @@ def _do_sub(self): "run again and set force_subtract = True.") return do_sub + + def save_container(self, container, suffix="", asn_id="", force=True): + """Save all models in container for intermediate background subtraction""" + for i, model in enumerate(container): + self.save_model(model, suffix=suffix, force=force, asn_id=asn_id, idx=i) def copy_background_to_surf_bright(spectrum): @@ -240,8 +250,6 @@ def copy_background_to_surf_bright(spectrum): def split_container(container): """Divide a ModelContainer with science and background into one of each """ - asn = container.meta.asn_table.instance - background = ModelContainer() science = ModelContainer() @@ -252,12 +260,11 @@ def split_container(container): background.append(container._models[ind_bkgd]) # Pass along the association table to the output science container - science.meta.asn_table = {} science.asn_pool_name = container.asn_pool_name science.asn_table_name = container.asn_table_name - merge_tree(science.meta.asn_table.instance, asn) + merge_tree(science.asn_table, container.asn_table) # Prune the background members from the table - for p in science.meta.asn_table.instance['products']: + for p in science.asn_table['products']: p['members'] = [m for m in p['members'] if m['exptype'].lower() != 'background'] return science, background diff --git a/jwst/outlier_detection/outlier_detection_step.py b/jwst/outlier_detection/outlier_detection_step.py index 94a1ca7e21..1b6b42f733 100644 --- a/jwst/outlier_detection/outlier_detection_step.py +++ b/jwst/outlier_detection/outlier_detection_step.py @@ -159,7 +159,7 @@ def _guess_mode(self, input_models): return self.mode # guess mode from input type - if isinstance(input_models, (str, dict)): + if isinstance(input_models, (str, dict, list)): input_models = datamodels.open(input_models, asn_n_members=1) # Select which version of OutlierDetection @@ -202,6 +202,8 @@ def _get_asn_id(self, input_models): try: if isinstance(input_models, ModelLibrary): asn_id = input_models.asn["asn_id"] + elif isinstance(input_models, ModelContainer): + asn_id = input_models.asn_table["asn_id"] else: asn_id = input_models.meta.asn_table.asn_id except (AttributeError, KeyError): @@ -223,7 +225,7 @@ def _get_asn_id(self, input_models): def _set_status(self, input_models, status): # this might be called with the input which might be a filename or path - if not isinstance(input_models, (datamodels.JwstDataModel, ModelLibrary)): + if not isinstance(input_models, (datamodels.JwstDataModel, ModelLibrary, ModelContainer)): input_models = datamodels.open(input_models) record_step_status(input_models, "outlier_detection", status) diff --git a/jwst/outlier_detection/spec.py b/jwst/outlier_detection/spec.py index e019f456fe..6467daadd4 100644 --- a/jwst/outlier_detection/spec.py +++ b/jwst/outlier_detection/spec.py @@ -40,7 +40,7 @@ def detect_outliers( See `OutlierDetectionStep.spec` for documentation of these arguments. """ if not isinstance(input_models, ModelContainer): - input_models = ModelContainer(input_models, save_open=in_memory) + input_models = ModelContainer(input_models) if len(input_models) < 2: log.warning(f"Input only contains {len(input_models)} exposures") diff --git a/jwst/pipeline/calwebb_coron3.py b/jwst/pipeline/calwebb_coron3.py index b35cceecf8..ac5bb28cd3 100644 --- a/jwst/pipeline/calwebb_coron3.py +++ b/jwst/pipeline/calwebb_coron3.py @@ -90,14 +90,14 @@ def process(self, user_input): # This asn_id assignment is important as it allows outlier detection # to know the asn_id since that step receives the cube as input. - self.asn_id = input_models.meta.asn_table.asn_id + self.asn_id = input_models.asn_table["asn_id"] # Store the output file for future use - self.output_file = input_models.meta.asn_table.products[0].name + self.output_file = input_models.asn_table["products"][0]["name"] # Find all the member types in the product members_by_type = defaultdict(list) - prod = input_models.meta.asn_table.products[0].instance + prod = input_models.asn_table["products"][0] for member in prod['members']: members_by_type[member['exptype'].lower()].append(member['expname']) @@ -218,7 +218,7 @@ def process(self, user_input): model_blender.finalize_model(result) try: - result.meta.asn.pool_name = input_models.meta.asn_table.asn_pool + result.meta.asn.pool_name = input_models.asn_pool_name result.meta.asn.table_name = op.basename(user_input) except AttributeError: self.log.debug('Cannot set association information on final') diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index 2644bee233..d62bcc926f 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -128,12 +128,12 @@ def process(self, input): # products until the individual tasks work and do it themselves exptype = input_models[0].meta.exposure.type model_type = input_models[0].meta.model_type - output_file = input_models.meta.asn_table.products[0].name + output_file = input_models.asn_table["products"][0]["name"] self.output_file = output_file # Find all the member types in the product members_by_type = defaultdict(list) - product = input_models.meta.asn_table.products[0].instance + product = input_models.asn_table["products"][0] for member in product['members']: members_by_type[member['exptype'].lower()].append(member['expname']) @@ -145,7 +145,7 @@ def process(self, input): # If background data are present, call the master background step if members_by_type['background']: source_models = self.master_background(input_models) - source_models.meta.asn_table = input_models.meta.asn_table + source_models.asn_table = input_models.asn_table # If the step is skipped, do the container splitting that # would've been done in master_background diff --git a/jwst/pipeline/calwebb_tso3.py b/jwst/pipeline/calwebb_tso3.py index 2b9e14ae86..f5d974f48b 100644 --- a/jwst/pipeline/calwebb_tso3.py +++ b/jwst/pipeline/calwebb_tso3.py @@ -72,11 +72,11 @@ def process(self, input): return if self.output_file is None: - self.output_file = input_models.meta.asn_table.products[0].name + self.output_file = input_models.asn_table["products"][0]["name"] # This asn_id assignment is important as it allows outlier detection # to know the asn_id since that step receives the cube as input. - self.asn_id = input_models.meta.asn_table.asn_id + self.asn_id = input_models.asn_table["asn_id"] self.outlier_detection.mode = 'tso' # Input may consist of multiple exposures, so loop over each of them @@ -100,13 +100,13 @@ def process(self, input): original_filename = cube.meta.filename # ensure output filename will not have duplicate asn_id - if "_"+input_models.meta.asn_table.asn_id in original_filename: + if "_"+self.asn_id in original_filename: original_filename = original_filename.replace( - "_"+input_models.meta.asn_table.asn_id, '' + "_"+self.asn_id, '' ) self.save_model( cube, output_file=original_filename, suffix='crfints', - asn_id=input_models.meta.asn_table.asn_id + asn_id=self.asn_id ) cube.meta.filename = original_filename input_models[i] = cube @@ -173,7 +173,7 @@ def process(self, input): phot_result_list.append(self.white_light(result)) # Update some metadata from the association - x1d_result.meta.asn.pool_name = input_models.meta.asn_table.asn_pool + x1d_result.meta.asn.pool_name = input_models.asn_table["asn_pool"] x1d_result.meta.asn.table_name = op.basename(input) # Save the final x1d Multispec model diff --git a/jwst/pixel_replace/pixel_replace_step.py b/jwst/pixel_replace/pixel_replace_step.py index b76260d0a0..c7695440b8 100644 --- a/jwst/pixel_replace/pixel_replace_step.py +++ b/jwst/pixel_replace/pixel_replace_step.py @@ -52,8 +52,11 @@ def process(self, input): with datamodels.open(input) as input_model: # If more than one 2d spectrum exists in input, call replacement - if input_model.meta.model_type in ['MultiSlitModel', 'SlitModel', - 'ImageModel', 'IFUImageModel', 'CubeModel']: + if isinstance(input_model, (datamodels.MultiSlitModel, + datamodels.SlitModel, + datamodels.ImageModel, + datamodels.IFUImageModel, + datamodels.CubeModel)): self.log.debug('Input is a {input_model.meta.model_type}.') elif isinstance(input_model, datamodels.ModelContainer): self.log.debug('Input is a ModelContainer.') @@ -77,7 +80,7 @@ def process(self, input): # Setup output path naming if associations are involved. asn_id = None try: - asn_id = input_model.meta.asn_table.asn_id + asn_id = input_model.asn_table["asn_id"] except (AttributeError, KeyError): pass if asn_id is None: diff --git a/jwst/stpipe/tests/steps/__init__.py b/jwst/stpipe/tests/steps/__init__.py index 503b3e3d00..eab156a0e5 100644 --- a/jwst/stpipe/tests/steps/__init__.py +++ b/jwst/stpipe/tests/steps/__init__.py @@ -2,6 +2,7 @@ from stdatamodels.jwst import datamodels from stdatamodels.jwst.datamodels import ImageModel +from jwst.datamodels import ModelContainer class StepWithReference(Step): @@ -177,8 +178,12 @@ class StepWithContainer(Step): def process(self, *args): container = [] - model1 = ImageModel(args[0]).copy() - model2 = ImageModel(args[0]).copy() + if isinstance(args[0], ModelContainer): + model = args[0][0] + else: + model = args[0] + model1 = ImageModel(model).copy() + model2 = ImageModel(model).copy() model1.meta.filename = 'swc_model1.fits' model2.meta.filename = 'swc_model2.fits' container.append(model1) diff --git a/jwst/stpipe/tests/test_input.py b/jwst/stpipe/tests/test_input.py index 8586dd37db..a628382458 100644 --- a/jwst/stpipe/tests/test_input.py +++ b/jwst/stpipe/tests/test_input.py @@ -9,7 +9,7 @@ from jwst.stpipe import Step from jwst.datamodels import ModelContainer -from jwst.stpipe.tests.steps import StepWithModel +from jwst.stpipe.tests.steps import StepWithModel, StepWithContainer from jwst.stpipe.tests.util import t_path @@ -18,7 +18,7 @@ def test_default_input_with_container(mk_tmp_dirs): model_path = t_path('data/flat.fits') with ModelContainer([model_path]) as container: - step = StepWithModel() + step = StepWithContainer() step.run(container) assert step._input_filename is None diff --git a/jwst/tweakreg/tests/test_tweakreg.py b/jwst/tweakreg/tests/test_tweakreg.py index 9dc21267c9..57db989e16 100644 --- a/jwst/tweakreg/tests/test_tweakreg.py +++ b/jwst/tweakreg/tests/test_tweakreg.py @@ -290,7 +290,9 @@ def test_custom_catalog(custom_catalog_path, example_input, catfile, asn, meta, example_input[0].meta.tweakreg_catalog = "" # write out the ModelContainer and association (so the association table will be loaded) - example_input.save(dir_path=str(custom_catalog_path.parent)) + for model in example_input: + model.save(model.meta.filename, dir_path=str(custom_catalog_path.parent)) + model.close() asn_data = { 'asn_id': 'foo', 'asn_pool': 'bar', diff --git a/pyproject.toml b/pyproject.toml index 0ef1fc7253..1b980ddb5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "scipy>=1.9.3", "spherical-geometry>=1.2.22", "stcal @ git+https://github.com/spacetelescope/stcal.git@main", - "stdatamodels>=2.1.0,<2.2.0", + "stdatamodels @ git+https://github.com/spacetelescope/stdatamodels.git@main", "stpipe>=0.7.0,<0.8.0", "stsci.imagestats>=1.6.3", "synphot>=1.2",