diff --git a/changes/8909.scripts.rst b/changes/8909.scripts.rst new file mode 100644 index 0000000000..0addc426d1 --- /dev/null +++ b/changes/8909.scripts.rst @@ -0,0 +1 @@ +Remove the outdated schema_editor script. diff --git a/changes/8916.background.rst b/changes/8916.background.rst new file mode 100644 index 0000000000..e2e4414b14 --- /dev/null +++ b/changes/8916.background.rst @@ -0,0 +1 @@ +Apply bitwise operations in correct order when counting good pixels in the background mask during WFSS background subtraction. \ No newline at end of file diff --git a/changes/8935.general.rst b/changes/8935.general.rst new file mode 100644 index 0000000000..fe0b6a58e2 --- /dev/null +++ b/changes/8935.general.rst @@ -0,0 +1 @@ +Include xml and db test data as package data for lib tests. diff --git a/jwst/background/background_sub.py b/jwst/background/background_sub.py index b5fa267bfb..9edd641c23 100755 --- a/jwst/background/background_sub.py +++ b/jwst/background/background_sub.py @@ -272,6 +272,20 @@ def average_background(input_model, bkg_list, sigma, maxiters): return avg_bkg +def sufficient_background_pixels(dq_array, bkg_mask, min_pixels=100): + """Count number of good pixels for background use. + + Check DQ flags of pixels selected for bkg use - XOR the DQ values with + the DO_NOT_USE flag to flip the DO_NOT_USE bit. Then count the number + of pixels that AND with the DO_NOT_USE flag, i.e. initially did not have + the DO_NOT_USE bit set. + """ + return np.count_nonzero((dq_array[bkg_mask] + ^ pixel['DO_NOT_USE']) + & pixel['DO_NOT_USE'] + ) > min_pixels + + def subtract_wfss_bkg(input_model, bkg_filename, wl_range_name, mmag_extract=None): """Scale and subtract a background reference image from WFSS/GRISM data. @@ -310,18 +324,12 @@ def subtract_wfss_bkg(input_model, bkg_filename, wl_range_name, mmag_extract=Non # i.e. in regions we can use as background. if got_catalog: bkg_mask = mask_from_source_cat(input_model, wl_range_name, mmag_extract) - # Ensure mask has 100 pixels and that those pixels correspond to valid - # pixels using model DQ array - if np.count_nonzero(input_model.dq[bkg_mask] - ^ pixel['DO_NOT_USE'] - & pixel['DO_NOT_USE'] - ) < 100: + if not sufficient_background_pixels(input_model.dq, bkg_mask): log.warning("Not enough background pixels to work with.") log.warning("Step will be SKIPPED.") return None else: bkg_mask = np.ones(input_model.data.shape, dtype=bool) - # Compute the mean values of science image and background reference # image, including only regions where there are no identified sources. # Exclude pixel values in the lower and upper 25% of the histogram. diff --git a/jwst/background/tests/test_background.py b/jwst/background/tests/test_background.py index 7e895b7ddd..00b04c93a8 100644 --- a/jwst/background/tests/test_background.py +++ b/jwst/background/tests/test_background.py @@ -9,11 +9,13 @@ from numpy.testing import assert_allclose from stdatamodels.jwst import datamodels +from stdatamodels.jwst.datamodels.dqflags import pixel from jwst.assign_wcs import AssignWcsStep from jwst.background import BackgroundStep from jwst.stpipe import Step -from jwst.background.background_sub import robust_mean, mask_from_source_cat, no_NaN +from jwst.background.background_sub import (robust_mean, mask_from_source_cat, + no_NaN, sufficient_background_pixels) @pytest.fixture(scope="module") @@ -403,3 +405,23 @@ def test_no_nan(): # Make sure arrays are equal. assert np.array_equal(model.data, result.data) + + +def test_sufficient_background_pixels(): + model = datamodels.ImageModel(data=np.zeros((2048, 2048)), + dq=np.zeros((2048, 2048))) + refpix_flags = pixel['DO_NOT_USE'] | pixel['REFERENCE_PIXEL'] + model.dq[:4, :] = refpix_flags + model.dq[-4:, :] = refpix_flags + model.dq[:, :4] = refpix_flags + model.dq[:, -4:] = refpix_flags + + bkg_mask = np.ones((2048, 2048), dtype=bool) + # With full array minux refpix available for bkg, should be sufficient + assert sufficient_background_pixels(model.dq, bkg_mask) + + bkg_mask[4: -4, :] = 0 + bkg_mask[:, 4: -4] = 0 + # Now mask out entire array, mocking full source coverage of detector - + # no pixels should be available for bkg + assert not sufficient_background_pixels(model.dq, bkg_mask) diff --git a/jwst/datamodels/__init__.py b/jwst/datamodels/__init__.py index a3d64a70e3..508ac27400 100644 --- a/jwst/datamodels/__init__.py +++ b/jwst/datamodels/__init__.py @@ -31,7 +31,7 @@ _jwst_models = ["ModelContainer", "SourceModelContainer", "ModelLibrary"] # Deprecated modules in stdatamodels -_deprecated_modules = ['schema'] +_deprecated_modules = ['schema', 'schema_editor'] # Deprecated models in stdatamodels _deprecated_models: list[str] = [] @@ -52,6 +52,6 @@ sys.modules[f"jwst.datamodels.{attr}"] = obj # Add a few submodules to sys.modules without exposing them locally -for _submodule_name in ['schema_editor', 'validate']: +for _submodule_name in ['validate']: _submodule = importlib.import_module(f"stdatamodels.jwst.datamodels.{_submodule_name}") sys.modules[f"jwst.datamodels.{_submodule_name}"] = _submodule diff --git a/jwst/regtest/test_schema_editor.py b/jwst/regtest/test_schema_editor.py deleted file mode 100644 index bc111133a6..0000000000 --- a/jwst/regtest/test_schema_editor.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Tests for the schema editor - -Notes ------ -The tests do not have coverage. The original code was written without tests. -The tests here are for later modifications. -""" -import os -from pathlib import Path -import yaml - -import pytest - -import stdatamodels.jwst.datamodels.schema_editor as se - -from jwst.regtest.regtestdata import text_diff -from jwst.lib.file_utils import pushdir - -# Define data locations in artifactory -KEYWORD_DB = 'datamodels/keyword_db' -FIXED_SCHEMA = 'fixed' -SCHEMA_TRUTH = 'truth/test_schema_editor' - - -@pytest.fixture(scope='module') -def keyword_db(rtdata_module): - """Define the keyword database""" - rt = rtdata_module - - keyword_db = Path('keyword_db') - keyword_db.mkdir() - with pushdir(str(keyword_db)): - schemas = rt.data_glob(KEYWORD_DB, glob='*.json') - for schema in schemas: - rt.get_data(schema) - - return keyword_db - - -@pytest.fixture(scope='module') -def model_db(): - """Create a full Model_db""" - models = se.Model_db() - return models - - -@pytest.fixture(scope='module') -def run_editor_full(tmp_cwd_module, keyword_db): - """Just run the editor""" - - editor = se.Schema_editor( - input=str(keyword_db), output=FIXED_SCHEMA, - add=True, delete=True, edit=True, rename=True - ) - editor.change() - - # The fixed schema live in a date-stamped folder under FIXED_SCHEMA. - # It will be the only folder, so we can just return the first from the list. - return os.path.join(FIXED_SCHEMA, os.listdir(FIXED_SCHEMA)[0]) - - -@pytest.mark.bigdata -def test_limit_datamodels_from_file(tmp_cwd_module, model_db, keyword_db, rtdata_module): - """Test limiting datamodels from Schema_editor""" - - # Create the exclusion file. - exclude = list(model_db)[1:] - exclude_path = 'exclude.yaml' - with open(exclude_path, 'w') as fh: - yaml.dump(exclude, fh) - - # Run the editor - editor = se.Schema_editor(input=str(keyword_db), exclude_file=exclude_path, - list=True, rename=True) - editor.change() - assert len(editor.model_db) == 1 - - -def test_limit_datamodels(model_db): - """Limit the datamodel list""" - - # Reload the models but exclude everything except the first model. - exclude_list = list(model_db)[1:] - models_limited = se.Model_db(exclude=exclude_list) - assert len(models_limited) == 1 - - -@pytest.mark.bigdata -@pytest.mark.parametrize( - 'schema', - ['core.schema.yaml', - 'extract1dimage.schema.yaml', - 'guider_meta.schema.yaml', - 'ifucube.schema.yaml', - 'keyword_exptype.schema.yaml', 'keyword_pband.schema.yaml', 'keyword_readpatt.schema.yaml', - 'multiextract1d.schema.yaml', 'multispec.schema.yaml', - 'pathloss.schema.yaml', - 'referencefile.schema.yaml', - 'slitmeta.schema.yaml', - 'wcsinfo.schema.yaml', ] -) -def test_full_run(tmp_cwd_module, schema, run_editor_full, rtdata_module): - """Check fixed schema files""" - rt = rtdata_module - rt.output = os.path.join(run_editor_full, schema) - rt.get_truth(SCHEMA_TRUTH + '/' + schema) - - assert text_diff(rt.output, rt.truth) - - -@pytest.mark.bigdata -def test_no_option_warning(tmp_cwd_module, keyword_db): - """If no operations is given, raise an error""" - editor = se.Schema_editor( - input=keyword_db - ) - with pytest.raises(RuntimeError): - editor.change() diff --git a/jwst/scripts/schema_editor.py b/jwst/scripts/schema_editor.py deleted file mode 100755 index 941b04d751..0000000000 --- a/jwst/scripts/schema_editor.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2018 Association of Universities for Research in Astronomy (AURA) - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: - -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. - -# 2. Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. - -# 3. The name of AURA and its representatives may not be used to -# endorse or promote products derived from this software without -# specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY AURA ``AS IS'' AND ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL AURA BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS -# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR -# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -# DAMAGE. - -from stdatamodels.jwst.datamodels import schema_editor - - -def main(): - options = schema_editor.Options() - editor = schema_editor.Schema_editor() - editor.change(options) - - -if __name__ == '__main__': - main() diff --git a/jwst/scripts/tests/test_scripts.py b/jwst/scripts/tests/test_scripts.py index 18e702b597..b9ba4e8d08 100644 --- a/jwst/scripts/tests/test_scripts.py +++ b/jwst/scripts/tests/test_scripts.py @@ -11,7 +11,6 @@ 'create_data', 'okify_regtests', 'pointing_summary', - 'schema_editor', 'schemadoc', 'set_telescope_pointing', 'set_telescope_pointing.py', diff --git a/pyproject.toml b/pyproject.toml index 36bd029be8..b615cd49e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,6 @@ csvconvert = "jwst.csv_tools.csvconvert:CSVConvertScript" exp_to_source = "jwst.exp_to_source.main:Main" okify_regtests = "jwst.scripts.okify_regtests:main" pointing_summary = "jwst.scripts.pointing_summary:main" -schema_editor = "jwst.scripts.schema_editor:main" schemadoc = "jwst.scripts.schemadoc:main" set_telescope_pointing = "jwst.scripts.set_telescope_pointing:main" "set_telescope_pointing.py" = "jwst.scripts.set_telescope_pointing:deprecated_name" @@ -157,7 +156,8 @@ namespaces = false ] "jwst.lib" = [ "tests/data/*.asdf", - "tests/data/*.db", + "tests/data/**/*.db", + "tests/data/**/*.xml", "tests/data/*.ecsv", "tests/data/*.fits", ]