From e82661010ec2110393487c0ab76dcf02ff983f26 Mon Sep 17 00:00:00 2001 From: David Law Date: Wed, 27 Nov 2024 13:26:32 -0500 Subject: [PATCH 1/7] Change prior-group flagging to DO_NOT_USE instead of SATURATION --- src/stcal/saturation/saturation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/stcal/saturation/saturation.py b/src/stcal/saturation/saturation.py index c607c64a..5f97cadc 100644 --- a/src/stcal/saturation/saturation.py +++ b/src/stcal/saturation/saturation.py @@ -133,11 +133,11 @@ def flag_saturated_pixels( # Reset flag array to only pixels passing this gauntlet flagarray[:] = 0 - flagarray[indx] = 2 + flagarray[indx] = dnu # Grow the newly-flagged saturating pixels if n_pix_grow_sat > 0: - flagarray = adjacent_pixels(flagarray, saturated, n_pix_grow_sat) + flagarray = adjacent_pixels(flagarray, dnu, n_pix_grow_sat) # Add them to the gdq array np.bitwise_or(gdq[ints, group, :, :], flagarray, gdq[ints, group, :, :]) @@ -166,10 +166,10 @@ def flag_saturated_pixels( # Flag the 2nd group for the pixels passing that gauntlet flagarray = np.zeros_like(mask,dtype='uint8') - flagarray[mask] = saturated + flagarray[mask] = dnu # flag any pixels that border these new pixels if n_pix_grow_sat > 0: - flagarray = adjacent_pixels(flagarray, saturated, n_pix_grow_sat) + flagarray = adjacent_pixels(flagarray, dnu, n_pix_grow_sat) # Add them to the gdq array np.bitwise_or(gdq[ints, 1, :, :], flagarray, gdq[ints, 1, :, :]) From 0089d93354dd7336b5c8c747c3d717f4858a0138 Mon Sep 17 00:00:00 2001 From: David Law Date: Wed, 27 Nov 2024 14:40:24 -0500 Subject: [PATCH 2/7] Fix boundary issue --- src/stcal/saturation/saturation.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/stcal/saturation/saturation.py b/src/stcal/saturation/saturation.py index 5f97cadc..dcccf0ec 100644 --- a/src/stcal/saturation/saturation.py +++ b/src/stcal/saturation/saturation.py @@ -167,9 +167,6 @@ def flag_saturated_pixels( # Flag the 2nd group for the pixels passing that gauntlet flagarray = np.zeros_like(mask,dtype='uint8') flagarray[mask] = dnu - # flag any pixels that border these new pixels - if n_pix_grow_sat > 0: - flagarray = adjacent_pixels(flagarray, dnu, n_pix_grow_sat) # Add them to the gdq array np.bitwise_or(gdq[ints, 1, :, :], flagarray, gdq[ints, 1, :, :]) From a6646a5bad8dce523e02404c9e3ecb0c589485d4 Mon Sep 17 00:00:00 2001 From: David Law Date: Wed, 27 Nov 2024 14:44:28 -0500 Subject: [PATCH 3/7] Add change log entry --- changes/321.saturation.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/321.saturation.rst diff --git a/changes/321.saturation.rst b/changes/321.saturation.rst new file mode 100644 index 00000000..cc087d1f --- /dev/null +++ b/changes/321.saturation.rst @@ -0,0 +1 @@ +Change flagging of 'pre-saturation' grouped data to use DO_NOT_USE instead of SATURATION flag to avoid confusing the snowball routine downstream. From 36d1cc6eb82fbe5c06197be7d567ba3b2e6e6b41 Mon Sep 17 00:00:00 2001 From: David Law Date: Wed, 27 Nov 2024 15:17:01 -0500 Subject: [PATCH 4/7] Move change log entry --- changes/{321.saturation.rst => 321.bugfix.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{321.saturation.rst => 321.bugfix.rst} (100%) diff --git a/changes/321.saturation.rst b/changes/321.bugfix.rst similarity index 100% rename from changes/321.saturation.rst rename to changes/321.bugfix.rst From 1b49a43b80de861d4ab3476313694eaebc3efafb Mon Sep 17 00:00:00 2001 From: Tyler Pauly Date: Mon, 16 Dec 2024 14:54:10 -0500 Subject: [PATCH 5/7] xfail unit test, add coverage for this change --- tests/test_saturation.py | 42 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/test_saturation.py b/tests/test_saturation.py index cc1465a3..a3869600 100644 --- a/tests/test_saturation.py +++ b/tests/test_saturation.py @@ -4,7 +4,7 @@ """ from enum import IntEnum - +import pytest import numpy as np from stcal.saturation.saturation import flag_saturated_pixels @@ -40,6 +40,7 @@ def test_basic_saturation_flagging(): assert np.all(gdq[0, satindex:, 5, 5] == DQFLAGS["SATURATED"]) +@pytest.mark.xfail(reason="stcal PR#321 broke this test") def test_read_pattern_saturation_flagging(): """Check that the saturation threshold varies depending on how the reads are allocated into resultants.""" @@ -78,6 +79,45 @@ def test_read_pattern_saturation_flagging(): assert np.all(gdq[0, 2:, 5, 5] == DQFLAGS["SATURATED"]) +def test_read_pattern_saturation_flagging_dnu(): + """Check that the saturation threshold varies depending on how the reads + are allocated into resultants. First resultant expected to have DNU while + PR#321 band-aid still in place.""" + + # Create inputs, data, and saturation maps + data = np.zeros((1, 5, 20, 20)).astype("float32") + gdq = np.zeros((1, 5, 20, 20)).astype("uint32") + pdq = np.zeros((20, 20)).astype("uint32") + sat_thresh = np.ones((20, 20)) * 100000.0 + sat_dq = np.zeros((20, 20)).astype("uint32") + + # Add ramp values up to the saturation limit + data[0, 0, 5, 5] = 0 + data[0, 1, 5, 5] = 20000 + data[0, 2, 5, 5] = 40000 + data[0, 3, 5, 5] = 60000 # Signal reaches saturation limit + data[0, 4, 5, 5] = 62000 + + # Set saturation value in the saturation model + satvalue = 60000 + sat_thresh[5, 5] = satvalue + + # set read_pattern to have many reads in the third resultant, so that + # its mean exposure time is much smaller than its last read time + # (in this case, the ratio is 13 / 20). + # This means that the effective saturation for the third resultant + # is 60000 * 13 / 20 = 39000 and the third resultant should be marked + # saturated. + read_pattern = [[1], [2], [3, 4, 5, 6, 7, 8, 9, 10], [11], [12], [13]] + + gdq, pdq, _ = flag_saturated_pixels( + data, gdq, pdq, sat_thresh, sat_dq, ATOD_LIMIT, DQFLAGS, read_pattern=read_pattern + ) + + # Make sure that groups after the third get flagged + assert np.all(gdq[0, 2:, 5, 5] == [DQFLAGS["DO_NOT_USE"], DQFLAGS["SATURATED"], DQFLAGS["SATURATED"]]) + + def test_no_sat_check_at_limit(): """Test to verify that pixels at the A-to-D limit (65535), but flagged with NO_SAT_CHECK do NOT get flagged as saturated, and that their neighbors From 8f0186b9582d13fb2c492f01f44c78a7cffa1e0a Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 19 Dec 2024 14:57:15 -0500 Subject: [PATCH 6/7] update dependencies --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1b2c79e9..7c46ff17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,14 +13,14 @@ classifiers = [ "Programming Language :: Python :: 3", ] dependencies = [ - "astropy >=5.0.4", + "astropy >=6.0.0", "drizzle>=1.15.0", - "scipy >=1.7.2", - "scikit-image>=0.19", - "numpy >=1.21.2", + "scipy >=1.14.1", + "scikit-image>=0.20.0", + "numpy >=1.25.0", "opencv-python-headless >=4.6.0.66", - "asdf >=2.15.0", - "gwcs @ git+https://github.com/spacetelescope/gwcs.git@master", + "asdf >=3.3.0", + "gwcs >=0.22.0", "tweakwcs >=0.8.8", "requests >=2.22", ] From fc7183a367466ed711fb1cc916a1b1fa54943569 Mon Sep 17 00:00:00 2001 From: Tyler Pauly Date: Fri, 20 Dec 2024 12:52:50 -0500 Subject: [PATCH 7/7] prepare changelog for build release --- CHANGES.rst | 32 ++++++++++++++++++++++++++++++++ changes/306.apichange.rst | 2 -- changes/314.bugfix.rst | 1 - changes/318.bugfix.rst | 5 ----- changes/319.bugfix.rst | 1 - changes/321.bugfix.rst | 1 - 6 files changed, 32 insertions(+), 10 deletions(-) delete mode 100644 changes/306.apichange.rst delete mode 100644 changes/314.bugfix.rst delete mode 100644 changes/318.bugfix.rst delete mode 100644 changes/319.bugfix.rst delete mode 100644 changes/321.bugfix.rst diff --git a/CHANGES.rst b/CHANGES.rst index d8fa679f..896d9c8a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,35 @@ +1.11.0 (2024-12-20) +=================== + +Changes to API +-------------- + +- Add maximum_shower_amplitude parameter to MIRI cosmic rays showers routine + to fix accidental flagging of bright science pixels. (`#306 + `_) + + +Bug Fixes +--------- + +- Do not evaluate the inverse WCS transform within the bounding box in cases + where a resampled WCS is computed. Do not pass table columns to the WCS + Shared API. (`#314 `_) +- For `ramp_fitting`, the `CRMAG` element was not originally implemented in + the C-extension for ramp fitting. It is now implemented. A bug in the read + noise recalculation for CHARGELOSS when using the multiprocessing option has + been fixed. Further, in `JWST` regression tests have been added to test for + multiprocessing to ensure testing for anything that could affect + multiprocessing. (`#318 + `_) +- Update weight threshold calculation in outlier detection to work around numpy + bug that introduces small numerical differences for a mean of a masked array. + (`#319 `_) +- Change flagging of 'pre-saturation' grouped data to use DO_NOT_USE instead of + SATURATION flag to avoid confusing the snowball routine downstream. (`#321 + `_) + + 1.10.0 (2024-11-15) =================== diff --git a/changes/306.apichange.rst b/changes/306.apichange.rst deleted file mode 100644 index d04413a5..00000000 --- a/changes/306.apichange.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add maximum_shower_amplitude parameter to MIRI cosmic rays showers routine -to fix accidental flagging of bright science pixels. diff --git a/changes/314.bugfix.rst b/changes/314.bugfix.rst deleted file mode 100644 index 589d75d8..00000000 --- a/changes/314.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Do not evaluate the inverse WCS transform within the bounding box in cases where a resampled WCS is computed. Do not pass table columns to the WCS Shared API. diff --git a/changes/318.bugfix.rst b/changes/318.bugfix.rst deleted file mode 100644 index 286be7e1..00000000 --- a/changes/318.bugfix.rst +++ /dev/null @@ -1,5 +0,0 @@ -For `ramp_fitting`, the `CRMAG` element was not originally implemented in -the C-extension for ramp fitting. It is now implemented. A bug in the read -noise recalculation for CHARGELOSS when using the multiprocessing option has -been fixed. Further, in `JWST` regression tests have been added to test for -multiprocessing to ensure testing for anything that could affect multiprocessing. diff --git a/changes/319.bugfix.rst b/changes/319.bugfix.rst deleted file mode 100644 index 0fd87fd4..00000000 --- a/changes/319.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Update weight threshold calculation in outlier detection to work around numpy bug that introduces small numerical differences for a mean of a masked array. diff --git a/changes/321.bugfix.rst b/changes/321.bugfix.rst deleted file mode 100644 index cc087d1f..00000000 --- a/changes/321.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Change flagging of 'pre-saturation' grouped data to use DO_NOT_USE instead of SATURATION flag to avoid confusing the snowball routine downstream.