Merge branch 'main' into AL-850

.github/workflows/build.yml

@@ -23,5 +23,6 @@ jobs:
         # Until we have arm64 runners, we can't automatically test arm64 wheels
         - cp3*-macosx_arm64
       sdist: true
+      test_command: python -c "from stcal.ramp_fitting.ols_cas22 import _ramp, _jump, _fit; from stcal.ramp_fitting import slope_fitter"
     secrets:
       pypi_token: ${{ secrets.PYPI_PASSWORD_STSCI_MAINTAINER }}

.github/workflows/ci.yml

@@ -28,10 +28,10 @@ jobs:
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
     with:
       envs: |
-        - linux: py39-oldestdeps-cov-xdist
-        - linux: py39-xdist
+        - linux: py310-oldestdeps-cov-xdist
         - linux: py310-xdist
-        - linux: py311-cov-xdist
+        - linux: py311-xdist
+        - linux: py312-cov-xdist
           coverage: codecov
         - macos: py311-xdist
   test_downstream:

.github/workflows/ci_cron.yml

@@ -16,5 +16,5 @@ jobs:
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
     with:
       envs: |
-        - macos: py39-xdist
         - macos: py310-xdist
+        - macos: py311-xdist

.github/workflows/tests_devdeps.yml

@@ -24,9 +24,9 @@ jobs:
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
     with:
       envs: |
-        - linux: py39-devdeps-xdist
         - linux: py310-devdeps-xdist
         - linux: py311-devdeps-xdist
+        - linux: py312-devdeps-xdist
         - linux: py3-devdeps-xdist
   test_downstream:
     if: (github.repository == 'spacetelescope/stcal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests')))

CHANGES.rst

@@ -1,8 +1,164 @@
-1.5.3 (unreleased)
+1.7.3 (unreleased)
 ==================
 
+General
+-------
+
+-
+
+Changes to API
+--------------
+
 - 
 
+Bug Fixes
+---------
+
+- 
+
+1.7.2 (2024-06-12)
+==================
+
+General
+-------
+
+- build with Numpy 2.0 release candidate [#260]
+
+Bug Fixes
+---------
+
+jump
+~~~~
+- Flag asymmetrical snowballs that are missed by the current code (JP-3638). This was changed to
+  not require that the center of the snowball jump ellipse is a saturated
+  pixel. [#261]
+
+1.7.1 (2024-05-21)
+==================
+
+Bug Fixes
+---------
+
+jump
+~~~~
+
+- Catch some additional warnings about all-NaN slices. [#258]
+
+ramp_fitting
+~~~~~~~~~~~~
+
+- Fix a bug in Poisson variance calculation visible when providing an average
+  dark current value in which the specified dark current was not converted to the
+  appropriate units for pixels with negative slopes.  This resulted in
+  incorrect SCI, ERR, and VAR_POISSON values. Also required revising the approach
+  for catching all-zero variance cases when average dark current was not
+  specified. [#255]
+
+- Refactor ramp fitting using a C extension to improve performance. [#156]
+
+1.7.0 (2024-03-25)
+==================
+
+Changes to API
+--------------
+
+jump
+~~~~
+
+- Switch multiprocessing method to ``fork_server``. [#249]
+
+ramp_fitting
+~~~~~~~~~~~~
+
+- Switch multiprocessing method to ``fork_server``. [#249]
+
+Bug Fixes
+---------
+
+jump
+~~~~
+
+- Updated the shower flagging code to mask reference pixels, require a minimum
+  number of groups to trigger the detection, and use all integrations to determine
+  the median value. [#248]
+
+ramp_fitting
+~~~~~~~~~~~~
+
+- Changed the data type of three variables that are used in measuring
+  the jump free segments of integrations. The variables were uint8 and
+  they would yield wrong results for integrations with more than 256
+  groups. [#251]
+
+- Use ``sqrtf`` instead of ``sqrt`` in ols_cas22 ramp fitting with
+  jump detection to avoid small numerical errors on different systems
+  due to a cast to/from double. [#252]
+
+
+Other
+-----
+
+jump
+~~~~
+
+- Enable the use of multiple integrations to find outliers. Also,
+  when the number of groups is above a threshold, use single pass
+  outlier flagging rather than the iterative flagging. [#242]
+
+- Use ``sqrtf`` instead of ``sqrt`` in ols_cas22 ramp fitting with
+  jump detection to avoid small numerical errors on different systems
+  due to a cast to/from double. [#252]
+
+1.6.1 (2024-02-29)
+==================
+
+Changes to API
+--------------
+
+ramp_fitting
+~~~~~~~~~~~~
+
+- Add ``average_dark_current`` to calculations of poisson variance. [#243]
+
+1.6.0 (2024-02-15)
+==================
+
+Changes to API
+--------------
+
+jump
+~~~~
+
+- Add in the flagging of groups in the integration after a snowball
+  occurs. The saturated core of the snowball gets flagged as jump
+  for a number of groups passed in as a parameter [#238]
+
+Bug Fixes
+---------
+
+jump
+~~~~
+
+- Fixed the computation of the number of rows per slice for multiprocessing, which
+  was causing different results when running the step with multiprocess [#239]
+
+- Fix the code to at least always flag the group with the shower and the requested
+  groups after the primary shower. [#237]
+
+Other
+-----
+
+jump
+~~~~
+
+- Reorganize jump docs between the jwst and stcal repos. [#240]
+
+ramp_fitting
+~~~~~~~~~~~~
+
+- Reorganize ramp_fitting docs between the jwst and stcal repos. [#240]
+
+
 1.5.2 (2023-12-13)
 ==================
 
@@ -24,7 +180,7 @@ Other
 - Enable automatic linting and code style checks [#187]
 
 ramp_fitting
-------------
+~~~~~~~~~~~~
 
 - Refactor Casertano, et.al, 2022 uneven ramp fitting and incorporate the matching
   jump detection algorithm into it. [#215]
@@ -84,13 +240,16 @@ jump
   within a group. [#207]
 
 - Added more allowable selections for the number of cores to use for
-  multiprocessing [#183].
+  multiprocessing [#183]
+
+- Fixed the computation of the number of rows per slice for multiprocessing,
+  which caused different results when running the step with multiprocess [#239]
 
 ramp_fitting
 ~~~~~~~~~~~~
 
 - Added more allowable selections for the number of cores to use for
-  multiprocessing [#183].
+  multiprocessing [#183]
 
 - Updating variance computation for invalid integrations, as well as
   updating the median rate computation by excluding groups marked as

README.md

@@ -10,13 +10,14 @@ STScI Calibration algorithms and tools.
 
 ![STScI Logo](docs/_static/stsci_logo.png)
 
-**STCAL requires Python 3.9 or above and a C compiler for dependencies.**
+> [!IMPORTANT]
+> STCAL requires Python 3.10 or above and a C compiler for dependencies.
 
-**Linux and MacOS platforms are tested and supported. Windows is not currently supported.**
+> [!IMPORTANT]
+> Linux and MacOS platforms are tested and supported. Windows is not currently supported.**
 
-**If installing on MacOS Mojave 10.14, you must install
-into an environment with python 3.9. Installation will fail on python 3.10 due
-to lack of a stable build for dependency `opencv-python`.**
+> [!WARNING]
+> Installation on MacOS Mojave 10.14 will fail due to lack of a stable build for dependency ``opencv-python``.
 
 `STCAL` is intended to be used as a support package for calibration pipeline
 software, such as the `JWST` and `Roman` calibration pipelines. `STCAL` is a

docs/stcal/jump/description.rst

@@ -1,6 +1,8 @@
+.. _jump_algorithm:
+
 Algorithm
 ---------
-This routine detects jumps in an exposure by looking for outliers
+This routine detects jumps by looking for outliers
 in the up-the-ramp signal for each pixel in each integration within
 an input exposure. On output, the GROUPDQ array is updated with the DQ flag
 "JUMP_DET" to indicate the location of each jump that was found.
@@ -10,34 +12,39 @@ output PIXELDQ array.
 The SCI and ERR arrays of the input data are not modified.
 
 The current implementation uses the two-point difference method described
-in Anderson&Gordon2011_.
+in `Anderson & Gordon (2011) <https://ui.adsabs.harvard.edu/abs/2011PASP..123.1237A>`_.
 
 Two-Point Difference Method
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 The two-point difference method is applied to each integration as follows:
 
-* Compute the first differences for each pixel (the difference between
-  adjacent groups)
-* Compute the clipped (dropping the largest difference) median of the first differences for each pixel.
-* Use the median to estimate the Poisson noise for each group and combine it
-  with the read noise to arrive at an estimate of the total expected noise for
-  each difference.
-* Compute the "difference ratio" as the difference between the first differences
-  of each group and the median, divided by the expected noise.
-* If the largest "difference ratio" is greater than the rejection threshold,
-  flag the group corresponding to that ratio as having a jump.
-* If a jump is found in a given pixel, iterate the above steps with the
-  jump-impacted group excluded, looking for additional lower-level jumps
-  that still exceed the rejection threshold.
-* Stop iterating on a given pixel when no new jumps are found or only one
-  difference remains.
-* If the there are only three differences (four groups), the standard median
-  is used rather than the clipped median.
-* If there are only two differences (three groups), the smallest one is compared to the larger
-  one and if the larger one is above a threshold, it is flagged as a jump.
+#. Compute the first differences for each pixel (the difference between
+   adjacent groups)
+#. Compute the clipped median (dropping the largest difference) of the first differences for each pixel.
+   If there are only three first difference values (four groups), no clipping is
+   performed when computing the median.
+#. Use the median to estimate the Poisson noise for each group and combine it
+   with the read noise to arrive at an estimate of the total expected noise for
+   each difference.
+#. Compute the "difference ratio" as the difference between the first differences
+   of each group and the median, divided by the expected noise.
+#. If the largest "difference ratio" is greater than the rejection threshold,
+   flag the group corresponding to that ratio as having a jump.
+#. If a jump is found in a given pixel, iterate the above steps with the
+   jump-impacted group excluded, looking for additional lower-level jumps
+   that still exceed the rejection threshold.
+#. Stop iterating on a given pixel when no new jumps are found or only one
+   difference remains.
+#. If there are only two differences (three groups), the smallest one is compared to the larger
+   one and if the larger one is above a threshold, it is flagged as a jump.
+#. If flagging of the 4 neighbors is requested, then the 4 adjacent pixels will
+   have ramp jumps flagged in the same group as the central pixel as long as it has
+   a jump between the min and max requested levels for this option.
+#. If flagging of groups after a ramp jump is requested, then the groups in the
+   requested time since a detected ramp jump will be flagged as ramp jumps if
+   the ramp jump is above the requested threshold.  Two thresholds and times are
+   possible for this option.
 
-Note that any ramp values flagged as SATURATED in the input GROUPDQ array
+Note that any ramp groups flagged as SATURATED in the input GROUPDQ array
 are not used in any of the above calculations and hence will never be
 marked as containing a jump.
-
-.. _Anderson&Gordon2011: https://ui.adsabs.harvard.edu/abs/2011PASP..123.1237A