Skip to content

Commit

Permalink
parent 39d0e3f
Browse files Browse the repository at this point in the history 
author Ken MacDonald <[email protected]> 1697200039 -0400
committer Ken MacDonald <[email protected]> 1702991072 -0500

Updating testing for C implementation of ramp fitting.

Updating test for C code.

Updating first group orphan testing.

Updating tests.  Not sure if the four CR test is a valid test.

Removing test cases in ramp fitting.

Updating the CHARGELOSS read noise computation to only update pixels and integrations affected by CHARGELOSS.

Updating the change log for PR 8355.
  • Loading branch information
@kmacdonald-stsci
kmacdonald-stsci committed May 16, 2024
1 parent 3df35ef commit 9a90640
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 48 deletions.
4 changes: 4 additions & 0 deletions CHANGES.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,10 @@ ramp_fitting

- Modified one runtime warning filter. [#8320]

- Updated tests to properly handle the C extension (forcing arrays to be
of an expected type. Modified the CHARGELESS portion of the ramp fit
step code to update read noise ramps only affected by CHARGELOSS. [#8355]

refpix
------

Expand Down
9 changes: 7 additions & 2 deletions jwst/ramp_fitting/ramp_fit_step.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,11 +476,16 @@ def process(self, input):
# readnoise variances into copies of the original ramp fitting
# tuples.
image_info_new, integ_info_new = None, None
ch_int, ch_grp, ch_row, ch_col = wh_chargeloss

Check warning on line 479 in jwst/ramp_fitting/ramp_fit_step.py

View check run for this annotation

Codecov / codecov/patch

jwst/ramp_fitting/ramp_fit_step.py#L479 

Added line #L479 was not covered by tests
if image_info is not None and image_var_RN is not None:
image_info_new = (image_info[0], image_info[1], image_info[2], image_var_RN, image_info[4])
rnoise = image_info[3]
rnoise[ch_row, ch_col] = image_var_RN[ch_row, ch_col]
image_info_new = (image_info[0], image_info[1], image_info[2], rnoise, image_info[4])

Check warning on line 483 in jwst/ramp_fitting/ramp_fit_step.py

View check run for this annotation

Codecov / codecov/patch

jwst/ramp_fitting/ramp_fit_step.py#L481-L483 

Added lines #L481 - L483 were not covered by tests

if integ_info is not None and integ_var_RN is not None:
integ_info_new = (integ_info[0], integ_info[1], integ_info[2], integ_var_RN, integ_info[4])
rnoise = integ_info[3]
rnoise[ch_int, ch_row, ch_col] = integ_var_RN[ch_int, ch_row, ch_col]
integ_info_new = (integ_info[0], integ_info[1], integ_info[2], rnoise, integ_info[4])

Check warning on line 488 in jwst/ramp_fitting/ramp_fit_step.py

View check run for this annotation

Codecov / codecov/patch

jwst/ramp_fitting/ramp_fit_step.py#L486-L488 

Added lines #L486 - L488 were not covered by tests

image_info = image_info_new
integ_info = integ_info_new
Expand Down
105 changes: 59 additions & 46 deletions jwst/ramp_fitting/tests/test_ramp_fit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,14 +136,14 @@ def test_mixed_crs_and_donotuse():
max_seg, max_cr = calc_num_seg(gdq, 3, JUMP_DET, DO_NOT_USE)
assert max_seg == 3


def test_one_group_two_ints_fit_ols():
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(ngroups=1, gain=1, readnoise=10, nints=2)
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(
ngroups=1, gain=1., readnoise=10., nints=2)
model1.data[0, 0, 50, 50] = 10.0
model1.data[1, 0, 50, 50] = 12.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', dqflags.pixel)
model1, 1024 * 30000., False, rnoise, gain, 'OLS', 'optimal', 'none', dqflags.pixel)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], 11.0, 1e-6)
Expand All @@ -156,8 +156,8 @@ def test_multiprocessing():
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(
ngroups=ngroups, gain=1, readnoise=10, nints=nints, nrows=nrows, ncols=ncols)

delta_plane1 = np.zeros((nrows, ncols), dtype=np.float64)
delta_plane2 = np.zeros((nrows, ncols), dtype=np.float64)
delta_plane1 = np.zeros((nrows, ncols), dtype=np.float32)
delta_plane2 = np.zeros((nrows, ncols), dtype=np.float32)
delta_vec = np.asarray([x / 50.0 for x in range(nrows)])
for i in range(ncols):
delta_plane1[i, :] = delta_vec * i
Expand Down Expand Up @@ -192,8 +192,8 @@ def test_multiprocessing2():
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(
ngroups=ngroups, gain=1, readnoise=10, nints=nints, nrows=nrows, ncols=ncols)

delta_plane1 = np.zeros((nrows, ncols), dtype=np.float64)
delta_plane2 = np.zeros((nrows, ncols), dtype=np.float64)
delta_plane1 = np.zeros((nrows, ncols), dtype=np.float32)
delta_plane2 = np.zeros((nrows, ncols), dtype=np.float32)
delta_vec = np.asarray([x / 50.0 for x in range(nrows)])
for i in range(ncols):
delta_plane1[i, :] = delta_vec * i
Expand Down Expand Up @@ -242,6 +242,8 @@ def test_nocrs_noflux(self, method):
def test_nocrs_noflux_firstrows_are_nan(self, method):
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(ngroups=5)
model1.data[0, :, 0:12, :] = np.nan
# model1, gdq, rnoise, pixdq, err, gain = setup_inputs(ngroups=5, nrows=4, ncols=4)
# model1.data[0, :, 0:2, :] = np.nan

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 60000, False, rnoise, gain, 'OLS', 'optimal', 'none', dqflags.pixel)
Expand Down Expand Up @@ -306,26 +308,25 @@ def test_bad_gain_values(self, method):
# All pixel values are zero, so slope should be zero, except
# the pixels with invalid data. Those pixels should have
# NaN values.
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(ngroups=5)
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(ngroups=5, nrows=4, ncols=4)
model1.meta.exposure.ngroups = 11
gain.data[10, 10] = -10
gain.data[20, 20] = np.nan
gain.data[1, 1] = -10
gain.data[2, 2] = np.nan

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 64000, False, rnoise, gain, 'OLS', 'optimal', 'none', dqflags.pixel)

data = slopes[0]
dq = slopes[1]

data, dq, vp, vr, err = slopes
no_nan = np.zeros(data.shape, dtype=int)
no_nan[data != 0] = 1
tsum = sum(sum(no_nan))

assert tsum == 2
assert np.isnan(data[10, 10])
assert np.isnan(data[20, 20])
assert dq[10, 10] == NO_GAIN | DO_NOT_USE
assert dq[20, 20] == NO_GAIN | DO_NOT_USE
assert np.isnan(data[1, 1])
assert np.isnan(data[2, 2])
assert dq[1, 1] == NO_GAIN | DO_NOT_USE
assert dq[2, 2] == NO_GAIN | DO_NOT_USE

def test_simple_ramp(self, method):
# Here given a 10 group ramp with an exact slope of 20/group. The output slope should be 20.
Expand Down Expand Up @@ -412,6 +413,7 @@ def test_photon_noise_only_bad_last_frame_two_groups(self, method):
data = slopes.data
np.testing.assert_allclose(data[50, 50], cds_slope, 1e-6)

@pytest.mark.skip(reason="Unweighted fit not implemented.")
def test_photon_noise_with_unweighted_fit(self, method):
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(ngroups=5, gain=1000, readnoise=1)
model1.data[0, 0, 50, 50] = 10.0
Expand Down Expand Up @@ -491,23 +493,28 @@ def test_four_groups_two_CRs_at_end(self, method):
data = slopes[0]
np.testing.assert_allclose(data[50, 50], cds_slope, 1e-6)

@pytest.mark.skip(reason="Not ready for C implementation.")
def test_four_groups_four_CRs(self, method):
#
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(ngroups=4, gain=1, readnoise=10)
model1.data[0, 0, 50, 50] = 10.0
model1.data[0, 1, 50, 50] = 15.0
model1.data[0, 2, 50, 50] = 25.0
model1.data[0, 3, 50, 50] = 145.0
model1.groupdq[0, 0, 50, 50] = JUMP_DET
model1.groupdq[0, 1, 50, 50] = JUMP_DET
model1.groupdq[0, 2, 50, 50] = JUMP_DET
model1.groupdq[0, 3, 50, 50] = JUMP_DET
# Is this even a valid test?
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(
ngroups=4, nrows=1, ncols=2, gain=1., readnoise=10.)
row, col = 0, 1
model1.data[0, 0, row, col] = 10.0
model1.data[0, 1, row, col] = 15.0
model1.data[0, 2, row, col] = 25.0
model1.data[0, 3, row, col] = 145.0
model1.groupdq[0, 0, row, col] = JUMP_DET
model1.groupdq[0, 1, row, col] = JUMP_DET
model1.groupdq[0, 2, row, col] = JUMP_DET
model1.groupdq[0, 3, row, col] = JUMP_DET

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', dqflags.pixel)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], 0, 1e-6)
data, dq, vp, vr, err = slopes
# cdata, cdq, cvp, cvr, cerr = cube

np.testing.assert_allclose(data[row, col], 0, 1e-6)

def test_four_groups_three_CRs_at_end(self, method):
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(ngroups=4, gain=1, readnoise=10)
Expand All @@ -527,19 +534,23 @@ def test_four_groups_three_CRs_at_end(self, method):
np.testing.assert_allclose(data[50, 50], expected_slope, 1e-6)

def test_four_groups_CR_causes_orphan_1st_group(self, method):
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(ngroups=4, gain=.01, readnoise=10000)
model1.data[0, 0, 50, 50] = 10.0
model1.data[0, 1, 50, 50] = 125.0
model1.data[0, 2, 50, 50] = 145.0
model1.data[0, 3, 50, 50] = 165.0
model1.groupdq[0, 1, 50, 50] = JUMP_DET
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(
ngroups=4, nrows=2, ncols=2, gain=1.05, readnoise=12.5)

row, col = 1, 1
model1.data[0, 0, row, col] = 10.0
model1.data[0, 1, row, col] = 125.0
model1.data[0, 2, row, col] = 145.0
model1.data[0, 3, row, col] = 165.0
model1.groupdq[0, 1, row, col] = JUMP_DET

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', dqflags.pixel)

tol = 1.e-5
expected_slope = 20.0
data = slopes[0]
np.testing.assert_allclose(data[50, 50], expected_slope, 1e-6)
np.testing.assert_allclose(data[row, col], expected_slope, tol)

def test_one_group_fit(self, method):
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(ngroups=1, gain=1, readnoise=10)
Expand Down Expand Up @@ -609,7 +620,7 @@ def test_five_groups_unc(self, method):
# deltaDN = 50
delta_time = (ngroups - 1) * grouptime
# delta_electrons = median_slope * ingain *delta_time
single_sample_readnoise = np.float64(inreadnoise / np.sqrt(2))
single_sample_readnoise = np.float32(inreadnoise / np.sqrt(2))

data, dq, var_poisson, var_rnoise, err = slopes

Expand Down Expand Up @@ -653,7 +664,7 @@ def test_oneCR_10_groups_combination(self, method):
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', dqflags.pixel)

segment_groups = 5
single_sample_readnoise = np.float64(inreadnoise / np.sqrt(2))
single_sample_readnoise = np.float32(inreadnoise / np.sqrt(2))
# check that the segment variance is as expected

ovar_rnoise = opt_info[3]
Expand Down Expand Up @@ -747,6 +758,7 @@ def test_twenty_groups_two_segments():
check = model1.data[0, 0, 0, :] - oslope
tol = 1E-5
# Pixel 1 has zero slope, so ignore it.

np.testing.assert_allclose(opedestal[0, 0, 1:], check[0, 0, 0, 1:], tol)


Expand Down Expand Up @@ -882,7 +894,7 @@ def setup_inputs_ramp_model_new(dims, frame_data, timing, variance):
rampmodel.meta.subarray.ysize = nrows

# Set up the gain model
garray = np.ones(shape=(nrows, ncols), dtype=np.float64) * gain
garray = np.ones(shape=(nrows, ncols), dtype=np.float32) * gain
gmodel = GainModel(data=garray)
gmodel.meta.instrument.name = 'MIRI'
gmodel.meta.subarray.xstart = 1
Expand All @@ -891,7 +903,7 @@ def setup_inputs_ramp_model_new(dims, frame_data, timing, variance):
gmodel.meta.subarray.ysize = nrows

# Set up the read noise model
read_noise = np.full((nrows, ncols), rnoise, dtype=np.float64)
read_noise = np.full((nrows, ncols), rnoise, dtype=np.float32)
rnmodel = ReadnoiseModel(data=read_noise)
rnmodel.meta.instrument.name = 'MIRI'
rnmodel.meta.subarray.xstart = 1
Expand All @@ -907,11 +919,11 @@ def setup_small_cube(ngroups=10, nints=1, nrows=2, ncols=2, deltatime=10.,
"""
Create input MIRI datacube having the specified dimensions
"""
gain = np.ones(shape=(nrows, ncols), dtype=np.float64) * gain
err = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float64)
data = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float64)
gain = np.ones(shape=(nrows, ncols), dtype=np.float32) * gain
err = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float32)
data = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float32)
pixdq = np.zeros(shape=(nrows, ncols), dtype=np.int32)
rnoise = np.full((nrows, ncols), readnoise, dtype=np.float64)
rnoise = np.full((nrows, ncols), readnoise, dtype=np.float32)
gdq = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.uint8)
model1 = RampModel(data=data, err=err, pixeldq=pixdq, groupdq=gdq)

Expand All @@ -938,14 +950,15 @@ def setup_small_cube(ngroups=10, nints=1, nrows=2, ncols=2, deltatime=10.,


# Need test for multi-ints near zero with positive and negative slopes
def setup_inputs(ngroups=10, readnoise=10, nints=1, nrows=103, ncols=102,
nframes=1, grouptime=1.0, gain=1, deltatime=1):
# default dimensions are (1, 10, 103, 102)
def setup_inputs(ngroups=10, readnoise=10., nints=1, nrows=103, ncols=102,
nframes=1, grouptime=1.0, gain=1., deltatime=1.):

data = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float32)
err = np.ones(shape=(nints, ngroups, nrows, ncols), dtype=np.float32)
pixdq = np.zeros(shape=(nrows, ncols), dtype=np.uint32)
gdq = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.uint8)
gain = np.ones(shape=(nrows, ncols), dtype=np.float64) * gain
gain = np.ones(shape=(nrows, ncols), dtype=np.float32) * gain
rnoise = np.full((nrows, ncols), readnoise, dtype=np.float32)
int_times = np.zeros((nints,))

Expand Down

0 comments on commit 9a90640

Please sign in to comment.