From 30ad631c29a0db387e89471a313ac83c455b0a0e Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 1 May 2024 15:03:00 -0400 Subject: [PATCH 01/44] Fixed slits in MSA file working up through extract_2d --- jwst/assign_wcs/nirspec.py | 78 ++++++++++++++++++++++++++++++++------ jwst/extract_2d/nirspec.py | 7 +++- 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index c6e3e8c2ef..563413afee 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -35,6 +35,8 @@ log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) +FIXED_SLIT_NUMS = {'NONE': 0, 'S200A1': 1, 'S200A2': 2, + 'S400A1': 3, 'S1600A1': 4, 'S200B1': 5} __all__ = ["create_pipeline", "imaging", "ifu", "slits_wcs", "get_open_slits", "nrs_wcs_set_input", "nrs_ifu_wcs", "get_spectral_order_wrange"] @@ -437,12 +439,11 @@ def get_open_fixed_slits(input_model, slit_y_range=[-.55, .55]): if input_model.meta.instrument.fixed_slit is None: input_model.meta.instrument.fixed_slit = 'NONE' - slit_nums = {'NONE':0, 'S200A1':1, 'S200A2':2, 'S400A1':3, 'S1600A1':4, 'S200B1':5} primary_slit = input_model.meta.instrument.fixed_slit ylow, yhigh = slit_y_range # Slits are defined with hardwired source ID's, based on the assignments - # in the "slit_nums" dictionary. Exact assignments depend on whether the + # in the "FIXED_SLIT_NUMS" dictionary. Exact assignments depend on whether the # slit is the "primary" and hence contains the target of interest. The # source_id for the primary slit is always 1, while source_ids for secondary # slits is a two-digit value, where the first (tens) digit corresponds to @@ -452,15 +453,20 @@ def get_open_fixed_slits(input_model, slit_y_range=[-.55, .55]): # # Slit(Name, ShutterID, DitherPos, Xcen, Ycen, Ymin, Ymax, Quad, SourceID) s2a1 = Slit('S200A1', 0, 0, 0, 0, ylow, yhigh, 5, - 1 if primary_slit=='S200A1' else 10*slit_nums[primary_slit] + 1) + 1 if primary_slit == 'S200A1' + else 10 * FIXED_SLIT_NUMS[primary_slit] + 1) s2a2 = Slit('S200A2', 1, 0, 0, 0, ylow, yhigh, 5, - 1 if primary_slit=='S200A2' else 10*slit_nums[primary_slit] + 2) + 1 if primary_slit == 'S200A2' + else 10 * FIXED_SLIT_NUMS[primary_slit] + 2) s4a1 = Slit('S400A1', 2, 0, 0, 0, ylow, yhigh, 5, - 1 if primary_slit=='S400A1' else 10*slit_nums[primary_slit] + 3) + 1 if primary_slit == 'S400A1' + else 10 * FIXED_SLIT_NUMS[primary_slit] + 3) s16a1 = Slit('S1600A1', 3, 0, 0, 0, ylow, yhigh, 5, - 1 if primary_slit=='S1600A1' else 10*slit_nums[primary_slit] + 4) + 1 if primary_slit == 'S1600A1' + else 10 * FIXED_SLIT_NUMS[primary_slit] + 4) s2b1 = Slit('S200B1', 4, 0, 0, 0, ylow, yhigh, 5, - 1 if primary_slit=='S200B1' else 10*slit_nums[primary_slit] + 5) + 1 if primary_slit == 'S200B1' + else 10 * FIXED_SLIT_NUMS[primary_slit] + 5) # Decide which slits need to be added to this exposure subarray = input_model.meta.subarray.name.upper() @@ -637,7 +643,7 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, # add a margin to the slit y limits margin = 0.5 - # Now lets look at each unique slitlet id + # Now let's look at each unique MOS slitlet id for slitlet_id in slitlet_ids_unique: # Get the rows for the current slitlet_id slitlets_sid = [x for x in msa_data if x['slitlet_id'] == slitlet_id] @@ -646,11 +652,56 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, # How many shutters in the slitlet are labeled as "main" or "primary"? n_main_shutter = len([s for s in slitlets_sid if s['primary_source'] == 'Y']) + # Check for fixed slit + try: + # TODO: confirm default value for true MSA slitlets + is_fs = [x['fixed_slit'] != 'NONE' for x in slitlets_sid] + except (IndexError, ValueError, KeyError): + # May be old-style MSA file without a fixed_slit column + is_fs = [False] * len(slitlets_sid) + # In the next part we need to calculate, find, determine 5 things: # quadrant, xcen, ycen, ymin, ymax - # There are no main shutters: all are background - if n_main_shutter == 0: + # First check for a fixed slit + shutter_id = None + if all(is_fs) and len(slitlets_sid) == 1 and n_main_shutter == 1: + # One fixed slit open for the source, and it is marked 'primary' + slitlet = slitlets_sid[0] + slit_name = slitlet['fixed_slit'] + log.debug(f'Found fixed slit {slit_name}') + + # use standard number for fixed slit shutter id + slitlet_id = slit_name + shutter_id = FIXED_SLIT_NUMS[slit_name] - 1 + xcen = ycen = 0 + quadrant = 5 + + # no additional margin for fixed slit bounding boxes + ymin = ylow + ymax = yhigh + + # source position and id + # TODO: check if source position is offset by 0.5 + source_id = slitlet['source_id'] + source_xpos = slitlet['estimated_source_in_shutter_x'] + source_ypos = slitlet['estimated_source_in_shutter_y'] + + elif any(is_fs): + # Unsupported fixed slit configuration + message = ("For slitlet_id = {}, metadata_id = {}, " + "dither_index = {}".format( + slitlet_id, msa_metadata_id, dither_position)) + log.warning(message) + message = ("MSA configuration file has an unsupported " + "fixed slit configuration") + log.warning(message) + msa_file.close() + raise MSAFileError(message) + + # Now check for regular MSA slitlets + elif n_main_shutter == 0: + # There are no main shutters: all are background if len(open_shutters) == 1: jmin = jmax = j = open_shutters[0] else: @@ -662,6 +713,7 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, quadrant = slitlets_sid[0]['shutter_quadrant'] ycen = j xcen = slitlets_sid[0]['shutter_row'] # grab the first as they are all the same + source_xpos = 0.5 source_ypos = 0.5 source_id = _get_bkg_source_id(bkg_counter, max_source_id) @@ -682,7 +734,9 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, j = ycen ymax = yhigh + margin + (jmax - j) * 1.15 ymin = -(-ylow + margin) + (jmin - j) * 1.15 + # get the source_id from the primary shutter entry + source_id = None for i in range(len(slitlets_sid)): if slitlets_sid[i]['primary_source'] == 'Y': source_id = slitlets_sid[i]['source_id'] @@ -698,7 +752,8 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, raise MSAFileError(message) # subtract 1 because shutter numbers in the MSA reference file are 1-based. - shutter_id = xcen + (ycen - 1) * 365 + if shutter_id is None: + shutter_id = xcen + (ycen - 1) * 365 try: source_name, source_alias, stellarity, source_ra, source_dec = [ (s['source_name'], s['alias'], s['stellarity'], s['ra'], s['dec']) @@ -730,6 +785,7 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, slitlets.append(Slit(slitlet_id, shutter_id, dither_position, xcen, ycen, ymin, ymax, quadrant, source_id, all_shutters, source_name, source_alias, stellarity, source_xpos, source_ypos, source_ra, source_dec)) + msa_file.close() return slitlets diff --git a/jwst/extract_2d/nirspec.py b/jwst/extract_2d/nirspec.py index 571ad42563..c29ab427c2 100644 --- a/jwst/extract_2d/nirspec.py +++ b/jwst/extract_2d/nirspec.py @@ -165,7 +165,12 @@ def set_slit_attributes(output_model, slit, xlo, xhi, ylo, yhi): output_model.stellarity = float(slit.stellarity) output_model.source_xpos = float(slit.source_xpos) output_model.source_ypos = float(slit.source_ypos) - output_model.slitlet_id = int(slit.name) + try: + output_model.slitlet_id = int(slit.name) + except ValueError: + # Fixed slits in MOS data have string values for the name; + # use the shutter ID instead + output_model.slitlet_id = slit.shutter_id output_model.quadrant = int(slit.quadrant) output_model.xcen = int(slit.xcen) output_model.ycen = int(slit.ycen) From 4459b01e5699a45034564f388c0d02a2afb98cab Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 1 May 2024 17:32:39 -0400 Subject: [PATCH 02/44] Fixed slits via MSA file working through spec2, details TBD --- jwst/assign_wcs/nirspec.py | 5 ++-- jwst/pipeline/calwebb_spec2.py | 48 +++++++++++++++++++++++++++++----- jwst/wavecorr/wavecorr.py | 13 ++++++--- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index 563413afee..516e9057f4 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -664,9 +664,10 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, # quadrant, xcen, ycen, ymin, ymax # First check for a fixed slit + # TODO: check on how to handle 'primary' column shutter_id = None - if all(is_fs) and len(slitlets_sid) == 1 and n_main_shutter == 1: - # One fixed slit open for the source, and it is marked 'primary' + if all(is_fs) and len(slitlets_sid) == 1: # and n_main_shutter == 1: + # One fixed slit open for the source #, and it is marked 'primary' slitlet = slitlets_sid[0] slit_name = slitlet['fixed_slit'] log.debug(f'Found fixed slit {slit_name}') diff --git a/jwst/pipeline/calwebb_spec2.py b/jwst/pipeline/calwebb_spec2.py index aa88fcb87e..d7807f33ee 100644 --- a/jwst/pipeline/calwebb_spec2.py +++ b/jwst/pipeline/calwebb_spec2.py @@ -517,13 +517,49 @@ def _process_nirspec_slits(self, data): calibrated = self.extract_2d(data) calibrated = self.srctype(calibrated) calibrated = self.master_background_mos(calibrated) - calibrated = self.wavecorr(calibrated) - calibrated = self.flat_field(calibrated) - calibrated = self.pathloss(calibrated) - calibrated = self.barshadow(calibrated) - calibrated = self.photom(calibrated) - return calibrated + # Split the datamodel into 2 pieces: one with MOS slits and + # the other with FS slits + calib_mos = datamodels.MultiSlitModel() + calib_fss = datamodels.MultiSlitModel() + primary_fs = None + for slit in calibrated.slits: + if slit.quadrant == 5: + # Take the primary fixed slit from the first + # one encountered + if primary_fs is None: + primary_fs = slit.name + slit.meta.exposure.type = "NRS_FIXEDSLIT" + calib_fss.slits.append(slit) + else: + calib_mos.slits.append(slit) + + # First process MOS slits through all remaining steps + calib_mos.update(calibrated) + if len(calib_mos.slits) > 0: + calib_mos = self.wavecorr(calib_mos) + calib_mos = self.flat_field(calib_mos) + calib_mos = self.pathloss(calib_mos) + calib_mos = self.barshadow(calib_mos) + calib_mos = self.photom(calib_mos) + + # Now repeat for FS slits + if len(calib_fss.slits) > 0: + calib_fss.update(calibrated) + calib_fss.meta.exposure.type = "NRS_FIXEDSLIT" + calib_fss.meta.instrument.fixed_slit = primary_fs + + calib_fss = self.wavecorr(calib_fss) + calib_fss = self.flat_field(calib_fss) + calib_fss = self.pathloss(calib_fss) + calib_fss = self.barshadow(calib_fss) + calib_fss = self.photom(calib_fss) + + # Append the FS results to the MOS results + for slit in calib_fss.slits: + calib_mos.slits.append(slit) + + return calib_mos def _process_niriss_soss(self, data): """Process SOSS diff --git a/jwst/wavecorr/wavecorr.py b/jwst/wavecorr/wavecorr.py index 0174eb1f59..53fe507935 100644 --- a/jwst/wavecorr/wavecorr.py +++ b/jwst/wavecorr/wavecorr.py @@ -118,9 +118,16 @@ def apply_zero_point_correction(slit, reffile): # Get the source position in the slit and set the aperture name if slit.meta.exposure.type in ['NRS_FIXEDSLIT', 'NRS_BRIGHTOBJ']: - # pass lam = 2 microns - # needed for wavecorr with fixed slits - source_xpos = get_source_xpos(slit, slit_wcs, lam=2) + # Check for fixed slits defined via MSA files in + # MOS/FS combination processing: they will have a non-empty + # shutter state and should not have their source position + # overridden + if slit.shutter_state == "": + # pass lam = 2 microns + # needed for wavecorr with fixed slits + source_xpos = get_source_xpos(slit, slit_wcs, lam=2) + else: + source_xpos = slit.source_xpos aperture_name = slit.name else: source_xpos = slit.source_xpos From bf1967839789cd3c8fe5df7f08396438441d8563 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 3 May 2024 13:10:29 -0400 Subject: [PATCH 03/44] Updates for handling fixed slit point sources consistently with and without MSA file source definition --- jwst/extract_1d/extract.py | 7 -- jwst/flatfield/flat_field.py | 11 +-- jwst/master_background/expand_to_2d.py | 3 +- jwst/master_background/nirspec_utils.py | 74 +++++++-------- .../tests/test_nirspec_corrections.py | 2 +- jwst/photom/photom.py | 5 +- jwst/pipeline/calwebb_spec2.py | 50 +++++++++- jwst/wavecorr/tests/test_wavecorr.py | 2 +- jwst/wavecorr/wavecorr.py | 94 +++++++++++-------- 9 files changed, 146 insertions(+), 102 deletions(-) diff --git a/jwst/extract_1d/extract.py b/jwst/extract_1d/extract.py index c9529fb0c6..32442c8f83 100644 --- a/jwst/extract_1d/extract.py +++ b/jwst/extract_1d/extract.py @@ -3736,13 +3736,6 @@ def create_extraction(extract_ref_dict, use_source_posn = False log.info(f"Setting use_source_posn to False for source type {source_type}") - # Turn off use_source_posn if working on non-primary NRS fixed slits - if is_multiple_slits: - if exp_type == 'NRS_FIXEDSLIT' and slitname != slit.meta.instrument.fixed_slit: - use_source_posn = False - log.info("Can only compute source location for primary NIRSpec slit,") - log.info("so setting use_source_posn to False") - if photom_has_been_run: pixel_solid_angle = meta_source.meta.photometry.pixelarea_steradians if pixel_solid_angle is None: diff --git a/jwst/flatfield/flat_field.py b/jwst/flatfield/flat_field.py index 1c4c1db6ea..e93b6190b5 100644 --- a/jwst/flatfield/flat_field.py +++ b/jwst/flatfield/flat_field.py @@ -355,7 +355,6 @@ def nirspec_fs_msa(output_model, f_flat_model, s_flat_model, d_flat_model, dispa """ exposure_type = output_model.meta.exposure.type - primary_slit = output_model.meta.instrument.fixed_slit # Create a list to hold the list of slits. This will eventually be used # to extend the MultiSlitModel.slits attribute. We do it this way to @@ -376,12 +375,12 @@ def nirspec_fs_msa(output_model, f_flat_model, s_flat_model, d_flat_model, dispa if user_supplied_flat is not None: slit_flat = user_supplied_flat.slits[slit_idx] else: - if exposure_type == "NRS_FIXEDSLIT" and \ - slit.name == primary_slit and slit.source_type.upper() == "POINT": + if (exposure_type == "NRS_FIXEDSLIT" + and slit.source_type.upper() == "POINT"): - # For fixed-slit exposures, if this is the primary slit - # and it contains a point source, compute the flat-field - # corrections for both uniform (without wavecorr) and point + # For fixed-slit exposures, if this contains a point source, + # compute the flat-field corrections for both uniform + # (without wavecorr) and point # source (with wavecorr) modes, applying only the point # source version to the data. diff --git a/jwst/master_background/expand_to_2d.py b/jwst/master_background/expand_to_2d.py index 34c5871447..a267ae4a74 100644 --- a/jwst/master_background/expand_to_2d.py +++ b/jwst/master_background/expand_to_2d.py @@ -188,8 +188,7 @@ def bkg_for_multislit(input, tab_wavelength, tab_background): # if the slit contains a point source, in order to make the master bkg # match the calibrated science data in the slit if input.meta.exposure.type == 'NRS_FIXEDSLIT' and slit.source_type.upper() == 'POINT': - primary = True if slit.name == input.meta.instrument.fixed_slit else False - background.slits[k] = correct_nrs_fs_bkg(background.slits[k], primary) + background.slits[k] = correct_nrs_fs_bkg(background.slits[k]) return background diff --git a/jwst/master_background/nirspec_utils.py b/jwst/master_background/nirspec_utils.py index 66ccbcab23..57b6f63a27 100644 --- a/jwst/master_background/nirspec_utils.py +++ b/jwst/master_background/nirspec_utils.py @@ -166,7 +166,7 @@ def correct_nrs_ifu_bkg(input_model): return input_model -def correct_nrs_fs_bkg(input_model, primary_slit): +def correct_nrs_fs_bkg(input_model): """Apply point source vs. uniform source corrections to a NIRSpec Fixed-Slit 2D master background array. @@ -175,9 +175,6 @@ def correct_nrs_fs_bkg(input_model, primary_slit): input_model : `~jwst.datamodels.SlitModel` The input background data. - primary_slit : bool - Is this the primary slit in the exposure? - Returns ------- input_model : `~jwst.datamodels.SlitModel` @@ -201,44 +198,39 @@ def correct_nrs_fs_bkg(input_model, primary_slit): log.warning('Skipping background updates') return input_model - if primary_slit: - # If processing the primary slit, we also need flatfield and - # photom correction arrays - if 'flatfield_point' in input_model.instance: - ff_point = getattr(input_model, 'flatfield_point') - else: - log.warning('flatfield_point array not found in input') - log.warning('Skipping background updates') - return input_model - - if 'flatfield_uniform' in input_model.instance: - ff_uniform = getattr(input_model, 'flatfield_uniform') - else: - log.warning('flatfield_uniform array not found in input') - log.warning('Skipping background updates') - return input_model - - if 'photom_point' in input_model.instance: - ph_point = getattr(input_model, 'photom_point') - else: - log.warning('photom_point array not found in input') - log.warning('Skipping background updates') - return input_model - - if 'photom_uniform' in input_model.instance: - ph_uniform = getattr(input_model, 'photom_uniform') - else: - log.warning('photom_uniform array not found in input') - log.warning('Skipping background updates') - return input_model - - # Apply the corrections for the primary slit - input_model.data *= (pl_uniform / pl_point) * \ - (ff_uniform / ff_point) * \ - (ph_point / ph_uniform) + # If processing the primary slit, we also need flatfield and + # photom correction arrays + if 'flatfield_point' in input_model.instance: + ff_point = getattr(input_model, 'flatfield_point') + else: + log.warning('flatfield_point array not found in input') + log.warning('Skipping background updates') + return input_model + + if 'flatfield_uniform' in input_model.instance: + ff_uniform = getattr(input_model, 'flatfield_uniform') + else: + log.warning('flatfield_uniform array not found in input') + log.warning('Skipping background updates') + return input_model + if 'photom_point' in input_model.instance: + ph_point = getattr(input_model, 'photom_point') else: - # Apply the corrections for secondary slits - input_model.data *= (pl_uniform / pl_point) + log.warning('photom_point array not found in input') + log.warning('Skipping background updates') + return input_model + + if 'photom_uniform' in input_model.instance: + ph_uniform = getattr(input_model, 'photom_uniform') + else: + log.warning('photom_uniform array not found in input') + log.warning('Skipping background updates') + return input_model + + # Apply the corrections for the primary slit + input_model.data *= (pl_uniform / pl_point) * \ + (ff_uniform / ff_point) * \ + (ph_point / ph_uniform) return input_model diff --git a/jwst/master_background/tests/test_nirspec_corrections.py b/jwst/master_background/tests/test_nirspec_corrections.py index af44f2f0c6..ef4171d1a0 100644 --- a/jwst/master_background/tests/test_nirspec_corrections.py +++ b/jwst/master_background/tests/test_nirspec_corrections.py @@ -51,6 +51,6 @@ def test_fs_correction(): photom_point=ph_ps, photom_uniform=ph_un) corrected = input.data * (ff_un / ff_ps) * (pl_un / pl_ps) * (ph_ps / ph_un) - result = correct_nrs_fs_bkg(input, primary_slit=True) + result = correct_nrs_fs_bkg(input) assert np.allclose(corrected, result.data, rtol=1.e-7) diff --git a/jwst/photom/photom.py b/jwst/photom/photom.py index 7f7a6cc4da..c445d9601f 100644 --- a/jwst/photom/photom.py +++ b/jwst/photom/photom.py @@ -848,9 +848,8 @@ def photom_io(self, tabdata, order=None): slit = self.input.slits[self.slitnum] # The NIRSpec fixed-slit primary slit needs special handling if # it contains a point source - if self.exptype.upper() == 'NRS_FIXEDSLIT' and \ - slit.name == self.input.meta.instrument.fixed_slit and \ - slit.source_type.upper() == 'POINT': + if (self.exptype.upper() == 'NRS_FIXEDSLIT' + and slit.source_type.upper() == 'POINT'): # First, compute 2D array of photom correction values using # uncorrected wavelengths, which is appropriate for a uniform source diff --git a/jwst/pipeline/calwebb_spec2.py b/jwst/pipeline/calwebb_spec2.py index d7807f33ee..8ff70f971c 100644 --- a/jwst/pipeline/calwebb_spec2.py +++ b/jwst/pipeline/calwebb_spec2.py @@ -270,6 +270,8 @@ def process_exposure_product( # srctype and wavecorr before flat_field. if exp_type in GRISM_TYPES: calibrated = self._process_grism(calibrated) + elif exp_type == 'NRS_MSASPEC': + calibrated = self._process_nirspec_msa_slits(calibrated) elif exp_type in NRS_SLIT_TYPES: calibrated = self._process_nirspec_slits(calibrated) elif exp_type == 'NIS_SOSS': @@ -350,6 +352,10 @@ def process_exposure_product( x1d = self.photom(x1d) else: self.log.warning("Extract_1d did not return a DataModel - skipping photom.") + elif exp_type == 'NRS_MSASPEC': + # check for fixed slits mixed in with MSA spectra + x1d = resampled.copy() + x1d = self.extract_1d(x1d) else: x1d = resampled.copy() x1d = self.extract_1d(x1d) @@ -509,10 +515,40 @@ def _process_grism(self, data): return calibrated def _process_nirspec_slits(self, data): - """Process NIRSpec + """Process NIRSpec slits. + + This function handles FS, BOTS, and calibration modes. + MOS mode is handled separately, in order to do master + background subtraction and process fixed slits defined in + MSA files. - NIRSpec MOS and FS need srctype and wavecorr before flat_field. - Also have to deal with master background operations. + Note that NIRSpec MOS and FS need srctype and wavecorr before + flat_field. + """ + calibrated = self.extract_2d(data) + calibrated = self.srctype(calibrated) + calibrated = self.master_background_mos(calibrated) + calibrated = self.wavecorr(calibrated) + calibrated = self.flat_field(calibrated) + calibrated = self.pathloss(calibrated) + calibrated = self.barshadow(calibrated) + calibrated = self.photom(calibrated) + calibrated = self.pixel_replace(calibrated) + + return calibrated + + def _process_nirspec_msa_slits(self, data): + """Process NIRSpec MSA slits. + + The NRS_MSASPEC exposure type may contain fixed slit definitions + in addition to standard MSA slitlets. These are handled + separately internally to this function, in order to pull the + correct reference files and perform the right algorithms for + each slit type. Processed slits are recombined into a single + model with EXP_TYPE=NRS_MSASPEC on return. + + Note that NIRSpec MOS and FS need srctype and wavecorr before + flat_field. Also have to deal with master background operations. """ calibrated = self.extract_2d(data) calibrated = self.srctype(calibrated) @@ -559,6 +595,14 @@ def _process_nirspec_slits(self, data): for slit in calib_fss.slits: calib_mos.slits.append(slit) + if len(calib_mos.slits) == len(calib_fss.slits): + # update the MOS model with step completion status from the + # FS model, since there were no MOS slits to run + steps = ['wavecorr', 'flat_field', 'pathloss', 'barshadow', 'photom'] + for step in steps: + setattr(calib_mos.meta.cal_step, step, + getattr(calib_fss.meta.cal_step, step)) + return calib_mos def _process_niriss_soss(self, data): diff --git a/jwst/wavecorr/tests/test_wavecorr.py b/jwst/wavecorr/tests/test_wavecorr.py index e4a06cb625..95eeb1042e 100644 --- a/jwst/wavecorr/tests/test_wavecorr.py +++ b/jwst/wavecorr/tests/test_wavecorr.py @@ -162,7 +162,7 @@ def test_wavecorr_fs(): assert_allclose(result.slits[0].source_xpos, 0.127111, atol=1e-6) slit = result.slits[0] - source_xpos = wavecorr.get_source_xpos(slit, slit.meta.wcs, lam=2) + source_xpos = wavecorr.get_source_xpos(slit) assert_allclose(result.slits[0].source_xpos, source_xpos, atol=1e-6) mean_correction = np.abs(src_result.slits[0].wavelength - result.slits[0].wavelength) diff --git a/jwst/wavecorr/wavecorr.py b/jwst/wavecorr/wavecorr.py index 53fe507935..788914010b 100644 --- a/jwst/wavecorr/wavecorr.py +++ b/jwst/wavecorr/wavecorr.py @@ -60,38 +60,41 @@ def do_correction(input_model, wavecorr_file): output_model = input_model.copy() - # Get the primary slit for a FS exposure - if exp_type == 'NRS_FIXEDSLIT': - primary_slit = input_model.meta.instrument.fixed_slit - if primary_slit is None or primary_slit == 'NONE': - log.warning('Primary slit name not found in input') - log.warning('Skipping wavecorr correction') - input_model.meta.cal_step.wavecorr = 'SKIPPED' - return input_model - # For BRIGHTOBJ, operate on the single SlitModel if isinstance(input_model, datamodels.SlitModel): if _is_point_source(input_model, exp_type): apply_zero_point_correction(output_model, wavecorr_file) else: - # For FS only work on the primary slit + # For FS only work on point source slits with + # position information + corrected = False if exp_type == 'NRS_FIXEDSLIT': + primary_slit = input_model.meta.instrument.fixed_slit for slit in output_model.slits: - if slit.name == primary_slit: - if not hasattr(slit.meta, "dither"): - log.warning('meta.dither is not populated for the primary slit') - log.warning('Skipping wavecorr correction') - input_model.meta.cal_step.wavecorr = 'SKIPPED' - break - if slit.meta.dither.x_offset is None or slit.meta.dither.y_offset is None: - log.warning('dither.x(y)_offset values are None for primary slit') - log.warning('Skipping wavecorr correction') - input_model.meta.cal_step.wavecorr = 'SKIPPED' - break - if _is_point_source(slit, exp_type): - apply_zero_point_correction(slit, wavecorr_file) - output_model.meta.cal_step.wavecorr = 'COMPLETE' - break + if _is_point_source(slit, exp_type): + # If fixed slit was not defined via MSA file, + # it must have dither information to find the + # source position, and it must be the primary slit + if not _is_msa_fixed_slit(slit): + if slit.name != primary_slit: + log.warning(f'Skipping wavecorr correction for ' + f'non-primary slit {slit.name}') + continue + if not hasattr(slit.meta, "dither"): + log.warning('meta.dither is not populated for the primary slit') + log.warning('Skipping wavecorr correction') + continue + if slit.meta.dither.x_offset is None or slit.meta.dither.y_offset is None: + log.warning('dither.x(y)_offset values are None for primary slit') + log.warning('Skipping wavecorr correction') + input_model.meta.cal_step.wavecorr = 'SKIPPED' + continue + apply_zero_point_correction(slit, wavecorr_file) + corrected = True + if corrected: + output_model.meta.cal_step.wavecorr = 'COMPLETE' + else: + output_model.meta.cal_step.wavecorr = 'SKIPPED' # For MOS work on all slits containing a point source else: @@ -119,15 +122,14 @@ def apply_zero_point_correction(slit, reffile): # Get the source position in the slit and set the aperture name if slit.meta.exposure.type in ['NRS_FIXEDSLIT', 'NRS_BRIGHTOBJ']: # Check for fixed slits defined via MSA files in - # MOS/FS combination processing: they will have a non-empty - # shutter state and should not have their source position - # overridden - if slit.shutter_state == "": - # pass lam = 2 microns - # needed for wavecorr with fixed slits - source_xpos = get_source_xpos(slit, slit_wcs, lam=2) - else: + # MOS/FS combination processing: they should not have their + # source position overridden by dither keywords + if _is_msa_fixed_slit(slit): + # get the planned source position source_xpos = slit.source_xpos + else: + # get the source position from the dither offsets + source_xpos = get_source_xpos(slit) aperture_name = slit.name else: source_xpos = slit.source_xpos @@ -225,6 +227,26 @@ def compute_dispersion(wcs, xpix=None, ypix=None): return (lamright - lamleft) * 10 ** -6 +def _is_msa_fixed_slit(slit): + """ + Determine if a fixed slit source was defined via a MSA file. + + Parameters + ---------- + slit : `~stdatamodels.jwst.transforms.models.Slit` + A slit object. + """ + # Fixed slits defined via MSA files in MOS/FS combination + # processing will have a non-empty shutter state + # TODO: determine if there's a better test + if (not hasattr(slit, 'shutter_state') + or slit.shutter_state is None + or slit.shutter_state == ""): + return False + else: + return True + + def _is_point_source(slit, exp_type): """ Determine if a source is a point source. @@ -259,7 +281,7 @@ def _is_point_source(slit, exp_type): return result -def get_source_xpos(slit, slit_wcs, lam): +def get_source_xpos(slit): """ Compute the source position within the slit for a NIRSpec fixed slit. @@ -267,10 +289,6 @@ def get_source_xpos(slit, slit_wcs, lam): ---------- slit : `~jwst.datamodels.SlitModel` The slit object. - slit_wcs : `~gwcs.wcs.WCS` - The WCS object for this slit. - lam : float - Wavelength in microns. Returns ------- From 7f75f2b7797b4c76c2588f377615da899d3377c9 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 3 May 2024 14:54:26 -0400 Subject: [PATCH 04/44] Separate spec2 extraction by mode --- jwst/pipeline/calwebb_spec2.py | 42 +++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/jwst/pipeline/calwebb_spec2.py b/jwst/pipeline/calwebb_spec2.py index 8ff70f971c..02be8def46 100644 --- a/jwst/pipeline/calwebb_spec2.py +++ b/jwst/pipeline/calwebb_spec2.py @@ -353,9 +353,45 @@ def process_exposure_product( else: self.log.warning("Extract_1d did not return a DataModel - skipping photom.") elif exp_type == 'NRS_MSASPEC': - # check for fixed slits mixed in with MSA spectra - x1d = resampled.copy() - x1d = self.extract_1d(x1d) + # Check for fixed slits mixed in with MSA spectra + resamp_mos = datamodels.MultiSlitModel() + resamp_fss = datamodels.MultiSlitModel() + for slit in resampled.slits: + # Quadrant information is not preserved through resampling, + # but MSA slits have numbers for names, so use that to + # distinguish MSA from FS + try: + msa_name = int(slit.name) + except ValueError: + msa_name = None + if msa_name is None: + slit.meta.exposure.type = "NRS_FIXEDSLIT" + resamp_fss.slits.append(slit) + else: + slit.meta.exposure.type = "NRS_MSASPEC" + resamp_mos.slits.append(slit) + resamp_mos.update(resampled) + resamp_fss.update(resampled) + + # Extract the MOS slits + x1d = None + if len(resamp_mos.slits) > 0: + self.log.info(f'Extracting {len(resamp_mos.slits)} MSA slitlets') + x1d = self.extract_1d(resamp_mos) + + # Extract the FS slits + if len(resamp_fss.slits) > 0: + self.log.info(f'Extracting {len(resamp_fss.slits)} fixed slits') + resamp_fss.meta.exposure.type = "NRS_FIXEDSLIT" + x1d_fss = self.extract_1d(resamp_fss) + if x1d is None: + x1d = x1d_fss + x1d.meta.exposure.type = "NRS_MSASPEC" + else: + for spec in x1d_fss.spec: + x1d.spec.append(spec) + resamp_mos.close() + resamp_fss.close() else: x1d = resampled.copy() x1d = self.extract_1d(x1d) From 5983fd939d438ef1bd1060c92e2d1acc6bd75c7d Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 6 May 2024 15:22:40 -0400 Subject: [PATCH 05/44] Allow FS extraction as MOS in spec2 --- jwst/pipeline/calwebb_spec2.py | 40 ---------------------------------- 1 file changed, 40 deletions(-) diff --git a/jwst/pipeline/calwebb_spec2.py b/jwst/pipeline/calwebb_spec2.py index 02be8def46..b250258f89 100644 --- a/jwst/pipeline/calwebb_spec2.py +++ b/jwst/pipeline/calwebb_spec2.py @@ -352,46 +352,6 @@ def process_exposure_product( x1d = self.photom(x1d) else: self.log.warning("Extract_1d did not return a DataModel - skipping photom.") - elif exp_type == 'NRS_MSASPEC': - # Check for fixed slits mixed in with MSA spectra - resamp_mos = datamodels.MultiSlitModel() - resamp_fss = datamodels.MultiSlitModel() - for slit in resampled.slits: - # Quadrant information is not preserved through resampling, - # but MSA slits have numbers for names, so use that to - # distinguish MSA from FS - try: - msa_name = int(slit.name) - except ValueError: - msa_name = None - if msa_name is None: - slit.meta.exposure.type = "NRS_FIXEDSLIT" - resamp_fss.slits.append(slit) - else: - slit.meta.exposure.type = "NRS_MSASPEC" - resamp_mos.slits.append(slit) - resamp_mos.update(resampled) - resamp_fss.update(resampled) - - # Extract the MOS slits - x1d = None - if len(resamp_mos.slits) > 0: - self.log.info(f'Extracting {len(resamp_mos.slits)} MSA slitlets') - x1d = self.extract_1d(resamp_mos) - - # Extract the FS slits - if len(resamp_fss.slits) > 0: - self.log.info(f'Extracting {len(resamp_fss.slits)} fixed slits') - resamp_fss.meta.exposure.type = "NRS_FIXEDSLIT" - x1d_fss = self.extract_1d(resamp_fss) - if x1d is None: - x1d = x1d_fss - x1d.meta.exposure.type = "NRS_MSASPEC" - else: - for spec in x1d_fss.spec: - x1d.spec.append(spec) - resamp_mos.close() - resamp_fss.close() else: x1d = resampled.copy() x1d = self.extract_1d(x1d) From 489914172cc2bbacba027f509d4425c5c8829860 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 6 May 2024 15:23:00 -0400 Subject: [PATCH 06/44] Assign exp type by slit for spec3 --- jwst/exp_to_source/exp_to_source.py | 22 +++++++++++++++++----- jwst/pipeline/calwebb_spec3.py | 2 +- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/jwst/exp_to_source/exp_to_source.py b/jwst/exp_to_source/exp_to_source.py index b9540b1b16..8f3d3a4f39 100644 --- a/jwst/exp_to_source/exp_to_source.py +++ b/jwst/exp_to_source/exp_to_source.py @@ -40,6 +40,7 @@ def exp_to_source(inputs): log.debug(f'Copying source {slit.source_id}') result_slit = result[str(slit.source_id)] result_slit.exposures.append(slit) + # store values for later use (after merge_tree) # these values are incorrectly getting overwritten by # the top model. @@ -47,12 +48,14 @@ def exp_to_source(inputs): slit_bunit_err = slit.meta.bunit_err slit_model = slit.meta.model_type slit_wcsinfo = slit.meta.wcsinfo.instance - # exposure.meta.bunit_data and bunit_err does not exist - # before calling merge_tree save these values + slit_exptype = None + if hasattr(slit.meta, 'exposure'): + if hasattr(slit.meta.exposure, 'type'): + slit_exptype = slit.meta.exposure.type + # Before merge_tree the slits have a model_type of SlitModel. # After merge_tree it is overwritten with MultiSlitModel. # store the model type to undo overwriting of modeltype. - merge_tree(result_slit.exposures[-1].meta.instance, exposure.meta.instance) result_slit.exposures[-1].meta.bunit_data = slit_bunit @@ -60,8 +63,17 @@ def exp_to_source(inputs): result_slit.exposures[-1].meta.model_type = slit_model result_slit.exposures[-1].meta.wcsinfo = slit_wcsinfo - if result_slit.meta.instrument.name is None: - result_slit.update(exposure) + # make sure top-level exposure type matches slit exposure type + # (necessary for NIRSpec fixed slits defined as part of an MSA file) + if slit_exptype is not None: + result_slit.exposures[-1].meta.exposure.type = slit_exptype + + if result_slit.meta.instrument.name is None: + result_slit.update(exposure) + + result_slit.meta.exposure.type = slit_exptype + log.debug(f'Input exposure type: {exposure.meta.exposure.type}') + log.debug(f'Output exposure type: {result_slit.meta.exposure.type}') result_slit.meta.filename = None # Resulting merged data doesn't come from one file diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index 429f43da4a..b539c3f38b 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -223,7 +223,7 @@ def process(self, input): # also the slit name (for NIRSpec fixed-slit only). if isinstance(source, tuple): source_id, result = source - if result[0].meta.exposure.type == "NRS_FIXEDSLIT": + if exptype == "NRS_FIXEDSLIT": slit_name = self._create_nrsfs_slit_name(result) self.output_file = format_product( output_file, source_id=source_id.lower(), slit_name=slit_name) From 786082633abd61786759db4feef14e89c0d7232c Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 6 May 2024 16:44:45 -0400 Subject: [PATCH 07/44] Ignore non-primary fixed slits --- jwst/assign_wcs/nirspec.py | 44 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index 516e9057f4..fca1bb6b7d 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -652,25 +652,27 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, # How many shutters in the slitlet are labeled as "main" or "primary"? n_main_shutter = len([s for s in slitlets_sid if s['primary_source'] == 'Y']) - # Check for fixed slit - try: - # TODO: confirm default value for true MSA slitlets - is_fs = [x['fixed_slit'] != 'NONE' for x in slitlets_sid] - except (IndexError, ValueError, KeyError): - # May be old-style MSA file without a fixed_slit column - is_fs = [False] * len(slitlets_sid) + # Check for fixed slit sources defined in the MSA file + is_fs = [False] * len(slitlets_sid) + for i, slitlet in enumerate(slitlets_sid): + try: + if (slitlet['fixed_slit'] in FIXED_SLIT_NUMS.keys() + and slitlet['fixed_slit'] != 'NONE'): + is_fs[i] = True + except (IndexError, ValueError, KeyError): + # May be old-style MSA file without a fixed_slit column + pass # In the next part we need to calculate, find, determine 5 things: # quadrant, xcen, ycen, ymin, ymax # First check for a fixed slit - # TODO: check on how to handle 'primary' column shutter_id = None - if all(is_fs) and len(slitlets_sid) == 1: # and n_main_shutter == 1: - # One fixed slit open for the source #, and it is marked 'primary' + if all(is_fs) and len(slitlets_sid) == 1 and n_main_shutter == 1: + # One fixed slit open for the source and it is marked 'primary' slitlet = slitlets_sid[0] slit_name = slitlet['fixed_slit'] - log.debug(f'Found fixed slit {slit_name}') + log.debug(f'Found fixed slit {slit_name} with primary target') # use standard number for fixed slit shutter id slitlet_id = slit_name @@ -683,22 +685,20 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, ymax = yhigh # source position and id - # TODO: check if source position is offset by 0.5 source_id = slitlet['source_id'] source_xpos = slitlet['estimated_source_in_shutter_x'] source_ypos = slitlet['estimated_source_in_shutter_y'] elif any(is_fs): - # Unsupported fixed slit configuration - message = ("For slitlet_id = {}, metadata_id = {}, " - "dither_index = {}".format( - slitlet_id, msa_metadata_id, dither_position)) - log.warning(message) - message = ("MSA configuration file has an unsupported " - "fixed slit configuration") - log.warning(message) - msa_file.close() - raise MSAFileError(message) + # Ignore any fixed slit configuration not recognized + # as a primary source + message = (f"For slitlet_id = {slitlet_id}, " + f"metadata_id = {msa_metadata_id}, " + f"dither_index = {dither_position}, " + f"ignoring non-primary fixed slit " + f"{slitlets_sid[0]['fixed_slit']}") + log.debug(message) + continue # Now check for regular MSA slitlets elif n_main_shutter == 0: From 119c2142852d55f0cc481db2115eddbbe501f8f3 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 6 May 2024 18:15:11 -0400 Subject: [PATCH 08/44] Allow non-primary FS as background slits --- jwst/assign_wcs/nirspec.py | 39 ++++++++++++++++++++++------------ jwst/pipeline/calwebb_spec2.py | 8 +------ 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index fca1bb6b7d..8507f90d7b 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -668,8 +668,8 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, # First check for a fixed slit shutter_id = None - if all(is_fs) and len(slitlets_sid) == 1 and n_main_shutter == 1: - # One fixed slit open for the source and it is marked 'primary' + if all(is_fs) and len(slitlets_sid) == 1: + # One fixed slit open for the source slitlet = slitlets_sid[0] slit_name = slitlet['fixed_slit'] log.debug(f'Found fixed slit {slit_name} with primary target') @@ -685,20 +685,31 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, ymax = yhigh # source position and id - source_id = slitlet['source_id'] - source_xpos = slitlet['estimated_source_in_shutter_x'] - source_ypos = slitlet['estimated_source_in_shutter_y'] + if n_main_shutter == 1: + # source is marked primary + source_id = slitlet['source_id'] + source_xpos = slitlet['estimated_source_in_shutter_x'] + source_ypos = slitlet['estimated_source_in_shutter_y'] + else: + # source is background only + source_id = _get_bkg_source_id(bkg_counter, max_source_id) + source_xpos = 0.5 + source_ypos = 0.5 + log.info(f'Slitlet_id {slitlet_id} is background only; ' + f'assigned source_id = {source_id}') + bkg_counter += 1 elif any(is_fs): - # Ignore any fixed slit configuration not recognized - # as a primary source - message = (f"For slitlet_id = {slitlet_id}, " - f"metadata_id = {msa_metadata_id}, " - f"dither_index = {dither_position}, " - f"ignoring non-primary fixed slit " - f"{slitlets_sid[0]['fixed_slit']}") - log.debug(message) - continue + # Unsupported fixed slit configuration + message = ("For slitlet_id = {}, metadata_id = {}, " + "dither_index = {}".format( + slitlet_id, msa_metadata_id, dither_position)) + log.warning(message) + message = ("MSA configuration file has an unsupported " + "fixed slit configuration") + log.warning(message) + msa_file.close() + raise MSAFileError(message) # Now check for regular MSA slitlets elif n_main_shutter == 0: diff --git a/jwst/pipeline/calwebb_spec2.py b/jwst/pipeline/calwebb_spec2.py index b250258f89..a71bffbf20 100644 --- a/jwst/pipeline/calwebb_spec2.py +++ b/jwst/pipeline/calwebb_spec2.py @@ -548,19 +548,13 @@ def _process_nirspec_msa_slits(self, data): """ calibrated = self.extract_2d(data) calibrated = self.srctype(calibrated) - calibrated = self.master_background_mos(calibrated) # Split the datamodel into 2 pieces: one with MOS slits and # the other with FS slits calib_mos = datamodels.MultiSlitModel() calib_fss = datamodels.MultiSlitModel() - primary_fs = None for slit in calibrated.slits: if slit.quadrant == 5: - # Take the primary fixed slit from the first - # one encountered - if primary_fs is None: - primary_fs = slit.name slit.meta.exposure.type = "NRS_FIXEDSLIT" calib_fss.slits.append(slit) else: @@ -569,6 +563,7 @@ def _process_nirspec_msa_slits(self, data): # First process MOS slits through all remaining steps calib_mos.update(calibrated) if len(calib_mos.slits) > 0: + calib_mos = self.master_background_mos(calib_mos) calib_mos = self.wavecorr(calib_mos) calib_mos = self.flat_field(calib_mos) calib_mos = self.pathloss(calib_mos) @@ -579,7 +574,6 @@ def _process_nirspec_msa_slits(self, data): if len(calib_fss.slits) > 0: calib_fss.update(calibrated) calib_fss.meta.exposure.type = "NRS_FIXEDSLIT" - calib_fss.meta.instrument.fixed_slit = primary_fs calib_fss = self.wavecorr(calib_fss) calib_fss = self.flat_field(calib_fss) From ad3e39ebf637048b9972cc6308a8cfd01a71f80c Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 7 May 2024 12:37:30 -0400 Subject: [PATCH 09/44] Revert changes made in other branches --- jwst/assign_wcs/nirspec.py | 5 ++--- jwst/assign_wcs/tests/test_nirspec.py | 2 +- jwst/flatfield/flat_field.py | 7 +------ 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index 8507f90d7b..19f14c10d5 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -725,9 +725,8 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, quadrant = slitlets_sid[0]['shutter_quadrant'] ycen = j xcen = slitlets_sid[0]['shutter_row'] # grab the first as they are all the same - - source_xpos = 0.5 - source_ypos = 0.5 + source_xpos = 0.0 + source_ypos = 0.0 source_id = _get_bkg_source_id(bkg_counter, max_source_id) log.info(f'Slitlet_id {slitlet_id} is background only; assigned source_id = {source_id}') bkg_counter += 1 diff --git a/jwst/assign_wcs/tests/test_nirspec.py b/jwst/assign_wcs/tests/test_nirspec.py index a517916835..7fdbaffd1e 100644 --- a/jwst/assign_wcs/tests/test_nirspec.py +++ b/jwst/assign_wcs/tests/test_nirspec.py @@ -311,7 +311,7 @@ def test_msa_configuration_all_background(): slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) ref_slit = trmodels.Slit(57, 8281, 1, 251, 23, -2.15, 2.15, 4, 0, '1x1', 'background_57', 'bkg_57', - 0, 0.0, 0.0) + 0, -0.5, -0.5) _compare_slits(slitlet_info[0], ref_slit) diff --git a/jwst/flatfield/flat_field.py b/jwst/flatfield/flat_field.py index e93b6190b5..a64b78f442 100644 --- a/jwst/flatfield/flat_field.py +++ b/jwst/flatfield/flat_field.py @@ -441,12 +441,7 @@ def nirspec_fs_msa(output_model, f_flat_model, s_flat_model, d_flat_model, dispa # Make sure all DO_NOT_USE pixels are set to NaN, # including those flagged by this step - dnu = np.where(slit.dq & dqflags.pixel['DO_NOT_USE']) - slit.data[dnu] = np.nan - slit.err[dnu] = np.nan - slit.var_poisson[dnu] = np.nan - slit.var_rnoise[dnu] = np.nan - slit.var_flat[dnu] = np.nan + slit.data[np.where(slit.dq & dqflags.pixel['DO_NOT_USE'])] = np.nan any_updated = True From c268aa732eda4e9d508c9f0605d114d1d863b865 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 7 May 2024 13:22:04 -0400 Subject: [PATCH 10/44] Code clean up --- jwst/wavecorr/wavecorr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jwst/wavecorr/wavecorr.py b/jwst/wavecorr/wavecorr.py index 788914010b..6800663f43 100644 --- a/jwst/wavecorr/wavecorr.py +++ b/jwst/wavecorr/wavecorr.py @@ -117,7 +117,6 @@ def apply_zero_point_correction(slit, reffile): The ``wavecorr`` reference file. """ log.info(f'slit name {slit.name}') - slit_wcs = slit.meta.wcs # Get the source position in the slit and set the aperture name if slit.meta.exposure.type in ['NRS_FIXEDSLIT', 'NRS_BRIGHTOBJ']: From acb114a8dd9345d69e88e7313b593dce2d42278a Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 8 May 2024 13:56:37 -0400 Subject: [PATCH 11/44] Enable saving for intermediate FS/MOS results; separate extraction for FS/MOS --- jwst/pipeline/calwebb_spec2.py | 82 +++++++++++++++++++++++++++++++--- 1 file changed, 75 insertions(+), 7 deletions(-) diff --git a/jwst/pipeline/calwebb_spec2.py b/jwst/pipeline/calwebb_spec2.py index a71bffbf20..8c10929b15 100644 --- a/jwst/pipeline/calwebb_spec2.py +++ b/jwst/pipeline/calwebb_spec2.py @@ -352,6 +352,13 @@ def process_exposure_product( x1d = self.photom(x1d) else: self.log.warning("Extract_1d did not return a DataModel - skipping photom.") + elif exp_type == 'NRS_MSASPEC': + # Special handling for MSA spectra, to handle mixed-in + # fixed slits separately + if not self.extract_1d.skip: + x1d = self._extract_nirspec_msa_slits(resampled) + else: + x1d = resampled.copy() else: x1d = resampled.copy() x1d = self.extract_1d(x1d) @@ -575,11 +582,20 @@ def _process_nirspec_msa_slits(self, data): calib_fss.update(calibrated) calib_fss.meta.exposure.type = "NRS_FIXEDSLIT" - calib_fss = self.wavecorr(calib_fss) - calib_fss = self.flat_field(calib_fss) - calib_fss = self.pathloss(calib_fss) - calib_fss = self.barshadow(calib_fss) - calib_fss = self.photom(calib_fss) + # Run each step with an alternate suffix, + # to avoid overwriting previous products if save_results=True + fs_steps = ['wavecorr', 'flat_field', 'pathloss', 'barshadow', 'photom'] + for step_name in fs_steps: + # Set suffix + step = getattr(self, step_name) + current_suffix = step.suffix + step.suffix = f'{current_suffix}_fs' + + # Run step + calib_fss = step(calib_fss) + + # Reset suffix + step.suffix = current_suffix # Append the FS results to the MOS results for slit in calib_fss.slits: @@ -588,8 +604,7 @@ def _process_nirspec_msa_slits(self, data): if len(calib_mos.slits) == len(calib_fss.slits): # update the MOS model with step completion status from the # FS model, since there were no MOS slits to run - steps = ['wavecorr', 'flat_field', 'pathloss', 'barshadow', 'photom'] - for step in steps: + for step in fs_steps: setattr(calib_mos.meta.cal_step, step, getattr(calib_fss.meta.cal_step, step)) @@ -622,3 +637,56 @@ def _process_common(self, data): calibrated = self.residual_fringe(calibrated) # only run on MIRI_MRS data return calibrated + + def _extract_nirspec_msa_slits(self, resampled): + """Extract NIRSpec MSA slits with separate handling for FS slits.""" + + # Check for fixed slits mixed in with MSA spectra: + # they need separate reference files + resamp_mos = datamodels.MultiSlitModel() + resamp_fss = datamodels.MultiSlitModel() + for slit in resampled.slits: + # Quadrant information is not preserved through resampling, + # but MSA slits have numbers for names, so use that to + # distinguish MSA from FS + try: + msa_name = int(slit.name) + except ValueError: + msa_name = None + if msa_name is None: + slit.meta.exposure.type = "NRS_FIXEDSLIT" + resamp_fss.slits.append(slit) + else: + slit.meta.exposure.type = "NRS_MSASPEC" + resamp_mos.slits.append(slit) + resamp_mos.update(resampled) + resamp_fss.update(resampled) + + # Extract the MOS slits + x1d = None + save_x1d = self.extract_1d.save_results + self.extract_1d.save_results = False + if len(resamp_mos.slits) > 0: + self.log.info(f'Extracting {len(resamp_mos.slits)} MSA slitlets') + x1d = self.extract_1d(resamp_mos) + + # Extract the FS slits + if len(resamp_fss.slits) > 0: + self.log.info(f'Extracting {len(resamp_fss.slits)} fixed slits') + resamp_fss.meta.exposure.type = "NRS_FIXEDSLIT" + x1d_fss = self.extract_1d(resamp_fss) + if x1d is None: + x1d = x1d_fss + x1d.meta.exposure.type = "NRS_MSASPEC" + else: + for spec in x1d_fss.spec: + x1d.spec.append(spec) + + # save the composite model + if save_x1d: + self.save_model(x1d, suffix='x1d') + + resamp_mos.close() + resamp_fss.close() + + return x1d From 6314e5dc04bedf38a3afcbda7d1148bc954b5ed8 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 8 May 2024 14:14:39 -0400 Subject: [PATCH 12/44] Add pixel_replace to mos/fs processing --- jwst/pipeline/calwebb_spec2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jwst/pipeline/calwebb_spec2.py b/jwst/pipeline/calwebb_spec2.py index 8c10929b15..639b8ab5c8 100644 --- a/jwst/pipeline/calwebb_spec2.py +++ b/jwst/pipeline/calwebb_spec2.py @@ -576,6 +576,7 @@ def _process_nirspec_msa_slits(self, data): calib_mos = self.pathloss(calib_mos) calib_mos = self.barshadow(calib_mos) calib_mos = self.photom(calib_mos) + calib_mos = self.pixel_replace(calib_mos) # Now repeat for FS slits if len(calib_fss.slits) > 0: @@ -584,7 +585,8 @@ def _process_nirspec_msa_slits(self, data): # Run each step with an alternate suffix, # to avoid overwriting previous products if save_results=True - fs_steps = ['wavecorr', 'flat_field', 'pathloss', 'barshadow', 'photom'] + fs_steps = ['wavecorr', 'flat_field', 'pathloss', 'barshadow', + 'photom', 'pixel_replace'] for step_name in fs_steps: # Set suffix step = getattr(self, step_name) From 93dc171826eefd754de36c776d937228d565605a Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 8 May 2024 15:11:15 -0400 Subject: [PATCH 13/44] Re-fix MOS background slit location --- jwst/assign_wcs/nirspec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index 19f14c10d5..7a16f2a6fc 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -725,8 +725,8 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, quadrant = slitlets_sid[0]['shutter_quadrant'] ycen = j xcen = slitlets_sid[0]['shutter_row'] # grab the first as they are all the same - source_xpos = 0.0 - source_ypos = 0.0 + source_xpos = 0.5 + source_ypos = 0.5 source_id = _get_bkg_source_id(bkg_counter, max_source_id) log.info(f'Slitlet_id {slitlet_id} is background only; assigned source_id = {source_id}') bkg_counter += 1 From 3a9eccd05e942103a80f7263672960293e521651 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 16 May 2024 16:21:11 -0400 Subject: [PATCH 14/44] Add tests for MOS/FS WCS assignment --- .../tests/data/msa_fs_configuration.fits | Bin 0 -> 2016000 bytes jwst/assign_wcs/tests/test_nirspec.py | 54 +++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 jwst/assign_wcs/tests/data/msa_fs_configuration.fits diff --git a/jwst/assign_wcs/tests/data/msa_fs_configuration.fits b/jwst/assign_wcs/tests/data/msa_fs_configuration.fits new file mode 100644 index 0000000000000000000000000000000000000000..dd4c3fc6f2f16f2fd7adc260c8e3fbac2e031c62 GIT binary patch literal 2016000 zcmeI*O;cN0dH`T`PtQzMDqpM2Zon@}>yv=R`s zBzoGbEV9ccl}-M@)GSl^3q8LgyIE#URhHT0>K+cDVZ>#2 zeVoUux7+o%*;c(<&$@5-8-t^q+`6*-s=A(c2i8~4-kTV?x!Y)UntQFz zNsv?5Dz9pll_zJ!tDPD+=GD~~)o0a}Uc5W&we`(v{;)fz9(p%xwmRKr_n@0sl(ie3 z{k*W|>zziHqh_1!M!nl?Z3oyl&HUzC<@3t=YUKx?ER;Vf-#=YyTFK!4)9UAymF1$ofBfac`{l*^ z>=Jhzemx7(@z^i8wdxzpRs_1#8qBNs@6Llw8Kt*t+K^hM=GKHuW%;>XE+ziG64 zdq*!;KA=E*UGE%ez8iS$#p<&c%g=k;x2P^3ae9(>v;FqocDueC_rrH1^xyJDb-mi# zs(HM8mc@Pj@xEzxn%!}uW!dVZ^~#GcYAZ#&#R2j5UUwSpZ}QzNpYzdkrN5&7UQyph z^+}v>*c!(r-pxXN&6B;R|M6*WZKd{XW4-cs1E0SS^9{Sd*Ub07{5jBRe0|VpZ8nBi z`Fwr(@zZB><$>|$@_5C@oGWLoX1lZB*zD~@8};r+O!L!TzjsV)Zf`X6V_1l{u-Nkp znlX)?L3g-~009C7W=f!UI=Nn5?Y;h1pD#b_of{p$SmxL7 zX4T#Lb|Y(i+YM)>qb5J({P`z~_wsm?Jb#|gKWcXG-e1Ja zA2%T0gN5?D<44`H_IUN>N`8VhAYPXBzhBs|=jUyWZhk&EB zv3hCp_aD8}-QM}%`p;k1YLyoo#eQ?N)7(9^rxd5WLrN>|a=tN2$o&Ke5FkK+z_baJ zTnG>#K!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1g=$J=`f^E+iNXKasmVh5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+zy$=B4#R`f_5vY2lmGz&1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5STK7rNdBA+D=)S${|32009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pkH$ou&e!tki5ydN#009C72oNAZfB*pk1PBlyK!5-N0+%Qd*Gtp(5>pCI zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfWRdSq@M?h{Yy?OJOKg( z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009CwP$2$&leE2oQ&BPj0t5&UAV7cs0RjXHf%N@N?BgLofB*pk1PBlyK!5-N0t5&U zAV7csfvXjW>$7QlwN=PQfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAP@!8-}8w5c)yka0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+z%&ai z9fs=CcA9He8vz0Y2oNAZfWXBC((}LAzj$O32oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+0D-F#SUL=u()OzI%R_(w0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0ykSA{eAV=zuB`= zJplp)2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs zfyoFg9fmk*JDKRB5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5(LmO%Rb(y@P8`NSnafB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5;& zs}YFbFPFAglU)`91PBlyK!5-N0t5&UAV7cs0RjXF5FkK+0D;>skp7%1_HVn3`4b>O zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBnAW`X#Gvwc{`!M2oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&Um}Y_W?+nNOG}o*)0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&Um_32``L(p2y_uR30RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!Cty3Z&n69s8G=Q)~hR2oNAZfB*pk1PDyCKzv`6w$og*+6WLJK!5-N z0t5&Um`;K8eMRg~XSJ##K!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t9ZdK>VCy+TP?jsht1;0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBnAjKI=ih?BOHi7pBO0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PEM~K>Bmu*uSiN;u0W0fB*pk1PBlyK!5-N0tBW_Ag+g`?X=aY z9s&di5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB=E%7D&I}GWMstYSj_A z9)bA$NZaeFMpgm@2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zNCndGkB$9woIU{p1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZAQgz8Gf3NX zoIU{p1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C7W>p~lee2kt)%lti0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PIKQK>S|aw4JS4nhgO01PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAaEH1>F*=N{$=D6lK=q%1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+0Dor!MiC%DfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfWWi~EFFf5(stVFR1X0H z1PBlyKw#1W>G@CWPdbly1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7rb%GwFjSMa(^R8c2oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB=E35m-75dD8Z3vdcn%009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXFT&6(! z_XuMDGINSefB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAJ`vUQMEYf!NXKscB2oNAZfB*pk1PBly zK!5-N0v8cT*OOxZA~8Ld009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pkvm_8dXPvgQG)HqGK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1ZGPh{hUMW&(h5!Kq1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkKcS_GC3LnUcDE%m8`009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB=E36G%T-9Q#*SflLGl z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAJ34!?c=+br)vBe-jfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5Ex$|{d?1~KYj!w2@oJafB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjYWvq1d))U>_LCgx6n009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZV3a`m`PMmHC+m0RjXF z5FkK+!0i-B&u?S@c50Y40RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FjuO0`dEh(smkZ zQv(441PBlyK!5;&X%$G%pJV^B>y3HR_OkO%Dt=mb>`yAbI0Og~AV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7csf!ih!zt1UcZ<~Yp5+Fc;009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1ST!8bQmJ0?WFUFM}PnU0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7csfmsn)It(*O+gX{Pc@Q8#fB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RopVuyh!5r0wNrmVp2P0t5(LpFsM$7yH*&k<0`L z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7rbi%tzBX;A zr$Uw7WPx8$y^m*k@c5=C!y>7PZ?YCLC{(7g8HNNdOTAk)z>tsZ7?NPP1R(+La zpC-2#-H)@oS-st^zsFSH;Azt<90T+mtWkr3(_Pw*) zsc+=c_1?aF?jno&LcGbP?sUF7=;r&u#@7e+t#-ZDJ=eU8`a-I?BESKnf&Z||VJ**N!}7oH78eIefD>U&Vs z_qx9MPrYMO>%!095N~cdlzh%DaWvoF@#W*b?c3?ryN&Y(E&M{f#fjDTu&A%m={9%s z(zZ4_M_anJ5uelFj$3b0Ux-%;0@=6^JsRHMgyVC$cYGOE-`jDM6!py&@g`W`T(7>Z zX7{W7@VCF$Y~?Q^t*yq`Xs4ze>MP<+?)fn{->YxGop0Xu+oP>JF4(EsdRIezMZC#9 zKk^suyuLThZ+rV*{wjRV+D_^#;yswe^P}>rwzB-Z(yMR%=Pzru%8QNa%G1?Y;oVcG zq^{tKrr}?AOK|dZI>2JqL7Y2<;7Y2<;w>(a|Fla=&Fla=&N8_Xm zgGQtagGQu#JWjeWXhga&XhgavB689>B689 z>7I>~E({uxE({u)?m_+>9{DtngYtAo$AXim`SGzJ4jL=nA0I~=a8!wr{$$uu<>S)t z#lnvcJtvWl4?S_vP^UO(XyI|t(8A-Op@lyz{r*wmrLzyLanO%SanJzqIOxa227OZc zgXna2x;SX)29ASrr?U_CanOm=fWqUT-m66VvT?jW#X)~IY|zh2e^~nHgVOLF9yVwzKlFsl_hL=|`R`wS_OIXlOTNX|=gZ~gxl*ZdJm{mis~itnDF)4#N^gz_ zeSg59dm+;H@t_|J7&Nz7jQZ+$&<_U;%4>R**W^NAlz{c;<2N63^Y_nKFFxn`b9UBx z_KdDSXAdv5{=DDb+s@b0FSuTEvi|&wqb(cXs~&IY_qlRv{W-1*U1WJJtRIB+=gHM~ zwEi4ct$J_6&p+j2eIvx1FNXz$^9ReJ3%{^lT;FNdJNZo)i`Uz}A>P8#0?ftY_0}iD zdhurdbA_F~?Tan&7Jeb#Val0ON+>6b(=NIBlZv8K;7jHE-ivllRUx+uk`oeng ze!Gz`0B&Dw1;5z7A>P8oo*!YoxYKRy?9|)M?%0dDVSniRx2P|~iwkWR+8(FZ|5jhV zcwFgQ{>zW|CqI(Dmi52B$?FifVu8{JoiFoq><=E^FE8HT=zr5SKR2Jh={xf>= Date: Thu, 16 May 2024 17:05:27 -0400 Subject: [PATCH 15/44] Tests for exptype handling in exp_to_source --- jwst/exp_to_source/exp_to_source.py | 6 +-- .../exp_to_source/tests/test_exp_to_source.py | 40 +++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/jwst/exp_to_source/exp_to_source.py b/jwst/exp_to_source/exp_to_source.py index 8f3d3a4f39..20edde7d38 100644 --- a/jwst/exp_to_source/exp_to_source.py +++ b/jwst/exp_to_source/exp_to_source.py @@ -67,14 +67,12 @@ def exp_to_source(inputs): # (necessary for NIRSpec fixed slits defined as part of an MSA file) if slit_exptype is not None: result_slit.exposures[-1].meta.exposure.type = slit_exptype - - if result_slit.meta.instrument.name is None: - result_slit.update(exposure) - result_slit.meta.exposure.type = slit_exptype log.debug(f'Input exposure type: {exposure.meta.exposure.type}') log.debug(f'Output exposure type: {result_slit.meta.exposure.type}') + if result_slit.meta.instrument.name is None: + result_slit.update(exposure) result_slit.meta.filename = None # Resulting merged data doesn't come from one file exposure.close() diff --git a/jwst/exp_to_source/tests/test_exp_to_source.py b/jwst/exp_to_source/tests/test_exp_to_source.py index 0216140027..72f84a5df9 100644 --- a/jwst/exp_to_source/tests/test_exp_to_source.py +++ b/jwst/exp_to_source/tests/test_exp_to_source.py @@ -75,3 +75,43 @@ def test_container_structure(): container.close() for model in inputs: model.close() + for model in outputs.values(): + model.close() + + +def test_slit_exptype(): + """Test for slit exposure type handling.""" + + # Setup input + inputs = [MultiSlitModel(f) for f in helpers.INPUT_FILES] + container = ModelContainer(inputs) + + # Add a slit exposure type to each input + for model in container: + for slit in model.slits: + if slit.source_id == 1: + slit.meta.exposure = {'type': 'NRS_MSASPEC'} + else: + slit.meta.exposure = {'type': 'NRS_FIXEDSLIT'} + + # Make the source-based containers + outputs = multislit_to_container(container) + + # Check that exposure type was passed from input to output + assert len(container) == 3 + assert len(outputs) == 5 + for i, model in enumerate(container): + for slit in model.slits: + exposure = outputs[str(slit.source_id)][i] + assert exposure.meta.exposure.type == slit.meta.exposure.type + if slit.source_id == 1: + assert exposure.meta.exposure.type == 'NRS_MSASPEC' + else: + assert exposure.meta.exposure.type == 'NRS_FIXEDSLIT' + + # Closeout + container.close() + for model in inputs: + model.close() + for model in outputs.values(): + model.close() From 76c82b285b7d4614a7e3c48758b15e35786ad37b Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 17 May 2024 13:04:55 -0400 Subject: [PATCH 16/44] Add tests for NIRSpec extract2d --- jwst/extract_2d/tests/test_nirspec.py | 213 ++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 jwst/extract_2d/tests/test_nirspec.py diff --git a/jwst/extract_2d/tests/test_nirspec.py b/jwst/extract_2d/tests/test_nirspec.py new file mode 100644 index 0000000000..81e593ac99 --- /dev/null +++ b/jwst/extract_2d/tests/test_nirspec.py @@ -0,0 +1,213 @@ +import numpy as np +import pytest +from astropy.io import fits +from astropy.table import Table +from stdatamodels.jwst.datamodels import ImageModel, CubeModel, MultiSlitModel, SlitModel + +from jwst.assign_wcs import AssignWcsStep +from jwst.extract_2d.extract_2d_step import Extract2dStep + + +# WCS keywords, borrowed from NIRCam grism tests +WCS_KEYS = {'wcsaxes': 2, 'ra_ref': 53.1490299775, 'dec_ref': -27.8168745624, + 'v2_ref': 86.103458, 'v3_ref': -493.227512, 'roll_ref': 45.04234459270135, + 'crpix1': 1024.5, 'crpix2': 1024.5, + 'crval1': 53.1490299775, 'crval2': -27.8168745624, + 'cdelt1': 1.81661111111111e-05, 'cdelt2': 1.8303611111111e-05, + 'ctype1': 'RA---TAN', 'ctype2': 'DEC--TAN', + 'pc1_1': -0.707688557183348, 'pc1_2': 0.7065245261360363, + 'pc2_1': 0.7065245261360363, 'pc2_2': 1.75306861111111e-05, + 'cunit1': 'deg', 'cunit2': 'deg'} + + +def create_nirspec_hdul(detector='NRS1', grating='G395M', filter_name='F290LP', + exptype='NRS_MSASPEC', subarray='FULL', slit=None, nint=1, + wcskeys=None): + if wcskeys is None: + wcskeys = WCS_KEYS.copy() + + hdul = fits.HDUList() + phdu = fits.PrimaryHDU() + phdu.header['TELESCOP'] = 'JWST' + phdu.header['INSTRUME'] = 'NIRSPEC' + phdu.header['DETECTOR'] = detector + phdu.header['FILTER'] = filter_name + phdu.header['GRATING'] = grating + phdu.header['TIME-OBS'] = '8:59:37' + phdu.header['DATE-OBS'] = '2023-01-05' + phdu.header['EXP_TYPE'] = exptype + phdu.header['PATT_NUM'] = 1 + phdu.header['SUBARRAY'] = subarray + if subarray == 'SUBS200A1': + phdu.header['SUBSIZE1'] = 2048 + phdu.header['SUBSIZE2'] = 64 + phdu.header['SUBSTRT1'] = 1 + phdu.header['SUBSTRT2'] = 1041 + elif subarray == 'SUB2048': + phdu.header['SUBSIZE1'] = 2048 + phdu.header['SUBSIZE2'] = 32 + phdu.header['SUBSTRT1'] = 1 + phdu.header['SUBSTRT2'] = 946 + else: + phdu.header['SUBSIZE1'] = 2048 + phdu.header['SUBSIZE2'] = 2048 + phdu.header['SUBSTRT1'] = 1 + phdu.header['SUBSTRT2'] = 1 + + if exptype == 'NRS_MSASPEC': + phdu.header['MSAMETID'] = 1 + phdu.header['MSAMETFL'] = 'test_msa_01.fits' + + if slit is not None: + phdu.header['FXD_SLIT'] = slit + phdu.header['APERNAME'] = f'NRS_{slit}_SLIT' + + scihdu = fits.ImageHDU() + scihdu.header['EXTNAME'] = "SCI" + scihdu.header.update(wcskeys) + if nint > 1: + scihdu.data = np.ones((nint, phdu.header['SUBSIZE2'], phdu.header['SUBSIZE1'])) + else: + scihdu.data = np.ones((phdu.header['SUBSIZE2'], phdu.header['SUBSIZE1'])) + + hdul.append(phdu) + hdul.append(scihdu) + return hdul + + +def create_msa_hdul(): + # Two point sources, one in MSA, one fixed slit. + # Source locations for the fixed slit are placeholders, not realistic. + shutter_data = { + 'slitlet_id': [12, 12, 12, 100], + 'msa_metadata_id': [1, 1, 1, 1], + 'shutter_quadrant': [4, 4, 4, 0], + 'shutter_row': [251, 251, 251, 0], + 'shutter_column': [22, 23, 24, 0], + 'source_id': [1, 1, 1, 2], + 'background': ['Y', 'N', 'Y', 'N'], + 'shutter_state': ['OPEN', 'OPEN', 'OPEN', 'OPEN'], + 'estimated_source_in_shutter_x': [np.nan, 0.18283921, np.nan, 0.5], + 'estimated_source_in_shutter_y': [np.nan, 0.31907734, np.nan, 0.5], + 'dither_point_index': [1, 1, 1, 1], + 'primary_source': ['N', 'Y', 'N', 'Y'], + 'fixed_slit': ['NONE', 'NONE', 'NONE', 'S200A1']} + + source_data = { + 'program': [95065, 95065], + 'source_id': [1, 2], + 'source_name': ['95065_1', '95065_2'], + 'alias': ['2122', '2123'], + 'ra': [53.139904, 53.15], + 'dec': [-27.805002, -27.81], + 'preimage_id': ['95065001_000', '95065001_000'], + 'stellarity': [1.0, 1.0]} + + shutter_table = Table(shutter_data) + source_table = Table(source_data) + + hdul = fits.HDUList() + hdul.append(fits.PrimaryHDU()) + hdul.append(fits.ImageHDU()) + hdul.append(fits.table_to_hdu(shutter_table)) + hdul.append(fits.table_to_hdu(source_table)) + hdul[2].name = 'SHUTTER_INFO' + hdul[3].name = 'SOURCE_INFO' + + return hdul + + +@pytest.fixture +def nirspec_msa_rate(tmp_path): + hdul = create_nirspec_hdul() + hdul[0].header['MSAMETFL'] = str(tmp_path / 'test_msa_01.fits') + filename = str(tmp_path / 'test_nrs_msa_rate.fits') + hdul.writeto(filename, overwrite=True) + hdul.close() + return filename + + +@pytest.fixture +def nirspec_fs_rate(tmp_path): + hdul = create_nirspec_hdul( + exptype='NRS_FIXEDSLIT', subarray='SUBS200A1', slit='S200A1') + filename = str(tmp_path / 'test_nrs_fs_rate.fits') + hdul.writeto(filename, overwrite=True) + hdul.close() + return filename + + +@pytest.fixture +def nirspec_bots_rateints(tmp_path): + hdul = create_nirspec_hdul( + exptype='NRS_BRIGHTOBJ', subarray='SUB2048', slit='S1600A1', nint=3) + filename = str(tmp_path / 'test_nrs_bots_rateints.fits') + hdul.writeto(filename, overwrite=True) + hdul.close() + return filename + + +@pytest.fixture +def nirspec_msa_metfl(tmp_path): + hdul = create_msa_hdul() + filename = str(tmp_path / 'test_msa_01.fits') + hdul.writeto(filename, overwrite=True) + hdul.close() + return filename + + +def test_extract_2d_nirspec_msa_fs(nirspec_msa_rate, nirspec_msa_metfl): + model = ImageModel(nirspec_msa_rate) + result = AssignWcsStep.call(model) + result = Extract2dStep.call(result) + assert isinstance(result, MultiSlitModel) + + # there should be 2 slits extracted: one MSA, one FS + assert len(result.slits) == 2 + + # the MSA slit has an integer name, slitlet_id matches name + assert result.slits[0].name == '12' + assert result.slits[0].slitlet_id == 12 + assert result.slits[0].data.shape == (31, 1355) + + # the FS slit has a string name, slitlet_id matches shutter ID + assert result.slits[1].name == 'S200A1' + assert result.slits[1].slitlet_id == 0 + assert result.slits[1].data.shape == (45, 1254) + + model.close() + result.close() + + +def test_extract_2d_nirspec_fs(nirspec_fs_rate): + model = ImageModel(nirspec_fs_rate) + result = AssignWcsStep.call(model) + result = Extract2dStep.call(result) + assert isinstance(result, MultiSlitModel) + + # there should be 1 slit extracted: FS, S200A1 + assert len(result.slits) == 1 + + # the FS slit has a string name, slitlet_id matches shutter ID + assert result.slits[0].name == 'S200A1' + assert result.slits[0].slitlet_id == 0 + assert result.slits[0].data.shape == (45, 1254) + + model.close() + result.close() + + +def test_extract_2d_nirspec_bots(nirspec_bots_rateints): + model = CubeModel(nirspec_bots_rateints) + result = AssignWcsStep.call(model) + result = Extract2dStep.call(result) + + # output is a single slit + assert isinstance(result, SlitModel) + + # the BOTS slit has a string name, slitlet_id matches shutter ID + assert result.name == 'S1600A1' + assert result.data.shape == (3, 28, 1300) + + model.close() + result.close() From 8ad1b9df8e68e14a611ca7127818cb73b74da364 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 17 May 2024 13:06:52 -0400 Subject: [PATCH 17/44] Re-fix flat field DNU --- jwst/flatfield/flat_field.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jwst/flatfield/flat_field.py b/jwst/flatfield/flat_field.py index a64b78f442..e93b6190b5 100644 --- a/jwst/flatfield/flat_field.py +++ b/jwst/flatfield/flat_field.py @@ -441,7 +441,12 @@ def nirspec_fs_msa(output_model, f_flat_model, s_flat_model, d_flat_model, dispa # Make sure all DO_NOT_USE pixels are set to NaN, # including those flagged by this step - slit.data[np.where(slit.dq & dqflags.pixel['DO_NOT_USE'])] = np.nan + dnu = np.where(slit.dq & dqflags.pixel['DO_NOT_USE']) + slit.data[dnu] = np.nan + slit.err[dnu] = np.nan + slit.var_poisson[dnu] = np.nan + slit.var_rnoise[dnu] = np.nan + slit.var_flat[dnu] = np.nan any_updated = True From 309d09865c7ec9b37f0cc681791e6d3fd537ccc6 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 17 May 2024 18:29:05 -0400 Subject: [PATCH 18/44] Coverage tests for NIRSpec flat field --- jwst/flatfield/tests/test_flatfield.py | 236 +++++++++++++++++++++++++ 1 file changed, 236 insertions(+) diff --git a/jwst/flatfield/tests/test_flatfield.py b/jwst/flatfield/tests/test_flatfield.py index 8b0176ae89..4fc7d37315 100644 --- a/jwst/flatfield/tests/test_flatfield.py +++ b/jwst/flatfield/tests/test_flatfield.py @@ -1,5 +1,7 @@ import pytest import numpy as np +from astropy.io import fits +from astropy.table import Table from stdatamodels.jwst import datamodels @@ -92,3 +94,237 @@ def test_nirspec_flatfield_step_interface(exptype): data.meta.subarray.ysize = shape[0] FlatFieldStep.call(data) + + +def create_nirspec_flats(shape, msa=False): + flats = [] + for flat_name in ['f', 's', 'd']: + if flat_name == 'f': + f_data = np.full(shape[0], 1.0) + f_err = np.full(shape[0], 0.1) + else: + f_data = np.full(shape[0], 1.0) + f_err = np.full(shape[0], np.nan) + + # make a fast variation table + flat_table = Table({ + 'slit_name': ['ANY'], + 'nelem': [shape[0]], + 'wavelength': [np.arange(1, shape[0] + 1, dtype=float)], + 'data': [f_data], + 'error': [f_err]}) + + fflat_hdul = fits.HDUList([fits.PrimaryHDU()]) + if flat_name == 's' and not msa: + fflat_hdul.append(fits.ImageHDU(data=np.full(shape[-2:], 1.0), name='SCI')) + fflat_hdul.append(fits.ImageHDU(data=np.full(shape[-2:], 0), name='DQ')) + fflat_hdul.append(fits.ImageHDU(data=np.full(shape[-2:], 0.0), name='ERR')) + + elif flat_name == 's' and msa: + fflat_hdul.append(fits.ImageHDU(data=np.full((5, shape[1], shape[2]), 1.0), name='SCI')) + fflat_hdul.append(fits.ImageHDU(data=np.full((5, shape[1], shape[2]), 0), name='DQ')) + fflat_hdul.append(fits.ImageHDU(data=np.full((5, shape[1], shape[2]), 0.0), name='ERR')) + + fflat_hdul.append(fits.table_to_hdu( + Table({'wavelength': np.arange(1, shape[0] + 1)}))) + fflat_hdul[-1].name = 'WAVELENGTH' + + elif flat_name == 'd': + fflat_hdul.append(fits.ImageHDU(data=np.full(shape, 1.0), name='SCI')) + fflat_hdul.append(fits.ImageHDU(data=np.full(shape, 0), name='DQ')) + fflat_hdul.append(fits.ImageHDU(data=np.full(shape, 0.0), name='ERR')) + fflat_hdul.append(fits.table_to_hdu( + Table({'wavelength': np.arange(1, shape[0] + 1)}))) + fflat_hdul[-1].name = 'WAVELENGTH' + + if flat_name == 'f' and msa: + for quadrant in range(4): + fflat_hdul.append(fits.ImageHDU(data=np.full(shape, 1.0), + name='SCI', ver=(quadrant + 1))) + fflat_hdul.append(fits.ImageHDU(data=np.full(shape, 0), + name='DQ', ver=(quadrant + 1))) + fflat_hdul.append(fits.ImageHDU(data=np.full(shape, 0.0), + name='ERR', ver=(quadrant + 1))) + hdu = fits.table_to_hdu( + Table({'wavelength': [np.arange(1, shape[0] + 1)]})) + hdu.header['EXTNAME'] = 'WAVELENGTH' + hdu.header['EXTVER'] = quadrant + 1 + fflat_hdul.append(hdu) + + hdu = fits.table_to_hdu(flat_table) + hdu.header['EXTNAME'] = 'FAST_VARIATION' + hdu.header['EXTVER'] = quadrant + 1 + fflat_hdul.append(hdu) + + else: + fflat_hdul.append(fits.table_to_hdu(flat_table)) + fflat_hdul[-1].name = 'FAST_VARIATION' + + if msa and flat_name == 'f': + flat = datamodels.NirspecQuadFlatModel(fflat_hdul) + else: + flat = datamodels.NirspecFlatModel(fflat_hdul) + fflat_hdul.close() + + flat.meta.instrument.name = 'NIRSPEC' + flat.meta.subarray.xstart = 1 + flat.meta.subarray.ystart = 1 + flat.meta.subarray.xsize = shape[1] + flat.meta.subarray.ysize = shape[0] + + flats.append(flat) + + return flats + + +def test_nirspec_bots_flat(): + """Test that the interface works for NIRSpec BOTS data""" + shape = (3, 20, 20) + w_shape = (10, 20, 20) + + data = datamodels.SlitModel(shape) + data.meta.instrument.name = 'NIRSPEC' + data.meta.exposure.type = 'NRS_BRIGHTOBJ' + data.meta.subarray.xstart = 1 + data.meta.subarray.ystart = 1 + data.meta.subarray.xsize = shape[1] + data.meta.subarray.ysize = shape[0] + data.xstart = 1 + data.ystart = 1 + data.xsize = shape[1] + data.ysize = shape[0] + data.data += 1 + data.wavelength = np.ones(shape[-2:]) + data.wavelength[:] = np.linspace(1, 5, shape[-1], dtype=float) + + flats = create_nirspec_flats(w_shape) + result = FlatFieldStep.call(data, override_fflat=flats[0], override_sflat=flats[1], + override_dflat=flats[2], override_flat='N/A') + + # null flat, so data is the same, other than nan edges + nn = ~np.isnan(result.data) + assert np.allclose(result.data[nn], data.data[nn]) + + # check that NaNs match in every extension they should + for ext in ['data', 'err', 'var_rnoise', 'var_poisson', 'var_flat']: + test_data = getattr(result, ext) + assert np.all(np.isnan(test_data[~nn])) + + # error is propagated from non-empty fflat error + # (no other additive contribution, scale from data is 1.0) + assert result.var_flat.shape == shape + assert np.allclose(result.var_flat[nn], 0.1 ** 2) + assert result.meta.cal_step.flat_field == 'COMPLETE' + + result.close() + for flat in flats: + flat.close() + + +def test_nirspec_fs_flat(): + """Test that the interface works for NIRSpec FS data.""" + shape = (20, 20) + w_shape = (10, 20, 20) + + data = datamodels.MultiSlitModel() + data.meta.instrument.name = 'NIRSPEC' + data.meta.exposure.type = 'NRS_FIXEDSLIT' + data.meta.subarray.xstart = 1 + data.meta.subarray.ystart = 1 + data.meta.subarray.xsize = shape[1] + data.meta.subarray.ysize = shape[0] + + data.slits.append(datamodels.SlitModel(shape)) + data.slits[0].data = np.full(shape, 1.0) + data.slits[0].dq = np.full(shape, 0) + data.slits[0].err = np.full(shape, 0.0) + data.slits[0].var_poisson = np.full(shape, 0.0) + data.slits[0].var_rnoise = np.full(shape, 0.0) + data.slits[0].data = np.full(shape, 1.0) + data.slits[0].wavelength = np.ones(shape[-2:]) + data.slits[0].wavelength[:] = np.linspace(1, 5, shape[-1], dtype=float) + data.slits[0].source_type = 'UNKNOWN' + data.slits[0].name = 'S200A1' + data.slits[0].xstart = 1 + data.slits[0].ystart = 1 + data.slits[0].xsize = shape[1] + data.slits[0].ysize = shape[0] + + flats = create_nirspec_flats(w_shape) + result = FlatFieldStep.call(data, override_fflat=flats[0], override_sflat=flats[1], + override_dflat=flats[2], override_flat='N/A') + + # null flat, so data is the same, other than nan edges + nn = ~np.isnan(result.slits[0].data) + assert np.allclose(result.slits[0].data[nn], data.slits[0].data[nn]) + + # check that NaNs match in every extension they should + for ext in ['data', 'err', 'var_rnoise', 'var_poisson', 'var_flat']: + test_data = getattr(result.slits[0], ext) + assert np.all(np.isnan(test_data[~nn])) + + # error is propagated from non-empty fflat error + # (no other additive contribution, scale from data is 1.0) + assert result.slits[0].var_flat.shape == shape + assert np.allclose(result.slits[0].var_flat[nn], 0.1 ** 2) + assert result.meta.cal_step.flat_field == 'COMPLETE' + + result.close() + for flat in flats: + flat.close() + + +def test_nirspec_msa_flat(): + """Test that the interface works for NIRSpec MSA data.""" + shape = (20, 20) + w_shape = (10, 20, 20) + + data = datamodels.MultiSlitModel() + data.meta.instrument.name = 'NIRSPEC' + data.meta.exposure.type = 'NRS_MSASPEC' + data.meta.subarray.xstart = 1 + data.meta.subarray.ystart = 1 + data.meta.subarray.xsize = shape[1] + data.meta.subarray.ysize = shape[0] + + data.slits.append(datamodels.SlitModel(shape)) + data.slits[0].data = np.full(shape, 1.0) + data.slits[0].dq = np.full(shape, 0) + data.slits[0].err = np.full(shape, 0.0) + data.slits[0].var_poisson = np.full(shape, 0.0) + data.slits[0].var_rnoise = np.full(shape, 0.0) + data.slits[0].data = np.full(shape, 1.0) + data.slits[0].wavelength = np.ones(shape[-2:]) + data.slits[0].wavelength[:] = np.linspace(1, 5, shape[-1], dtype=float) + data.slits[0].source_type = 'UNKNOWN' + data.slits[0].name = '11' + data.slits[0].quadrant = 1 + data.slits[0].xcen = 10 + data.slits[0].ycen = 10 + data.slits[0].xstart = 1 + data.slits[0].ystart = 1 + data.slits[0].xsize = shape[1] + data.slits[0].ysize = shape[0] + + flats = create_nirspec_flats(w_shape, msa=True) + result = FlatFieldStep.call(data, override_fflat=flats[0], override_sflat=flats[1], + override_dflat=flats[2], override_flat='N/A') + + # null flat, so data is the same, other than nan edges + nn = ~np.isnan(result.slits[0].data) + assert np.allclose(result.slits[0].data[nn], data.slits[0].data[nn]) + + # check that NaNs match in every extension they should + for ext in ['data', 'err', 'var_rnoise', 'var_poisson', 'var_flat']: + test_data = getattr(result.slits[0], ext) + assert np.all(np.isnan(test_data[~nn])) + + # error is propagated from non-empty fflat error + # (no other additive contribution, scale from data is 1.0) + assert result.slits[0].var_flat.shape == shape + assert np.allclose(result.slits[0].var_flat[nn], 0.1 ** 2) + assert result.meta.cal_step.flat_field == 'COMPLETE' + + result.close() + for flat in flats: + flat.close() From a45292d59c5bc97209e1706205bbd9ecca3e0ca3 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 20 May 2024 11:32:16 -0400 Subject: [PATCH 19/44] Add flatfield test for NIRSpec IFU --- jwst/flatfield/tests/test_flatfield.py | 50 ++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/jwst/flatfield/tests/test_flatfield.py b/jwst/flatfield/tests/test_flatfield.py index 4fc7d37315..10722636ca 100644 --- a/jwst/flatfield/tests/test_flatfield.py +++ b/jwst/flatfield/tests/test_flatfield.py @@ -5,6 +5,8 @@ from stdatamodels.jwst import datamodels +from jwst.assign_wcs import AssignWcsStep +from jwst.assign_wcs.tests.test_nirspec import create_nirspec_ifu_file from jwst.flatfield import FlatFieldStep from jwst.flatfield.flat_field_step import NRS_IMAGING_MODES, NRS_SPEC_MODES @@ -221,7 +223,8 @@ def test_nirspec_bots_flat(): flat.close() -def test_nirspec_fs_flat(): +@pytest.mark.parametrize('srctype', ['POINT', 'EXTENDED']) +def test_nirspec_fs_flat(srctype): """Test that the interface works for NIRSpec FS data.""" shape = (20, 20) w_shape = (10, 20, 20) @@ -243,7 +246,7 @@ def test_nirspec_fs_flat(): data.slits[0].data = np.full(shape, 1.0) data.slits[0].wavelength = np.ones(shape[-2:]) data.slits[0].wavelength[:] = np.linspace(1, 5, shape[-1], dtype=float) - data.slits[0].source_type = 'UNKNOWN' + data.slits[0].source_type = srctype data.slits[0].name = 'S200A1' data.slits[0].xstart = 1 data.slits[0].ystart = 1 @@ -328,3 +331,46 @@ def test_nirspec_msa_flat(): result.close() for flat in flats: flat.close() + + +@pytest.mark.slow +def test_nirspec_ifu_flat(): + """Test that the interface works for NIRSpec IFU data. + + Larger data and more WCS operations required for testing make + this test take more than a minute, so marking this test 'slow'. + """ + shape = (2048, 2048) + w_shape = (10, 2048, 2048) + + # IFU mode requires WCS information, so make a more realistic model + hdul = create_nirspec_ifu_file(grating='PRISM', filter='CLEAR', + gwa_xtil=0.35986012, gwa_ytil=0.13448857, + gwa_tilt=37.1) + hdul['SCI'].data = np.ones(shape, dtype=float) + + data = datamodels.IFUImageModel(hdul) + data = AssignWcsStep.call(data) + + flats = create_nirspec_flats(w_shape) + result = FlatFieldStep.call(data, override_fflat=flats[0], override_sflat=flats[1], + override_dflat=flats[2], override_flat='N/A') + + # null flat, so data is the same, other than nan edges + nn = ~np.isnan(result.data) + assert np.allclose(result.data[nn], data.data[nn]) + + # check that NaNs match in every extension they should + for ext in ['data', 'err', 'var_rnoise', 'var_poisson', 'var_flat']: + test_data = getattr(result, ext) + assert np.all(np.isnan(test_data[~nn])) + + # error is propagated from non-empty fflat error + # (no other additive contribution, scale from data is 1.0) + assert result.var_flat.shape == shape + assert np.allclose(result.var_flat[nn], 0.1 ** 2) + assert result.meta.cal_step.flat_field == 'COMPLETE' + + result.close() + for flat in flats: + flat.close() From a56ceb4efbc43ea925480328e02d816691c83573 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 20 May 2024 13:23:24 -0400 Subject: [PATCH 20/44] Don't extract fixed slits if neither background nor primary --- jwst/assign_wcs/nirspec.py | 12 +++++++++--- .../tests/data/msa_fs_configuration.fits | Bin 2016000 -> 2016000 bytes jwst/assign_wcs/tests/test_nirspec.py | 10 ++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index 7a16f2a6fc..5e9efcafa0 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -650,7 +650,9 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, open_shutters = [x['shutter_column'] for x in slitlets_sid] # How many shutters in the slitlet are labeled as "main" or "primary"? + # How many are labeled background? n_main_shutter = len([s for s in slitlets_sid if s['primary_source'] == 'Y']) + n_background = len([s for s in slitlets_sid if s['background'] == 'Y']) # Check for fixed slit sources defined in the MSA file is_fs = [False] * len(slitlets_sid) @@ -672,7 +674,6 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, # One fixed slit open for the source slitlet = slitlets_sid[0] slit_name = slitlet['fixed_slit'] - log.debug(f'Found fixed slit {slit_name} with primary target') # use standard number for fixed slit shutter id slitlet_id = slit_name @@ -690,14 +691,19 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, source_id = slitlet['source_id'] source_xpos = slitlet['estimated_source_in_shutter_x'] source_ypos = slitlet['estimated_source_in_shutter_y'] - else: + log.info(f'Found fixed slit {slitlet_id} with source_id = {source_id}.') + elif n_background == 1: # source is background only source_id = _get_bkg_source_id(bkg_counter, max_source_id) source_xpos = 0.5 source_ypos = 0.5 log.info(f'Slitlet_id {slitlet_id} is background only; ' - f'assigned source_id = {source_id}') + f'assigned source_id = {source_id}.') bkg_counter += 1 + else: + log.info(f'Fixed slit {slitlet_id} is neither background nor source; ' + f'skipping it.') + continue elif any(is_fs): # Unsupported fixed slit configuration diff --git a/jwst/assign_wcs/tests/data/msa_fs_configuration.fits b/jwst/assign_wcs/tests/data/msa_fs_configuration.fits index dd4c3fc6f2f16f2fd7adc260c8e3fbac2e031c62..5a1f5b6e55be4ba7ca4da17125829f11621e9542 100644 GIT binary patch delta 77 zcmWNok7^4lW)(0U;4F2`L#l1tpS- YnueB+o&m+k#LU8a>&=Se*{y!n2h5Kc&Hw-a delta 77 zcmWN Date: Tue, 21 May 2024 10:48:02 -0400 Subject: [PATCH 21/44] Add check for dq when data is NaN --- jwst/flatfield/tests/test_flatfield.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jwst/flatfield/tests/test_flatfield.py b/jwst/flatfield/tests/test_flatfield.py index 10722636ca..a332632bf2 100644 --- a/jwst/flatfield/tests/test_flatfield.py +++ b/jwst/flatfield/tests/test_flatfield.py @@ -211,6 +211,7 @@ def test_nirspec_bots_flat(): for ext in ['data', 'err', 'var_rnoise', 'var_poisson', 'var_flat']: test_data = getattr(result, ext) assert np.all(np.isnan(test_data[~nn])) + assert np.all(result.dq[~nn] | datamodels.dqflags.pixel['DO_NOT_USE']) # error is propagated from non-empty fflat error # (no other additive contribution, scale from data is 1.0) @@ -265,6 +266,7 @@ def test_nirspec_fs_flat(srctype): for ext in ['data', 'err', 'var_rnoise', 'var_poisson', 'var_flat']: test_data = getattr(result.slits[0], ext) assert np.all(np.isnan(test_data[~nn])) + assert np.all(result.slits[0].dq[~nn] | datamodels.dqflags.pixel['DO_NOT_USE']) # error is propagated from non-empty fflat error # (no other additive contribution, scale from data is 1.0) @@ -321,6 +323,7 @@ def test_nirspec_msa_flat(): for ext in ['data', 'err', 'var_rnoise', 'var_poisson', 'var_flat']: test_data = getattr(result.slits[0], ext) assert np.all(np.isnan(test_data[~nn])) + assert np.all(result.slits[0].dq[~nn] | datamodels.dqflags.pixel['DO_NOT_USE']) # error is propagated from non-empty fflat error # (no other additive contribution, scale from data is 1.0) @@ -364,6 +367,7 @@ def test_nirspec_ifu_flat(): for ext in ['data', 'err', 'var_rnoise', 'var_poisson', 'var_flat']: test_data = getattr(result, ext) assert np.all(np.isnan(test_data[~nn])) + assert np.all(result.dq[~nn] | datamodels.dqflags.pixel['DO_NOT_USE']) # error is propagated from non-empty fflat error # (no other additive contribution, scale from data is 1.0) From 642da2db8dca5d79f872ae7b76c599791bca9b10 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 23 May 2024 17:05:44 -0400 Subject: [PATCH 22/44] Regression tests for MOS/FS spec2 and spec3 --- jwst/regtest/test_nirspec_mos_fs_spec2.py | 55 +++++++++++++++++++++++ jwst/regtest/test_nirspec_mos_fs_spec3.py | 42 +++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 jwst/regtest/test_nirspec_mos_fs_spec2.py create mode 100644 jwst/regtest/test_nirspec_mos_fs_spec3.py diff --git a/jwst/regtest/test_nirspec_mos_fs_spec2.py b/jwst/regtest/test_nirspec_mos_fs_spec2.py new file mode 100644 index 0000000000..bcceb0d4f6 --- /dev/null +++ b/jwst/regtest/test_nirspec_mos_fs_spec2.py @@ -0,0 +1,55 @@ +import pytest + +from astropy.io.fits.diff import FITSDiff + +from jwst.stpipe import Step + + +@pytest.fixture(scope="module") +def run_pipeline(rtdata_module): + """Run the calwebb_spec2 pipeline on a single NIRSpec MOS/FS exposure.""" + + rtdata = rtdata_module + + # Get the MSA metadata file referenced in the input exposure + rtdata.get_data("nirspec/mos/jw02674004001_01_msa.fits") + + # Get the input ASN file and exposures + rtdata.get_data("nirspec/mos/jw02674004001_03101_00001_nrs1_rate.fits") + + # Run the calwebb_spec2 pipeline; save results from intermediate steps + args = ["calwebb_spec2", rtdata.input, + "--steps.assign_wcs.save_results=true", + "--steps.msa_flagging.save_results=true", + "--steps.master_background_mos.save_results=true", + "--steps.extract_2d.save_results=true", + "--steps.srctype.save_results=true", + "--steps.wavecorr.save_results=true", + "--steps.flat_field.save_results=true", + "--steps.pathloss.save_results=true", + "--steps.barshadow.save_results=true"] + Step.from_cmdline(args) + + return rtdata + + +@pytest.mark.bigdata +@pytest.mark.parametrize("suffix", [ + "assign_wcs", "msa_flagging", "extract_2d", "srctype", + "master_background_mos", "wavecorr", "flat_field", "pathloss", "barshadow", + "wavecorr_fs", "flat_field_fs", "pathloss_fs", "barshadow_fs", + "cal", "s2d", "x1d"]) +def test_nirspec_mos_fs_spec2(run_pipeline, fitsdiff_default_kwargs, suffix): + """Regression test for calwebb_spec2 on a NIRSpec MOS/FS exposure.""" + + # Run the pipeline and retrieve outputs + rtdata = run_pipeline + output = f"jw02674004001_03101_00001_nrs1_{suffix}.fits" + rtdata.output = output + + # Get the truth files + rtdata.get_truth("truth/test_nirspec_mos_fs_spec2/" + output) + + # Compare the results + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() diff --git a/jwst/regtest/test_nirspec_mos_fs_spec3.py b/jwst/regtest/test_nirspec_mos_fs_spec3.py new file mode 100644 index 0000000000..dcea194948 --- /dev/null +++ b/jwst/regtest/test_nirspec_mos_fs_spec3.py @@ -0,0 +1,42 @@ +import pytest +from astropy.io.fits.diff import FITSDiff +import numpy as np +from gwcs import wcstools + +from jwst.stpipe import Step +from stdatamodels.jwst import datamodels + + +@pytest.fixture(scope="module") +def run_pipeline(rtdata_module): + """Run calwebb_spec3 on NIRSpec MOS data.""" + rtdata = rtdata_module + rtdata.get_asn("nirspec/mos/jw02674-o004_20240305t054741_spec3_00001_asn.json") + + # Run the calwebb_spec3 pipeline on the association + args = ["calwebb_spec3", rtdata.input] + Step.from_cmdline(args) + + return rtdata + + +@pytest.mark.bigdata +@pytest.mark.parametrize("suffix", ["cal", "crf", "s2d", "x1d"]) +@pytest.mark.parametrize("source_id", ["s01354", "s12105", "s34946", + "s34949", "s34950", "s34951", "s34952", + "s34953", "s34954", "s34955"]) +def test_nirspec_mos_fs_spec3(run_pipeline, suffix, source_id, fitsdiff_default_kwargs): + """Check results of calwebb_spec3""" + rtdata = run_pipeline + + output = f"jw02674-o004_{source_id}_nirspec_f290lp-g395m_{suffix}.fits" + rtdata.output = output + rtdata.get_truth(f"truth/test_nirspec_mos_fs_spec3/{output}") + + # Adjust tolerance for machine precision with float32 drizzle code + if suffix == "s2d": + fitsdiff_default_kwargs["rtol"] = 1e-4 + fitsdiff_default_kwargs["atol"] = 1e-5 + + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() From 4fa1151342098cfad278c57cba73ef3b5a2d0395 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 29 May 2024 14:24:43 -0400 Subject: [PATCH 23/44] Don't block FS region for MOS if there are fixed slits defined --- jwst/nsclean/nsclean.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/jwst/nsclean/nsclean.py b/jwst/nsclean/nsclean.py index 3117eb624a..bbd3b0d4bf 100644 --- a/jwst/nsclean/nsclean.py +++ b/jwst/nsclean/nsclean.py @@ -156,8 +156,20 @@ def create_mask(input_model, mask_spectral_regions, n_sigma): mask[nan_pix] = False # If IFU or MOS, mask the fixed-slit area of the image; uses hardwired indexes - if exptype in ['nrs_ifu', 'nrs_msaspec']: + if exptype == 'nrs_ifu': + log.info("Masking the fixed slit region for IFU data.") mask[922:1116, :] = False + elif exptype == 'nrs_msaspec': + # check for any slits defined in the fixed slit quadrant: + # if there is nothing there of interest, mask the whole FS region + slit2msa = input_model.meta.wcs.get_transform('slit_frame', 'msa_frame') + is_fs = [s.quadrant == 5 for s in slit2msa.slits] + if not any(is_fs): + log.info("Masking the fixed slit region for MOS data.") + mask[922:1116, :] = False + else: + log.info("Fixed slits found in MSA definition; " + "not masking the fixed slit region for MOS data.") # Use left/right reference pixel columns (first and last 4). Can only be # applied to data that uses all 2048 columns of the detector. From 53cc90df8076f09d83443148d434cf408f36948b Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 29 May 2024 15:31:39 -0400 Subject: [PATCH 24/44] Catch warnings for invalid values --- jwst/master_background/nirspec_utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/jwst/master_background/nirspec_utils.py b/jwst/master_background/nirspec_utils.py index 57b6f63a27..1987b36988 100644 --- a/jwst/master_background/nirspec_utils.py +++ b/jwst/master_background/nirspec_utils.py @@ -1,4 +1,5 @@ import logging +import warnings from stdatamodels.jwst import datamodels @@ -229,8 +230,11 @@ def correct_nrs_fs_bkg(input_model): return input_model # Apply the corrections for the primary slit - input_model.data *= (pl_uniform / pl_point) * \ - (ff_uniform / ff_point) * \ - (ph_point / ph_uniform) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "invalid value*", RuntimeWarning) + warnings.filterwarnings("ignore", "divide by zero*", RuntimeWarning) + input_model.data *= (pl_uniform / pl_point) * \ + (ff_uniform / ff_point) * \ + (ph_point / ph_uniform) return input_model From 68a01b61cc9bcf8b954844ff38b151362aab110f Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 29 May 2024 17:31:27 -0400 Subject: [PATCH 25/44] Disable master background in spec3 for NIRSpec MOS --- jwst/master_background/expand_to_2d.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/jwst/master_background/expand_to_2d.py b/jwst/master_background/expand_to_2d.py index a267ae4a74..20fe2cd996 100644 --- a/jwst/master_background/expand_to_2d.py +++ b/jwst/master_background/expand_to_2d.py @@ -160,8 +160,29 @@ def bkg_for_multislit(input, tab_wavelength, tab_background): min_wave = np.amin(tab_wavelength) max_wave = np.amax(tab_wavelength) + # Check whether the photom step has been run + try: + s_photom = str(input.meta.cal_step.photom).upper() + except AttributeError: + s_photom = 'INCOMPLETE' + for (k, slit) in enumerate(input.slits): log.info(f'Expanding background for slit {slit.name}') + + # Check exposure type and calibration status for + # special handling + exp_type = slit.meta.exposure.type + if exp_type is None: + exp_type = input.meta.exposure.type + + if exp_type == 'NRS_MSASPEC' and s_photom == 'COMPLETE': + log.warning('Master background subtraction is not supported ' + 'for flux-calibrated NIRSpec MOS spectra.') + log.warning('Setting the background to 0.0') + background.slits[k].data[:] = 0.0 + background.slits[k].dq[:] = 0 + continue + wl_array = get_wavelengths(slit, input.meta.exposure.type) if wl_array is None: raise RuntimeError(f"Can't determine wavelengths for {type(slit)}") @@ -187,7 +208,7 @@ def bkg_for_multislit(input, tab_wavelength, tab_background): # NIRSpec fixed slits need corrections applied to the 2D background # if the slit contains a point source, in order to make the master bkg # match the calibrated science data in the slit - if input.meta.exposure.type == 'NRS_FIXEDSLIT' and slit.source_type.upper() == 'POINT': + if exp_type == 'NRS_FIXEDSLIT' and slit.source_type.upper() == 'POINT': background.slits[k] = correct_nrs_fs_bkg(background.slits[k]) return background From 4623f32580a6b9fb9a1587bd2e54144f34846748 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 29 May 2024 17:32:00 -0400 Subject: [PATCH 26/44] Fix combined error propagation; improve log message --- jwst/combine_1d/combine1d.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/jwst/combine_1d/combine1d.py b/jwst/combine_1d/combine1d.py index 4155bb1e0e..c4a6021e9d 100644 --- a/jwst/combine_1d/combine1d.py +++ b/jwst/combine_1d/combine1d.py @@ -68,6 +68,7 @@ def __init__(self, ms, spec, exptime_key): self.unit_weight = False # may be reset below self.right_ascension = np.zeros_like(self.wavelength) self.declination = np.zeros_like(self.wavelength) + self.name = spec.name self.source_id = spec.source_id self.source_type = spec.source_type self.flux_unit = spec.spec_table.columns['flux'].unit @@ -204,7 +205,11 @@ def accumulate_sums(self, input_spectra): ninputs = 0 for in_spec in input_spectra: ninputs += 1 - log.info(f'Accumulating data from input spectrum {ninputs}') + if in_spec.name is not None: + slit_name = f'{ninputs}, slit {in_spec.name}' + else: + slit_name = ninputs + log.info(f'Accumulating data from input spectrum {slit_name}') # Get the pixel numbers in the output corresponding to the # wavelengths of the current input spectrum. out_pixel = self.wcs.invert(in_spec.right_ascension, @@ -263,8 +268,8 @@ def compute_combination(self): sum_weight = np.where(self.weight > 0., self.weight, 1.) self.surf_bright /= sum_weight self.flux /= sum_weight - self.flux_error = np.sqrt(self.flux_error / sum_weight) - self.sb_error = np.sqrt(self.sb_error / sum_weight) + self.flux_error = np.sqrt(self.flux_error) / sum_weight + self.sb_error = np.sqrt(self.sb_error) / sum_weight self.normalized = True def create_output_data(self): From 47fe7baec1592a38af336c7cd5448206ca794096 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 30 May 2024 10:06:42 -0400 Subject: [PATCH 27/44] Fix attribute error for slit.meta.exposure --- jwst/master_background/expand_to_2d.py | 5 ++++- stpipe-log.cfg | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 stpipe-log.cfg diff --git a/jwst/master_background/expand_to_2d.py b/jwst/master_background/expand_to_2d.py index 20fe2cd996..9637180a2b 100644 --- a/jwst/master_background/expand_to_2d.py +++ b/jwst/master_background/expand_to_2d.py @@ -171,7 +171,10 @@ def bkg_for_multislit(input, tab_wavelength, tab_background): # Check exposure type and calibration status for # special handling - exp_type = slit.meta.exposure.type + try: + exp_type = slit.meta.exposure.type + except AttributeError: + exp_type = None if exp_type is None: exp_type = input.meta.exposure.type diff --git a/stpipe-log.cfg b/stpipe-log.cfg new file mode 100644 index 0000000000..41d3cf4c99 --- /dev/null +++ b/stpipe-log.cfg @@ -0,0 +1,3 @@ +[*] +level = DEBUG +handler = stdout From 034f6e91ece363819c49ec9ffd6166e89c81738a Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 30 May 2024 10:06:53 -0400 Subject: [PATCH 28/44] Clean up style --- jwst/regtest/test_nirspec_mos_fs_spec3.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/jwst/regtest/test_nirspec_mos_fs_spec3.py b/jwst/regtest/test_nirspec_mos_fs_spec3.py index dcea194948..2543558cce 100644 --- a/jwst/regtest/test_nirspec_mos_fs_spec3.py +++ b/jwst/regtest/test_nirspec_mos_fs_spec3.py @@ -1,10 +1,7 @@ import pytest from astropy.io.fits.diff import FITSDiff -import numpy as np -from gwcs import wcstools from jwst.stpipe import Step -from stdatamodels.jwst import datamodels @pytest.fixture(scope="module") From eae956f3f20964c01cc84a2ababf21428d1cc6bd Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 30 May 2024 10:12:10 -0400 Subject: [PATCH 29/44] Move pixel_replace out of nirspec-specific processing --- jwst/pipeline/calwebb_spec2.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/jwst/pipeline/calwebb_spec2.py b/jwst/pipeline/calwebb_spec2.py index 639b8ab5c8..9457d62ee1 100644 --- a/jwst/pipeline/calwebb_spec2.py +++ b/jwst/pipeline/calwebb_spec2.py @@ -536,7 +536,6 @@ def _process_nirspec_slits(self, data): calibrated = self.pathloss(calibrated) calibrated = self.barshadow(calibrated) calibrated = self.photom(calibrated) - calibrated = self.pixel_replace(calibrated) return calibrated @@ -576,7 +575,6 @@ def _process_nirspec_msa_slits(self, data): calib_mos = self.pathloss(calib_mos) calib_mos = self.barshadow(calib_mos) calib_mos = self.photom(calib_mos) - calib_mos = self.pixel_replace(calib_mos) # Now repeat for FS slits if len(calib_fss.slits) > 0: @@ -585,8 +583,7 @@ def _process_nirspec_msa_slits(self, data): # Run each step with an alternate suffix, # to avoid overwriting previous products if save_results=True - fs_steps = ['wavecorr', 'flat_field', 'pathloss', 'barshadow', - 'photom', 'pixel_replace'] + fs_steps = ['wavecorr', 'flat_field', 'pathloss', 'barshadow', 'photom'] for step_name in fs_steps: # Set suffix step = getattr(self, step_name) From 52359a0a69b2fc01e4086980c4be5807d2b3c394 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 30 May 2024 12:08:17 -0400 Subject: [PATCH 30/44] Better solution for allowing MOS master bg in spec2 only --- jwst/master_background/expand_to_2d.py | 95 ++++++++++++++++--------- jwst/master_background/nirspec_utils.py | 2 +- 2 files changed, 64 insertions(+), 33 deletions(-) diff --git a/jwst/master_background/expand_to_2d.py b/jwst/master_background/expand_to_2d.py index 9637180a2b..de221eab9a 100644 --- a/jwst/master_background/expand_to_2d.py +++ b/jwst/master_background/expand_to_2d.py @@ -18,7 +18,7 @@ WFSS_EXPTYPES = ['NIS_WFSS', 'NRC_WFSS', 'NRC_GRISM', 'NRC_TSGRISM'] -def expand_to_2d(input, m_bkg_spec): +def expand_to_2d(input, m_bkg_spec, allow_mos=False): """Expand a 1-D background to 2-D. Parameters @@ -30,6 +30,14 @@ def expand_to_2d(input, m_bkg_spec): Either the name of a file containing a 1-D background spectrum, or a data model containing such a spectrum. + allow_mos : bool + If True, NIRSpec MOS data is supported. If False, + background is set to 0.0 for any slit marked as exposure + type NRS_MSASPEC. This parameter should be set to True only + for the master_background_mos step in the spec2 pipeline; + MOS data is not supported via the master_background step + in the spec3 pipeline. + Returns ------- background : `~jwst.datamodels.JwstDataModel` @@ -56,15 +64,17 @@ def expand_to_2d(input, m_bkg_spec): # Handle associations, or input ModelContainers if isinstance(input, ModelContainer): - background = bkg_for_container(input, tab_wavelength, tab_background) + background = bkg_for_container(input, tab_wavelength, tab_background, + allow_mos=allow_mos) else: - background = create_bkg(input, tab_wavelength, tab_background) + background = create_bkg(input, tab_wavelength, tab_background, + allow_mos=allow_mos) return background -def bkg_for_container(input, tab_wavelength, tab_background): +def bkg_for_container(input, tab_wavelength, tab_background, allow_mos=False): """Create a 2-D background for a container object. Parameters @@ -78,6 +88,14 @@ def bkg_for_container(input, tab_wavelength, tab_background): tab_background : 1-D ndarray The surf_bright column read from the 1-D background table. + allow_mos : bool + If True, NIRSpec MOS data is supported. If False, + background is set to 0.0 for any slit marked as exposure + type NRS_MSASPEC. This parameter should be set to True only + for the master_background_mos step in the spec2 pipeline; + MOS data is not supported via the master_background step + in the spec3 pipeline. + Returns ------- background : `~jwst.datamodels.ModelContainer` @@ -87,13 +105,14 @@ def bkg_for_container(input, tab_wavelength, tab_background): background = ModelContainer() for input_model in input: - temp = create_bkg(input_model, tab_wavelength, tab_background) + temp = create_bkg(input_model, tab_wavelength, tab_background, + allow_mos=allow_mos) background.append(temp) return background -def create_bkg(input, tab_wavelength, tab_background): +def create_bkg(input, tab_wavelength, tab_background, allow_mos=False): """Create a 2-D background. Parameters @@ -107,6 +126,14 @@ def create_bkg(input, tab_wavelength, tab_background): tab_background : 1-D ndarray The surf_bright column read from the 1-D background table. + allow_mos : bool + If True, NIRSpec MOS data is supported. If False, + background is set to 0.0 for any slit marked as exposure + type NRS_MSASPEC. This parameter should be set to True only + for the master_background_mos step in the spec2 pipeline; + MOS data is not supported via the master_background step + in the spec3 pipeline. + Returns ------- background : `~jwst.datamodels.JwstDataModel` @@ -116,7 +143,8 @@ def create_bkg(input, tab_wavelength, tab_background): # Handle individual NIRSpec FS, NIRSpec MOS if isinstance(input, datamodels.MultiSlitModel): - background = bkg_for_multislit(input, tab_wavelength, tab_background) + background = bkg_for_multislit(input, tab_wavelength, tab_background, + allow_mos=allow_mos) # Handle MIRI LRS elif isinstance(input, datamodels.ImageModel): @@ -134,7 +162,7 @@ def create_bkg(input, tab_wavelength, tab_background): return background -def bkg_for_multislit(input, tab_wavelength, tab_background): +def bkg_for_multislit(input, tab_wavelength, tab_background, allow_mos=False): """Create a 2-D background for a MultiSlitModel. Parameters @@ -148,6 +176,14 @@ def bkg_for_multislit(input, tab_wavelength, tab_background): tab_background : 1-D ndarray The surf_bright column read from the 1-D background table. + allow_mos : bool + If True, NIRSpec MOS data is supported. If False, + background is set to 0.0 for any slit marked as exposure + type NRS_MSASPEC. This parameter should be set to True only + for the master_background_mos step in the spec2 pipeline; + MOS data is not supported via the master_background step + in the spec3 pipeline. + Returns ------- background : `~jwst.datamodels.MultiSlitModel` @@ -160,32 +196,9 @@ def bkg_for_multislit(input, tab_wavelength, tab_background): min_wave = np.amin(tab_wavelength) max_wave = np.amax(tab_wavelength) - # Check whether the photom step has been run - try: - s_photom = str(input.meta.cal_step.photom).upper() - except AttributeError: - s_photom = 'INCOMPLETE' - for (k, slit) in enumerate(input.slits): log.info(f'Expanding background for slit {slit.name}') - # Check exposure type and calibration status for - # special handling - try: - exp_type = slit.meta.exposure.type - except AttributeError: - exp_type = None - if exp_type is None: - exp_type = input.meta.exposure.type - - if exp_type == 'NRS_MSASPEC' and s_photom == 'COMPLETE': - log.warning('Master background subtraction is not supported ' - 'for flux-calibrated NIRSpec MOS spectra.') - log.warning('Setting the background to 0.0') - background.slits[k].data[:] = 0.0 - background.slits[k].dq[:] = 0 - continue - wl_array = get_wavelengths(slit, input.meta.exposure.type) if wl_array is None: raise RuntimeError(f"Can't determine wavelengths for {type(slit)}") @@ -208,9 +221,27 @@ def bkg_for_multislit(input, tab_wavelength, tab_background): background.slits[k].dq[mask_limit] = np.bitwise_or(background.slits[k].dq[mask_limit], dqflags.pixel['DO_NOT_USE']) + # Check exposure type for special handling + try: + exp_type = slit.meta.exposure.type + except AttributeError: + exp_type = None + if exp_type is None: + exp_type = input.meta.exposure.type + + # NIRSpec MOS should only have backgrounds assigned in spec2, + # via the master_background_mos step. + if exp_type == 'NRS_MSASPEC' and not allow_mos: + log.warning('Master background subtraction is not supported ' + 'for NIRSpec MOS spectra.') + log.warning('Setting the background to 0.0') + background.slits[k].data[:] = 0.0 + background.slits[k].dq[:] = 0 + continue + # NIRSpec fixed slits need corrections applied to the 2D background # if the slit contains a point source, in order to make the master bkg - # match the calibrated science data in the slit + # match the calibrated science data in the slit. if exp_type == 'NRS_FIXEDSLIT' and slit.source_type.upper() == 'POINT': background.slits[k] = correct_nrs_fs_bkg(background.slits[k]) diff --git a/jwst/master_background/nirspec_utils.py b/jwst/master_background/nirspec_utils.py index 1987b36988..3b553f2a56 100644 --- a/jwst/master_background/nirspec_utils.py +++ b/jwst/master_background/nirspec_utils.py @@ -71,7 +71,7 @@ def map_to_science_slits(input_model, master_bkg): # Loop over all input slits, creating 2D master background to # match each 2D slitlet cutout - output_model = expand_to_2d(input_model, master_bkg) + output_model = expand_to_2d(input_model, master_bkg, allow_mos=True) return output_model From 9aa340cc76bd4130b4c297f1d74794ec72210e39 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 3 Jun 2024 11:37:31 -0400 Subject: [PATCH 31/44] Update MOS/FS handling for new source naming scheme --- jwst/assign_wcs/nirspec.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index 1610430ec6..3812b2c575 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -635,37 +635,56 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, # quadrant, xcen, ycen, ymin, ymax # First check for a fixed slit - shutter_id = None if all(is_fs) and len(slitlets_sid) == 1: # One fixed slit open for the source slitlet = slitlets_sid[0] slit_name = slitlet['fixed_slit'] - # use standard number for fixed slit shutter id + # Use standard number for fixed slit shutter id slitlet_id = slit_name shutter_id = FIXED_SLIT_NUMS[slit_name] - 1 xcen = ycen = 0 quadrant = 5 - # no additional margin for fixed slit bounding boxes + # No additional margin for fixed slit bounding boxes ymin = ylow ymax = yhigh - # source position and id + # Source position and id if n_main_shutter == 1: - # source is marked primary + # Source is marked primary source_id = slitlet['source_id'] source_xpos = slitlet['estimated_source_in_shutter_x'] source_ypos = slitlet['estimated_source_in_shutter_y'] log.info(f'Found fixed slit {slitlet_id} with source_id = {source_id}.') + + # Get source info for this slitlet: + # note that slits with a real source assigned have source_id > 0, + # while slits with source_id < 0 contain "virtual" sources + try: + source_name, source_alias, stellarity, source_ra, source_dec = [ + (s['source_name'], s['alias'], s['stellarity'], s['ra'], s['dec']) + for s in msa_source if s['source_id'] == source_id][0] + except IndexError: + log.warning("Could not retrieve source info from MSA file") + + if source_id < 0: + log.info(f'Slitlet {slitlet_id} contains virtual source, ' + f'with source_id={source_id}') + elif n_background == 1: - # source is background only - source_id = _get_bkg_source_id(bkg_counter, max_source_id) + # Source is background only: + # assign a unique id based on the slitlet_id + source_id = slitlet_id source_xpos = 0.5 source_ypos = 0.5 + source_name = f"{prog_id}_BKG{slitlet_id}" + source_alias = "BKG{}".format(slitlet_id) + stellarity = 0.0 + source_ra = 0.0 + source_dec = 0.0 log.info(f'Slitlet_id {slitlet_id} is background only; ' f'assigned source_id = {source_id}.') - bkg_counter += 1 else: log.info(f'Fixed slit {slitlet_id} is neither background nor source; ' f'skipping it.') @@ -678,7 +697,7 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, slitlet_id, msa_metadata_id, dither_position)) log.warning(message) message = ("MSA configuration file has an unsupported " - "fixed slit configuration") + "fixed slit configuration.") log.warning(message) msa_file.close() raise MSAFileError(message) @@ -731,6 +750,7 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, ymin = -(-ylow + margin) + (jmin - j) * 1.15 # Get the source_id from the primary shutter entry + source_id = None for i in range(len(slitlets_sid)): if slitlets_sid[i]['primary_source'] == 'Y': source_id = slitlets_sid[i]['source_id'] From 4cc91417b99e1503d5d10e29e74f09bebae587fd Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 3 Jun 2024 14:51:50 -0400 Subject: [PATCH 32/44] Updates for MOS/FS: no background or slitlet_id expected for FS --- jwst/assign_wcs/nirspec.py | 122 +++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 52 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index 3812b2c575..6b83bd5f61 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -603,26 +603,48 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, log.info(f'Retrieving open MSA slitlets for msa_metadata_id = {msa_metadata_id} ' f'and dither_index = {dither_position}') - # Get the unique slitlet_ids - slitlet_ids_unique = list(set([x['slitlet_id'] for x in msa_data])) + # Sort the MSA rows by slitlet_id + slitlet_sets = {} + for row in msa_data: + # Check for fixed slit: if set, then slitlet_id is null + is_fs = False + try: + fixed_slit = row['fixed_slit'] + if (fixed_slit in FIXED_SLIT_NUMS.keys() + and fixed_slit != 'NONE'): + is_fs = True + except (IndexError, ValueError, KeyError): + # May be old-style MSA file without a fixed_slit column + fixed_slit = None + + if is_fs: + # Fixed slit - use the slit name as the ID + slitlet_id = fixed_slit + else: + # MSA - use the slitlet ID + slitlet_id = row['slitlet_id'] - # add a margin to the slit y limits + # Append the row for the slitlet + if slitlet_id in slitlet_sets: + slitlet_sets[slitlet_id].append(row) + else: + slitlet_sets[slitlet_id] = [row] + + # Add a margin to the slit y limits margin = 0.5 # Now let's look at each unique MOS slitlet id - for slitlet_id in slitlet_ids_unique: - # Get the rows of shutter info for the current slitlet_id - slitlets_sid = [x for x in msa_data if x['slitlet_id'] == slitlet_id] - open_shutters = [x['shutter_column'] for x in slitlets_sid] + for slitlet_id, slitlet_rows in slitlet_sets.items(): + # Get the open shutter information from the slitlet rows + open_shutters = [x['shutter_column'] for x in slitlet_rows] # How many shutters in the slitlet are labeled as "main" or "primary"? # How many are labeled background? - n_main_shutter = len([s for s in slitlets_sid if s['primary_source'] == 'Y']) - n_background = len([s for s in slitlets_sid if s['background'] == 'Y']) + n_main_shutter = len([s for s in slitlet_rows if s['primary_source'] == 'Y']) # Check for fixed slit sources defined in the MSA file - is_fs = [False] * len(slitlets_sid) - for i, slitlet in enumerate(slitlets_sid): + is_fs = [False] * len(slitlet_rows) + for i, slitlet in enumerate(slitlet_rows): try: if (slitlet['fixed_slit'] in FIXED_SLIT_NUMS.keys() and slitlet['fixed_slit'] != 'NONE'): @@ -634,15 +656,13 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, # In the next part we need to calculate, find, or determine 5 things for each slit: # quadrant, xcen, ycen, ymin, ymax - # First check for a fixed slit - if all(is_fs) and len(slitlets_sid) == 1: + # First, check for a fixed slit + if all(is_fs) and len(slitlet_rows) == 1: # One fixed slit open for the source - slitlet = slitlets_sid[0] - slit_name = slitlet['fixed_slit'] + slitlet = slitlet_rows[0] - # Use standard number for fixed slit shutter id - slitlet_id = slit_name - shutter_id = FIXED_SLIT_NUMS[slit_name] - 1 + # Use a standard number for fixed slit shutter id + shutter_id = FIXED_SLIT_NUMS[slitlet_id] - 1 xcen = ycen = 0 quadrant = 5 @@ -666,28 +686,17 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, (s['source_name'], s['alias'], s['stellarity'], s['ra'], s['dec']) for s in msa_source if s['source_id'] == source_id][0] except IndexError: + # Missing source information: assign a virtual source name log.warning("Could not retrieve source info from MSA file") + source_name = f"{prog_id}_VRT{slitlet_id}" + source_alias = "VRT{}".format(slitlet_id) + stellarity = 0.0 + source_ra = 0.0 + source_dec = 0.0 - if source_id < 0: - log.info(f'Slitlet {slitlet_id} contains virtual source, ' - f'with source_id={source_id}') - - elif n_background == 1: - # Source is background only: - # assign a unique id based on the slitlet_id - source_id = slitlet_id - source_xpos = 0.5 - source_ypos = 0.5 - source_name = f"{prog_id}_BKG{slitlet_id}" - source_alias = "BKG{}".format(slitlet_id) - stellarity = 0.0 - source_ra = 0.0 - source_dec = 0.0 - log.info(f'Slitlet_id {slitlet_id} is background only; ' - f'assigned source_id = {source_id}.') else: - log.info(f'Fixed slit {slitlet_id} is neither background nor source; ' - f'skipping it.') + log.warning(f'Fixed slit {slitlet_id} is not a primary source; ' + f'skipping it.') continue elif any(is_fs): @@ -708,14 +717,14 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, if len(open_shutters) == 1: jmin = jmax = j = open_shutters[0] else: - jmin = min([s['shutter_column'] for s in slitlets_sid]) - jmax = max([s['shutter_column'] for s in slitlets_sid]) + jmin = min([s['shutter_column'] for s in slitlet_rows]) + jmax = max([s['shutter_column'] for s in slitlet_rows]) j = jmin + (jmax - jmin) // 2 ymax = yhigh + margin + (jmax - j) * 1.15 ymin = -(-ylow + margin) + (jmin - j) * 1.15 - quadrant = slitlets_sid[0]['shutter_quadrant'] + quadrant = slitlet_rows[0]['shutter_quadrant'] ycen = j - xcen = slitlets_sid[0]['shutter_row'] # grab the first as they are all the same + xcen = slitlet_rows[0]['shutter_row'] # grab the first as they are all the same shutter_id = xcen + (ycen - 1) * 365 # shutter numbers in MSA file are 1-indexed # Background slits all have source_id=0 in the msa_file, @@ -739,21 +748,21 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, (s['shutter_row'], s['shutter_column'], s['shutter_quadrant'], s['estimated_source_in_shutter_x'], s['estimated_source_in_shutter_y']) - for s in slitlets_sid if s['background'] == 'N'][0] + for s in slitlet_rows if s['background'] == 'N'][0] shutter_id = xcen + (ycen - 1) * 365 # shutter numbers in MSA file are 1-indexed # y-size - jmin = min([s['shutter_column'] for s in slitlets_sid]) - jmax = max([s['shutter_column'] for s in slitlets_sid]) + jmin = min([s['shutter_column'] for s in slitlet_rows]) + jmax = max([s['shutter_column'] for s in slitlet_rows]) j = ycen ymax = yhigh + margin + (jmax - j) * 1.15 ymin = -(-ylow + margin) + (jmin - j) * 1.15 # Get the source_id from the primary shutter entry source_id = None - for i in range(len(slitlets_sid)): - if slitlets_sid[i]['primary_source'] == 'Y': - source_id = slitlets_sid[i]['source_id'] + for i in range(len(slitlet_rows)): + if slitlet_rows[i]['primary_source'] == 'Y': + source_id = slitlet_rows[i]['source_id'] # Get source info for this slitlet; # note that slits with a real source assigned have source_id > 0, @@ -763,17 +772,24 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, (s['source_name'], s['alias'], s['stellarity'], s['ra'], s['dec']) for s in msa_source if s['source_id'] == source_id][0] except IndexError: + # Missing source information: assign a virtual source name log.warning("Could not retrieve source info from MSA file") + source_name = f"{prog_id}_VRT{slitlet_id}" + source_alias = "VRT{}".format(slitlet_id) + stellarity = 0.0 + source_ra = 0.0 + source_dec = 0.0 if source_id < 0: - log.info(f'Slitlet {slitlet_id} contains virtual source, with source_id={source_id}') + log.info(f'Slitlet {slitlet_id} contains virtual source, ' + f'with source_id={source_id}') # More than 1 main shutter: Not allowed! else: message = ("For slitlet_id = {}, metadata_id = {}, " "and dither_index = {}".format(slitlet_id, msa_metadata_id, dither_position)) log.warning(message) - message = ("MSA configuration file has more than 1 shutter with primary source") + message = "MSA configuration file has more than 1 shutter with primary source" log.warning(message) msa_file.close() raise MSAFileError(message) @@ -794,9 +810,11 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, # Create the shutter_state string all_shutters = _shutter_id_to_str(open_shutters, ycen) - slitlets.append(Slit(slitlet_id, shutter_id, dither_position, xcen, ycen, ymin, ymax, - quadrant, source_id, all_shutters, source_name, source_alias, - stellarity, source_xpos, source_ypos, source_ra, source_dec)) + slit_parameters = (slitlet_id, shutter_id, dither_position, xcen, ycen, ymin, ymax, + quadrant, source_id, all_shutters, source_name, source_alias, + stellarity, source_xpos, source_ypos, source_ra, source_dec) + log.debug(f'Appending slit: {slit_parameters}') + slitlets.append(Slit(*slit_parameters)) msa_file.close() return slitlets From c74bf9d4147da66d834faf500f9fe458fd396d13 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 3 Jun 2024 14:52:01 -0400 Subject: [PATCH 33/44] Test updates --- jwst/assign_wcs/tests/test_nirspec.py | 26 +++++++++++--------------- jwst/extract_2d/tests/test_nirspec.py | 1 + 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/jwst/assign_wcs/tests/test_nirspec.py b/jwst/assign_wcs/tests/test_nirspec.py index 8b4e49dad9..281e467bdc 100644 --- a/jwst/assign_wcs/tests/test_nirspec.py +++ b/jwst/assign_wcs/tests/test_nirspec.py @@ -360,31 +360,26 @@ def test_msa_fs_configuration(): """ Test the get_open_msa_slits function with FS and MSA slits defined. """ + prog_id = '1234' msa_meta_id = 12 msaconfl = get_file_path('msa_fs_configuration.fits') dither_position = 1 - slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, - slit_y_range=[-.5, .5]) + slitlet_info = nirspec.get_open_msa_slits( + prog_id, msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) # MSA slit: reads in as normal ref_slit = trmodels.Slit(55, 9376, 1, 251, 26, -5.6, 1.0, 4, 1, '1111x', '95065_1', '2122', 0.13, -0.31716078999999997, -0.18092266) - _compare_slits(slitlet_info[-1], ref_slit) + _compare_slits(slitlet_info[0], ref_slit) # FS primary: S200A1, shutter id 0, quadrant 5 ref_slit = trmodels.Slit('S200A1', 0, 1, 0, 0, -0.5, 0.5, 5, 3, 'x', '95065_3', '3', 1.0, -0.161, -0.229, 53.139904, -27.805002) - _compare_slits(slitlet_info[0], ref_slit) - - # FS background: S200A2, shutter id 1, quadrant 5 - ref_slit = trmodels.Slit('S200A2', 1, 1, 0, 0, -0.5, 0.5, 5, 8, 'x', - 'background_S200A2', 'bkg_S200A2', - 0.0, 0.0, 0.0, 0.0, 0.0) _compare_slits(slitlet_info[1], ref_slit) - # FS S200B1 is in the MSA file but neither background nor primary: - # it should not be defined. The rest should be there. - fs_slits_defined = ['S200A1', 'S200A2', 'S400A1', 'S1600A1'] + # The remaining fixed slits may be in the MSA file but not primary: + # they should not be defined. + fs_slits_defined = ['S200A1'] n_fixed = 0 for slit in slitlet_info: if slit.quadrant == 5: @@ -397,7 +392,7 @@ def test_msa_fs_configuration_unsupported(tmp_path): """ Test the get_open_msa_slits function with unsupported FS defined. """ - # modify an existing MSA file to add a bad ros + # modify an existing MSA file to add a bad row msaconfl = get_file_path('msa_fs_configuration.fits') bad_confl = tmp_path / 'bad_msa_fs_configuration.fits' shutil.copy(msaconfl, bad_confl) @@ -409,11 +404,12 @@ def test_msa_fs_configuration_unsupported(tmp_path): msa_hdu_list[2].name = 'SHUTTER_INFO' msa_hdu_list.writeto(bad_confl, overwrite=True) + prog_id = '1234' msa_meta_id = 12 dither_position = 1 with pytest.raises(MSAFileError, match='unsupported fixed slit'): - nirspec.get_open_msa_slits(bad_confl, msa_meta_id, dither_position, - slit_y_range=[-.5, .5]) + nirspec.get_open_msa_slits( + prog_id, bad_confl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) diff --git a/jwst/extract_2d/tests/test_nirspec.py b/jwst/extract_2d/tests/test_nirspec.py index 81e593ac99..ab6c2747de 100644 --- a/jwst/extract_2d/tests/test_nirspec.py +++ b/jwst/extract_2d/tests/test_nirspec.py @@ -33,6 +33,7 @@ def create_nirspec_hdul(detector='NRS1', grating='G395M', filter_name='F290LP', phdu.header['DETECTOR'] = detector phdu.header['FILTER'] = filter_name phdu.header['GRATING'] = grating + phdu.header['PROGRAM'] = '01234' phdu.header['TIME-OBS'] = '8:59:37' phdu.header['DATE-OBS'] = '2023-01-05' phdu.header['EXP_TYPE'] = exptype From 05d2096247b70f9fe62807d6687726b855e7c136 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 3 Jun 2024 16:21:34 -0400 Subject: [PATCH 34/44] Add test for missing source data --- jwst/assign_wcs/tests/test_nirspec.py | 37 ++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/jwst/assign_wcs/tests/test_nirspec.py b/jwst/assign_wcs/tests/test_nirspec.py index 281e467bdc..2f154cd878 100644 --- a/jwst/assign_wcs/tests/test_nirspec.py +++ b/jwst/assign_wcs/tests/test_nirspec.py @@ -394,7 +394,7 @@ def test_msa_fs_configuration_unsupported(tmp_path): """ # modify an existing MSA file to add a bad row msaconfl = get_file_path('msa_fs_configuration.fits') - bad_confl = tmp_path / 'bad_msa_fs_configuration.fits' + bad_confl = str(tmp_path / 'bad_msa_fs_configuration.fits') shutil.copy(msaconfl, bad_confl) with fits.open(bad_confl) as msa_hdu_list: @@ -412,6 +412,41 @@ def test_msa_fs_configuration_unsupported(tmp_path): prog_id, bad_confl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) +def test_msa_missing_source(tmp_path): + """ + Test the get_open_msa_slits function with missing source information. + """ + # modify an existing MSA file to remove source info + msaconfl = get_file_path('msa_fs_configuration.fits') + bad_confl = str(tmp_path / 'bad_msa_fs_configuration.fits') + shutil.copy(msaconfl, bad_confl) + + with fits.open(bad_confl) as msa_hdu_list: + source_table = table.Table(msa_hdu_list['SOURCE_INFO'].data) + source_table.remove_rows(slice(0, -1)) + msa_hdu_list['SOURCE_INFO'] = fits.table_to_hdu(source_table) + msa_hdu_list[3].name = 'SOURCE_INFO' + msa_hdu_list.writeto(bad_confl, overwrite=True) + + prog_id = '1234' + msa_meta_id = 12 + dither_position = 1 + + slitlet_info = nirspec.get_open_msa_slits( + prog_id, bad_confl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) + + # MSA slit: virtual source name assigned + ref_slit = trmodels.Slit(55, 9376, 1, 251, 26, -5.6, 1.0, 4, 1, '1111x', + '1234_VRT55', 'VRT55', 0.0, + -0.31716078999999997, -0.18092266, 0.0, 0.0) + _compare_slits(slitlet_info[0], ref_slit) + + # FS primary: S200A1, virtual source name assigned + ref_slit = trmodels.Slit('S200A1', 0, 1, 0, 0, -0.5, 0.5, 5, 3, 'x', + '1234_VRTS200A1', 'VRTS200A1', 0.0, + -0.161, -0.229, 0.0, 0.0) + _compare_slits(slitlet_info[1], ref_slit) + open_shutters = [[24], [23, 24], [22, 23, 25, 27], [22, 23, 25, 27, 28]] main_shutter = [24, 23, 25, 28] From ee566457c1cd5cb19bcdeddf563dbe16a3403fb6 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 3 Jun 2024 16:55:49 -0400 Subject: [PATCH 35/44] Update source ids in regression test --- jwst/regtest/test_nirspec_mos_fs_spec3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jwst/regtest/test_nirspec_mos_fs_spec3.py b/jwst/regtest/test_nirspec_mos_fs_spec3.py index 2543558cce..33e82e5a2c 100644 --- a/jwst/regtest/test_nirspec_mos_fs_spec3.py +++ b/jwst/regtest/test_nirspec_mos_fs_spec3.py @@ -19,9 +19,9 @@ def run_pipeline(rtdata_module): @pytest.mark.bigdata @pytest.mark.parametrize("suffix", ["cal", "crf", "s2d", "x1d"]) -@pytest.mark.parametrize("source_id", ["s01354", "s12105", "s34946", - "s34949", "s34950", "s34951", "s34952", - "s34953", "s34954", "s34955"]) +@pytest.mark.parametrize("source_id", ["b000000003", "b000000004", "b000000048", + "b000000052", "s000001354", "s000012105", + "s000034946"]) def test_nirspec_mos_fs_spec3(run_pipeline, suffix, source_id, fitsdiff_default_kwargs): """Check results of calwebb_spec3""" rtdata = run_pipeline From ec8b8ce0bc7eda749389739006e70ffa323e63ce Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 3 Jun 2024 16:56:46 -0400 Subject: [PATCH 36/44] Remove accidental commit --- stpipe-log.cfg | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 stpipe-log.cfg diff --git a/stpipe-log.cfg b/stpipe-log.cfg deleted file mode 100644 index 41d3cf4c99..0000000000 --- a/stpipe-log.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[*] -level = DEBUG -handler = stdout From 88a21beb1c4ac2b4fac07e1640e0ba276560c241 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 4 Jun 2024 14:16:14 -0400 Subject: [PATCH 37/44] Remove barshadow processing for FS slits defined in MSA --- jwst/pipeline/calwebb_spec2.py | 2 +- jwst/regtest/test_nirspec_mos_fs_spec2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jwst/pipeline/calwebb_spec2.py b/jwst/pipeline/calwebb_spec2.py index 9457d62ee1..2ae7d2d0cc 100644 --- a/jwst/pipeline/calwebb_spec2.py +++ b/jwst/pipeline/calwebb_spec2.py @@ -583,7 +583,7 @@ def _process_nirspec_msa_slits(self, data): # Run each step with an alternate suffix, # to avoid overwriting previous products if save_results=True - fs_steps = ['wavecorr', 'flat_field', 'pathloss', 'barshadow', 'photom'] + fs_steps = ['wavecorr', 'flat_field', 'pathloss', 'photom'] for step_name in fs_steps: # Set suffix step = getattr(self, step_name) diff --git a/jwst/regtest/test_nirspec_mos_fs_spec2.py b/jwst/regtest/test_nirspec_mos_fs_spec2.py index bcceb0d4f6..82dd419888 100644 --- a/jwst/regtest/test_nirspec_mos_fs_spec2.py +++ b/jwst/regtest/test_nirspec_mos_fs_spec2.py @@ -37,7 +37,7 @@ def run_pipeline(rtdata_module): @pytest.mark.parametrize("suffix", [ "assign_wcs", "msa_flagging", "extract_2d", "srctype", "master_background_mos", "wavecorr", "flat_field", "pathloss", "barshadow", - "wavecorr_fs", "flat_field_fs", "pathloss_fs", "barshadow_fs", + "wavecorr_fs", "flat_field_fs", "pathloss_fs", "cal", "s2d", "x1d"]) def test_nirspec_mos_fs_spec2(run_pipeline, fitsdiff_default_kwargs, suffix): """Regression test for calwebb_spec2 on a NIRSpec MOS/FS exposure.""" From 6973992e3bd27d5fb8ac896d575bd4cc632e914d Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 4 Jun 2024 15:06:50 -0400 Subject: [PATCH 38/44] Update change notes --- CHANGES.rst | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index c353bf0c74..7d6e63abfe 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -24,6 +24,10 @@ assign_wcs data file to properly handle background and virtual slits, and assign appropriate meta data to them for use downstream. [#8442] +- Added handling for fixed slit sources defined in a MSA metadata file, for combined + NIRSpec MOS and fixed slit observations. Slits are now appended to the data + product in the order they appear in the MSA file. [#8467] + associations ------------ @@ -61,6 +65,10 @@ exp_to_source in order to support changes in `source_id` handling for NIRSpec MOS exposures that contain background and virtual slits. [#8442] +- Update the top-level model exposure type from the slit exposure type, + to support processing for combined NIRSpec MOS and fixed slit + observations. [#8467] + extract_1d ---------- @@ -81,11 +89,19 @@ extract_1d - Add propagation of uncertainty when annular backgrounds are subtracted from source spectra during IFU spectral extraction. [#8515] +- Removed a check for the primary slit for NIRSpec fixed slit mode: + all slits containing point sources are now handled consistently, + whether they are marked primary or not. [#8467] + extract_2d ---------- - Added handling for NIRCam GRISM time series pointing offsets. [#8449] +- Added support for slit names that have string values instead of integer + values, necessary for processing combined NIRSpec MOS and fixed slit + data products. [#8467] + flat_field ---------- @@ -95,6 +111,10 @@ flat_field - Update NIRSpec flatfield code for all modes to ensure SCI=ERR=NaN wherever the DO_NOT_USE flag is set in the DQ array. [#8463] +- Removed a check for the primary slit for NIRSpec fixed slit mode: + all slits containing point sources are now handled consistently, + whether they are marked primary or not. [#8467] + general ------- @@ -103,6 +123,25 @@ general - Increase minimum required scipy. [#8441] +master_background +----------------- + +- Removed a check for the primary slit for NIRSpec fixed slit mode: + all slits containing point sources are now handled consistently, + whether they are marked primary or not. [#8467] + +- Disabled support for master background correction for NIRSpec MOS + slits in the ``master_background``, called in ``calwebb_spec3``. + Master background correction for MOS mode should be performed + via ``master_background_mos``, called in ``calwebb_spec2``. [#8467] + +nsclean +------- + +- Added a check for combined NIRSpec MOS and fixed slit products: if fixed + slits are defined in a MOS product, the central fixed slit quadrant + is not automatically masked. [#8467] + outlier_detection ----------------- @@ -135,6 +174,10 @@ photom - Ensure that NaNs in MRS photom files are not replaced with ones by pipeline code for consistency with other modes [#8453] +- Removed a check for the primary slit for NIRSpec fixed slit mode: + all slits containing point sources are now handled consistently, + whether they are marked primary or not. [#8467] + pipeline -------- @@ -149,6 +192,12 @@ pipeline virtual ("v") slits and the construction of output file names for each type. [#8442] +- Added ``calwebb_spec2`` pipeline handling for combined NIRSpec MOS and + fixed slit observations. Steps that require different reference files + for MOS and FS are run twice, first for all MOS slits, then for all + FS slits. Final output products (``cal``, ``s2d``, ``x1d``) contain the + combined products. [#8467] + pixel_replace ------------- @@ -217,6 +266,14 @@ tweakreg - Change code default to use IRAF StarFinder instead of DAO StarFinder [#8487] +wavecorr +-------- + +- Added a check for fixed slits that already have source position information, + assigned via a MSA metafile, for combined NIRSpec MOS and fixed slit processing. + Point source position is calculated from dither offsets only for standard + fixed slit processing. [#8467] + wfss_contam ----------- From 9fb36a313298bf02ba80cbd115c2dcff404cd4dd Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 7 Jun 2024 10:17:54 -0400 Subject: [PATCH 39/44] Code style fixes --- jwst/assign_wcs/tests/test_nirspec.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/jwst/assign_wcs/tests/test_nirspec.py b/jwst/assign_wcs/tests/test_nirspec.py index f706c33ce9..3c6d3591f5 100644 --- a/jwst/assign_wcs/tests/test_nirspec.py +++ b/jwst/assign_wcs/tests/test_nirspec.py @@ -2,28 +2,25 @@ Test functions for NIRSPEC WCS - all modes. """ import functools -import shutil -from math import cos, sin import os.path import shutil +from math import cos, sin -import pytest +import astropy.units as u +import astropy.coordinates as coords import numpy as np -from numpy.testing import assert_allclose +import pytest from astropy.io import fits from astropy.modeling import models as astmodels from astropy import table from astropy import wcs as astwcs -import astropy.units as u -import astropy.coordinates as coords -from gwcs import wcs -from gwcs import wcstools +from gwcs import wcs, wcstools +from numpy.testing import assert_allclose from stdatamodels.jwst import datamodels from stdatamodels.jwst.transforms import models as trmodels -from jwst.assign_wcs import nirspec -from jwst.assign_wcs import assign_wcs_step +from jwst.assign_wcs import nirspec, assign_wcs_step from jwst.assign_wcs.tests import data from jwst.assign_wcs.util import MSAFileError, in_ifu_slice From 5b91b44d1feb5e3006f639f0434046a14f77168a Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 7 Jun 2024 11:12:50 -0400 Subject: [PATCH 40/44] Clean up code comments --- jwst/assign_wcs/nirspec.py | 3 +-- jwst/wavecorr/wavecorr.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index 5f33e15709..740782a099 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -633,13 +633,12 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, # Add a margin to the slit y limits margin = 0.5 - # Now let's look at each unique MOS slitlet id + # Now let's look at each unique slitlet id for slitlet_id, slitlet_rows in slitlet_sets.items(): # Get the open shutter information from the slitlet rows open_shutters = [x['shutter_column'] for x in slitlet_rows] # How many shutters in the slitlet are labeled as "main" or "primary"? - # How many are labeled background? n_main_shutter = len([s for s in slitlet_rows if s['primary_source'] == 'Y']) # Check for fixed slit sources defined in the MSA file diff --git a/jwst/wavecorr/wavecorr.py b/jwst/wavecorr/wavecorr.py index 6800663f43..8bfe553555 100644 --- a/jwst/wavecorr/wavecorr.py +++ b/jwst/wavecorr/wavecorr.py @@ -237,7 +237,6 @@ def _is_msa_fixed_slit(slit): """ # Fixed slits defined via MSA files in MOS/FS combination # processing will have a non-empty shutter state - # TODO: determine if there's a better test if (not hasattr(slit, 'shutter_state') or slit.shutter_state is None or slit.shutter_state == ""): From 1cc17487f9c3ce9c8b61dd81c34ebb03d44db0f6 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 7 Jun 2024 18:49:39 -0400 Subject: [PATCH 41/44] Documentation updates for NIRSpec MOS/FS --- docs/jwst/data_products/msa_metadata.rst | 73 +++++++++++++-------- docs/jwst/flatfield/main.rst | 17 +++-- docs/jwst/master_background/description.rst | 26 ++++---- docs/jwst/photom/main.rst | 7 ++ docs/jwst/pipeline/calwebb_spec2.rst | 42 ++++++++++++ docs/jwst/wavecorr/description.rst | 8 ++- jwst/srctype/srctype.py | 2 +- 7 files changed, 127 insertions(+), 48 deletions(-) diff --git a/docs/jwst/data_products/msa_metadata.rst b/docs/jwst/data_products/msa_metadata.rst index b251402c80..5173000aa0 100644 --- a/docs/jwst/data_products/msa_metadata.rst +++ b/docs/jwst/data_products/msa_metadata.rst @@ -43,7 +43,7 @@ The overall structure of the MSA FITS file is as follows: +-----+---------------+----------+-----------+--------------------+ | 1 | SHUTTER_IMAGE | IMAGE | float32 | 342 x 730 | +-----+---------------+----------+-----------+--------------------+ -| 2 | SHUTTER_INFO | BINTABLE | N/A | variable x 12 cols | +| 2 | SHUTTER_INFO | BINTABLE | N/A | variable x 13 cols | +-----+---------------+----------+-----------+--------------------+ | 3 | SOURCE_INFO | BINTABLE | N/A | variable x 8 cols | +-----+---------------+----------+-----------+--------------------+ @@ -89,6 +89,8 @@ The structure of the ``SHUTTER_INFO`` table extension is as follows: +-------------------------------+-----------+----------------------+ | PRIMARY_SOURCE | string | Primary source flag | +-------------------------------+-----------+----------------------+ +| FIXED_SLIT | string | Fixed slit name | ++-------------------------------+-----------+----------------------+ - SLITLET_ID: integer ID of each slitlet consisting of one or more open shutters. @@ -102,8 +104,8 @@ The structure of the ``SHUTTER_INFO`` table extension is as follows: quadrant. - SOURCE_ID: unique integer ID for each source in each slitlet, used for matching to an entry in the ``SOURCE_INFO`` table. -- BACKGROUND: boolean indicating whether the shutter is open to background - only or contains a known source. +- BACKGROUND: Y or N. Y indicates that the shutter is open to background + only. - SHUTTER_STATE: OPEN or CLOSED. Generally will always be OPEN, unless the shutter is part of a long slit that is not contiguous. - ESTIMATED_SOURCE_IN_SHUTTER_X/Y: the position of the source within the @@ -111,8 +113,11 @@ The structure of the ``SHUTTER_INFO`` table extension is as follows: and 1,1 is upper-right, as planned in the MPT. - DITHER_POINT_INDEX: integer index of the nod sequence; matches to header keyword `PATT_NUM`. -- PRIMARY_SOURCE: boolean indicating whether the shutter contains the +- PRIMARY_SOURCE: Y or N. Y indicates that the shutter contains the primary science source. +- FIXED_SLIT: string name of a fixed slit containing the source; set to + NONE for MSA slitlets. This column may not appear in older versions of + the MSA metadata files. It is the :ref:`assign_wcs ` step in the :ref:`calwebb_spec2 ` pipeline that opens and loads all @@ -140,29 +145,29 @@ slitlet 2 is comprised of 3 shutters. Because a 3-point nod pattern has been use there are 3 different sets of metadata for each slitlet (one set for each dither/nod position) and hence a total of 9 entries (3 shutters x 3 dithers). -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| Slit | Meta | | | | Src | | | X | Y | Dith | Pri | -| | | | | | | | | | | | | -| ID | ID | Quad | Row | Col | ID | Bkg | State | pos | pos | Pt | Src | -+======+======+======+=====+=====+========+=====+=======+=======+=======+======+=====+ -| 2 | 1 | 2 | 10 | 154 | 0 | Y | OPEN | NaN | NaN | 1 | N | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 155 | 42 | N | OPEN | 0.399 | 0.702 | 1 | Y | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 156 | 0 | Y | OPEN | NaN | NaN | 1 | N | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 154 | 42 | N | OPEN | 0.410 | 0.710 | 2 | Y | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 155 | 0 | Y | OPEN | NaN | NaN | 2 | N | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 156 | 0 | Y | OPEN | NaN | NaN | 2 | N | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 154 | 0 | Y | OPEN | NaN | NaN | 3 | N | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 155 | 0 | Y | OPEN | NaN | NaN | 3 | N | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 156 | 42 | N | OPEN | 0.389 | 0.718 | 3 | Y | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| Slit | Meta | | | | Src | | | X | Y | Dith | Pri | Fxd | +| | | | | | | | | | | | | | +| ID | ID | Quad | Row | Col | ID | Bkg | State | pos | pos | Pt | Src | Slit | ++======+======+======+=====+=====+========+=====+=======+=======+=======+======+=====+======+ +| 2 | 1 | 2 | 10 | 154 | 0 | Y | OPEN | NaN | NaN | 1 | N | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 155 | 42 | N | OPEN | 0.399 | 0.702 | 1 | Y | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 156 | 0 | Y | OPEN | NaN | NaN | 1 | N | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 154 | 42 | N | OPEN | 0.410 | 0.710 | 2 | Y | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 155 | 0 | Y | OPEN | NaN | NaN | 2 | N | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 156 | 0 | Y | OPEN | NaN | NaN | 2 | N | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 154 | 0 | Y | OPEN | NaN | NaN | 3 | N | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 155 | 0 | Y | OPEN | NaN | NaN | 3 | N | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 156 | 42 | N | OPEN | 0.389 | 0.718 | 3 | Y | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ The values in the `slitlet_id` column show that we're only looking at table rows for slitlet 2, all of which come from MSA configuration (`msa_metadata_id`) 1. @@ -229,6 +234,20 @@ from a virtual slitlet with `source_id` = -42 will look like: jw12345-o066_v000000042_nirspec_f170lp_g235m_x1d.fits +Fixed Slits +~~~~~~~~~~~ + +It is possible to plan fixed slit sources alongside standard MOS targets. In this case, +a unique `slitlet_id` is not assigned in the MSA file. Instead, the slit is identified +by the value of the `fixed_slit` column. This value may be set to any of the NIRSpec +fixed slit names used for science: S200A1, S200A2, S400A1, or S1600A1. + +Fixed slit targets must always have `primary_source` = "Y" and `background` = "N". +They will never be extracted as background sources. + +The `shutter_quadrant`, `shutter_row`, and `shutter_column` fields are set to placeholder +values. All other values have the same meaning and values as for MSA slitlets. + The SOURCE_INFO Metadata ------------------------ diff --git a/docs/jwst/flatfield/main.rst b/docs/jwst/flatfield/main.rst index 3cd4111764..38df509446 100644 --- a/docs/jwst/flatfield/main.rst +++ b/docs/jwst/flatfield/main.rst @@ -78,12 +78,12 @@ pixel. This interpolation requires knowledge of the dispersion direction, which is read from keyword "DISPAXIS." See the Reference File section for further details. -For NIRSpec Fixed-Slit and MOS exposures, an on-the-fly flat-field is +For NIRSpec Fixed Slit (FS) and MOS exposures, an on-the-fly flat-field is constructed to match each of the slits/slitlets contained in the science exposure. For NIRSpec IFU exposures, a single full-frame flat-field is constructed, which is applied to the entire science image. -NIRSpec NRS_BRIGHTOBJ data are processed just like NIRSpec Fixed-Slit +NIRSpec NRS_BRIGHTOBJ data are processed just like NIRSpec fixed slit data, except that NRS_BRIGHTOBJ data are stored in a CubeModel, rather than a MultiSlitModel. A 2-D flat-field image is constructed on-the-fly as usual, but this image is then divided into each plane of @@ -92,9 +92,9 @@ the 3-D science data arrays. In all cases, there is a step option that allows for saving the on-the-fly flatfield to a file, if desired. -NIRSpec Fixed-Slit Primary Slit +NIRSpec Fixed Slit Primary Slit ------------------------------- -The primary slit in a NIRSpec fixed-slit exposure receives special handling. +The primary slit in a NIRSpec fixed slit exposure receives special handling. If the primary slit, as given by the "FXD_SLIT" keyword value, contains a point source, as given by the "SRCTYPE" keyword, it is necessary to know the flatfield conversion factors for both a point source and a uniform source @@ -116,8 +116,15 @@ applied by the :ref:`wavecorr ` step to account for any source mis-centering in the slit and the flatfield conversion factors are wavelength-dependent. A uniform source does not require wavelength corrections and hence the flatfield conversions will differ for point and uniform -sources. Any secondary slits that may be included in a fixed-slit exposure +sources. Any secondary slits that may be included in a fixed slit exposure do not have source centering information available, so the :ref:`wavecorr ` step is not applied, and hence there's no difference between the point source and uniform source flatfield conversions for those slits. + +Fixed slits planned as part of a combined MOS and FS observation are an +exception to this rule. These targets may each be identified as +point sources, with location information for each given in the +:ref:`MSA metadata file `. Point sources in fixed slits planned +this way are treated in the same manner as the primary fixed slit in standard +FS observations. diff --git a/docs/jwst/master_background/description.rst b/docs/jwst/master_background/description.rst index a436ecdc1d..2e21d58b0d 100644 --- a/docs/jwst/master_background/description.rst +++ b/docs/jwst/master_background/description.rst @@ -295,27 +295,25 @@ source centering within the slit, hence slits containing uniform sources receive the same flat-field and photometric calibrations as background spectra and therefore don't require corrections for those two calibrations. Furthermore, the source position in the slit is only known for the primary slit in an exposure, so -even if the secondary slits contain point sources, no wavelength correction can -be applied, and therefore again the flat-field and photometric calibrations are -the same as for background spectra. This means only the pathloss correction -difference between uniform and point sources needs to be accounted for in the -secondary slits. +secondary slits are always handled as extended sources, and no wavelength correction is +applied, and therefore again the flat-field and photometric calibrations are +the same as for background spectra. -Therefore if the primary slit (as given by the FXD_SLIT keyword) contains a point source -(as given by the SRCTYPE keyword) the corrections that need to be applied to the 2-D -master background for that slit are: +Fixed slits planned as part of a combined MOS and FS observation are an +exception to this rule. These targets may each be identified as +point sources, with location information for each given in the +:ref:`MSA metadata file `. Point sources in fixed slits planned +this way are treated in the same manner as the primary fixed slit in standard +FS observations. + +Therefore if a fixed slit contains a point source (as given by the SRCTYPE keyword) +the corrections that need to be applied to the 2-D master background for that slit are: .. math:: bkg(corr) = bkg &* [flatfield(uniform) / flatfield(point)]\\ &* [pathloss(uniform) / pathloss(point)]\\ &* [photom(point) / photom(uniform)] -For secondary slits that contain a point source, the correction applied to the -2-D master background is simply: - -.. math:: - bkg(corr) = bkg * pathloss(uniform) / pathloss(point) - The uniform and point source versions of the flat-field, pathloss, and photom corrections are retrieved from the input :ref:`cal ` product. They are computed and stored there during the execution of each of those steps diff --git a/docs/jwst/photom/main.rst b/docs/jwst/photom/main.rst index 1d98734cb5..b18cd82c29 100644 --- a/docs/jwst/photom/main.rst +++ b/docs/jwst/photom/main.rst @@ -114,6 +114,13 @@ do not have source centering information available, so the difference between the point source and uniform source photometric conversions for those slits. +Fixed slits planned as part of a combined MOS and FS observation are an +exception to this rule. These targets may each be identified as +point sources, with location information for each given in the +:ref:`MSA metadata file `. Point sources in fixed slits planned +this way are all treated in the same manner as the primary fixed slit in standard +FS observations. + Pixel Area Data ^^^^^^^^^^^^^^^ For all instrument modes other than NIRSpec the photom step loads a 2-D pixel diff --git a/docs/jwst/pipeline/calwebb_spec2.rst b/docs/jwst/pipeline/calwebb_spec2.rst index fa53774c89..aef4e85ec1 100644 --- a/docs/jwst/pipeline/calwebb_spec2.rst +++ b/docs/jwst/pipeline/calwebb_spec2.rst @@ -114,6 +114,48 @@ performed on the original (unresampled) data. The :ref:`cube_build ` step. +Combined NIRSpec MOS and FS Exposures +------------------------------------- + +It is possible to observe one or more fixed slit sources as part of a NIRSpec MOS +observation. Fixed slits observed this way are mostly handled as if they were +MOS slitlets: they are assigned a WCS, extracted from the full-frame image, calibrated, +and appended to the output data products. + +However, since FS and MOS modes require different pipeline steps and reference files +at various points in the ``calwebb_spec2`` pipeline, the processing path is not as +straightforward as for a standard MOS exposure. Internal to the pipeline, the data +product is sorted into MOS slits and FS slits. MOS slits are processed together first, +then FS slits are processed through the same steps. At the end of the processing, +the calibrated images and spectra are recombined into final data products containing +all observed slits. + +The detailed processing flow is as follows: + +- Fixed slits containing primary sources are identified in the input + :ref:`MSA metadata file ` in the :ref:`assign_wcs ` + step, via the "fixed_slit" column in the MSA SHUTTER_INFO table. +- All slits (MOS and FS) are processed together through the :ref:`srctype ` step. +- MOS slits are processed together through the :ref:`master_background `, + :ref:`wavecorr `, :ref:`flat_field `, + :ref:`pathloss `, :ref:`barshadow `, and + :ref:`photom ` steps. +- FS slits are processed together through the :ref:`wavecorr `, + :ref:`flat_field `, :ref:`pathloss `, and + :ref:`photom ` steps. If intermediate products from these + steps are saved, they will have an additional "_fs" suffix appended to + their file names. +- MOS and FS slits are recombined and processed together through the + :ref:`resample_spec ` step. +- MOS slits are processed through the :ref:`extract_1d ` step, + then FS slits are processed through the same step. The extracted spectra are + recombined into a final data product. + +The combined, calibrated output product for this mode may be used as input for +the :ref:`calwebb_spec3 ` pipeline. Since that pipeline sorts and +separates the data by source, the fixed slit and MOS targets are independently handled +through all pipeline steps with no further accommodation necessary. + NIRSpec Lamp Exposures ---------------------- diff --git a/docs/jwst/wavecorr/description.rst b/docs/jwst/wavecorr/description.rst index 999ee6a6db..6412ccbec5 100644 --- a/docs/jwst/wavecorr/description.rst +++ b/docs/jwst/wavecorr/description.rst @@ -43,7 +43,7 @@ estimate the fractional location of the source within the given slit. Note that this computation can only be performed for the primary slit in the exposure, which is given in the "FXD_SLIT" keyword. The positions of sources in any additional slits cannot be estimated and therefore -the wavelength correction is only applied to the primary slit. +the wavelength correction is only applied to the primary slit.\ :sup:`1` The estimated position of the source within the primary slit (in the dispersion direction) is then used in the same manner as described above @@ -52,3 +52,9 @@ grid for the primary slit. Upon successful completion of the step, the status keyword "S_WAVCOR" is set to "COMPLETE". + +:sup:`1`\ Note that fixed slits that are planned as part of a combined +MOS and FS observation do have *a priori* estimates of their source +locations, via the :ref:`MSA metadata file`. When available, +these source locations are directly used, instead of recomputing the source +position in the ``wavecorr`` step. diff --git a/jwst/srctype/srctype.py b/jwst/srctype/srctype.py index 46f192e533..8448f1109f 100644 --- a/jwst/srctype/srctype.py +++ b/jwst/srctype/srctype.py @@ -131,7 +131,7 @@ def set_source_type(input_model, source_type=None): # NIRSpec fixed-slit is a special case: Apply the source type # determined above to only the primary slit (the one in which # the target is located). Set all other slits to the default - # value, which for NRS_FIXEDSLIT is 'POINT'. + # value, which for NRS_FIXEDSLIT is 'EXTENDED'. default_type = 'EXTENDED' primary_slit = input_model.meta.instrument.fixed_slit log.debug(f' primary_slit = {primary_slit}') From 3f723b3cfe29239206572c01a7ad6ec741fe4323 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 10 Jun 2024 11:00:42 -0400 Subject: [PATCH 42/44] Update wavecorr test --- jwst/wavecorr/tests/test_wavecorr.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jwst/wavecorr/tests/test_wavecorr.py b/jwst/wavecorr/tests/test_wavecorr.py index 125d492a79..e7681fb6d2 100644 --- a/jwst/wavecorr/tests/test_wavecorr.py +++ b/jwst/wavecorr/tests/test_wavecorr.py @@ -190,8 +190,9 @@ def test_mos_slit_status(): im_src.slits[0].source_type = 'EXTENDED' im_wave = WavecorrStep.call(im_src) - # check that the step is recorded as completed - assert im_wave.meta.cal_step.wavecorr == 'COMPLETE' + # check that the step is recorded as skipped, + # since no slits were corrected + assert im_wave.meta.cal_step.wavecorr == 'SKIPPED' # check that the step is listed as skipped for extended mos sources assert im_wave.slits[0].meta.cal_step.wavecorr == 'SKIPPED' @@ -260,7 +261,7 @@ def test_wavecorr_fs(): corrected_wavelength = wavecorr.compute_wavelength(slit.meta.wcs, x, y) assert_allclose(slit.wavelength, corrected_wavelength) - # test the roundtripping on the wavelength correction transform + # test the round-tripping on the wavelength correction transform ref_name = result.meta.ref_file.wavecorr.name freference = datamodels.WaveCorrModel(WavecorrStep.reference_uri_to_cache_path(ref_name, im.crds_observatory)) From fe37e3721986932dc5444995586fd0c78387c5d3 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 10 Jun 2024 12:16:22 -0400 Subject: [PATCH 43/44] Documentation updates for NIRSpec MOS/FS --- docs/jwst/exp_to_source/main.rst | 15 +++++++++++---- docs/jwst/flatfield/main.rst | 4 ++-- docs/jwst/master_background/description.rst | 8 ++++---- docs/jwst/nsclean/main.rst | 4 ++++ docs/jwst/photom/main.rst | 8 ++++---- docs/jwst/pipeline/calwebb_spec3.rst | 16 ++++++++++++++++ 6 files changed, 41 insertions(+), 14 deletions(-) diff --git a/docs/jwst/exp_to_source/main.rst b/docs/jwst/exp_to_source/main.rst index 0216274adf..2e862079d2 100644 --- a/docs/jwst/exp_to_source/main.rst +++ b/docs/jwst/exp_to_source/main.rst @@ -34,14 +34,21 @@ configuration of MSA slitlets with a source assigned to each slitlet. The source-to-slitlet linkage is carried along in the information contained in the MSA metadata file used during :ref:`calwebb_spec2 ` processing. Each slitlet instance created by the :ref:`extract_2d ` -step stores the source ID (a simple integer number) in the SOURCEID keyword of +step stores the source name in the SRCNAME keyword of the SCI extension header for the slitlet. The ``exp_to_source`` tool uses -the SOURCEID values to sort the data from each input product into an +the SRCNAME values to sort the data from each input product into an appropriate source-based output product. -NIRSpec Fixed-Slit +If fixed slit targets are planned as part of a NIRSpec MOS exposure, they +will also have sources identified by the MSA metadata file. The associated +SRCNAME values are used to sort the data from these slits in the same way the +MSA slitlets are sorted. In this combined mode, the products containing +MOS slitlets are marked with EXP_TYPE = "NRS_MSASPEC" after sorting; the models +containing fixed slits are marked with EXP_TYPE = "NRS_FIXEDSLIT". + +NIRSpec Fixed Slit ^^^^^^^^^^^^^^^^^^ -NIRSpec fixed-slit observations do not have sources identified with each +NIRSpec fixed slit observations do not have sources identified with each slit, so the slit names, e.g. S200A1, S1600A1, etc., are mapped to predefined source ID values, as follows: diff --git a/docs/jwst/flatfield/main.rst b/docs/jwst/flatfield/main.rst index 38df509446..f6592b2105 100644 --- a/docs/jwst/flatfield/main.rst +++ b/docs/jwst/flatfield/main.rst @@ -104,7 +104,7 @@ is applied to the slit data, but that correction is not appropriate for the background signal contained in the slit, and hence corrections must be applied later in the :ref:`master background ` step. -So in this case the `flatfield` step will compute 2D arrays of conversion +In this case, the `flatfield` step will compute 2D arrays of conversion factors that are appropriate for a uniform source and for a point source, and store those correction factors in the "FLATFIELD_UN" and "FLATFIELD_PS" extensions, respectively, of the output data product. The point source @@ -118,7 +118,7 @@ wavelength-dependent. A uniform source does not require wavelength corrections and hence the flatfield conversions will differ for point and uniform sources. Any secondary slits that may be included in a fixed slit exposure do not have source centering information available, so the -:ref:`wavecorr ` step is not applied, and hence there's no +:ref:`wavecorr ` step is not applied, and there is no difference between the point source and uniform source flatfield conversions for those slits. diff --git a/docs/jwst/master_background/description.rst b/docs/jwst/master_background/description.rst index 2e21d58b0d..6a5d39b10b 100644 --- a/docs/jwst/master_background/description.rst +++ b/docs/jwst/master_background/description.rst @@ -295,9 +295,9 @@ source centering within the slit, hence slits containing uniform sources receive the same flat-field and photometric calibrations as background spectra and therefore don't require corrections for those two calibrations. Furthermore, the source position in the slit is only known for the primary slit in an exposure, so -secondary slits are always handled as extended sources, and no wavelength correction is -applied, and therefore again the flat-field and photometric calibrations are -the same as for background spectra. +secondary slits are always handled as extended sources, no wavelength correction is +applied, and therefore again the flat-field, photometric, and pathloss calibrations +are the same as for background spectra. Fixed slits planned as part of a combined MOS and FS observation are an exception to this rule. These targets may each be identified as @@ -306,7 +306,7 @@ point sources, with location information for each given in the this way are treated in the same manner as the primary fixed slit in standard FS observations. -Therefore if a fixed slit contains a point source (as given by the SRCTYPE keyword) +Therefore, if a fixed slit contains a point source (as given by the SRCTYPE keyword) the corrections that need to be applied to the 2-D master background for that slit are: .. math:: diff --git a/docs/jwst/nsclean/main.rst b/docs/jwst/nsclean/main.rst index 48a9c615d9..c61e406094 100644 --- a/docs/jwst/nsclean/main.rst +++ b/docs/jwst/nsclean/main.rst @@ -103,6 +103,10 @@ processing MOS and IFU images. The masked region is currently hardwired in the step to image indexes [1:2048, 923:1116], where the indexes are in x, y order and in 1-indexed values. +Note, however, that it is possible to plan one or more fixed slit targets +alongside MSA slitlets in MOS observations. In this situation, the fixed +slit region is not automatically masked. + Left/Right Reference Pixel Columns ---------------------------------- Full-frame images contain 4 columns of reference pixels on the left and diff --git a/docs/jwst/photom/main.rst b/docs/jwst/photom/main.rst index b18cd82c29..0e0b478b64 100644 --- a/docs/jwst/photom/main.rst +++ b/docs/jwst/photom/main.rst @@ -84,9 +84,9 @@ conversion factor. If the time-dependent coefficients are present in the reference file, the photom step will apply the correction based on the observation date of the exposure being processed. -NIRSpec Fixed-Slit Primary Slit +NIRSpec Fixed Slit Primary Slit ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The primary slit in a NIRSpec fixed-slit exposure receives special handling. +The primary slit in a NIRSpec fixed slit exposure receives special handling. If the primary slit, as given by the "FXD_SLIT" keyword value, contains a point source, as given by the "SRCTYPE" keyword, it is necessary to know the photometric conversion factors for both a point source and a uniform source @@ -96,7 +96,7 @@ is applied to the slit data, but that correction is not appropriate for the background signal contained in the slit, and hence corrections must be applied later in the :ref:`master background ` step. -So in this case the `photom` step will compute 2D arrays of conversion +In this case, the `photom` step will compute 2D arrays of conversion factors that are appropriate for a uniform source and for a point source, and store those correction factors in the "PHOTOM_UN" and "PHOTOM_PS" extensions, respectively, of the output data product. The point source @@ -105,7 +105,7 @@ correction array is also applied to the slit data. Note that this special handling is only needed when the slit contains a point source, because in that case corrections to the wavelength grid are applied by the :ref:`wavecorr ` step to account for any -source mis-centering in the slit and the photometric conversion factors are +source offsets in the slit and the photometric conversion factors are wavelength-dependent. A uniform source does not require wavelength corrections and hence the photometric conversions will differ for point and uniform sources. Any secondary slits that may be included in a fixed-slit exposure diff --git a/docs/jwst/pipeline/calwebb_spec3.rst b/docs/jwst/pipeline/calwebb_spec3.rst index 4331170368..307311bc58 100644 --- a/docs/jwst/pipeline/calwebb_spec3.rst +++ b/docs/jwst/pipeline/calwebb_spec3.rst @@ -51,6 +51,9 @@ to observations of a moving target (TARGTYPE='moving'). :sup:`2`\ The master background subtraction step is applied to NIRSpec MOS exposures in the :ref:`calwebb_spec2 ` pipeline. +WFSS and SOSS Processing +------------------------ + Notice that NIRCam and NIRISS WFSS, as well as NIRISS SOSS data, receive only minimal processing by ``calwebb_spec3``. WFSS 2D input data are reorganized into source-based products by the @@ -66,6 +69,19 @@ obtained in TSO mode. TSO mode NIRISS SOSS exposures should be processed with the :ref:`calwebb_tso3 ` pipeline. +Combined NIRSpec MOS and FS Exposures +------------------------------------- + +It is possible to observe NIRSpec fixed slit targets alongside MSA +slitlets in NIRSpec MOS exposures. In this case, the input files produced by the +:ref:`calwebb_spec2 ` pipeline contain both MOS and FS data. Any +:ref:`master_background ` corrections applied in this pipeline +will operate on the fixed slits only, since background corrections for MOS slitlets should +be applied in the :ref:`calwebb_spec2 ` pipeline. After the sources are +separated in the :ref:`exp_to_source ` step, each data product will contain +only a fixed slit or a MOS target, so all further processing follows the standard flow +for each exposure type. + Arguments --------- From 316526443f5885e9b981dfd817f65057e03d1408 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 10 Jun 2024 13:11:11 -0400 Subject: [PATCH 44/44] Use source_name to determine if FS slit comes from MSA file --- jwst/wavecorr/wavecorr.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwst/wavecorr/wavecorr.py b/jwst/wavecorr/wavecorr.py index df47cb6c75..d514355397 100644 --- a/jwst/wavecorr/wavecorr.py +++ b/jwst/wavecorr/wavecorr.py @@ -331,10 +331,10 @@ def _is_msa_fixed_slit(slit): A slit object. """ # Fixed slits defined via MSA files in MOS/FS combination - # processing will have a non-empty shutter state - if (not hasattr(slit, 'shutter_state') - or slit.shutter_state is None - or slit.shutter_state == ""): + # processing will have a non-empty source name + if (not hasattr(slit, 'source_name') + or slit.source_name is None + or slit.source_name == ""): return False else: return True