diff --git a/CHANGES.rst b/CHANGES.rst index af89562bb6..b9bdbf8fce 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,6 +20,10 @@ assign_wcs - Move the assigned source position for dedicated NIRSpec MOS background slits from the lower left corner of the slit to the middle of the slit. [#8461] +- Updated the routines that load NIRSpec MOS slit and source data from the MSA meta + data file to properly handle background and virtual slits, and assign appropriate + meta data to them for use downstream. [#8442] + associations ------------ @@ -29,6 +33,9 @@ associations - Match NIRSpec imprint observations to science exposures on mosaic tile location and dither pointing, ``MOSTILNO`` and ``DITHPTIN``. [#8410] +- Updated Level3 rules for new handling of NIRSpec MOS source_id formatting when + constructing output file names. [#8442] + dark_current ------------ @@ -45,6 +52,13 @@ documentation - Added documentation for NIRCam GRISM time series pointing offsets. [#8449] +exp_to_source +------------- + +- Modified slit sorting to use `source_name` as the key, rather than `source_id`, + in order to support changes in `source_id` handling for NIRSpec MOS exposures + that contain background and virtual slits. [#8442] + extract_1d ---------- @@ -103,6 +117,10 @@ outlier_detection finished, unless save_intermediate_results is True. This PR also addressed the _i2d files not being saved in the specified output directory. [#8464] +- Removed the setting of `self.skip = True` when the step gets skipped (due to + inappropriate inputs), so that the step still executes when called again + while processing a list of multiple sources. [#8442] + - Added tests for changes made in #8464. [#8481] - Added the option to use a rolling median instead of a simple median @@ -124,6 +142,11 @@ pipeline - Removed unused ``scale_detection`` argument from ``calwebb_tso3`` pipeline. [#8438] +- Updated the ``calwebb_spec3`` pipeline handling of NIRSpec MOS inputs, to + comply with the new scheme for source ("s"), background ("b"), and + virtual ("v") slits and the construction of output file names for each + type. [#8442] + pixel_replace ------------- diff --git a/docs/jwst/data_products/file_naming.rst b/docs/jwst/data_products/file_naming.rst index 0f8b6ca30b..c6a48e8a83 100644 --- a/docs/jwst/data_products/file_naming.rst +++ b/docs/jwst/data_products/file_naming.rst @@ -42,14 +42,14 @@ Just as for Stage 2, the suffix distinguishes the different file products of Sta The FITS file naming scheme for Stage 3 "source-based" products is as follows, where items in parentheses are optional: - jw-_[<"t"TargID | "s"SourceID>](-<"epoch"X>)__(-)_(-).fits + jw-_[<"t"TargID | [<"s" | "b" | "v">]](-<"epoch"X>)__(-)_(-).fits where - ppppp: Program ID number - AC_ID: Association candidate ID - TargID: 3-digit Target ID (either TargID or SourceID must be present) - - SourceID: 5-digit Source ID + - SourceID: 9-digit Source ID - epochX: The text "epoch" followed by a single digit epoch number (optional) - instr: Science instrument name (e.g. 'nircam', 'miri') - optElements: A single or hyphen-separated list of optical elements (e.g. filter, grating) @@ -57,10 +57,12 @@ where - prodType: Product type identifier (e.g. 'i2d', 's3d', 'x1d') - ACT_ID: 2-digit activity ID (optional) -An example Stage 3 product FITS file name is: +Example Stage 3 product FITS file names are: jw87600-a3001_t001_niriss_f480m-nrm_amiavg.fits + jw54321-o066_s000123456_nirspec_f170lp_g235m_s2d.fits + Optional Components """"""""""""""""""" @@ -72,7 +74,14 @@ TargID vs SourceID For single-target modes, this is the target identifier as defined in the APT proposal. - For multi-object modes, such as NIRSpec MOS, this will be the slit ID for each object. + For multi-object modes, such as NIRSpec MOS, this will be the source ID for each object. + Note that the SourceID value is preceded by one of three characters "s", "b", or "v". + For most multi-source observation modes, such as Wide Field Slitless Spectroscopy (WFSS) and + NIRSpec Fixed-Slit (FS) spectroscopy, the "s" prefix is used to indicate that the data + correspond to a defined source. In some NIRSpec MOS observations, however, + there can also be "background" and "virtual" sources + (see :ref:`NIRSpec MSA slitlets `). + These cases use the "b" and "v" SourceID prefixes. epochX diff --git a/docs/jwst/data_products/msa_metadata.rst b/docs/jwst/data_products/msa_metadata.rst index edaa703b2f..b251402c80 100644 --- a/docs/jwst/data_products/msa_metadata.rst +++ b/docs/jwst/data_products/msa_metadata.rst @@ -2,7 +2,7 @@ MSA Metadata File: ``msa`` ^^^^^^^^^^^^^^^^^^^^^^^^^^ -While not containing any actual science data, the NIRSpec MSA metadata file is nonetheless +While it doesn't contain any actual science data, the NIRSpec MSA metadata file is nonetheless a crucial component of calibration processing for NIRSpec MOS exposures. It contains all the slitlet, shutter, and source configuration information that's needed by the :ref:`calwebb_spec2 ` pipeline to process a MOS exposure. @@ -128,6 +128,8 @@ where the values of `MSAMETID` and `PATT_NUM` in the science exposure match the values of `msa_metdata_id` and `dither_point_index`, respectively, are loaded. +Slitlets with a catalog source +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To better understand the ways in which these metadata are used, it's useful to reference a hypothetical example of data within a ``SHUTTER_INFO`` table. The table below shows the first 9 rows of a ``SHUTTER_INFO`` table for a MOS exposure @@ -186,13 +188,46 @@ within each slitlet should be considered the "primary" shutter. This is especial important for slitlets that contain extended sources and hence the `source_id` and `background` entries may indicate that the source is present in multiple shutters. -When a slitlet is found that has no shutters with a primary source (i.e. no shutters -having `primary_source` = "Y"), it is classified as a background slitlet and assigned -a source ID value that's greater than the maximum source ID assigned to other slitlets -(because such slitlets all have a source ID of zero in the MSA metadata coming from -the ground system). -These background slitlets can then be used in :ref:`master background ` -subtraction. +.. _msa_background_and_virtual_slits: + +Slitlets without a catalog source +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +It is possible for users to define slitlets that do not contain a source that's defined +in the MPT catalog when constructing an MSA configuration for an observation. These +kinds of slitlets can be used for two purposes. First, slitlets in which all of the +constituent shutters only contain background can be used to perform "master background" +subtraction during the :ref:`calwebb_spec2 ` pipeline processing +(see :ref:`NIRSpec Master Background ` for more details). +Second, slitlets made up of open shutters that may contain signal from some uncataloged +source in the field can also be created. These are referred to as "virtual" slitlets. + +Background and virtual slitlets have unique metadata in the shutter information table. +The primary defining piece of data is their assigned `source_id` value, because these +slitlets don't have a corresponding source listed in the source information table. +During creation with the MPT, all background slitlets are given a `source_id` of zero. +Virtual slitlets, on the other hand, are assigned *negative* `source_id` values, starting +with -1 and counting downwards from there (i.e. each virtual slit has a unique negative +`source_id` value). + +During the parsing of shutter information described in the previous section, when a +slitlet is found that has no shutters with a primary source (i.e. no shutters +having `primary_source` = "Y"), it is recognized as a background slitlet. In order to +be able to track multiple background slitlets through the remaining processing, they +are reassigned a new `source_id` value equal to their `slitlet_id`. Virtual slitlets, +meanwhile, retain their unique negative `source_id` values throughout processing. + +During Stage 3 processing with the :ref:`calwebb_spec3 ` pipeline, +unique source-based product file names will be created that distinguish data from the +three different kinds of slitlets: source, background, and virtual. As described in +:ref:`source-based file names `, the `SourceID` field of Stage 3 +file names consists of the 9-digit `source_id` number assigned to each MOS slitlet, +preceded by one of the three characters "s", "b", or "v", which identifies whether +the data are from a source, background, or virtual slitlet, respectively. Note that, +as described above, the `source_id` number used here for background slitlets is a +copy of their `slitlet_id` number. For example, a Stage 3 file name for data taken +from a virtual slitlet with `source_id` = -42 will look like: + + jw12345-o066_v000000042_nirspec_f170lp_g235m_x1d.fits The SOURCE_INFO Metadata diff --git a/docs/jwst/exp_to_source/main.rst b/docs/jwst/exp_to_source/main.rst index 14aaa0c8dd..0216274adf 100644 --- a/docs/jwst/exp_to_source/main.rst +++ b/docs/jwst/exp_to_source/main.rst @@ -83,8 +83,8 @@ on the meaning of each field in the file names. The FITS file naming scheme for the source-based "cal" products follows the standard Stage 3 syntax, such as:: - jw10006-o010_s00061_nirspec_f170lp-g235m_cal.fits + jw10006-o010_s000000061_nirspec_f170lp-g235m_cal.fits -where "s00061" in this example is the source ID. +where "s000000061" in this example is the source ID. See :ref:`source-based file names ` for more details on the meaning of each field in this type of file name. diff --git a/docs/jwst/pipeline/calwebb_spec3.rst b/docs/jwst/pipeline/calwebb_spec3.rst index 8b387c480c..4331170368 100644 --- a/docs/jwst/pipeline/calwebb_spec3.rst +++ b/docs/jwst/pipeline/calwebb_spec3.rst @@ -114,7 +114,7 @@ input exposure-based products. The source-based collections of data are saved in intermediate files, one per source/slit. The root names of the source-based files contain the source ID as an identifier and use the same "_cal" suffix as the input calibrated exposure files. An example source-based file name is -"jw00042-o001_s0002_niriss_gr150r_f150w_cal.fits", where "s0002" is the source id. +"jw00042-o001_s00000002_niriss_gr150r_f150w_cal.fits", where "s00000002" is the source id. The reorganized sets of data are sent to subsequent steps to process and combine all the data for one source at a time. diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index c6e3e8c2ef..e08834e824 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -405,20 +405,29 @@ def get_open_slits(input_model, reference_files=None, slit_y_range=[-.55, .55]): lamp_mode = lamp_mode.lower() else: lamp_mode = 'none' + + # MOS/MSA exposure requiring MSA metadata file if exp_type in ["nrs_msaspec", "nrs_autoflat"] or ((exp_type in ["nrs_lamp", "nrs_autowave"]) and (lamp_mode == "msaspec")): - msa_metadata_file, msa_metadata_id, dither_point = get_msa_metadata( - input_model, reference_files) - slits = get_open_msa_slits(msa_metadata_file, msa_metadata_id, dither_point, slit_y_range) + prog_id = input_model.meta.observation.program_number.lstrip("0") + msa_metadata_file, msa_metadata_id, dither_point = get_msa_metadata(input_model, reference_files) + slits = get_open_msa_slits(prog_id, msa_metadata_file, msa_metadata_id, dither_point, slit_y_range) + + # Fixed slits exposure (non-TSO) elif exp_type == "nrs_fixedslit": slits = get_open_fixed_slits(input_model, slit_y_range) + + # Bright object (TSO) exposure in S1600A1 fixed slit elif exp_type == "nrs_brightobj": slits = [Slit('S1600A1', 3, 0, 0, 0, slit_y_range[0], slit_y_range[1], 5, 1)] + + # Lamp exposure using fixed slits elif exp_type in ["nrs_lamp", "nrs_autowave"]: if lamp_mode in ['fixedslit', 'brightobj']: slits = get_open_fixed_slits(input_model, slit_y_range) else: raise ValueError("EXP_TYPE {0} is not supported".format(exp_type.upper())) + if reference_files is not None and slits: slits = validate_open_slits(input_model, slits, reference_files) log.info("Slits projected on detector {0}: {1}".format(input_model.meta.instrument.detector, @@ -510,27 +519,7 @@ def get_msa_metadata(input_model, reference_files): return msa_config, msa_metadata_id, dither_position -def _get_bkg_source_id(bkg_counter, shift_by): - """ - Compute a ``source_id`` for background slitlets. - - All background slitlets are assigned a source_id of 0. - A unique ``source_id`` is necessary to keep them separate in exp_to_source. - A counter is used to assign a unique ``source_id`` that's - greater than the max ID number of all defined sources. - - Parameters - ---------- - bkg_counter : int - The current value of the counter. - shift_by : int - The highest of all source_id values. - """ - - return bkg_counter + shift_by - - -def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, +def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, slit_y_range=[-.55, .55]): """ Return the opened MOS slitlets. @@ -555,10 +544,10 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, For example, something like: (12, 2, 4, 251, 22, 1, 'Y', 'OPEN', nan, nan, 1, 'N'), - column - Parameters ---------- + prog_id : str + The program number msa_file : str MSA meta data file name, FITS keyword ``MSAMETFL``. msa_metadata_id : int @@ -596,32 +585,14 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, log.error(message) raise MSAFileError(message) - # Get the configuration header from the _msa.fits file. The EXTNAME should be 'SHUTTER_INFO' - msa_conf = msa_file[('SHUTTER_INFO', 1)] - msa_source = msa_file[("SOURCE_INFO", 1)].data + # Get the shutter and source info tables from the _msa.fits file. + msa_conf = msa_file[('SHUTTER_INFO', 1)] # EXTNAME = 'SHUTTER_INFO' + msa_source = msa_file[("SOURCE_INFO", 1)].data # EXTNAME = 'SOURCE_INFO' # First we are going to filter the msa_file data on the msa_metadata_id # and dither_point_index. msa_data = [x for x in msa_conf.data if x['msa_metadata_id'] == msa_metadata_id and x['dither_point_index'] == dither_position] - - # Get all source_ids for slitlets with sources. - # These should not be used when assigning source_id to background slitlets. - source_ids = set([x[5] for x in msa_conf.data if x['msa_metadata_id'] == msa_metadata_id - and x['dither_point_index'] == dither_position]) - # All BKG shutters in the msa metafile have a source_id value of 0. - # Remove it from the list of source ids. - if 0 in source_ids: - source_ids.remove(0) - if source_ids: - max_source_id = max(source_ids) + 1 - else: - max_source_id = 0 - - # define a counter for "all background" slitlets. - # It will be used to assign a "source_id". - bkg_counter = 0 - log.debug(f'msa_data with msa_metadata_id = {msa_metadata_id} {msa_data}') log.info(f'Retrieving open MSA slitlets for msa_metadata_id = {msa_metadata_id} ' f'and dither_index = {dither_position}') @@ -629,25 +600,20 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, # Get the unique slitlet_ids slitlet_ids_unique = list(set([x['slitlet_id'] for x in msa_data])) - # SDP may assign a value of "-1" to ``slitlet_id`` - these need to be ignored. - # JP-436 - if -1 in slitlet_ids_unique: - slitlet_ids_unique.remove(-1) - # add a margin to the slit y limits margin = 0.5 # Now lets look at each unique slitlet id for slitlet_id in slitlet_ids_unique: - # Get the rows for the current slitlet_id + # Get the rows of shutter info for the current slitlet_id slitlets_sid = [x for x in msa_data if x['slitlet_id'] == slitlet_id] open_shutters = [x['shutter_column'] for x in slitlets_sid] # How many shutters in the slitlet are labeled as "main" or "primary"? n_main_shutter = len([s for s in slitlets_sid if s['primary_source'] == 'Y']) - # In the next part we need to calculate, find, determine 5 things: - # quadrant, xcen, ycen, ymin, ymax + # In the next part we need to calculate, find, or determine 5 things for each slit: + # quadrant, xcen, ycen, ymin, ymax # There are no main shutters: all are background if n_main_shutter == 0: @@ -662,19 +628,31 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, quadrant = slitlets_sid[0]['shutter_quadrant'] ycen = j xcen = slitlets_sid[0]['shutter_row'] # grab the first as they are all the same + shutter_id = xcen + (ycen - 1) * 365 # shutter numbers in MSA file are 1-indexed + + # Background slits all have source_id=0 in the msa_file, + # so assign a unique id based on the slitlet_id + source_id = slitlet_id + + # Hardwire the source info for background slits, because there's + # no source info for them in the msa_file source_xpos = 0.5 source_ypos = 0.5 - source_id = _get_bkg_source_id(bkg_counter, max_source_id) - log.info(f'Slitlet_id {slitlet_id} is background only; assigned source_id = {source_id}') - bkg_counter += 1 + source_name = f"{prog_id}_BKG{slitlet_id}" + source_alias = "BKG{}".format(slitlet_id) + stellarity = 0.0 + source_ra = 0.0 + source_dec = 0.0 + log.info(f'Slitlet {slitlet_id} is background only; assigned source_id={source_id}') - # There is 1 main shutter: phew, that makes it easier. + # There is 1 main shutter: this is a slit containing either a real or virtual source elif n_main_shutter == 1: xcen, ycen, quadrant, source_xpos, source_ypos = [ (s['shutter_row'], s['shutter_column'], s['shutter_quadrant'], s['estimated_source_in_shutter_x'], s['estimated_source_in_shutter_y']) for s in slitlets_sid if s['background'] == 'N'][0] + shutter_id = xcen + (ycen - 1) * 365 # shutter numbers in MSA file are 1-indexed # y-size jmin = min([s['shutter_column'] for s in slitlets_sid]) @@ -682,11 +660,25 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, j = ycen ymax = yhigh + margin + (jmax - j) * 1.15 ymin = -(-ylow + margin) + (jmin - j) * 1.15 - # get the source_id from the primary shutter entry + + # Get the source_id from the primary shutter entry for i in range(len(slitlets_sid)): if slitlets_sid[i]['primary_source'] == 'Y': source_id = slitlets_sid[i]['source_id'] + # Get source info for this slitlet; + # note that slits with a real source assigned have source_id > 0, + # while slits with source_id < 0 contain "virtual" sources + try: + source_name, source_alias, stellarity, source_ra, source_dec = [ + (s['source_name'], s['alias'], s['stellarity'], s['ra'], s['dec']) + for s in msa_source if s['source_id'] == source_id][0] + except IndexError: + log.warning("Could not retrieve source info from MSA file") + + if source_id < 0: + log.info(f'Slitlet {slitlet_id} contains virtual source, with source_id={source_id}') + # More than 1 main shutter: Not allowed! else: message = ("For slitlet_id = {}, metadata_id = {}, " @@ -697,20 +689,6 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, msa_file.close() raise MSAFileError(message) - # subtract 1 because shutter numbers in the MSA reference file are 1-based. - shutter_id = xcen + (ycen - 1) * 365 - try: - source_name, source_alias, stellarity, source_ra, source_dec = [ - (s['source_name'], s['alias'], s['stellarity'], s['ra'], s['dec']) - for s in msa_source if s['source_id'] == source_id][0] - except IndexError: - # all background shutters - source_name = "background_{}".format(slitlet_id) - source_alias = "bkg_{}".format(slitlet_id) - stellarity = 0.0 - source_ra = 0.0 - source_dec = 0.0 - # Create the output list of tuples that contain the required # data for further computations """ diff --git a/jwst/assign_wcs/tests/test_nirspec.py b/jwst/assign_wcs/tests/test_nirspec.py index a517916835..b89ccdcc4b 100644 --- a/jwst/assign_wcs/tests/test_nirspec.py +++ b/jwst/assign_wcs/tests/test_nirspec.py @@ -70,6 +70,7 @@ def create_hdul(detector='NRS1'): phdu.header['detector'] = detector phdu.header['time-obs'] = '8:59:37' phdu.header['date-obs'] = '2016-09-05' + phdu.header['program'] = '1234' scihdu = fits.ImageHDU() scihdu.header['EXTNAME'] = "SCI" @@ -276,10 +277,11 @@ def test_msa_configuration_normal(): """ # Test 1: Reasonably normal as well + prog_id = '1234' msa_meta_id = 12 msaconfl = get_file_path('msa_configuration.fits') dither_position = 1 - slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, + slitlet_info = nirspec.get_open_msa_slits(prog_id, msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) ref_slit = trmodels.Slit(55, 9376, 1, 251, 26, -5.6, 1.0, 4, 1, '1111x', '95065_1', '2122', 0.13, -0.31716078999999997, -0.18092266) @@ -291,11 +293,12 @@ def test_msa_configuration_no_background(): Test the get_open_msa_slits function. """ # Test 2: Two main shutters, not allowed and should fail + prog_id = '1234' msa_meta_id = 13 msaconfl = get_file_path('msa_configuration.fits') dither_position = 1 with pytest.raises(MSAFileError): - nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, + nirspec.get_open_msa_slits(prog_id, msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) @@ -305,13 +308,14 @@ def test_msa_configuration_all_background(): """ # Test 3: No non-background, not acceptable. + prog_id = '1234' msa_meta_id = 14 msaconfl = get_file_path('msa_configuration.fits') dither_position = 1 - slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, + slitlet_info = nirspec.get_open_msa_slits(prog_id, msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) - ref_slit = trmodels.Slit(57, 8281, 1, 251, 23, -2.15, 2.15, 4, 0, '1x1', 'background_57', 'bkg_57', - 0, 0.0, 0.0) + ref_slit = trmodels.Slit(57, 8281, 1, 251, 23, -2.15, 2.15, 4, 57, '1x1', '1234_BKG57', 'BKG57', + 0.0, 0.0, 0.0) _compare_slits(slitlet_info[0], ref_slit) @@ -321,10 +325,11 @@ def test_msa_configuration_row_skipped(): """ # Test 4: One row is skipped, should be acceptable. + prog_id = '1234' msa_meta_id = 15 msaconfl = get_file_path('msa_configuration.fits') dither_position = 1 - slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, + slitlet_info = nirspec.get_open_msa_slits(prog_id, msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) ref_slit = trmodels.Slit(58, 8646, 1, 251, 24, -3.3, 5.6, 4, 1, '11x1011', '95065_1', '2122', 0.130, -0.31716078999999997, -0.18092266) @@ -336,10 +341,11 @@ def test_msa_configuration_multiple_returns(): Test the get_open_msa_slits function. """ # Test 4: One row is skipped, should be acceptable. + prog_id = '1234' msa_meta_id = 16 msaconfl = get_file_path('msa_configuration.fits') dither_position = 1 - slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, + slitlet_info = nirspec.get_open_msa_slits(prog_id, msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) ref_slit1 = trmodels.Slit(59, 8651, 1, 256, 24, -3.3, 5.6, 4, 1, '11x1011', '95065_1', '2122', 0.13000000000000003, -0.31716078999999997, -0.18092266) diff --git a/jwst/associations/lib/rules_level3_base.py b/jwst/associations/lib/rules_level3_base.py index f342e080b3..ce7c5c631b 100644 --- a/jwst/associations/lib/rules_level3_base.py +++ b/jwst/associations/lib/rules_level3_base.py @@ -583,7 +583,7 @@ def finalize(associations): # Define default product name filling format_product = FormatTemplate( key_formats={ - 'source_id': ['s{:05d}', 's{:s}'], + 'source_id': ['{:s}'], 'expspcin': ['{:0>2s}'], 'slit_name': ['{:s}'] } diff --git a/jwst/exp_to_source/exp_to_source.py b/jwst/exp_to_source/exp_to_source.py index b9540b1b16..baaaa1ce20 100644 --- a/jwst/exp_to_source/exp_to_source.py +++ b/jwst/exp_to_source/exp_to_source.py @@ -37,8 +37,15 @@ def exp_to_source(inputs): log.info(f'Reorganizing data from exposure {exposure.meta.filename}') for slit in exposure.slits: - log.debug(f'Copying source {slit.source_id}') - result_slit = result[str(slit.source_id)] + if slit.source_name is None: + # All MultiSlit data other than NIRSpec MOS get sorted by + # source_id (source_name is not populated) + key = slit.source_id + else: + # NIRSpec MOS slits get sorted by source_name + key = slit.source_name + log.debug(f'Copying source {key}') + result_slit = result[str(key)] result_slit.exposures.append(slit) # store values for later use (after merge_tree) # these values are incorrectly getting overwritten by diff --git a/jwst/msaflagopen/tests/test_msa_open.py b/jwst/msaflagopen/tests/test_msa_open.py index ca8611f248..50e6379984 100644 --- a/jwst/msaflagopen/tests/test_msa_open.py +++ b/jwst/msaflagopen/tests/test_msa_open.py @@ -158,6 +158,7 @@ def test_msaflagopen_step(): 'msa_metadata_id': 12} im.meta.observation = { + 'program_number': '1234', 'date': '2016-09-05', 'time': '8:59:37'} diff --git a/jwst/outlier_detection/outlier_detection_step.py b/jwst/outlier_detection/outlier_detection_step.py index 18140b1c69..8add2f32a9 100644 --- a/jwst/outlier_detection/outlier_detection_step.py +++ b/jwst/outlier_detection/outlier_detection_step.py @@ -163,7 +163,6 @@ def process(self, input_data): model.meta.cal_step.outlier_detection = "SKIPPED" else: self.input_models.meta.cal_step.outlier_detection = "SKIPPED" - self.skip = True return self.input_models self.log.debug(f"Using {detection_step.__name__} class for outlier_detection") diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index 429f43da4a..a5fcd59dc6 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -2,7 +2,6 @@ from collections import defaultdict from functools import wraps import os.path as op -import numpy as np from stdatamodels.jwst import datamodels @@ -182,54 +181,34 @@ def process(self, input): for name, model in multislit_to_container(source_models).items() ] - # Check for negative and large source_id values - if len(sources) > 99999: - self.log.critical("Data contain more than 100,000 sources;" - "filename does not support 6 digit source ids.") - raise Exception - - available_src_ids = set(np.arange(99999) + 1) - used_src_ids = set() - for src in sources: - src_id, model = src - src_id = int(src_id) - used_src_ids.add(src_id) - if 0 < src_id <= 99999: - available_src_ids.remove(src_id) - - hotfixed_sources = [] - # now find and reset bad source_id values - for src in sources: - src_id, model = src - src_id = int(src_id) - # Replace ids that aren't positive 5-digit integers - if src_id < 0 or src_id > 99999: - src_id_new = available_src_ids.pop() - self.log.info(f"Source ID {src_id} falls outside allowed range.") - self.log.info(f"Reassigning {src_id} to {str(src_id_new).zfill(5)}.") - # Replace source_id for each model in the SourceModelContainers - for contained_model in model: - contained_model.source_id = src_id_new - src_id = src_id_new - hotfixed_sources.append((str(src_id), model)) - - sources = hotfixed_sources - # Process each source for source in sources: # If each source is a SourceModelContainer, - # the output name needs to be updated with the source ID, and potentially - # also the slit name (for NIRSpec fixed-slit only). + # the output name needs to be updated based on the source ID, + # and potentially also the slit name (for NIRSpec fixed-slit only). if isinstance(source, tuple): source_id, result = source + + # NIRSpec fixed-slit data if result[0].meta.exposure.type == "NRS_FIXEDSLIT": + # Output file name is constructed using the source_id and the slit name slit_name = self._create_nrsfs_slit_name(result) - self.output_file = format_product( - output_file, source_id=source_id.lower(), slit_name=slit_name) + srcid = f's{source_id:>09s}' + self.output_file = format_product(output_file, source_id=srcid, slit_name=slit_name) + + # NIRSpec MOS/MSA data + elif result[0].meta.exposure.type == "NRS_MSASPEC": + # Construct the specially formatted source_id to use in the output file + # name that separates source, background, and virtual slits + srcid = self._create_nrsmos_source_id(result) + self.output_file = format_product(output_file, source_id=srcid) + self.log.debug(f"output_file = {self.output_file}") + else: - self.output_file = format_product( - output_file, source_id=source_id.lower()) + # All other types just use the source_id directly in the file name + srcid = f's{source_id:>09s}' + self.output_file = format_product(output_file, source_id=srcid) else: result = source @@ -338,6 +317,35 @@ def _create_nrsfs_slit_name(self, source_models): return slit_name + def _create_nrsmos_source_id(self, source_models): + """Create the complete source_id product field for NIRSpec MOS products. + + The source_id value gets a "s", "b", or "v" character prepended + to uniquely identify source, background, and virtual slits. + """ + + # Get the original source name and ID from the input models + source_name = source_models[0].source_name + source_id = source_models[0].source_id + + # MOS background sources have "BKG" in the source name + if "BKG" in source_name: + # prepend "b" to the source_id number and format to 9 chars + srcid = f'b{str(source_id):>09s}' + self.log.debug(f"Source {source_name} is a MOS background slitlet: ID={srcid}") + + # MOS virtual sources have a negative source_id value + elif source_id < 0: + # prepend "v" to the source_id number and remove the leading negative sign + srcid = f'v{str(source_id)[1:]:>09s}' + self.log.debug(f"Source {source_name} is a MOS virtual slitlet: ID={srcid}") + + # Regular MOS sources + else: + # prepend "s" to the source_id number and format to 9 chars + srcid = f's{str(source_id):>09s}' + + return srcid # ######### # Utilities @@ -359,4 +367,4 @@ def save_model(model, **kwargs): return result - return save_model + return save_model \ No newline at end of file diff --git a/jwst/regtest/test_niriss_wfss.py b/jwst/regtest/test_niriss_wfss.py index b723690a2f..d2ab05f4d4 100644 --- a/jwst/regtest/test_niriss_wfss.py +++ b/jwst/regtest/test_niriss_wfss.py @@ -74,7 +74,7 @@ def run_nis_wfss_spec3(run_nis_wfss_spec2, rtdata_module): @pytest.mark.bigdata @pytest.mark.parametrize('suffix', ['cal', 'x1d', 'c1d']) -@pytest.mark.parametrize('source_id', ['s00015', 's00104']) +@pytest.mark.parametrize('source_id', ['s000000015', 's000000104']) def test_nis_wfss_spec3(run_nis_wfss_spec3, rtdata_module, suffix, source_id, fitsdiff_default_kwargs): """Regression test of the calwebb_spec3 pipeline applied to NIRISS WFSS data""" rtdata = rtdata_module diff --git a/jwst/regtest/test_nirspec_fs_spec3.py b/jwst/regtest/test_nirspec_fs_spec3.py index 869d6c3949..fd6bb4e783 100644 --- a/jwst/regtest/test_nirspec_fs_spec3.py +++ b/jwst/regtest/test_nirspec_fs_spec3.py @@ -27,8 +27,8 @@ def run_pipeline(rtdata_module): @pytest.mark.bigdata @pytest.mark.parametrize("suffix", ["cal", "crf", "s2d", "x1d"]) -@pytest.mark.parametrize("source_id,slit_name", [("s00001","s200a2"), ("s00021","s200a1"), ("s00023","s400a1"), - ("s00024","s1600a1"), ("s00025","s200b1")]) +@pytest.mark.parametrize("source_id,slit_name", [("s000000001","s200a2"), ("s000000021","s200a1"), + ("s000000023","s400a1"), ("s000000024","s1600a1"), ("s000000025","s200b1")]) def test_nirspec_fs_spec3(run_pipeline, rtdata_module, fitsdiff_default_kwargs, suffix, source_id, slit_name): """Test spec3 pipeline on a set of NIRSpec FS exposures.""" rtdata = rtdata_module diff --git a/jwst/regtest/test_nirspec_mos_spec2.py b/jwst/regtest/test_nirspec_mos_spec2.py index 8bc6f909d6..48e9185fc2 100644 --- a/jwst/regtest/test_nirspec_mos_spec2.py +++ b/jwst/regtest/test_nirspec_mos_spec2.py @@ -12,10 +12,10 @@ def run_pipeline(rtdata_module): rtdata = rtdata_module # Get the MSA metadata file referenced in the input exposure - rtdata.get_data("nirspec/mos/jw01180025001_01_msa.fits") + rtdata.get_data("nirspec/mos/jw01345066001_01_msa.fits") # Get the input ASN file and exposures - rtdata.get_asn("nirspec/mos/jw01180-o025_20221129t204108_spec2_00037_asn.json") + rtdata.get_asn("nirspec/mos/jw01345-o066_20230831t181155_spec2_00010_asn.json") # Run the calwebb_spec2 pipeline; save results from intermediate steps args = ["calwebb_spec2", rtdata.input, @@ -41,11 +41,13 @@ def run_pipeline(rtdata_module): "srctype", "pathloss", "barshadow", "cal", "s2d", "x1d"]) def test_nirspec_mos_spec2(run_pipeline, fitsdiff_default_kwargs, suffix): """Regression test of the calwebb_spec2 pipeline on a - NIRSpec MOS exposure.""" + NIRSpec MOS exposure. Using an exposure that's part of a + 3-shutter nod sequence, so there are nodded exposures available + with which to do background subtraction.""" # Run the pipeline and retrieve outputs rtdata = run_pipeline - output = f"jw01180025001_05101_00001_nrs2_{suffix}.fits" + output = f"jw01345066001_05101_00003_nrs1_{suffix}.fits" rtdata.output = output # Get the truth files diff --git a/jwst/regtest/test_nirspec_mos_spec3.py b/jwst/regtest/test_nirspec_mos_spec3.py index 507eb5c1c5..4e54e30a11 100644 --- a/jwst/regtest/test_nirspec_mos_spec3.py +++ b/jwst/regtest/test_nirspec_mos_spec3.py @@ -11,7 +11,7 @@ def run_pipeline(rtdata_module): """Run calwebb_spec3 on NIRSpec MOS data.""" rtdata = rtdata_module - rtdata.get_asn("nirspec/mos/jw00626-o030_20191210t193826_spec3_001_asn.json") + rtdata.get_asn("nirspec/mos/jw01345-o066_20230831t181155_spec3_00002_asn.json") # Run the calwebb_spec3 pipeline on the association args = ["calwebb_spec3", rtdata.input] @@ -22,13 +22,15 @@ def run_pipeline(rtdata_module): @pytest.mark.bigdata @pytest.mark.parametrize("suffix", ["cal", "crf", "s2d", "x1d"]) -@pytest.mark.parametrize("source_id", ["s00000", "s00227", "s00279", "s00443", - "s00482", "s02315"]) +@pytest.mark.parametrize("source_id", ["b000000030", "b000000031", + "s000004385", "s000007380", + "v000000048", "v000000049", + "v000000053", "v000000056"]) def test_nirspec_mos_spec3(run_pipeline, suffix, source_id, fitsdiff_default_kwargs): """Check results of calwebb_spec3""" rtdata = run_pipeline - output = f"jw00626-o030_{source_id}_nirspec_f170lp-g235m_{suffix}.fits" + output = f"jw01345-o066_{source_id}_nirspec_f170lp-g235m_{suffix}.fits" rtdata.output = output rtdata.get_truth(f"truth/test_nirspec_mos_spec3/{output}") diff --git a/jwst/resample/tests/test_resample_step.py b/jwst/resample/tests/test_resample_step.py index 39585c713b..fc9cd101b8 100644 --- a/jwst/resample/tests/test_resample_step.py +++ b/jwst/resample/tests/test_resample_step.py @@ -70,6 +70,7 @@ def nirspec_rate(): 'ysize': 416, 'ystart': 529} im.meta.observation = { + 'program_number': '1234', 'date': '2016-09-05', 'time': '8:59:37'} im.meta.exposure = {