Merge branch master into JP-3690

CHANGES.rst

@@ -23,6 +23,12 @@ assign_wcs
 - Moved `update_s_region_imaging`, `update_s_region_keyword`, and `wcs_from_footprints`
   into stcal. [#8624]
 
+associations
+------------
+
+- Restored slit name to level 3 product names for NIRSpec BOTS and background
+  fixed slit targets. [#8699]
+
 cube_build
 ----------
 
@@ -48,6 +54,8 @@ general
 
 - Update required stcal version to 1.8.0. [#8706]
 
+- Increase minimum required stpipe. [#8713]
+
 master_background
 -----------------
 
@@ -105,6 +113,13 @@ resample_spec
   so that the spectral resampling step only exposes parameters that are appropriate
   for spectral data. [#8622]
 
+resample_step
+-------------
+
+- Fixed a typo in ``load_custom_wcs`` from ``array_shape`` to ``pixel_shape`` and
+  changed to use values in the top-level ASDF structure if the values in the WCS
+  are ``None``. [#8698]
+
 scripts
 -------
 

jwst/ami/leastsqnrm.py

@@ -542,15 +542,17 @@ def matrix_operations(img, model, flux=None, linfit=False, dqm=None):
     flatimg = img.reshape(np.shape(img)[0] * np.shape(img)[1])
     flatdqm = dqm.reshape(np.shape(img)[0] * np.shape(img)[1])
     log.info("fringefitting.leastsqnrm.matrix_operations(): ")
-    log.info(f"\timg {img.shape:} \n\tdqm {dqm.shape:}")
+    log.info(f"\timg {img.shape:}")
+    log.info(f"\tdqm {dqm.shape:}")
     log.info(
         f"\tL x W = {img.shape[0]:d} x {img.shape[1]:d} = {img.shape[0] * img.shape[1]:d}",
     )
     log.info(f"\tflatimg {flatimg.shape:}")
     log.info(f"\tflatdqm {flatdqm.shape:}")
 
 
-    log.info("\n\ttype(dqm) %s", type(dqm))
+    log.info("")
+    log.info("\ttype(dqm) %s", type(dqm))
     if dqm is not None:
         nanlist = np.where(flatdqm)  # where DO_NOT_USE up.
     else:

jwst/assign_wcs/assign_wcs_step.py

@@ -118,7 +118,8 @@ def process(self, input, *args, **kwargs):
 
             except (ValueError, RuntimeError) as e:
                 log.warning("Failed to update 'meta.wcsinfo' with FITS SIP "
-                            f'approximation. Reported error is:\n"{e.args[0]}"')
+                            "approximation. Reported error is:")
+                log.warning(f'"{e.args[0]}"')
         else:  # WFSS modes
             try:
                 # A bounding_box is needed for the imaging WCS
@@ -137,6 +138,7 @@ def process(self, input, *args, **kwargs):
                                  )
             except (ValueError, RuntimeError) as e:
                 log.warning("Failed to update 'meta.wcsinfo' with FITS SIP "
-                            f'approximation. Reported error is:\n"{e.args[0]}"')
+                            "approximation. Reported error is:")
+                log.warning(f'"{e.args[0]}"')
 
         return result

jwst/associations/association.py

@@ -198,7 +198,8 @@ def validate(cls, asn):
         try:
             jsonschema.validate(asn_data, asn_schema)
         except (AttributeError, jsonschema.ValidationError) as err:
-            logger.debug('Validation failed:\n%s', err)
+            logger.debug('Validation failed:')
+            logger.debug(str(err))
             raise AssociationNotValidError('Validation failed') from err
 
         # Validate no path data for expnames

jwst/associations/lib/dms_base.py

@@ -765,17 +765,15 @@ def _get_slit_name(self):
             else:
                 values.sort(key=str.lower)
                 value = format_list(values)
-                if value not in _EMPTY:
+                if value not in _EMPTY and value not in slit_names:
                     slit_names.append(value)
 
         # Build the string. Sort the elements in order to
         # create data-independent results
         slit_names.sort(key=str.lower)
         slit_name = '-'.join(slit_names)
 
-        if slit_name == '':
-            slit_name = None
-
+        # Slit name may be empty string
         return slit_name
 
     def _get_subarray(self):

jwst/associations/lib/rules_level3_base.py

@@ -158,6 +158,10 @@ def _dms_product_name(association):
 
         opt_elem = association._get_opt_element()
 
+        slit_name = association._get_slit_name()
+        if slit_name:
+            slit_name = '-' + slit_name
+
         exposure = association._get_exposure()
         if exposure:
             exposure = '-' + exposure
@@ -170,14 +174,15 @@ def _dms_product_name(association):
             'jw{program}-{acid}'
             '_{target}'
             '_{instrument}'
-            '_{opt_elem}{subarray}'
+            '_{opt_elem}{slit_name}{subarray}'
         )
         product_name = product_name.format(
             program=association.data['program'],
             acid=association.acid.id,
             target=target,
             instrument=instrument,
             opt_elem=opt_elem,
+            slit_name=slit_name,
             subarray=subarray,
             exposure=exposure
         )
@@ -633,6 +638,10 @@ def dms_product_name_sources(asn):
 
     opt_elem = asn._get_opt_element()
 
+    slit_name = asn._get_slit_name()
+    if slit_name:
+        slit_name = '-' + slit_name
+
     subarray = asn._get_subarray()
     if subarray:
         subarray = '-' + subarray
@@ -641,23 +650,26 @@ def dms_product_name_sources(asn):
         'jw{program}-{acid}'
         '_{source_id}'
         '_{instrument}'
-        '_{opt_elem}{subarray}'
+        '_{opt_elem}{slit_name}{subarray}'
     )
     product_name = format_product(
         product_name_format,
         program=asn.data['program'],
         acid=asn.acid.id,
         instrument=instrument,
         opt_elem=opt_elem,
+        slit_name=slit_name,
         subarray=subarray,
     )
 
     return product_name.lower()
 
 
 def dms_product_name_nrsfs_sources(asn):
-    """Produce source-based product names for
-       NIRSpec fixed-slit observations.
+    """Produce source-based product names for NIRSpec fixed-slit observations.
+
+    For this mode, the product names have a placeholder for the
+    slit name, to be filled in later by the pipeline.
 
     Parameters
     ---------
@@ -674,10 +686,6 @@ def dms_product_name_nrsfs_sources(asn):
 
     opt_elem = asn._get_opt_element()
 
-    slit_name = asn._get_slit_name()
-    if slit_name:
-        slit_name = '-' + slit_name
-
     subarray = asn._get_subarray()
     if subarray:
         subarray = '-' + subarray

jwst/associations/tests/test_level3_product_names.py

@@ -9,7 +9,7 @@
     t_path,
 )
 
-from jwst.associations import (AssociationPool, generate)
+from jwst.associations import generate
 from jwst.associations.lib.dms_base import DMSAttrConstraint
 
 
@@ -96,7 +96,7 @@ def test_level3_names(pool_file, global_constraints):
     rules = registry_level3_only(
         global_constraints=global_constraints
     )
-    pool = AssociationPool.read(pool_file)
+    pool = combine_pools(pool_file)
     asns = generate(pool, rules)
     for asn in asns:
         product_name = asn['products'][0]['name']
@@ -110,18 +110,64 @@ def test_level3_names(pool_file, global_constraints):
 
 def test_multiple_optelems(pool_file):
     rules = registry_level3_only()
-    pool = AssociationPool.read(pool_file)
-    asns = generate(pool, rules)
+    pool = combine_pools(pool_file)
+    asns = generate(pool, rules, finalize=False)
     for asn in asns:
         product_name = asn['products'][0]['name']
         if asn['asn_rule'] != 'Asn_Lv3MIRMRS':
             m = re.match(LEVEL3_PRODUCT_NAME_REGEX, product_name)
             assert m is not None
+
+            # there should always be an opt_elem
+            values = ['-'.join(asn.constraints['opt_elem'].found_values)]
+
+            # there may also be an opt_elem2, fixed slit or 2, or a subarray
+            for extra in ['opt_elem2', 'fxd_slit', 'fxd_slit2', 'subarray']:
+
+                # special rules for fixed slit for NRS FS:
+                # it gets a format placeholder instead of the value
+                if asn['asn_rule'] == 'Asn_Lv3NRSFSS':
+                    if extra == 'fxd_slit':
+                        values.append('{slit_name}')
+                        continue
+                    elif extra == 'fxd_slit2':
+                        continue
+
+                try:
+                    value = '-'.join(asn.constraints[extra].found_values)
+                except KeyError:
+                    value = None
+
+                # empty values and subarray = full are not recorded
+                if value not in EMPTY and value != 'full':
+                    values.append(value)
+
+            assert m.groupdict()['opt_elem'] == '-'.join(values)
+
+
+def test_tso3_names():
+    rules = registry_level3_only()
+    tso_pool = t_path('data/pool_021_tso.csv')
+    pool = combine_pools(tso_pool)
+    asns = generate(pool, rules, finalize=False)
+    for asn in asns:
+        product_name = asn['products'][0]['name']
+
+        m = re.match(LEVEL3_PRODUCT_NAME_REGEX, product_name)
+        assert m is not None
+
+        # there should always be an opt_elem
+        values = ['-'.join(asn.constraints['opt_elem'].found_values)]
+
+        # there may also be an opt_elem2, fixed slit or 2, or a subarray
+        for extra in ['opt_elem2', 'fxd_slit', 'fxd_slit2', 'subarray']:
             try:
-                value = '-'.join(asn.constraints['opt_elem2'].found_values)
+                value = '-'.join(asn.constraints[extra].found_values)
             except KeyError:
                 value = None
-            if value in EMPTY:
-                assert '-' not in m.groupdict()['opt_elem']
-            else:
-                assert '-' in m.groupdict()['opt_elem']
+
+            # empty values and subarray = full are not recorded
+            if value not in EMPTY and value != 'full':
+                values.append(value)
+
+        assert m.groupdict()['opt_elem'] == '-'.join(values)

jwst/dark_current/dark_current_step.py

@@ -100,8 +100,8 @@ def set_average_dark_current(self, input_model, dark_model):
             if np.sum(dark_model.average_dark_current) == 0.0:
                 input_model.average_dark_current[:, :] = dark_model.meta.exposure.average_dark_current
             elif np.shape(input_model.average_dark_current) != np.shape(dark_model.average_dark_current):
-                self.log.warning("DarkModel average_dark_current does not match shape of data.\n"
-                                 "Dark current from reference file cannot be applied.")
+                self.log.warning("DarkModel average_dark_current does not match shape of data.")
+                self.log.warning("Dark current from reference file cannot be applied.")
             else:
                 input_model.average_dark_current = dark_model.average_dark_current
 

jwst/extract_1d/extract.py

@@ -805,13 +805,12 @@ def sanity_check_limits(
     """
     if (ap_wcs.xstart >= ap_ref.xstop or ap_wcs.xstop <= ap_ref.xstart or
             ap_wcs.ystart >= ap_ref.ystop or ap_wcs.ystop <= ap_ref.ystart):
-        log.warning(
-            f"The WCS bounding box is outside the aperture:\n\t"
-            f"aperture: {ap_ref.xstart}, {ap_ref.xstop}, {ap_ref.ystart}, {ap_ref.ystop}\n\t"
-            f"wcs: {ap_wcs.xstart}, {ap_wcs.xstop}, {ap_wcs.ystart}, {ap_wcs.ystop}\n"
-            f"so the wcs bounding box will be ignored."
-        )
-
+        log.warning("The WCS bounding box is outside the aperture:")
+        log.warning(f"\taperture: {ap_ref.xstart}, {ap_ref.xstop}, "
+                    f"{ap_ref.ystart}, {ap_ref.ystop}")
+        log.warning(f"\twcs: {ap_wcs.xstart}, {ap_wcs.xstop}, "
+                    f"{ap_wcs.ystart}, {ap_wcs.ystop}")
+        log.warning("so the wcs bounding box will be ignored.")
         flag = False
     else:
         flag = True

jwst/extract_1d/extract1d.py

@@ -269,8 +269,8 @@ def extract1d(image, var_poisson, var_rnoise, var_flat, lambdas, disp_range,
             elif len(bkg_model) < bkg_order:
                 log.debug(f"Not enough valid pixels to determine background "
                           f"with the required order for lambda={lam:.6f} "
-                          f"(column {x:d})\n"
-                          f"Lowering background order to {len(bkg_model)}")
+                          f"(column {x:d})")
+                log.debug(f"Lowering background order to {len(bkg_model)}")
 
         # Extract the source, and optionally subtract background using the
         # fit to the background for this column.  Even if

jwst/lib/set_telescope_pointing.py

@@ -603,9 +603,9 @@ def add_wcs(filename, allow_any_file=False, force_level1bmodel=False,
 
     try:
         if type(model) not in EXPECTED_MODELS:
-            logger.warning(f'Input {model} is not of an expected type (uncal, rate, rateints)'
-                           '\n    Updating pointing may have no effect or detrimental effects on the WCS information,'
-                           '\n    especially if the input is the result of Level2b or higher calibration.')
+            logger.warning(f'Input {model} is not of an expected type (uncal, rate, rateints)')
+            logger.warning('    Updating pointing may have no effect or detrimental effects on the WCS information,')
+            logger.warning('    especially if the input is the result of Level2b or higher calibration.')
             if not allow_any_file:
                 raise TypeError(f'Input model {model} is not one of {EXPECTED_MODELS} and `allow_any_file` is `False`.'
                                 '\n\tFailing WCS processing.')
@@ -814,10 +814,10 @@ def update_wcs_from_fgs_guiding(model, t_pars, default_roll_ref=0.0, default_vpa
         model, t_pars, default_roll_ref, default_vparity, default_v3yangle
     )
 
-    logger.info('WCS info:'
-                f'\n\tcrpix1: {crpix1} crpix2: {crpix2}'
-                f'\n\tcrval1: {crval1} crval2: {crval2}'
-                f'\n\tpc_matrix: {pc_matrix}')
+    logger.info('WCS info:')
+    logger.info(f'\tcrpix1: {crpix1} crpix2: {crpix2}')
+    logger.info(f'\tcrval1: {crval1} crval2: {crval2}')
+    logger.info(f'\tpc_matrix: {pc_matrix}')
 
     model.meta.wcsinfo.crpix1 = crpix1
     model.meta.wcsinfo.crpix2 = crpix2
@@ -1242,8 +1242,8 @@ def calc_transforms_track_tr_202111(t_pars: TransformParameters):
     # Check on telemetry for FGS ID. If invalid, use either user-specified or default to 1.
     fgsid = t_pars.pointing.fgsid
     if fgsid not in FGSIDS:
-        logger.warning(f'Method {t_pars.method} requires a valid FGS ID in telementry.'
-                       '\nHowever telemetry reports an invalid id of {fgsid}')
+        logger.warning(f'Method {t_pars.method} requires a valid FGS ID in telementry.')
+        logger.warning('However telemetry reports an invalid id of {fgsid}')
         if t_pars.fgsid in FGSIDS:
             fgsid = t_pars.fgsid
             logger.warning(f'Using user-specified ID of {fgsid}')