diff --git a/.gitignore b/.gitignore index db6b3a8c4b..11d23b8394 100644 --- a/.gitignore +++ b/.gitignore @@ -52,7 +52,7 @@ tst*.fits # Logs and databases # ###################### *.log -!pypeit/data/spectrographs/keck_deimos/gain_ronoise/*.log +!pypeit/data/spectrographs/keck_deimos/gain_ronoise/*/*.log *.sql *.sqlite diff --git a/CHANGES.rst b/CHANGES.rst index e8e3b3eeca..1306a4a6b7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -58,8 +58,7 @@ RELEASE FILE IN doc/releases files that have frametype None (this prevent ``run_pypeit`` to crash) - Added a function ``check_spectrograph()`` (currently only defined for LRIS), that checks (during ``pypeit_setup``) if the selected spectrograph is the - corrected one for the data used. - + corrected one for the data used. 1.13.0 (2 June 2023) -------------------- diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000..02b0303597 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,9 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +authors: + - family-names: Prochaska, Hennawi, Westfall, Cooke, Wang, Hsyu, Davies, Farina, Pelliccia + given-names: J. Xavier, Joseph, Kyle, Ryan, Feige, Tiffany, Fred, Emanuele, Debora +title: "PypeIt" +version: 1.14.0 +doi: 10.21105/joss.02308 +date-released: 2020-08-28 \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 019c6268bc..f7b4bd542a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,7 +15,7 @@ recursive-include scripts * prune build prune docs/_build prune docs/api -prune pypeit/deprecated +prune deprecated global-exclude *.pyc *.o *.so *.DS_Store diff --git a/pypeit/deprecated/.bumpversion.cfg b/deprecated/.bumpversion.cfg similarity index 100% rename from pypeit/deprecated/.bumpversion.cfg rename to deprecated/.bumpversion.cfg diff --git a/pypeit/deprecated/.travis.yml b/deprecated/.travis.yml similarity index 100% rename from pypeit/deprecated/.travis.yml rename to deprecated/.travis.yml diff --git a/pypeit/deprecated/ararclines.py b/deprecated/ararclines.py similarity index 100% rename from pypeit/deprecated/ararclines.py rename to deprecated/ararclines.py diff --git a/pypeit/deprecated/arc_old.py b/deprecated/arc_old.py similarity index 100% rename from pypeit/deprecated/arc_old.py rename to deprecated/arc_old.py diff --git a/pypeit/deprecated/arcid_plot.py b/deprecated/arcid_plot.py similarity index 100% rename from pypeit/deprecated/arcid_plot.py rename to deprecated/arcid_plot.py diff --git a/pypeit/aroutput.yml b/deprecated/aroutput.yml similarity index 100% rename from pypeit/aroutput.yml rename to deprecated/aroutput.yml diff --git a/pypeit/deprecated/arproc.py b/deprecated/arproc.py similarity index 100% rename from pypeit/deprecated/arproc.py rename to deprecated/arproc.py diff --git a/pypeit/deprecated/arsciexp.py b/deprecated/arsciexp.py similarity index 100% rename from pypeit/deprecated/arsciexp.py rename to deprecated/arsciexp.py diff --git a/pypeit/deprecated/arspecobj.py b/deprecated/arspecobj.py similarity index 100% rename from pypeit/deprecated/arspecobj.py rename to deprecated/arspecobj.py diff --git a/pypeit/deprecated/artrace.py b/deprecated/artrace.py similarity index 100% rename from pypeit/deprecated/artrace.py rename to deprecated/artrace.py diff --git a/pypeit/deprecated/biasframe.py b/deprecated/biasframe.py similarity index 100% rename from pypeit/deprecated/biasframe.py rename to deprecated/biasframe.py diff --git a/pypeit/deprecated/bin/pypeit_arcid_plot b/deprecated/bin/pypeit_arcid_plot similarity index 100% rename from pypeit/deprecated/bin/pypeit_arcid_plot rename to deprecated/bin/pypeit_arcid_plot diff --git a/pypeit/deprecated/bin/pypeit_chk_2dslits b/deprecated/bin/pypeit_chk_2dslits similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_2dslits rename to deprecated/bin/pypeit_chk_2dslits diff --git a/pypeit/deprecated/bin/pypeit_chk_alignments b/deprecated/bin/pypeit_chk_alignments similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_alignments rename to deprecated/bin/pypeit_chk_alignments diff --git a/pypeit/deprecated/bin/pypeit_chk_edges b/deprecated/bin/pypeit_chk_edges similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_edges rename to deprecated/bin/pypeit_chk_edges diff --git a/pypeit/deprecated/bin/pypeit_chk_flats b/deprecated/bin/pypeit_chk_flats similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_flats rename to deprecated/bin/pypeit_chk_flats diff --git a/pypeit/deprecated/bin/pypeit_chk_for_calibs b/deprecated/bin/pypeit_chk_for_calibs similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_for_calibs rename to deprecated/bin/pypeit_chk_for_calibs diff --git a/pypeit/deprecated/bin/pypeit_chk_tilts b/deprecated/bin/pypeit_chk_tilts similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_tilts rename to deprecated/bin/pypeit_chk_tilts diff --git a/pypeit/deprecated/bin/pypeit_chk_wavecalib b/deprecated/bin/pypeit_chk_wavecalib similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_wavecalib rename to deprecated/bin/pypeit_chk_wavecalib diff --git a/pypeit/deprecated/bin/pypeit_coadd_1dspec b/deprecated/bin/pypeit_coadd_1dspec similarity index 100% rename from pypeit/deprecated/bin/pypeit_coadd_1dspec rename to deprecated/bin/pypeit_coadd_1dspec diff --git a/pypeit/deprecated/bin/pypeit_coadd_2dspec b/deprecated/bin/pypeit_coadd_2dspec similarity index 100% rename from pypeit/deprecated/bin/pypeit_coadd_2dspec rename to deprecated/bin/pypeit_coadd_2dspec diff --git a/pypeit/deprecated/bin/pypeit_coadd_datacube b/deprecated/bin/pypeit_coadd_datacube similarity index 100% rename from pypeit/deprecated/bin/pypeit_coadd_datacube rename to deprecated/bin/pypeit_coadd_datacube diff --git a/pypeit/deprecated/bin/pypeit_compare_sky b/deprecated/bin/pypeit_compare_sky similarity index 100% rename from pypeit/deprecated/bin/pypeit_compare_sky rename to deprecated/bin/pypeit_compare_sky diff --git a/pypeit/deprecated/bin/pypeit_find_objects b/deprecated/bin/pypeit_find_objects similarity index 100% rename from pypeit/deprecated/bin/pypeit_find_objects rename to deprecated/bin/pypeit_find_objects diff --git a/pypeit/deprecated/bin/pypeit_flux_calib b/deprecated/bin/pypeit_flux_calib similarity index 100% rename from pypeit/deprecated/bin/pypeit_flux_calib rename to deprecated/bin/pypeit_flux_calib diff --git a/pypeit/deprecated/bin/pypeit_flux_setup b/deprecated/bin/pypeit_flux_setup similarity index 100% rename from pypeit/deprecated/bin/pypeit_flux_setup rename to deprecated/bin/pypeit_flux_setup diff --git a/pypeit/deprecated/bin/pypeit_identify b/deprecated/bin/pypeit_identify similarity index 100% rename from pypeit/deprecated/bin/pypeit_identify rename to deprecated/bin/pypeit_identify diff --git a/pypeit/deprecated/bin/pypeit_lowrdx_pixflat b/deprecated/bin/pypeit_lowrdx_pixflat similarity index 100% rename from pypeit/deprecated/bin/pypeit_lowrdx_pixflat rename to deprecated/bin/pypeit_lowrdx_pixflat diff --git a/pypeit/deprecated/bin/pypeit_lowrdx_skyspec b/deprecated/bin/pypeit_lowrdx_skyspec similarity index 100% rename from pypeit/deprecated/bin/pypeit_lowrdx_skyspec rename to deprecated/bin/pypeit_lowrdx_skyspec diff --git a/pypeit/deprecated/bin/pypeit_obslog b/deprecated/bin/pypeit_obslog similarity index 100% rename from pypeit/deprecated/bin/pypeit_obslog rename to deprecated/bin/pypeit_obslog diff --git a/pypeit/deprecated/bin/pypeit_qa_html b/deprecated/bin/pypeit_qa_html similarity index 100% rename from pypeit/deprecated/bin/pypeit_qa_html rename to deprecated/bin/pypeit_qa_html diff --git a/pypeit/deprecated/bin/pypeit_ql_keck_mosfire b/deprecated/bin/pypeit_ql_keck_mosfire similarity index 100% rename from pypeit/deprecated/bin/pypeit_ql_keck_mosfire rename to deprecated/bin/pypeit_ql_keck_mosfire diff --git a/pypeit/deprecated/bin/pypeit_ql_keck_nires b/deprecated/bin/pypeit_ql_keck_nires similarity index 100% rename from pypeit/deprecated/bin/pypeit_ql_keck_nires rename to deprecated/bin/pypeit_ql_keck_nires diff --git a/pypeit/deprecated/bin/pypeit_ql_mos b/deprecated/bin/pypeit_ql_mos similarity index 100% rename from pypeit/deprecated/bin/pypeit_ql_mos rename to deprecated/bin/pypeit_ql_mos diff --git a/pypeit/deprecated/bin/pypeit_sensfunc b/deprecated/bin/pypeit_sensfunc similarity index 100% rename from pypeit/deprecated/bin/pypeit_sensfunc rename to deprecated/bin/pypeit_sensfunc diff --git a/pypeit/deprecated/bin/pypeit_setup b/deprecated/bin/pypeit_setup similarity index 100% rename from pypeit/deprecated/bin/pypeit_setup rename to deprecated/bin/pypeit_setup diff --git a/pypeit/deprecated/bin/pypeit_show_1dspec b/deprecated/bin/pypeit_show_1dspec similarity index 100% rename from pypeit/deprecated/bin/pypeit_show_1dspec rename to deprecated/bin/pypeit_show_1dspec diff --git a/pypeit/deprecated/bin/pypeit_show_2dspec b/deprecated/bin/pypeit_show_2dspec similarity index 100% rename from pypeit/deprecated/bin/pypeit_show_2dspec rename to deprecated/bin/pypeit_show_2dspec diff --git a/pypeit/deprecated/bin/pypeit_show_arxiv b/deprecated/bin/pypeit_show_arxiv similarity index 100% rename from pypeit/deprecated/bin/pypeit_show_arxiv rename to deprecated/bin/pypeit_show_arxiv diff --git a/pypeit/deprecated/bin/pypeit_show_wvcalib b/deprecated/bin/pypeit_show_wvcalib similarity index 100% rename from pypeit/deprecated/bin/pypeit_show_wvcalib rename to deprecated/bin/pypeit_show_wvcalib diff --git a/pypeit/deprecated/bin/pypeit_skysub_regions b/deprecated/bin/pypeit_skysub_regions similarity index 100% rename from pypeit/deprecated/bin/pypeit_skysub_regions rename to deprecated/bin/pypeit_skysub_regions diff --git a/pypeit/deprecated/bin/pypeit_tellfit b/deprecated/bin/pypeit_tellfit similarity index 100% rename from pypeit/deprecated/bin/pypeit_tellfit rename to deprecated/bin/pypeit_tellfit diff --git a/pypeit/deprecated/bin/pypeit_trace_edges b/deprecated/bin/pypeit_trace_edges similarity index 100% rename from pypeit/deprecated/bin/pypeit_trace_edges rename to deprecated/bin/pypeit_trace_edges diff --git a/pypeit/deprecated/bin/pypeit_view_fits b/deprecated/bin/pypeit_view_fits similarity index 100% rename from pypeit/deprecated/bin/pypeit_view_fits rename to deprecated/bin/pypeit_view_fits diff --git a/pypeit/deprecated/bin/run_pypeit b/deprecated/bin/run_pypeit similarity index 100% rename from pypeit/deprecated/bin/run_pypeit rename to deprecated/bin/run_pypeit diff --git a/pypeit/deprecated/bpmimage.py b/deprecated/bpmimage.py similarity index 100% rename from pypeit/deprecated/bpmimage.py rename to deprecated/bpmimage.py diff --git a/pypeit/deprecated/check_requirements.py b/deprecated/check_requirements.py similarity index 100% rename from pypeit/deprecated/check_requirements.py rename to deprecated/check_requirements.py diff --git a/pypeit/deprecated/chk_tilts.py b/deprecated/chk_tilts.py similarity index 100% rename from pypeit/deprecated/chk_tilts.py rename to deprecated/chk_tilts.py diff --git a/pypeit/deprecated/coadd.py b/deprecated/coadd.py similarity index 100% rename from pypeit/deprecated/coadd.py rename to deprecated/coadd.py diff --git a/pypeit/deprecated/coadd2d.py b/deprecated/coadd2d.py similarity index 100% rename from pypeit/deprecated/coadd2d.py rename to deprecated/coadd2d.py diff --git a/pypeit/deprecated/coadd_1dspec_old.py b/deprecated/coadd_1dspec_old.py similarity index 100% rename from pypeit/deprecated/coadd_1dspec_old.py rename to deprecated/coadd_1dspec_old.py diff --git a/pypeit/deprecated/combine.py b/deprecated/combine.py similarity index 100% rename from pypeit/deprecated/combine.py rename to deprecated/combine.py diff --git a/pypeit/deprecated/datacube.py b/deprecated/datacube.py similarity index 100% rename from pypeit/deprecated/datacube.py rename to deprecated/datacube.py diff --git a/pypeit/deprecated/debugger.py b/deprecated/debugger.py similarity index 100% rename from pypeit/deprecated/debugger.py rename to deprecated/debugger.py diff --git a/pypeit/deprecated/defs.py b/deprecated/defs.py similarity index 100% rename from pypeit/deprecated/defs.py rename to deprecated/defs.py diff --git a/pypeit/deprecated/ech_coadd.py b/deprecated/ech_coadd.py similarity index 100% rename from pypeit/deprecated/ech_coadd.py rename to deprecated/ech_coadd.py diff --git a/pypeit/deprecated/extract.py b/deprecated/extract.py similarity index 100% rename from pypeit/deprecated/extract.py rename to deprecated/extract.py diff --git a/pypeit/deprecated/filter.py b/deprecated/filter.py similarity index 100% rename from pypeit/deprecated/filter.py rename to deprecated/filter.py diff --git a/deprecated/find_objects.py b/deprecated/find_objects.py new file mode 100644 index 0000000000..46263e217a --- /dev/null +++ b/deprecated/find_objects.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python +# +# See top-level LICENSE file for Copyright information +# +# -*- coding: utf-8 -*- +""" +This script enables the viewing of a processed FITS file +with extras. Run above the Science/ folder. +""" + +import os +import argparse +import numpy as np + +from astropy.table import Table + +from pypeit import msgs +from pypeit import io +from pypeit import slittrace +from pypeit.core import gui +from pypeit.core.parse import get_dnum + + +def parse_args(options=None, return_parser=False): + + parser = argparse.ArgumentParser(description='Display sky subtracted, spec2d image in the' + 'interactive object finding GUI. Run above' + 'the Science/ folder', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument('file', type = str, default=None, help='PYPEIT spec2d file') + parser.add_argument("--list", default=False, help="List the extensions only?", + action="store_true") + parser.add_argument('--det', default=1, type=int, help="Detector") + parser.add_argument("--old", default=False, action="store_true", help="Used old slit tracing") + + if return_parser: + return parser + + return parser.parse_args() if options is None else parser.parse_args(options) + + +def parse_traces(hdulist_1d, det_nm): + """Extract the relevant trace information + """ + traces = dict(traces=[], fwhm=[]) + pkflux = [] + for hdu in hdulist_1d: + if det_nm in hdu.name: + tbl = Table(hdu.data) + trace = tbl['TRACE_SPAT'] + fwhm = tbl['FWHMFIT'] + obj_id = hdu.name.split('-')[0] + traces['traces'].append(trace.copy()) + traces['fwhm'].append(np.median(fwhm)) + pkflux.append(np.median(tbl['BOX_COUNTS'])) + traces['pkflux'] = np.array(pkflux) + return traces + + +def main(args): + + raise NotImplementedError('This script is currently out of date.') + + # List only? + hdu = io.fits_open(args.file) + head0 = hdu[0].header + if args.list: + hdu.info() + return + + # Init + sdet = get_dnum(args.det, prefix=False) + + # One detector, sky sub for now + names = [hdu[i].name for i in range(len(hdu))] + + try: + exten = names.index('DET{:s}-PROCESSED'.format(sdet)) + except: # Backwards compatability + msgs.error('Requested detector {:s} was not processed.\n' + 'Maybe you chose the wrong one to view?\n' + 'Set with --det= or check file contents with --list'.format(sdet)) + sciimg = hdu[exten].data + try: + exten = names.index('DET{:s}-SKY'.format(sdet)) + except: # Backwards compatability + msgs.error('Requested detector {:s} has no sky model.\n' + 'Maybe you chose the wrong one to view?\n' + 'Set with --det= or check file contents with --list'.format(sdet)) + skymodel = hdu[exten].data + try: + exten = names.index('DET{:s}-MASK'.format(sdet)) + except ValueError: # Backwards compatability + msgs.error('Requested detector {:s} has no bit mask.\n' + 'Maybe you chose the wrong one to view?\n' + 'Set with --det= or check file contents with --list'.format(sdet)) + + mask = hdu[exten].data + frame = (sciimg - skymodel) * (mask == 0) + + mdir = head0['PYPMFDIR'] + mkey = head0['FRAMMKEY'] + mast_key = '{0}_{1:02d}'.format(mkey, args.det) + if not os.path.exists(mdir): + mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir)) + msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base)) + mdir = mdir_base + + # Assumes a MasterSlit file has been written + #slits = slittrace.SlitTraceSet.from_master('{0}_{1:02d}'.format(head0['TRACMKEY'], args.det), + # mdir) + # Load the slits information + slits = slittrace.SlitTraceSet.from_master(mast_key, mdir) + + # Object traces + left, right, mask = slits.select_edges() + msgs.error("You need to choose which slits you care about here") + + # Get object traces + spec1d_file = args.file.replace('spec2d', 'spec1d') + if os.path.isfile(spec1d_file): + hdulist_1d = io.fits_open(spec1d_file) + else: + hdulist_1d = [] + msgs.warn('Could not find spec1d file: {:s}'.format(spec1d_file) + msgs.newline() + + ' No objects were extracted.') + + msgs.error("This code needs to be refactored since tslits_dict was removed...") + import pdb + pdb.set_trace() + tslits_dict['objtrc'] = parse_traces(hdulist_1d, det_nm) + obj_trace = parse_traces(hdulist_1d, 'DET{:s}'.format(sdet)) + + # TODO :: Need to include standard star trace in the spec2d files + std_trace = None + + # Extract some trace models + fwhm = 2 # Start with some default value + # TODO: Dictionaries like this are a pet peeve of mine. I'd prefer + # either individual objects or a class with a well-formed data model. + # TODO: Why do all of these dictionary elements need fwhm? Can they + # be different? + trace_models = dict() + # Brightest object on slit + trace_models['object'] = dict(trace_model=None, fwhm=fwhm) + if len(obj_trace['pkflux']) > 0: + smash_peakflux = obj_trace['pkflux'] + ibri = smash_peakflux.argmax() + trace_models['object']['trace_model'] = obj_trace['traces'][ibri] + trace_models['object']['fwhm'] = obj_trace['fwhm'][ibri] + # Standard star trace + trace_models['std'] = dict(trace_model=std_trace, fwhm=trace_models['object']['fwhm']) + # Trace of the slit edge + # TODO: Any particular reason to use the lefts? + trace_models['slit'] = dict(trace_model=left.copy(), fwhm=trace_models['object']['fwhm']) + + # Finally, initialise the GUI + gui.object_find.initialise(args.det, frame, left, right, obj_trace, trace_models, None, + printout=True, slit_ids=slits.id) + + ofgui = gui_object_find.initialise(args.det, frame, tslits_dict, None, printout=True, slit_ids=slits.id) + + + +def illum_profile_spatial(self, skymask=None, trim_edg=(0, 0), debug=False): + """ + Calculate the residual spatial illumination profile using the sky regions. + + The residual is calculated using the differential: + + .. code-block:: python + + correction = amplitude * (1 + spatial_shift * (dy/dx)/y) + + where ``y`` is the spatial profile determined from illumflat, and + spatial_shift is the residual spatial flexure shift in units of pixels. + + Args: + skymask (`numpy.ndarray`_): + Mask of sky regions where the spatial illumination will be determined + trim_edg (:obj:`tuple`): + A tuple of two ints indicated how much of the slit edges should be + trimmed when fitting to the spatial profile. + debug (:obj:`bool`): + Show debugging plots? + """ + + msgs.info("Performing spatial sensitivity correction") + # Setup some helpful parameters + skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) + hist_trim = 0 # Trim the edges of the histogram to take into account edge effects + gpm = self.sciImg.select_flag(invert=True) + slitid_img_init = self.slits.slit_img(pad=0, initial=True, flexure=self.spat_flexure_shift) + spatScaleImg = np.ones_like(self.sciImg.image) + # For each slit, grab the spatial coordinates and a spline + # representation of the spatial profile from the illumflat + rawimg = self.sciImg.image.copy() + numbins = int(np.max(self.slits.get_slitlengths(initial=True, median=True))) + spatbins = np.linspace(0.0, 1.0, numbins + 1) + spat_slit = 0.5 * (spatbins[1:] + spatbins[:-1]) + slitlength = np.median(self.slits.get_slitlengths(median=True)) + coeff_fit = np.zeros((self.slits.nslits, 2)) + for sl, slitnum in enumerate(self.slits.spat_id): + msgs.info("Deriving spatial correction for slit {0:d}/{1:d}".format(sl + 1, self.slits.spat_id.size)) + # Get the initial slit locations + onslit_b_init = (slitid_img_init == slitnum) + + # Synthesize ximg, and edgmask from slit boundaries. Doing this outside this + # routine would save time. But this is pretty fast, so we just do it here to make the interface simpler. + spatcoord, edgmask = pixels.ximg_and_edgemask(self.slits_left[:, sl], self.slits_right[:, sl], + onslit_b_init, trim_edg=trim_edg) + + # Make the model histogram + xspl = np.linspace(0.0, 1.0, 10 * int(slitlength)) # Sub sample each pixel with 10 subpixels + # TODO: caliBrate is no longer a dependency. If you need these b-splines pass them in. + modspl = self.caliBrate.flatimages.illumflat_spat_bsplines[sl].value(xspl)[0] + gradspl = interpolate.interp1d(xspl, np.gradient(modspl) / modspl, kind='linear', bounds_error=False, + fill_value='extrapolate') + + # Ignore skymask + coord_msk = onslit_b_init & gpm + hist, _ = np.histogram(spatcoord[coord_msk], bins=spatbins, weights=rawimg[coord_msk]) + cntr, _ = np.histogram(spatcoord[coord_msk], bins=spatbins) + hist_slit_all = hist / (cntr + (cntr == 0)) + histmod, _ = np.histogram(spatcoord[coord_msk], bins=spatbins, weights=gradspl(spatcoord[coord_msk])) + hist_model = histmod / (cntr + (cntr == 0)) + + # Repeat with skymask + coord_msk = onslit_b_init & gpm & skymask_now + hist, _ = np.histogram(spatcoord[coord_msk], bins=spatbins, weights=rawimg[coord_msk]) + cntr, _ = np.histogram(spatcoord[coord_msk], bins=spatbins) + hist_slit = hist / (cntr + (cntr == 0)) + + # Prepare for fit - take the non-zero elements and trim slit edges + if hist_trim == 0: + ww = (hist_slit != 0) + xfit = spat_slit[ww] + yfit = hist_slit_all[ww] + mfit = hist_model[ww] + else: + ww = (hist_slit[hist_trim:-hist_trim] != 0) + xfit = spat_slit[hist_trim:-hist_trim][ww] + yfit = hist_slit_all[hist_trim:-hist_trim][ww] + mfit = hist_model[hist_trim:-hist_trim][ww] + + # Fit the function + spat_func = lambda par, ydata, model: par[0]*(1 + par[1] * model) - ydata + res_lsq = least_squares(spat_func, [np.median(yfit), 0.0], args=(yfit, mfit)) + spatnorm = spat_func(res_lsq.x, 0.0, gradspl(spatcoord[onslit_b_init])) + spatnorm /= spat_func(res_lsq.x, 0.0, gradspl(0.5)) + # Set the scaling factor + spatScaleImg[onslit_b_init] = spatnorm + coeff_fit[sl, :] = res_lsq.x + + if debug: + from matplotlib import pyplot as plt + xplt = np.arange(24) + plt.subplot(121) + plt.plot(xplt[0::2], coeff_fit[::2, 0], 'rx') + plt.plot(xplt[1::2], coeff_fit[1::2, 0], 'bx') + plt.subplot(122) + plt.plot(xplt[0::2], coeff_fit[::2, 1]/10, 'rx') + plt.plot(xplt[1::2], coeff_fit[1::2, 1]/10, 'bx') + plt.show() + plt.imshow(spatScaleImg, vmin=0.99, vmax=1.01) + plt.show() + plt.subplot(133) + plt.plot(xplt[0::2], coeff_fit[::2, 2], 'rx') + plt.plot(xplt[1::2], coeff_fit[1::2, 2], 'bx') + plt.show() + # Apply the relative scale correction + self.apply_relative_scale(spatScaleImg) + +def illum_profile_spectral(self, global_sky, skymask=None): + """Calculate the residual spectral illumination profile using the sky regions. + This uses the same routine as the flatfield spectral illumination profile. + + Args: + global_sky (`numpy.ndarray`_): + Model of the sky + skymask (`numpy.ndarray`_, optional): + Mask of sky regions where the spectral illumination will be determined + """ + trim = self.par['calibrations']['flatfield']['slit_trim'] + sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] + smooth_npix = self.par['calibrations']['flatfield']['slit_illum_smooth_npix'] + gpm = self.sciImg.select_flag(invert=True) + # Note :: Need to provide wavelength to illum_profile_spectral (not the tilts) so that the + # relative spectral sensitivity is calculated at a given wavelength for all slits simultaneously. + scaleImg = flatfield.illum_profile_spectral(self.sciImg.image.copy(), self.waveimg, self.slits, + slit_illum_ref_idx=sl_ref, model=global_sky, gpmask=gpm, + skymask=skymask, trim=trim, flexure=self.spat_flexure_shift, + smooth_npix=smooth_npix) + # Now apply the correction to the science frame + self.apply_relative_scale(scaleImg) + + + +def global_skysub_old(self, skymask=None, update_crmask=True, trim_edg=(0,0), + previous_sky=None, show_fit=False, show=False, show_objs=False, objs_not_masked=False, + reinit_bpm:bool=True): + """ + Perform global sky subtraction. This SlicerIFU-specific routine ensures that the + edges of the slits are not trimmed, and performs a spatial and spectral + correction using the sky spectrum, if requested. See Reduce.global_skysub() + for parameter definitions. + + See base class method for description of parameters. + + Args: + reinit_bpm (:obj:`bool`, optional): + If True (default), the bpm is reinitialized to the initial bpm + Should be False on the final run in case there was a failure + upstream and no sources were found in the slit/order + """ + if self.par['reduce']['findobj']['skip_skysub']: + msgs.info("Skipping global sky sub as per user request") + return np.zeros_like(self.sciImg.image) + + # Generate a global sky sub for all slits separately + global_sky_sep = super().global_skysub(skymask=skymask, update_crmask=update_crmask, + trim_edg=trim_edg, show_fit=show_fit, show=show, + show_objs=show_objs, reinit_bpm=reinit_bpm) + # Check if any slits failed + if np.any(global_sky_sep[self.slitmask >= 0] == 0) and not self.bkg_redux: + # Cannot continue without a sky model for all slits + msgs.error("Global sky subtraction has failed for at least one slit.") + + # Check if flexure or a joint fit is requested + if not self.par['reduce']['skysub']['joint_fit'] and self.par['flexure']['spec_method'] == 'skip': + return global_sky_sep + if self.wv_calib is None: + msgs.error("A wavelength calibration is needed (wv_calib) if a joint sky fit is requested.") + msgs.info("Generating wavelength image") + + self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spat_flexure=self.spat_flexure_shift) + # Calculate spectral flexure + method = self.par['flexure']['spec_method'] + # TODO :: Perhaps include a new label for IFU flexure correction - e.g. 'slitcen_relative' or 'slitcenIFU' or 'IFU' + # :: If a new label is introduced, change the other instances of 'method' (see below), and in flexure.spec_flexure_qa() + if method in ['slitcen']: + self.slitshift = self.calculate_flexure(global_sky_sep) + # Recalculate the wavelength image, and the global sky taking into account the spectral flexure + msgs.info("Generating wavelength image, now accounting for spectral flexure") + self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spec_flexure=self.slitshift, + spat_flexure=self.spat_flexure_shift) + + + # If the joint fit or spec/spat sensitivity corrections are not being performed, return the separate slits sky + if not self.par['reduce']['skysub']['joint_fit']: + return global_sky_sep + + # Do the spatial scaling first + # if self.par['scienceframe']['process']['use_illumflat']: + # # Perform the correction + # self.illum_profile_spatial(skymask=skymask) + # # Re-generate a global sky sub for all slits separately + # global_sky_sep = Reduce.global_skysub(self, skymask=skymask, update_crmask=update_crmask, trim_edg=trim_edg, + # show_fit=show_fit, show=show, show_objs=show_objs) + + # Use sky information in all slits to perform a joint sky fit + global_sky = self.joint_skysub(skymask=skymask, update_crmask=update_crmask, trim_edg=trim_edg, + show_fit=show_fit, show=show, show_objs=show_objs, + objs_not_masked=objs_not_masked) + + return global_sky diff --git a/pypeit/deprecated/flat.py b/deprecated/flat.py similarity index 90% rename from pypeit/deprecated/flat.py rename to deprecated/flat.py index ac9451216e..f59334af70 100644 --- a/pypeit/deprecated/flat.py +++ b/deprecated/flat.py @@ -1025,3 +1025,102 @@ def norm_slits(mstrace, datasec_img, lordloc, rordloc, pixwid, # # return slit_profiles, mstracenrm, extrap_blz +# Deprecated because it is slow, inefficient, and needs more work. The idea is to use a spline fit to control points +# along the spectral direction to construct a map between modelimg and rawimg. See core.flat.poly_map for a faster +# approach. +def spline_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, + slit_illum_ref_idx=0, gpmask=None, thismask=None, nbins=20, debug=False): + """ + Use a spline fit to control points along the spectral direction to construct a map between modelimg and + rawimg. Currently, this routine is only used for image slicer IFUs. + + This problem needs to be recast into a smoothing problem, that can take advantage of the following: + https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.UnivariateSpline.html#scipy.interpolate.UnivariateSpline + where: + y = science/model + w = model/science_error + resid = w*(y-f(x)) + Then, iterate over this. The reason to iterate is that there should be a mapping between science and + science_error. Once we have an estimate of f(x), we can do the following: + spl = spline(science, science_error) + new_science_error = spl(model*f(x)) + Now iterate a few times until new_science_error (and the fit) is converged. + + Parameters + ---------- + rawimg : `numpy.ndarray`_ + Image data that will be used to estimate the spectral relative sensitivity + rawivar : `numpy.ndarray`_ + Inverse variance image of rawimg + waveimg : `numpy.ndarray`_ + Wavelength image + slitmask : `numpy.ndarray`_ + A 2D int mask, the same shape as rawimg, indicating which pixels are on a slit. A -1 value + indicates not on a slit, while any pixels on a slit should have the value of the slit spatial ID + number. + slitmask_trim : `numpy.ndarray`_ + Same as slitmask, but the slit edges are trimmed. + modelimg : `numpy.ndarray`_ + A model of the rawimg data. + slit_illum_ref_idx : :obj:`int` + Index of slit that is used as the reference. + gpmask : `numpy.ndarray`_, optional + Boolean good pixel mask (True = Good) + thismask : `numpy.ndarray`_, optional + A boolean mask (True = good) that indicates all pixels where the scaleImg should be constructed. + If None, the slitmask that is generated by this routine will be used. + nbins : :obj:`int` + Number of bins in the spectral direction to sample the relative spectral sensitivity + debug : :obj:`bool` + If True, some plots will be output to test if the fitting is working correctly. + + Returns + ------- + scale_model: `numpy.ndarray`_ + An image containing the appropriate scaling + """ + # Some variables to consider putting as function arguments + numiter = 4 + + # Start by calculating a ratio of the raming and the modelimg + nspec = rawimg.shape[0] + _ratio = rawimg * utils.inverse(modelimg) + _ratio_ivar = rawivar * modelimg**2 + _fit_wghts = modelimg * np.sqrt(rawivar) + + # Generate the mask + _thismask = thismask if (thismask is not None) else (slitmask > 0) + gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool) + # Extract the list of spatial IDs from the slitmask + slitmask_spatid = np.unique(slitmask) + slitmask_spatid = np.sort(slitmask_spatid[slitmask_spatid > 0]) + + # Create a spline between the raw data and the error + # TODO :: This is slow and inefficient. + embed() + flxsrt = np.argsort(np.ravel(rawimg)) + spl = scipy.interpolate.interp1d(np.ravel(rawimg)[flxsrt], np.ravel(rawivar)[flxsrt], kind='linear', + bounds_error=False, fill_value=0.0, assume_sorted=True) + modelmap = np.zeros_like(rawimg) + for sl, spatid in enumerate(slitmask_spatid): + print("sl") + # Prepare the masks, edges, and fitting variables + this_slit = (slitmask == spatid) + this_slit_trim = (slitmask_trim == spatid) + this_slit_mask = gpm & this_slit_trim + this_wave = waveimg[this_slit_mask] + this_wghts = _fit_wghts[this_slit_mask] + asrt = np.argsort(this_wave) + for ii in range(numiter): + # Generate the map between model and data + splmap = scipy.interpolate.UnivariateSpline(this_wave[asrt], _ratio[this_slit_mask][asrt], w=this_wghts[asrt], + bbox=[None, None], k=3, s=nspec, ext=0, check_finite=False) + # Construct the mapping, and use this to make a model of the rawdata + this_modmap = splmap(this_wave[this_slit_mask]) + this_modflx = modelimg[this_slit_mask] * this_modmap + # Update the fit weights + this_wghts = modelimg[this_slit_mask] * np.sqrt(spl(this_modflx)) + # Produce the final model for this slit + modelmap[this_slit] = splmap(this_wave[this_slit]) + return modelmap + diff --git a/pypeit/deprecated/flux.py b/deprecated/flux.py similarity index 100% rename from pypeit/deprecated/flux.py rename to deprecated/flux.py diff --git a/pypeit/deprecated/flux_old.py b/deprecated/flux_old.py similarity index 100% rename from pypeit/deprecated/flux_old.py rename to deprecated/flux_old.py diff --git a/pypeit/deprecated/fluxcalibrate.py b/deprecated/fluxcalibrate.py similarity index 100% rename from pypeit/deprecated/fluxcalibrate.py rename to deprecated/fluxcalibrate.py diff --git a/pypeit/deprecated/fluxspec.py b/deprecated/fluxspec.py similarity index 100% rename from pypeit/deprecated/fluxspec.py rename to deprecated/fluxspec.py diff --git a/pypeit/deprecated/load.py b/deprecated/load.py similarity index 100% rename from pypeit/deprecated/load.py rename to deprecated/load.py diff --git a/pypeit/deprecated/lowrdx_pixflat.py b/deprecated/lowrdx_pixflat.py similarity index 100% rename from pypeit/deprecated/lowrdx_pixflat.py rename to deprecated/lowrdx_pixflat.py diff --git a/pypeit/deprecated/masters.py b/deprecated/masters.py similarity index 100% rename from pypeit/deprecated/masters.py rename to deprecated/masters.py diff --git a/pypeit/deprecated/old_test_objfind.py b/deprecated/old_test_objfind.py similarity index 100% rename from pypeit/deprecated/old_test_objfind.py rename to deprecated/old_test_objfind.py diff --git a/pypeit/deprecated/orig_save.py b/deprecated/orig_save.py similarity index 100% rename from pypeit/deprecated/orig_save.py rename to deprecated/orig_save.py diff --git a/pypeit/deprecated/orig_specobjs.py b/deprecated/orig_specobjs.py similarity index 100% rename from pypeit/deprecated/orig_specobjs.py rename to deprecated/orig_specobjs.py diff --git a/pypeit/deprecated/par.py b/deprecated/par.py similarity index 100% rename from pypeit/deprecated/par.py rename to deprecated/par.py diff --git a/pypeit/deprecated/parse.py b/deprecated/parse.py similarity index 100% rename from pypeit/deprecated/parse.py rename to deprecated/parse.py diff --git a/pypeit/deprecated/parse_calib_id.py b/deprecated/parse_calib_id.py similarity index 100% rename from pypeit/deprecated/parse_calib_id.py rename to deprecated/parse_calib_id.py diff --git a/pypeit/deprecated/pca.py b/deprecated/pca.py similarity index 100% rename from pypeit/deprecated/pca.py rename to deprecated/pca.py diff --git a/pypeit/deprecated/pixels.py b/deprecated/pixels.py similarity index 100% rename from pypeit/deprecated/pixels.py rename to deprecated/pixels.py diff --git a/pypeit/deprecated/processimages.py b/deprecated/processimages.py similarity index 100% rename from pypeit/deprecated/processimages.py rename to deprecated/processimages.py diff --git a/pypeit/deprecated/procimg.py b/deprecated/procimg.py similarity index 100% rename from pypeit/deprecated/procimg.py rename to deprecated/procimg.py diff --git a/pypeit/deprecated/pydl.py b/deprecated/pydl.py similarity index 100% rename from pypeit/deprecated/pydl.py rename to deprecated/pydl.py diff --git a/pypeit/deprecated/pypsetup.py b/deprecated/pypsetup.py similarity index 100% rename from pypeit/deprecated/pypsetup.py rename to deprecated/pypsetup.py diff --git a/pypeit/deprecated/qa.py b/deprecated/qa.py similarity index 100% rename from pypeit/deprecated/qa.py rename to deprecated/qa.py diff --git a/pypeit/deprecated/ql_keck_deimos.py b/deprecated/ql_keck_deimos.py similarity index 100% rename from pypeit/deprecated/ql_keck_deimos.py rename to deprecated/ql_keck_deimos.py diff --git a/pypeit/deprecated/ql_keck_lris.py b/deprecated/ql_keck_lris.py similarity index 100% rename from pypeit/deprecated/ql_keck_lris.py rename to deprecated/ql_keck_lris.py diff --git a/pypeit/deprecated/ql_keck_mosfire.py b/deprecated/ql_keck_mosfire.py similarity index 100% rename from pypeit/deprecated/ql_keck_mosfire.py rename to deprecated/ql_keck_mosfire.py diff --git a/pypeit/deprecated/ql_keck_nires.py b/deprecated/ql_keck_nires.py similarity index 100% rename from pypeit/deprecated/ql_keck_nires.py rename to deprecated/ql_keck_nires.py diff --git a/pypeit/deprecated/ql_multislit.py b/deprecated/ql_multislit.py similarity index 100% rename from pypeit/deprecated/ql_multislit.py rename to deprecated/ql_multislit.py diff --git a/pypeit/deprecated/ql_vlt_fors2.py b/deprecated/ql_vlt_fors2.py similarity index 100% rename from pypeit/deprecated/ql_vlt_fors2.py rename to deprecated/ql_vlt_fors2.py diff --git a/pypeit/deprecated/rawimage.py b/deprecated/rawimage.py similarity index 100% rename from pypeit/deprecated/rawimage.py rename to deprecated/rawimage.py diff --git a/pypeit/deprecated/reduce.py b/deprecated/reduce.py similarity index 100% rename from pypeit/deprecated/reduce.py rename to deprecated/reduce.py diff --git a/pypeit/deprecated/requirements.txt b/deprecated/requirements.txt similarity index 100% rename from pypeit/deprecated/requirements.txt rename to deprecated/requirements.txt diff --git a/pypeit/deprecated/requirements_doc.txt b/deprecated/requirements_doc.txt similarity index 100% rename from pypeit/deprecated/requirements_doc.txt rename to deprecated/requirements_doc.txt diff --git a/pypeit/deprecated/save.py b/deprecated/save.py similarity index 100% rename from pypeit/deprecated/save.py rename to deprecated/save.py diff --git a/pypeit/deprecated/sciimgstack.py b/deprecated/sciimgstack.py similarity index 100% rename from pypeit/deprecated/sciimgstack.py rename to deprecated/sciimgstack.py diff --git a/pypeit/deprecated/script_utils.py b/deprecated/script_utils.py similarity index 100% rename from pypeit/deprecated/script_utils.py rename to deprecated/script_utils.py diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.armed b/deprecated/settings/archive/settings.2017-02-06.armed similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.armed rename to deprecated/settings/archive/settings.2017-02-06.armed diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.armlsd b/deprecated/settings/archive/settings.2017-02-06.armlsd similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.armlsd rename to deprecated/settings/archive/settings.2017-02-06.armlsd diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.baseargflag b/deprecated/settings/archive/settings.2017-02-06.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.baseargflag rename to deprecated/settings/archive/settings.2017-02-06.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.basespect b/deprecated/settings/archive/settings.2017-02-06.basespect similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.basespect rename to deprecated/settings/archive/settings.2017-02-06.basespect diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.isis_blue b/deprecated/settings/archive/settings.2017-02-06.isis_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.isis_blue rename to deprecated/settings/archive/settings.2017-02-06.isis_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.kast_blue b/deprecated/settings/archive/settings.2017-02-06.kast_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.kast_blue rename to deprecated/settings/archive/settings.2017-02-06.kast_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.kast_red b/deprecated/settings/archive/settings.2017-02-06.kast_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.kast_red rename to deprecated/settings/archive/settings.2017-02-06.kast_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.lris_blue b/deprecated/settings/archive/settings.2017-02-06.lris_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.lris_blue rename to deprecated/settings/archive/settings.2017-02-06.lris_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.lris_red b/deprecated/settings/archive/settings.2017-02-06.lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.lris_red rename to deprecated/settings/archive/settings.2017-02-06.lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.basespect b/deprecated/settings/archive/settings.2017-02-21.basespect similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.basespect rename to deprecated/settings/archive/settings.2017-02-21.basespect diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.isis_blue b/deprecated/settings/archive/settings.2017-02-21.isis_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.isis_blue rename to deprecated/settings/archive/settings.2017-02-21.isis_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.kast_blue b/deprecated/settings/archive/settings.2017-02-21.kast_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.kast_blue rename to deprecated/settings/archive/settings.2017-02-21.kast_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.kast_red b/deprecated/settings/archive/settings.2017-02-21.kast_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.kast_red rename to deprecated/settings/archive/settings.2017-02-21.kast_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.levy b/deprecated/settings/archive/settings.2017-02-21.levy similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.levy rename to deprecated/settings/archive/settings.2017-02-21.levy diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.lris_blue b/deprecated/settings/archive/settings.2017-02-21.lris_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.lris_blue rename to deprecated/settings/archive/settings.2017-02-21.lris_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.lris_red b/deprecated/settings/archive/settings.2017-02-21.lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.lris_red rename to deprecated/settings/archive/settings.2017-02-21.lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-03-24.baseargflag b/deprecated/settings/archive/settings.2017-03-24.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-03-24.baseargflag rename to deprecated/settings/archive/settings.2017-03-24.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-03-24.kast_blue b/deprecated/settings/archive/settings.2017-03-24.kast_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-03-24.kast_blue rename to deprecated/settings/archive/settings.2017-03-24.kast_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-03-24.kast_red b/deprecated/settings/archive/settings.2017-03-24.kast_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-03-24.kast_red rename to deprecated/settings/archive/settings.2017-03-24.kast_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-03-24.kast_red_ret b/deprecated/settings/archive/settings.2017-03-24.kast_red_ret similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-03-24.kast_red_ret rename to deprecated/settings/archive/settings.2017-03-24.kast_red_ret diff --git a/pypeit/deprecated/settings/archive/settings.2017-03-24.lris_blue b/deprecated/settings/archive/settings.2017-03-24.lris_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-03-24.lris_blue rename to deprecated/settings/archive/settings.2017-03-24.lris_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-03-24.lris_red b/deprecated/settings/archive/settings.2017-03-24.lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-03-24.lris_red rename to deprecated/settings/archive/settings.2017-03-24.lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-04-12.armed b/deprecated/settings/archive/settings.2017-04-12.armed similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-04-12.armed rename to deprecated/settings/archive/settings.2017-04-12.armed diff --git a/pypeit/deprecated/settings/archive/settings.2017-04-12.armlsd b/deprecated/settings/archive/settings.2017-04-12.armlsd similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-04-12.armlsd rename to deprecated/settings/archive/settings.2017-04-12.armlsd diff --git a/pypeit/deprecated/settings/archive/settings.2017-04-12.baseargflag b/deprecated/settings/archive/settings.2017-04-12.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-04-12.baseargflag rename to deprecated/settings/archive/settings.2017-04-12.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-04-24.lris_red b/deprecated/settings/archive/settings.2017-04-24.lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-04-24.lris_red rename to deprecated/settings/archive/settings.2017-04-24.lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-05-19.armed b/deprecated/settings/archive/settings.2017-05-19.armed similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-05-19.armed rename to deprecated/settings/archive/settings.2017-05-19.armed diff --git a/pypeit/deprecated/settings/archive/settings.2017-05-19.baseargflag b/deprecated/settings/archive/settings.2017-05-19.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-05-19.baseargflag rename to deprecated/settings/archive/settings.2017-05-19.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-05-19.levy b/deprecated/settings/archive/settings.2017-05-19.levy similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-05-19.levy rename to deprecated/settings/archive/settings.2017-05-19.levy diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.basespect b/deprecated/settings/archive/settings.2017-06-05.basespect similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.basespect rename to deprecated/settings/archive/settings.2017-06-05.basespect diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.isis_blue b/deprecated/settings/archive/settings.2017-06-05.isis_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.isis_blue rename to deprecated/settings/archive/settings.2017-06-05.isis_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.kast_blue b/deprecated/settings/archive/settings.2017-06-05.kast_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.kast_blue rename to deprecated/settings/archive/settings.2017-06-05.kast_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.kast_red b/deprecated/settings/archive/settings.2017-06-05.kast_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.kast_red rename to deprecated/settings/archive/settings.2017-06-05.kast_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.kast_red_ret b/deprecated/settings/archive/settings.2017-06-05.kast_red_ret similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.kast_red_ret rename to deprecated/settings/archive/settings.2017-06-05.kast_red_ret diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.keck_hires b/deprecated/settings/archive/settings.2017-06-05.keck_hires similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.keck_hires rename to deprecated/settings/archive/settings.2017-06-05.keck_hires diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.levy b/deprecated/settings/archive/settings.2017-06-05.levy similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.levy rename to deprecated/settings/archive/settings.2017-06-05.levy diff --git a/pypeit/deprecated/settings/archive/settings.2017-07-08.isis_blue b/deprecated/settings/archive/settings.2017-07-08.isis_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-07-08.isis_blue rename to deprecated/settings/archive/settings.2017-07-08.isis_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-07-16.armlsd b/deprecated/settings/archive/settings.2017-07-16.armlsd similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-07-16.armlsd rename to deprecated/settings/archive/settings.2017-07-16.armlsd diff --git a/pypeit/deprecated/settings/archive/settings.2017-07-16.baseargflag b/deprecated/settings/archive/settings.2017-07-16.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-07-16.baseargflag rename to deprecated/settings/archive/settings.2017-07-16.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-08-25.baseargflag b/deprecated/settings/archive/settings.2017-08-25.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-08-25.baseargflag rename to deprecated/settings/archive/settings.2017-08-25.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-08-29.dolores b/deprecated/settings/archive/settings.2017-08-29.dolores similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-08-29.dolores rename to deprecated/settings/archive/settings.2017-08-29.dolores diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.apf_levy b/deprecated/settings/archive/settings.2017-11-17.apf_levy similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.apf_levy rename to deprecated/settings/archive/settings.2017-11-17.apf_levy diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.baseargflag b/deprecated/settings/archive/settings.2017-11-17.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.baseargflag rename to deprecated/settings/archive/settings.2017-11-17.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.keck_hires b/deprecated/settings/archive/settings.2017-11-17.keck_hires similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.keck_hires rename to deprecated/settings/archive/settings.2017-11-17.keck_hires diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.keck_lris_blue b/deprecated/settings/archive/settings.2017-11-17.keck_lris_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.keck_lris_blue rename to deprecated/settings/archive/settings.2017-11-17.keck_lris_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.keck_lris_red b/deprecated/settings/archive/settings.2017-11-17.keck_lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.keck_lris_red rename to deprecated/settings/archive/settings.2017-11-17.keck_lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.shane_kast_blue b/deprecated/settings/archive/settings.2017-11-17.shane_kast_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.shane_kast_blue rename to deprecated/settings/archive/settings.2017-11-17.shane_kast_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.shane_kast_red b/deprecated/settings/archive/settings.2017-11-17.shane_kast_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.shane_kast_red rename to deprecated/settings/archive/settings.2017-11-17.shane_kast_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.shane_kast_red_ret b/deprecated/settings/archive/settings.2017-11-17.shane_kast_red_ret similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.shane_kast_red_ret rename to deprecated/settings/archive/settings.2017-11-17.shane_kast_red_ret diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.tng_dolores b/deprecated/settings/archive/settings.2017-11-17.tng_dolores similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.tng_dolores rename to deprecated/settings/archive/settings.2017-11-17.tng_dolores diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.wht_isis_blue b/deprecated/settings/archive/settings.2017-11-17.wht_isis_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.wht_isis_blue rename to deprecated/settings/archive/settings.2017-11-17.wht_isis_blue diff --git a/pypeit/deprecated/settings/archive/settings.2018-04-05.armlsd b/deprecated/settings/archive/settings.2018-04-05.armlsd similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-04-05.armlsd rename to deprecated/settings/archive/settings.2018-04-05.armlsd diff --git a/pypeit/deprecated/settings/archive/settings.2018-04-05.baseargflag b/deprecated/settings/archive/settings.2018-04-05.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-04-05.baseargflag rename to deprecated/settings/archive/settings.2018-04-05.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2018-04-05.keck_deimos b/deprecated/settings/archive/settings.2018-04-05.keck_deimos similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-04-05.keck_deimos rename to deprecated/settings/archive/settings.2018-04-05.keck_deimos diff --git a/pypeit/deprecated/settings/archive/settings.2018-04-30.keck_deimos b/deprecated/settings/archive/settings.2018-04-30.keck_deimos similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-04-30.keck_deimos rename to deprecated/settings/archive/settings.2018-04-30.keck_deimos diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-10.armed b/deprecated/settings/archive/settings.2018-05-10.armed similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-10.armed rename to deprecated/settings/archive/settings.2018-05-10.armed diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-10.armlsd b/deprecated/settings/archive/settings.2018-05-10.armlsd similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-10.armlsd rename to deprecated/settings/archive/settings.2018-05-10.armlsd diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-10.keck_deimos b/deprecated/settings/archive/settings.2018-05-10.keck_deimos similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-10.keck_deimos rename to deprecated/settings/archive/settings.2018-05-10.keck_deimos diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-10.keck_lris_blue b/deprecated/settings/archive/settings.2018-05-10.keck_lris_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-10.keck_lris_blue rename to deprecated/settings/archive/settings.2018-05-10.keck_lris_blue diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-10.keck_lris_red b/deprecated/settings/archive/settings.2018-05-10.keck_lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-10.keck_lris_red rename to deprecated/settings/archive/settings.2018-05-10.keck_lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-22.keck_deimos b/deprecated/settings/archive/settings.2018-05-22.keck_deimos similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-22.keck_deimos rename to deprecated/settings/archive/settings.2018-05-22.keck_deimos diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-22.keck_lris_red b/deprecated/settings/archive/settings.2018-05-22.keck_lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-22.keck_lris_red rename to deprecated/settings/archive/settings.2018-05-22.keck_lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-22.shane_kast_red_ret b/deprecated/settings/archive/settings.2018-05-22.shane_kast_red_ret similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-22.shane_kast_red_ret rename to deprecated/settings/archive/settings.2018-05-22.shane_kast_red_ret diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-31.keck_nirspec b/deprecated/settings/archive/settings.2018-05-31.keck_nirspec similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-31.keck_nirspec rename to deprecated/settings/archive/settings.2018-05-31.keck_nirspec diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.arms b/deprecated/settings/archive/settings.2018-06-07.arms similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.arms rename to deprecated/settings/archive/settings.2018-06-07.arms diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.baseargflag b/deprecated/settings/archive/settings.2018-06-07.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.baseargflag rename to deprecated/settings/archive/settings.2018-06-07.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.keck_deimos b/deprecated/settings/archive/settings.2018-06-07.keck_deimos similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.keck_deimos rename to deprecated/settings/archive/settings.2018-06-07.keck_deimos diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.keck_lris_blue b/deprecated/settings/archive/settings.2018-06-07.keck_lris_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.keck_lris_blue rename to deprecated/settings/archive/settings.2018-06-07.keck_lris_blue diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.keck_lris_red b/deprecated/settings/archive/settings.2018-06-07.keck_lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.keck_lris_red rename to deprecated/settings/archive/settings.2018-06-07.keck_lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.keck_nirspec b/deprecated/settings/archive/settings.2018-06-07.keck_nirspec similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.keck_nirspec rename to deprecated/settings/archive/settings.2018-06-07.keck_nirspec diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.shane_kast_blue b/deprecated/settings/archive/settings.2018-06-07.shane_kast_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.shane_kast_blue rename to deprecated/settings/archive/settings.2018-06-07.shane_kast_blue diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.shane_kast_red b/deprecated/settings/archive/settings.2018-06-07.shane_kast_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.shane_kast_red rename to deprecated/settings/archive/settings.2018-06-07.shane_kast_red diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.shane_kast_red_ret b/deprecated/settings/archive/settings.2018-06-07.shane_kast_red_ret similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.shane_kast_red_ret rename to deprecated/settings/archive/settings.2018-06-07.shane_kast_red_ret diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.tng_dolores b/deprecated/settings/archive/settings.2018-06-07.tng_dolores similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.tng_dolores rename to deprecated/settings/archive/settings.2018-06-07.tng_dolores diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.wht_isis_blue b/deprecated/settings/archive/settings.2018-06-07.wht_isis_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.wht_isis_blue rename to deprecated/settings/archive/settings.2018-06-07.wht_isis_blue diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-19.baseargflag b/deprecated/settings/archive/settings.2018-06-19.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-19.baseargflag rename to deprecated/settings/archive/settings.2018-06-19.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-26.baseargflag b/deprecated/settings/archive/settings.2018-06-26.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-26.baseargflag rename to deprecated/settings/archive/settings.2018-06-26.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2018-07-03.keck_nires b/deprecated/settings/archive/settings.2018-07-03.keck_nires similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-07-03.keck_nires rename to deprecated/settings/archive/settings.2018-07-03.keck_nires diff --git a/pypeit/deprecated/settings/settings.apf_levy b/deprecated/settings/settings.apf_levy similarity index 100% rename from pypeit/deprecated/settings/settings.apf_levy rename to deprecated/settings/settings.apf_levy diff --git a/pypeit/deprecated/settings/settings.armed b/deprecated/settings/settings.armed similarity index 100% rename from pypeit/deprecated/settings/settings.armed rename to deprecated/settings/settings.armed diff --git a/pypeit/deprecated/settings/settings.arms b/deprecated/settings/settings.arms similarity index 100% rename from pypeit/deprecated/settings/settings.arms rename to deprecated/settings/settings.arms diff --git a/pypeit/deprecated/settings/settings.baseargflag b/deprecated/settings/settings.baseargflag similarity index 100% rename from pypeit/deprecated/settings/settings.baseargflag rename to deprecated/settings/settings.baseargflag diff --git a/pypeit/deprecated/settings/settings.basespect b/deprecated/settings/settings.basespect similarity index 100% rename from pypeit/deprecated/settings/settings.basespect rename to deprecated/settings/settings.basespect diff --git a/pypeit/deprecated/settings/settings.keck_deimos b/deprecated/settings/settings.keck_deimos similarity index 100% rename from pypeit/deprecated/settings/settings.keck_deimos rename to deprecated/settings/settings.keck_deimos diff --git a/pypeit/deprecated/settings/settings.keck_hires b/deprecated/settings/settings.keck_hires similarity index 100% rename from pypeit/deprecated/settings/settings.keck_hires rename to deprecated/settings/settings.keck_hires diff --git a/pypeit/deprecated/settings/settings.keck_lris_blue b/deprecated/settings/settings.keck_lris_blue similarity index 100% rename from pypeit/deprecated/settings/settings.keck_lris_blue rename to deprecated/settings/settings.keck_lris_blue diff --git a/pypeit/deprecated/settings/settings.keck_lris_red b/deprecated/settings/settings.keck_lris_red similarity index 100% rename from pypeit/deprecated/settings/settings.keck_lris_red rename to deprecated/settings/settings.keck_lris_red diff --git a/pypeit/deprecated/settings/settings.keck_nires b/deprecated/settings/settings.keck_nires similarity index 100% rename from pypeit/deprecated/settings/settings.keck_nires rename to deprecated/settings/settings.keck_nires diff --git a/pypeit/deprecated/settings/settings.keck_nirspec b/deprecated/settings/settings.keck_nirspec similarity index 100% rename from pypeit/deprecated/settings/settings.keck_nirspec rename to deprecated/settings/settings.keck_nirspec diff --git a/pypeit/deprecated/settings/settings.py b/deprecated/settings/settings.py similarity index 100% rename from pypeit/deprecated/settings/settings.py rename to deprecated/settings/settings.py diff --git a/pypeit/deprecated/settings/settings.shane_kast_blue b/deprecated/settings/settings.shane_kast_blue similarity index 100% rename from pypeit/deprecated/settings/settings.shane_kast_blue rename to deprecated/settings/settings.shane_kast_blue diff --git a/pypeit/deprecated/settings/settings.shane_kast_red b/deprecated/settings/settings.shane_kast_red similarity index 100% rename from pypeit/deprecated/settings/settings.shane_kast_red rename to deprecated/settings/settings.shane_kast_red diff --git a/pypeit/deprecated/settings/settings.shane_kast_red_ret b/deprecated/settings/settings.shane_kast_red_ret similarity index 100% rename from pypeit/deprecated/settings/settings.shane_kast_red_ret rename to deprecated/settings/settings.shane_kast_red_ret diff --git a/pypeit/deprecated/settings/settings.tng_dolores b/deprecated/settings/settings.tng_dolores similarity index 100% rename from pypeit/deprecated/settings/settings.tng_dolores rename to deprecated/settings/settings.tng_dolores diff --git a/pypeit/deprecated/settings/settings.wht_isis_blue b/deprecated/settings/settings.wht_isis_blue similarity index 100% rename from pypeit/deprecated/settings/settings.wht_isis_blue rename to deprecated/settings/settings.wht_isis_blue diff --git a/pypeit/deprecated/setup.py b/deprecated/setup.py similarity index 100% rename from pypeit/deprecated/setup.py rename to deprecated/setup.py diff --git a/pypeit/deprecated/trace_slits.py b/deprecated/trace_slits.py similarity index 100% rename from pypeit/deprecated/trace_slits.py rename to deprecated/trace_slits.py diff --git a/pypeit/deprecated/traceslits.py b/deprecated/traceslits.py similarity index 100% rename from pypeit/deprecated/traceslits.py rename to deprecated/traceslits.py diff --git a/pypeit/deprecated/tracewave_old.py b/deprecated/tracewave_old.py similarity index 100% rename from pypeit/deprecated/tracewave_old.py rename to deprecated/tracewave_old.py diff --git a/pypeit/deprecated/utils.py b/deprecated/utils.py similarity index 100% rename from pypeit/deprecated/utils.py rename to deprecated/utils.py diff --git a/pypeit/deprecated/waveimage.py b/deprecated/waveimage.py similarity index 100% rename from pypeit/deprecated/waveimage.py rename to deprecated/waveimage.py diff --git a/pypeit/deprecated/waveio_old.py b/deprecated/waveio_old.py similarity index 100% rename from pypeit/deprecated/waveio_old.py rename to deprecated/waveio_old.py diff --git a/pypeit/deprecated/wvutils.py b/deprecated/wvutils.py similarity index 100% rename from pypeit/deprecated/wvutils.py rename to deprecated/wvutils.py diff --git a/doc/Makefile b/doc/Makefile index c71afd4a43..61786be58d 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -56,7 +56,7 @@ clean: rm -rf $(LOCALFILES) apirst: - SPHINX_APIDOC_OPTIONS=$(SPHINXAPIOPT) $(SPHINXAPI) --separate -o ./api ../pypeit ../pypeit/tests/* ../pypeit/deprecated/* ../pypeit/version.py ../pypeit/compiler_version* + SPHINX_APIDOC_OPTIONS=$(SPHINXAPIOPT) $(SPHINXAPI) --separate -o ./api ../pypeit ../pypeit/tests/* ../pypeit/version.py ../pypeit/compiler_version* wget -O ./include/dev_suite_readme.rst https://raw.githubusercontent.com/pypeit/PypeIt-development-suite/main/README.rst python ./scripts/build_datacontainer_datamodels.py python ./scripts/build_dependency_rst.py diff --git a/doc/api/pypeit.scripts.print_bpm.rst b/doc/api/pypeit.scripts.print_bpm.rst new file mode 100644 index 0000000000..d042ad948d --- /dev/null +++ b/doc/api/pypeit.scripts.print_bpm.rst @@ -0,0 +1,8 @@ +pypeit.scripts.print\_bpm module +================================ + +.. automodule:: pypeit.scripts.print_bpm + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.scripts.rst b/doc/api/pypeit.scripts.rst index b94e122b86..4f768ea755 100644 --- a/doc/api/pypeit.scripts.rst +++ b/doc/api/pypeit.scripts.rst @@ -35,6 +35,7 @@ Submodules pypeit.scripts.multislit_flexure pypeit.scripts.obslog pypeit.scripts.parse_slits + pypeit.scripts.print_bpm pypeit.scripts.qa_html pypeit.scripts.ql pypeit.scripts.run_pypeit diff --git a/doc/calibrations/wvcalib.rst b/doc/calibrations/wvcalib.rst index 1f7367c706..148268c118 100644 --- a/doc/calibrations/wvcalib.rst +++ b/doc/calibrations/wvcalib.rst @@ -31,13 +31,13 @@ with the **pypeit_chk_wavecalib** script, e.g. : $ pypeit_chk_wavecalib Calibrations/WaveCalib_A_1_MSC03.fits - N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) mesured_fwhm RMS - --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------ ----- - 0 35 6422.5 7753.8 9053.2 0.325 48 6508.325 - 9047.930 96.5 3.5 0.046 - 1 93 6310.0 7641.4 8940.8 0.325 49 6336.179 - 8931.145 98.6 3.6 0.036 - 2 140 6440.8 7772.1 9071.5 0.325 47 6508.325 - 9047.930 96.5 3.6 0.049 - 3 184 6301.2 7632.6 8932.0 0.325 50 6306.533 - 8931.145 99.8 3.6 0.037 - 4 243 6257.1 7588.5 8887.9 0.325 49 6268.229 - 8821.832 97.1 3.6 0.034 + N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) measured_fwhm RMS + --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------- ----- + 0 35 6422.5 7753.8 9053.2 0.325 48 6508.325 - 9047.930 96.5 3.5 0.046 + 1 93 6310.0 7641.4 8940.8 0.325 49 6336.179 - 8931.145 98.6 3.6 0.036 + 2 140 6440.8 7772.1 9071.5 0.325 47 6508.325 - 9047.930 96.5 3.6 0.049 + 3 184 6301.2 7632.6 8932.0 0.325 50 6306.533 - 8931.145 99.8 3.6 0.037 + 4 243 6257.1 7588.5 8887.9 0.325 49 6268.229 - 8821.832 97.1 3.6 0.034 - ``SpatID`` is the spatial position of the slit/order. @@ -50,7 +50,7 @@ with the **pypeit_chk_wavecalib** script, e.g. : number, the wavelength range, and the spectral coverage of the identified and fitted arc lines. -- ``mesured_fwhm`` is the measured arc lines FWHM (in binned pixels of the input +- ``measured_fwhm`` is the measured arc lines FWHM (in binned pixels of the input arc frame), i.e, the approximate spectral resolution. Note that this not necessarily the ``fwhm`` used to identify the arc lines during the wavelength calibration, see :ref:`wvcalib-fwhm`. diff --git a/doc/dev/deimosconfig.rst b/doc/dev/deimosconfig.rst index 3401cd76ef..995140c065 100644 --- a/doc/dev/deimosconfig.rst +++ b/doc/dev/deimosconfig.rst @@ -48,7 +48,7 @@ as determined by :func:`pypeit.metadata.PypeItMetaData.unique_configurations`. The ``AMPMODE`` value is included, even though ``PypeIt`` (currently) restricts itself to only attempting to reduce frames read by the B -amplifier; see +and A amplifiers; see :func:`~pypeit.spectrographs.keck_deimos.KeckDEIMOSSpectrograph.valid_configuration_values`. Additionally, ``PypeIt`` requires all frames to have ``MOSMODE == 'Spectral'``. Frames that do not match these header keyword @@ -178,7 +178,7 @@ The algorithm for this test is as follows: 5. Check that "cleaning" the configurations of frames that cannot be reduced by ``PypeIt`` (those with ``MOSMODE != 'Spectral'`` - or ``AMPMODE != SINGLE:B``), using + or ``AMPMODE != SINGLE:B`` or ``AMPMODE != SINGLE:A``), using :func:`~pypeit.metadata.PypeItMetaData.clean_configurations` does not remove any file because all of the dev-suite files are valid. diff --git a/doc/dev/deimosframes.rst b/doc/dev/deimosframes.rst index 5206aa2286..a1b1024a36 100644 --- a/doc/dev/deimosframes.rst +++ b/doc/dev/deimosframes.rst @@ -79,7 +79,7 @@ Also note that ``PypeIt`` will ignore frames observed under conditions that do not meet the restricted values set by :func:`~pypeit.spectrographs.keck_deimos.KeckDEIMOSSpectrograph.valid_configuration_values`. This currently requires all frames to have ``MOSMODE == 'Spectral'`` -and ``AMPMODE == 'SINGLE:B'``. Frames that do not meet these criteria +and ``AMPMODE == 'SINGLE:B'`` or ``AMPMODE == 'SINGLE:A'``. Frames that do not meet these criteria will not be included in the automatically generated :ref:`pypeit_file` created by :ref:`pypeit_setup`. diff --git a/doc/dev/development.rst b/doc/dev/development.rst index 0f62f73c29..935051d54a 100644 --- a/doc/dev/development.rst +++ b/doc/dev/development.rst @@ -422,6 +422,12 @@ are as follows: * The docstrings for any changes to existing methods that were altered must have been modified so that they are up-to-date and accurate. + * The documentation must be successfully recompiled, either using the + ``update_docs`` scripts or but running ``make clean ; make html`` in the + ``doc/`` directory. (We plan for this to be added to the dev-suite testing; + in the meantime, PR authors simply need to affirm that the documentation + builds successfully.) + * Spurious commented code used for debugging or testing is fine, but please let us know if you want it to be kept by adding a relevant comment, something like ``# TODO: Keep this around for now``, at the @@ -491,7 +497,11 @@ tagging process is as follows: # Push the new tag git push --tags - * The tag is released for `pip`_ installation. + Similarly, a matching tag is executed for the dev-suite code (these tags only + exist for versions 1.15 and later). + + * The tag of the ``pypeit`` code-base (not the dev-suite) is released for + `pip`_ installation. .. code-block:: bash diff --git a/doc/help/pypeit_chk_alignments.rst b/doc/help/pypeit_chk_alignments.rst index a68e12c7ca..2f58ffa5b7 100644 --- a/doc/help/pypeit_chk_alignments.rst +++ b/doc/help/pypeit_chk_alignments.rst @@ -1,7 +1,7 @@ .. code-block:: console $ pypeit_chk_alignments -h - usage: pypeit_chk_alignments [-h] [--chname CHNAME] file + usage: pypeit_chk_alignments [-h] [--chname CHNAME] [--try_old] file Display Alignment image and the trace data @@ -11,4 +11,6 @@ options: -h, --help show this help message and exit --chname CHNAME Channel name for image in Ginga (default: Alignments) + --try_old Attempt to load old datamodel versions. A crash may ensue.. + (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_chk_noise_1dspec.rst b/doc/help/pypeit_chk_noise_1dspec.rst index 80fa4e2613..51b54fbfaf 100644 --- a/doc/help/pypeit_chk_noise_1dspec.rst +++ b/doc/help/pypeit_chk_noise_1dspec.rst @@ -6,6 +6,7 @@ [--z [Z ...]] [--maskdef_objname MASKDEF_OBJNAME] [--pypeit_name PYPEIT_NAME] [--wavemin WAVEMIN] [--wavemax WAVEMAX] [--plot_or_save PLOT_OR_SAVE] + [--try_old] [files ...] Examine the noise in a PypeIt spectrum @@ -41,4 +42,6 @@ window. If you choose save, a folder called spec1d*_noisecheck will be created and all the relevant plot will be placed there. (default: plot) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_chk_noise_2dspec.rst b/doc/help/pypeit_chk_noise_2dspec.rst index 7fc7fd75d5..d0a6662125 100644 --- a/doc/help/pypeit_chk_noise_2dspec.rst +++ b/doc/help/pypeit_chk_noise_2dspec.rst @@ -5,7 +5,7 @@ [--maskdef_id MASKDEF_ID] [--pypeit_id PYPEIT_ID] [--pad PAD] [--aspect_ratio ASPECT_RATIO] [--wavemin WAVEMIN] [--wavemax WAVEMAX] - [--mode MODE] [--list] + [--mode MODE] [--list] [--try_old] [files ...] Examine the noise in a PypeIt slit/order @@ -43,4 +43,6 @@ placed. "print" will cause the check noise values to be printed in the terminal. (default: plot) --list List the extensions only? (default: False) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_chk_scattlight.rst b/doc/help/pypeit_chk_scattlight.rst index 73f9cddab3..08cd077c40 100644 --- a/doc/help/pypeit_chk_scattlight.rst +++ b/doc/help/pypeit_chk_scattlight.rst @@ -2,6 +2,7 @@ $ pypeit_chk_scattlight -h usage: pypeit_chk_scattlight [-h] [--spec2d SPEC2D] [--det DET] [--mask MASK] + [--try_old] file slits Display the scattered light image in a Ginga viewer @@ -21,4 +22,6 @@ --mask MASK If True, the detector pixels that are considered on the slit will be masked to highlight the scattered light regions (default: False) + --try_old Attempt to load old datamodel versions. A crash may ensue.. + (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_chk_tilts.rst b/doc/help/pypeit_chk_tilts.rst index b286323586..a7440699a2 100644 --- a/doc/help/pypeit_chk_tilts.rst +++ b/doc/help/pypeit_chk_tilts.rst @@ -17,5 +17,5 @@ Ginga). If not set, only the fitted, masked and rejected in the fit tilts are shown. (default: False) --try_old Attempt to load old datamodel versions. A crash may ensue.. - (default: True) + (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_chk_wavecalib.rst b/doc/help/pypeit_chk_wavecalib.rst index 51d396d998..fe49166847 100644 --- a/doc/help/pypeit_chk_wavecalib.rst +++ b/doc/help/pypeit_chk_wavecalib.rst @@ -1,7 +1,7 @@ .. code-block:: console $ pypeit_chk_wavecalib -h - usage: pypeit_chk_wavecalib [-h] input_file [input_file ...] + usage: pypeit_chk_wavecalib [-h] [--try_old] input_file [input_file ...] Print QA on Wavelength Calib to the screen @@ -11,4 +11,6 @@ options: -h, --help show this help message and exit + --try_old Attempt to load old datamodel versions. A crash may ensue.. + (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_collate_1d.rst b/doc/help/pypeit_collate_1d.rst index d61fa3a918..1054c1ee55 100644 --- a/doc/help/pypeit_collate_1d.rst +++ b/doc/help/pypeit_collate_1d.rst @@ -36,8 +36,6 @@ value are skipped, else all wavelength rms values are accepted. refframe Perform reference frame correction prior to coadding. Options are ['observed', 'heliocentric', 'barycentric']. Defaults to None. - chk_version If true, spec1ds and archival sensfuncs must match the currently - supported versions. If false (the default) version numbers are not checked. spec1d read @@ -88,8 +86,14 @@ --refframe {observed,heliocentric,barycentric} Perform reference frame correction prior to coadding. Options are: observed, heliocentric, barycentric - --chk_version Whether to check the data model versions of spec1d files - and sensfunc files. + --chk_version If True enforce strict PypeIt version checking to ensure + that all files were created with the current version of + PypeIt. If set to False, the code will attempt to read + out-of-date files and keep going. Beware (!!) that this + can lead to unforeseen bugs that either cause the code + to crash or lead to erroneous results. I.e., you really + need to know what you are doing if you set this to + False! -v VERBOSITY, --verbosity VERBOSITY Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename diff --git a/doc/help/pypeit_edge_inspector.rst b/doc/help/pypeit_edge_inspector.rst index f975fedd18..ca482dfafa 100644 --- a/doc/help/pypeit_edge_inspector.rst +++ b/doc/help/pypeit_edge_inspector.rst @@ -1,7 +1,7 @@ .. code-block:: console $ pypeit_edge_inspector -h - usage: pypeit_edge_inspector [-h] trace_file + usage: pypeit_edge_inspector [-h] [--try_old] trace_file Interactively inspect/edit slit edge traces @@ -10,4 +10,6 @@ options: -h, --help show this help message and exit + --try_old Attempt to load old datamodel versions. A crash may ensue.. + (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_flux_calib.rst b/doc/help/pypeit_flux_calib.rst index 9bdb6d363f..162dab294c 100644 --- a/doc/help/pypeit_flux_calib.rst +++ b/doc/help/pypeit_flux_calib.rst @@ -1,7 +1,8 @@ .. code-block:: console $ pypeit_flux_calib -h - usage: pypeit_flux_calib [-h] [--par_outfile] [-v VERBOSITY] flux_file + usage: pypeit_flux_calib [-h] [--par_outfile] [-v VERBOSITY] [--try_old] + flux_file Flux calibrate 1D spectra produced by PypeIt @@ -53,4 +54,6 @@ Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename flux_calib_YYYYMMDD-HHMM.log + --try_old Attempt to load old datamodel versions. A crash may + ensue.. \ No newline at end of file diff --git a/doc/help/pypeit_identify.rst b/doc/help/pypeit_identify.rst index da9e7a8bab..51874f8498 100644 --- a/doc/help/pypeit_identify.rst +++ b/doc/help/pypeit_identify.rst @@ -5,6 +5,7 @@ [--slit SLIT] [--det DET] [--rmstol RMSTOL] [--fwhm FWHM] [--sigdetect SIGDETECT] [--pixtol PIXTOL] [--linear] [--force_save] [--rescale_resid] [-v VERBOSITY] + [--try_old] arc_file slits_file Launch PypeIt identify tool, display extracted Arc, and load linelist. @@ -38,4 +39,6 @@ Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename identify_YYYYMMDD- HHMM.log (default: 1) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_parse_slits.rst b/doc/help/pypeit_parse_slits.rst index f58c4230f4..19dbd2c1dd 100644 --- a/doc/help/pypeit_parse_slits.rst +++ b/doc/help/pypeit_parse_slits.rst @@ -1,7 +1,7 @@ .. code-block:: console $ pypeit_parse_slits -h - usage: pypeit_parse_slits [-h] input_file + usage: pypeit_parse_slits [-h] [--try_old] input_file Print info on slits from a input file @@ -10,4 +10,6 @@ options: -h, --help show this help message and exit + --try_old Attempt to load old datamodel versions. A crash may ensue.. + (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_print_bpm.rst b/doc/help/pypeit_print_bpm.rst new file mode 100644 index 0000000000..73e2508173 --- /dev/null +++ b/doc/help/pypeit_print_bpm.rst @@ -0,0 +1,25 @@ +.. code-block:: console + + $ pypeit_print_bpm -h + usage: pypeit_print_bpm [-h] [--file FILE] [--det DET] bit + + Print out an informative description of a bad pixel masked value. Usually, you + should run pypeit_show_2dspec --showmask first to see the bad pixel mask values. + Then, call this script with the BPM value that you want to findmore information + about. + + positional arguments: + bit Bad pixel mask value to describe in plain text + + options: + -h, --help show this help message and exit + --file FILE PypeIt spec2d file to use for the description(optional). If + provided, the bitmask contained in the spec2d file will be used + to describe the bad pixel mask value. If not provided, the + default pypeit bad pixel mask will be used. (default: None) + --det DET Detector name or number. If a number, the name is constructed + assuming the reduction is for a single detector. If a string, it + must match the name of the detector object (e.g., DET01 for a + detector, MSC01 for a mosaic). This is not required, and the + value is acceptable. Default is 1. (default: 1) + \ No newline at end of file diff --git a/doc/help/pypeit_ql.rst b/doc/help/pypeit_ql.rst index 609095f598..5e84f548a1 100644 --- a/doc/help/pypeit_ql.rst +++ b/doc/help/pypeit_ql.rst @@ -11,7 +11,7 @@ [--ignore_std] [--skip_display] [--coadd2d] [--only_slits ONLY_SLITS [ONLY_SLITS ...]] [--offsets OFFSETS] [--weights WEIGHTS] [--spec_samp_fact SPEC_SAMP_FACT] - [--spat_samp_fact SPAT_SAMP_FACT] + [--spat_samp_fact SPAT_SAMP_FACT] [--try_old] spectrograph Script to produce quick-look PypeIt reductions @@ -133,4 +133,6 @@ If coadding, adjust the spatial grid sampling by this factor. For a finer grid, set value to <1.0; for coarser sampling, set value to >1.0). (default: 1.0) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_show_2dspec.rst b/doc/help/pypeit_show_2dspec.rst index 77d84129bd..1ab710aebf 100644 --- a/doc/help/pypeit_show_2dspec.rst +++ b/doc/help/pypeit_show_2dspec.rst @@ -2,10 +2,10 @@ $ pypeit_show_2dspec -h usage: pypeit_show_2dspec [-h] [--list] [--det DET] [--spat_id SPAT_ID] - [--maskID MASKID] [--showmask] [--removetrace] - [--embed] [--ignore_extract_mask] + [--maskID MASKID] [--showmask [SHOWMASK ...]] + [--removetrace] [--embed] [--ignore_extract_mask] [--channels CHANNELS] [--prefix PREFIX] [--no_clear] - [-v VERBOSITY] + [-v VERBOSITY] [--try_old] file Display sky subtracted, spec2d image in a ginga viewer. @@ -24,7 +24,16 @@ --spat_id SPAT_ID Restrict plotting to this slit (PypeIt ID notation) (default: None) --maskID MASKID Restrict plotting to this maskID (default: None) - --showmask Overplot masked pixels (default: False) + --showmask [SHOWMASK ...] + Include a channel showing the mask. If no arguments are + provided, the mask bit values are provided directly. You + can also specify one or more mask flags used to + construct an image identifying which pixels are flagged + by any of these issues. E.g., to show pixels flagged by + the instrument specific bad-pixel mask or cosmic arrays, + use --showmask BPM CR . See + https://pypeit.readthedocs.io/en/release/out_masks.html + for the list of flags. (default: None) --removetrace Do not overplot traces in the skysub, sky_resid, and resid channels (default: False) --embed Upon completion embed in ipython shell (default: False) @@ -38,4 +47,6 @@ -v VERBOSITY, --verbosity VERBOSITY Verbosity level between 0 [none] and 2 [all] (default: 1) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_show_wvcalib.rst b/doc/help/pypeit_show_wvcalib.rst index 43684e9cc2..b993d3fd27 100644 --- a/doc/help/pypeit_show_wvcalib.rst +++ b/doc/help/pypeit_show_wvcalib.rst @@ -1,7 +1,7 @@ .. code-block:: console $ pypeit_show_wvcalib -h - usage: pypeit_show_wvcalib [-h] [--slit_file SLIT_FILE] [--is_order] + usage: pypeit_show_wvcalib [-h] [--slit_file SLIT_FILE] [--is_order] [--try_old] file slit_order Show the result of wavelength calibration @@ -15,4 +15,6 @@ --slit_file SLIT_FILE Slit file (default: None) --is_order Input slit/order is an order (default: False) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_skysub_regions.rst b/doc/help/pypeit_skysub_regions.rst index 92b9b0ff8b..99e96054be 100644 --- a/doc/help/pypeit_skysub_regions.rst +++ b/doc/help/pypeit_skysub_regions.rst @@ -2,6 +2,7 @@ $ pypeit_skysub_regions -h usage: pypeit_skysub_regions [-h] [--det DET] [-o] [-i] [-f] [-s] [-v VERBOSITY] + [--try_old] file Display a spec2d frame and interactively define the sky regions using a GUI. Run @@ -22,4 +23,6 @@ Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename skysub_regions_YYYYMMDD-HHMM.log (default: 1) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_tellfit.rst b/doc/help/pypeit_tellfit.rst index 28158d4068..134ba35556 100644 --- a/doc/help/pypeit_tellfit.rst +++ b/doc/help/pypeit_tellfit.rst @@ -4,6 +4,7 @@ usage: pypeit_tellfit [-h] [--objmodel {qso,star,poly}] [-r REDSHIFT] [-g TELL_GRID] [-p PCA_FILE] [-t TELL_FILE] [--debug] [--plot] [--par_outfile PAR_OUTFILE] [-v VERBOSITY] + [--chk_version] spec1dfile Telluric correct a spectrum @@ -74,4 +75,8 @@ Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename tellfit_YYYYMMDD- HHMM.log + --chk_version Ensure the datamodels are from the current PypeIt + version. By default (consistent with previous + functionality) this is not enforced and crashes may + ensue ... \ No newline at end of file diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index 6a5293f4c7..43055026a6 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -4,7 +4,7 @@ usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c] pypeit_file - ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev477+g4a02a85ca + ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev729+g948bfc229.d20240202 ## ## Available spectrographs include: ## bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, diff --git a/doc/include/class_datamodel_detectorcontainer.rst b/doc/include/class_datamodel_detectorcontainer.rst index 9154cbe8d9..e5a4d49b7b 100644 --- a/doc/include/class_datamodel_detectorcontainer.rst +++ b/doc/include/class_datamodel_detectorcontainer.rst @@ -1,26 +1,26 @@ **Version**: 1.0.1 -================= ================ ================= ======================================================================================================================================================================================================================================================== -Attribute Type Array Type Description -================= ================ ================= ======================================================================================================================================================================================================================================================== -``binning`` str Binning in PypeIt orientation (not the original) -``darkcurr`` int, float Dark current (e-/pixel/hour) -``dataext`` int Index of fits extension containing data -``datasec`` `numpy.ndarray`_ str Either the data sections or the header keyword where the valid data sections can be obtained, one per amplifier. If defined explicitly should be in FITS format (e.g., [1:2048,10:4096]). -``det`` int PypeIt designation for detector number (1-based). -``gain`` `numpy.ndarray`_ `numpy.floating`_ Inverse gain (e-/ADU). A list should be provided if a detector contains more than one amplifier. -``mincounts`` int, float Counts (e-) in a pixel below this value will be ignored as being unphysical. -``nonlinear`` int, float Percentage of detector range which is linear (i.e. everything above ``nonlinear*saturation`` will be flagged as saturated) -``numamplifiers`` int Number of amplifiers -``oscansec`` `numpy.ndarray`_ str Either the overscan section or the header keyword where the valid data sections can be obtained, one per amplifier. If defined explicitly should be in FITS format (e.g., [1:2048,10:4096]). -``platescale`` int, float arcsec per pixel in the spatial dimension for an unbinned pixel -``ronoise`` `numpy.ndarray`_ `numpy.floating`_ Read-out noise (e-). A list should be provided if a detector contains more than one amplifier. If any element of this list is <=0, the readout noise will be determined from the overscan regions defined by oscansec. -``saturation`` int, float The detector saturation level in ADU/DN -``spatflip`` bool If this is True then the spatial dimension will be flipped. PypeIt expects echelle orders to increase with increasing pixel number. I.e., setting spatflip=True can reorder images so that blue orders appear on the left and red orders on the right. -``specaxis`` int Spectra are dispersed along this axis. Allowed values are 0 (first dimension for a numpy array shape) or 1 (second dimension for numpy array shape). -``specflip`` bool If this is True then the dispersion dimension (specified by the specaxis) will be flipped. PypeIt expects wavelengths to increase with increasing pixel number. If this is not the case for this instrument, set specflip to True. -``xgap`` int, float Gap between the square detector pixels (expressed as a fraction of the x pixel size -- x is predominantly the spatial axis) -``ygap`` int, float Gap between the square detector pixels (expressed as a fraction of the y pixel size -- y is predominantly the spectral axis) -``ysize`` int, float The size of a pixel in the y-direction as a multiple of the x pixel size (i.e. xsize = 1.0 -- x is predominantly the dispersion axis) -================= ================ ================= ======================================================================================================================================================================================================================================================== +================= ===================== ================= ======================================================================================================================================================================================================================================================== +Attribute Type Array Type Description +================= ===================== ================= ======================================================================================================================================================================================================================================================== +``binning`` str Binning in PypeIt orientation (not the original) +``darkcurr`` int, float Dark current (e-/pixel/hour) +``dataext`` int Index of fits extension containing data +``datasec`` `numpy.ndarray`_ str Either the data sections or the header keyword where the valid data sections can be obtained, one per amplifier. If defined explicitly should be in FITS format (e.g., [1:2048,10:4096]). +``det`` int, `numpy.integer`_ PypeIt designation for detector number (1-based). +``gain`` `numpy.ndarray`_ `numpy.floating`_ Inverse gain (e-/ADU). A list should be provided if a detector contains more than one amplifier. +``mincounts`` int, float Counts (e-) in a pixel below this value will be ignored as being unphysical. +``nonlinear`` int, float Percentage of detector range which is linear (i.e. everything above ``nonlinear*saturation`` will be flagged as saturated) +``numamplifiers`` int Number of amplifiers +``oscansec`` `numpy.ndarray`_ str Either the overscan section or the header keyword where the valid data sections can be obtained, one per amplifier. If defined explicitly should be in FITS format (e.g., [1:2048,10:4096]). +``platescale`` int, float arcsec per pixel in the spatial dimension for an unbinned pixel +``ronoise`` `numpy.ndarray`_ `numpy.floating`_ Read-out noise (e-). A list should be provided if a detector contains more than one amplifier. If any element of this list is <=0, the readout noise will be determined from the overscan regions defined by oscansec. +``saturation`` int, float The detector saturation level in ADU/DN +``spatflip`` bool If this is True then the spatial dimension will be flipped. PypeIt expects echelle orders to increase with increasing pixel number. I.e., setting spatflip=True can reorder images so that blue orders appear on the left and red orders on the right. +``specaxis`` int Spectra are dispersed along this axis. Allowed values are 0 (first dimension for a numpy array shape) or 1 (second dimension for numpy array shape). +``specflip`` bool If this is True then the dispersion dimension (specified by the specaxis) will be flipped. PypeIt expects wavelengths to increase with increasing pixel number. If this is not the case for this instrument, set specflip to True. +``xgap`` int, float Gap between the square detector pixels (expressed as a fraction of the x pixel size -- x is predominantly the spatial axis) +``ygap`` int, float Gap between the square detector pixels (expressed as a fraction of the y pixel size -- y is predominantly the spectral axis) +``ysize`` int, float The size of a pixel in the y-direction as a multiple of the x pixel size (i.e. xsize = 1.0 -- x is predominantly the dispersion axis) +================= ===================== ================= ======================================================================================================================================================================================================================================================== diff --git a/doc/include/class_datamodel_onespec.rst b/doc/include/class_datamodel_onespec.rst index eff409ec4d..a773421d33 100644 --- a/doc/include/class_datamodel_onespec.rst +++ b/doc/include/class_datamodel_onespec.rst @@ -1,5 +1,5 @@ -**Version**: 1.0.1 +**Version**: 1.0.2 ================= ================ ================= ========================================================================================================================================== Attribute Type Array Type Description @@ -11,6 +11,7 @@ Attribute Type Array Type Description ``ivar`` `numpy.ndarray`_ `numpy.floating`_ Inverse variance array (matches units of flux) ``mask`` `numpy.ndarray`_ `numpy.integer`_ Mask array (1=Good,0=Bad) ``obj_model`` `numpy.ndarray`_ `numpy.floating`_ Object model for tellurics +``sigma`` `numpy.ndarray`_ `numpy.floating`_ One sigma noise array, equivalent to 1/sqrt(ivar) (matches units of flux) ``spect_meta`` dict header dict ``telluric`` `numpy.ndarray`_ `numpy.floating`_ Telluric model ``wave`` `numpy.ndarray`_ `numpy.floating`_ Wavelength array (angstroms in vacuum), weighted by pixel contributions diff --git a/doc/include/class_datamodel_slittraceset.rst b/doc/include/class_datamodel_slittraceset.rst index f245b38ad4..8b3f4e5bb9 100644 --- a/doc/include/class_datamodel_slittraceset.rst +++ b/doc/include/class_datamodel_slittraceset.rst @@ -1,5 +1,5 @@ -**Version**: 1.1.4 +**Version**: 1.1.5 ===================== ============================ ===================== ==================================================================================================================================================== Attribute Type Array Type Description @@ -28,7 +28,6 @@ Attribute Type Array Type Desc ``pypeline`` str PypeIt pypeline name ``right_init`` `numpy.ndarray`_ `numpy.floating`_ Spatial coordinates (pixel indices) of all right edges, one per slit. Derived from the TraceImage. Shape is Nspec by Nslits. ``right_tweak`` `numpy.ndarray`_ `numpy.floating`_ Spatial coordinates (pixel indices) of all right edges, one per slit. These traces have been adjusted by the flat-field. Shape is Nspec by Nslits. -``slitbitm`` str List of BITMASK keys from SlitTraceBitMask ``spat_id`` `numpy.ndarray`_ int, `numpy.integer`_ Slit ID number from SPAT measured at half way point. ``specmax`` `numpy.ndarray`_ `numpy.floating`_ Maximum spectral position (pixel units) allowed for each slit/order. Shape is Nslits. ``specmin`` `numpy.ndarray`_ `numpy.floating`_ Minimum spectral position (pixel units) allowed for each slit/order. Shape is Nslits. diff --git a/doc/include/class_datamodel_telluric.rst b/doc/include/class_datamodel_telluric.rst index 7143c793ac..f9f749f007 100644 --- a/doc/include/class_datamodel_telluric.rst +++ b/doc/include/class_datamodel_telluric.rst @@ -20,8 +20,10 @@ Attribute Type Array Type Description ``std_name`` str Type of standard source ``std_ra`` float RA of the standard source ``std_src`` str Name of the standard source -``telgrid`` str File containing grid of HITRAN atmosphere models +``telgrid`` str File containing PCA components or grid of HITRAN atmosphere models ``tell_norm_thresh`` float ?? +``tell_npca`` int Number of telluric PCA components used +``teltype`` str Type of telluric model, `pca` or `grid` ``tol`` float Relative tolerance for converage of the differential evolution optimization. ``ubound_norm`` float Flux normalization upper bound ``z_qso`` float Redshift of the QSO diff --git a/doc/include/datamodel_onespec.rst b/doc/include/datamodel_onespec.rst index 6eac2e340a..302e3a1fe5 100644 --- a/doc/include/datamodel_onespec.rst +++ b/doc/include/datamodel_onespec.rst @@ -1,5 +1,5 @@ -Version 1.0.1 +Version 1.0.2 ============ ============================== ========= =========================================================== HDU Name HDU Type Data Type Description diff --git a/doc/include/datamodel_sensfunc.rst b/doc/include/datamodel_sensfunc.rst index 2d9e9b5bf2..165d92e3b9 100644 --- a/doc/include/datamodel_sensfunc.rst +++ b/doc/include/datamodel_sensfunc.rst @@ -15,31 +15,28 @@ HDU Name HDU Type Data Type Description TELLURIC table (if present) -================= ========= =========================================================== -Column Data Type Description -================= ========= =========================================================== -``WAVE`` float64 Wavelength vector -``TELLURIC`` float64 Best-fitting telluric model spectrum -``OBJ_MODEL`` float64 Best-fitting object model spectrum -``TELL_THETA`` float64 Best-fitting telluric model parameters -``TELL_PRESS`` float64 Best-fitting telluric model pressure -``TELL_TEMP`` float64 Best-fitting telluric model temperature -``TELL_H2O`` float64 Best-fitting telluric model humidity -``TELL_AIRMASS`` float64 Best-fitting telluric model airmass -``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution -``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum -``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum -``OBJ_THETA`` float64 Best-fitting object model parameters -``CHI2`` float64 Chi-square of the best-fit model -``SUCCESS`` bool Flag that fit was successful -``NITER`` int64 Number of fit iterations -``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) -``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) -``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit -``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit -``WAVE_MIN`` float64 Minimum wavelength included in the fit -``WAVE_MAX`` float64 Maximum wavelength included in the fit -================= ========= =========================================================== +================= ========= ================================================================ +Column Data Type Description +================= ========= ================================================================ +``WAVE`` float64 Wavelength vector +``TELLURIC`` float64 Best-fitting telluric model spectrum +``OBJ_MODEL`` float64 Best-fitting object model spectrum +``TELL_THETA`` float64 Best-fitting telluric model parameters +``TELL_PARAM`` float64 Best-fitting telluric atmospheric parameters or PCA coefficients +``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution +``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum +``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum +``OBJ_THETA`` float64 Best-fitting object model parameters +``CHI2`` float64 Chi-square of the best-fit model +``SUCCESS`` bool Flag that fit was successful +``NITER`` int64 Number of fit iterations +``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) +``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) +``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit +``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit +``WAVE_MIN`` float64 Minimum wavelength included in the fit +``WAVE_MAX`` float64 Maximum wavelength included in the fit +================= ========= ================================================================ SENS table diff --git a/doc/include/datamodel_slittraceset.rst b/doc/include/datamodel_slittraceset.rst index e542c3e310..afd6996d75 100644 --- a/doc/include/datamodel_slittraceset.rst +++ b/doc/include/datamodel_slittraceset.rst @@ -1,5 +1,5 @@ -Version 1.1.4 +Version 1.1.5 ===================== ============================== ========= ============================================================================================================= HDU Name HDU Type Data Type Description diff --git a/doc/include/datamodel_telluric.rst b/doc/include/datamodel_telluric.rst index aac5007b4e..d22cbf732c 100644 --- a/doc/include/datamodel_telluric.rst +++ b/doc/include/datamodel_telluric.rst @@ -11,28 +11,25 @@ HDU Name HDU Type Data Type Description TELLURIC table -================= ========= =========================================================== -Column Data Type Description -================= ========= =========================================================== -``WAVE`` float64 Wavelength vector -``TELLURIC`` float64 Best-fitting telluric model spectrum -``OBJ_MODEL`` float64 Best-fitting object model spectrum -``TELL_THETA`` float64 Best-fitting telluric model parameters -``TELL_PRESS`` float64 Best-fitting telluric model pressure -``TELL_TEMP`` float64 Best-fitting telluric model temperature -``TELL_H2O`` float64 Best-fitting telluric model humidity -``TELL_AIRMASS`` float64 Best-fitting telluric model airmass -``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution -``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum -``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum -``OBJ_THETA`` float64 Best-fitting object model parameters -``CHI2`` float64 Chi-square of the best-fit model -``SUCCESS`` bool Flag that fit was successful -``NITER`` int64 Number of fit iterations -``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) -``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) -``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit -``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit -``WAVE_MIN`` float64 Minimum wavelength included in the fit -``WAVE_MAX`` float64 Maximum wavelength included in the fit -================= ========= =========================================================== +================= ========= ================================================================ +Column Data Type Description +================= ========= ================================================================ +``WAVE`` float64 Wavelength vector +``TELLURIC`` float64 Best-fitting telluric model spectrum +``OBJ_MODEL`` float64 Best-fitting object model spectrum +``TELL_THETA`` float64 Best-fitting telluric model parameters +``TELL_PARAM`` float64 Best-fitting telluric atmospheric parameters or PCA coefficients +``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution +``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum +``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum +``OBJ_THETA`` float64 Best-fitting object model parameters +``CHI2`` float64 Chi-square of the best-fit model +``SUCCESS`` bool Flag that fit was successful +``NITER`` int64 Number of fit iterations +``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) +``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) +``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit +``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit +``WAVE_MIN`` float64 Minimum wavelength included in the fit +``WAVE_MAX`` float64 Maximum wavelength included in the fit +================= ========= ================================================================ diff --git a/doc/include/links.rst b/doc/include/links.rst index e850dcc513..91e0d4fa7a 100644 --- a/doc/include/links.rst +++ b/doc/include/links.rst @@ -36,6 +36,7 @@ .. _scipy.optimize.curve_fit: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.curve_fit.html .. _scipy.optimize.leastsq: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.leastsq.html .. _scipy.optimize.least_squares: http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.least_squares.html +.. _scipy.optimize.linear_sum_assignment: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.linear_sum_assignment.html .. _scipy.optimize.OptimizeResult: http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.OptimizeResult.html .. _scipy.optimize.differential_evolution: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.differential_evolution.html .. _scipy.interpolate.interp1d: https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.interp1d.html diff --git a/doc/out_masks.rst b/doc/out_masks.rst index fefde58c76..1cb6e41b7d 100644 --- a/doc/out_masks.rst +++ b/doc/out_masks.rst @@ -28,65 +28,98 @@ the following bits: .. include:: include/imagebitmask_table.rst -Interpreting the bit values ---------------------------- +Viewing and interpreting the bit mask image (python) +---------------------------------------------------- -You can construct the appropriate bitmask in two ways: +To access the bitmask array, we recommend using +:class:`~pypeit.spec2dobj.Spec2DObj` directly: - - (Recommended) Instantiate the bitmask directly: +.. code-block:: python - .. code-block:: python + from pathlib import Path + from pypeit.spec2dobj import Spec2DObj - from pypeit.images.imagebitmask import ImageBitMask - bm = ImageBitMask() + f = Path('spec2d_b27-J1217p3905_KASTb_20150520T045733.560.fits').resolve() + detname = 'DET01' + spec2d = Spec2DObj.from_file(f, detname) + mask = spec2d.bpmmask - - Specifically for the ``spec2d*`` files, the names of the bits - and their order is saved to the header. You can instantiate a - generic :class:`~pypeit.bitmask.BitMask` from the header; - however, it's not clear how long this behavior will remain. To - instantiate the relevant :class:`~pypeit.bitmask.BitMask` from - the header: +The bitmask array is held by the ``spec2d.bpmmask`` attribute. - .. code-block:: python +But you can read it directly from the spec2d file: - from astropy.io import fits - from pypeit.bitmask import BitMask - hdu = fits.open('spec2d_DE.20100913.22358-CFHQS1_DEIMOS_20100913T061231.334.fits') - bm = BitMask(hdu[1].header['IMGBITM'].split(',')) +.. code-block:: python + from astropy.io import fits + from pypeit.images.imagebitmask import ImageBitMaskArray -With the :class:`~pypeit.bitmask.BitMask` or -:class:`~pypeit.images.imagebitmask.ImageBitMask` instantiated, you -can interpret the meaning of the ``BPMMASK`` extensions as follows: + f = Path('spec2d_b27-J1217p3905_KASTb_20150520T045733.560.fits').resolve() + detname = 'DET01' + hdu = fits.open(f) + mask = ImageBitMaskArray.from_array(hdu[f'{detname}-BPMMASK'].data) - - Use the :func:`~pypeit.bitmask.BitMask.flagged` method to - produce a boolean array that selects pixels flagged with a - given bit: +To show all the bit values directly: - .. code-block:: python +.. code-block:: python - extract_flag = bm.flagged(hdu['DET01-BPMMASK'].data, flag='EXTRACT') + from matplotlib import pyplot as plt - - Use the same method to select multiple bits: + plt.imshow(mask, origin='lower', interpolation='nearest') + plt.show() - .. code-block:: python +However, this isn't necessarily as useful as creating boolean arrays that +identify which pixels are flagged due to one or more reasons. - process_flag = bm.flagged(hdu['DET01-BPMMASK'].data, flag=['BPM', 'CR', 'SATURATION']) +E.g., to show all pixels flagged for having cosmic rays: - - Or select bits that are flagged for *any* reason: +.. code-block:: python - .. code-block:: python + plt.imshow(mask.flagged(flag='CR').astype(int), origin='lower', interpolation='nearest') + plt.show() - # You can use the `flagged` method for this - any_flag = bm.flagged(hdu['DET01-BPMMASK'].data) - # or equivalently - _any_flag = hdu['DET01-BPMMASK'].data > 0 +or as being part of the instrument-specific bad-pixel mask *or* not part of any slit: - - To get the human-readable reason that any given value is flagged, use the - :func:`~pypeit.bitmask.BitMask.flagged_bits` method. Currently this can - only be used with a *single* bit value: +.. code-block:: python - .. code-block:: python + plt.imshow(mask.flagged(flag=['BPM', 'OFFSLITS']).astype(int), + origin='lower', interpolation='nearest') + plt.show() + +You can also use the :ref:`pypeit_show_2dspec` script to include an image that +shows the full mask or an image that selects specific flags. + +To print the human-readable reason(s) any given value is flagged: + +.. code-block:: python + + coo = (0,0) # Tuple with the 2D coordinate of the pixel + print(mask.flagged_bits(coo)) + +.. _pypeit_print_bpm: + +pypeit_print_bpm +---------------- + +This simple executable allows you to effectively do the above via a command-line +script. The script usage can be displayed by calling the script with the +``-h`` option: + +.. include:: help/pypeit_print_bpm.rst + +A typical call and its output looks like this: + +.. code-block:: bash + + % pypeit_print_bpm 23 + [INFO] :: Using the default PypeIt bad pixel mask. + [INFO] :: The bad pixel mask value (23) corresponds to the following: + + * BPM : Component of the instrument-specific bad pixel mask + * CR : Cosmic ray detected + * SATURATION : Saturated pixel + * OFFSLITS : Pixel does not belong to any slit + + [INFO] :: Please see the following website for more information: + https://pypeit.readthedocs.io/en/release/out_masks.html - print(bm.flagged_bits(hdu['DET01-BPMMASK'].data[0,0])) diff --git a/doc/outputs.rst b/doc/outputs.rst index 5e0171820d..cb503788bb 100644 --- a/doc/outputs.rst +++ b/doc/outputs.rst @@ -62,6 +62,7 @@ they're produced, and their current datamodel. calibrations/calibrations out_spec2D out_spec1D + out_masks .. note:: diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index da16794add..ce1532dd1d 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -323,7 +323,7 @@ Key Type Options ``slit_illum_finecorr`` bool .. True If True, a fine correction to the spatial illumination profile will be performed. The fine correction is a low order 2D polynomial fit to account for a gradual change to the spatial illumination profile as a function of wavelength. ``slit_illum_pad`` int, float .. 5.0 The number of pixels to pad the slit edges when constructing the slit-illumination profile. Single value applied to both edges. ``slit_illum_ref_idx`` int .. 0 The index of a reference slit (0-indexed) used for estimating the relative spectral sensitivity (or the relative blaze). This parameter is only used if ``slit_illum_relative = True``. -``slit_illum_relative`` bool .. False Generate an image of the relative spectral illumination for a multi-slit setup. If you set ``use_slitillum = True`` for any of the frames that use the flatfield model, this *must* be set to True. Currently, this is only used for SlicerIFU reductions. +``slit_illum_relative`` bool .. False Generate an image of the relative spectral illumination for a multi-slit setup. If you set ``use_specillum = True`` for any of the frames that use the flatfield model, this *must* be set to True. Currently, this is only used for SlicerIFU reductions. ``slit_illum_smooth_npix`` int .. 10 The number of pixels used to determine smoothly varying relative weights is given by ``nspec/slit_illum_smooth_npix``, where nspec is the number of spectral pixels. ``slit_trim`` int, float, tuple .. 3.0 The number of pixels to trim each side of the slit when selecting pixels to use for fitting the spectral response function. Single values are used for both slit edges; a two-tuple can be used to trim the left and right sides differently. ``spat_samp`` int, float .. 5.0 Spatial sampling for slit illumination function. This is the width of the median filter in pixels used to determine the slit illumination function, and thus sets the minimum scale on which the illumination function will have features. @@ -411,6 +411,7 @@ Key Type Options ``sync_predict`` str ``pca``, ``nearest``, ``auto`` ``pca`` Mode to use when predicting the form of the trace to insert. Use `pca` to use the PCA decomposition, `nearest` to reproduce the shape of the nearest trace, or `auto` to let PypeIt decide which mode to use between `pca` and `nearest`. In general, it will first try `pca`, and if that is not possible, it will use `nearest`. ``sync_to_edge`` bool .. True If adding a first left edge or a last right edge, ignore `center_mode` for these edges and place them at the edge of the detector (with the relevant shape). ``trace_median_frac`` int, float .. .. After detection of peaks in the rectified Sobel-filtered image and before refitting the edge traces, the rectified image is median filtered with a kernel width of `trace_median_frac*nspec` along the spectral dimension. +``trace_rms_tol`` int, float .. .. After retracing edges using peaks detected in the rectified and collapsed image, the RMS difference (in pixels) between the original and refit traces are calculated. This sets the upper limit of the RMS for traces that will be removed. If None, no limit is set and all new traces are kept. ``trace_thresh`` int, float .. .. After rectification and median filtering of the Sobel-filtered image (see `trace_median_frac`), values in the median-filtered image *below* this threshold are masked in the refitting of the edge trace data. If None, no masking applied. ``use_maskdesign`` bool .. False Use slit-mask designs to identify slits. =========================== ================ =========================================== ============== ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== @@ -506,40 +507,40 @@ Coadd1DPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.Coadd1DPar` -==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= -Key Type Options Default Description -==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= -``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that spec1d*.fits files were createdwith the current version of PypeIt -``coaddfile`` str .. .. Output filename -``dloglam`` int, float .. .. Dispersion in units of log10(wave) in case you want to specify it in the get_wave_grid (for the 'velocity' or 'log10' options), otherwise a median value is computed from the data. -``dv`` int, float .. .. Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), otherwise a median value is computed from the data. -``dwave`` int, float .. .. Dispersion in Angstroms in case you want to specify it in the get_wave_grid (for the 'linear' option), otherwise a median value is computed from the data. -``ex_value`` str .. ``OPT`` The extraction to coadd, i.e. optimal or boxcar. Must be either 'OPT' or 'BOX' -``filter`` str .. ``none`` Filter for scaling. See flux_calib.load_fitler_file() for naming. Ignore if none -``filter_mag`` float .. .. Magnitude of the source in the given filter -``filter_mask`` str, list .. .. List of wavelength regions to mask when doing the scaling (`i.e.`, occasional junk pixels). Colon and comma separateed, e.g. 5552:5559,6010:6030 -``flux_value`` bool .. True If True (default), the code will coadd the fluxed spectra (i.e. the FLAM) in the spec1d files. If False, it will coadd the counts. -``lower`` int, float .. 3.0 Lower rejection threshold used for rejecting pixels when combining spectra in units of sigma. -``mag_type`` str .. ``AB`` Magnitude type. AB is the only option currently allowed -``maxiter_reject`` int .. 5 Maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen successive iterations or when maxiter_reject is reached. -``maxiter_scale`` int .. 5 Maximum number of iterations performed for rescaling spectra. -``maxrej`` int .. .. Coadding performs iterative rejection by comparing each exposure to a preliminary stack of all the exposures. If this parameter is set then it will not reject more than maxrej pixels per iteration of this rejection. The default is None, which means no maximum on rejected pixels. -``nbests`` list, int .. .. Number of orders to use for estimating the per exposure weights. Default is None, which will just use one fourth of the total number of orders. This is only used for Echelle -``nmaskedge`` int .. 2 Number of edge pixels to mask. This should be removed/fixed. -``ref_percentile`` int, float .. 70.0 Percentile used for selecting the minimum SNR cut from a reference spectrum used to robustly determine the median ratio between spectra. This parameter is used by coadd1d.robust_median_ratio as part of the automatic rescaling procedure. Pixels above this percentile cut are deemed the "good" pixels and are used to compute the ratio of two spectra. This must be a number between 0 and 100. -``scale_method`` str .. ``auto`` Method used to rescale the spectra prior to coadding. The options are: 'auto' -- Determine the scaling method automatically based on the S/N ratio which works well. 'poly' -- Polynomial rescaling. 'median' -- Median rescaling 'none' -- Do not rescale. 'hand' -- Pass in hand scaling factors. This option is not well tested. -``sigrej_exp`` int, float .. .. Rejection threshold used for rejecting exposures with S/N more than sigrej_exp*sigma above the median S/N. If None (the default), no rejection is performed. Currently, only available for multi-slit observations. -``sigrej_scale`` int, float .. 3.0 Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. -``sn_clip`` int, float .. 30.0 Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectrum which neverthless differ at a level greater than the formal S/N due to systematics. -``sn_min_medscale`` int, float .. 0.5 For scale method set to ``auto``, this sets the minimum SNR for which median scaling is attempted. -``sn_min_polyscale`` int, float .. 2.0 For scale method set to ``auto``, this sets the minimum SNR for which polynomial scaling is attempted. -``sn_smooth_npix`` int, float .. .. Number of pixels to median filter by when computing S/N used to decide how to scale and weight spectra. If set to None (default), the code will determine the effective number of good pixels per spectrum in the stack that is being co-added and use 10% of this neff. -``spec_samp_fact`` float .. 1.0 Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or coarser (spec_samp_fact > 1.0) by this sampling factor. This basically multiples the 'native' spectral pixels by spec_samp_fact, i.e. units spec_samp_fact are pixels. -``upper`` int, float .. 3.0 Upper rejection threshold used for rejecting pixels when combining spectra in units of sigma. -``wave_grid_max`` int, float .. .. Used in case you want to specify the maximum wavelength in your wavelength grid, default=None computes from data -``wave_grid_min`` int, float .. .. Used in case you want to specify the minimum wavelength in your wavelength grid, default=None computes from data -``wave_method`` str .. ``linear`` Method used to construct wavelength grid for coadding spectra. The routine that creates the wavelength is :func:`~pypeit.core.wavecal.wvutils.get_wave_grid`. The options are: 'iref' -- Use the first wavelength array. 'velocity' -- Grid is uniform in velocity. 'log10' -- Grid is uniform in log10(wave). This is the same as velocity. 'linear' -- Grid is uniform in lambda. 'concatenate' -- Meld the input wavelength arrays -==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= +==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +Key Type Options Default Description +==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +``coaddfile`` str .. .. Output filename +``dloglam`` int, float .. .. Dispersion in units of log10(wave) in case you want to specify it in the get_wave_grid (for the 'velocity' or 'log10' options), otherwise a median value is computed from the data. +``dv`` int, float .. .. Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), otherwise a median value is computed from the data. +``dwave`` int, float .. .. Dispersion in Angstroms in case you want to specify it in the get_wave_grid (for the 'linear' option), otherwise a median value is computed from the data. +``ex_value`` str .. ``OPT`` The extraction to coadd, i.e. optimal or boxcar. Must be either 'OPT' or 'BOX' +``filter`` str .. ``none`` Filter for scaling. See flux_calib.load_fitler_file() for naming. Ignore if none +``filter_mag`` float .. .. Magnitude of the source in the given filter +``filter_mask`` str, list .. .. List of wavelength regions to mask when doing the scaling (`i.e.`, occasional junk pixels). Colon and comma separateed, e.g. 5552:5559,6010:6030 +``flux_value`` bool .. True If True (default), the code will coadd the fluxed spectra (i.e. the FLAM) in the spec1d files. If False, it will coadd the counts. +``lower`` int, float .. 3.0 Lower rejection threshold used for rejecting pixels when combining spectra in units of sigma. +``mag_type`` str .. ``AB`` Magnitude type. AB is the only option currently allowed +``maxiter_reject`` int .. 5 Maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen successive iterations or when maxiter_reject is reached. +``maxiter_scale`` int .. 5 Maximum number of iterations performed for rescaling spectra. +``maxrej`` int .. .. Coadding performs iterative rejection by comparing each exposure to a preliminary stack of all the exposures. If this parameter is set then it will not reject more than maxrej pixels per iteration of this rejection. The default is None, which means no maximum on rejected pixels. +``nbests`` list, int .. .. Number of orders to use for estimating the per exposure weights. Default is None, which will just use one fourth of the total number of orders. This is only used for Echelle +``nmaskedge`` int .. 2 Number of edge pixels to mask. This should be removed/fixed. +``ref_percentile`` int, float .. 70.0 Percentile used for selecting the minimum SNR cut from a reference spectrum used to robustly determine the median ratio between spectra. This parameter is used by coadd1d.robust_median_ratio as part of the automatic rescaling procedure. Pixels above this percentile cut are deemed the "good" pixels and are used to compute the ratio of two spectra. This must be a number between 0 and 100. +``scale_method`` str .. ``auto`` Method used to rescale the spectra prior to coadding. The options are: 'auto' -- Determine the scaling method automatically based on the S/N ratio which works well. 'poly' -- Polynomial rescaling. 'median' -- Median rescaling 'none' -- Do not rescale. 'hand' -- Pass in hand scaling factors. This option is not well tested. +``sigrej_exp`` int, float .. .. Rejection threshold used for rejecting exposures with S/N more than sigrej_exp*sigma above the median S/N. If None (the default), no rejection is performed. Currently, only available for multi-slit observations. +``sigrej_scale`` int, float .. 3.0 Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. +``sn_clip`` int, float .. 30.0 Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectrum which neverthless differ at a level greater than the formal S/N due to systematics. +``sn_min_medscale`` int, float .. 0.5 For scale method set to ``auto``, this sets the minimum SNR for which median scaling is attempted. +``sn_min_polyscale`` int, float .. 2.0 For scale method set to ``auto``, this sets the minimum SNR for which polynomial scaling is attempted. +``sn_smooth_npix`` int, float .. .. Number of pixels to median filter by when computing S/N used to decide how to scale and weight spectra. If set to None (default), the code will determine the effective number of good pixels per spectrum in the stack that is being co-added and use 10% of this neff. +``spec_samp_fact`` float .. 1.0 Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or coarser (spec_samp_fact > 1.0) by this sampling factor. This basically multiples the 'native' spectral pixels by spec_samp_fact, i.e. units spec_samp_fact are pixels. +``upper`` int, float .. 3.0 Upper rejection threshold used for rejecting pixels when combining spectra in units of sigma. +``wave_grid_max`` int, float .. .. Used in case you want to specify the maximum wavelength in your wavelength grid, default=None computes from data +``wave_grid_min`` int, float .. .. Used in case you want to specify the minimum wavelength in your wavelength grid, default=None computes from data +``wave_method`` str .. ``linear`` Method used to construct wavelength grid for coadding spectra. The routine that creates the wavelength is :func:`~pypeit.core.wavecal.wvutils.get_wave_grid`. The options are: 'iref' -- Use the first wavelength array. 'velocity' -- Grid is uniform in velocity. 'log10' -- Grid is uniform in log10(wave). This is the same as velocity. 'linear' -- Grid is uniform in lambda. 'concatenate' -- Meld the input wavelength arrays +``weight_method`` str .. ``auto`` Method used to weight the spectra for coadding. The options are: 'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent.'constant' -- Constant weights based on rms_sn**2'uniform' -- Uniform weighting'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option will not work well at low S/N ratio although it is useful for objects where only a small fraction of the spectral coverage has high S/N ratio (like high-z quasars).'relative' -- Apply relative weights implying one reference exposure will receive unit weight at all wavelengths and all others receive relatively wavelength dependent weights . Note, relative weighting will only work well when there is at least one spectrum with a reasonable S/N, and a continuum. This option may only be better when the object being used has a strong continuum + emission lines. This is particularly useful if you are dealing with highly variable spectra (e.g. emission lines) andrequire a precision better than ~1 per cent.'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated. +==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= ---- @@ -578,7 +579,6 @@ Class Instantiation: :class:`~pypeit.par.pypeitpar.Collate1DPar` ========================= ========== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== Key Type Options Default Description ========================= ========== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== -``chk_version`` bool .. False Whether to check the data model versions of spec1d files and sensfunc files. ``dry_run`` bool .. False If set, the script will display the matching File and Object Ids but will not flux, coadd or archive. ``exclude_serendip`` bool .. False Whether to exclude SERENDIP objects from collating. ``exclude_slit_trace_bm`` list, str .. [] A list of slit trace bitmask bits that should be excluded. @@ -643,21 +643,22 @@ ReduxPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.ReduxPar` -====================== ============== ======= ============================================ =============================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -====================== ============== ======= ============================================ =============================================================================================================================================================================================================================================================================================================================================================== -``calwin`` int, float .. 0 The window of time in hours to search for calibration frames for a science frame -``detnum`` int, list .. .. Restrict reduction to a list of detector indices. In case of mosaic reduction (currently only available for Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of tuples of the detector indices that are mosaiced together. E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]`` -``ignore_bad_headers`` bool .. False Ignore bad headers (NOT recommended unless you know it is safe). -``maskIDs`` str, int, list .. .. Restrict reduction to a set of slitmask IDs Example syntax -- ``maskIDs = 818006,818015`` This must be used with detnum (for now). -``qadir`` str .. ``QA`` Directory relative to calling directory to write quality assessment files. -``quicklook`` bool .. False Run a quick look reduction? This is usually good if you want to quickly reduce the data (usually at the telescope in real time) to get an initial estimate of the data quality. -``redux_path`` str .. ``/Users/westfall/Work/packages/pypeit/doc`` Path to folder for performing reductions. Default is the current working directory. -``scidir`` str .. ``Science`` Directory relative to calling directory to write science files. -``slitspatnum`` str, list .. .. Restrict reduction to a set of slit DET:SPAT values (closest slit is used). Example syntax -- slitspatnum = DET01:175,DET01:205 or MSC02:2234 If you are re-running the code, (i.e. modifying one slit) you *must* have the precise SPAT_ID index. -``sortroot`` str .. .. A filename given to output the details of the sorted files. If None, the default is the root name of the pypeit file. If off, no output is produced. -``spectrograph`` str .. .. Spectrograph that provided the data to be reduced. See :ref:`instruments` for valid options. -====================== ============== ======= ============================================ =============================================================================================================================================================================================================================================================================================================================================================== +====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== +``calwin`` int, float .. 0 The window of time in hours to search for calibration frames for a science frame +``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that all files were created with the current version of PypeIt. If set to False, the code will attempt to read out-of-date files and keep going. Beware (!!) that this can lead to unforeseen bugs that either cause the code to crash or lead to erroneous results. I.e., you really need to know what you are doing if you set this to False! +``detnum`` int, list .. .. Restrict reduction to a list of detector indices. In case of mosaic reduction (currently only available for Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of tuples of the detector indices that are mosaiced together. E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]`` +``ignore_bad_headers`` bool .. False Ignore bad headers (NOT recommended unless you know it is safe). +``maskIDs`` str, int, list .. .. Restrict reduction to a set of slitmask IDs Example syntax -- ``maskIDs = 818006,818015`` This must be used with detnum (for now). +``qadir`` str .. ``QA`` Directory relative to calling directory to write quality assessment files. +``quicklook`` bool .. False Run a quick look reduction? This is usually good if you want to quickly reduce the data (usually at the telescope in real time) to get an initial estimate of the data quality. +``redux_path`` str .. ``/Users/westfall/Work/packages/pypeit/doc`` Path to folder for performing reductions. Default is the current working directory. +``scidir`` str .. ``Science`` Directory relative to calling directory to write science files. +``slitspatnum`` str, list .. .. Restrict reduction to a set of slit DET:SPAT values (closest slit is used). Example syntax -- slitspatnum = DET01:175,DET01:205 or MSC02:2234 If you are re-running the code, (i.e. modifying one slit) you *must* have the precise SPAT_ID index. +``sortroot`` str .. .. A filename given to output the details of the sorted files. If None, the default is the root name of the pypeit file. If off, no output is produced. +``spectrograph`` str .. .. Spectrograph that provided the data to be reduced. See :ref:`instruments` for valid options. +====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== ---- @@ -690,34 +691,35 @@ CubePar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.CubePar` -==================== ===== ===================== ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -==================== ===== ===================== ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -``align`` bool .. False If set to True, the input frames will be spatially aligned by cross-correlating the whitelight images with either a reference image (see ``reference_image``) or the whitelight image that is generated using the first spec2d listed in the coadd3d file. Alternatively, the user can specify the offsets (i.e. Delta RA x cos(dec) and Delta Dec, both in arcsec) in the spec2d block of the coadd3d file. See the documentation for examples of this usage. -``astrometric`` bool .. True If true, an astrometric correction will be applied using the alignment frames. -``combine`` bool .. False If set to True, the input frames will be combined. Otherwise, a separate datacube will be generated for each input spec2d file, and will be saved as a spec3d file. -``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. -``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. -``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. -``method`` str ``subpixel``, ``ngp`` ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "ngp" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). -``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` -``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. -``ra_min`` float .. .. Minimum RA to use when generating the WCS. If None, the default is minimum RA based on the WCS of all spaxels. Units should be degrees. -``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). -``relative_weights`` bool .. False If set to True, the combined frames will use a relative weighting scheme. This only works well if there is a common continuum source in the field of view of all input observations, and is generally only required if high relative precision is desired. -``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". -``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. -``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. -``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. -``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel. -``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. -``spec_subpixel`` int .. 5 When method=subpixel, spec_subpixel sets the subpixellation scale of each detector pixel in the spectral direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spat_subpixel. -``standard_cube`` str .. .. Filename of a standard star datacube. This cube will be used to correct the relative scales of the slits, and to flux calibrate the science datacube. -``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. -``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. -``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. -``whitelight_range`` list .. None, None A two element list specifying the wavelength range over which to generate the white light image. The first (second) element is the minimum (maximum) wavelength to use. If either of these elements are None, PypeIt will automatically use a wavelength range that ensures all spaxels have the same wavelength coverage. Note, if you are using a reference_image to align all frames, it is preferable to use the same white light wavelength range for all white light images. For example, you may wish to use an emission line map to register two frames. -==================== ===== ===================== ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== +==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +Key Type Options Default Description +==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +``align`` bool .. False If set to True, the input frames will be spatially aligned by cross-correlating the whitelight images with either a reference image (see ``reference_image``) or the whitelight image that is generated using the first spec2d listed in the coadd3d file. Alternatively, the user can specify the offsets (i.e. Delta RA x cos(dec) and Delta Dec, both in arcsec) in the spec2d block of the coadd3d file. See the documentation for examples of this usage. +``astrometric`` bool .. True If true, an astrometric correction will be applied using the alignment frames. +``combine`` bool .. False If set to True, the input frames will be combined. Otherwise, a separate datacube will be generated for each input spec2d file, and will be saved as a spec3d file. +``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. +``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. +``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. +``method`` str ``subpixel``, ``ngp`` ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "ngp" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). +``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` +``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. +``ra_min`` float .. .. Minimum RA to use when generating the WCS. If None, the default is minimum RA based on the WCS of all spaxels. Units should be degrees. +``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). +``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". +``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. +``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. +``slice_subpixel`` int .. 5 When method=subpixel, slice_subpixel sets the subpixellation scale of each IFU slice. The default option is to divide each slice into 5 sub-slices during datacube creation. See also, spec_subpixel and spat_subpixel. +``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. +``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel and slice_subpixel. +``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. +``spec_subpixel`` int .. 5 When method=subpixel, spec_subpixel sets the subpixellation scale of each detector pixel in the spectral direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spat_subpixel and slice_subpixel. +``standard_cube`` str .. .. Filename of a standard star datacube. This cube will be used to correct the relative scales of the slits, and to flux calibrate the science datacube. +``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. +``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. +``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. +``weight_method`` str ``auto``, ``constant``, ``uniform``, ``wave_dependent``, ``relative``, ``ivar`` ``auto`` Method used to weight the spectra for coadding. The options are: 'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent.'constant' -- Constant weights based on rms_sn**2'uniform' -- Uniform weighting'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option will not work well at low S/N ratio although it is useful for objects where only a small fraction of the spectral coverage has high S/N ratio (like high-z quasars).'relative' -- Apply relative weights implying one reference exposure will receive unit weight at all wavelengths and all others receive relatively wavelength dependent weights . Note, relative weighting will only work well when there is at least one spectrum with a reasonable S/N, and a continuum. This option may only be better when the object being used has a strong continuum + emission lines. This is particularly useful if you are dealing with highly variable spectra (e.g. emission lines) andrequire a precision better than ~1 per cent.'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated. +``whitelight_range`` list .. None, None A two element list specifying the wavelength range over which to generate the white light image. The first (second) element is the minimum (maximum) wavelength to use. If either of these elements are None, PypeIt will automatically use a wavelength range that ensures all spaxels have the same wavelength coverage. Note, if you are using a reference_image to align all frames, it is preferable to use the same white light wavelength range for all white light images. For example, you may wish to use an emission line map to register two frames. +==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= ---- @@ -901,15 +903,15 @@ ScatteredLightPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.ScatteredLightPar` -================== ========= ================================= ========= =============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -================== ========= ================================= ========= =============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -``finecorr`` bool .. True If True, a fine correction to the scattered light will be performed. However, the fine correction will only be applied if the model/frame/archive correction is performed. -``finecorr_mask`` int, list .. .. A list containing the inter-slit regions that the user wishes to mask during the fine correction to the scattered light. Each integer corresponds to an inter-slit region. For example, "0" corresponds to all pixels left of the leftmost slit, while a value of "1" corresponds to all pixels between the first and second slit (counting from the left). It should be either a single integer value, or a list of integer values. The default (None) means that no inter-slit regions will be masked. -``finecorr_order`` int .. 2 Polynomial order to use for the fine correction to the scattered light subtraction. It should be a low value. -``finecorr_pad`` int .. 2 Number of unbinned pixels to extend the slit edges by when masking the slits for thefine correction to the scattered light. -``method`` str ``model``, ``frame``, ``archive`` ``model`` Method used to fit the overscan. Options are: model, frame, archive.'model' will the scattered light model parameters derived from a user-specified frame during their reduction (note, you will need to make sure that you set appropriate scattlight frames in your .pypeit file for this option). 'frame' will use each individual frame to determine the scattered light that affects this frame. 'archive' will use an archival model parameter solution for the scattered light (note that this option is not currently available for all spectrographs). -================== ========= ================================= ========= =============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== +=================== ========= ================================= ========= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ +Key Type Options Default Description +=================== ========= ================================= ========= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ +``finecorr_mask`` int, list .. .. A list containing the inter-slit regions that the user wishes to mask during the fine correction to the scattered light. Each integer corresponds to an inter-slit region. For example, "0" corresponds to all pixels left of the leftmost slit, while a value of "1" corresponds to all pixels between the first and second slit (counting from the left). It should be either a single integer value, or a list of integer values. The default (None) means that no inter-slit regions will be masked. +``finecorr_method`` str ``median``, ``poly`` .. If None, a fine correction to the scattered light will not be performed. Otherwise, the allowed methods include: median, poly. 'median' will subtract a constant value from an entire CCD row, based on a median of the pixels that are not on slits (see also, 'finecorr_pad'). 'poly' will fit a polynomial to the scattered light in each row, based on the pixels that are not on slits (see also, 'finecorr_pad'). +``finecorr_order`` int .. 2 Polynomial order to use for the fine correction to the scattered light subtraction. It should be a low value. +``finecorr_pad`` int .. 4 Number of unbinned pixels to extend the slit edges by when masking the slits for the fine correction to the scattered light. +``method`` str ``model``, ``frame``, ``archive`` ``model`` Method used to fit the overscan. Options are: model, frame, archive. 'model' will the scattered light model parameters derived from a user-specified frame during their reduction (note, you will need to make sure that you set appropriate scattlight frames in your .pypeit file for this option). 'frame' will use each individual frame to determine the scattered light that affects this frame. 'archive' will use an archival model parameter solution for the scattered light (note that this option is not currently available for all spectrographs). +=================== ========= ================================= ========= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ ---- @@ -1006,7 +1008,7 @@ Key Type Options Default ``popsize`` int .. 30 A multiplier for setting the total population size for the differential evolution optimization. See scipy.optimize.differential_evolution for details. ``recombination`` int, float .. 0.7 The recombination constant for the differential evolution optimization. This should be in the range [0, 1]. See scipy.optimize.differential_evolution for details. ``redshift`` int, float .. 0.0 The redshift for the object model. This is currently only used by objmodel=qso -``resln_frac_bounds`` tuple .. (0.5, 1.5) Bounds for the resolution fit optimization which is part of the telluric model. This range is in units of the resln_guess, so the (0.5, 1.5) would bound the spectral resolution fit to be within the range bounds_resln = (0.5*resln_guess, 1.5*resln_guess) +``resln_frac_bounds`` tuple .. (0.6, 1.4) Bounds for the resolution fit optimization which is part of the telluric model. This range is in units of the resln_guess, so the (0.6, 1.4) would bound the spectral resolution fit to be within the range bounds_resln = (0.6*resln_guess, 1.4*resln_guess) ``resln_guess`` int, float .. .. A guess for the resolution of your spectrum expressed as lambda/dlambda. The resolution is fit explicitly as part of the telluric model fitting, but this guess helps determine the bounds for the optimization (see next). If not provided, the wavelength sampling of your spectrum will be used and the resolution calculated using a typical sampling of 3 spectral pixels per resolution element. ``seed`` int .. 777 An initial seed for the differential evolution optimization, which is a random process. The default is a seed = 777 which will be used to generate a unique seed for every order. A specific seed is used because otherwise the random number generator will use the time for the seed, and the results will not be reproducible. ``sn_clip`` int, float .. 30.0 This adds an error floor to the ivar, preventing too much rejection at high-S/N (`i.e.`, standard stars, bright objects) using the function utils.clip_ivar. A small erorr is added to the input ivar so that the output ivar_out will never give S/N greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectra which neverthless differ at a level greater than the formal S/N due to the fact that our telluric models are only good to about 3%. @@ -1017,6 +1019,8 @@ Key Type Options Default ``sticky`` bool .. True Sticky parameter for the utils.djs_reject algorithm for iterative model fit rejection. If set to True then points rejected from a previous iteration are kept rejected, in other words the bad pixel mask is the OR of all previous iterations and rejected pixels accumulate. If set to False, the bad pixel mask is the mask from the previous iteration, and if the model fit changes between iterations, points can alternate from being rejected to not rejected. At present this code only performs optimizations with differential evolution and experience shows that sticky needs to be True in order for these to converge. This is because the outliers can be so large that they dominate the loss function, and one never iteratively converges to a good model fit. In other words, the deformations in the model between iterations with sticky=False are too small to approach a reasonable fit. ``telgridfile`` str .. .. File containing the telluric grid for the observatory in question. These grids are generated from HITRAN models for each observatory using nominal site parameters. They must be downloaded from the GoogleDrive and installed in your PypeIt installation via the pypeit_install_telluric script. NOTE: This parameter no longer includes the full pathname to the Telluric Grid file, but is just the filename of the grid itself. ``tell_norm_thresh`` int, float .. 0.9 Threshold of telluric absorption region +``tell_npca`` int .. 5 Number of telluric PCA components used. Can be set to any number from 1 to 10. +``teltype`` str .. ``pca`` Method used to evaluate telluric models, either pca or grid. The grid option uses a fixed grid of pre-computed HITRAN+LBLRTM atmospheric transmission models for each observatory, whereas the pca option uses principal components of a larger model grid to compute an accurate pseudo-telluric model with a much lighter telgridfile. ``tol`` float .. 0.001 Relative tolerance for converage of the differential evolution optimization. See scipy.optimize.differential_evolution for details. ``upper`` int, float .. 3.0 Upper rejection threshold in units of sigma_corr*sigma, where sigma is the formal noise of the spectrum, and sigma_corr is an empirically determined correction to the formal error. See above for description. ======================= ================== ======= ========================== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= @@ -1425,7 +1429,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gemini_gmos_north_e2v: @@ -1786,7 +1790,7 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gemini_gnirs_echelle: @@ -1915,7 +1919,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 6 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gemini_gnirs_ifu: @@ -2058,7 +2062,7 @@ Alterations to the default parameters are: [[UVIS]] extinct_correct = False [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gtc_maat: @@ -2134,7 +2138,7 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 180, + exprng = None, 300, [[[process]]] mask_cr = True noise_floor = 0.01 @@ -2347,7 +2351,7 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 180, + exprng = None, 300, [[[process]]] mask_cr = True noise_floor = 0.01 @@ -2665,7 +2669,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-keck_esi: @@ -2930,7 +2934,11 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_10500_R120000.fits + pix_shift_bounds = (-40.0, 40.0) + [telluric] + resln_frac_bounds = (0.25, 1.25) + pix_shift_bounds = (-40.0, 40.0) .. _instr_par-keck_kcrm: @@ -3035,6 +3043,8 @@ Alterations to the default parameters are: [sensfunc] [[UVIS]] extinct_correct = False + [[IR]] + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-keck_kcwi: @@ -3117,6 +3127,7 @@ Alterations to the default parameters are: use_pattern = True [[flatfield]] spec_samp_coarse = 20.0 + spat_samp = 1.0 tweak_slits_thresh = 0.0 tweak_slits_maxfrac = 0.0 slit_illum_relative = True @@ -3139,7 +3150,7 @@ Alterations to the default parameters are: use_pattern = True subtract_scattlight = True [[[scattlight]]] - finecorr_mask = 12 + finecorr_method = median [reduce] [[extraction]] skip_extraction = True @@ -3148,6 +3159,8 @@ Alterations to the default parameters are: [sensfunc] [[UVIS]] extinct_correct = False + [[IR]] + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-keck_lris_blue: @@ -3249,7 +3262,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_blue_orig: @@ -3351,7 +3364,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_red: @@ -3464,7 +3477,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 9 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_red_mark4: @@ -3577,7 +3590,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 9 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_red_orig: @@ -3690,7 +3703,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 9 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_mosfire: @@ -3817,7 +3830,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 13 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_nires: @@ -3960,7 +3973,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_nirspec_low: @@ -4089,7 +4102,10 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-8.0, 8.0) + [telluric] + pix_shift_bounds = (-8.0, 8.0) .. _instr_par-lbt_luci1: @@ -5018,7 +5034,7 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-magellan_fire_long: @@ -5148,7 +5164,7 @@ Alterations to the default parameters are: find_trim_edge = 50, 50, [sensfunc] [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-magellan_mage: @@ -5665,7 +5681,7 @@ Alterations to the default parameters are: [sensfunc] polyorder = 7 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-mmt_bluechannel: @@ -5914,7 +5930,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-not_alfosc: @@ -6411,7 +6427,7 @@ Alterations to the default parameters are: [[UVIS]] polycorrect = False [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-p200_tspec: @@ -6557,7 +6573,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-shane_kast_blue: @@ -6658,7 +6674,7 @@ Alterations to the default parameters are: spectrum = sky_kastb_600.fits [sensfunc] [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-shane_kast_red: @@ -6750,7 +6766,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-shane_kast_red_ret: @@ -6844,7 +6860,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-soar_goodman_blue: @@ -6947,7 +6963,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-soar_goodman_red: @@ -7052,7 +7068,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-tng_dolores: @@ -7242,7 +7258,7 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_Paranal_VIS_9800_25000_R25000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-vlt_sinfoni: @@ -7383,7 +7399,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 7 [[IR]] - telgridfile = TelFit_Paranal_NIR_9800_25000_R25000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-vlt_xshooter_nir: @@ -7534,7 +7550,11 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_Paranal_NIR_9800_25000_R25000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-10.0, 10.0) + [telluric] + resln_frac_bounds = (0.4, 2.0) + pix_shift_bounds = (-10.0, 10.0) .. _instr_par-vlt_xshooter_uvb: @@ -7676,7 +7696,10 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-8.0, 8.0) + [telluric] + pix_shift_bounds = (-8.0, 8.0) .. _instr_par-vlt_xshooter_vis: @@ -7820,7 +7843,11 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_Paranal_VIS_4900_11100_R25000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-10.0, 10.0) + [telluric] + resln_frac_bounds = (0.4, 2.0) + pix_shift_bounds = (-10.0, 10.0) .. _instr_par-wht_isis_blue: diff --git a/doc/releases/1.15.0.rst b/doc/releases/1.15.0.rst index 119f1e328c..83d0e236a8 100644 --- a/doc/releases/1.15.0.rst +++ b/doc/releases/1.15.0.rst @@ -1,6 +1,6 @@ -Version 1.14.1dev -================= +Version 1.15.0 +============== Installation Changes -------------------- @@ -34,7 +34,6 @@ Functionality/Performance Improvements and Additions - Add a sensible error message to the pypeit Spectrum1D loaders in the event a user inadvertently tries to use Spectrum1D instead of SpectrumList for a ``spec1d`` file. -- Add support for the R4K detector for MDM OSMOS - Enabled interpolation and extrapolation of missing echelle orders. This is currently only used for Keck/HIRES, but the code is general. - Allow the specification of wavelength limits on the flexure cross-correlation @@ -44,11 +43,15 @@ Functionality/Performance Improvements and Additions to reflect the *type* of polynomial being fit. Both names point to the same code, but the name ``'polynomial'`` is deprecated and will be removed in the future. -- Added a new GUI for creating and editing PypeIt input files: ``pypeit_setup_gui`` +- Introduced PCA method for telluric corrections +- Added slicer subpixel sampling for DataCube generation +- Added ``trace_rms_tol`` parameter for edge tracing, which helps guard against + poorly constrained traces for spectrally truncated slits/orders. Instrument-specific Updates --------------------------- +- Add support for the R4K detector for MDM OSMOS - Updated archival sensitivity functions for DEIMOS 1200G, 800G, and 600ZD gratings. - Keck/KCWI and Keck/KCRM: Turned on polynomial correction for sky subtraction. - We now support the reduction of VLT/FORS2 data taken in MOS mode. @@ -78,14 +81,20 @@ Script Changes - Column ``SpatID`` in the output of ``pypeit_chk_wavecalib`` changed to ``SpatOrderID`` and now show the echelle order number, if applicable, otherwise the slit number. - ``pypeit_chk_edges`` now load SlitTraceSet (if available) to be able to overlay the echelle order numbers. +- Added a new GUI for creating and editing PypeIt input files: ``pypeit_setup_gui`` - Added a -G option to ``pypeit_setup`` and ``pypeit_obslog`` that will start the new - Setup GUI. + Setup GUI. +- Improvements and bug fixes for how the mask is displayed by + ``pypeit_show_2dspec``, and added ``pypeit_print_bpm`` to allow for a quick + report on the flags associated with a given bit value. Datamodel Changes ----------------- - A wavelength array is now stored for DataCube() -- Wavecalib and Wavefit datacontainers now store information about echelle order number, if applicable. +- WaveCalib and WaveFit datacontainers now store information about echelle order + number, if applicable. +- Change to how SlitTraceSet datamodel stores and checks bit flag values. Under-the-hood Improvements --------------------------- @@ -96,6 +105,10 @@ Under-the-hood Improvements - Improvements to echelle wavelength calibration. Code changes in: ``pypeit/core/wavecal/wvutils.py``, ``pypeit/core/wavecal/echelle.py``, ``pypeit/core/wavecal/autoid.py``, ``pypeit/wavecalib.py``. +- Fixed some failure modes when matching measured order positions to expected + positions for fixed-format echelle spectrographs. +- More extensive propagation of turning off datamodel version checking (using + ``try_old`` and ``chk_version``) throughout the code. Bug Fixes --------- diff --git a/doc/spectrographs/soar_goodman.rst b/doc/spectrographs/soar_goodman.rst index d8f6f34726..c7a95de3b3 100644 --- a/doc/spectrographs/soar_goodman.rst +++ b/doc/spectrographs/soar_goodman.rst @@ -6,13 +6,16 @@ Overview ======== This file summarizes several instrument specific -items for the SOAR/Goodman spectrograph. Note that -there are two ways to download your data from the +items for the SOAR/Goodman spectrograph. + +Note that there are two ways to download your data from the archive: -(1) in .fits format; and -(2) in .fits.fz format. +(1) in .fits format -- these have been pre-processed and are not PypeIt compliant +(2) in .fits.fz format -- these are the raw frames from the telescope PypeIt will only work with the .fz format, so please ensure that you are only using data in this format. +We recommend downloading the data from here: +https://archive.lco.global/ \ No newline at end of file diff --git a/doc/tutorials/lris_howto.rst b/doc/tutorials/lris_howto.rst index 3bb07e011c..61e46535f9 100644 --- a/doc/tutorials/lris_howto.rst +++ b/doc/tutorials/lris_howto.rst @@ -287,13 +287,13 @@ and it prints on screen the following (here truncated) table: .. code-block:: bash - N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) mesured_fwhm RMS - --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------ ----- - 0 212 6835.4 8469.9 10121.3 1.610 43 6931.379 - 9925.919 91.1 2.7 0.027 - 1 265 6038.3 7674.1 9327.6 1.612 45 6336.179 - 9227.030 87.9 2.7 0.025 - 2 335 7169.5 8831.4 10485.5 1.611 39 7427.339 - 10472.923 91.8 2.7 0.348 - 3 384 0.0 0.0 0.0 0.000 0 0.000 - 0.000 0.0 0.0 0.000 - 4 474 0.0 0.0 0.0 0.000 0 0.000 - 0.000 0.0 0.0 0.000 + N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) measured_fwhm RMS + --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------- ----- + 0 212 6835.4 8469.9 10121.3 1.610 43 6931.379 - 9925.919 91.1 2.7 0.027 + 1 265 6038.3 7674.1 9327.6 1.612 45 6336.179 - 9227.030 87.9 2.7 0.025 + 2 335 7169.5 8831.4 10485.5 1.611 39 7427.339 - 10472.923 91.8 2.7 0.348 + 3 384 0.0 0.0 0.0 0.000 0 0.000 - 0.000 0.0 0.0 0.000 + 4 474 0.0 0.0 0.0 0.000 0 0.000 - 0.000 0.0 0.0 0.000 See :ref:`pypeit-chk-wavecalib` for a detailed description of all the columns. Note that the slits with ``SpatID`` 384 and 474 have all the values set to 0.0. This is because diff --git a/doc/tutorials/mosfire_howto.rst b/doc/tutorials/mosfire_howto.rst index afb605aed2..3843f7afee 100644 --- a/doc/tutorials/mosfire_howto.rst +++ b/doc/tutorials/mosfire_howto.rst @@ -348,16 +348,16 @@ and it prints on screen the following: .. code-block:: bash - N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) mesured_fwhm RMS - --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------ ----- - 0 306 20252.3 22473.7 24689.0 2.167 32 20275.839 - 23914.989 82.0 2.7 0.041 - 1 656 19145.6 21364.7 23574.6 2.164 49 19193.537 - 22741.961 80.1 2.6 0.048 - 2 766 19963.1 22181.5 24392.9 2.163 33 20007.951 - 23914.989 88.2 2.6 0.028 - 3 877 19232.6 21450.8 23659.5 2.163 48 19250.306 - 22741.961 78.9 2.5 0.050 - 4 1010 19213.0 21430.9 23641.0 2.163 48 19250.306 - 22741.961 78.9 2.6 0.047 - 5 1297 20173.6 22392.2 24603.7 2.164 32 20193.227 - 23914.989 84.0 2.6 0.036 - 6 1653 19244.3 21465.0 23678.2 2.166 49 19250.306 - 22741.961 78.7 2.7 0.060 - 7 1920 19251.7 21475.0 23689.8 2.169 47 19350.119 - 22741.961 76.4 2.7 0.042 + N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) measured_fwhm RMS + --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------- ----- + 0 306 20252.3 22473.7 24689.0 2.167 32 20275.839 - 23914.989 82.0 2.7 0.041 + 1 656 19145.6 21364.7 23574.6 2.164 49 19193.537 - 22741.961 80.1 2.6 0.048 + 2 766 19963.1 22181.5 24392.9 2.163 33 20007.951 - 23914.989 88.2 2.6 0.028 + 3 877 19232.6 21450.8 23659.5 2.163 48 19250.306 - 22741.961 78.9 2.5 0.050 + 4 1010 19213.0 21430.9 23641.0 2.163 48 19250.306 - 22741.961 78.9 2.6 0.047 + 5 1297 20173.6 22392.2 24603.7 2.164 32 20193.227 - 23914.989 84.0 2.6 0.036 + 6 1653 19244.3 21465.0 23678.2 2.166 49 19250.306 - 22741.961 78.7 2.7 0.060 + 7 1920 19251.7 21475.0 23689.8 2.169 47 19350.119 - 22741.961 76.4 2.7 0.042 See :ref:`pypeit-chk-wavecalib` for a detailed description of all the columns. diff --git a/doc/tutorials/nires_howto.rst b/doc/tutorials/nires_howto.rst index f17a10677d..6fd9923892 100644 --- a/doc/tutorials/nires_howto.rst +++ b/doc/tutorials/nires_howto.rst @@ -236,13 +236,13 @@ terminal to see the full output): .. code-block:: bash - N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) mesured_fwhm RMS - --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------ ----- - 0 234 8131.7 9408.9 10646.5 1.225 30 9793.676 - 10527.657 29.2 2.1 0.085 - 1 416 9496.8 10961.4 12401.6 1.419 91 9793.676 - 12351.597 88.1 2.1 0.078 - 2 574 11380.8 13133.2 14860.1 1.699 95 11439.783 - 14833.093 97.5 2.1 0.129 - 3 720 14203.1 16389.5 18547.8 2.122 106 14227.201 - 18526.181 98.9 2.2 0.106 - 4 885 18895.7 21806.9 24686.8 2.827 84 18914.824 - 24627.748 98.6 2.0 0.165 + N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) measured_fwhm RMS + --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------- ----- + 0 234 8131.7 9408.9 10646.5 1.225 30 9793.676 - 10527.657 29.2 2.1 0.085 + 1 416 9496.8 10961.4 12401.6 1.419 91 9793.676 - 12351.597 88.1 2.1 0.078 + 2 574 11380.8 13133.2 14860.1 1.699 95 11439.783 - 14833.093 97.5 2.1 0.129 + 3 720 14203.1 16389.5 18547.8 2.122 106 14227.201 - 18526.181 98.9 2.2 0.106 + 4 885 18895.7 21806.9 24686.8 2.827 84 18914.824 - 24627.748 98.6 2.0 0.165 See :ref:`pypeit-chk-wavecalib` for a detailed description of all the columns. diff --git a/pypeit/alignframe.py b/pypeit/alignframe.py index 598a27855c..9a28491d21 100644 --- a/pypeit/alignframe.py +++ b/pypeit/alignframe.py @@ -143,6 +143,10 @@ def __init__(self, rawalignimg, slits, spectrograph, alignpar, det=1, qa_path=No # Attributes unique to this object self._alignprof = None + # Create a bad pixel mask + self.slit_bpm = self.slits.bitmask.flagged(self.slits.mask, + and_not=self.slits.bitmask.exclude_for_reducing) + # Completed steps self.steps = [] @@ -180,8 +184,14 @@ def build_traces(self, show_peaks=False, debug=False): slitid_img_init = self.slits.slit_img(initial=True) left, right, _ = self.slits.select_edges(initial=True) align_prof = dict({}) + # Go through the slits for slit_idx, slit_spat in enumerate(self.slits.spat_id): + if self.slit_bpm[slit_idx]: + msgs.info(f'Skipping bad slit/order {self.slits.slitord_id[slit_idx]} ({slit_idx+1}/{self.slits.nslits})') + self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADALIGNCALIB') + continue + specobj_dict = {'SLITID': slit_idx, 'DET': self.rawalignimg.detector.name, 'OBJTYPE': "align_profile", 'PYPELINE': self.spectrograph.pypeline} msgs.info("Fitting alignment traces in slit {0:d}/{1:d}".format(slit_idx+1, self.slits.nslits)) @@ -221,12 +231,19 @@ def generate_traces(self, align_prof): nbars = len(self.alignpar['locations']) # Generate an array containing the centroid of all bars align_traces = np.zeros((self.nspec, nbars, self.nslits)) - for sl in range(self.nslits): - sls = '{0:d}'.format(sl) + + + # Go through the slits + for slit_idx, slit_spat in enumerate(self.slits.spat_id): + if self.slit_bpm[slit_idx]: + msgs.info(f'Skipping bad slit/order {self.slits.slitord_id[slit_idx]} ({slit_idx+1}/{self.slits.nslits})') + self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADALIGNCALIB') + continue + sls = '{0:d}'.format(slit_idx) for bar in range(nbars): - if align_prof[sls][bar].SLITID != sl: + if align_prof[sls][bar].SLITID != slit_idx: msgs.error("Alignment profiling failed to generate traces") - align_traces[:, bar, sl] = align_prof[sls][bar].TRACE_SPAT + align_traces[:, bar, slit_idx] = align_prof[sls][bar].TRACE_SPAT return align_traces def run(self, show=False): diff --git a/pypeit/archive.py b/pypeit/archive.py old mode 100755 new mode 100644 diff --git a/pypeit/bitmask.py b/pypeit/bitmask.py index caa92c96d5..fd35e61c0f 100644 --- a/pypeit/bitmask.py +++ b/pypeit/bitmask.py @@ -216,11 +216,12 @@ def minimum_dtype(self, asuint=False): return numpy.uint32 if asuint else numpy.int32 return numpy.uint64 if asuint else numpy.int64 - def flagged(self, value, flag=None): + def flagged(self, value, flag=None, exclude=None, and_not=None): """ - Determine if a bit is on in the provided bitmask value. The - function can be used to determine if any individual bit is on or - any one of many bits is on. + + Determine if a bit is on in the provided integer(s). The function can + be used to determine if any individual bit is on or any one of many bits + is on. Args: value (int, array-like): @@ -229,28 +230,116 @@ def flagged(self, value, flag=None): flag (str, array-like, optional): One or more bit names to check. If None, then it checks if *any* bit is on. + exclude (str, array-like, optional): + One or more bit names to *exclude* from the check. If None, all + flags are included in the check. + and_not (str, array-like, optional): + One or more bit names to ensure are *not* selected by the check. + I.e., if this bit is flagged the returned value is false, even + if other selected bits *are* flagged. See examples. If None, + functionality ignored. Returns: - bool: Boolean flags that the provided flags (or any flag) is - on for the provided bitmask value. Shape is the same as - `value`. + bool, `numpy.ndarray`_: Boolean flags that the provided flags (or + any flag) is on for the provided bitmask value. If a numpy array, + the shape is the same as ``value``. Raises: KeyError: Raised by the dict data type if the input *flag* is not one of the valid bitmask names. TypeError: Raised if the provided *flag* does not contain one or more strings. + + Example: + + Let the BitMask object have two flags, 'A' and 'B', and a bit + array ``v``, such that: + + >>> import numpy + >>> from pypeit.bitmask import BitMask + >>> bm = BitMask(['A', 'B']) + >>> v = numpy.arange(4).astype(numpy.int16) + >>> v + array([0, 1, 2, 3], dtype=int16) + >>> print([bm.flagged_bits(_v) for _v in v]) + [[], ['A'], ['B'], ['A', 'B']] + + This function will return a boolean array that indicates where flags + are turned on in each bit value. + + - To find if a specific bit is on: + + >>> bm.flagged(v, flag='A') + array([False, True, False, True]) + + - To find if *any* bit is on: + + >>> bm.flagged(v) + array([False, True, True, True]) + + This is identical to: + + >>> bm.flagged(v, flag=['A', 'B']) + array([False, True, True, True]) + + or a logical-or combination of all the bits: + + >>> bm.flagged(v, flag='A') | bm.flagged(v, flag='B') + array([False, True, True, True]) + + - To find if any bit is on except for a given subset, use the + ``exclude`` keyword: + + >>> bm.flagged(v, exclude='A') + array([False, False, True, True]) + + This is identical to: + + >>> bm.flagged(v, flag='B') + array([False, False, True, True]) + + Obviously, this is more useful when there are many flags and + you're only trying to exclude a few. + + - To find if a set of bits are on *and* a different set of bits are + *not* on, use the ``and_not`` keyword: + + >>> bm.flagged(v, and_not='B') + array([False, True, False, False]) + + This is equivalent to: + + >>> bm.flagged(v, flag=['A', 'B'], and_not='B') + array([False, True, False, False]) + + and: + + >>> bm.flagged(v) & numpy.logical_not(bm.flagged(v, flag='B')) + array([False, True, False, False]) + + Note that the returned values are False if the bit indicates that + flag 'B' is on, even though flag 'B' was in the list provided to + the ``flag`` keyword; i.e., the use of ``and_not`` supersedes any + elements of ``flag``. + + - Currently there is no functionality that performs a logical-and + combination of flags. """ _flag = self._prep_flags(flag) + if exclude is not None: + # Remove the bits to exclude + _exclude = numpy.atleast_1d(exclude) + if not numpy.all(numpy.isin(_exclude, self.keys())): + raise ValueError(f'Not all exclude flags are valid: {exclude}') + _flag = numpy.setdiff1d(_flag, _exclude) - out = value & (1 << self.bits[_flag[0]]) != 0 - if len(_flag) == 1: - return out + # Bits to expunge + _and_not = None if and_not is None else self.flagged(value, and_not) - nn = len(_flag) - for i in range(1,nn): + out = value & (1 << self.bits[_flag[0]]) != 0 + for i in range(1,len(_flag)): out |= (value & (1 << self.bits[_flag[i]]) != 0) - return out + return out if _and_not is None else out & numpy.logical_not(_and_not) def flagged_bits(self, value): """ @@ -532,5 +621,30 @@ def parse_bits_from_hdr(hdr, prefix): values += [i] descr += [hdr.comments[k]] return keys, values, descr + + def correct_flag_order(self, flags): + """ + Check if the provided flags are in the correct order compared to the + current definition of the object. + + Args: + flags (:obj:`list`): + A list of strings that *must* be in the order of the bit + numbers. I.e., bit 0 uses the string in the first element of + the list, bit 1 uses the second element, etc. The number of + flags does not need to exactly match the current set of flags. + It can be longer or shorter, so long as it begins with the first + flag. + + Returns: + :obj:`bool`: Indicates if the provides flags are in the correct order. + """ + cls_flags = list(self.keys()) + for i, flag in enumerate(flags): + if i >= self.nbits: + break + if cls_flags[i] != flag: + return False + return True diff --git a/pypeit/calibframe.py b/pypeit/calibframe.py index 30826d8683..a626a4482d 100644 --- a/pypeit/calibframe.py +++ b/pypeit/calibframe.py @@ -169,15 +169,10 @@ def from_hdu(cls, hdu, chk_version=True, **kwargs): **kwargs: Passed directly to :func:`~pypeit.datamodel.DataContainer._parse`. """ + # Parse d, dm_version_passed, dm_type_passed, parsed_hdus = cls._parse(hdu, **kwargs) - # Check version and type? - if not dm_type_passed: - msgs.error(f'The HDU(s) cannot be parsed by a {cls.__name__} object!') - if not dm_version_passed: - _f = msgs.error if chk_version else msgs.warn - _f(f'Current version of {cls.__name__} object in code ({cls.version}) ' - 'does not match version used to write your HDU(s)!') - + # Check + cls._check_parsed(dm_version_passed, dm_type_passed, chk_version=chk_version) # Instantiate self = cls.from_dict(d=d) @@ -207,7 +202,6 @@ def calib_keys_from_header(self, hdr): self.calib_id = self.ingest_calib_id(hdr['CALIBID']) else: msgs.warn('Header does not have CALIBID card; cannot parse calibration IDs.') - return self @staticmethod def parse_key_dir(inp, from_filename=False): diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index cdcc3318d5..07f15a32cf 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -64,6 +64,11 @@ class Calibrations: user_slits (:obj:`dict`, optional): A limited set of slits selected by the user for analysis. See :func:`~pypeit.slittrace.SlitTraceSet.user_mask`. + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict + version checking to ensure a valid file. If False, the code will + try to keep going, but this may lead to faults and quiet failures. + User beware! Attributes: fitstbl (:class:`~pypeit.metadata.PypeItMetaData`): @@ -134,7 +139,7 @@ def get_instance(fitstbl, par, spectrograph, caldir, **kwargs): return calibclass(fitstbl, par, spectrograph, caldir, **kwargs) def __init__(self, fitstbl, par, spectrograph, caldir, qadir=None, - reuse_calibs=False, show=False, user_slits=None): + reuse_calibs=False, show=False, user_slits=None, chk_version=True): # Check the types # TODO -- Remove this None option once we have data models for all the Calibrations @@ -153,6 +158,7 @@ def __init__(self, fitstbl, par, spectrograph, caldir, qadir=None, # Calibrations self.reuse_calibs = reuse_calibs + self.chk_version = chk_version self.calib_dir = Path(caldir).resolve() if not self.calib_dir.exists(): self.calib_dir.mkdir(parents=True) @@ -314,7 +320,7 @@ def get_arc(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - self.msarc = frame['class'].from_file(cal_file) + self.msarc = frame['class'].from_file(cal_file, chk_version=self.chk_version) return self.msarc # Reset the BPM @@ -357,7 +363,7 @@ def get_tiltimg(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - self.mstilt = frame['class'].from_file(cal_file) + self.mstilt = frame['class'].from_file(cal_file, chk_version=self.chk_version) return self.mstilt # Reset the BPM @@ -405,7 +411,7 @@ def get_align(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - self.alignments = frame['class'].from_file(cal_file) + self.alignments = frame['class'].from_file(cal_file, chk_version=self.chk_version) self.alignments.is_synced(self.slits) return self.alignments @@ -456,7 +462,7 @@ def get_bias(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - self.msbias = frame['class'].from_file(cal_file) + self.msbias = frame['class'].from_file(cal_file, chk_version=self.chk_version) return self.msbias # Otherwise, create the processed file. @@ -495,7 +501,7 @@ def get_dark(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - self.msdark = frame['class'].from_file(cal_file) + self.msdark = frame['class'].from_file(cal_file, chk_version=self.chk_version) return self.msdark # TODO: If a bias has been constructed and it will be subtracted from @@ -571,7 +577,7 @@ def get_scattlight(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - self.msscattlight = frame['class'].from_file(cal_file) + self.msscattlight = frame['class'].from_file(cal_file, chk_version=self.chk_version) return self.msscattlight # Scattered light model does not exist or we're not reusing it. @@ -689,7 +695,8 @@ def get_flats(self): setup = illum_setup if pixel_setup is None else pixel_setup calib_id = illum_calib_id if pixel_calib_id is None else pixel_calib_id if cal_file.exists() and self.reuse_calibs: - self.flatimages = flatfield.FlatImages.from_file(cal_file) + self.flatimages = flatfield.FlatImages.from_file(cal_file, + chk_version=self.chk_version) self.flatimages.is_synced(self.slits) # Load user defined files if self.par['flatfield']['pixelflat_file'] is not None: @@ -845,7 +852,7 @@ def get_slits(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - self.slits = frame['class'].from_file(cal_file) + self.slits = frame['class'].from_file(cal_file, chk_version=self.chk_version) self.slits.mask = self.slits.mask_init.copy() if self.user_slits is not None: self.slits.user_mask(detname, self.user_slits) @@ -858,7 +865,8 @@ def get_slits(self): # If so, reuse it? if edges_file.exists() and self.reuse_calibs: # Yep! Load it and parse it into slits. - self.slits = edgetrace.EdgeTraceSet.from_file(edges_file).get_slits() + self.slits = edgetrace.EdgeTraceSet.from_file(edges_file, + chk_version=self.chk_version).get_slits() # Write the slits calibration file self.slits.to_file() if self.user_slits is not None: @@ -961,7 +969,7 @@ def get_wv_calib(self): # we want to reuse it, do so (or just load it): if cal_file.exists() and self.reuse_calibs: # Load the file - self.wv_calib = wavecalib.WaveCalib.from_file(cal_file) + self.wv_calib = wavecalib.WaveCalib.from_file(cal_file, chk_version=self.chk_version) self.wv_calib.chk_synced(self.slits) self.slits.mask_wvcalib(self.wv_calib) # Return @@ -1033,7 +1041,7 @@ def get_tilts(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - self.wavetilts = wavetilts.WaveTilts.from_file(cal_file) + self.wavetilts = wavetilts.WaveTilts.from_file(cal_file, chk_version=self.chk_version) self.wavetilts.is_synced(self.slits) self.slits.mask_wavetilts(self.wavetilts) return self.wavetilts diff --git a/pypeit/coadd1d.py b/pypeit/coadd1d.py index 34b03974a5..cd1f7c1944 100644 --- a/pypeit/coadd1d.py +++ b/pypeit/coadd1d.py @@ -16,6 +16,7 @@ from pypeit.spectrographs.util import load_spectrograph from pypeit.onespec import OneSpec +from pypeit import utils from pypeit import sensfunc from pypeit import specobjs from pypeit import msgs @@ -23,22 +24,23 @@ from pypeit.history import History + class CoAdd1D: @classmethod - def get_instance(cls, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, - debug=False, show=False): + def get_instance(cls, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, + setup_id=None, debug=False, show=False, chk_version=True): """ - Superclass factory method which generates the subclass instance. See :class:`CoAdd1D` instantiation for - argument descriptions. + Superclass factory method which generates the subclass instance. See + :class:`CoAdd1D` instantiation for argument descriptions. """ pypeline = fits.getheader(spec1dfiles[0])['PYPELINE'] + 'CoAdd1D' - return next(c for c in cls.__subclasses__() if c.__name__ == pypeline)( - spec1dfiles, objids, spectrograph=spectrograph, par=par, sensfuncfile=sensfuncfile, setup_id=setup_id, - debug=debug, show=show) + return next(c for c in utils.all_subclasses(CoAdd1D) if c.__name__ == pypeline)( + spec1dfiles, objids, spectrograph=spectrograph, par=par, sensfuncfile=sensfuncfile, + setup_id=setup_id, debug=debug, show=show, chk_version=chk_version) - def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, - debug=False, show=False): + def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, + setup_id=None, debug=False, show=False, chk_version=True): """ Args: @@ -64,6 +66,11 @@ def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfil Debug. Default = False show (bool, optional): Debug. Default = True + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict + version checking to ensure a valid file. If False, the code + will try to keep going, but this may lead to faults and quiet + failures. User beware! """ # Instantiate attributes self.spec1dfiles = spec1dfiles @@ -82,6 +89,7 @@ def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfil # self.debug = debug self.show = show + self.chk_version = chk_version self.nexp = len(self.spec1dfiles) # Number of exposures self.coaddfile = None self.gpm_exp = np.ones(self.nexp, dtype=bool).tolist() # list of bool indicating the exposures that have been coadded @@ -123,11 +131,11 @@ def save(self, coaddfile, telluric=None, obj_model=None, overwrite=True): Overwrite existing file? """ self.coaddfile = coaddfile - wave_gpm = self.wave_coadd > 1.0 # Generate the spectrum container object - onespec = OneSpec(wave=self.wave_coadd[wave_gpm], wave_grid_mid=self.wave_grid_mid[wave_gpm], flux=self.flux_coadd[wave_gpm], - PYP_SPEC=self.spectrograph.name, ivar=self.ivar_coadd[wave_gpm], - mask=self.gpm_coadd[wave_gpm].astype(int), + onespec = OneSpec(wave=self.wave_coadd, wave_grid_mid=self.wave_grid_mid, flux=self.flux_coadd, + PYP_SPEC=self.spectrograph.name, ivar=self.ivar_coadd, + sigma = np.sqrt(utils.inverse(self.ivar_coadd)), + mask=self.gpm_coadd.astype(int), ext_mode=self.par['ex_value'], fluxed=self.par['flux_value']) # TODO This is a hack, not sure how to merge the headers at present @@ -139,9 +147,9 @@ def save(self, coaddfile, telluric=None, obj_model=None, overwrite=True): # Add on others if telluric is not None: - onespec.telluric = telluric[wave_gpm] + onespec.telluric = telluric if obj_model is not None: - onespec.obj_model = obj_model[wave_gpm] + onespec.obj_model = obj_model # Write onespec.to_file(coaddfile, history=history, overwrite=overwrite) @@ -157,14 +165,6 @@ class MultiSlitCoAdd1D(CoAdd1D): Child of CoAdd1d for Multislit and Longslit reductions. """ - def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, debug=False, show=False): - """ - See :class:`CoAdd1D` instantiation for argument descriptions. - """ - super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par = par, sensfuncfile = sensfuncfile, - setup_id=setup_id, debug = debug, show = show) - - def load(self): """ Load the arrays we need for performing coadds. @@ -188,7 +188,8 @@ def load(self): """ waves, fluxes, ivars, gpms, headers = [], [], [], [], [] for iexp in range(self.nexp): - sobjs = specobjs.SpecObjs.from_fitsfile(self.spec1dfiles[iexp], chk_version=self.par['chk_version']) + sobjs = specobjs.SpecObjs.from_fitsfile(self.spec1dfiles[iexp], + chk_version=self.chk_version) indx = sobjs.name_indices(self.objids[iexp]) if not np.any(indx): msgs.error( @@ -197,7 +198,7 @@ def load(self): msgs.error("Error in spec1d file for exposure {:d}: " "More than one object was identified with the OBJID={:s} in file={:s}".format( iexp, self.objids[iexp], self.spec1dfiles[iexp])) - wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, trace_spec, trace_spat, meta_spec, header = \ + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, _, _, _, header = \ sobjs[indx].unpack_object(ret_flam=self.par['flux_value'], extract_type=self.par['ex_value']) waves.append(wave_iexp) fluxes.append(flux_iexp) @@ -267,8 +268,8 @@ def check_exposures(self): # check if there is any bad exposure by comparing the rms_sn with the median rms_sn among all exposures if len(_fluxes) > 2: - # Evaluate the sn_weights. - rms_sn, weights = coadd.sn_weights(_fluxes, _ivars, _gpms, const_weights=True) + # Evaluate the rms_sn + rms_sn, _ = coadd.calc_snr(_fluxes, _ivars, _gpms) # some stats mean, med, sigma = stats.sigma_clipped_stats(rms_sn, sigma_lower=2., sigma_upper=2.) _sigrej = self.par['sigrej_exp'] if self.par['sigrej_exp'] is not None else 10.0 @@ -320,9 +321,9 @@ def coadd(self): spec_samp_fact=self.par['spec_samp_fact'], ref_percentile=self.par['ref_percentile'], maxiter_scale=self.par['maxiter_scale'], sigrej_scale=self.par['sigrej_scale'], scale_method=self.par['scale_method'], sn_min_medscale=self.par['sn_min_medscale'], - sn_min_polyscale=self.par['sn_min_polyscale'], maxiter_reject=self.par['maxiter_reject'], - lower=self.par['lower'], upper=self.par['upper'], maxrej=self.par['maxrej'], sn_clip=self.par['sn_clip'], - debug=self.debug, show=self.show) + sn_min_polyscale=self.par['sn_min_polyscale'], weight_method = self.par['weight_method'], + maxiter_reject=self.par['maxiter_reject'], lower=self.par['lower'], upper=self.par['upper'], + maxrej=self.par['maxrej'], sn_clip=self.par['sn_clip'], debug=self.debug, show=self.show) @@ -331,15 +332,14 @@ class EchelleCoAdd1D(CoAdd1D): Child of CoAdd1d for Echelle reductions. """ - def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, - debug=False, show=False): + def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, + setup_id=None, debug=False, show=False, chk_version=True): """ See :class:`CoAdd1D` instantiation for argument descriptions. - - """ - super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par = par, sensfuncfile = sensfuncfile, - setup_id=setup_id, debug = debug, show = show) + super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par=par, + sensfuncfile=sensfuncfile, setup_id=setup_id, debug=debug, show=show, + chk_version=chk_version) if sensfuncfile is None: msgs.error('sensfuncfile is a required argument for echelle coadding') @@ -394,7 +394,6 @@ def coadd(self): = coadd.ech_combspec(self.waves, self.fluxes, self.ivars, self.gpms, self.weights_sens, setup_ids=self.unique_setups, nbests=self.par['nbests'], - sn_smooth_npix=self.par['sn_smooth_npix'], wave_method=self.par['wave_method'], dv=self.par['dv'], dwave=self.par['dwave'], dloglam=self.par['dloglam'], wave_grid_min=self.par['wave_grid_min'], @@ -434,13 +433,20 @@ def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): """ nexp = len(spec1dfiles) for iexp in range(nexp): - sobjs = specobjs.SpecObjs.from_fitsfile(spec1dfiles[iexp], chk_version=self.par['chk_version']) + sobjs = specobjs.SpecObjs.from_fitsfile(spec1dfiles[iexp], chk_version=self.chk_version) indx = sobjs.name_indices(objids[iexp]) if not np.any(indx): msgs.error("No matching objects for {:s}. Odds are you input the wrong OBJID".format(objids[iexp])) - wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, trace_spec, trace_spat, meta_spec, header = \ + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, _, _, _, header = \ sobjs[indx].unpack_object(ret_flam=self.par['flux_value'], extract_type=self.par['ex_value']) - weights_sens_iexp = sensfunc.SensFunc.sensfunc_weights(sensfuncfiles[iexp], wave_iexp, debug=self.debug) + # This np.atleast2d hack deals with the situation where we are wave_iexp is actually Multislit data, i.e. we are treating + # it like an echelle spectrograph with a single order. This usage case arises when we want to use the + # echelle coadding code to combine echelle and multislit data + if wave_iexp.ndim == 1: + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp = np.atleast_2d(wave_iexp).T, np.atleast_2d(flux_iexp).T, np.atleast_2d(ivar_iexp).T, np.atleast_2d(gpm_iexp).T + weights_sens_iexp = sensfunc.SensFunc.sensfunc_weights(sensfuncfiles[iexp], wave_iexp, + debug=self.debug, + chk_version=self.chk_version) # Allocate arrays on first iteration # TODO :: We should refactor to use a list of numpy arrays, instead of a 2D numpy array. if iexp == 0: @@ -457,6 +463,8 @@ def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): # Store the information waves[...,iexp], fluxes[...,iexp], ivars[..., iexp], gpms[...,iexp], weights_sens[...,iexp] \ = wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, weights_sens_iexp + + return waves, fluxes, ivars, gpms, weights_sens, header_out @@ -509,3 +517,14 @@ def load(self): return waves, fluxes, ivars, gpms, weights_sens, headers +class SlicerIFUCoAdd1D(MultiSlitCoAdd1D): + """ + Child of MultiSlitCoAdd1d for SlicerIFU reductions. + """ + + def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, debug=False, show=False): + """ + See :class:`CoAdd1D` instantiation for argument descriptions. + """ + super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par = par, sensfuncfile = sensfuncfile, + setup_id=setup_id, debug = debug, show = show) diff --git a/pypeit/coadd2d.py b/pypeit/coadd2d.py index 16e19ed909..871e33e09b 100644 --- a/pypeit/coadd2d.py +++ b/pypeit/coadd2d.py @@ -420,13 +420,14 @@ def good_slitindx(self, only_slits=None, exclude_slits=None): # This creates a unified bpm common to all frames slits0 = self.stack_dict['slits_list'][0] # bpm for the first frame - reduce_bpm = (slits0.mask > 0) & (np.invert(slits0.bitmask.flagged(slits0.mask, - flag=slits0.bitmask.exclude_for_reducing))) + + reduce_bpm = slits0.bitmask.flagged(slits0.mask, + and_not=slits0.bitmask.exclude_for_reducing) for i in range(1, self.nexp): # update bpm with the info from the other frames slits = self.stack_dict['slits_list'][i] - reduce_bpm |= (slits.mask > 0) & (np.invert(slits.bitmask.flagged(slits.mask, - flag=slits.bitmask.exclude_for_reducing))) + reduce_bpm |= slits.bitmask.flagged(slits.mask, + and_not=slits.bitmask.exclude_for_reducing) # these are the good slit index according to the bpm mask good_slitindx = np.where(np.logical_not(reduce_bpm))[0] @@ -465,7 +466,7 @@ def good_slitindx(self, only_slits=None, exclude_slits=None): # these are the good slit index excluding the slits that are selected by the user return np.delete(good_slitindx, exclude_slitindx) - def optimal_weights(self, slitorderid, objid, const_weights=False): + def optimal_weights(self, slitorderid, objid, weight_method='auto'): """ Determine optimal weights for 2d coadds. This script grabs the information from SpecObjs list for the object with specified slitid and objid and passes to coadd.sn_weights to determine the optimal weights for @@ -474,23 +475,54 @@ def optimal_weights(self, slitorderid, objid, const_weights=False): Parameters ---------- slitorderid : :obj:`int` - The slit or order id that has the brightest object whose - S/N will be used to determine the weight for each frame. + The slit or order id that has the brightest object whose + S/N will be used to determine the weight for each frame. objid : `numpy.ndarray`_ - Array of object indices with shape = (nexp,) of the - brightest object whose S/N will be used to determine the - weight for each frame. - const_weights : :obj:`bool` - Use constant weights for coadding the exposures. - Default=False + Array of object indices with shape = (nexp,) of the + brightest object whose S/N will be used to determine the + weight for each frame. + weight_method : `str`, optional + Weight method to be used in :func:`~pypeit.coadd.sn_weights`. + Options are ``'auto'``, ``'constant'``, ``'relative'``, or + ``'ivar'``. The default is ``'auto'``. Behavior is as follows: + + - ``'auto'``: Use constant weights if rms_sn < 3.0, otherwise + use wavelength dependent. + + - ``'constant'``: Constant weights based on rms_sn**2 + + - ``'uniform'``: Uniform weighting. + + - ``'wave_dependent'``: Wavelength dependent weights will be + used irrespective of the rms_sn ratio. This option will not + work well at low S/N ratio although it is useful for objects + where only a small fraction of the spectral coverage has high + S/N ratio (like high-z quasars). + + - ``'relative'``: Calculate weights by fitting to the ratio of + spectra? Note, relative weighting will only work well when + there is at least one spectrum with a reasonable S/N, and a + continuum. RJC note - This argument may only be better when + the object being used has a strong continuum + emission lines. + The reference spectrum is assigned a value of 1 for all + wavelengths, and the weights of all other spectra will be + determined relative to the reference spectrum. This is + particularly useful if you are dealing with highly variable + spectra (e.g. emission lines) and require a precision better + than ~1 per cent. + + - ``'ivar'``: Use inverse variance weighting. This is not well + tested and should probably be deprecated. Returns ------- - rms_sn : ndarray, shape = (len(specobjs_list),) - Root mean square S/N value for each input spectra - weights : ndarray, shape (len(specobjs_list),) - Weights to be applied to the spectra. These are - signal-to-noise squared weights. + rms_sn : `numpy.ndarray`_ + Array of root-mean-square S/N value for each input spectra. Shape = (nexp,) + weights : list + List of len(nexp) containing the signal-to-noise squared weights to be + applied to the spectra. This output is aligned with the vector (or + vectors) provided in waves which is read in by this routine, i.e. it is a + list of arrays of type `numpy.ndarray`_ with the same shape as those in waves. """ nexp = len(self.stack_dict['specobjs_list']) @@ -523,7 +555,7 @@ def optimal_weights(self, slitorderid, objid, const_weights=False): f'flux not available in slit/order = {slitorderid}') # TODO For now just use the zero as the reference for the wavelengths? Perhaps we should be rebinning the data though? - rms_sn, weights = coadd.sn_weights(fluxes, ivars, gpms, self.sn_smooth_npix, const_weights=const_weights) + rms_sn, weights = coadd.sn_weights(fluxes, ivars, gpms, sn_smooth_npix=self.sn_smooth_npix, weight_method=weight_method) return rms_sn, weights def coadd(self, interp_dspat=True): @@ -778,8 +810,8 @@ def reduce(self, pseudo_dict, show=False, clear_ginga=True, show_peaks=False, sh # Make changes to parset specific to 2d coadds parcopy = copy.deepcopy(self.par) - parcopy['reduce']['findobj']['trace_npoly'] = 3 # Low order traces since we are rectified - + # Enforce low order traces since we are rectified + parcopy['reduce']['findobj']['trace_npoly'] = int(np.clip(parcopy['reduce']['findobj']['trace_npoly'],None,3)) # Manual extraction. manual_obj = None if self.par['coadd2d']['manual'] is not None and len(self.par['coadd2d']['manual']) > 0: @@ -787,8 +819,6 @@ def reduce(self, pseudo_dict, show=False, clear_ginga=True, show_peaks=False, sh # Get bpm mask. There should not be any masked slits because we excluded those already # before the coadd, but we need to pass a bpm to FindObjects and Extract slits = pseudo_dict['slits'] - #pseudo_reduce_bpm = (slits.mask > 0) & (np.invert(slits.bitmask.flagged(slits.mask, - # flag=slits.bitmask.exclude_for_reducing))) # Initiate FindObjects object objFind = find_objects.FindObjects.get_instance(sciImage, pseudo_dict['slits'], self.spectrograph, parcopy, @@ -1397,7 +1427,7 @@ def compute_weights(self, weights): # adjustment for multislit to case 3) Bright object exists and parset `weights` is equal to 'auto' if (self.objid_bri is not None) and (weights == 'auto'): # compute weights using bright object - _, self.use_weights = self.optimal_weights(self.spatid_bri, self.objid_bri, const_weights=True) + _, self.use_weights = self.optimal_weights(self.spatid_bri, self.objid_bri, weight_method='constant') if self.par['coadd2d']['user_obj'] is not None: msgs.info(f'Weights computed using a unique reference object in slit={self.spatid_bri} provided by the user') else: @@ -1464,7 +1494,7 @@ def get_brightest_obj(self, specobjs_list, spat_ids): #remove_indx.append(iobj) # if there are objects on this slit left, we can proceed with computing rms_sn if len(fluxes) > 0: - rms_sn, weights = coadd.sn_weights(fluxes, ivars, gpms, const_weights=True) + rms_sn, _ = coadd.calc_snr(fluxes, ivars, gpms) imax = np.argmax(rms_sn) slit_snr_max[islit, iexp] = rms_sn[imax] objid_max[islit, iexp] = objid_this[imax] @@ -1748,8 +1778,8 @@ def get_brightest_obj(self, specobjs_list, nslits): f'object {sobjs[ind][0].ECH_OBJID} in order {sobjs[ind][0].ECH_ORDER}.') continue if flux is not None: - rms_sn, weights = coadd.sn_weights([flux], [ivar], [mask], const_weights=True) - order_snr[iord, iobj] = rms_sn + rms_sn, _ = coadd.calc_snr([flux], [ivar], [mask]) + order_snr[iord, iobj] = rms_sn[0] bpm[iord, iobj] = False # If there are orders that have bpm = True for some objs and not for others, set bpm = True for all objs diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 3119b8edd6..5174fc0207 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -169,17 +169,26 @@ def to_file(self, ofile, primary_hdr=None, hdr=None, **kwargs): super(DataCube, self).to_file(ofile, primary_hdr=primary_hdr, hdr=hdr, **kwargs) @classmethod - def from_file(cls, ifile): + def from_file(cls, ifile, verbose=True, chk_version=True, **kwargs): """ + Instantiate the object from an extension in the specified fits file. + Over-load :func:`~pypeit.datamodel.DataContainer.from_file` to deal with the header - + Args: - ifile (str): Filename holding the object + ifile (:obj:`str`, `Path`_): + Fits file with the data to read + verbose (:obj:`bool`, optional): + Print informational messages (not currently used) + chk_version (:obj:`bool`, optional): + Passed to :func:`from_hdu`. + kwargs (:obj:`dict`, optional): + Arguments passed directly to :func:`from_hdu`. """ with io.fits_open(ifile) as hdu: # Read using the base class - self = super().from_hdu(hdu) + self = cls.from_hdu(hdu, chk_version=chk_version, **kwargs) # Internals self.filename = ifile self.head0 = hdu[1].header # Actually use the first extension here, since it contains the WCS @@ -331,8 +340,9 @@ class CoAdd3D: """ # Superclass factory method generates the subclass instance @classmethod - def get_instance(cls, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, dec_offsets=None, - spectrograph=None, det=1, overwrite=False, show=False, debug=False): + def get_instance(cls, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, + dec_offsets=None, spectrograph=None, det=1, overwrite=False, show=False, + debug=False): """ Instantiate the subclass appropriate for the provided spectrograph. @@ -345,15 +355,15 @@ def get_instance(cls, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_o :class:`CoAdd3D`: One of the subclasses with :class:`CoAdd3D` as its base. """ - return next(c for c in cls.__subclasses__() if c.__name__ == (spectrograph.pypeline + 'CoAdd3D'))( - spec2dfiles, par, skysub_frame=skysub_frame, scale_corr=scale_corr, ra_offsets=ra_offsets, - dec_offsets=dec_offsets, spectrograph=spectrograph, det=det, overwrite=overwrite, - show=show, debug=debug) + spec2dfiles, par, skysub_frame=skysub_frame, scale_corr=scale_corr, + ra_offsets=ra_offsets, dec_offsets=dec_offsets, spectrograph=spectrograph, + det=det, overwrite=overwrite, show=show, debug=debug) - def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, dec_offsets=None, - spectrograph=None, det=None, overwrite=False, show=False, debug=False): + def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, + dec_offsets=None, spectrograph=None, det=None, overwrite=False, show=False, + debug=False): """ Args: @@ -373,10 +383,10 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs same length as spec2dfiles. ra_offsets (:obj:`list`, optional): If not None, this should be a list of relative RA offsets of each frame. It should be the - same length as spec2dfiles. + same length as spec2dfiles. The units should be degrees. dec_offsets (:obj:`list`, optional): If not None, this should be a list of relative Dec offsets of each frame. It should be the - same length as spec2dfiles. + same length as spec2dfiles. The units should be degrees. spectrograph (:obj:`str`, :class:`~pypeit.spectrographs.spectrograph.Spectrograph`, optional): The name or instance of the spectrograph used to obtain the data. If None, this is pulled from the file header. @@ -388,31 +398,63 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs Show results in ginga debug (:obj:`bool`, optional): Show QA for debugging. - """ # TODO :: Consider loading all calibrations into a single variable within the main CoAdd3D parent class. + # Set the variables self.spec2d = spec2dfiles self.numfiles = len(spec2dfiles) self.par = par self.overwrite = overwrite + self.chk_version = self.par['rdx']['chk_version'] + # Extract some parsets for simplicity + self.cubepar = self.par['reduce']['cube'] + self.flatpar = self.par['calibrations']['flatfield'] + self.senspar = self.par['sensfunc'] + # Extract some commonly used variables + self.method = self.cubepar['method'] + self.combine = self.cubepar['combine'] + self.align = self.cubepar['align'] # Do some quick checks on the input options - if skysub_frame is not None: - if len(skysub_frame) != len(spec2dfiles): - msgs.error("The skysub_frame list should be identical length to the spec2dfiles list") - if scale_corr is not None: - if len(scale_corr) != len(spec2dfiles): - msgs.error("The scale_corr list should be identical length to the spec2dfiles list") - if ra_offsets is not None: - if len(ra_offsets) != len(spec2dfiles): - msgs.error("The ra_offsets list should be identical length to the spec2dfiles list") - if dec_offsets is not None: - if len(dec_offsets) != len(spec2dfiles): - msgs.error("The dec_offsets list should be identical length to the spec2dfiles list") - # Set the frame-specific options + if skysub_frame is not None and len(skysub_frame) != self.numfiles: + msgs.error("The skysub_frame list should be identical length to the spec2dfiles list") + if scale_corr is not None and len(scale_corr) != self.numfiles: + msgs.error("The scale_corr list should be identical length to the spec2dfiles list") + if ra_offsets is not None and len(ra_offsets) != self.numfiles: + msgs.error("The ra_offsets list should be identical length to the spec2dfiles list") + if dec_offsets is not None and len(dec_offsets) != self.numfiles: + msgs.error("The dec_offsets list should be identical length to the spec2dfiles list") + # Make sure both ra_offsets and dec_offsets are either both None or both lists + if ra_offsets is None and dec_offsets is not None: + msgs.error("If you provide dec_offsets, you must also provide ra_offsets") + if ra_offsets is not None and dec_offsets is None: + msgs.error("If you provide ra_offsets, you must also provide dec_offsets") + # Set the frame specific options self.skysub_frame = skysub_frame self.scale_corr = scale_corr - self.ra_offsets = np.array(ra_offsets) if isinstance(ra_offsets, list) else ra_offsets - self.dec_offsets = np.array(dec_offsets) if isinstance(dec_offsets, list) else dec_offsets + self.ra_offsets = list(ra_offsets) if isinstance(ra_offsets, np.ndarray) else ra_offsets + self.dec_offsets = list(dec_offsets) if isinstance(dec_offsets, np.ndarray) else dec_offsets + # If there is only one frame being "combined" AND there's no reference image, then don't compute the translation. + if self.numfiles == 1 and self.cubepar["reference_image"] is None: + if self.align: + msgs.warn("Parameter 'align' should be False when there is only one frame and no reference image") + msgs.info("Setting 'align' to False") + self.align = False + if self.ra_offsets is not None: + if not self.align: + msgs.warn("When 'ra_offset' and 'dec_offset' are set, 'align' must be True.") + msgs.info("Setting 'align' to True") + self.align = True + # If no ra_offsets or dec_offsets have been provided, initialise the lists + self.user_alignment = True + if self.ra_offsets is None and self.dec_offsets is None: + msgs.info("No RA or Dec offsets have been provided.") + if self.align: + msgs.info("An automatic alignment will be performed using WCS information from the headers.") + # User offsets are not provided, so turn off the user_alignment + self.user_alignment = False + # Initialise the lists of ra_offsets and dec_offsets + self.ra_offsets = [0.0]*self.numfiles + self.dec_offsets = [0.0]*self.numfiles # Check on Spectrograph input if spectrograph is None: @@ -422,48 +464,30 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs self.spec = load_spectrograph(spectrograph) self.specname = self.spec.name - # Extract some parsets for simplicity - self.cubepar = self.par['reduce']['cube'] - self.flatpar = self.par['calibrations']['flatfield'] - self.senspar = self.par['sensfunc'] - # Initialise arrays for storage self.ifu_ra, self.ifu_dec = np.array([]), np.array([]) # The RA and Dec at the centre of the IFU, as stored in the header - self.all_ra, self.all_dec, self.all_wave = np.array([]), np.array([]), np.array([]) - self.all_sci, self.all_ivar, self.all_idx, self.all_wghts = np.array([]), np.array([]), np.array([]), np.array([]) - self.all_spatpos, self.all_specpos, self.all_spatid = np.array([], dtype=int), np.array([], dtype=int), np.array([], dtype=int) + + self.all_sci, self.all_ivar, self.all_wave, self.all_slitid, self.all_wghts = [], [], [], [], [] self.all_tilts, self.all_slits, self.all_align = [], [], [] - self.all_wcs, self.all_dar = [], [] + self.all_wcs, self.all_ra, self.all_dec, self.all_dar = [], [], [], [] self.weights = np.ones(self.numfiles) # Weights to use when combining cubes self._dspat = None if self.cubepar['spatial_delta'] is None else self.cubepar['spatial_delta'] / 3600.0 # binning size on the sky (/3600 to convert to degrees) self._dwv = self.cubepar['wave_delta'] # linear binning size in wavelength direction (in Angstroms) - # Extract some commonly used variables - self.method = self.cubepar['method'] - self.combine = self.cubepar['combine'] - self.align = self.cubepar['align'] - # If there is only one frame being "combined" AND there's no reference image, then don't compute the translation. - if self.numfiles == 1 and self.cubepar["reference_image"] is None: - if self.align: - msgs.warn("Parameter 'align' should be False when there is only one frame and no reference image") - msgs.info("Setting 'align' to False") - self.align = False - if self.ra_offsets is not None: - if not self.align: - msgs.warn("When 'ra_offset' and 'dec_offset' are set, 'align' must be True.") - msgs.info("Setting 'align' to True") - self.align = True # TODO :: The default behaviour (combine=False, align=False) produces a datacube that uses the instrument WCS # It should be possible (and perhaps desirable) to do a spatial alignment (i.e. align=True), apply this to the # RA,Dec values of each pixel, and then use the instrument WCS to save the output (or, just adjust the crval). # At the moment, if the user wishes to spatially align the frames, a different WCS is generated. # Determine what method is requested - self.spec_subpixel, self.spat_subpixel = 1, 1 + self.spec_subpixel, self.spat_subpixel, self.slice_subpixel = 1, 1, 1 if self.method == "subpixel": - msgs.info("Adopting the subpixel algorithm to generate the datacube.") - self.spec_subpixel, self.spat_subpixel = self.cubepar['spec_subpixel'], self.cubepar['spat_subpixel'] + self.spec_subpixel, self.spat_subpixel, self.slice_subpixel = self.cubepar['spec_subpixel'], self.cubepar['spat_subpixel'], self.cubepar['slice_subpixel'] + msgs.info("Adopting the subpixel algorithm to generate the datacube, with subpixellation scales:" + msgs.newline() + + f" Spectral: {self.spec_subpixel}" + msgs.newline() + + f" Spatial: {self.spat_subpixel}" + msgs.newline() + + f" Slices: {self.slice_subpixel}") elif self.method == "ngp": msgs.info("Adopting the nearest grid point (NGP) algorithm to generate the datacube.") else: @@ -578,7 +602,9 @@ def set_default_scalecorr(self): msgs.info("Loading default scale image for relative spectral illumination correction:" + msgs.newline() + self.cubepar['scale_corr']) try: - spec2DObj = spec2dobj.Spec2DObj.from_file(self.cubepar['scale_corr'], self.detname) + spec2DObj = spec2dobj.Spec2DObj.from_file(self.cubepar['scale_corr'], + self.detname, + chk_version=self.chk_version) except Exception as e: msgs.warn(f'Loading spec2d file raised {type(e).__name__}:\n{str(e)}') msgs.warn("Could not load scaleimg from spec2d file:" + msgs.newline() + @@ -639,7 +665,8 @@ def get_current_scalecorr(self, spec2DObj, scalecorr=None): msgs.info("Loading the following frame for the relative spectral illumination correction:" + msgs.newline() + scalecorr) try: - spec2DObj_scl = spec2dobj.Spec2DObj.from_file(scalecorr, self.detname) + spec2DObj_scl = spec2dobj.Spec2DObj.from_file(scalecorr, self.detname, + chk_version=self.chk_version) except Exception as e: msgs.warn(f'Loading spec2d file raised {type(e).__name__}:\n{str(e)}') msgs.error("Could not load skysub image from spec2d file:" + msgs.newline() + scalecorr) @@ -670,7 +697,9 @@ def set_default_skysub(self): msgs.info("Loading default image for sky subtraction:" + msgs.newline() + self.cubepar['skysub_frame']) try: - spec2DObj = spec2dobj.Spec2DObj.from_file(self.cubepar['skysub_frame'], self.detname) + spec2DObj = spec2dobj.Spec2DObj.from_file(self.cubepar['skysub_frame'], + self.detname, + chk_version=self.chk_version) skysub_exptime = self.spec.get_meta_value([spec2DObj.head0], 'exptime') except: msgs.error("Could not load skysub image from spec2d file:" + msgs.newline() + self.cubepar['skysub_frame']) @@ -740,7 +769,8 @@ def get_current_skysub(self, spec2DObj, exptime, opts_skysub=None): # Load a user specified frame for sky subtraction msgs.info("Loading skysub frame:" + msgs.newline() + opts_skysub) try: - spec2DObj_sky = spec2dobj.Spec2DObj.from_file(opts_skysub, self.detname) + spec2DObj_sky = spec2dobj.Spec2DObj.from_file(opts_skysub, self.detname, + chk_version=self.chk_version) skysub_exptime = self.spec.get_meta_value([spec2DObj_sky.head0], 'exptime') except: msgs.error("Could not load skysub image from spec2d file:" + msgs.newline() + opts_skysub) @@ -772,13 +802,14 @@ def add_grating_corr(self, flatfile, waveimg, slits, spat_flexure=None): spat_flexure : :obj:`float`, optional: Spatial flexure in pixels """ + # Check if the Flat file exists + if not os.path.exists(flatfile): + msgs.warn("Grating correction requested, but the following file does not exist:" + msgs.newline() + flatfile) + return if flatfile not in self.flat_splines.keys(): msgs.info("Calculating relative sensitivity for grating correction") - # Check if the Flat file exists - if not os.path.exists(flatfile): - msgs.error("Grating correction requested, but the following file does not exist:" + msgs.newline() + flatfile) # Load the Flat file - flatimages = flatfield.FlatImages.from_file(flatfile) + flatimages = flatfield.FlatImages.from_file(flatfile, chk_version=self.chk_version) total_illum = flatimages.fit2illumflat(slits, finecorr=False, frametype='illum', initial=True, spat_flexure=spat_flexure) * \ flatimages.fit2illumflat(slits, finecorr=True, frametype='illum', initial=True, spat_flexure=spat_flexure) flatframe = flatimages.pixelflat_raw / total_illum @@ -786,8 +817,7 @@ def add_grating_corr(self, flatfile, waveimg, slits, spat_flexure=None): # Calculate the relative scale scale_model = flatfield.illum_profile_spectral(flatframe, waveimg, slits, slit_illum_ref_idx=self.flatpar['slit_illum_ref_idx'], - model=None, - skymask=None, trim=self.flatpar['slit_trim'], + model=None, trim=self.flatpar['slit_trim'], flexure=spat_flexure, smooth_npix=self.flatpar['slit_illum_smooth_npix']) else: @@ -843,11 +873,12 @@ class SlicerIFUCoAdd3D(CoAdd3D): - White light images are also produced, if requested. """ - def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, dec_offsets=None, - spectrograph=None, det=1, overwrite=False, show=False, debug=False): - super().__init__(spec2dfiles, par, skysub_frame=skysub_frame, scale_corr=scale_corr, ra_offsets=ra_offsets, - dec_offsets=dec_offsets, spectrograph=spectrograph, det=det, overwrite=overwrite, - show=show, debug=debug) + def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, + dec_offsets=None, spectrograph=None, det=1, overwrite=False, show=False, + debug=False): + super().__init__(spec2dfiles, par, skysub_frame=skysub_frame, scale_corr=scale_corr, + ra_offsets=ra_offsets, dec_offsets=dec_offsets, spectrograph=spectrograph, + det=det, overwrite=overwrite, show=show, debug=debug) self.mnmx_wv = None # Will be used to store the minimum and maximum wavelengths of every slit and frame. self._spatscale = np.zeros((self.numfiles, 2)) # index 0, 1 = pixel scale, slicer scale self._specscale = np.zeros(self.numfiles) @@ -881,7 +912,8 @@ def get_alignments(self, spec2DObj, slits, spat_flexure=None): alignfile = os.path.join(spec2DObj.calibs['DIR'], spec2DObj.calibs[key]) if os.path.exists(alignfile) and self.cubepar['astrometric']: msgs.info("Loading alignments") - alignments = alignframe.Alignments.from_file(alignfile) + alignments = alignframe.Alignments.from_file(alignfile, + chk_version=self.chk_version) else: msgs.warn(f'Processed alignment frame not recorded or not found!') msgs.info("Using slit edges for astrometric transform") @@ -920,27 +952,25 @@ def load(self): As well as the primary arrays that store the pixel information for multiple spec2d frames, including: - * self.all_ra - * self.all_dec - * self.all_wave * self.all_sci * self.all_ivar - * self.all_idx + * self.all_wave + * self.all_slitid * self.all_wghts - * self.all_spatpos - * self.all_specpos - * self.all_spatid * self.all_tilts * self.all_slits * self.all_align - * self.all_dar * self.all_wcs + * self.all_ra + * self.all_dec + * self.all_dar """ # Load all spec2d files and prepare the data for making a datacube for ff, fil in enumerate(self.spec2d): # Load it up msgs.info("Loading PypeIt spec2d frame:" + msgs.newline() + fil) - spec2DObj = spec2dobj.Spec2DObj.from_file(fil, self.detname) + spec2DObj = spec2dobj.Spec2DObj.from_file(fil, self.detname, + chk_version=self.chk_version) detector = spec2DObj.detector spat_flexure = None # spec2DObj.sci_spat_flexure @@ -1039,34 +1069,31 @@ def load(self): # TODO: This should use the mask function to figure out which elements are masked. onslit_gpm = (slitid_img_init > 0) & (bpmmask.mask == 0) & sky_is_good - # Grab the WCS of this frame - self.all_wcs.append(self.spec.get_wcs(spec2DObj.head0, slits, detector.platescale, wave0, dwv)) - # Generate the alignment splines, and then retrieve images of the RA and Dec of every pixel, # and the number of spatial pixels in each slit alignSplines = self.get_alignments(spec2DObj, slits, spat_flexure=spat_flexure) - raimg, decimg, minmax = slits.get_radec_image(self.all_wcs[ff], alignSplines, spec2DObj.tilts, - initial=True, flexure=spat_flexure) - # Get copies of arrays to be saved - ra_ext = raimg[onslit_gpm] - dec_ext = decimg[onslit_gpm] + # Grab the WCS of this frame, and generate the RA and Dec images + # NOTE :: These RA and Dec images are only used to setup the WCS of the datacube. The actual RA and Dec + # of each pixel in the datacube is calculated in the datacube.subpixellate() method. + crval_wv = self.cubepar['wave_min'] if self.cubepar['wave_min'] is not None else wave0 + cd_wv = self.cubepar['wave_delta'] if self.cubepar['wave_delta'] is not None else dwv + self.all_wcs.append(self.spec.get_wcs(spec2DObj.head0, slits, detector.platescale, crval_wv, cd_wv)) + ra_img, dec_img, minmax = slits.get_radec_image(self.all_wcs[ff], alignSplines, spec2DObj.tilts, initial=True, flexure=spat_flexure) + + # Extract wavelength and delta wavelength arrays from the images wave_ext = waveimg[onslit_gpm] - flux_ext = sciImg[onslit_gpm] - ivar_ext = ivar[onslit_gpm] dwav_ext = dwaveimg[onslit_gpm] - # From here on out, work in sorted wavelengths + # For now, work in sorted wavelengths wvsrt = np.argsort(wave_ext) wave_sort = wave_ext[wvsrt] dwav_sort = dwav_ext[wvsrt] - ra_sort = ra_ext[wvsrt] - dec_sort = dec_ext[wvsrt] # Here's an array to get back to the original ordering resrt = np.argsort(wvsrt) # Compute the DAR correction - cosdec = np.cos(np.mean(dec_sort) * np.pi / 180.0) + cosdec = np.cos(self.ifu_dec[ff] * np.pi / 180.0) airmass = self.spec.get_meta_value([spec2DObj.head0], 'airmass') # unitless parangle = self.spec.get_meta_value([spec2DObj.head0], 'parangle') pressure = self.spec.get_meta_value([spec2DObj.head0], 'pressure') # units are pascals @@ -1074,11 +1101,11 @@ def load(self): humidity = self.spec.get_meta_value([spec2DObj.head0], 'humidity') # Expressed as a percentage (not a fraction!) darcorr = DARcorrection(airmass, parangle, pressure, temperature, humidity, cosdec) - # Perform extinction correction + # Compute the extinction correction msgs.info("Applying extinction correction") - longitude = self.spec.telescope['longitude'] - latitude = self.spec.telescope['latitude'] - extinct = flux_calib.load_extinction_data(longitude, latitude, self.senspar['UVIS']['extinct_file']) + extinct = flux_calib.load_extinction_data(self.spec.telescope['longitude'], + self.spec.telescope['latitude'], + self.senspar['UVIS']['extinct_file']) # extinction_correction requires the wavelength is sorted extcorr_sort = flux_calib.extinction_correction(wave_sort * units.AA, airmass, extinct) @@ -1105,27 +1132,26 @@ def load(self): # Convert the flux_sav to counts/s, correct for the relative sensitivity of different setups extcorr_sort *= sensfunc_sort / (exptime * gratcorr_sort) # Correct for extinction - flux_sort = flux_ext[wvsrt] * extcorr_sort - ivar_sort = ivar_ext[wvsrt] / extcorr_sort ** 2 + sciImg[onslit_gpm] *= extcorr_sort[resrt] + ivar[onslit_gpm] /= extcorr_sort[resrt] ** 2 # Convert units to Counts/s/Ang/arcsec2 # Slicer sampling * spatial pixel sampling sl_deg = np.sqrt(self.all_wcs[ff].wcs.cd[0, 0] ** 2 + self.all_wcs[ff].wcs.cd[1, 0] ** 2) px_deg = np.sqrt(self.all_wcs[ff].wcs.cd[1, 1] ** 2 + self.all_wcs[ff].wcs.cd[0, 1] ** 2) scl_units = dwav_sort * (3600.0 * sl_deg) * (3600.0 * px_deg) - flux_sort /= scl_units - ivar_sort *= scl_units ** 2 + sciImg[onslit_gpm] /= scl_units[resrt] + ivar[onslit_gpm] *= scl_units[resrt] ** 2 # Calculate the weights relative to the zeroth cube self.weights[ff] = 1.0 # exptime #np.median(flux_sav[resrt]*np.sqrt(ivar_sav[resrt]))**2 + wghts = self.weights[ff] * np.ones(sciImg.shape) # Get the slit image and then unset pixels in the slit image that are bad - this_specpos, this_spatpos = np.where(onslit_gpm) - this_spatid = slitid_img_init[onslit_gpm] + slitid_img_gpm = slitid_img_init * onslit_gpm.astype(int) # If individual frames are to be output without aligning them, # there's no need to store information, just make the cubes now - numpix = ra_sort.size if not self.combine and not self.align: # Get the output filename if self.numfiles == 1 and self.cubepar['output_filename'] != "": @@ -1136,30 +1162,25 @@ def load(self): slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) numwav = int((np.max(waveimg) - wave0) / dwv) bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) - # Generate the output WCS for the datacube - tmp_crval_wv = (self.all_wcs[ff].wcs.crval[2] * self.all_wcs[ff].wcs.cunit[2]).to(units.Angstrom).value - tmp_cd_wv = (self.all_wcs[ff].wcs.cd[2,2] * self.all_wcs[ff].wcs.cunit[2]).to(units.Angstrom).value - crval_wv = self.cubepar['wave_min'] if self.cubepar['wave_min'] is not None else tmp_crval_wv - cd_wv = self.cubepar['wave_delta'] if self.cubepar['wave_delta'] is not None else tmp_cd_wv - output_wcs = self.spec.get_wcs(spec2DObj.head0, slits, detector.platescale, crval_wv, cd_wv) # Set the wavelength range of the white light image. wl_wvrng = None if self.cubepar['save_whitelight']: wl_wvrng = datacube.get_whitelight_range(np.max(self.mnmx_wv[ff, :, 0]), - np.min(self.mnmx_wv[ff, :, 1]), - self.cubepar['whitelight_range']) + np.min(self.mnmx_wv[ff, :, 1]), + self.cubepar['whitelight_range']) # Make the datacube if self.method in ['subpixel', 'ngp']: # Generate the datacube flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(outfile, output_wcs, ra_sort[resrt], dec_sort[resrt], wave_sort[resrt], - flux_sort[resrt], ivar_sort[resrt], np.ones(numpix), - this_spatpos, this_specpos, this_spatid, - spec2DObj.tilts, slits, alignSplines, darcorr, bins, all_idx=None, - overwrite=self.overwrite, whitelight_range=wl_wvrng, - spec_subpixel=self.spec_subpixel, spat_subpixel=self.spat_subpixel) + datacube.generate_cube_subpixel(self.all_wcs[ff], bins, sciImg, ivar, waveimg, slitid_img_gpm, wghts, + self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, + self.ra_offsets[ff], self.dec_offsets[ff], + overwrite=self.overwrite, whitelight_range=wl_wvrng, outfile=outfile, + spec_subpixel=self.spec_subpixel, + spat_subpixel=self.spat_subpixel, + slice_subpixel=self.slice_subpixel) # Prepare the header - hdr = output_wcs.to_header() + hdr = self.all_wcs[ff].to_header() if self.fluxcal: hdr['FLUXUNIT'] = (flux_calib.PYPEIT_FLUX_SCALE, "Flux units -- erg/s/cm^2/Angstrom/arcsec^2") else: @@ -1173,16 +1194,13 @@ def load(self): continue # Store the information if we are combining multiple frames - self.all_ra = np.append(self.all_ra, ra_sort[resrt]) - self.all_dec = np.append(self.all_dec, dec_sort[resrt]) - self.all_wave = np.append(self.all_wave, wave_sort[resrt]) - self.all_sci = np.append(self.all_sci, flux_sort[resrt]) - self.all_ivar = np.append(self.all_ivar, ivar_sort[resrt].copy()) - self.all_idx = np.append(self.all_idx, ff * np.ones(numpix)) - self.all_wghts = np.append(self.all_wghts, self.weights[ff] * np.ones(numpix) / self.weights[0]) - self.all_spatpos = np.append(self.all_spatpos, this_spatpos) - self.all_specpos = np.append(self.all_specpos, this_specpos) - self.all_spatid = np.append(self.all_spatid, this_spatid) + self.all_sci.append(sciImg.copy()) + self.all_ivar.append(ivar.copy()) + self.all_wave.append(waveimg.copy()) + self.all_ra.append(ra_img.copy()) + self.all_dec.append(dec_img.copy()) + self.all_slitid.append(slitid_img_gpm.copy()) + self.all_wghts.append(wghts.copy()) self.all_tilts.append(spec2DObj.tilts) self.all_slits.append(slits) self.all_align.append(alignSplines) @@ -1197,29 +1215,29 @@ def run_align(self): `numpy.ndarray`_: A new set of Dec values that has been aligned """ # Grab cos(dec) for convenience - cosdec = np.cos(np.mean(self.all_dec) * np.pi / 180.0) + cosdec = np.cos(np.mean(self.ifu_dec[0]) * np.pi / 180.0) + # Initialize the RA and Dec offset arrays + ra_offsets, dec_offsets = [0.0]*self.numfiles, [0.0]*self.numfiles # Register spatial offsets between all frames - if self.ra_offsets is not None: - # Calculate the offsets - new_ra, new_dec = datacube.align_user_offsets(self.all_ra, self.all_dec, self.all_idx, - self.ifu_ra, self.ifu_dec, - self.ra_offsets, self.dec_offsets) + if self.user_alignment: + # The user has specified offsets - update these values accounting for the difference in header RA/DEC + ra_offsets, dec_offsets = datacube.align_user_offsets(self.ifu_ra, self.ifu_dec, + self.ra_offsets, self.dec_offsets) else: - new_ra, new_dec = self.all_ra.copy(), self.all_dec.copy() # Find the wavelength range where all frames overlap min_wl, max_wl = datacube.get_whitelight_range(np.max(self.mnmx_wv[:, :, 0]), # The max blue wavelength np.min(self.mnmx_wv[:, :, 1]), # The min red wavelength self.cubepar['whitelight_range']) # The user-specified values (if any) - # Get the good whitelight pixels - ww, wavediff = datacube.get_whitelight_pixels(self.all_wave, min_wl, max_wl) + # Get the good white light pixels + slitid_img_gpm, wavediff = datacube.get_whitelight_pixels(self.all_wave, self.all_slitid, min_wl, max_wl) # Iterate over white light image generation and spatial shifting numiter = 2 for dd in range(numiter): msgs.info(f"Iterating on spatial translation - ITERATION #{dd+1}/{numiter}") - ref_idx = None # Don't use an index - This is the default behaviour when a reference image is supplied # Generate the WCS image_wcs, voxedge, reference_image = \ - datacube.create_wcs(new_ra[ww], new_dec[ww], self.all_wave[ww], self._dspat, wavediff, + datacube.create_wcs(self.all_ra, self.all_dec, self.all_wave, slitid_img_gpm, self._dspat, wavediff, + ra_offsets=ra_offsets, dec_offsets=dec_offsets, ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], wave_min=self.cubepar['wave_min'], wave_max=self.cubepar['wave_max'], @@ -1228,12 +1246,13 @@ def run_align(self): if voxedge[2].size != 2: msgs.error("Spectral range for WCS is incorrect for white light image") - wl_imgs = datacube.generate_image_subpixel(image_wcs, new_ra[ww], new_dec[ww], self.all_wave[ww], - self.all_sci[ww], self.all_ivar[ww], self.all_wghts[ww], - self.all_spatpos[ww], self.all_specpos[ww], self.all_spatid[ww], + wl_imgs = datacube.generate_image_subpixel(image_wcs, voxedge, self.all_sci, self.all_ivar, self.all_wave, + slitid_img_gpm, self.all_wghts, self.all_wcs, self.all_tilts, self.all_slits, self.all_align, self.all_dar, - voxedge, all_idx=self.all_idx[ww], - spec_subpixel=self.spec_subpixel, spat_subpixel=self.spat_subpixel) + ra_offsets, dec_offsets, + spec_subpixel=self.spec_subpixel, + spat_subpixel=self.spat_subpixel, + slice_subpixel=self.slice_subpixel) if reference_image is None: # ref_idx will be the index of the cube with the highest S/N ref_idx = np.argmax(self.weights) @@ -1251,10 +1270,10 @@ def run_align(self): dec_shift *= self._dspat msgs.info("Spatial shift of cube #{0:d}:".format(ff + 1) + msgs.newline() + "RA, DEC (arcsec) = {0:+0.3f} E, {1:+0.3f} N".format(ra_shift*3600.0, dec_shift*3600.0)) - # Apply the shift - new_ra[self.all_idx == ff] += ra_shift - new_dec[self.all_idx == ff] += dec_shift - return new_ra, new_dec + # Store the shift in the RA and DEC offsets in degrees + ra_offsets[ff] += ra_shift + dec_offsets[ff] += dec_shift + return ra_offsets, dec_offsets def compute_weights(self): """ @@ -1263,19 +1282,22 @@ def compute_weights(self): Returns: `numpy.ndarray`_: The individual pixel weights for each detector pixel, and every frame. """ + # If there is only one file, then all pixels have the same weight + if self.numfiles == 1: + return np.ones_like(self.all_sci) + # Calculate the relative spectral weights of all pixels - return np.ones_like(self.all_sci) if self.numfiles == 1 else \ - datacube.compute_weights_frompix(self.all_ra, self.all_dec, self.all_wave, self.all_sci, self.all_ivar, - self.all_idx, self._dspat, self._dwv, self.mnmx_wv, self.all_wghts, - self.all_spatpos, self.all_specpos, self.all_spatid, - self.all_tilts, self.all_slits, self.all_align, self.all_dar, - ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], - dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], - wave_min=self.cubepar['wave_min'], wave_max=self.cubepar['wave_max'], - relative_weights=self.cubepar['relative_weights'], - whitelight_range=self.cubepar['whitelight_range'], - reference_image=self.cubepar['reference_image'], - specname=self.specname) + return datacube.compute_weights_frompix(self.all_ra, self.all_dec, self.all_wave, self.all_sci, self.all_ivar, + self.all_slitid, self._dspat, self._dwv, self.mnmx_wv, self.all_wghts, + self.all_wcs, self.all_tilts, self.all_slits, self.all_align, self.all_dar, + self.ra_offsets, self.dec_offsets, + ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], + dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], + wave_min=self.cubepar['wave_min'], wave_max=self.cubepar['wave_max'], + weight_method=self.cubepar['weight_method'], + whitelight_range=self.cubepar['whitelight_range'], + reference_image=self.cubepar['reference_image'], + specname=self.specname) def run(self): """ @@ -1320,14 +1342,15 @@ def run(self): # Align the frames if self.align: - self.all_ra, self.all_dec = self.run_align() + self.ra_offsets, self.dec_offsets = self.run_align() # Compute the relative weights on the spectra self.all_wghts = self.compute_weights() # Generate the WCS, and the voxel edges cube_wcs, vox_edges, _ = \ - datacube.create_wcs(self.all_ra, self.all_dec, self.all_wave, self._dspat, self._dwv, + datacube.create_wcs(self.all_ra, self.all_dec, self.all_wave, self.all_slitid, self._dspat, self._dwv, + ra_offsets=self.ra_offsets, dec_offsets=self.dec_offsets, ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], wave_min=self.cubepar['wave_min'], wave_max=self.cubepar['wave_max'], @@ -1354,12 +1377,14 @@ def run(self): outfile = datacube.get_output_filename("", self.cubepar['output_filename'], True, -1) # Generate the datacube flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(outfile, cube_wcs, self.all_ra, self.all_dec, self.all_wave, - self.all_sci, self.all_ivar, np.ones(self.all_wghts.size), - self.all_spatpos, self.all_specpos, self.all_spatid, - self.all_tilts, self.all_slits, self.all_align, self.all_dar, vox_edges, - all_idx=self.all_idx, overwrite=self.overwrite, whitelight_range=wl_wvrng, - spec_subpixel=self.spec_subpixel, spat_subpixel=self.spat_subpixel) + datacube.generate_cube_subpixel(cube_wcs, vox_edges, self.all_sci, self.all_ivar, self.all_wave, + self.all_slitid, self.all_wghts, self.all_wcs, + self.all_tilts, self.all_slits, self.all_align, self.all_dar, + self.ra_offsets, self.dec_offsets, + outfile=outfile, overwrite=self.overwrite, whitelight_range=wl_wvrng, + spec_subpixel=self.spec_subpixel, + spat_subpixel=self.spat_subpixel, + slice_subpixel=self.slice_subpixel) # Prepare the header hdr = cube_wcs.to_header() if self.fluxcal: @@ -1374,15 +1399,17 @@ def run(self): else: for ff in range(self.numfiles): outfile = datacube.get_output_filename("", self.cubepar['output_filename'], False, ff) - ww = np.where(self.all_idx == ff) # Generate the datacube flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(outfile, cube_wcs, self.all_ra[ww], self.all_dec[ww], self.all_wave[ww], - self.all_sci[ww], self.all_ivar[ww], np.ones(ww[0].size), - self.all_spatpos[ww], self.all_specpos[ww], self.all_spatid[ww], - self.all_tilts[ff], self.all_slits[ff], self.all_align[ff], self.all_dar[ff], vox_edges, - all_idx=self.all_idx[ww], overwrite=self.overwrite, whitelight_range=wl_wvrng, - spec_subpixel=self.spec_subpixel, spat_subpixel=self.spat_subpixel) + datacube.generate_cube_subpixel(cube_wcs, vox_edges, + self.all_sci[ff], self.all_ivar[ff], self.all_wave[ff], + self.all_slitid[ff], self.all_wghts[ff], self.all_wcs[ff], + self.all_tilts[ff], self.all_slits[ff], self.all_align[ff], self.all_dar[ff], + self.ra_offsets[ff], self.dec_offsets[ff], + overwrite=self.overwrite, whitelight_range=wl_wvrng, + outfile=outfile, spec_subpixel=self.spec_subpixel, + spat_subpixel=self.spat_subpixel, + slice_subpixel=self.slice_subpixel) # Prepare the header hdr = cube_wcs.to_header() if self.fluxcal: diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index e3158372ab..0d6cfff77e 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -180,7 +180,7 @@ def poly_ratio_fitfunc_chi2(theta, gpm, arg_dict): There are two non-standard things implemented here which increase ther robustness. The first is a non-standard error used for the chi, which adds robustness and increases the stability of the optimization. This was taken from the idlutils solve_poly_ratio code. The second thing is that the chi is remapped using the scipy huber loss function to - reduce sensitivity to outliers, ased on the scipy cookbook on robust optimization. + reduce sensitivity to outliers, based on the scipy cookbook on robust optimization. Args: theta (`numpy.ndarray`_): parameter vector for the polymomial fit @@ -304,7 +304,7 @@ def poly_ratio_fitfunc(flux_ref, gpm, arg_dict, init_from_last=None, **kwargs_op except KeyError: debug = False - sigma_corr, maskchi = renormalize_errors(chi, mask=gpm, title = 'poly_ratio_fitfunc', debug=debug) + sigma_corr, maskchi = renormalize_errors(chi, mask=mask_both, title = 'poly_ratio_fitfunc', debug=debug) ivartot = ivartot1/sigma_corr**2 return result, flux_scale, ivartot @@ -747,10 +747,58 @@ def smooth_weights(inarr, gdmsk, sn_smooth_npix): sig_res = np.fmax(sn_smooth_npix / 10.0, 3.0) gauss_kernel = convolution.Gaussian1DKernel(sig_res) sn_conv = convolution.convolve(sn_med2, gauss_kernel, boundary='extend') + # TODO Should we be setting a floor on the weights to prevent tiny numbers? return sn_conv -def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, const_weights=False, - ivar_weights=False, relative_weights=False, verbose=False): +# TODO add a wave_min, wave_max option here to deal with objects like high-z QSOs etc. which only have flux in a given wavelength range. +def calc_snr(fluxes, ivars, gpms): + + """ + Calculate the rms S/N of each input spectrum. + + Parameters + ---------- + fluxes : list + List of length nexp containing the `numpy.ndarray`_ 1d float spectra. The + shapes in the list can be different. nexp = len(fluxes) + ivars : list + List of length nexp containing the `numpy.ndarray`_ 1d float inverse + variances of the spectra. + gpms : list + List of length nexp containing the `numpy.ndarray`_ 1d float boolean good + pixel masks of the spectra. + verbose : bool, optional + Verbosity of print out. + + Returns + ------- + rms_sn : `numpy.ndarray`_ + Array of shape (nexp,) of root-mean-square S/N value for each input spectra where nexp=len(fluxes). + sn_val : list + List of length nexp containing the wavelength dependent S/N arrays for each input spectrum, i.e. + each element contains the array flux*sqrt(ivar) + """ + + nexp = len(fluxes) + + # Calculate S/N + sn_val, rms_sn, sn2 = [], [], [] + for iexp, (flux, ivar, gpm) in enumerate(zip(fluxes, ivars, gpms)): + sn_val_iexp = flux*np.sqrt(ivar) + sn_val.append(sn_val_iexp) + sn_val_ma = np.ma.array(sn_val_iexp, mask=np.logical_not(gpm)) + sn_sigclip = stats.sigma_clip(sn_val_ma, sigma=3, maxiters=5) + sn2_iexp = sn_sigclip.mean()**2 # S/N^2 value for each spectrum + if sn2_iexp is np.ma.masked: + msgs.error(f'No unmasked value in iexp={iexp+1}/{nexp}. Check inputs.') + else: + sn2.append(sn2_iexp) + rms_sn.append(np.sqrt(sn2_iexp)) # Root Mean S/N**2 value for all spectra + + return np.array(rms_sn), sn_val + +# TODO add a wave_min, wave_max option here to deal with objects like high-z QSOs etc. which only have flux in a given wavelength range. +def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, weight_method='auto', verbose=False): """ Calculate the S/N of each input spectrum and create an array of @@ -769,22 +817,41 @@ def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, const_weights=False, pixel masks of the spectra. sn_smooth_npix : float, optional Number of pixels used for determining smoothly varying S/N ratio - weights. This can be set to None if const_weights is True, since then + weights. This must be passed for all weight methods excpet for weight_method='constant' or 'uniform', since then wavelength dependent weights are not used. - const_weights : bool, optional - Use a constant weights for each spectrum? - ivar_weights : bool, optional - Use inverse variance weighted scheme? - relative_weights : bool, optional - Calculate weights by fitting to the ratio of spectra? Note, relative - weighting will only work well when there is at least one spectrum with a - reasonable S/N, and a continuum. RJC note - This argument may only be - better when the object being used has a strong continuum + emission - lines. The reference spectrum is assigned a value of 1 for all - wavelengths, and the weights of all other spectra will be determined - relative to the reference spectrum. This is particularly useful if you - are dealing with highly variable spectra (e.g. emission lines) and - require a precision better than ~1 per cent. + weight_method : str, optional + + The weighting method to be used. Options are ``'auto'``, ``'constant'``, ``'uniform'``, ``'wave_dependent'``, + ``'relative'``, or ``'ivar'``. The default is ``'auto'``. Behavior is + as follows: + + - ``'auto'``: Use constant weights if rms_sn < 3.0, otherwise use + wavelength dependent. + + - ``'constant'``: Constant weights based on rms_sn**2 + + - ``'uniform'``: Uniform weighting. + + - ``'wave_dependent'``: Wavelength dependent weights will be used + irrespective of the rms_sn ratio. This option will not work well + at low S/N ratio although it is useful for objects where only a + small fraction of the spectral coverage has high S/N ratio (like + high-z quasars). + + - ``'relative'``: Calculate weights by fitting to the ratio of + spectra? Note, relative weighting will only work well when there + is at least one spectrum with a reasonable S/N, and a continuum. + RJC note - This argument may only be better when the object being + used has a strong continuum + emission lines. The reference + spectrum is assigned a value of 1 for all wavelengths, and the + weights of all other spectra will be determined relative to the + reference spectrum. This is particularly useful if you are + dealing with highly variable spectra (e.g. emission lines) and + require a precision better than ~1 per cent. + + - ``'ivar'``: Use inverse variance weighting. This is not well + tested and should probably be deprecated. + verbose : bool, optional Verbosity of print out. @@ -798,37 +865,29 @@ def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, const_weights=False, vectors) provided in waves, i.e. it is a list of arrays of type `numpy.ndarray`_ with the same shape as those in waves. """ + if weight_method not in ['auto', 'constant', 'uniform', 'wave_dependent', 'relative', 'ivar']: + msgs.error('Unrecognized option for weight_method=%s').format(weight_method) nexp = len(fluxes) # Check that all the input lists have the same length if len(ivars) != nexp or len(gpms) != nexp: msgs.error("Input lists of spectra must have the same length") - # Check sn_smooth_npix is set if const_weights=False - if sn_smooth_npix is None and not const_weights: - msgs.error('sn_smooth_npix cannot be None if const_weights=False') + # Check that sn_smooth_npix if weight_method = constant or uniform + if sn_smooth_npix is None and weight_method not in ['constant', 'uniform']: + msgs.error("sn_smooth_npix cannot be None unless the weight_method='constant' or weight_method='uniform'") - # Give preference to ivar_weights - if ivar_weights and relative_weights: - msgs.warn("Performing inverse variance weights instead of relative weighting") - relative_weights = False - - # Calculate S/N - sn_val, rms_sn, sn2 = [], [], [] - for iexp in range(nexp): - sn_val_iexp = fluxes[iexp]*np.sqrt(ivars[iexp]) - sn_val.append(sn_val_iexp) - sn_val_ma = np.ma.array(sn_val_iexp, mask=np.logical_not(gpms[iexp])) - sn_sigclip = stats.sigma_clip(sn_val_ma, sigma=3, maxiters=5) - sn2_iexp = sn_sigclip.mean()**2 # S/N^2 value for each spectrum - if sn2_iexp is np.ma.masked: - msgs.error(f'No unmasked value in iexp={iexp+1}/{nexp}. Check inputs.') - else: - sn2.append(sn2_iexp) - rms_sn.append(np.sqrt(sn2_iexp)) # Root Mean S/N**2 value for all spectra + rms_sn, sn_val = calc_snr(fluxes, ivars, gpms) + sn2 = np.square(rms_sn) # Check if relative weights input - if relative_weights: + if verbose: + msgs.info('Computing weights with weight_method=%s'.format(weight_method)) + + weights = [] + + weight_method_used = [] if weight_method == 'auto' else nexp*[weight_method] + if weight_method == 'relative': # Relative weights are requested, use the highest S/N spectrum as a reference ref_spec = np.argmax(sn2) if verbose: @@ -836,101 +895,44 @@ def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, const_weights=False, "The reference spectrum (ref_spec={0:d}) has a typical S/N = {1:.3f}".format(ref_spec, sn2[ref_spec])) # Adjust the arrays to be relative refscale = utils.inverse(sn_val[ref_spec]) - gpms_rel = [] for iexp in range(nexp): # Compute the relative (S/N)^2 and update the mask sn2[iexp] /= sn2[ref_spec] - gpms_rel.append(gpms[iexp] & ((gpms[ref_spec]) | (sn_val[ref_spec] != 0))) - sn_val[iexp] *= refscale - - sn_gpms = gpms_rel - else: - sn_gpms = gpms - - # TODO: Should ivar weights be deprecated?? - # Initialise weights - weights = [] - if ivar_weights: - if verbose: - msgs.info("Using ivar weights for merging orders") + gpm_rel = gpms[iexp] & ((gpms[ref_spec]) | (sn_val[ref_spec] != 0)) + sn_val_rescaled = sn_val[iexp]*refscale + weights.append(smooth_weights(sn_val_rescaled** 2, gpm_rel, sn_smooth_npix)) + elif weight_method == 'ivar': + # TODO: Should ivar weights be deprecated?? for ivar, mask in zip(ivars, gpms): weights.append(smooth_weights(ivar, mask, sn_smooth_npix)) - else: + elif weight_method == 'constant': + for iexp in range(nexp): + weights.append(np.full(fluxes[iexp].size, np.fmax(sn2[iexp], 1e-2))) # set the minimum to be 1e-2 to avoid zeros + elif weight_method == 'uniform': for iexp in range(nexp): - # Now - if (rms_sn[iexp] < 3.0) or const_weights: - weight_method = 'constant' + weights.append( + np.full(fluxes[iexp].size, 1.0)) + elif weight_method == 'wave_dependent': + for iexp in range(nexp): + weights.append(smooth_weights(sn_val[iexp] ** 2, gpms[iexp], sn_smooth_npix)) + elif weight_method == 'auto': + for iexp in range(nexp): + if rms_sn[iexp] < 3.0: weights.append(np.full(fluxes[iexp].size, np.fmax(sn2[iexp], 1e-2))) # set the minimum to be 1e-2 to avoid zeros + weight_method_used.append('constant') else: - weight_method = 'wavelength dependent' - # JFH THis line is experimental but it deals with cases where the spectrum drops to zero. We thus - # transition to using ivar_weights. This needs more work because the spectra are not rescaled at this point. - # RJC - also note that nothing should be changed to sn_val is relative_weights=True - #sn_val[sn_val[:, iexp] < 1.0, iexp] = ivar_stack[sn_val[:, iexp] < 1.0, iexp] - weights.append(smooth_weights(sn_val[iexp]**2, sn_gpms[iexp], sn_smooth_npix)) - if verbose: - msgs.info('Using {:s} weights for coadding, S/N '.format(weight_method) + - '= {:4.2f}, weight = {:4.2f} for {:}th exposure'.format( - rms_sn[iexp], np.mean(weights[iexp]), iexp)) + weights.append(smooth_weights(sn_val[iexp]**2, gpms[iexp], sn_smooth_npix)) + weight_method_used.append('wavelength dependent') + if verbose: + for iexp in range(nexp): + msgs.info('Using {:s} weights for coadding, S/N '.format(weight_method_used[iexp]) + + '= {:4.2f}, weight = {:4.2f} for {:}th exposure'.format(rms_sn[iexp], np.mean(weights[iexp]), iexp)) # Finish return np.array(rms_sn), weights -# TODO: This was commented out and would need to be refactored if brought back -# because of changes to the SensFunc and Telluric datamodels. -## TODO Rename this function to something sensfunc related -#def get_tell_from_file(sensfile, waves, masks, iord=None): -# ''' -# Get the telluric model from the sensfile. -# -# Args: -# sensfile (str): the name of your fits format sensfile -# waves (ndarray): wavelength grid for your output telluric model -# masks (ndarray, bool): mask for the wave -# iord (int or None): if None returns telluric model for all orders, otherwise return the order you want -# -# Returns: -# ndarray: telluric model on your wavelength grid -# ''' -# -# -# sens_param = Table.read(sensfile, 1) -# sens_table = Table.read(sensfile, 2) -# telluric = np.zeros_like(waves) -# -# if (waves.ndim == 1) and (iord is None): -# msgs.info('Loading Telluric from Longslit sensfiles.') -# tell_interp = scipy.interpolate.interp1d(sens_table[0]['WAVE'], sens_table[0]['TELLURIC'], kind='cubic', -# bounds_error=False, fill_value=np.nan)(waves[masks]) -# telluric[masks] = tell_interp -# elif (waves.ndim == 1) and (iord is not None): -# msgs.info('Loading order {:} Telluric from Echelle sensfiles.'.format(iord)) -# wave_tell_iord = sens_table[iord]['WAVE'] -# tell_mask = (wave_tell_iord > 1.0) -# tell_iord = sens_table[iord]['TELLURIC'] -# tell_iord_interp = scipy.interpolate.interp1d(wave_tell_iord[tell_mask], tell_iord[tell_mask], kind='cubic', -# bounds_error=False, fill_value=np.nan)(waves[masks]) -# telluric[masks] = tell_iord_interp -# else: -# norder = np.shape(waves)[1] -# for iord in range(norder): -# wave_iord = waves[:, iord] -# mask_iord = masks[:, iord] -# -# # Interpolate telluric to the same grid with waves -# # Since it will be only used for plotting, I just simply interpolate it rather than evaluate it based on the model -# wave_tell_iord = sens_table[iord]['WAVE'] -# tell_mask = (wave_tell_iord > 1.0) -# tell_iord = sens_table[iord]['TELLURIC'] -# tell_iord_interp = scipy.interpolate.interp1d(wave_tell_iord[tell_mask], tell_iord[tell_mask], kind='cubic', -# bounds_error=False, fill_value=np.nan)(wave_iord[mask_iord]) -# telluric[mask_iord, iord] = tell_iord_interp -# -# return telluric - - def robust_median_ratio(flux, ivar, flux_ref, ivar_ref, mask=None, mask_ref=None, ref_percentile=70.0, min_good=0.05, maxiters=5, sigrej=3.0, max_factor=10.0, snr_do_not_rescale=1.0, verbose=False): @@ -1170,7 +1172,7 @@ def scale_spec(wave, flux, ivar, sn, wave_ref, flux_ref, ivar_ref, mask=None, ma msgs.error("Scale method not recognized! Check documentation for available options") # Finish if show: - scale_spec_qa(wave, flux, ivar, wave_ref, flux_ref, ivar_ref, scale, method_used, mask = mask, mask_ref=mask_ref, + scale_spec_qa(wave, flux*mask, ivar*mask, wave_ref, flux_ref*mask_ref, ivar_ref*mask_ref, scale, method_used, mask = mask, mask_ref=mask_ref, title='Scaling Applied to the Data') return flux_scale, ivar_scale, scale, method_used @@ -1400,7 +1402,7 @@ def scale_spec_qa(wave, flux, ivar, wave_ref, flux_ref, ivar_ref, ymult, # TODO: Change mask to gpm def coadd_iexp_qa(wave, flux, rejivar, mask, wave_stack, flux_stack, ivar_stack, mask_stack, - outmask, norder=None, title='', qafile=None): + outmask, norder=None, title='', qafile=None, show_telluric=False): """ Routine to creqate QA for showing the individual spectrum @@ -1436,6 +1438,8 @@ def coadd_iexp_qa(wave, flux, rejivar, mask, wave_stack, flux_stack, ivar_stack, Plot title qafile (:obj:`str`, optional): QA file name + show_telluric (:obj:`bool`, optional): + Show the atmospheric absorption if wavelengths > 9000A are covered by the spectrum """ @@ -1464,7 +1468,7 @@ def coadd_iexp_qa(wave, flux, rejivar, mask, wave_stack, flux_stack, ivar_stack, # TODO Use one of our telluric models here instead # Plot transmission - if (np.max(wave[mask]) > 9000.0): + if (np.max(wave[mask]) > 9000.0) and show_telluric: skytrans_file = data.get_skisim_filepath('atm_transmission_secz1.5_1.6mm.dat') skycat = np.genfromtxt(skytrans_file, dtype='float') scale = 0.8 * ymax @@ -1537,7 +1541,7 @@ def weights_qa(waves, weights, gpms, title='', colors=None): plt.show() def coadd_qa(wave, flux, ivar, nused, gpm=None, tell=None, - title=None, qafile=None): + title=None, qafile=None, show_telluric=False): """ Routine to make QA plot of the final stacked spectrum. It works for both longslit/mulitslit, coadded individual order spectrum of the Echelle data @@ -1562,6 +1566,8 @@ def coadd_qa(wave, flux, ivar, nused, gpm=None, tell=None, plot title qafile : str, optional QA file name + show_telluric : bool, optional + Show a telluric absorptoin model on top of the data if wavelengths cover > 9000A """ #TODO: This routine should take a parset @@ -1592,7 +1598,7 @@ def coadd_qa(wave, flux, ivar, nused, gpm=None, tell=None, ymin, ymax = get_ylim(flux, ivar, gpm) # Plot transmission - if (np.max(wave[gpm])>9000.0) and (tell is None): + if (np.max(wave[gpm])>9000.0) and (tell is None) and show_telluric: skytrans_file = data.get_skisim_filepath('atm_transmission_secz1.5_1.6mm.dat') skycat = np.genfromtxt(skytrans_file,dtype='float') scale = 0.8*ymax @@ -1851,8 +1857,6 @@ def spec_reject_comb(wave_grid, wave_grid_mid, waves_list, fluxes_list, ivars_li qdone = False while (not qdone) and (iter < maxiter_reject): # Compute the stack - #from IPython import embed - #embed() wave_stack, flux_stack, ivar_stack, gpm_stack, nused = compute_stack( wave_grid, waves_list, fluxes_list, ivars_list, utils.array_to_explist(this_gpms, nspec_list=nspec_list), weights_list) # Interpolate the individual spectra onto the wavelength grid of the stack. Use wave_grid_mid for this @@ -2022,10 +2026,9 @@ def combspec(waves, fluxes, ivars, gpms, sn_smooth_npix, ref_percentile=70.0, maxiter_scale=5, wave_grid_input=None, sigrej_scale=3.0, scale_method='auto', hand_scale=None, sn_min_polyscale=2.0, sn_min_medscale=0.5, - const_weights=False, maxiter_reject=5, sn_clip=30.0, + weight_method='auto', maxiter_reject=5, sn_clip=30.0, lower=3.0, upper=3.0, maxrej=None, qafile=None, title='', debug=False, - debug_scale=False, debug_order_stack=False, - debug_global_stack=False, show_scale=False, show=False, verbose=True): + debug_scale=False, show_scale=False, show=False, verbose=True): ''' Main driver routine for coadding longslit/multi-slit spectra. @@ -2100,8 +2103,9 @@ def combspec(waves, fluxes, ivars, gpms, sn_smooth_npix, maximum SNR for perforing median scaling sn_min_medscale : float, optional, default = 0.5 minimum SNR for perforing median scaling - const_weights : bool, optional - If True, apply constant weight + weight_method (str): + Weight method to use for coadding spectra (see + :func:`~pypeit.core.coadd.sn_weights`) for documentation. Default='auto' sn_clip: float, optional, default=30.0 Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio @@ -2157,6 +2161,12 @@ def combspec(waves, fluxes, ivars, gpms, sn_smooth_npix, spectrum on wave_stack wavelength grid. True=Good. shape=(ngrid,) ''' + + #debug_scale=True + #show_scale=True + #debug=True + #show=True + #from IPython import embed #embed() # We cast to float64 because of a bug in np.histogram @@ -2172,7 +2182,7 @@ def combspec(waves, fluxes, ivars, gpms, sn_smooth_npix, dwave=dwave, dv=dv, dloglam=dloglam, spec_samp_fact=spec_samp_fact) # Evaluate the sn_weights. This is done once at the beginning - rms_sn, weights = sn_weights(_fluxes, _ivars, gpms, sn_smooth_npix=sn_smooth_npix, const_weights=const_weights, verbose=verbose) + rms_sn, weights = sn_weights(_fluxes, _ivars, gpms, sn_smooth_npix=sn_smooth_npix, weight_method=weight_method, verbose=verbose) fluxes_scale, ivars_scale, scales, scale_method_used = scale_spec_stack( wave_grid, wave_grid_mid, _waves, _fluxes, _ivars, gpms, rms_sn, weights, ref_percentile=ref_percentile, maxiter_scale=maxiter_scale, sigrej_scale=sigrej_scale, scale_method=scale_method, hand_scale=hand_scale, @@ -2192,7 +2202,7 @@ def multi_combspec(waves, fluxes, ivars, masks, sn_smooth_npix=None, wave_method='linear', dwave=None, dv=None, dloglam=None, spec_samp_fact=1.0, wave_grid_min=None, wave_grid_max=None, ref_percentile=70.0, maxiter_scale=5, sigrej_scale=3.0, scale_method='auto', hand_scale=None, sn_min_polyscale=2.0, sn_min_medscale=0.5, - const_weights=False, maxiter_reject=5, sn_clip=30.0, lower=3.0, upper=3.0, + weight_method='auto', maxiter_reject=5, sn_clip=30.0, lower=3.0, upper=3.0, maxrej=None, qafile=None, debug=False, debug_scale=False, show_scale=False, show=False): """ @@ -2267,8 +2277,9 @@ def multi_combspec(waves, fluxes, ivars, masks, sn_smooth_npix=None, maximum SNR for perforing median scaling sn_min_medscale : float, optional minimum SNR for perforing median scaling - const_weights : `numpy.ndarray`_, optional - Constant weight factors specified + weight_method (str): + Weight method to use for coadding spectra (see + :func:`~pypeit.core.coadd.sn_weights`) for documentation. Default='auto' maxiter_reject : int, optional maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen @@ -2333,7 +2344,7 @@ def multi_combspec(waves, fluxes, ivars, masks, sn_smooth_npix=None, spec_samp_fact=spec_samp_fact, wave_grid_min=wave_grid_min, wave_grid_max=wave_grid_max, ref_percentile=ref_percentile, maxiter_scale=maxiter_scale, sigrej_scale=sigrej_scale, scale_method=scale_method, hand_scale=hand_scale, sn_min_medscale=sn_min_medscale, sn_min_polyscale=sn_min_polyscale, sn_smooth_npix=sn_smooth_npix, - const_weights=const_weights, maxiter_reject=maxiter_reject, sn_clip=sn_clip, lower=lower, upper=upper, + weight_method=weight_method, maxiter_reject=maxiter_reject, sn_clip=sn_clip, lower=lower, upper=upper, maxrej=maxrej, qafile=qafile, title='multi_combspec', debug=debug, debug_scale=debug_scale, show_scale=show_scale, show=show) @@ -2348,7 +2359,7 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se ref_percentile=70.0, maxiter_scale=5, niter_order_scale=3, sigrej_scale=3.0, scale_method='auto', hand_scale=None, sn_min_polyscale=2.0, sn_min_medscale=0.5, - sn_smooth_npix=None, const_weights=False, maxiter_reject=5, + maxiter_reject=5, sn_clip=30.0, lower=3.0, upper=3.0, maxrej=None, qafile=None, debug_scale=False, debug_order_stack=False, debug_global_stack=False, debug=False, @@ -2436,8 +2447,6 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se maximum SNR for perforing median scaling sn_min_medscale : float, optional, default = 0.5 minimum SNR for perforing median scaling - const_weights : bool, optional - If True, apply constant weight maxiter_reject : int, optional, default=5 maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen @@ -2573,17 +2582,17 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se # nspec, norder, nexp = shape # Decide how much to smooth the spectra by if this number was not passed in - nspec_good = [] - ngood = [] - if sn_smooth_npix is None: - # Loop over setups - for wave, norder, nexp in zip(waves_arr_setup, norders, nexps): - # This is the effective good number of spectral pixels in the stack - nspec_good.append(np.sum(wave > 1.0)) - ngood.append(norder*nexp) - nspec_eff = np.sum(nspec_good)/np.sum(ngood) - sn_smooth_npix = int(np.round(0.1 * nspec_eff)) - msgs.info('Using a sn_smooth_pix={:d} to decide how to scale and weight your spectra'.format(sn_smooth_npix)) + #nspec_good = [] + #ngood = [] + #if sn_smooth_npix is None: + # # Loop over setups + # for wave, norder, nexp in zip(waves_arr_setup, norders, nexps): + # # This is the effective good number of spectral pixels in the stack + # nspec_good.append(np.sum(wave > 1.0)) + # ngood.append(norder*nexp) + # nspec_eff = np.sum(nspec_good)/np.sum(ngood) + # sn_smooth_npix = int(np.round(0.1 * nspec_eff)) + # msgs.info('Using a sn_smooth_pix={:d} to decide how to scale and weight your spectra'.format(sn_smooth_npix)) # Create the setup lists waves_setup_list = [utils.echarr_to_echlist(wave)[0] for wave in waves_arr_setup] @@ -2607,8 +2616,8 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se rms_sn_setup_list = [] colors = [] for isetup in range(nsetups): - rms_sn_vec, _ = sn_weights(fluxes_setup_list[isetup], ivars_setup_list[isetup], gpms_setup_list[isetup], - sn_smooth_npix=sn_smooth_npix, const_weights=const_weights, verbose=verbose) + ## Need to feed have optional wave_min, wave_max here to deal with sources like high-z quasars? + rms_sn_vec, _ = calc_snr(fluxes_setup_list[isetup], ivars_setup_list[isetup], gpms_setup_list[isetup]) rms_sn = rms_sn_vec.reshape(norders[isetup], nexps[isetup]) mean_sn_ord = np.mean(rms_sn, axis=1) best_orders = np.argsort(mean_sn_ord)[::-1][0:_nbests[isetup]] @@ -2620,6 +2629,7 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se colors.append([setup_colors[isetup]]*norders[isetup]*nexps[isetup]) + # Create the waves_setup_list weights_setup_list = [utils.echarr_to_echlist(weight)[0] for weight in weights] diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 475db4f93a..2105e2d0a1 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -10,6 +10,7 @@ from astropy.coordinates import AltAz, SkyCoord from astropy.io import fits import scipy.optimize as opt +from scipy import signal from scipy.interpolate import interp1d import numpy as np @@ -428,31 +429,73 @@ def get_output_whitelight_filename(outfile): return os.path.splitext(outfile)[0] + "_whitelight.fits" -def get_whitelight_pixels(all_wave, min_wl, max_wl): +def get_whitelight_pixels(all_wave, all_slitid, min_wl, max_wl): """ Determine which pixels are included within the specified wavelength range Args: - all_wave (`numpy.ndarray`_): - The wavelength of each individual pixel + all_wave (`numpy.ndarray`_, list): + List of `numpy.ndarray`_ wavelength images. The length of the list is the number of spec2d frames. + Each element of the list contains a wavelength image that provides the wavelength at each pixel on + the detector, with shape is (nspec, nspat). + all_slitid (`numpy.ndarray`_, list): + List of `numpy.ndarray`_ slitid images. The length of the list is the number of spec2d frames. + Each element of the list contains a slitid image that provides the slit number at each pixel on + the detector, with shape (nspec, nspat). min_wl (float): Minimum wavelength to consider max_wl (float): Maximum wavelength to consider Returns: - :obj:`tuple`: A `numpy.ndarray`_ object with the indices of all_wave - that contain pixels within the requested wavelength range, and a float - with the wavelength range (i.e. maximum wavelength - minimum wavelength) + :obj:`tuple`: The first element of the tuple is a list of `numpy.ndarray`_ slitid images + (or a single `numpy.ndarray`_ slitid image if only one spec2d frame is provided), + shape is (nspec, nspat), where a zero value corresponds to an excluded pixel + (either outside the desired wavelength range, a bad pixel, a pixel not on the slit). + All other pixels have a value equal to the slit number. The second element of the tuple + is the wavelength difference between the maximum and minimum wavelength in the desired + wavelength range. """ - wavediff = np.max(all_wave) - np.min(all_wave) + # Check if lists or numpy arrays are input + list_inputs = [all_wave, all_slitid] + if all([isinstance(l, list) for l in list_inputs]): + numframes = len(all_wave) + if not all([len(l) == numframes for l in list_inputs]): + msgs.error("All input lists must have the same length") + # Store in the following variables + _all_wave, _all_slitid = all_wave, all_slitid + elif all([not isinstance(l, list) for l in list_inputs]): + _all_wave, _all_slitid = [all_wave], [all_slitid] + numframes = 1 + else: + msgs.error("The input lists must either all be lists (of the same length) or all be numpy arrays") + if max_wl < min_wl: + msgs.error("The maximum wavelength must be greater than the minimum wavelength") + # Initialise the output + out_slitid = [np.zeros(_all_slitid[0].shape, dtype=int) for _ in range(numframes)] + # Loop over all frames and find the pixels that are within the wavelength range if min_wl < max_wl: - ww = np.where((all_wave > min_wl) & (all_wave < max_wl)) - wavediff = max_wl - min_wl + # Loop over files and determine which pixels are within the wavelength range + for ff in range(numframes): + ww = np.where((_all_wave[ff] > min_wl) & (_all_wave[ff] < max_wl)) + out_slitid[ff][ww] = _all_slitid[ff][ww] else: - msgs.warn("Datacubes do not completely overlap in wavelength. Offsets may be unreliable...") - ww = (np.arange(all_wave.size),) - return ww, wavediff + msgs.warn("Datacubes do not completely overlap in wavelength.") + out_slitid = _all_slitid + min_wl, max_wl = None, None + for ff in range(numframes): + this_wave = _all_wave[ff][_all_slitid[ff] > 0] + tmp_min = np.min(this_wave) + tmp_max = np.max(this_wave) + if min_wl is None or tmp_min < min_wl: + min_wl = tmp_min + if max_wl is None or tmp_max > max_wl: + max_wl = tmp_max + # Determine the wavelength range + wavediff = max_wl - min_wl + # Need to return a single slitid image if only one frame, otherwise return a list of slitid images. + # Also return the wavelength difference + return out_slitid[0] if numframes == 1 else out_slitid, wavediff def get_whitelight_range(wavemin, wavemax, wl_range): @@ -562,7 +605,7 @@ def load_imageWCS(filename, ext=0): return image, imgwcs -def align_user_offsets(all_ra, all_dec, all_idx, ifu_ra, ifu_dec, ra_offset, dec_offset): +def align_user_offsets(ifu_ra, ifu_dec, ra_offset, dec_offset): """ Align the RA and DEC of all input frames, and then manually shift the cubes based on user-provided offsets. @@ -570,42 +613,37 @@ def align_user_offsets(all_ra, all_dec, all_idx, ifu_ra, ifu_dec, ra_offset, dec ra_offset should include the cos(dec) factor. Args: - all_ra (`numpy.ndarray`_): - A 1D array containing the RA values of each detector pixel of every frame. - all_dec (`numpy.ndarray`_): - A 1D array containing the Dec values of each detector pixel of every frame. - Same size as all_ra. - all_idx (`numpy.ndarray`_): - A 1D array containing an ID value for each detector frame (0-indexed). - Same size as all_ra. ifu_ra (`numpy.ndarray`_): A list of RA values of the IFU (one value per frame) ifu_dec (`numpy.ndarray`_): A list of Dec values of the IFU (one value per frame) ra_offset (`numpy.ndarray`_): A list of RA offsets to be applied to the input pixel values (one value per frame). - Note, the ra_offset MUST contain the cos(dec) factor. This is the number of arcseconds + Note, the ra_offset MUST contain the cos(dec) factor. This is the number of degrees on the sky that represents the telescope offset. dec_offset (`numpy.ndarray`_): A list of Dec offsets to be applied to the input pixel values (one value per frame). + This is the number of degrees on the sky that represents the telescope offset. Returns: - A tuple containing a new set of RA and Dec values that have been aligned. Both arrays - are of type `numpy.ndarray`_. + A tuple containing a new set of RA and Dec offsets for each frame. + Both arrays are of type `numpy.ndarray`_, and are in units of degrees. """ # First, translate all coordinates to the coordinates of the first frame # Note: You do not need cos(dec) here, this just overrides the IFU coordinate centre of each frame # The cos(dec) factor should be input by the user, and should be included in the self.opts['ra_offset'] ref_shift_ra = ifu_ra[0] - ifu_ra ref_shift_dec = ifu_dec[0] - ifu_dec - numfiles = ra_offset.size + numfiles = len(ra_offset) + out_ra_offsets = [0.0 for _ in range(numfiles)] + out_dec_offsets = [0.0 for _ in range(numfiles)] for ff in range(numfiles): # Apply the shift - all_ra[all_idx == ff] += ref_shift_ra[ff] + ra_offset[ff] / 3600.0 - all_dec[all_idx == ff] += ref_shift_dec[ff] + dec_offset[ff] / 3600.0 + out_ra_offsets[ff] = ref_shift_ra[ff] + ra_offset[ff] + out_dec_offsets[ff] = ref_shift_dec[ff] + dec_offset[ff] msgs.info("Spatial shift of cube #{0:d}:".format(ff + 1) + msgs.newline() + - "RA, DEC (arcsec) = {0:+0.3f} E, {1:+0.3f} N".format(ra_offset[ff], dec_offset[ff])) - return all_ra, all_dec + "RA, DEC (arcsec) = {0:+0.3f} E, {1:+0.3f} N".format(ra_offset[ff]*3600.0, dec_offset[ff]*3600.0)) + return out_ra_offsets, out_dec_offsets def set_voxel_sampling(spatscale, specscale, dspat=None, dwv=None): @@ -663,7 +701,41 @@ def set_voxel_sampling(spatscale, specscale, dspat=None, dwv=None): return _dspat, _dwv -def wcs_bounds(all_ra, all_dec, all_wave, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None): +def check_inputs(list_inputs): + """ + This function checks the inputs to several of the cube building routines, and makes sure they are all consistent. + Often, this is to make check if all inputs are lists of the same length, or if all inputs are 2D `numpy.ndarray`_. + The goal of the routine is to return a consistent set of lists of the input. + + Parameters + ---------- + list_inputs : :obj:`list` + A list of inputs to check. + + Returns + ------- + list_inputs : :obj:`list` + A list of inputs that have been checked for consistency. + """ + if all([isinstance(l, list) for l in list_inputs]): + # Several frames are being combined. Check the lists have the same length + numframes = len(list_inputs[0]) + if not all([len(l) == numframes for l in list_inputs]): + msgs.error("All input lists must have the same length") + # The inputs are good, return as is + return tuple(list_inputs) + elif all([not isinstance(l, list) for l in list_inputs]): + # Just a single frame - store as single element lists + ret_list = () + for l in list_inputs: + ret_list += ([l],) + return ret_list + else: + msgs.error("The input arguments should all be of type 'list', or all not be of type 'list':") + + +def wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, ra_offsets=None, dec_offsets=None, + ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None): """ Calculate the bounds of the WCS and the expected edges of the voxels, based on user-specified parameters or the extremities of the data. This is a @@ -672,15 +744,20 @@ def wcs_bounds(all_ra, all_dec, all_wave, ra_min=None, ra_max=None, dec_min=None Parameters ---------- - all_ra : `numpy.ndarray`_ - 1D flattened array containing the RA values of each pixel from all - spec2d files - all_dec : `numpy.ndarray`_ - 1D flattened array containing the DEC values of each pixel from all - spec2d files - all_wave : `numpy.ndarray`_ - 1D flattened array containing the wavelength values of each pixel from - all spec2d files + raImg : (`numpy.ndarray`_, list): + A list of 2D array containing the RA of each pixel, with shape (nspec, nspat) + decImg : (`numpy.ndarray`_, list): + A list of 2D array containing the Dec of each pixel, with shape (nspec, nspat) + waveImg (`numpy.ndarray`_, list): + A list of 2D array containing the wavelength of each pixel, with shape (nspec, nspat) + slitid_img_gpm : (`numpy.ndarray`_, list): + A list of 2D array containing the spat ID of each pixel, with shape (nspec, nspat). + A value of 0 indicates that the pixel is not on a slit. All other values indicate the + slit spatial ID. + ra_offsets : list, optional + A list of the RA offsets for each frame + dec_offsets : list, optional + A list of the Dec offsets for each frame ra_min : :obj:`float`, optional Minimum RA of the WCS ra_max : :obj:`float`, optional @@ -688,11 +765,11 @@ def wcs_bounds(all_ra, all_dec, all_wave, ra_min=None, ra_max=None, dec_min=None dec_min : :obj:`float`, optional Minimum Dec of the WCS dec_max : :obj:`float`, optional - Maximum RA of the WCS - wav_min : :obj:`float`, optional + Maximum Dec of the WCS + wave_min : :obj:`float`, optional Minimum wavelength of the WCS - wav_max : :obj:`float`, optional - Maximum RA of the WCS + wave_max : :obj:`float`, optional + Maximum wavelength of the WCS Returns ------- @@ -703,23 +780,59 @@ def wcs_bounds(all_ra, all_dec, all_wave, ra_min=None, ra_max=None, dec_min=None _dec_min : :obj:`float` Minimum Dec of the WCS _dec_max : :obj:`float` - Maximum RA of the WCS - _wav_min : :obj:`float` + Maximum Dec of the WCS + _wave_min : :obj:`float` Minimum wavelength of the WCS - _wav_max : :obj:`float` - Maximum RA of the WCS + _wave_max : :obj:`float` + Maximum wavelength of the WCS """ - # Setup the cube ranges - _ra_min = ra_min if ra_min is not None else np.min(all_ra) - _ra_max = ra_max if ra_max is not None else np.max(all_ra) - _dec_min = dec_min if dec_min is not None else np.min(all_dec) - _dec_max = dec_max if dec_max is not None else np.max(all_dec) - _wav_min = wave_min if wave_min is not None else np.min(all_wave) - _wav_max = wave_max if wave_max is not None else np.max(all_wave) - return _ra_min, _ra_max, _dec_min, _dec_max, _wav_min, _wav_max - - -def create_wcs(all_ra, all_dec, all_wave, dspat, dwave, + # Check if the ra_offsets and dec_offsets are specified + if ra_offsets is None or dec_offsets is None: + if isinstance(raImg, list): + ra_offsets = [0.0]*len(raImg) + dec_offsets = [0.0]*len(raImg) + else: + ra_offsets = 0.0 + dec_offsets = 0.0 + # Check the inputs + _raImg, _decImg, _waveImg, _slitid_img_gpm, _ra_offsets, _dec_offsets = \ + check_inputs([raImg, decImg, waveImg, slitid_img_gpm, ra_offsets, dec_offsets]) + numframes = len(_raImg) + + # Loop over the frames and get the bounds - start by setting the default values + _ra_min, _ra_max = ra_min, ra_max + _dec_min, _dec_max = dec_min, dec_max + _wave_min, _wave_max = wave_min, wave_max + for fr in range(numframes): + # Only do calculations if the min/max inputs are not specified + # Get the RA, Dec, and wavelength of the pixels on the slit + if ra_min is None or ra_max is None: + this_ra = _raImg[fr][_slitid_img_gpm[fr] > 0] + tmp_min, tmp_max = np.min(this_ra)+_ra_offsets[fr], np.max(this_ra)+_ra_offsets[fr] + if fr == 0 or tmp_min < _ra_min: + _ra_min = tmp_min + if fr == 0 or tmp_max > _ra_max: + _ra_max = tmp_max + if dec_min is None or dec_max is None: + this_dec = _decImg[fr][_slitid_img_gpm[fr] > 0] + tmp_min, tmp_max = np.min(this_dec)+_dec_offsets[fr], np.max(this_dec)+_dec_offsets[fr] + if fr == 0 or tmp_min < _dec_min: + _dec_min = tmp_min + if fr == 0 or tmp_max > _dec_max: + _dec_max = tmp_max + if wave_min is None or wave_max is None: + this_wave = _waveImg[fr][_slitid_img_gpm[fr] > 0] + tmp_min, tmp_max = np.min(this_wave), np.max(this_wave) + if fr == 0 or tmp_min < _wave_min: + _wave_min = tmp_min + if fr == 0 or tmp_max > _wave_max: + _wave_max = tmp_max + # Return the bounds + return _ra_min, _ra_max, _dec_min, _dec_max, _wave_min, _wave_max + + +def create_wcs(raImg, decImg, waveImg, slitid_img_gpm, dspat, dwave, + ra_offsets=None, dec_offsets=None, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, reference=None, collapse=False, equinox=2000.0, specname="PYP_SPEC"): """ @@ -728,20 +841,25 @@ def create_wcs(all_ra, all_dec, all_wave, dspat, dwave, Parameters ---------- - all_ra : `numpy.ndarray`_ - 1D flattened array containing the RA values of each pixel from all - spec2d files - all_dec : `numpy.ndarray`_ - 1D flattened array containing the DEC values of each pixel from all - spec2d files - all_wave : `numpy.ndarray`_ - 1D flattened array containing the wavelength values of each pixel from - all spec2d files + raImg : (`numpy.ndarray`_, list): + A list of 2D array containing the RA of each pixel, with shape (nspec, nspat) + decImg : (`numpy.ndarray`_, list): + A list of 2D array containing the Dec of each pixel, with shape (nspec, nspat) + waveImg (`numpy.ndarray`_, list): + A list of 2D array containing the wavelength of each pixel, with shape (nspec, nspat) + slitid_img_gpm : (`numpy.ndarray`_, list): + A list of 2D array containing the spat ID of each pixel, with shape (nspec, nspat). + A value of 0 indicates that the pixel is not on a slit. All other values indicate the + slit spatial ID. dspat : float Spatial size of each square voxel (in arcsec). The default is to use the values in cubepar. dwave : float Linear wavelength step of each voxel (in Angstroms) + ra_offsets : list, optional + List of RA offsets for each frame (degrees) + dec_offsets : list, optional + List of Dec offsets for each frame (degrees) ra_min : float, optional Minimum RA of the WCS (degrees) ra_max : float, optional @@ -774,28 +892,27 @@ def create_wcs(all_ra, all_dec, all_wave, dspat, dwave, reference_image : `numpy.ndarray`_ The reference image to be used for the cross-correlation. Can be None. """ - # Grab cos(dec) for convenience - cosdec = np.cos(np.mean(all_dec) * np.pi / 180.0) - # Setup the cube ranges - _ra_min, _ra_max, _dec_min, _dec_max, _wav_min, _wav_max = \ - wcs_bounds(all_ra, all_dec, all_wave, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, - wave_min=wave_min, wave_max=wave_max) + _ra_min, _ra_max, _dec_min, _dec_max, _wave_min, _wave_max = \ + wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, + ra_offsets=ra_offsets, dec_offsets=dec_offsets, + ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max) + + # Grab cos(dec) for convenience. Use the average of the min and max dec + cosdec = np.cos(0.5*(_dec_min+_dec_max) * np.pi / 180.0) # Number of voxels in each dimension numra = int((_ra_max - _ra_min) * cosdec / dspat) numdec = int((_dec_max - _dec_min) / dspat) - numwav = int(np.round((_wav_max - _wav_min) / dwave)) + numwav = int(np.round((_wave_max - _wave_min) / dwave)) # If a white light WCS is being generated, make sure there's only 1 wavelength bin if collapse: - _wav_min = np.min(all_wave) - _wav_max = np.max(all_wave) - dwave = _wav_max - _wav_min + dwave = _wave_max - _wave_min numwav = 1 # Generate a master WCS to register all frames - coord_min = [_ra_min, _dec_min, _wav_min] + coord_min = [_ra_min, _dec_min, _wave_min] coord_dlt = [dspat, dspat, dwave] # If a reference image is being used and a white light image is requested (collapse=True) update the celestial parts @@ -813,7 +930,7 @@ def create_wcs(all_ra, all_dec, all_wave, dspat, dwave, msgs.newline() + "Parameters of the WCS:" + msgs.newline() + "RA min = {0:f}".format(coord_min[0]) + msgs.newline() + "DEC min = {0:f}".format(coord_min[1]) + - msgs.newline() + "WAVE min, max = {0:f}, {1:f}".format(_wav_min, _wav_max) + + msgs.newline() + "WAVE min, max = {0:f}, {1:f}".format(_wave_min, _wave_max) + msgs.newline() + "Spaxel size = {0:f} arcsec".format(3600.0 * dspat) + msgs.newline() + "Wavelength step = {0:f} A".format(dwave) + msgs.newline() + "-" * 40) @@ -862,10 +979,10 @@ def generate_WCS(crval, cdelt, equinox=2000.0, name="PYP_SPEC"): return w -def compute_weights_frompix(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, dspat, dwv, mnmx_wv, all_wghts, - all_spatpos, all_specpos, all_spatid, all_tilts, all_slits, all_align, all_dar, +def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, dspat, dwv, mnmx_wv, wghtsImg, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, - sn_smooth_npix=None, relative_weights=False, reference_image=None, whitelight_range=None, + sn_smooth_npix=None, weight_method='auto', reference_image=None, whitelight_range=None, specname="PYPSPEC"): r""" Calculate wavelength dependent optimal weights. The weighting is currently @@ -873,192 +990,283 @@ def compute_weights_frompix(all_ra, all_dec, all_wave, all_sci, all_ivar, all_id first prepares a whitelight image, and then calls compute_weights() to determine the appropriate weights of each pixel. - Args: - all_ra (`numpy.ndarray`_): - 1D flattened array containing the RA values of each pixel from all - spec2d files - all_dec (`numpy.ndarray`_): - 1D flattened array containing the DEC values of each pixel from all - spec2d files - all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength values of each pixel - from all spec2d files - all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all - spec2d files - all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel - from all spec2d files - all_idx (`numpy.ndarray`_): - 1D flattened array containing an integer identifier indicating which - spec2d file each pixel originates from. For example, a 0 would - indicate that a pixel originates from the first spec2d frame listed - in the input file. a 1 would indicate that this pixel originates - from the second spec2d file, and so forth. - dspat (float): - The size of each spaxel on the sky (in degrees) - dwv (float): - The size of each wavelength pixel (in Angstroms) - mnmx_wv (`numpy.ndarray`_): - The minimum and maximum wavelengths of every slit and frame. The shape is (Nframes, Nslits, 2), - The minimum and maximum wavelengths are stored in the [:,:,0] and [:,:,1] indices, respectively. - all_wghts (`numpy.ndarray`_): - 1D flattened array containing the weights of each pixel to be used - in the combination - all_spatpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spatial direction - all_specpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spectral direction - all_spatid (`numpy.ndarray`_): - 1D flattened array containing the spatid of each pixel - tilts (`numpy.ndarray`_, list): - 2D wavelength tilts frame, or a list of tilt frames (see all_idx) - slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): - Information stored about the slits, or a list of SlitTraceSet (see - all_idx) - astrom_trans (:class:`~pypeit.alignframe.AlignmentSplines`, list): - A Class containing the transformation between detector pixel - coordinates and WCS pixel coordinates, or a list of Alignment - Splines (see all_idx) - all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): - A Class containing the DAR correction information, or a list of DARcorrection - classes. If a list, it must be the same length as astrom_trans. - ra_min (float, optional): - Minimum RA of the WCS (degrees) - ra_max (float, optional): - Maximum RA of the WCS (degrees) - dec_min (float, optional): - Minimum Dec of the WCS (degrees) - dec_max (float, optional): - Maximum Dec of the WCS (degrees) - wave_min (float, optional): - Minimum wavelength of the WCS (degrees) - wave_max (float, optional): - Maximum wavelength of the WCS (degrees) - sn_smooth_npix (float, optional): - Number of pixels used for determining smoothly varying S/N ratio - weights. This is currently not required, since a relative weighting - scheme with a polynomial fit is used to calculate the S/N weights. - relative_weights (bool, optional): - Calculate weights by fitting to the ratio of spectra? - reference_image (`numpy.ndarray`_): - Reference image to use for the determination of the highest S/N spaxel in the image. - specname (str): - Name of the spectrograph + Parameters + ---------- - Returns: - `numpy.ndarray`_ : a 1D array the same size as all_sci, containing - relative wavelength dependent weights of each input pixel. + raImg : `numpy.ndarray`_, list + A list of 2D array containing the RA of each pixel, with shape (nspec, nspat) + decImg : `numpy.ndarray`_, list + A list of 2D array containing the Dec of each pixel, with shape (nspec, nspat) + waveImg : `numpy.ndarray`_, list + A list of 2D array containing the wavelength of each pixel, with shape (nspec, nspat) + sciImg : `numpy.ndarray`_, list + A list of 2D array containing the science image of each pixel, with shape (nspec, nspat) + ivarImg : `numpy.ndarray`_, list + A list of 2D array containing the inverse variance image of each pixel, with shape (nspec, nspat) + slitidImg : `numpy.ndarray`_, list + A list of 2D array containing the slit ID of each pixel, with shape (nspec, nspat) + dspat : float + The size of each spaxel on the sky (in degrees) + dwv : float + The size of each wavelength pixel (in Angstroms) + mnmx_wv : `numpy.ndarray`_ + The minimum and maximum wavelengths of every slit and frame. The shape is (Nframes, Nslits, 2), + The minimum and maximum wavelengths are stored in the [:,:,0] and [:,:,1] indices, respectively. + wghtsImg : `numpy.ndarray`_, list + A list of 2D array containing the weights of each pixel, with shape (nspec, nspat) + all_wcs : `astropy.wcs.WCS`_, list + A list of WCS objects, one for each frame. + all_tilts : `numpy.ndarray`_, list + 2D wavelength tilts frame, or a list of tilt frames + all_slits : :class:`~pypeit.slittrace.SlitTraceSet`, list + Information stored about the slits, or a list of SlitTraceSet objects + all_align : :class:`~pypeit.alignframe.AlignmentSplines`, list + A Class containing the transformation between detector pixel + coordinates and WCS pixel coordinates, or a list of Alignment + Splines. + all_dar : :class:`~pypeit.coadd3d.DARcorrection`, list + A Class containing the DAR correction information, or a list of DARcorrection + classes. If a list, it must be the same length as astrom_trans. + ra_offsets : float, list + RA offsets for each frame in units of degrees + dec_offsets : float, list + Dec offsets for each frame in units of degrees + ra_min : float, optional + Minimum RA of the WCS (degrees) + ra_max : float, optional + Maximum RA of the WCS (degrees) + dec_min : float, optional + Minimum Dec of the WCS (degrees) + dec_max : float, optional + Maximum Dec of the WCS (degrees) + wave_min : float, optional + Minimum wavelength of the WCS (degrees) + wave_max : float, optional + Maximum wavelength of the WCS (degrees) + sn_smooth_npix : float, optional + Number of pixels used for determining smoothly varying S/N ratio + weights. This is currently not required, since a relative weighting + scheme with a polynomial fit is used to calculate the S/N weights. + weight_method : `str`, optional + Weight method to be used in :func:`~pypeit.coadd.sn_weights`. + Options are ``'auto'``, ``'constant'``, ``'uniform'``, ``'wave_dependent'``, ``'relative'``, or + ``'ivar'``. The default is ``'auto'``. Behavior is as follows: + + - ``'auto'``: Use constant weights if rms_sn < 3.0, otherwise + use wavelength dependent. + + - ``'constant'``: Constant weights based on rms_sn**2 + + - ``'uniform'``: Uniform weighting. + + - ``'wave_dependent'``: Wavelength dependent weights will be + used irrespective of the rms_sn ratio. This option will not + work well at low S/N ratio although it is useful for objects + where only a small fraction of the spectral coverage has high + S/N ratio (like high-z quasars). + + - ``'relative'``: Calculate weights by fitting to the ratio of + spectra? Note, relative weighting will only work well when + there is at least one spectrum with a reasonable S/N, and a + continuum. RJC note - This argument may only be better when + the object being used has a strong continuum + emission lines. + The reference spectrum is assigned a value of 1 for all + wavelengths, and the weights of all other spectra will be + determined relative to the reference spectrum. This is + particularly useful if you are dealing with highly variable + spectra (e.g. emission lines) and require a precision better + than ~1 per cent. + + - ``'ivar'``: Use inverse variance weighting. This is not well + tested and should probably be deprecated. + + reference_image : `numpy.ndarray`_ + Reference image to use for the determination of the highest S/N spaxel in the image. + specname : str + Name of the spectrograph + + Returns + ------- + weights : `numpy.ndarray`_ + a 1D array the same size as all_sci, containing relative wavelength + dependent weights of each input pixel. """ # Find the wavelength range where all frames overlap min_wl, max_wl = get_whitelight_range(np.max(mnmx_wv[:, :, 0]), # The max blue wavelength np.min(mnmx_wv[:, :, 1]), # The min red wavelength whitelight_range) # The user-specified values (if any) # Get the good white light pixels - ww, wavediff = get_whitelight_pixels(all_wave, min_wl, max_wl) + slitid_img_gpm, wavediff = get_whitelight_pixels(waveImg, slitidImg, min_wl, max_wl) # Generate the WCS image_wcs, voxedge, reference_image = \ - create_wcs(all_ra, all_dec, all_wave, dspat, wavediff, + create_wcs(raImg, decImg, waveImg, slitid_img_gpm, dspat, wavediff, + ra_offsets=ra_offsets, dec_offsets=dec_offsets, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max, - reference=reference_image, collapse=True, equinox=2000.0, - specname=specname) + reference=reference_image, collapse=True, equinox=2000.0, specname=specname) # Generate the white light image # NOTE: hard-coding subpixel=1 in both directions for speed, and combining into a single image - wl_full = generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, - all_spatpos, all_specpos, all_spatid, all_tilts, all_slits, all_align, all_dar, - voxedge, all_idx=all_idx, spec_subpixel=1, spat_subpixel=1, combine=True) - # Compute the weights - return compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, wl_full[:, :, 0], dspat, dwv, - sn_smooth_npix=sn_smooth_npix, relative_weights=relative_weights) - + wl_full = generate_image_subpixel(image_wcs, voxedge, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtsImg, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + spec_subpixel=1, spat_subpixel=1, slice_subpixel=1, combine=True) -def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, whitelight_img, dspat, dwv, - sn_smooth_npix=None, relative_weights=False): + # Compute the weights + return compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + wl_full[:, :, 0], dspat, dwv, + ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, + sn_smooth_npix=sn_smooth_npix, weight_method=weight_method) + + +def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + whitelight_img, dspat, dwv, + ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, + sn_smooth_npix=None, weight_method='auto'): r""" Calculate wavelength dependent optimal weights. The weighting is currently based on a relative :math:`(S/N)^2` at each wavelength - Args: - all_ra (`numpy.ndarray`_): - 1D flattened array containing the RA values of each pixel from all - spec2d files - all_dec (`numpy.ndarray`_): - 1D flattened array containing the DEC values of each pixel from all - spec2d files - all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength values of each pixel - from all spec2d files - all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all - spec2d files - all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel - from all spec2d files - all_idx (`numpy.ndarray`_): - 1D flattened array containing an integer identifier indicating which - spec2d file each pixel originates from. For example, a 0 would - indicate that a pixel originates from the first spec2d frame listed - in the input file. a 1 would indicate that this pixel originates - from the second spec2d file, and so forth. - whitelight_img (`numpy.ndarray`_): - A 2D array containing a whitelight image, that was created with the - input ``all_`` arrays. - dspat (float): - The size of each spaxel on the sky (in degrees) - dwv (float): - The size of each wavelength pixel (in Angstroms) - sn_smooth_npix (float, optional): - Number of pixels used for determining smoothly varying S/N ratio - weights. This is currently not required, since a relative weighting - scheme with a polynomial fit is used to calculate the S/N weights. - relative_weights (bool, optional): - Calculate weights by fitting to the ratio of spectra? + Parameters + ---------- - Returns: - `numpy.ndarray`_ : a 1D array the same size as all_sci, containing - relative wavelength dependent weights of each input pixel. + raImg : `numpy.ndarray`_, list + A list of 2D array containing the RA of each pixel, with shape (nspec, nspat) + decImg : `numpy.ndarray`_, list + A list of 2D array containing the Dec of each pixel, with shape (nspec, nspat) + waveImg : `numpy.ndarray`_, list + A list of 2D array containing the wavelength of each pixel, with shape (nspec, nspat) + sciImg : `numpy.ndarray`_, list + A list of 2D array containing the science image of each pixel, with shape (nspec, nspat) + ivarImg : `numpy.ndarray`_, list + A list of 2D array containing the inverse variance image of each pixel, with shape (nspec, nspat) + slitidImg : `numpy.ndarray`_, list + A list of 2D array containing the slit ID of each pixel, with shape (nspec, nspat) + all_wcs : `astropy.wcs.WCS`_, list + A list of WCS objects, one for each frame. + all_tilts : `numpy.ndarray`_, list + 2D wavelength tilts frame, or a list of tilt frames + all_slits : :class:`~pypeit.slittrace.SlitTraceSet`, list + Information stored about the slits, or a list of SlitTraceSet objects + all_align : :class:`~pypeit.alignframe.AlignmentSplines`, list + A Class containing the transformation between detector pixel + coordinates and WCS pixel coordinates, or a list of Alignment + Splines. + all_dar : :class:`~pypeit.coadd3d.DARcorrection`, list + A Class containing the DAR correction information, or a list of DARcorrection + classes. If a list, it must be the same length as astrom_trans. + ra_offsets : float, list + RA offsets for each frame in units of degrees + dec_offsets : float, list + Dec offsets for each frame in units of degrees + whitelight_img : `numpy.ndarray`_ + A 2D array containing a white light image, that was created with the + input ``all`` arrays. + dspat : float + The size of each spaxel on the sky (in degrees) + dwv : float + The size of each wavelength pixel (in Angstroms) + sn_smooth_npix : float, optional + Number of pixels used for determining smoothly varying S/N ratio + weights. This is currently not required, since a relative weighting + scheme with a polynomial fit is used to calculate the S/N weights. + weight_method : `str`, optional + Weight method to be used in :func:`~pypeit.coadd.sn_weights`. + Options are ``'auto'``, ``'constant'``, ``'uniform'``, ``'wave_dependent'``, ``'relative'``, or + ``'ivar'``. The default is ``'auto'``. Behavior is as follows: + + - ``'auto'``: Use constant weights if rms_sn < 3.0, otherwise + use wavelength dependent. + + - ``'constant'``: Constant weights based on rms_sn**2 + + - ``'uniform'``: Uniform weighting. + + - ``'wave_dependent'``: Wavelength dependent weights will be + used irrespective of the rms_sn ratio. This option will not + work well at low S/N ratio although it is useful for objects + where only a small fraction of the spectral coverage has high + S/N ratio (like high-z quasars). + + - ``'relative'``: Calculate weights by fitting to the ratio of + spectra? Note, relative weighting will only work well when + there is at least one spectrum with a reasonable S/N, and a + continuum. RJC note - This argument may only be better when + the object being used has a strong continuum + emission lines. + The reference spectrum is assigned a value of 1 for all + wavelengths, and the weights of all other spectra will be + determined relative to the reference spectrum. This is + particularly useful if you are dealing with highly variable + spectra (e.g. emission lines) and require a precision better + than ~1 per cent. + + - ``'ivar'``: Use inverse variance weighting. This is not well + tested and should probably be deprecated. + + Returns + ------- + all_wghts: list + Either a 2D `numpy.ndarray`_ or a list of 2D `numpy.ndarray`_ arrays + containing the optimal weights of each pixel for all frames, with shape + (nspec, nspat). """ msgs.info("Calculating the optimal weights of each pixel") - # Determine number of files - numfiles = np.unique(all_idx).size + # Check the inputs for combinations of lists or not, and then determine the number of frames + _raImg, _decImg, _waveImg, _sciImg, _ivarImg, _slitidImg, \ + _all_wcs, _all_tilts, _all_slits, _all_align, _all_dar, _ra_offsets, _dec_offsets = \ + check_inputs([raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets]) + numframes = len(_sciImg) + + # If there's only one frame, use uniform weighting + if numframes == 1: + msgs.warn("Only one frame provided. Using uniform weighting.") + return np.ones_like(sciImg) + + # Check the WCS bounds + _ra_min, _ra_max, _dec_min, _dec_max, _wave_min, _wave_max = \ + wcs_bounds(_raImg, _decImg, _waveImg, _slitidImg, ra_offsets=_ra_offsets, dec_offsets=_dec_offsets, + ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max) # Find the location of the object with the highest S/N in the combined white light image - idx_max = np.unravel_index(np.argmax(whitelight_img), whitelight_img.shape) + med_filt_whitelight = signal.medfilt2d(whitelight_img, kernel_size=3) + idx_max = np.unravel_index(np.argmax(med_filt_whitelight), med_filt_whitelight.shape) + # TODO: Taking the maximum pixel of the whitelight image is extremely brittle to the case where + # their are hot pixels in the white light image, which there are plenty of since the edges of the slits are very + # poorly behaved. + #idx_max = np.unravel_index(np.argmax(whitelight_img), whitelight_img.shape) msgs.info("Highest S/N object located at spaxel (x, y) = {0:d}, {1:d}".format(idx_max[0], idx_max[1])) # Generate a 2D WCS to register all frames - coord_min = [np.min(all_ra), np.min(all_dec), np.min(all_wave)] + coord_min = [_ra_min, _dec_min, _wave_min] coord_dlt = [dspat, dspat, dwv] whitelightWCS = generate_WCS(coord_min, coord_dlt) + wcs_scale = (1.0 * whitelightWCS.spectral.wcs.cunit[0]).to(units.Angstrom).value # Ensures the WCS is in Angstroms # Make the bin edges to be at +/- 1 pixels around the maximum (i.e. summing 9 pixels total) - numwav = int((np.max(all_wave) - np.min(all_wave)) / dwv) + numwav = int((_wave_max - _wave_min) / dwv) xbins = np.array([idx_max[0]-1, idx_max[0]+2]) - 0.5 ybins = np.array([idx_max[1]-1, idx_max[1]+2]) - 0.5 spec_bins = np.arange(1 + numwav) - 0.5 bins = (xbins, ybins, spec_bins) # Extract the spectrum of the highest S/N object - flux_stack = np.zeros((numwav, numfiles)) - ivar_stack = np.zeros((numwav, numfiles)) - for ff in range(numfiles): - msgs.info("Extracting spectrum of highest S/N detection from frame {0:d}/{1:d}".format(ff + 1, numfiles)) - ww = (all_idx == ff) - # Extract the spectrum - pix_coord = whitelightWCS.wcs_world2pix(np.vstack((all_ra[ww], all_dec[ww], all_wave[ww] * 1.0E-10)).T, 0) - spec, edges = np.histogramdd(pix_coord, bins=bins, weights=all_sci[ww]) - var, edges = np.histogramdd(pix_coord, bins=bins, weights=1/all_ivar[ww]) - norm, edges = np.histogramdd(pix_coord, bins=bins) - normspec = (norm > 0) / (norm + (norm == 0)) - var_spec = var[0, 0, :] - ivar_spec = (var_spec > 0) / (var_spec + (var_spec == 0)) - # Calculate the S/N in a given spectral bin - flux_stack[:, ff] = spec[0, 0, :] * np.sqrt(normspec) # Note: sqrt(nrmspec), is because we want the S/N in a _single_ pixel (i.e. not spectral bin) - ivar_stack[:, ff] = ivar_spec - + flux_stack = np.zeros((numwav, numframes)) + ivar_stack = np.zeros((numwav, numframes)) + for ff in range(numframes): + msgs.info("Extracting spectrum of highest S/N detection from frame {0:d}/{1:d}".format(ff + 1, numframes)) + flxcube, sigcube, bpmcube, wave = \ + generate_cube_subpixel(whitelightWCS, bins, _sciImg[ff], _ivarImg[ff], _waveImg[ff], + _slitidImg[ff], np.ones(_sciImg[ff].shape), _all_wcs[ff], + _all_tilts[ff], _all_slits[ff], _all_align[ff], _all_dar[ff], + _ra_offsets[ff], _dec_offsets[ff], + spec_subpixel=1, spat_subpixel=1, slice_subpixel=1) + # Store the flux and ivar spectra of the highest S/N object. + # TODO :: This is the flux per spectral pixel, and not per detector pixel. Is this correct? + flux_stack[:, ff] = flxcube[:, 0, 0] + ivar_stack[:, ff] = utils.inverse(sigcube[:, 0, 0])**2 + + # Mask out any pixels that are zero in the flux or ivar stack mask_stack = (flux_stack != 0.0) & (ivar_stack != 0.0) # Obtain a wavelength of each pixel wcs_res = whitelightWCS.wcs_pix2world(np.vstack((np.zeros(numwav), np.zeros(numwav), np.arange(numwav))).T, 0) @@ -1068,53 +1276,48 @@ def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, white if sn_smooth_npix is None: sn_smooth_npix = int(np.round(0.1 * wave_spec.size)) rms_sn, weights = coadd.sn_weights(utils.array_to_explist(flux_stack), utils.array_to_explist(ivar_stack), utils.array_to_explist(mask_stack), - sn_smooth_npix=sn_smooth_npix, relative_weights=relative_weights) + sn_smooth_npix=sn_smooth_npix, weight_method=weight_method) # Because we pass back a weights array, we need to interpolate to assign each detector pixel a weight - all_wghts = np.ones(all_idx.size) - for ff in range(numfiles): - ww = (all_idx == ff) - all_wghts[ww] = interp1d(wave_spec, weights[ff], kind='cubic', - bounds_error=False, fill_value="extrapolate")(all_wave[ww]) + all_wghts = [np.ones(_sciImg[0].shape) for _ in range(numframes)] + for ff in range(numframes): + ww = (slitidImg[ff] > 0) + all_wghts[ff][ww] = interp1d(wave_spec, weights[ff], kind='cubic', + bounds_error=False, fill_value="extrapolate")(waveImg[ff][ww]) msgs.info("Optimal weighting complete") return all_wghts -def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, - all_spatpos, all_specpos, all_spatid, tilts, slits, astrom_trans, all_dar, bins, - all_idx=None, spec_subpixel=10, spat_subpixel=10, combine=False): +def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, combine=False): """ Generate a white light image from the input pixels Args: image_wcs (`astropy.wcs.WCS`_): World coordinate system to use for the white light images. - all_ra (`numpy.ndarray`_): - 1D flattened array containing the right ascension of each pixel - (units = degrees) - all_dec (`numpy.ndarray`_): - 1D flattened array containing the declination of each pixel (units = - degrees) - all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength of each pixel (units = - Angstroms) - all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all - spec2d files - all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel - from all spec2d files - all_wghts (`numpy.ndarray`_): - 1D flattened array containing the weights of each pixel to be used - in the combination - all_spatpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spatial direction - all_specpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spectral direction - all_spatid (`numpy.ndarray`_): - 1D flattened array containing the spatid of each pixel + bins (tuple): + A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial + and z wavelength coordinates + sciImg (`numpy.ndarray`_, list): + A list of 2D science images, or a single 2D image containing the + science data. + ivarImg (`numpy.ndarray`_, list): + A list of 2D inverse variance images, or a single 2D image + containing the inverse variance data. + waveImg (`numpy.ndarray`_, list): + A list of 2D wavelength images, or a single 2D image containing the + wavelength data. + slitid_img_gpm (`numpy.ndarray`_, list): + A list of 2D slit ID images, or a single 2D image containing the + slit ID data. + wghtImg (`numpy.ndarray`_, list): + A list of 2D weight images, or a single 2D image containing the + weight data. + all_wcs (`astropy.wcs.WCS`_, list): + A list of WCS objects, or a single WCS object containing the WCS + information of each image. tilts (`numpy.ndarray`_, list): 2D wavelength tilts frame, or a list of tilt frames (see all_idx) slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): @@ -1127,27 +1330,28 @@ def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_i all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): A Class containing the DAR correction information, or a list of DARcorrection classes. If a list, it must be the same length as astrom_trans. - bins (tuple): - A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial - and z wavelength coordinates - all_idx (`numpy.ndarray`_, optional): - If tilts, slits, and astrom_trans are lists, this should contain a - 1D flattened array, of the same length as all_sci, containing the - index the tilts, slits, and astrom_trans lists that corresponds to - each pixel. Note that, in this case all of these lists need to be - the same length. + ra_offset (:obj:`float`, list): + The RA offset to apply to each image, or a list of RA offsets. + dec_offset (:obj:`float`, list): + The DEC offset to apply to each image, or a list of DEC offsets. spec_subpixel (:obj:`int`, optional): What is the subpixellation factor in the spectral direction. Higher values give more reliable results, but note that the time required - goes as (``spec_subpixel * spat_subpixel``). The default value is 5, - which divides each detector pixel into 5 subpixels in the spectral - direction. + goes as (``spec_subpixel * spat_subpixel * slice_subpixel``). The + default value is 5, which divides each detector pixel into 5 subpixels + in the spectral direction. spat_subpixel (:obj:`int`, optional): What is the subpixellation factor in the spatial direction. Higher values give more reliable results, but note that the time required - goes as (``spec_subpixel * spat_subpixel``). The default value is 5, - which divides each detector pixel into 5 subpixels in the spatial - direction. + goes as (``spec_subpixel * spat_subpixel * slice_subpixel``). The + default value is 5, which divides each detector pixel into 5 subpixels + in the spatial direction. + slice_subpixel (:obj:`int`, optional): + What is the subpixellation factor in the slice direction. Higher + values give more reliable results, but note that the time required + goes as (``spec_subpixel * spat_subpixel * slice_subpixel``). The + default value is 5, which divides each IFU slice into 5 subpixels + in the slice direction. combine (:obj:`bool`, optional): If True, all of the input frames will be combined into a single output. Otherwise, individual images will be generated. @@ -1156,84 +1360,74 @@ def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_i `numpy.ndarray`_: The white light images for all frames """ # Perform some checks on the input -- note, more complete checks are performed in subpixellate() - _all_idx = np.zeros(all_sci.size) if all_idx is None else all_idx - if combine: - numfr = 1 - else: - numfr = np.unique(_all_idx).size - if len(tilts) != numfr or len(slits) != numfr or len(astrom_trans) != numfr or len(all_dar) != numfr: - msgs.error("The following arguments must be the same length as the expected number of frames to be combined:" - + msgs.newline() + "tilts, slits, astrom_trans, all_dar") + _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ + check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) + numframes = len(_sciImg) + # Prepare the array of white light images to be stored numra = bins[0].size-1 numdec = bins[1].size-1 - all_wl_imgs = np.zeros((numra, numdec, numfr)) + all_wl_imgs = np.zeros((numra, numdec, numframes)) # Loop through all frames and generate white light images - for fr in range(numfr): - msgs.info(f"Creating image {fr+1}/{numfr}") + for fr in range(numframes): + msgs.info(f"Creating image {fr+1}/{numframes}") if combine: # Subpixellate - img, _, _ = subpixellate(image_wcs, all_ra, all_dec, all_wave, - all_sci, all_ivar, all_wghts, all_spatpos, - all_specpos, all_spatid, tilts, slits, astrom_trans, all_dar, bins, - spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, all_idx=_all_idx) + img, _, _ = subpixellate(image_wcs, bins, _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, + _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset, + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel) else: - ww = np.where(_all_idx == fr) # Subpixellate - img, _, _ = subpixellate(image_wcs, all_ra[ww], all_dec[ww], all_wave[ww], - all_sci[ww], all_ivar[ww], all_wghts[ww], all_spatpos[ww], - all_specpos[ww], all_spatid[ww], tilts[fr], slits[fr], astrom_trans[fr], - all_dar[fr], bins, spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) + img, _, _ = subpixellate(image_wcs, bins, _sciImg[fr], _ivarImg[fr], _waveImg[fr], _slitid_img_gpm[fr], _wghtImg[fr], + _all_wcs[fr], _tilts[fr], _slits[fr], _astrom_trans[fr], _all_dar[fr], _ra_offset[fr], _dec_offset[fr], + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel) all_wl_imgs[:, :, fr] = img[:, :, 0] # Return the constructed white light images return all_wl_imgs -def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, - all_spatpos, all_specpos, all_spatid, tilts, slits, astrom_trans, all_dar, bins, - all_idx=None, spec_subpixel=10, spat_subpixel=10, overwrite=False, - whitelight_range=None, debug=False): +def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, + all_wcs, tilts, slits, astrom_trans, all_dar, + ra_offset, dec_offset, + spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, + overwrite=False, outfile=None, whitelight_range=None, debug=False): """ Save a datacube using the subpixel algorithm. Refer to the subpixellate() docstring for further details about this algorithm Args: - outfile (str): - Filename to be used to save the datacube output_wcs (`astropy.wcs.WCS`_): Output world coordinate system. - all_ra (`numpy.ndarray`_): - 1D flattened array containing the right ascension of each pixel - (units = degrees) - all_dec (`numpy.ndarray`_): - 1D flattened array containing the declination of each pixel (units = - degrees) - all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength of each pixel (units = - Angstroms) - all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all - spec2d files - all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel - from all spec2d files - all_wghts (`numpy.ndarray`_): - 1D flattened array containing the weights of each pixel to be used - in the combination - all_spatpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spatial direction - all_specpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spectral direction - all_spatid (`numpy.ndarray`_): - 1D flattened array containing the spatid of each pixel - tilts (`numpy.ndarray`_, list): - 2D wavelength tilts frame, or a list of tilt frames (see all_idx) - slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): - Information stored about the slits, or a list of SlitTraceSet (see - all_idx) + bins (tuple): + A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial + and z wavelength coordinates + sciImg (`numpy.ndarray`_, list): + A list of 2D array containing the counts of each pixel. If a list, + the shape of each numpy array is (nspec, nspat). + ivarImg (`numpy.ndarray`_, list): + A list of 2D array containing the inverse variance of each pixel. If a list, + the shape of each numpy array is (nspec, nspat). + waveImg (`numpy.ndarray`_, list): + A list of 2D array containing the wavelength of each pixel. If a list, + the shape of each numpy array is (nspec, nspat). + slitid_img_gpm (`numpy.ndarray`_, list): + A list of 2D array containing the slitmask of each pixel. If a list, + the shape of each numpy array is (nspec, nspat). + A zero value indicates that a pixel is either not on a slit or it is a bad pixel. + All other values are the slit spatial ID number. + wghtImg (`numpy.ndarray`_, list): + A list of 2D array containing the weights of each pixel to be used in the + combination. If a list, the shape of each numpy array is (nspec, nspat). + all_wcs (`astropy.wcs.WCS`_, list): + A list of `astropy.wcs.WCS`_ objects, one for each spec2d file. + tilts (list): + A list of `numpy.ndarray`_ objects, one for each spec2d file, + containing the tilts of each pixel. The shape of each numpy array + is (nspec, nspat). + slits (:class:`pypeit.slittrace.SlitTraceSet`, list): + A list of :class:`pypeit.slittrace.SlitTraceSet` objects, one for each + spec2d file, containing the properties of the slit for each spec2d file astrom_trans (:class:`~pypeit.alignframe.AlignmentSplines`, list): A Class containing the transformation between detector pixel coordinates and WCS pixel coordinates, or a list of Alignment @@ -1241,15 +1435,10 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): A Class containing the DAR correction information, or a list of DARcorrection classes. If a list, it must be the same length as astrom_trans. - bins (tuple): - A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial - and z wavelength coordinates - all_idx (`numpy.ndarray`_, optional): - If tilts, slits, and astrom_trans are lists, this should contain a - 1D flattened array, of the same length as all_sci, containing the - index the tilts, slits, and astrom_trans lists that corresponds to - each pixel. Note that, in this case all of these lists need to be - the same length. + ra_offset (float, list): + A float or list of floats containing the RA offset of each spec2d file + dec_offset (float, list): + A float or list of floats containing the DEC offset of each spec2d file spec_subpixel (int, optional): What is the subpixellation factor in the spectral direction. Higher values give more reliable results, but note that the time required @@ -1262,8 +1451,15 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s goes as (``spec_subpixel * spat_subpixel``). The default value is 5, which divides each detector pixel into 5 subpixels in the spatial direction. + slice_subpixel (int, optional): + What is the subpixellation factor in the slice direction. Higher + values give more reliable results, but note that the time required + goes as (``slice_subpixel``). The default value is 5, which divides + each IFU slice into 5 subslices in the slice direction. overwrite (bool, optional): If True, the output cube will be overwritten. + outfile (str, optional): + Filename to be used to save the datacube whitelight_range (None, list, optional): A two element list that specifies the minimum and maximum wavelengths (in Angstroms) to use when constructing the white light @@ -1279,18 +1475,28 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s Returns: :obj:`tuple`: Four `numpy.ndarray`_ objects containing - (1) the datacube generated from the subpixellated inputs, - (2) the corresponding error cube (standard deviation), - (3) the corresponding bad pixel mask cube, and - (4) a 1D array containing the wavelength at each spectral coordinate of the datacube. + (1) the datacube generated from the subpixellated inputs. The shape of + the datacube is (nwave, nspat1, nspat2). + (2) the corresponding error cube (standard deviation). The shape of the + error cube is (nwave, nspat1, nspat2). + (3) the corresponding bad pixel mask cube. The shape of the bad pixel + mask cube is (nwave, nspat1, nspat2). + (4) a 1D array containing the wavelength at each spectral coordinate of the datacube. The + shape of the wavelength array is (nwave,). """ + # Check the inputs + if whitelight_range is not None or debug: + if outfile is None: + msgs.error("Must provide an outfile name if either whitelight_range or debug are set") + # Prepare the header, and add the unit of flux to the header hdr = output_wcs.to_header() # Subpixellate - subpix = subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, all_spatpos, all_specpos, - all_spatid, tilts, slits, astrom_trans, all_dar, bins, all_idx=all_idx, - spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, debug=debug) + subpix = subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel, + debug=debug) # Extract the variables that we need if debug: flxcube, varcube, bpmcube, residcube = subpix @@ -1313,9 +1519,9 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s whitelight_wcs = output_wcs.celestial # Determine the wavelength range of the whitelight image if whitelight_range[0] is None: - whitelight_range[0] = np.min(all_wave) + whitelight_range[0] = wave[0] if whitelight_range[1] is None: - whitelight_range[1] = np.max(all_wave) + whitelight_range[1] = wave[-1] msgs.info("White light image covers the wavelength range {0:.2f} A - {1:.2f} A".format( whitelight_range[0], whitelight_range[1])) # Get the output filename for the white light image @@ -1324,60 +1530,60 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s msgs.info("Saving white light image as: {0:s}".format(out_whitelight)) img_hdu = fits.PrimaryHDU(whitelight_img.T, header=whitelight_wcs.to_header()) img_hdu.writeto(out_whitelight, overwrite=overwrite) + # TODO :: Avoid transposing these large cubes return flxcube.T, np.sqrt(varcube.T), bpmcube.T, wave -def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, all_spatpos, all_specpos, - all_spatid, tilts, slits, astrom_trans, all_dar, bins, all_idx=None, - spec_subpixel=10, spat_subpixel=10, debug=False): +def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, debug=False): r""" Subpixellate the input data into a datacube. This algorithm splits each - detector pixel into multiple subpixels, and then assigns each subpixel to a - voxel. For example, if ``spec_subpixel = spat_subpixel = 10``, then each - detector pixel is divided into :math:`10^2=100` subpixels. Alternatively, - when spec_subpixel = spat_subpixel = 1, this corresponds to the nearest grid + detector pixel into multiple subpixels and each IFU slice into multiple subslices. + Then, the algorithm assigns each subdivided detector pixel to a + voxel. For example, if ``spec_subpixel = spat_subpixel = slice_subpixel = 5``, then each + detector pixel is divided into :math:`5^3=125` subpixels. Alternatively, + when spec_subpixel = spat_subpixel = slice_subpixel = 1, this corresponds to the nearest grid point (NGP) algorithm. - Important Note: If spec_subpixel > 1 or spat_subpixel > 1, the errors will - be correlated, and the covariance is not being tracked, so the errors will - not be (quite) right. There is a tradeoff one has to make between sampling - and better looking cubes, versus no sampling and better behaved errors. + Important Note: If spec_subpixel > 1 or spat_subpixel > 1 or slice_subpixel > 1, + the errors will be correlated, and the covariance is not being tracked, so the + errors will not be (quite) right. There is a tradeoff one has to make between + sampling and better looking cubes, versus no sampling and better behaved errors. Args: output_wcs (`astropy.wcs.WCS`_): Output world coordinate system. - all_ra (`numpy.ndarray`_): - 1D flattened array containing the right ascension of each pixel - (units = degrees) - all_dec (`numpy.ndarray`_): - 1D flattened array containing the declination of each pixel (units = - degrees) - all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength of each pixel (units = - Angstroms) - all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all - spec2d files - all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel - from all spec2d files - all_wghts (`numpy.ndarray`_): - 1D flattened array containing the weights of each pixel to be used - in the combination - all_spatpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spatial direction - all_specpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spectral direction - all_spatid (`numpy.ndarray`_): - 1D flattened array containing the spatid of each pixel - tilts (`numpy.ndarray`_, list): - 2D wavelength tilts frame, or a list of tilt frames (see all_idx) - slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): - Information stored about the slits, or a list of SlitTraceSet (see - all_idx) + bins (tuple): + A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial + and z wavelength coordinates + sciImg (`numpy.ndarray`_, list): + A list of 2D array containing the counts of each pixel. The shape of + each 2D array is (nspec, nspat). + ivarImg (`numpy.ndarray`_, list): + A list of 2D array containing the inverse variance of each pixel. The shape of + each 2D array is (nspec, nspat). + waveImg (`numpy.ndarray`_, list): + A list of 2D array containing the wavelength of each pixel. The shape of + each 2D array is (nspec, nspat). + slitid_img_gpm (`numpy.ndarray`_, list): + A list of 2D array containing the slitmask of each pixel. The shape of + each 2D array is (nspec, nspat). + A zero value indicates that a pixel is either not on a slit or it is a bad pixel. + All other values are the slit spatial ID number. + wghtImg (`numpy.ndarray`_, list): + A list of 2D array containing the weights of each pixel to be used in the + combination. The shape of each 2D array is (nspec, nspat). + all_wcs (`astropy.wcs.WCS`_, list): + A list of `astropy.wcs.WCS`_ objects, one for each spec2d file + tilts (list): + A list of `numpy.ndarray`_ objects, one for each spec2d file, + containing the tilts of each pixel. The shape of each 2D array is + (nspec, nspat). + slits (:class:`pypeit.slittrace.SlitTraceSet`, list): + A list of :class:`pypeit.slittrace.SlitTraceSet` objects, one for each + spec2d file, containing the properties of the slit for each spec2d file astrom_trans (:class:`~pypeit.alignframe.AlignmentSplines`, list): A Class containing the transformation between detector pixel coordinates and WCS pixel coordinates, or a list of Alignment @@ -1385,28 +1591,30 @@ def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_w all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): A Class containing the DAR correction information, or a list of DARcorrection classes. If a list, it must be the same length as astrom_trans. - bins (tuple): - A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial - and z wavelength coordinates - all_idx (`numpy.ndarray`_, optional): - If tilts, slits, and astrom_trans are lists, this should contain a - 1D flattened array, of the same length as all_sci, containing the - index the tilts, slits, and astrom_trans lists that corresponds to - each pixel. Note that, in this case all of these lists need to be - the same length. - spec_subpixel (:obj:`int`, optional): + ra_offset (float, list): + A float or list of floats containing the RA offset of each spec2d file + relative to the first spec2d file + dec_offset (float, list): + A float or list of floats containing the DEC offset of each spec2d file + relative to the first spec2d file + spec_subpixel (int, optional): What is the subpixellation factor in the spectral direction. Higher values give more reliable results, but note that the time required goes as (``spec_subpixel * spat_subpixel``). The default value is 5, which divides each detector pixel into 5 subpixels in the spectral direction. - spat_subpixel (:obj:`int`, optional): + spat_subpixel (int, optional): What is the subpixellation factor in the spatial direction. Higher values give more reliable results, but note that the time required goes as (``spec_subpixel * spat_subpixel``). The default value is 5, which divides each detector pixel into 5 subpixels in the spatial direction. - debug (bool): + slice_subpixel (int, optional): + What is the subpixellation factor in the slice direction. Higher + values give more reliable results, but note that the time required + goes as (``slice_subpixel``). The default value is 5, which divides + each IFU slice into 5 subslices in the slice direction. + debug (bool, optional): If True, a residuals cube will be output. If the datacube generation is correct, the distribution of pixels in the residual cube with no flux should have mean=0 and std=1. @@ -1417,33 +1625,11 @@ def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_w variance cube, (3) the corresponding bad pixel mask cube, and (4) the residual cube. The latter is only returned if debug is True. """ - # Check for combinations of lists or not - if all([isinstance(l, list) for l in [tilts, slits, astrom_trans, all_dar]]): - # Several frames are being combined. Check the lists have the same length - numframes = len(tilts) - if len(slits) != numframes or len(astrom_trans) != numframes or len(all_dar) != numframes: - msgs.error("The following lists must have the same length:" + msgs.newline() + - "tilts, slits, astrom_trans, all_dar") - # Check all_idx has been set - if all_idx is None: - if numframes != 1: - msgs.error("Missing required argument for combining frames: all_idx") - else: - all_idx = np.zeros(all_sci.size) - else: - tmp = np.unique(all_idx).size - if tmp != numframes: - msgs.warn("Indices in argument 'all_idx' does not match the number of frames expected.") - # Store in the following variables - _tilts, _slits, _astrom_trans, _all_dar = tilts, slits, astrom_trans, all_dar - elif all([not isinstance(l, list) for l in [tilts, slits, astrom_trans, all_dar]]): - # Just a single frame - store as lists for this code - _tilts, _slits, _astrom_trans, _all_dar = [tilts], [slits], [astrom_trans], [all_dar] - all_idx = np.zeros(all_sci.size) - numframes = 1 - else: - msgs.error("The following input arguments should all be of type 'list', or all not be type 'list':" + - msgs.newline() + "tilts, slits, astrom_trans, all_dar") + # Check the inputs for combinations of lists or not + _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ + check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) + numframes = len(_sciImg) + # Prepare the output arrays outshape = (bins[0].size-1, bins[1].size-1, bins[2].size-1) binrng = [[bins[0][0], bins[0][-1]], [bins[1][0], bins[1][-1]], [bins[2][0], bins[2][-1]]] @@ -1453,59 +1639,79 @@ def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_w # Divide each pixel into subpixels spec_offs = np.arange(0.5/spec_subpixel, 1, 1/spec_subpixel) - 0.5 # -0.5 is to offset from the centre of each pixel. spat_offs = np.arange(0.5/spat_subpixel, 1, 1/spat_subpixel) - 0.5 # -0.5 is to offset from the centre of each pixel. + slice_offs = np.arange(0.5/slice_subpixel, 1, 1/slice_subpixel) - 0.5 # -0.5 is to offset from the centre of each slice. spat_x, spec_y = np.meshgrid(spat_offs, spec_offs) - num_subpixels = spec_subpixel * spat_subpixel - area = 1 / num_subpixels - all_wght_subpix = all_wghts * area - all_var = utils.inverse(all_ivar) + num_subpixels = spec_subpixel * spat_subpixel # Number of subpixels per detector pixel + area = 1 / (num_subpixels * slice_subpixel) # Area of each subpixel # Loop through all exposures for fr in range(numframes): + onslit_gpm = _gpmImg[fr] + this_onslit_gpm = onslit_gpm > 0 + this_specpos, this_spatpos = np.where(this_onslit_gpm) + this_spatid = onslit_gpm[this_onslit_gpm] + # Extract tilts and slits for convenience this_tilts = _tilts[fr] this_slits = _slits[fr] - # Loop through all slits - for sl, spatid in enumerate(this_slits.spat_id): - if numframes == 1: - msgs.info(f"Resampling slit {sl+1}/{this_slits.nslits}") - else: - msgs.info(f"Resampling slit {sl+1}/{this_slits.nslits} of frame {fr+1}/{numframes}") - this_sl = np.where((all_spatid == spatid) & (all_idx == fr)) - wpix = (all_specpos[this_sl], all_spatpos[this_sl]) - # Generate a spline between spectral pixel position and wavelength - yspl = this_tilts[wpix]*(this_slits.nspec - 1) - tiltpos = np.add.outer(yspl, spec_y).flatten() - wspl = all_wave[this_sl] - asrt = np.argsort(yspl) - wave_spl = interp1d(yspl[asrt], wspl[asrt], kind='linear', bounds_error=False, fill_value='extrapolate') - # Calculate the wavelength at each subpixel - this_wave = wave_spl(tiltpos) - # Calculate the DAR correction at each sub pixel - ra_corr, dec_corr = _all_dar[fr].correction(this_wave) # This routine needs the wavelengths to be expressed in Angstroms - # Calculate spatial and spectral positions of the subpixels - spat_xx = np.add.outer(wpix[1], spat_x.flatten()).flatten() - spec_yy = np.add.outer(wpix[0], spec_y.flatten()).flatten() - # Transform this to spatial location - spatpos_subpix = _astrom_trans[fr].transform(sl, spat_xx, spec_yy) - spatpos = _astrom_trans[fr].transform(sl, all_spatpos[this_sl], all_specpos[this_sl]) - # Interpolate the RA/Dec over the subpixel spatial positions - ssrt = np.argsort(spatpos) - tmp_ra = all_ra[this_sl] - tmp_dec = all_dec[this_sl] - ra_spl = interp1d(spatpos[ssrt], tmp_ra[ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') - dec_spl = interp1d(spatpos[ssrt], tmp_dec[ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') - this_ra = ra_spl(spatpos_subpix) - this_dec = dec_spl(spatpos_subpix) - # Now apply the DAR correction - this_ra += ra_corr - this_dec += dec_corr - # Convert world coordinates to voxel coordinates, then histogram - vox_coord = output_wcs.wcs_world2pix(np.vstack((this_ra, this_dec, this_wave * 1.0E-10)).T, 0) - # Use the "fast histogram" algorithm, that assumes regular bin spacing - flxcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_sci[this_sl] * all_wght_subpix[this_sl], num_subpixels)) - varcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_var[this_sl] * all_wght_subpix[this_sl]**2, num_subpixels)) - normcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_wght_subpix[this_sl], num_subpixels)) - if debug: - residcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_sci[this_sl] * np.sqrt(all_ivar[this_sl]), num_subpixels)) + this_wcs = _all_wcs[fr] + this_astrom_trans = _astrom_trans[fr] + this_wght_subpix = _wghtImg[fr][this_onslit_gpm] * area + this_sci = _sciImg[fr][this_onslit_gpm] + this_var = utils.inverse(_ivarImg[fr][this_onslit_gpm]) + this_wav = _waveImg[fr][this_onslit_gpm] + # Loop over the subslices + for ss in range(slice_subpixel): + if slice_subpixel > 1: + # Only print this if there are multiple subslices + msgs.info(f"Resampling subslice {ss+1}/{slice_subpixel}") + # Generate an RA/Dec image for this subslice + raimg, decimg, minmax = this_slits.get_radec_image(this_wcs, this_astrom_trans, this_tilts, slice_offset=slice_offs[ss]) + this_ra = raimg[this_onslit_gpm] + this_dec = decimg[this_onslit_gpm] + # Loop through all slits + for sl, spatid in enumerate(this_slits.spat_id): + if numframes == 1: + msgs.info(f"Resampling slit {sl+1}/{this_slits.nslits}") + else: + msgs.info(f"Resampling slit {sl+1}/{this_slits.nslits} of frame {fr+1}/{numframes}") + this_sl = np.where(this_spatid == spatid) + wpix = (this_specpos[this_sl], this_spatpos[this_sl]) + # Generate a spline between spectral pixel position and wavelength + yspl = this_tilts[wpix]*(this_slits.nspec - 1) + tiltpos = np.add.outer(yspl, spec_y).flatten() + wspl = this_wav[this_sl] + asrt = np.argsort(yspl) + wave_spl = interp1d(yspl[asrt], wspl[asrt], kind='linear', bounds_error=False, fill_value='extrapolate') + # Calculate the wavelength at each subpixel + this_wave_subpix = wave_spl(tiltpos) + # Calculate the DAR correction at each sub pixel + ra_corr, dec_corr = _all_dar[fr].correction(this_wave_subpix) # This routine needs the wavelengths to be expressed in Angstroms + # Calculate spatial and spectral positions of the subpixels + spat_xx = np.add.outer(wpix[1], spat_x.flatten()).flatten() + spec_yy = np.add.outer(wpix[0], spec_y.flatten()).flatten() + # Transform this to spatial location + spatpos_subpix = _astrom_trans[fr].transform(sl, spat_xx, spec_yy) + spatpos = _astrom_trans[fr].transform(sl, wpix[1], wpix[0]) + # Interpolate the RA/Dec over the subpixel spatial positions + ssrt = np.argsort(spatpos) + tmp_ra = this_ra[this_sl] + tmp_dec = this_dec[this_sl] + ra_spl = interp1d(spatpos[ssrt], tmp_ra[ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') + dec_spl = interp1d(spatpos[ssrt], tmp_dec[ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') + # Evaluate the RA/Dec at the subpixel spatial positions + this_ra_int = ra_spl(spatpos_subpix) + this_dec_int = dec_spl(spatpos_subpix) + # Now apply the DAR correction and any user-supplied offsets + this_ra_int += ra_corr + _ra_offset[fr] + this_dec_int += dec_corr + _dec_offset[fr] + # Convert world coordinates to voxel coordinates, then histogram + vox_coord = output_wcs.wcs_world2pix(np.vstack((this_ra_int, this_dec_int, this_wave_subpix * 1.0E-10)).T, 0) + # Use the "fast histogram" algorithm, that assumes regular bin spacing + flxcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(this_sci[this_sl] * this_wght_subpix[this_sl], num_subpixels)) + varcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(this_var[this_sl] * this_wght_subpix[this_sl]**2, num_subpixels)) + normcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(this_wght_subpix[this_sl], num_subpixels)) + if debug: + residcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(this_sci[this_sl] * np.sqrt(utils.inverse(this_var[this_sl])), num_subpixels)) # Normalise the datacube and variance cube nc_inverse = utils.inverse(normcube) flxcube *= nc_inverse diff --git a/pypeit/core/flat.py b/pypeit/core/flat.py index 3d8b640fc5..1c104d1d96 100644 --- a/pypeit/core/flat.py +++ b/pypeit/core/flat.py @@ -19,6 +19,7 @@ from pypeit.core import parse from pypeit.core import pixels from pypeit.core import tracewave +from pypeit.core import coadd from pypeit import utils from pypeit.core import pydl @@ -285,7 +286,7 @@ def construct_illum_profile(norm_spec, spat_coo, slitwidth, spat_gpm=None, spat_ def illum_profile_spectral_poly(rawimg, waveimg, slitmask, slitmask_trim, model, slit_illum_ref_idx=0, - gpmask=None, thismask=None, nbins=20): + gpmask=None, thismask=None, nbins=20, debug=False): """ Use a polynomial fit to control points along the spectral direction to determine the relative spectral illumination of all slits. Currently, this routine is only used for image slicer IFUs. @@ -300,19 +301,21 @@ def illum_profile_spectral_poly(rawimg, waveimg, slitmask, slitmask_trim, model, A 2D int mask, the same shape as rawimg, indicating which pixels are on a slit. A -1 value indicates not on a slit, while any pixels on a slit should have the value of the slit spatial ID number. - slitmask_trim : + slitmask_trim : `numpy.ndarray`_ Same as slitmask, but the slit edges are trimmed. model : `numpy.ndarray`_ A model of the rawimg data. - slit_illum_ref_idx : int + slit_illum_ref_idx : :obj:`int` Index of slit that is used as the reference. gpmask : `numpy.ndarray`_, optional Boolean good pixel mask (True = Good) thismask : `numpy.ndarray`_, optional A boolean mask (True = good) that indicates all pixels where the scaleImg should be constructed. If None, the slitmask that is generated by this routine will be used. - nbins : int + nbins : :obj:`int` Number of bins in the spectral direction to sample the relative spectral sensitivity + debug : :obj:`bool` + If True, some plots will be output to test if the fitting is working correctly. Returns ------- @@ -347,6 +350,11 @@ def illum_profile_spectral_poly(rawimg, waveimg, slitmask, slitmask_trim, model, wgd = np.where(scale_err > 0) coeff = np.polyfit(wavcen[wgd], scale_bin[wgd], w=1/scale_err[wgd], deg=2) scaleImg[this_slit] *= np.polyval(coeff, waveimg[this_slit]) + if debug: + mod = np.polyval(coeff, wavcen[wgd]) + plt.errorbar(wavcen[wgd], scale_bin[wgd], yerr=scale_err[wgd], fmt='o') + plt.plot(wavcen[wgd], mod, 'r-') + plt.show() if sl == slit_illum_ref_idx: scaleImg[_thismask] *= utils.inverse(np.polyval(coeff, waveimg[_thismask])) minv, maxv = np.min(scaleImg[_thismask]), np.max(scaleImg[_thismask]) @@ -354,6 +362,149 @@ def illum_profile_spectral_poly(rawimg, waveimg, slitmask, slitmask_trim, model, return scaleImg +def smooth_scale(arr, wave_ref=None, polydeg=None, sn_smooth_npix=None): + """ + Smooth the relative sensitivity array using a polynomial fit or a boxcar filter. + + Parameters + ---------- + arr : `numpy.ndarray`_ + Array containing the relative sensitivity + wave_ref : `numpy.ndarray`_, optional + Wavelength array corresponding to the relative sensitivity array. Only used if polydeg is not None. + polydeg : :obj:`int`, optional + Degree of the polynomial fit to the relative sensitivity array. If None, a boxcar filter will be used. + sn_smooth_npix : :obj:`int`, optional + Number of pixels to use for the boxcar filter. Only used if polydeg is None. + + Returns + ------- + arr_smooth : `numpy.ndarray` + Smoothed relative sensitivity array + """ + # Do some checks on the input + if polydeg is not None and wave_ref is None: + msgs.error("Must provide a wavelength array if polydeg is not None") + if polydeg is None and sn_smooth_npix is None: + msgs.error("Must provide either polydeg or sn_smooth_npix") + # Smooth the relative sensitivity array + if polydeg is not None: + gd = (arr != 0) + wave_norm = (wave_ref - wave_ref[0]) / (wave_ref[1] - wave_ref[0]) + coeff = np.polyfit(wave_norm[gd], arr[gd], polydeg) + ref_relscale = np.polyval(coeff, wave_norm) + else: + ref_relscale = coadd.smooth_weights(arr, (arr != 0), sn_smooth_npix) + # Return the smoothed relative sensitivity array + return ref_relscale + + +# TODO:: See pypeit/deprecated/flat.py for a spline version. The following polynomial version is faster, but +# the spline version is more versatile. +def poly_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, deg=3, + slit_illum_ref_idx=0, gpmask=None, thismask=None, debug=False): + """ + Use a polynomial fit to control points along the spectral direction to construct a map between modelimg and + rawimg. Currently, this routine is only used for image slicer IFUs. + + This problem needs to be recast into a chi-squared problem, that can take advantage of polynomial fitting. + Refer to the following for variable assignments: + https://numpy.org/doc/stable/reference/generated/numpy.polyfit.html + where: + y = science/model + w = model/science_error + resid = w*(y-f(x)) + Then, iterate over this. The reason to iterate is that there should be a mapping between science and + science_error. Once we have an estimate of f(x), we can do the following: + spl = spline(science, science_error) + new_science_error = spl(model*f(x)) + Now iterate a few times until new_science_error (and the fit) is converged. + + Parameters + ---------- + rawimg : `numpy.ndarray`_ + Image data that will be used to estimate the spectral relative sensitivity + rawivar : `numpy.ndarray`_ + Inverse variance image of rawimg + waveimg : `numpy.ndarray`_ + Wavelength image + slitmask : `numpy.ndarray`_ + A 2D int mask, the same shape as rawimg, indicating which pixels are on a slit. A -1 value + indicates not on a slit, while any pixels on a slit should have the value of the slit spatial ID + number. + slitmask_trim : `numpy.ndarray`_ + Same as slitmask, but the slit edges are trimmed. + model : `numpy.ndarray`_ + A model of the rawimg data. + slit_illum_ref_idx : :obj:`int` + Index of slit that is used as the reference. + gpmask : `numpy.ndarray`_, optional + Boolean good pixel mask (True = Good) + thismask : `numpy.ndarray`_, optional + A boolean mask (True = good) that indicates all pixels where the scaleImg should be constructed. + If None, the slitmask that is generated by this routine will be used. + debug : :obj:`bool` + If True, some plots will be output to test if the fitting is working correctly. + + Returns + ------- + modelmap : `numpy.ndarray`_ + A 2D image with the same shape as rawimg, that contains the modelimg mapped to the rawimg + relscale : `numpy.ndarray`_ + A 2D image with the same shape as rawimg, that contains the relative spectral sensitivity + """ + # Some variables to consider putting as function arguments + numiter = 4 + + # Start by calculating a ratio of the raming and the modelimg + nspec = rawimg.shape[0] + _ratio = rawimg * utils.inverse(modelimg) + _ratio_ivar = rawivar * modelimg**2 + _fit_wghts = modelimg * np.sqrt(rawivar) + + # Generate the mask + _thismask = thismask if (thismask is not None) else (slitmask > 0) + gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool) + # Extract the list of spatial IDs from the slitmask + slitmask_spatid = np.unique(slitmask) + slitmask_spatid = np.sort(slitmask_spatid[slitmask_spatid > 0]) + + # Create a spline between the raw data and the error + flxsrt = np.argsort(np.ravel(rawimg)) + spl = scipy.interpolate.interp1d(np.ravel(rawimg)[flxsrt], np.ravel(rawivar)[flxsrt], kind='linear', + bounds_error=False, fill_value=0.0, assume_sorted=True) + modelmap = np.ones_like(rawimg) + relscale = np.ones_like(rawimg) + msgs.info("Generating a polynomial map between the model and the raw data") + for sl, spatid in enumerate(slitmask_spatid): + # Prepare the masks, edges, and fitting variables + this_slit = (slitmask == spatid) + this_slit_trim = (slitmask_trim == spatid) + this_slit_mask = gpm & this_slit_trim + this_wave = waveimg[this_slit_mask] + wmin, wmax = np.min(this_wave), np.max(this_wave) + this_wghts = _fit_wghts[this_slit_mask] + for ii in range(numiter): + # Generate the map between model and data + coeff = np.polyfit((this_wave-wmin)/(wmax-wmin), _ratio[this_slit_mask], deg, w=this_wghts) + # Construct the mapping, and use this to make a model of the rawdata + this_modmap = np.polyval(coeff, (this_wave-wmin)/(wmax-wmin)) + this_modflx = modelimg[this_slit_mask] * this_modmap + # Update the fit weights + this_wghts = modelimg[this_slit_mask] * np.sqrt(spl(this_modflx)) + # Produce the final model for this slit + modelmap[this_slit] *= np.polyval(coeff, (waveimg[this_slit]-wmin)/(wmax-wmin)) + relscale[this_slit] *= np.polyval(coeff, (waveimg[this_slit]-wmin)/(wmax-wmin)) + # Check if this is the reference slit, and if so, set the scale relative to the reference slit. + if sl == slit_illum_ref_idx: + for slidx, spatid in enumerate(slitmask_spatid): + # Get the slit pixels + norm_slit = (slitmask == spatid) + relscale[norm_slit] /= np.polyval(coeff, (waveimg[norm_slit]-wmin)/(wmax-wmin)) + # Return the modelmap and the relative scale + return modelmap, relscale + + # TODO: See pypeit/deprecated/flat.py for the previous version. We need # to continue to vet this algorithm to make sure there are no # unforeseen corner cases that cause errors. diff --git a/pypeit/core/procimg.py b/pypeit/core/procimg.py index 0f20a719b8..9133bd6eac 100644 --- a/pypeit/core/procimg.py +++ b/pypeit/core/procimg.py @@ -705,7 +705,6 @@ def subtract_overscan(rawframe, datasec_img, oscansec_img, method='savgol', para no_overscan[data_slice][:,0::2] -= even[:,None] else: msgs.error('Not ready for this approach, please contact the Developers') - # Subtract along the appropriate axis no_overscan[data_slice] -= (ossub[:, None] if compress_axis == 1 else ossub[None, :]) @@ -816,7 +815,9 @@ def subtract_pattern(rawframe, datasec_img, oscansec_img, frequency=None, axis=1 sgnl = overscan[ii,:] LSfreq, power = LombScargle(pixels, sgnl).autopower(minimum_frequency=use_fr*(1-100/frame_orig.shape[1]), maximum_frequency=use_fr*(1+100/frame_orig.shape[1]), samples_per_peak=10) bst = np.argmax(power) - cc = np.polyfit(LSfreq[bst-2:bst+3],power[bst-2:bst+3],2) + imin = np.clip(bst-2,0,None) + imax = np.clip(bst+3,None,overscan.shape[1]) + cc = np.polyfit(LSfreq[imin:imax],power[imin:imax],2) all_freq[ii] = -0.5*cc[1]/cc[0] cc = np.polyfit(all_rows, all_freq, 1) frq_mod = np.polyval(cc, all_rows) * (overscan.shape[1]-1) @@ -832,7 +833,7 @@ def subtract_pattern(rawframe, datasec_img, oscansec_img, frequency=None, axis=1 # Convert result to amplitude and phase amps = (np.abs(tmpamp))[idx] * (2.0 / overscan.shape[1]) - # STEP 2 - Using th emodel frequency, calculate how amplitude depends on pixel row (usually constant) + # STEP 2 - Using the model frequency, calculate how amplitude depends on pixel row (usually constant) # Use the above to as initial guess parameters for a chi-squared minimisation of the amplitudes msgs.info("Measuring amplitude-pixel dependence of amplifier {0:d}".format(amp)) nspec = overscan.shape[0] diff --git a/pypeit/core/scattlight.py b/pypeit/core/scattlight.py index 03376ed88d..394af7166d 100644 --- a/pypeit/core/scattlight.py +++ b/pypeit/core/scattlight.py @@ -12,6 +12,68 @@ from pypeit import msgs, utils +def pad_frame(_frame, detpad=300): + """ + Clean the edges of the input frame and then pad the frame to avoid edge effects. + + Parameters + ---------- + _frame : `numpy.ndarray`_ + Frame to be padded + detpad : int + Number of pixels to pad the frame on each side + + Returns + ------- + _frame_pad : `numpy.ndarray`_ + Padded frame + """ + _frame[0, :] = np.median(_frame[0:10, :], axis=0) + _frame[-1, :] = np.median(_frame[-10:, :], axis=0) + return np.pad(_frame, detpad, mode='edge') # Model should be generated on padded data + + +def scattered_light_model_pad(param, img, detpad=300): + """ + Construct a scattered light model for the input image, with the model parameters + defined by param. This function is used to generate a model of the scattered light, + based on a set of model parameters that have first been optimized using scattered_light(). + The model is generated on a padded version of the input image, and then trimmed to + match the input image size. + + Parameters + ---------- + param : `numpy.ndarray`_ + Model parameters that determine the scattered light based on the input img. + Here is a list of the individual parameter meanings: + + * param[0] = Gaussian kernel width in the spectral direction + * param[1] = Gaussian kernel width in the spatial direction + * param[2] = Lorentzian kernel width in the spectral direction + * param[3] = Lorentzian kernel width in the spatial direction + * param[4] = Pixel shift of the scattered light in the spectral direction + * param[5] = Pixel shift of the scattered light in the spatial direction + * param[6] = Zoom factor of the scattered light (~1) + * param[7] = constant offset for scattered light (independent of img) + * param[8] = Kernel angle + * param[9] = Relative importance of Gaussian vs Lorentzian. + 0 < value < 1 means Lorentzian is weighted more + value > 1 means Gaussian is weighted more. + * param[10:] = Polynomial scaling coefficients + img : `numpy.ndarray`_ + Image used to generate the scattered light model + detpad : int + Number of pixels to pad the frame on each side + + Returns + ------- + model : `numpy.ndarray`_ + Model of the scattered light for the input + """ + _frame_pad = pad_frame(img, detpad=detpad) + return scattered_light_model(param, _frame_pad)[detpad:-detpad, detpad:-detpad] + + def scattered_light_model(param, img): """ Model used to calculate the scattered light. @@ -34,11 +96,12 @@ def scattered_light_model(param, img): * param[4] = Pixel shift of the scattered light in the spectral direction * param[5] = Pixel shift of the scattered light in the spatial direction * param[6] = Zoom factor of the scattered light (~1) - * param[7] = Kernel angle - * param[8] = Relative importance of Gaussian vs Lorentzian. - 0 < value < 1 means Lorentzian is weighted more - value > 1 means Gaussian is weighted more. - * param[9:] = Polynomial scaling coefficients + * param[7] = constant offset for scattered light (independent of img) + * param[8] = Kernel angle + * param[9] = Relative importance of Gaussian vs Lorentzian. + 0 < value < 1 means Lorentzian is weighted more + value > 1 means Gaussian is weighted more. + * param[10:] = Polynomial scaling coefficients img : `numpy.ndarray`_ Raw image that you want to compute the scattered light model. shape is (nspec, nspat) @@ -151,9 +214,7 @@ def scattered_light(frame, bpm, offslitmask, x0, bounds, detpad=300, debug=False _frame = utils.replace_bad(frame, bpm) # Pad the edges of the data - _frame[0, :] = np.median(_frame[0:10, :], axis=0) - _frame[-1, :] = np.median(_frame[-10:, :], axis=0) - _frame_pad = np.pad(_frame, detpad, mode='edge') # Model should be generated on padded data + _frame_pad = pad_frame(_frame, detpad) offslitmask_pad = np.pad(offslitmask * gpm, detpad, mode='constant', constant_values=0) # but don't include padded data in the fit # Grab the pixels to be included in the fit wpix = np.where(offslitmask_pad) @@ -252,7 +313,7 @@ def mask_slit_regions(offslitmask, centrace, mask_regions=None): return offslitmask & np.logical_not(bad_mask) -def fine_correction(frame, bpm, offslitmask, polyord=2, debug=False): +def fine_correction(frame, bpm, offslitmask, method='median', polyord=2, debug=False): """ Calculate a fine correction to the residual scattered light of the input frame. Parameters @@ -265,6 +326,11 @@ def fine_correction(frame, bpm, offslitmask, polyord=2, debug=False): 2D boolean array indicating the bad pixels (True=bad), same shape as frame offslitmask : `numpy.ndarray`_ A boolean mask indicating the pixels that are on/off the slit (True = off the slit), same shape as frame + method : :obj:`str`, optional + Method to use to determine the fine correction to the scattered light. Options are: + - 'median': Use the median of the off-slit pixels to determine the scattered light + - 'poly': Use a polynomial fit to the off-slit pixels to determine the scattered light. If this + option is chosen, the polynomial order must be specified. See `polyord` below. polyord : :obj:`int`, optional Polynomial order to use for fitting the residual scattered light in the spatial direction. debug : :obj:`bool`, optional @@ -275,25 +341,33 @@ def fine_correction(frame, bpm, offslitmask, polyord=2, debug=False): scatt_img : `numpy.ndarray`_ A 2D image (nspec, nspat) of the fine correction to the scattered light determined from the input frame. """ - msgs.info("Performing a fine correction to the scattered light") - # Convert the BPM to a GPM for convenience - gpm = np.logical_not(bpm) - - # Define some useful variables + if method not in ['median', 'poly']: + msgs.error("Unrecognized method to determine the fine correction to the scattered light: {:s}".format(method)) + msgs.info("Performing a fine correction to the scattered light using the {:s} method".format(method)) nspec, nspat = frame.shape - xspat = np.linspace(0, 1, nspat) - model = np.zeros_like(frame) - - # Loop over the residual scattered light in the spectral direction and perform - # a low order polynomial fit to the scattered light in the spatial direction. - for yy in range(nspec): - ext = frame[yy, :] - gd = np.where(offslitmask[yy, :] & gpm[yy, :]) - coeff = np.polyfit(xspat[gd], ext[gd], polyord) - model[yy, :] = np.polyval(coeff, xspat) - # Median filter in the spectral direction to smooth out irregularities in the fine correction - model_med = ndimage.median_filter(model, size=(50, 1)) # Median filter to get rid of CRs - scatt_light_fine = ndimage.gaussian_filter(model_med, sigma=10) # Gaussian filter to smooth median filter + if method == 'median': + # Use the median of the off-slit pixels to determine the scattered light + mask = bpm | np.logical_not(offslitmask) + frame_msk = np.ma.array(frame, mask=mask) + scatt_light_fine = np.repeat(np.ma.median(frame_msk, axis=1).data[:, np.newaxis], nspat, axis=1) + elif method == 'poly': + # Convert the BPM to a GPM for convenience + gpm = np.logical_not(bpm) + + # Define some useful variables + xspat = np.linspace(0, 1, nspat) + model = np.zeros_like(frame) + + # Loop over the residual scattered light in the spectral direction and perform + # a low order polynomial fit to the scattered light in the spatial direction. + for yy in range(nspec): + ext = frame[yy, :] + gd = np.where(offslitmask[yy, :] & gpm[yy, :]) + coeff = np.polyfit(xspat[gd], ext[gd], polyord) + model[yy, :] = np.polyval(coeff, xspat) + # Median filter in the spectral direction to smooth out irregularities in the fine correction + model_med = ndimage.median_filter(model, size=(50, 1)) # Median filter to get rid of CRs + scatt_light_fine = ndimage.gaussian_filter(model_med, sigma=10) # Gaussian filter to smooth median filter if debug: from matplotlib import pyplot as plt vmin, vmax = -np.max(scatt_light_fine), np.max(scatt_light_fine) diff --git a/pypeit/core/slitdesign_matching.py b/pypeit/core/slitdesign_matching.py index 3c8e50fc33..eda3db0682 100644 --- a/pypeit/core/slitdesign_matching.py +++ b/pypeit/core/slitdesign_matching.py @@ -14,6 +14,7 @@ from IPython import embed import numpy as np +from scipy import optimize from matplotlib import pyplot as plt from astropy.stats import sigma_clipped_stats @@ -362,3 +363,60 @@ def plot_matches(edgetrace, ind, x_model, yref, slit_index, nspat=2048, duplicat plt.ylim(0, edgetrace.shape[0]) plt.legend(loc=1) plt.show() + + +def match_positions_1D(measured, nominal, tol=None): + """ + Match a set of measured 1D positions against a nominal expectation with + uniqueness (i.e., more than one measured position cannot be matched to the + same nominal position). + + The function primarily uses `scipy.optimize.linear_sum_assignment`_ to + perform the matching, where the matrix of separations of each measured + position to every nominal position is used as the cost matrix. + + Args: + measured (`numpy.ndarray`_): + Measured positions. Shape is ``(n,)``. + nominal (`numpy.ndarray`_): + Expected positions. Shape is ``(m,)``. + tol (:obj:`float`, optional): + Maximum separation between measured and nominal positions to be + considered a match. If None, no limit is applied. + + Returns: + `numpy.ndarray`_: Indices of the elements in ``measured`` that are + matched to the elements of ``nominal``. Shape is ``(m,)``. If the + tolerance is set or the number of measurements is less than the nominal + set (i.e., ``n < m``), any element of ``nominal`` that does not have an + appropriate match in ``measured`` is given an index of -1. + """ + # Calculate the (m,n) separation matrix + # NOTE: This is the brute force approach. For *lots* of measurements, this + # can be sped up by using a KDTree to build a sparse matrix with only a + # subset of the separations calculated. + sep = np.absolute(nominal[:,None] - measured[None,:]) + + # Perform the match + n_m, m_m = optimize.linear_sum_assignment(sep) + + # Remove any matches that don't meet the provided tolerance. + # NOTE: It's possible this approach yields a non-optimal match. I.e., when + # multiple matches are near the tolerance, removing the largest separations + # *before* performing the match (above) might ultimately yield a more + # optimal result for the ones that remain. But this approach has worked so + # far. + if tol is not None: + indx = sep[n_m,m_m] < tol + n_m = n_m[indx] + m_m = m_m[indx] + + # If there aren't any missing matches, just return the match vector + if n_m.size == nominal.size: + return m_m + + # Otherwise, insert -1 placeholders. + _m_m = np.full(nominal.size, -1, dtype=int) + _m_m[n_m] = m_m + return _m_m + diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 378e57ce81..074fe88e3c 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -39,7 +39,7 @@ # TODO These codes should probably be in a separate qso_pca module. Also pickle functionality needs to be removed. # The prior is not used (that was the reason for pickling), so the components could be stored in fits format. # npca is not actually required here. -def init_pca(filename,wave_grid,redshift, npca): +def qso_init_pca(filename,wave_grid,redshift,npca): """ This routine reads in the pickle file created by coarse_pca.create_coarse_pca. The relevant pieces are the wavelengths (wave_pca_c), the PCA components (pca_comp_c), and the Gaussian mixture @@ -72,19 +72,18 @@ def init_pca(filename,wave_grid,redshift, npca): pca_table = table.Table.read(file_with_path) wave_pca_c = pca_table['WAVE_PCA'][0].flatten() pca_comp_c = pca_table['PCA_COMP'][0][0,:,:] - coeffs_c =pca_table['PCA_COEFFS'][0][0,:,:] - #wave_pca_c, cont_all_c, pca_comp_c, coeffs_c, mean_pca, covar_pca, diff_pca, mix_fit, chi2, dof = pickle.load(open(filename,'rb')) + coeffs_c = pca_table['PCA_COEFFS'][0][0,:,:] num_comp = pca_comp_c.shape[0] # number of PCA components # Interpolate PCA components onto wave_grid - pca_interp = scipy.interpolate.interp1d(wave_pca_c*(1.0 + redshift),pca_comp_c, bounds_error=False, fill_value=0.0, axis=1) + pca_interp = scipy.interpolate.interp1d(wave_pca_c*(1.0+redshift), pca_comp_c, bounds_error=False, fill_value=0.0, axis=1) pca_comp_new = pca_interp(wave_grid) # Generate a mixture model for the coefficients prior, what should ngauss be? #prior = mixture.GaussianMixture(n_components = npca-1).fit(coeffs_c[:, 1:npca]) # Construct the PCA dict - pca_dict = {'npca': npca, 'components': pca_comp_new, 'coeffs': coeffs_c, 'z_fid': redshift, 'dloglam': dloglam} - return pca_dict + qso_pca_dict = {'npca': npca, 'components': pca_comp_new, 'coeffs': coeffs_c, 'z_fid': redshift, 'dloglam': dloglam} + return qso_pca_dict -def pca_eval(theta,pca_dict): +def qso_pca_eval(theta,qso_pca_dict): """ Function for evaluating the quasar PCA @@ -92,18 +91,18 @@ def pca_eval(theta,pca_dict): theta (`numpy.ndarray`_): Parameter vector, where theta_pca[0] is redshift, theta_pca[1] is the normalization, and theta_pca[2:npca+1] are the PCA coefficients, where npca is the PCA dimensionality - pca_dict (dict): - Dictionary continaing the PCA information generated by init_pca + qso_pca_dict (dict): + Dictionary continaing the PCA information generated by qso_init_pca Returns: `numpy.ndarray`_: Evaluated PCA for the QSO """ - C = pca_dict['components'] - z_fid = pca_dict['z_fid'] - dloglam = pca_dict['dloglam'] - npca = pca_dict['npca'] # Size of the PCA currently being used, original PCA in the dict could be larger + C = qso_pca_dict['components'] + z_fid = qso_pca_dict['z_fid'] + dloglam = qso_pca_dict['dloglam'] + npca = qso_pca_dict['npca'] # Size of the PCA currently being used, original PCA in the dict could be larger z_qso = theta[0] norm = theta[1] A = theta[2:] @@ -112,7 +111,7 @@ def pca_eval(theta,pca_dict): return norm*np.exp(np.dot(np.append(1.0,A),C_now)) # TODO The prior is not currently used, but this is left in here anyway. -#def pca_lnprior(theta,pca_dict): +#def qso_pca_lnprior(theta,qso_pca_dict): # """ # Routine to evaluate the the ln of the prior probability lnPrior, # from the Gaussian mixture model fit to the distriution of PCA @@ -124,9 +123,9 @@ def pca_eval(theta,pca_dict): # ``theta_pca[1]`` is the normalization, and # ``theta_pca[2:npca+1]`` are the PCA coefficients, where npca # is the PCA dimensionality -# pca_dict (dict): +# qso_pca_dict (dict): # Dictionary continaing the PCA information generated by -# ``init_pca`` +# ``qso_init_pca`` # # # Returns: @@ -135,30 +134,82 @@ def pca_eval(theta,pca_dict): # theta_pca[2:npca+1]`` # # """ -# gaussian_mixture_model = pca_dict['prior'] +# gaussian_mixture_model = qso_pca_dict['prior'] # A = theta[2:] # return gaussian_mixture_model.score_samples(A.reshape(1,-1)) +def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): + """ + Reads in the telluric PCA components from a file. + + Optionally, this method also trims wavelength to be in within ``wave_min`` + and ``wave_max`` and pads the data (see ``pad_frac``). + + Args: + filename (:obj:`str`): + Telluric PCA filename + wave_min (:obj:`float`, optional): + Minimum wavelength at which the grid is desired + wave_max (:obj:`float`, optional): + Maximum wavelength at which the grid is desired. + pad_frac (:obj:`float`, optional): + Percentage padding to be added to the grid boundaries if + ``wave_min`` or ``wave_max`` are input; ignored otherwise. The + resulting grid will extend from ``(1.0 - pad_frac)*wave_min`` to + ``(1.0 + pad_frac)*wave_max``. + + Returns: + :obj:`dict`: Dictionary containing the telluric PCA components. + - wave_grid: Telluric model wavelength grid + - dloglam: Wavelength sampling of telluric model + - tell_pad_pix: Number of pixels to pad at edges for convolution + - ncomp_tell_pca: Number of PCA components + - tell_pca: PCA component vectors + - bounds_tell_pca: Maximum/minimum coefficient + - coefs_tell_pca: Set of model coefficient values (for prior in future) + - teltype: Type of telluric model, i.e. 'pca' + """ + # load_telluric_grid() takes care of path and existance check + hdul = data.load_telluric_grid(filename) + wave_grid_full = hdul[1].data + pca_comp_full = hdul[0].data + nspec_full = wave_grid_full.size + ncomp = hdul[0].header['NCOMP'] + bounds = hdul[2].data + model_coefs = hdul[3].data + + ind_lower = np.argmin(np.abs(wave_grid_full - (1.0 - pad_frac)*wave_min)) \ + if wave_min is not None else 0 + ind_upper = np.argmin(np.abs(wave_grid_full - (1.0 + pad_frac)*wave_max)) \ + if wave_max is not None else nspec_full + wave_grid = wave_grid_full[ind_lower:ind_upper] + pca_comp_grid = pca_comp_full[:,ind_lower:ind_upper] + + dwave, dloglam, resln_guess, pix_per_sigma = wvutils.get_sampling(wave_grid) + tell_pad_pix = int(np.ceil(10.0 * pix_per_sigma)) + + return dict(wave_grid=wave_grid, dloglam=dloglam, + tell_pad_pix=tell_pad_pix, ncomp_tell_pca=ncomp, + tell_pca=pca_comp_grid, bounds_tell_pca=bounds, + coefs_tell_pca=model_coefs, teltype='pca') + def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): """ - Reads in the telluric grid from a file. + Reads in the telluric grid from a file. This method is no longer the + preferred approach; see "read_telluric_pca" for the PCA mode. Optionally, this method also trims the grid to be in within ``wave_min`` and ``wave_max`` and pads the data (see ``pad_frac``). - .. todo:: - List and describe the contents of the dictionary in the return - description. - Args: filename (:obj:`str`): Telluric grid filename - wave_min (:obj:`float`): + wave_min (:obj:`float`, optional): Minimum wavelength at which the grid is desired - wave_max (:obj:`float`): + wave_max (:obj:`float`, optional): Maximum wavelength at which the grid is desired. - pad_frac (:obj:`float`): + pad_frac (:obj:`float`, optional): Percentage padding to be added to the grid boundaries if ``wave_min`` or ``wave_max`` are input; ignored otherwise. The resulting grid will extend from ``(1.0 - pad_frac)*wave_min`` to @@ -166,16 +217,15 @@ def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): Returns: :obj:`dict`: Dictionary containing the telluric grid - - wave_grid= - - dloglam= - - resln_guess= - - pix_per_sigma= - - tell_pad_pix= - - pressure_grid= - - temp_grid= - - h2o_grid= - - airmass_grid= - - tell_grid= + - wave_grid: Telluric model wavelength grid + - dloglam: Wavelength sampling of telluric model + - tell_pad_pix: Number of pixels to pad at edges for convolution + - pressure_grid: Atmospheric pressure values in telluric grid [mbar] + - temp_grid: Temperature values in telluric grid [degrees C] + - h2o_grid: Humidity values in telluric grid [%] + - airmass_grid: Airmass values in telluric grid + - tell_grid: Grid of telluric models + - teltype: Type of telluric model, i.e. 'grid' """ # load_telluric_grid() takes care of path and existance check hdul = data.load_telluric_grid(filename) @@ -201,17 +251,9 @@ def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): dwave, dloglam, resln_guess, pix_per_sigma = wvutils.get_sampling(wave_grid) tell_pad_pix = int(np.ceil(10.0 * pix_per_sigma)) - - return dict(wave_grid=wave_grid, - dloglam=dloglam, - resln_guess=resln_guess, - pix_per_sigma=pix_per_sigma, - tell_pad_pix=tell_pad_pix, - pressure_grid=pg, - temp_grid=tg, - h2o_grid=hg, - airmass_grid=ag, - tell_grid=model_grid) + return dict(wave_grid=wave_grid, dloglam=dloglam, tell_pad_pix=tell_pad_pix, + pressure_grid=pg, temp_grid=tg, h2o_grid=hg, airmass_grid=ag, + tell_grid=model_grid, teltype='grid') def interp_telluric_grid(theta, tell_dict): @@ -259,8 +301,8 @@ def conv_telluric(tell_model, dloglam, res): Args: tell_model (`numpy.ndarray`_): Input telluric model at the native resolution of the telluric model grid. The shape of this input is in - general different from the size of the telluric grid (read in by read_telluric_grid above) because it is - trimmed to relevant wavelenghts using ind_lower, ind_upper. See eval_telluric below. + general different from the size of the raw telluric model (read in by read_telluric_* above) because it is + trimmed to relevant wavelengths using ind_lower, ind_upper. See eval_telluric below. dloglam (float): Wavelength spacing of the telluric grid expressed as a dlog10(lambda), i.e. stored in the tell_dict as tell_dict['dloglam'] @@ -294,12 +336,13 @@ def conv_telluric(tell_model, dloglam, res): def shift_telluric(tell_model, loglam, dloglam, shift, stretch): """ Routine to apply a shift to the telluric model. Note that the shift can be sub-pixel, i.e this routine interpolates. + Note that the shift units are pixels *of the telluric model*. Args: tell_model (`numpy.ndarray`_): - Input telluric model. The shape of this input is in general different from the size of the telluric grid - (read in by read_telluric_grid above) because it is trimmed to relevant wavelenghts using ind_lower, ind_upper. - See eval_telluric below. + Input telluric model. The shape of this input is in general different from the size of the telluric models + (read in by read_telluric_* above) because it is trimmed to relevant wavelengths using ind_lower, ind_upper. + See eval_telluric_* below. loglam (`numpy.ndarray`_): The log10 of the wavelength grid on which the tell_model is evaluated. @@ -320,43 +363,43 @@ def shift_telluric(tell_model, loglam, dloglam, shift, stretch): tell_model_shift = np.interp(loglam_shift, loglam, tell_model) return tell_model_shift - def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): """ - Evaluate the telluric model at an arbitrary location in parameter space. - - The full atmosphere model parameter space is either 5 or 7 dimensional, - which is the size of the ``theta_tell`` input parameter vector. + Evaluate the telluric model. - The parameters provided by ``theta_tell`` must be: pressure, temperature, - humidity, airmass, spectral resolution, shift, and stretch. The latter two - can be omitted if a shift and stretch of the telluric model are not - included. + Parameters from ``theta_tell`` are: (if teltype == 'pca') the PCA coefficients + or (if teltype == 'grid') the telluric grid parameters pressure, temperature, + humidity, and airmass, in both cases followed by spectral resolution, shift, + and stretch. This routine performs the following steps: - 1. nearest grid point interpolation of the telluric model onto a - new location in the 4-d space (pressure, temperature, - humidity, airmass) + 1. summation of telluric PCA components multiplied by coefficients + + 2. transformation of telluric PCA model from arsinh(tau) to transmission - 2. convolution of the atmosphere model to the resolution set by + 3. convolution of the atmosphere model to the resolution set by the spectral resolution. - 3. (Optional) shift and stretch the telluric model. + 4. shift and stretch the telluric model. Args: theta_tell (`numpy.ndarray`_): - Vector with the telluric model parameters. Must be 5 or 7 - elements long. See method description. + Vector with tell_npca PCA coefficients (if teltype='pca') + or pressure, temperature, humidity, and airmass (if teltype='grid'), + followed by spectral resolution, shift, and stretch. + Final length is then tell_npca+3 or 7. tell_dict (:obj:`dict`): - Dictionary containing the telluric grid data. See - :func:`read_telluric_grid`. + Dictionary containing the telluric data. See + :func:`read_telluric_pca` if teltype=='pca'. + or + :func:`read_telluric_grid` if teltype=='grid'. ind_lower (:obj:`int`, optional): The index of the first pixel to include in the model. Selecting a wavelength region for the modeling makes things faster because we only need to convolve the portion that is needed for the current model fit. - ind_upper: + ind_upper (:obj:`int`, optional): The index (inclusive) of the last pixel to include in the model. Selecting a wavelength region for the modeling makes things faster because we only need to convolve the portion that is @@ -364,13 +407,19 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): Returns: `numpy.ndarray`_: Telluric model evaluated at the desired location - theta_tell in model atmosphere parameter space. + theta_tell in model atmosphere parameter space. Shape is given by + the size of ``wave_grid`` plus ``tell_pad_pix`` padding from the input + tell_dict. + """ ntheta = len(theta_tell) - if ntheta not in [5, 7]: - msgs.error('Input model atmosphere parameters must have length 5 or 7.') - - tellmodel_hires = interp_telluric_grid(theta_tell[:4], tell_dict) + teltype = tell_dict['teltype'] + # FD: Currently assumes that shift and stretch are on. + # TODO: Make this work without shift and stretch. + if teltype == 'pca': + ntell = ntheta-3 + elif teltype == 'grid': + ntell = 4 # Set the wavelength range if not provided ind_lower = 0 if ind_lower is None else ind_lower @@ -382,16 +431,32 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): ## FW: There is an extreme case with ind_upper == ind_upper_pad, the previous -0 won't work ind_lower_final = ind_lower_pad if ind_lower_pad == ind_lower else ind_lower - ind_lower_pad ind_upper_final = ind_upper_pad if ind_upper_pad == ind_upper else ind_upper - ind_upper_pad - tellmodel_conv = conv_telluric(tellmodel_hires[ind_lower_pad:ind_upper_pad+1], - tell_dict['dloglam'], theta_tell[4]) - if ntheta == 5: - return tellmodel_conv[ind_lower_final:ind_upper_final] + if teltype == 'pca': + # Evaluate PCA model after truncating the wavelength range + tellmodel_hires = np.zeros_like(tell_dict['tell_pca'][0]) + tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.dot(np.append(1,theta_tell[:ntell]), + tell_dict['tell_pca'][:ntell+1][:,ind_lower_pad:ind_upper_pad+1]) + + # PCA model is inverse sinh of the optical depth, convert to transmission here + tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.sinh(tellmodel_hires[ind_lower_pad:ind_upper_pad+1]) + # It should generally be very rare, but trim negative optical depths here just in case. + clip = tellmodel_hires < 0 + tellmodel_hires[clip] = 0 + tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.exp(-tellmodel_hires[ind_lower_pad:ind_upper_pad+1]) + elif teltype == 'grid': + # Interpolate within the telluric grid + tellmodel_hires = interp_telluric_grid(theta_tell[:ntell], tell_dict) + + tellmodel_conv = conv_telluric(tellmodel_hires[ind_lower_pad:ind_upper_pad+1], + tell_dict['dloglam'], theta_tell[-3]) + + # Stretch and shift telluric wavelength grid tellmodel_out = shift_telluric(tellmodel_conv, np.log10(tell_dict['wave_grid'][ind_lower_pad:ind_upper_pad+1]), - tell_dict['dloglam'], theta_tell[5], theta_tell[6]) - return tellmodel_out[ind_lower_final:ind_upper_final] + tell_dict['dloglam'], theta_tell[-2], theta_tell[-1]) + return tellmodel_out[ind_lower_final:ind_upper_final] ############################ # Fitting routines # @@ -405,7 +470,45 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): Args: theta (`numpy.ndarray`_): - Parameter vector for the object + telluric model. See documentation of tellfit for a detailed description. + + Parameter vector for the object + telluric model. + + This is actually two concatenated parameter vectors, one for + the object and one for the telluric, i.e.: + + (in PCA mode) + theta_obj = theta[:-(tell_npca+3)] + theta_tell = theta[-(tell_npca+3):] + + (in grid mode) + theta_obj = theta[:-7] + theta_tell = theta[-7:] + + The telluric model theta_tell includes a either user-specified + number of PCA coefficients (in PCA mode) or ambient pressure, + temperature, humidity, and airmass (in grid mode) followed by + spectral resolution, shift, and stretch. + + That is, in PCA mode, + + pca_coeffs = theta_tell[:tell_npca] + + while in grid mode, + + pressure = theta_tell[0] + temperature = theta_tell[1] + humidity = theta_tell[2] + airmass = theta_tell[3] + + with the last three indices of the array corresponding to + + resolution = theta_tell[-3] + shift = theta_tell[-2] + stretch = theta_tell[-1] + + The object model theta_obj can have an arbitrary size and is + provided as an argument to obj_model_func + flux (`numpy.ndarray`_): The flux of the object being fit thismask (`numpy.ndarray`_, boolean): @@ -419,14 +522,23 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): that is minimized to perform the fit. """ - + obj_model_func = arg_dict['obj_model_func'] flux_ivar = arg_dict['ivar'] + teltype = arg_dict['tell_dict']['teltype'] + + # TODO: make this work without shift and stretch? + # Number of telluric model parameters, plus shift, stretch, and resolution + if teltype == 'pca': + nfit = arg_dict['tell_npca']+3 + elif teltype == 'grid': + nfit = 4+3 + + theta_obj = theta[:-nfit] + theta_tell = theta[-nfit:] - theta_obj = theta[:-7] - theta_tell = theta[-7:] tell_model = eval_telluric(theta_tell, arg_dict['tell_dict'], - ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) + ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) obj_model, model_gpm = obj_model_func(theta_obj, arg_dict['obj_dict']) totalmask = thismask & model_gpm @@ -446,24 +558,6 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): fits for any object model that the user provides. Args: - - theta (`numpy.ndarray`_): - - Parameter vector for the object + telluric model. - - This is actually two concatenated paramter vectors, one for - the object and one for the telluric, i.e:: - - theta_obj = theta[:-7] - theta_tell = theta[-7:] - - The telluric model theta_tell is currently hard wired to be six dimensional:: - - pressure, temperature, humidity, airmass, resln, shift = theta_tell - - The object model theta_obj can have an arbitrary size and is - provided as an argument to obj_model_func - flux (`numpy.ndarray`_): The flux of the object being fit thismask (`numpy.ndarray`_, boolean): @@ -477,8 +571,8 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): - ``arg_dict['flux_ivar']``: Inverse variance for the flux array - ``arg_dict['tell_dict']``: Dictionary containing the - telluric grid and its parameters read in by - read_telluric_grid + telluric model and its parameters read in by + read_telluric_pca or read_telluric_grid - ``arg_dict['ind_lower']``: Lower index into the telluric model wave_grid to trim down the telluric model. @@ -491,14 +585,11 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): object model arguments which is passed to the obj_model_func - init_from_last (object): + init_from_last (object, optional): Optional. Result object returned by the differential evolution optimizer for the last iteration. If this is passed the code will initialize from the previous best-fit for faster convergence. - **kwargs_opt (dict): - Optional arguments for the differential evolution - optimization Returns: tuple: Returns three objects: @@ -525,6 +616,13 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): nparams = len(bounds) # Number of parameters in the model popsize = arg_dict['popsize'] # Note this does nothing if the init is done from a previous iteration or optimum nsamples = arg_dict['popsize']*nparams + teltype = arg_dict['tell_dict']['teltype'] + # FD: Assumes shift and stretch are turned on. + if teltype == 'pca': + ntheta_tell = arg_dict['tell_npca']+3 # Total number of telluric model parameters in PCA mode + elif teltype == 'grid': + ntheta_tell = 4+3 # Total number of telluric model parameters in grid mode + # Decide how to initialize if init_from_last is not None: # Use a Gaussian ball about the optimum from a previous iteration @@ -538,9 +636,9 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): init_obj = np.array([[np.clip(param + ballsize*(bounds_obj[i][1] - bounds_obj[i][0]) * rng.standard_normal(1)[0], bounds_obj[i][0], bounds_obj[i][1]) for i, param in enumerate(arg_dict['obj_dict']['init_obj_opt_theta'])] for jsamp in range(nsamples)]) - tell_lhs = utils.lhs(7, samples=nsamples, seed_or_rng=rng) + tell_lhs = utils.lhs(ntheta_tell, samples=nsamples) init_tell = np.array([[bounds[-idim][0] + tell_lhs[isamp, idim] * (bounds[-idim][1] - bounds[-idim][0]) - for idim in range(7)] for isamp in range(nsamples)]) + for idim in range(ntheta_tell)] for isamp in range(nsamples)]) init = np.hstack((init_obj, init_tell)) else: # If this is the first iteration and no object model optimum is presented, use a latin hypercube which is the default @@ -550,10 +648,11 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): init = init, updating='immediate', popsize=popsize, recombination=arg_dict['recombination'], maxiter=arg_dict['diff_evol_maxiter'], polish=arg_dict['polish'], disp=arg_dict['disp']) - theta_obj = result.x[:-7] - theta_tell = result.x[-7:] + + theta_obj = result.x[:-ntheta_tell] + theta_tell = result.x[-ntheta_tell:] tell_model = eval_telluric(theta_tell, arg_dict['tell_dict'], - ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) + ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) obj_model, modelmask = obj_model_func(theta_obj, arg_dict['obj_dict']) totalmask = thismask & modelmask chi_vec = totalmask*(flux - tell_model*obj_model)*np.sqrt(flux_ivar) @@ -571,7 +670,7 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): # TODO This should be a general reader once we get our act together with the data model. # For echelle: read in all the orders into a (nspec, nporders) array -# FOr longslit: read in the stanard into a (nspec, 1) array +# For longslit: read in the standard into a (nspec, 1) array def unpack_orders(sobjs, ret_flam=False): """ Utility function to unpack the sobjs object and return the @@ -620,7 +719,7 @@ def unpack_orders(sobjs, ret_flam=False): # TODO: This function needs to be revisited. Better yet, it would useful to # brainstorm about whether or not it's worth revisiting the spec1d datamodel. -def general_spec_reader(specfile, ret_flam=False): +def general_spec_reader(specfile, ret_flam=False, chk_version=True): """ Read a spec1d file or a coadd spectrum file. @@ -629,6 +728,11 @@ def general_spec_reader(specfile, ret_flam=False): File with the data ret_flam (:obj:`bool`, optional): Return FLAM instead of COUNTS. + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict + version checking to ensure a valid file. If False, the code will + try to keep going, but this may lead to faults and quiet failures. + User beware! Returns: :obj:`tuple`: Seven objects are returned. The first five are @@ -643,7 +747,7 @@ def general_spec_reader(specfile, ret_flam=False): hdul = fits.open(specfile) if 'DMODCLS' in hdul[1].header and hdul[1].header['DMODCLS'] == 'OneSpec': # Load - spec = onespec.OneSpec.from_file(specfile) + spec = onespec.OneSpec.from_file(specfile, chk_version=chk_version) # Unpack wave = spec.wave # wavelength grid evaluated at the bin centers, uniformly-spaced in lambda or log10-lambda/velocity. @@ -656,7 +760,8 @@ def general_spec_reader(specfile, ret_flam=False): spect_dict = spec.spect_meta head = spec.head0 else: - sobjs = specobjs.SpecObjs.from_fitsfile(specfile, chk_version=False) + sobjs = specobjs.SpecObjs.from_fitsfile(specfile, chk_version=chk_version) + # TODO: What bug? Is it fixed now? How can we test if it's fixed? if np.sum(sobjs.OPT_WAVE) is None: raise ValueError("This is an ugly hack until the DataContainer bug is fixed") head = sobjs.header @@ -722,7 +827,8 @@ def save_coadd1d_tofits(outfile, wave, flux, ivar, gpm, wave_grid_mid=None, spec wave_gpm = wave > 1.0 spec = onespec.OneSpec(wave=wave[wave_gpm], wave_grid_mid=None if wave_grid_mid is None else wave_grid_mid[wave_gpm], - flux=flux[wave_gpm], PYP_SPEC=spectrograph, ivar=ivar[wave_gpm], mask=gpm[wave_gpm].astype(int), + flux=flux[wave_gpm], PYP_SPEC=spectrograph, ivar=ivar[wave_gpm], sigma=np.sqrt(utils.inverse(ivar[wave_gpm])), + mask=gpm[wave_gpm].astype(int), telluric=None if telluric is None else telluric[wave_gpm], obj_model=None if obj_model is None else obj_model[wave_gpm], ext_mode=ex_value, fluxed=True) @@ -888,8 +994,8 @@ def init_qso_model(obj_params, iord, wave, flux, ivar, mask, tellmodel): tellmodel : array shape (nspec,) This is a telluric model computed on the wave wavelength grid. Initialization usually requires some initial - best guess for the telluric absorption, which is computed from the midpoint of the telluric model grid parameter - space using the resolution of the spectrograph and the airmass of the observations. + best guess for the telluric absorption, which is computed from the mean of the telluric model grid using + the resolution of the spectrograph. Returns ------- @@ -905,17 +1011,17 @@ def init_qso_model(obj_params, iord, wave, flux, ivar, mask, tellmodel): """ - pca_dict = init_pca(obj_params['pca_file'], wave, obj_params['z_qso'], obj_params['npca']) - pca_mean = np.exp(pca_dict['components'][0, :]) + qso_pca_dict = qso_init_pca(obj_params['pca_file'], wave, obj_params['z_qso'], obj_params['npca']) + qso_pca_mean = np.exp(qso_pca_dict['components'][0, :]) tell_mask = tellmodel > obj_params['tell_norm_thresh'] # Create a reference model and bogus noise - flux_ref = pca_mean * tellmodel - ivar_ref = utils.inverse((pca_mean/100.0) ** 2) + flux_ref = qso_pca_mean * tellmodel + ivar_ref = utils.inverse((qso_pca_mean/100.0) ** 2) flam_norm_inv = coadd.robust_median_ratio(flux, ivar, flux_ref, ivar_ref, mask=mask, mask_ref=tell_mask) flam_norm = 1.0/flam_norm_inv # Set the bounds for the PCA and truncate to the right dimension - coeffs = pca_dict['coeffs'][:,1:obj_params['npca']] + coeffs = qso_pca_dict['coeffs'][:,1:obj_params['npca']] # Compute the min and max arrays of the coefficients which are not the norm, i.e. grab the coeffs that aren't the first one coeff_min = np.amin(coeffs, axis=0) # only coeff_max = np.amax(coeffs, axis=0) @@ -925,7 +1031,7 @@ def init_qso_model(obj_params, iord, wave, flux, ivar, mask, tellmodel): bounds_pca = [(i, j) for i, j in zip(coeff_min, coeff_max)] # Coefficients: determined from PCA model bounds_obj = bounds_z + bounds_flam + bounds_pca # Create the obj_dict - obj_dict = dict(npca=obj_params['npca'], pca_dict=pca_dict) + obj_dict = dict(npca=obj_params['npca'], pca_dict=qso_pca_dict) return obj_dict, bounds_obj @@ -946,23 +1052,21 @@ def eval_qso_model(theta, obj_dict): Returns ------- - pca_model : `numpy.ndarray`_ - PCA vectors were already interpolated onto the telluric model grid by init_qso_model. - array with same shape as the PCA vectors (tored in the obj_dict['pca_dict']) + qso_pca_model : array with same shape as the PCA vectors (stored in the obj_dict['pca_dict']) + PCA vectors were already interpolated onto the telluric model grid by init_qso_model - gpm : `numpy.ndarray`_ - Good pixel mask indicating where the model is valid. - array with same shape as the pca_model + gpm : `numpy.ndarray`_ : array with same shape as the qso_pca_model + Good pixel mask indicating where the model is valid """ - pca_model = pca_eval(theta, obj_dict['pca_dict']) + qso_pca_model = qso_pca_eval(theta, obj_dict['pca_dict']) # TODO Is the prior evaluation slowing things down?? # TODO Disablingthe prior for now as I think it slows things down for no big gain #ln_pca_pri = qso_pca.pca_lnprior(theta_PCA, arg_dict['pca_dict']) #ln_pca_pri = 0.0 #flux_model, tell_model, spec_model, modelmask - return pca_model, (pca_model > 0.0) + return qso_pca_model, (qso_pca_model > 0.0) ############## @@ -1202,8 +1306,10 @@ def eval_poly_model(theta, obj_dict): def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, - telgridfile, log10_blaze_function=None, ech_orders=None, polyorder=8, mask_hydrogen_lines=True, - mask_helium_lines=False, hydrogen_mask_wid=10., resln_guess=None, resln_frac_bounds=(0.5, 1.5), + telgridfile, log10_blaze_function=None, ech_orders=None, polyorder=8, + tell_npca=4, teltype='pca', + mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., + resln_guess=None, resln_frac_bounds=(0.6, 1.4), pix_shift_bounds=(-5.0, 5.0), delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), sn_clip=30.0, ballsize=5e-4, only_orders=None, maxiter=3, lower=3.0, upper=3.0, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, @@ -1252,6 +1358,10 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, If passed, provides the true order numbers for the spectra provided. polyorder : :obj:`int`, optional, default = 8 Polynomial order for the sensitivity function fit. + teltype : :obj:`str`, optional, default = 'pca' + Method for evaluating telluric models, either `pca` or `grid`. + tell_npca : :obj:`int`, optional, default = 4 + Number of telluric PCA components used, must be <= 10 mask_hydrogen_lines : :obj:`bool`, optional If True, mask stellar hydrogen absorption lines before fitting sensitivity function. Default = True mask_helium_lines : :obj:`bool`, optional @@ -1365,7 +1475,9 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, # Since we are fitting a sensitivity function, first compute counts per second per angstrom. TelObj = Telluric(wave, counts, counts_ivar, mask_tot, telgridfile, obj_params, - init_sensfunc_model, eval_sensfunc_model, log10_blaze_function=log10_blaze_function, ech_orders=ech_orders, + init_sensfunc_model, eval_sensfunc_model, log10_blaze_function=log10_blaze_function, + teltype=teltype, tell_npca=tell_npca, + ech_orders=ech_orders, pix_shift_bounds=pix_shift_bounds, resln_guess=resln_guess, resln_frac_bounds=resln_frac_bounds, sn_clip=sn_clip, maxiter=maxiter, lower=lower, upper=upper, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, @@ -1408,11 +1520,13 @@ def create_bal_mask(wave, bal_wv_min_max): -def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, npca=8, +def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, npca=8, pca_lower=1220.0, pca_upper=3100.0, bal_wv_min_max=None, delta_zqso=0.1, + teltype='pca', tell_npca=4, bounds_norm=(0.1, 3.0), tell_norm_thresh=0.9, sn_clip=30.0, only_orders=None, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, - debug_init=False, debug=False, show=False): + pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False, + chk_version=True): """ Fit and correct a QSO spectrum for telluric absorption. @@ -1445,6 +1559,10 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, delta_zqso : :obj:`float`, optional During the fit, the QSO redshift is allowed to vary within ``+/-delta_zqso``. + teltype : :obj:`str`, optional, default = 'pca' + Method for evaluating telluric models, either `pca` or `grid`. + tell_npca : :obj:`int`, optional, default = 4 + Number of telluric PCA components used, must be <=10 bounds_norm : :obj:`tuple`, optional A two-tuple with the lower and upper bounds on the fractional adjustment of the flux in the QSO model spectrum. For example, a value of ``(0.1, @@ -1497,6 +1615,10 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, fits. show : :obj:`bool`, optional Show a QA plot of the final fit. + chk_version : :obj:`bool`, optional + When reading in existing files written by PypeIt, perform strict version + checking to ensure a valid file. If False, the code will try to keep + going, but this may lead to faults and quiet failures. User beware! Returns ------- @@ -1515,7 +1637,8 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, 'lbound_norm', 'ubound_norm', 'tell_norm_thresh'), debug_init=debug_init) - wave, wave_grid_mid, flux, ivar, mask, meta_spec, header = general_spec_reader(spec1dfile, ret_flam=True) + wave, wave_grid_mid, flux, ivar, mask, meta_spec, header \ + = general_spec_reader(spec1dfile, ret_flam=True, chk_version=chk_version) header = fits.getheader(spec1dfile) # clean this up! # Mask the IGM and mask wavelengths that extend redward of our PCA @@ -1527,14 +1650,16 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, # parameters lowered for testing TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_qso_model, - eval_qso_model, sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, - recombination=recombination, polish=polish, disp=disp, debug=debug) + eval_qso_model, pix_shift_bounds=pix_shift_bounds, + sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, teltype=teltype, + tell_npca=tell_npca, recombination=recombination, polish=polish, + disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) # Apply the telluric correction telluric = TelObj.model['TELLURIC'][0,:] - pca_model = TelObj.model['OBJ_MODEL'][0,:] + qso_pca_model = TelObj.model['OBJ_MODEL'][0,:] # Plot the telluric corrected and rescaled orders flux_corr = flux/(telluric + (telluric == 0.0)) @@ -1554,11 +1679,11 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, alpha=0.7, zorder=5) plt.plot(wave, sig_corr, drawstyle='steps-mid', color='r', label='noise', alpha=0.3, zorder=1) - plt.plot(wave, pca_model, color='cornflowerblue', linewidth=1.0, label='PCA model', + plt.plot(wave, qso_pca_model, color='cornflowerblue', linewidth=1.0, label='PCA model', zorder=7, alpha=0.7) - plt.plot(wave, pca_model.max()*0.9*telluric, color='magenta', drawstyle='steps-mid', + plt.plot(wave, qso_pca_model.max()*0.9*telluric, color='magenta', drawstyle='steps-mid', label='telluric', alpha=0.4) - plt.ylim(-0.1*pca_model.max(), 1.5*pca_model.max()) + plt.ylim(-0.1*qso_pca_model.max(), 1.5*qso_pca_model.max()) plt.legend() plt.xlabel('Wavelength') plt.ylabel('Flux') @@ -1566,18 +1691,19 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, # save the telluric corrected spectrum save_coadd1d_tofits(outfile, wave, flux_corr, ivar_corr, mask_corr, wave_grid_mid=wave_grid_mid, - spectrograph=header['PYP_SPEC'], telluric=telluric, obj_model=pca_model, + spectrograph=header['PYP_SPEC'], telluric=telluric, obj_model=qso_pca_model, header=header, ex_value='OPT', overwrite=True) return TelObj -def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, star_mag=None, - star_ra=None, star_dec=None, func='legendre', model='exp', polyorder=5, - mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., - delta_coeff_bounds=(-20.0, 20.0), +def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, + star_mag=None, star_ra=None, star_dec=None, func='legendre', model='exp', + polyorder=5, teltype='pca', tell_npca=4, mask_hydrogen_lines=True, + mask_helium_lines=False, hydrogen_mask_wid=10., delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, - debug_init=False, debug=False, show=False): + pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False, + chk_version=True): """ This needs a doc string. @@ -1594,7 +1720,8 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, disp = True # Read in the data - wave, wave_grid_mid, flux, ivar, mask, meta_spec, header = general_spec_reader(spec1dfile, ret_flam=False) + wave, wave_grid_mid, flux, ivar, mask, meta_spec, header \ + = general_spec_reader(spec1dfile, ret_flam=False, chk_version=chk_version) # Read in standard star dictionary and interpolate onto regular telluric wave_grid star_ra = meta_spec['core']['RA'] if star_ra is None else star_ra star_dec = meta_spec['core']['DEC'] if star_dec is None else star_dec @@ -1634,7 +1761,9 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, # parameters lowered for testing TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_star_model, - eval_star_model, sn_clip=sn_clip, tol=tol, popsize=popsize, + eval_star_model, pix_shift_bounds=pix_shift_bounds, + teltype=teltype, tell_npca=tell_npca, + sn_clip=sn_clip, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -1679,11 +1808,11 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, return TelObj def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func='legendre', - model='exp', polyorder=3, fit_wv_min_max=None, mask_lyman_a=True, - delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), + model='exp', polyorder=3, fit_wv_min_max=None, mask_lyman_a=True, teltype='pca', + tell_npca=4, delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, - recombination=0.7, polish=True, disp=False, debug_init=False, debug=False, - show=False): + recombination=0.7, polish=True, disp=False, pix_shift_bounds=(-5.0,5.0), + debug_init=False, debug=False, show=False, chk_version=True): """ This needs a doc string. @@ -1697,7 +1826,8 @@ def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func disp = True # Read in the data - wave, wave_grid_mid, flux, ivar, mask, meta_spec, header = general_spec_reader(spec1dfile, ret_flam=False) + wave, wave_grid_mid, flux, ivar, mask, meta_spec, header \ + = general_spec_reader(spec1dfile, ret_flam=False, chk_version=chk_version) if flux.ndim == 2: norders = flux.shape[1] @@ -1732,8 +1862,9 @@ def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func # parameters lowered for testing TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_poly_model, - eval_poly_model, sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, - recombination=recombination, polish=polish, disp=disp, debug=debug) + eval_poly_model, pix_shift_bounds=pix_shift_bounds, + sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, teltype=teltype, + tell_npca=tell_npca, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -1783,7 +1914,12 @@ class Telluric(datamodel.DataContainer): The model telluric absorption spectra for the atmosphere are generated by HITRAN atmosphere models with a baseline atmospheric profile model of the - observatory in question. The modeling can be applied to both multi-slit + observatory in question. One can interpolate over grids of these absorption + spectra directly, which have been created for each observatory, or instead + (by default) use a PCA decomposition of the telluric models across all + observatories, which is more flexible and much lighter in file size. + + The modeling can be applied to both multi-slit or echelle data. The code performs a differential evolution optimization using `scipy.optimize.differential_evolution`_. Several iterations are performed with rejection of outliers (governed by the maxiter parameter @@ -1814,26 +1950,29 @@ class Telluric(datamodel.DataContainer): object model, the bounds depend on the nature of the object model, which is why ``init_obj_model`` must be provided. - The telluric model is governed by seven parameters --- ``pressure``, - ``temperature``, ``humidity``, ``airmass``, ``resln``, ``shift``, + The telluric model is governed by seven parameters --- for the `grid` + approach: ``pressure``, ``temperature``, ``humidity``, ``airmass``; for + the `pca` approach 4 PCA coefficients; plus ``resln``, ``shift``, and ``stretch`` --- where ``resln`` is the resolution of the spectrograph and ``shift`` is a shift in pixels. The ``stretch`` is a stretch of the pixel scale. The airmass of the object will be used to initalize the fit (this helps with initalizing the object model), but the models are sufficiently flexible that often the best-fit airmass actually differs from the - airmass of the spectrum. + airmass of the spectrum. In the `pca` approach, the number of PCA + components used can be adjusted between 1 and 10, with a tradeoff between + speed and accuracy. This code can be run on stacked spectra covering a large range of - airmasses and will still provide good results. The resulting airmass will - be an effective value, and as per above may not have much relation to the - true airmass of the observation. The exception to this rule is extremely - high signal-to-noise ratio data S/N > 30, for which it can be difficult - to obtain a good fit within the noise of the data. In such cases, the - user should split up the data into chunks taken around the same airmass, - fit those individually with this class, and then combine the resulting - telluric corrected spectra. This will, in general, result in better fits, - and will also average down the residuals from the telluric model fit in - the final averaged spectrum. + airmasses and will still provide good results. The resulting airmass + (in the `grid` approach) will be an effective value, and as per above may + not have much relation to the true airmass of the observation. The + exception to this rule is extremely high signal-to-noise ratio data + (S/N > 30), for which it can be difficult to obtain a good fit within the + noise of the data. In such cases, the user should split up the data into + chunks taken around the same airmass, fit those individually with this + class, and then combine the resulting telluric corrected spectra. This + will, in general, result in better fits, and will also average down the + residuals from the telluric model fit in the final averaged spectrum. The datamodel attributes are: @@ -1856,8 +1995,19 @@ class Telluric(datamodel.DataContainer): Good pixel gpm for the object in question. Same shape as ``wave``. telgridfile (:obj:`str`): - File containing grid of HITRAN atmosphere models; see + File containing grid of HITRAN atmosphere models or PCA + decomposition of such models, see :class:`~pypeit.par.pypeitpar.TelluricPar`. + teltype (:obj:`str`, optional): + Method for evaluating telluric models, either `pca` or `grid`. + The `grid` method directly interpolates a pre-computed grid of + HITRAN atmospheric models with nearest grid-point interpolation, + while the `pca` method instead fits for the coefficients of a + PCA decomposition of HITRAN atmospheric models, enabling a more + flexible and far more file-size efficient interpolation + of the telluric absorption model space. + tell_npca (:obj:`int`, optional): + Number of telluric PCA components used, must be <=10 obj_params (:obj:`dict`): Dictionary of parameters for initializing the object model. init_obj_model (callable): @@ -2043,7 +2193,12 @@ class Telluric(datamodel.DataContainer): """Datamodel version.""" datamodel = {'telgrid': dict(otype=str, - descr='File containing grid of HITRAN atmosphere models'), + descr='File containing PCA components or grid of ' + 'HITRAN atmosphere models'), + 'teltype': dict(otype=str, + descr='Type of telluric model, `pca` or `grid`'), + 'tell_npca': dict(otype=int, + descr='Number of telluric PCA components used'), 'std_src': dict(otype=str, descr='Name of the standard source'), 'std_name': dict(otype=str, descr='Type of standard source'), 'std_cal': dict(otype=str, @@ -2141,7 +2296,7 @@ class Telluric(datamodel.DataContainer): ] @staticmethod - def empty_model_table(norders, nspec, n_obj_par=0): + def empty_model_table(norders, nspec, tell_npca=4, n_obj_par=0): """ Construct an empty `astropy.table.Table`_ for the telluric model results. @@ -2151,6 +2306,8 @@ def empty_model_table(norders, nspec, n_obj_par=0): The number of slits/orders on the detector. nspec (:obj:`int`): The number of spectral pixels on the detector. + tell_npca (:obj:`int`): + Number of telluric model parameters n_obj_par (:obj:`int`, optional): The number of parameters used to construct the object model spectrum. @@ -2166,16 +2323,10 @@ def empty_model_table(norders, nspec, n_obj_par=0): table.Column(name='OBJ_MODEL', dtype=float, length=norders, shape=(nspec,), description='Best-fitting object model spectrum'), # TODO: Why do we need both TELL_THETA and all the individual parameters... - table.Column(name='TELL_THETA', dtype=float, length=norders, shape=(7,), + table.Column(name='TELL_THETA', dtype=float, length=norders, shape=(tell_npca+3,), description='Best-fitting telluric model parameters'), - table.Column(name='TELL_PRESS', dtype=float, length=norders, - description='Best-fitting telluric model pressure'), - table.Column(name='TELL_TEMP', dtype=float, length=norders, - description='Best-fitting telluric model temperature'), - table.Column(name='TELL_H2O', dtype=float, length=norders, - description='Best-fitting telluric model humidity'), - table.Column(name='TELL_AIRMASS', dtype=float, length=norders, - description='Best-fitting telluric model airmass'), + table.Column(name='TELL_PARAM', dtype=float, length=norders, shape=(tell_npca,), + description='Best-fitting telluric atmospheric parameters or PCA coefficients'), table.Column(name='TELL_RESLN', dtype=float, length=norders, description='Best-fitting telluric model spectral resolution'), table.Column(name='TELL_SHIFT', dtype=float, length=norders, @@ -2203,9 +2354,9 @@ def empty_model_table(norders, nspec, n_obj_par=0): table.Column(name='WAVE_MAX', dtype=float, length=norders, description='Maximum wavelength included in the fit')]) - def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_model, - eval_obj_model, log10_blaze_function=None, ech_orders=None, sn_clip=30.0, airmass_guess=1.5, - resln_guess=None, resln_frac_bounds=(0.5, 1.5), pix_shift_bounds=(-5.0, 5.0), + def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_model, eval_obj_model, + log10_blaze_function=None, ech_orders=None, sn_clip=30.0, teltype='pca', tell_npca=4, + airmass_guess=1.5, resln_guess=None, resln_frac_bounds=(0.3, 1.5), pix_shift_bounds=(-5.0, 5.0), pix_stretch_bounds=(0.9,1.1), maxiter=2, sticky=True, lower=3.0, upper=3.0, seed=777, ballsize = 5e-4, tol=1e-3, diff_evol_maxiter=1000, popsize=30, recombination=0.7, polish=True, disp=False, sensfunc=False, debug=False): @@ -2223,8 +2374,10 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode # 1) Assign arguments self.telgrid = telgridfile + self.teltype = teltype self.obj_params = obj_params self.init_obj_model = init_obj_model + self.tell_npca = tell_npca self.airmass_guess = airmass_guess self.eval_obj_model = eval_obj_model self.ech_orders = ech_orders @@ -2263,8 +2416,15 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode wave, flux, ivar, gpm) # 3) Read the telluric grid and initalize associated parameters wv_gpm = self.wave_in_arr > 1.0 - self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), - wave_max=self.wave_in_arr[wv_gpm].max()) + if self.teltype == 'pca': + self.tell_dict = read_telluric_pca(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), + wave_max=self.wave_in_arr[wv_gpm].max()) + elif self.teltype == 'grid': + self.tell_npca = 4 + self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), + wave_max=self.wave_in_arr[wv_gpm].max()) + + self.wave_grid = self.tell_dict['wave_grid'] self.ngrid = self.wave_grid.size self.resln_guess = wvutils.get_sampling(self.wave_in_arr)[2] \ @@ -2313,8 +2473,8 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode msgs.info(f'Initializing object model for order: {iord}, {counter}/{self.norders}' + f' with user supplied function: {self.init_obj_model.__name__}') tellmodel = eval_telluric(self.tell_guess, self.tell_dict, - ind_lower=self.ind_lower[iord], - ind_upper=self.ind_upper[iord]) + ind_lower=self.ind_lower[iord], + ind_upper=self.ind_upper[iord]) # TODO This is a pretty ugly way to pass in the blaze function. Particularly since now all the other models # (star, qso, poly) are going to have this parameter set in their obj_params dictionary. # Is there something more elegant that can be done with e.g. functools.partial? @@ -2336,6 +2496,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode arg_dict_iord = dict(ivar=self.ivar_arr[self.ind_lower[iord]:self.ind_upper[iord]+1,iord], tell_dict=self.tell_dict, ind_lower=self.ind_lower[iord], ind_upper=self.ind_upper[iord], + tell_npca=self.tell_npca, obj_model_func=self.eval_obj_model, obj_dict=obj_dict, ballsize=self.ballsize, bounds=bounds_iord, rng=self.rng, diff_evol_maxiter=self.diff_evol_maxiter, tol=self.tol, @@ -2380,8 +2541,12 @@ def run(self, only_orders=None): inmask=self.mask_arr[self.ind_lower[iord]:self.ind_upper[iord]+1,iord], maxiter=self.maxiter, lower=self.lower, upper=self.upper, sticky=self.sticky) - self.theta_obj_list[iord] = self.result_list[iord].x[:-7] - self.theta_tell_list[iord] = self.result_list[iord].x[-7:] + if self.teltype == 'pca': + self.theta_obj_list[iord] = self.result_list[iord].x[:-(self.tell_npca+3)] + self.theta_tell_list[iord] = self.result_list[iord].x[-(self.tell_npca+3):] + elif self.teltype == 'grid': + self.theta_obj_list[iord] = self.result_list[iord].x[:-(4+3)] + self.theta_tell_list[iord] = self.result_list[iord].x[-(4+3):] self.obj_model_list[iord], modelmask \ = self.eval_obj_model(self.theta_obj_list[iord], self.obj_dict_list[iord]) self.tellmodel_list[iord] = eval_telluric(self.theta_tell_list[iord], self.tell_dict, @@ -2432,7 +2597,7 @@ def init_output(self): """ self.model = self.empty_model_table(self.norders, self.nspec_in, - n_obj_par=self.max_ntheta_obj) + tell_npca=self.tell_npca, n_obj_par=self.max_ntheta_obj) if 'output_meta_keys' in self.obj_params: for key in self.obj_params['output_meta_keys']: if key.lower() in self.datamodel.keys(): @@ -2460,6 +2625,11 @@ def assign_output(self, iord): gdwave = self.wave_in_arr[:,iord] > 1.0 wave_in_gd = self.wave_in_arr[gdwave,iord] wave_grid_now = self.wave_grid[self.ind_lower[iord]:self.ind_upper[iord]+1] + + if self.teltype == 'pca': + ntell = self.tell_npca + elif self.teltype == 'grid': + ntell = 4 self.model['WAVE'][iord] = self.wave_in_arr[:,iord] self.model['TELLURIC'][iord][gdwave] \ @@ -2471,13 +2641,10 @@ def assign_output(self, iord): kind='linear', bounds_error=False, fill_value=0.0)(wave_in_gd) self.model['TELL_THETA'][iord] = self.theta_tell_list[iord] - self.model['TELL_PRESS'][iord] = self.theta_tell_list[iord][0] - self.model['TELL_TEMP'][iord] = self.theta_tell_list[iord][1] - self.model['TELL_H2O'][iord] = self.theta_tell_list[iord][2] - self.model['TELL_AIRMASS'][iord] = self.theta_tell_list[iord][3] - self.model['TELL_RESLN'][iord] = self.theta_tell_list[iord][4] - self.model['TELL_SHIFT'][iord] = self.theta_tell_list[iord][5] - self.model['TELL_STRETCH'][iord] = self.theta_tell_list[iord][6] + self.model['TELL_PARAM'][iord] = self.theta_tell_list[iord][:ntell] + self.model['TELL_RESLN'][iord] = self.theta_tell_list[iord][ntell] + self.model['TELL_SHIFT'][iord] = self.theta_tell_list[iord][ntell+1] + self.model['TELL_STRETCH'][iord] = self.theta_tell_list[iord][ntell+2] ntheta_iord = len(self.theta_obj_list[iord]) self.model['OBJ_THETA'][iord][:ntheta_iord] = self.theta_obj_list[iord] self.model['CHI2'][iord] = self.result_list[iord].fun @@ -2552,14 +2719,20 @@ def get_tell_guess(self): function. Returns: - :obj:`tuple`: The guess pressure, temperature, humidity, - airmass, resolution, shift, and stretch parameters. The first - three are the median of the parameters covered by the telluric - model grid. + :obj:`tuple`: The guess telluric model parameters, + resolution, shift, and stretch parameters. """ - return np.median(self.tell_dict['pressure_grid']), np.median(self.tell_dict['temp_grid']), \ - np.median(self.tell_dict['h2o_grid']), self.airmass_guess, self.resln_guess, \ - 0.0, 1.0 + if self.teltype == 'grid': + guess = [np.median(self.tell_dict['pressure_grid'])] + guess.append(np.median(self.tell_dict['temp_grid'])) + guess.append(np.median(self.tell_dict['h2o_grid'])) + guess.append(self.airmass_guess) + else: + guess = list(np.zeros(self.tell_npca)) + guess.append(self.resln_guess) + guess.append(0.0) + guess.append(1.0) + return tuple(guess) def get_bounds_tell(self): """ @@ -2567,18 +2740,30 @@ def get_bounds_tell(self): Returns: :obj:`list`: A list of two-tuples, where each two-tuple proives - the minimum and maximum allowed model values for the pressure, - temperature, humidity, airmass, resolution, shift, and - stretch parameters. + the minimum and maximum allowed model values for the PCA coefficients, + (or for the `grid` approach: pressure, temperature, humidity, airmass) + as well as the resolution, shift, and stretch parameters. """ # Set the bounds for the optimization - return [(self.tell_dict['pressure_grid'].min(), self.tell_dict['pressure_grid'].max()), - (self.tell_dict['temp_grid'].min(), self.tell_dict['temp_grid'].max()), - (self.tell_dict['h2o_grid'].min(), self.tell_dict['h2o_grid'].max()), - (self.tell_dict['airmass_grid'].min(), self.tell_dict['airmass_grid'].max()), - (self.resln_guess * self.resln_frac_bounds[0], - self.resln_guess * self.resln_frac_bounds[1]), - self.pix_shift_bounds, self.pix_stretch_bounds] + bounds = [] + if self.teltype == 'grid': + bounds.append((self.tell_dict['pressure_grid'].min(), + self.tell_dict['pressure_grid'].max())) + bounds.append((self.tell_dict['temp_grid'].min(), + self.tell_dict['temp_grid'].max())) + bounds.append((self.tell_dict['h2o_grid'].min(), + self.tell_dict['h2o_grid'].max())) + bounds.append((self.tell_dict['airmass_grid'].min(), + self.tell_dict['airmass_grid'].max())) + else: # i.e. for teltype == 'pca' + for ii in range(self.tell_npca): + bounds.append((self.tell_dict['bounds_tell_pca'][0][ii+1], + self.tell_dict['bounds_tell_pca'][1][ii+1])) + bounds.append((self.resln_guess * self.resln_frac_bounds[0], + self.resln_guess * self.resln_frac_bounds[1])) + bounds.append(self.pix_shift_bounds) + bounds.append(self.pix_stretch_bounds) + return bounds def sort_telluric(self): """ @@ -2600,10 +2785,14 @@ def sort_telluric(self): # Do a quick loop over all the orders to sort them in order of strongest # to weakest telluric absorption for iord in range(self.norders): - tm_grid = self.tell_dict['tell_grid'][...,self.ind_lower[iord]:self.ind_upper[iord]+1] - tell_model_mid = tm_grid[tm_grid.shape[0]//2, tm_grid.shape[1]//2, - tm_grid.shape[2]//2, tm_grid.shape[3]//2, :] - tell_med[iord] = np.mean(tell_model_mid) + if self.teltype == 'grid': + tm_grid = self.tell_dict['tell_grid'][...,self.ind_lower[iord]:self.ind_upper[iord]+1] + tell_model_mid = tm_grid[tm_grid.shape[0]//2, tm_grid.shape[1]//2, + tm_grid.shape[2]//2, tm_grid.shape[3]//2, :] + tell_med[iord] = np.mean(tell_model_mid) + else: + tell_model_mean = self.tell_dict['tell_pca'][0,self.ind_lower[iord]:self.ind_upper[iord]+1] + tell_med[iord] = np.mean(np.exp(-np.sinh(tell_model_mean))) # Perform fits in order of telluric strength return tell_med.argsort() diff --git a/pypeit/core/wavecal/autoid.py b/pypeit/core/wavecal/autoid.py index d9a2cfcfb7..fcd00cd5d2 100644 --- a/pypeit/core/wavecal/autoid.py +++ b/pypeit/core/wavecal/autoid.py @@ -52,7 +52,7 @@ def arc_fit_qa(waveFit, outfile (:obj:`str`, optional): Name of output file or 'show' to show on screen ids_only (bool, optional): - ?? + Only show the main panel with the arc spectrum and the identified lines title (:obj:`str`, optional): Add a title to the spectrum plot log (:obj:`bool`, optional): @@ -142,6 +142,8 @@ def arc_fit_qa(waveFit, # Title if title is not None: fig.suptitle(title, fontsize='x-large', va='top') + + # If we're only plotting the ID panel, save the figure and return if ids_only: plt.tight_layout(pad=0.2, h_pad=0.0, w_pad=0.0) if outfile is None: @@ -911,7 +913,7 @@ def map_fwhm(image, gpm, slits_left, slits_right, slitmask, npixel=None, nsample msgs.info(f"Calculating spectral resolution of slit {sl + 1}/{nslits}") # Fraction along the slit in the spatial direction to sample the arc line width nmeas = int(0.5+slit_lengths[sl]/_npixel) if nsample is None else nsample - slitsamp = np.linspace(0.01, 0.99, nmeas) + slitsamp = np.linspace(0.05, 0.95, nmeas) this_samp, this_cent, this_fwhm = np.array([]), np.array([]), np.array([]) for ss in range(nmeas): spat_vec = np.atleast_2d((1-slitsamp[ss]) * slits_left[:, sl] + slitsamp[ss] * slits_right[:, sl]).T diff --git a/pypeit/core/wavecal/templates.py b/pypeit/core/wavecal/templates.py index 893bc9489d..44bb9ca0be 100644 --- a/pypeit/core/wavecal/templates.py +++ b/pypeit/core/wavecal/templates.py @@ -266,6 +266,7 @@ def pypeit_arcspec(in_file, slit, binspec, binning=None): # minx=iwv_calib['fmin'], maxx=iwv_calib['fmax']) flux = np.array(iwv_calib['spec']).flatten() elif '.fits' in in_file: + # TODO: This should not default to chk_version=False ... wvcalib = wavecalib.WaveCalib.from_file(in_file, chk_version=False) idx = np.where(wvcalib.spat_ids == slit)[0][0] flux = wvcalib.arc_spectra[:,idx] diff --git a/pypeit/core/wavecal/wv_fitting.py b/pypeit/core/wavecal/wv_fitting.py index 7ab7fc40f0..c8d0eb242a 100644 --- a/pypeit/core/wavecal/wv_fitting.py +++ b/pypeit/core/wavecal/wv_fitting.py @@ -68,8 +68,35 @@ class WaveFit(datamodel.DataContainer): @staticmethod def hduext_prefix_from_spatid(spat_id): - """ Naming for HDU extensions""" + """ + Use the slit spatial ID number to construct the prefix for an output + HDU. + + Args: + spat_id (:obj:`int`): + Slit/order spatial ID number + + Returns: + :obj:`str`: The prefix to use for HDU extensions associated with + this data container. + """ return 'SPAT_ID-{}_'.format(spat_id) + + @staticmethod + def parse_spatid_from_hduext(ext): + """ + Parse the slit spatial ID integer from a *full* HDU extension name. + + Args: + ext (:obj:`str`): + Full extension name. + + Returns: + :obj:`int`: The parsed slit spatial ID number. Returns None if the + extension name does not start with ``'SPAT_ID-'``. + """ + return int(ext.replace('SPAT_ID-', '').split('_')[0]) \ + if ext.startswith('SPAT_ID-') else None def __init__(self, spat_id, ech_order=None, pypeitfit=None, pixel_fit=None, wave_fit=None, ion_bits=None, cen_wave=None, cen_disp=None, spec=None, wave_soln=None, @@ -96,8 +123,7 @@ def _bundle(self, **kwargs): # Extension prefix (for being unique with slits) hdue_pref = self.hduext_prefix_from_spatid(self.spat_id) # Without PypeItFit - _d = super(WaveFit, self)._bundle( - ext=hdue_pref+'WAVEFIT', **kwargs) + _d = super()._bundle(ext=hdue_pref+'WAVEFIT', **kwargs) # Deal with PypeItFit if _d[0][hdue_pref+'WAVEFIT']['pypeitfit'] is not None: _d.append({hdue_pref+'PYPEITFIT': _d[0][hdue_pref + 'WAVEFIT'].pop('pypeitfit')}) @@ -129,31 +155,61 @@ def to_hdu(self, **kwargs): return super().to_hdu(force_to_bintbl=True, **kwargs) @classmethod - def from_hdu(cls, hdu, **kwargs): + def from_hdu(cls, hdu, hdu_prefix=None, spat_id=None, **kwargs): """ - Parse the data from the provided HDU. + Parse the data from one or more fits objects. - See the base-class :func:`~pypeit.datamodel.DataContainer.from_hdu` for - the argument descriptions. + .. note:: + ``spat_id`` and ``hdu_prefix`` are mutually exclusive, with + ``hdu_prefix`` taking precedence. If *both* are None, the code + attempts to read the spatial ID number from the HDU extension + name(s) to construct the appropriate prefix. In the latter + case, only the first :class:`WaveFit` object will be read if the + HDU object contains many. + Args: + hdu (`astropy.io.fits.HDUList`_, `astropy.io.fits.ImageHDU`_, `astropy.io.fits.BinTableHDU`_): + The HDU(s) with the data to use for instantiation. + hdu_prefix (:obj:`str`, optional): + Only parse HDUs with extension names matched to this prefix. If + None, ``spat_id`` will be used to construct the prefix if it is + provided, otherwise all extensions are parsed. + spat_id (:obj:`int`, optional): + Spatial ID number for the relevant slit/order for this + wavelength fit. If provided, this is used to construct the + ``hdu_prefix`` using + :func:`~pypeit.core.wavecal.wv_fitting.WaveFit.hduext_prefix_from_spatid`. kwargs: Passed directly to the base-class - :class:`~pypeit.datamodel.DataContainer.from_hdu`. Should not - include ``hdu_prefix`` because this is set directly by the - class, read from the SPAT_ID card in a relevant header. + :class:`~pypeit.datamodel.DataContainer.from_hdu`. Returns: :class:`WaveFit`: Object instantiated from data in the provided HDU. """ - if 'hdu_prefix' in kwargs: - kwargs.pop('hdu_prefix') - # Set hdu_prefix - spat_id = hdu[1].header['SPAT_ID'] if isinstance(hdu, fits.HDUList)\ - else hdu.header['SPAT_ID'] - hdu_prefix = cls.hduext_prefix_from_spatid(spat_id) - # Run the default parser to get the data - return super(WaveFit, cls).from_hdu(hdu, hdu_prefix=hdu_prefix, **kwargs) + # Get the HDU prefix + if hdu_prefix is not None: + _hdu_prefix = hdu_prefix + elif spat_id is not None: + _hdu_prefix = cls.hduext_prefix_from_spatid(spat_id) + else: + # Try to get it from the provided HDU(s) + _hdu = hdu if isinstance(hdu, fits.HDUList) else [hdu] + _hdu_prefix = None + for h in _hdu: + _spat_id = cls.parse_spatid_from_hduext(h.name) + if _spat_id is None: + continue + _hdu_prefix = cls.hduext_prefix_from_spatid(_spat_id) + break + # At this point, _hdu_prefix will either be None because it couldn't + # find parse the slit ID, or it will be the correct prefix. + # TODO: Try to read the data in either case (what is currently + # done), or fault because the _bundle function *always* includes a + # prefix? + + # Construct and return the object + return super().from_hdu(hdu, hdu_prefix=_hdu_prefix, **kwargs) @property def ions(self): diff --git a/pypeit/core/wavecal/wvutils.py b/pypeit/core/wavecal/wvutils.py index 2f9a2b21af..e332c89d79 100644 --- a/pypeit/core/wavecal/wvutils.py +++ b/pypeit/core/wavecal/wvutils.py @@ -244,7 +244,6 @@ def get_wave_grid(waves=None, gpms=None, wave_method='linear', iref=0, wave_grid dloglam_pix = dloglam else: dloglam_pix = dloglam_data - # Generate wavelength array wave_grid, wave_grid_mid, dsamp = wavegrid(wave_grid_min, wave_grid_max, dloglam_pix, spec_samp_fact=spec_samp_fact, log10=True) diff --git a/pypeit/data/sky_spec/sky_LRISr_600_7500_5460_7950.fits b/pypeit/data/sky_spec/sky_LRISr_600_7500_5460_7950.fits index 0d926ff155..0231ab5a88 100644 Binary files a/pypeit/data/sky_spec/sky_LRISr_600_7500_5460_7950.fits and b/pypeit/data/sky_spec/sky_LRISr_600_7500_5460_7950.fits differ diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Dec-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Dec-17.log new file mode 100644 index 0000000000..7e604036cd --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Dec-17.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2008-Dec-17.log +# datestr = Wed Dec 17 16:23:10 2008 +# mode = Direct +# flatfile1 = d1218_0040.fits +# flatfile2 = d1218_0041.fits +# flatfile3 = d1218_0042.fits +# flatfile4 = d1218_0043.fits +# flatfile5 = d1218_0044.fits +# zerofile1 = d1218_0037.fits +# zerofile2 = d1218_0038.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2024 0.0024215 2.5444 0.013172 + 2 1 1 B 1.2112 0.0035957 2.5395 0.0075393 + 3 4 2 A 1.1864 0.0024307 2.4875 0.0050964 + 4 3 2 B 1.1736 0.0060077 2.4716 0.012121 + 5 6 3 A 1.1687 0.0015475 2.5022 0.032810 + 6 5 3 B 1.2725 0.0034453 2.6680 0.0072236 + 7 8 4 A 1.2260 0.0010171 2.5729 0.0035058 + 8 7 4 B 1.2137 0.00081794 2.5801 0.0037044 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Jul-09.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Jul-09.log new file mode 100644 index 0000000000..769cd7a7ad --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Jul-09.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2008-Jul-09.log +# datestr = Wed Jul 9 20:30:03 2008 +# mode = Direct +# flatfile1 = d0710_0004.fits +# flatfile2 = d0710_0005.fits +# flatfile3 = d0710_0006.fits +# flatfile4 = d0710_0007.fits +# flatfile5 = d0710_0008.fits +# zerofile1 = d0710_0001.fits +# zerofile2 = d0710_0002.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2009 0.0064159 2.5179 0.013452 + 2 1 1 B 1.2260 0.0035678 2.5706 0.0074806 + 3 4 2 A 1.1919 0.0043200 2.4990 0.0090574 + 4 3 2 B 1.1760 0.0022843 2.4656 0.0047896 + 5 6 3 A 1.1750 0.0013504 2.4636 0.0028317 + 6 5 3 B 1.2672 0.0044587 2.6569 0.0093487 + 7 8 4 A 1.2208 0.0016011 2.5596 0.0033569 + 8 7 4 B 1.2106 0.0016594 2.5382 0.0034793 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Nov-19.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Nov-19.log new file mode 100644 index 0000000000..04043087ed --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Nov-19.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2008-Nov-19.log +# datestr = Wed Nov 19 18:21:21 2008 +# mode = Direct +# flatfile1 = d1120_0040.fits +# flatfile2 = d1120_0041.fits +# flatfile3 = d1120_0042.fits +# flatfile4 = d1120_0043.fits +# flatfile5 = d1120_0044.fits +# zerofile1 = d1120_0037.fits +# zerofile2 = d1120_0038.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2134 0.0011958 2.5441 0.0025072 + 2 1 1 B 1.2110 0.0019706 2.5390 0.0041318 + 3 4 2 A 1.1996 0.00058394 2.5151 0.0012244 + 4 3 2 B 1.1848 0.0027476 2.4842 0.0057606 + 5 6 3 A 1.1806 0.0025219 2.4752 0.0052876 + 6 5 3 B 1.2855 0.0029711 2.6952 0.0062293 + 7 8 4 A 1.2272 0.0069947 2.5731 0.014666 + 8 7 4 B 1.2223 0.0021113 2.5627 0.0044266 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Apr-15.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Apr-15.log new file mode 100644 index 0000000000..14b08a486a --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Apr-15.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2009-Apr-15.log +# datestr = Wed Apr 15 23:58:57 2009 +# mode = Direct +# flatfile1 = d0410_0405.fits +# flatfile2 = d0410_0406.fits +# flatfile3 = d0410_0407.fits +# flatfile4 = d0410_0408.fits +# flatfile5 = d0410_0409.fits +# zerofile1 = d0410_0402.fits +# zerofile2 = d0410_0403.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2265 0.016408 2.5715 0.034402 + 2 1 1 B 1.2128 0.0032306 2.5429 0.0067733 + 3 4 2 A 1.2521 0.044030 2.6252 0.092317 + 4 3 2 B 1.1725 0.0064645 2.4584 0.013554 + 5 6 3 A 1.2533 0.045157 2.6278 0.094680 + 6 5 3 B 1.2916 0.011757 2.7080 0.024651 + 7 8 4 A 1.3476 0.074654 2.8255 0.15653 + 8 7 4 B 1.1962 0.0043081 2.5081 0.0090330 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Dec-08.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Dec-08.log new file mode 100644 index 0000000000..276da76604 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Dec-08.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2009-Dec-08.log +# datestr = Tue Dec 8 15:28:23 2009 +# mode = Direct +# flatfile1 = d1209_0011.fits +# flatfile2 = d1209_0012.fits +# flatfile3 = d1209_0013.fits +# flatfile4 = d1209_0014.fits +# flatfile5 = d1209_0015.fits +# zerofile1 = d1209_0008.fits +# zerofile2 = d1209_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2127 0.011772 2.5426 0.024683 + 2 1 1 B 1.2241 0.0095496 2.5666 0.020023 + 3 4 2 A 1.1937 0.020458 2.5029 0.042894 + 4 3 2 B 1.1922 0.013859 2.4996 0.029058 + 5 6 3 A 1.1806 0.018605 2.4753 0.039008 + 6 5 3 B 1.2891 0.022812 2.7028 0.047830 + 7 8 4 A 1.2198 0.010162 2.5576 0.021306 + 8 7 4 B 1.2326 0.015397 2.5845 0.032283 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jan-21.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jan-21.log new file mode 100644 index 0000000000..6050af87f4 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jan-21.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2009-Jan-21.log +# datestr = Wed Jan 21 09:40:08 2009 +# mode = Direct +# flatfile1 = d0122_0024.fits +# flatfile2 = d0122_0025.fits +# flatfile3 = d0122_0026.fits +# flatfile4 = d0122_0027.fits +# flatfile5 = d0122_0028.fits +# zerofile1 = d0122_0021.fits +# zerofile2 = d0122_0022.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2171 0.0053002 2.5519 0.011113 + 2 1 1 B 1.2170 0.0042223 2.5517 0.0088528 + 3 4 2 A 1.1975 0.0069933 2.5108 0.014663 + 4 3 2 B 1.1870 0.00068344 2.4887 0.0014330 + 5 6 3 A 1.1794 0.0041459 2.4729 0.0086926 + 6 5 3 B 1.2890 0.00057864 2.7027 0.0012135 + 7 8 4 A 1.2213 0.0015275 2.5607 0.0032028 + 8 7 4 B 1.2251 0.00097453 2.5687 0.0020431 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jun-18.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jun-18.log new file mode 100644 index 0000000000..551f4903a7 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jun-18.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2009-Jun-18.log +# datestr = Thu Jun 18 18:33:56 2009 +# mode = Direct +# flatfile1 = d0619_0004.fits +# flatfile2 = d0619_0005.fits +# flatfile3 = d0619_0006.fits +# flatfile4 = d0619_0007.fits +# flatfile5 = d0619_0008.fits +# zerofile1 = d0619_0001.fits +# zerofile2 = d0619_0002.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2179 0.0032398 2.5536 0.0067927 + 2 1 1 B 1.2157 0.0061805 2.5490 0.012959 + 3 4 2 A 1.2046 0.0019324 2.5256 0.0040515 + 4 3 2 B 1.1876 0.00090507 2.4900 0.0018975 + 5 6 3 A 1.1806 0.0052213 2.4753 0.010948 + 6 5 3 B 1.2764 0.0054817 2.6763 0.011493 + 7 8 4 A 1.2302 0.0040342 2.5793 0.0084583 + 8 7 4 B 1.2193 0.00030576 2.5564 0.00064120 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Mar-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Mar-17.log new file mode 100644 index 0000000000..891f643ff1 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Mar-17.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2009-Mar-17.log +# datestr = Tue Mar 17 18:51:42 2009 +# mode = Direct +# flatfile1 = d0318_0013.fits +# flatfile2 = d0318_0014.fits +# flatfile3 = d0318_0015.fits +# flatfile4 = d0318_0016.fits +# flatfile5 = d0318_0017.fits +# zerofile1 = d0318_0010.fits +# zerofile2 = d0318_0011.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2151 0.0021988 2.5476 0.0046100 + 2 1 1 B 1.2082 0.0013784 2.5332 0.0028900 + 3 4 2 A 1.1972 0.0052019 2.5102 0.010907 + 4 3 2 B 1.1881 0.0012605 2.4911 0.0026426 + 5 6 3 A 1.1790 0.0035966 2.4720 0.0075410 + 6 5 3 B 1.2847 0.0057194 2.6937 0.011992 + 7 8 4 A 1.2175 0.010660 2.5527 0.022351 + 8 7 4 B 1.2256 0.0014185 2.5697 0.0029741 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Sep-30.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Sep-30.log new file mode 100644 index 0000000000..a15cf1e592 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Sep-30.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2009-Sep-30.log +# datestr = Wed Sep 30 11:59:22 2009 +# mode = Direct +# flatfile1 = d1001_0004.fits +# flatfile2 = d1001_0005.fits +# flatfile3 = d1001_0006.fits +# flatfile4 = d1001_0007.fits +# flatfile5 = d1001_0008.fits +# zerofile1 = d1001_0001.fits +# zerofile2 = d1001_0002.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2181 0.014474 2.5539 0.030348 + 2 1 1 B 1.2283 0.025599 2.5754 0.053672 + 3 4 2 A 1.1979 0.013589 2.5116 0.028492 + 4 3 2 B 1.1929 0.016964 2.5011 0.035569 + 5 6 3 A 1.1819 0.011609 2.4781 0.024341 + 6 5 3 B 1.2943 0.011172 2.7138 0.023424 + 7 8 4 A 1.2275 0.014618 2.5737 0.030649 + 8 7 4 B 1.2285 0.013163 2.5757 0.027599 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Feb-02.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Feb-02.log new file mode 100644 index 0000000000..6290526c73 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Feb-02.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2010-Feb-02.log +# datestr = Tue Feb 2 15:36:23 2010 +# mode = Direct +# flatfile1 = d0203_0023.fits +# flatfile2 = d0203_0024.fits +# flatfile3 = d0203_0025.fits +# flatfile4 = d0203_0026.fits +# flatfile5 = d0203_0027.fits +# zerofile1 = d0203_0020.fits +# zerofile2 = d0203_0021.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2259 0.015300 2.5704 0.032080 + 2 1 1 B 1.2463 0.022173 2.6131 0.046489 + 3 4 2 A 1.2265 0.018076 2.5717 0.037899 + 4 3 2 B 1.2089 0.010507 2.5347 0.022030 + 5 6 3 A 1.1995 0.025882 2.5150 0.054266 + 6 5 3 B 1.3028 0.025062 2.7315 0.052547 + 7 8 4 A 1.2350 0.013071 2.5893 0.027406 + 8 7 4 B 1.2363 0.010014 2.5921 0.020996 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jan-07.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jan-07.log new file mode 100644 index 0000000000..5ab65bdfd1 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jan-07.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2010-Jan-07.log +# datestr = Thu Jan 7 18:49:08 2010 +# mode = Direct +# flatfile1 = d0108_0017.fits +# flatfile2 = d0108_0018.fits +# flatfile3 = d0108_0019.fits +# flatfile4 = d0108_0020.fits +# flatfile5 = d0108_0021.fits +# zerofile1 = d0108_0014.fits +# zerofile2 = d0108_0015.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2128 0.014558 3.8075 0.052824 + 2 1 1 B 1.2333 0.015789 2.5857 0.033105 + 3 4 2 A 1.2017 0.012996 2.5830 0.11089 + 4 3 2 B 1.1966 0.013201 2.5417 0.097460 + 5 6 3 A 1.1857 0.013864 3.7070 0.072131 + 6 5 3 B 1.2848 0.021610 2.7633 0.16351 + 7 8 4 A 1.2273 0.014075 3.7922 0.94343 + 8 7 4 B 1.2287 0.017110 3.8168 0.14523 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jun-06.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jun-06.log new file mode 100644 index 0000000000..dc77843920 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jun-06.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2010-Jun-06.log +# datestr = Sun Jun 6 18:32:17 2010 +# mode = Direct +# flatfile1 = d0607_0023.fits +# flatfile2 = d0607_0024.fits +# flatfile3 = d0607_0025.fits +# flatfile4 = d0607_0026.fits +# flatfile5 = d0607_0027.fits +# zerofile1 = d0607_0020.fits +# zerofile2 = d0607_0021.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2053 0.021831 2.5271 0.045774 + 2 1 1 B 1.2251 0.017808 2.5686 0.037339 + 3 4 2 A 1.1922 0.019690 2.4996 0.041285 + 4 3 2 B 1.1790 0.017103 2.4720 0.035860 + 5 6 3 A 1.1781 0.014910 2.4701 0.031260 + 6 5 3 B 1.2919 0.017971 2.7087 0.037680 + 7 8 4 A 1.2159 0.012018 2.5493 0.025198 + 8 7 4 B 1.2086 0.016910 2.5341 0.035454 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Mar-02.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Mar-02.log new file mode 100644 index 0000000000..86c15117a3 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Mar-02.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2010-Mar-02.log +# datestr = Tue Mar 2 11:56:58 2010 +# mode = Direct +# flatfile1 = d0303_0025.fits +# flatfile2 = d0303_0026.fits +# flatfile3 = d0303_0027.fits +# flatfile4 = d0303_0028.fits +# flatfile5 = d0303_0029.fits +# zerofile1 = d0303_0022.fits +# zerofile2 = d0303_0023.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2301 0.015997 2.5792 0.033540 + 2 1 1 B 1.2431 0.012607 2.6064 0.026432 + 3 4 2 A 1.2162 0.017342 2.5499 0.036360 + 4 3 2 B 1.2022 0.017339 2.5206 0.036355 + 5 6 3 A 1.1957 0.020572 2.5491 0.093522 + 6 5 3 B 1.2961 0.025214 2.7175 0.052866 + 7 8 4 A 1.2345 0.010414 2.5884 0.021834 + 8 7 4 B 1.2352 0.017529 2.5898 0.036752 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-May-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-May-05.log new file mode 100644 index 0000000000..9c4daa7b92 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-May-05.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2010-May-05.log +# datestr = Wed May 5 11:13:58 2010 +# mode = Direct +# flatfile1 = d0506_0025.fits +# flatfile2 = d0506_0026.fits +# flatfile3 = d0506_0027.fits +# flatfile4 = d0506_0028.fits +# flatfile5 = d0506_0029.fits +# zerofile1 = d0506_0022.fits +# zerofile2 = d0506_0023.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2130 0.017112 2.5434 0.035878 + 2 1 1 B 1.2354 0.0080350 2.5902 0.016847 + 3 4 2 A 1.1952 0.016490 2.5060 0.034575 + 4 3 2 B 1.1975 0.015944 2.5107 0.033430 + 5 6 3 A 1.1855 0.0090983 2.4855 0.019076 + 6 5 3 B 1.2856 0.019140 2.6955 0.040130 + 7 8 4 A 1.2275 0.018102 2.5736 0.037954 + 8 7 4 B 1.2193 0.012532 2.5564 0.026274 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Sep-01.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Sep-01.log new file mode 100644 index 0000000000..4c126e2be3 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Sep-01.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2010-Sep-01.log +# datestr = Wed Sep 1 07:29:09 2010 +# mode = Direct +# flatfile1 = d0901_0039.fits +# flatfile2 = d0901_0040.fits +# flatfile3 = d0901_0041.fits +# flatfile4 = d0901_0042.fits +# flatfile5 = d0901_0043.fits +# zerofile1 = d0901_0036.fits +# zerofile2 = d0901_0037.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2164 0.015648 2.5504 0.032810 + 2 1 1 B 1.2322 0.019232 2.5836 0.040324 + 3 4 2 A 1.2026 0.020414 2.5215 0.042801 + 4 3 2 B 1.1900 0.010322 2.4950 0.021643 + 5 6 3 A 1.1850 0.015011 2.4845 0.031474 + 6 5 3 B 1.2883 0.019664 2.7012 0.041229 + 7 8 4 A 1.2340 0.018733 2.5873 0.039277 + 8 7 4 B 1.2239 0.010707 2.5662 0.022449 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Feb-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Feb-26.log new file mode 100644 index 0000000000..5cb616e3f0 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Feb-26.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2011-Feb-26.log +# datestr = Sat Feb 26 21:44:10 2011 +# mode = Direct +# flatfile1 = d0227_0052.fits +# flatfile2 = d0227_0053.fits +# flatfile3 = d0227_0054.fits +# flatfile4 = d0227_0055.fits +# flatfile5 = d0227_0056.fits +# zerofile1 = d0227_0049.fits +# zerofile2 = d0227_0050.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2369 0.015895 2.5933 0.033327 + 2 1 1 B 1.2451 0.017819 2.6105 0.037362 + 3 4 2 A 1.2236 0.015896 2.5654 0.033330 + 4 3 2 B 1.2179 0.019184 2.5535 0.040223 + 5 6 3 A 1.2036 0.014354 2.5235 0.030096 + 6 5 3 B 1.2991 0.022734 2.7239 0.047666 + 7 8 4 A 1.2369 0.012008 2.5934 0.025177 + 8 7 4 B 1.2435 0.0092990 2.6073 0.019497 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Mar-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Mar-22.log new file mode 100644 index 0000000000..1beb9486e4 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Mar-22.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2011-Mar-22.log +# datestr = Tue Mar 22 16:21:02 2011 +# mode = Direct +# flatfile1 = d0323_0024.fits +# flatfile2 = d0323_0025.fits +# flatfile3 = d0323_0026.fits +# flatfile4 = d0323_0027.fits +# flatfile5 = d0323_0028.fits +# zerofile1 = d0323_0021.fits +# zerofile2 = d0323_0022.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2192 0.018642 2.5563 0.039087 + 2 1 1 B 1.2375 0.014153 2.5946 0.029674 + 3 4 2 A 1.2025 0.020976 2.5213 0.043980 + 4 3 2 B 1.1922 0.014358 2.4997 0.030103 + 5 6 3 A 1.1836 0.018015 2.4815 0.037772 + 6 5 3 B 1.2859 0.017405 2.6961 0.036493 + 7 8 4 A 1.2277 0.011987 2.5741 0.025133 + 8 7 4 B 1.2268 0.017227 2.5723 0.036119 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-May-25.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-May-25.log new file mode 100644 index 0000000000..7e3191c310 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-May-25.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2011-May-25.log +# datestr = Wed May 25 15:27:19 2011 +# mode = Direct +# flatfile1 = d0526_0011.fits +# flatfile2 = d0526_0012.fits +# flatfile3 = d0526_0013.fits +# flatfile4 = d0526_0014.fits +# flatfile5 = d0526_0015.fits +# zerofile1 = d0526_0008.fits +# zerofile2 = d0526_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2142 0.013147 2.5458 0.027565 + 2 1 1 B 1.2394 0.0086886 2.5985 0.018217 + 3 4 2 A 1.2016 0.020383 2.5193 0.042736 + 4 3 2 B 1.1954 0.014341 2.5065 0.030069 + 5 6 3 A 1.1895 0.020515 2.4940 0.043013 + 6 5 3 B 1.2912 0.019684 2.7073 0.041271 + 7 8 4 A 1.2209 0.014744 2.5598 0.030912 + 8 7 4 B 1.2300 0.016366 2.5789 0.034315 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Oct-25.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Oct-25.log new file mode 100644 index 0000000000..47cd9540dc --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Oct-25.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2011-Oct-25.log +# datestr = Tue Oct 25 11:14:27 2011 +# mode = Direct +# flatfile1 = d1026_0011.fits +# flatfile2 = d1026_0012.fits +# flatfile3 = d1026_0013.fits +# flatfile4 = d1026_0014.fits +# flatfile5 = d1026_0015.fits +# zerofile1 = d1026_0008.fits +# zerofile2 = d1026_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2100 0.011826 2.5369 0.024796 + 2 1 1 B 1.2397 0.021519 2.5992 0.045119 + 3 4 2 A 1.2029 0.016268 2.5222 0.034109 + 4 3 2 B 1.2011 0.013787 2.5184 0.028908 + 5 6 3 A 1.1895 0.019960 2.4939 0.041850 + 6 5 3 B 1.2840 0.017495 2.6921 0.036681 + 7 8 4 A 1.2334 0.013424 2.5861 0.028146 + 8 7 4 B 1.2275 0.017398 2.5737 0.036478 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Apr-12.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Apr-12.log new file mode 100644 index 0000000000..2c6f8bc0cd --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Apr-12.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2012-Apr-12.log +# datestr = Thu Apr 12 17:05:56 2012 +# mode = Direct +# flatfile1 = d0413_0010.fits +# flatfile2 = d0413_0011.fits +# flatfile3 = d0413_0012.fits +# flatfile4 = d0413_0013.fits +# flatfile5 = d0413_0014.fits +# zerofile1 = d0413_0007.fits +# zerofile2 = d0413_0008.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 0.0015744 0.10255 0.0033011 0.21502 + 2 1 1 B 1.2924 0.025125 2.7098 0.052679 + 3 4 2 A -0.49111 0.072159 -1.0297 0.15129 + 4 3 2 B -0.52790 0.073904 -1.1068 0.15495 + 5 6 3 A -0.55877 0.11742 -1.1716 0.24618 + 6 5 3 B -0.67065 0.12005 -1.4061 0.25171 + 7 8 4 A 1.2825 0.039860 2.6891 0.083574 + 8 7 4 B 0.011546 0.095399 0.024209 0.20002 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Jun-08.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Jun-08.log new file mode 100644 index 0000000000..63951d04c7 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Jun-08.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2012-Jun-08.log +# datestr = Fri Jun 8 19:18:21 2012 +# mode = Direct +# flatfile1 = d0609_0004.fits +# flatfile2 = d0609_0005.fits +# flatfile3 = d0609_0006.fits +# flatfile4 = d0609_0007.fits +# flatfile5 = d0609_0008.fits +# zerofile1 = d0609_0001.fits +# zerofile2 = d0609_0002.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2753 0.020584 2.6739 0.043158 + 2 1 1 B 1.2887 0.018639 2.7019 0.039080 + 3 4 2 A 0.011702 0.044785 0.024535 0.093901 + 4 3 2 B -0.017178 0.033837 -0.036017 0.070946 + 5 6 3 A -0.015898 0.046156 -0.033332 0.096775 + 6 5 3 B -0.064485 0.052428 -0.13520 0.10993 + 7 8 4 A 1.2819 0.015164 2.6877 0.031793 + 8 7 4 B 1.2765 1.0085 2.6765 2.1145 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Mar-14.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Mar-14.log new file mode 100644 index 0000000000..c2e01638fe --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Mar-14.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2012-Mar-14.log +# datestr = Wed Mar 14 18:33:02 2012 +# mode = Direct +# flatfile1 = d0315_0021.fits +# flatfile2 = d0315_0022.fits +# flatfile3 = d0315_0023.fits +# flatfile4 = d0315_0024.fits +# flatfile5 = d0315_0025.fits +# zerofile1 = d0315_0018.fits +# zerofile2 = d0315_0019.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2684 0.036559 2.6595 0.076653 + 2 1 1 B 1.2802 0.017248 2.6842 0.036163 + 3 4 2 A -0.0083733 0.033011 -0.017556 0.069214 + 4 3 2 B -0.029809 0.024391 -0.062501 0.051141 + 5 6 3 A -0.035762 0.037714 -0.074982 0.079074 + 6 5 3 B -0.077832 0.042671 -0.16319 0.089468 + 7 8 4 A 1.2809 0.015975 2.6856 0.033494 + 8 7 4 B -0.042963 0.18241 -0.090079 0.38246 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Nov-11.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Nov-11.log new file mode 100644 index 0000000000..ae60fe911b --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Nov-11.log @@ -0,0 +1,26 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2012-Nov-11.log +# datestr = Sun Nov 11 08:45:03 2012 +# mode = Direct +# flatfile1 = d1112_0005.fits +# flatfile2 = d1112_0006.fits +# flatfile3 = d1112_0007.fits +# flatfile4 = d1112_0008.fits +# zerofile1 = d1112_0002.fits +# zerofile2 = d1112_0003.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2237 0.014181 2.5658 0.029733 + 2 1 1 B 1.2329 0.010559 2.5850 0.022139 + 3 4 2 A 1.2081 0.014732 2.5330 0.030888 + 4 3 2 B 1.1995 0.013442 2.5150 0.028185 + 5 6 3 A 1.1934 0.016256 2.5021 0.034084 + 6 5 3 B 1.2831 0.014538 2.6903 0.030481 + 7 8 4 A 1.2268 0.011326 2.5723 0.023747 + 8 7 4 B 1.2297 0.018440 2.5784 0.038663 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Aug-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Aug-26.log new file mode 100644 index 0000000000..acc17c7c2a --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Aug-26.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2013-Aug-26.log +# datestr = Mon Aug 26 14:04:49 2013 +# mode = Direct +# flatfile1 = d0827_0016.fits +# flatfile2 = d0827_0017.fits +# flatfile3 = d0827_0018.fits +# flatfile4 = d0827_0019.fits +# flatfile5 = d0827_0020.fits +# zerofile1 = d0827_0013.fits +# zerofile2 = d0827_0014.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2209 0.012625 2.5599 0.026470 + 2 1 1 B 1.2344 0.018013 2.5882 0.037767 + 3 4 2 A 1.2075 0.023451 2.5318 0.049169 + 4 3 2 B 1.1921 0.013585 2.4994 0.028484 + 5 6 3 A 1.1868 0.013606 2.4883 0.028528 + 6 5 3 B 1.2842 0.015307 2.6926 0.032095 + 7 8 4 A 1.2260 0.015683 2.5706 0.032883 + 8 7 4 B 1.2259 0.013076 2.5703 0.027417 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Dec-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Dec-22.log new file mode 100644 index 0000000000..74ba15646e --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Dec-22.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2013-Dec-22.log +# datestr = Sun Dec 22 22:06:59 2013 +# mode = Direct +# flatfile1 = d1223_0021.fits +# flatfile2 = d1223_0022.fits +# flatfile3 = d1223_0023.fits +# flatfile4 = d1223_0024.fits +# flatfile5 = d1223_0025.fits +# zerofile1 = d1223_0018.fits +# zerofile2 = d1223_0019.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2124 0.010797 2.5420 0.022638 + 2 1 1 B 1.2309 0.010179 2.5809 0.021342 + 3 4 2 A 1.1983 0.013589 2.5125 0.028491 + 4 3 2 B 1.1886 0.013343 2.4921 0.027976 + 5 6 3 A 1.1883 0.020742 2.4916 0.043490 + 6 5 3 B 1.2811 0.024592 2.6861 0.051562 + 7 8 4 A 1.2208 0.014893 2.5597 0.031225 + 8 7 4 B 1.2209 0.016619 2.5599 0.034846 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Feb-04.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Feb-04.log new file mode 100644 index 0000000000..096849e094 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Feb-04.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2013-Feb-04.log +# datestr = Mon Feb 4 16:52:46 2013 +# mode = Direct +# flatfile1 = d0205_0020.fits +# flatfile2 = d0205_0021.fits +# flatfile3 = d0205_0022.fits +# flatfile4 = d0205_0023.fits +# flatfile5 = d0205_0024.fits +# zerofile1 = d0205_0017.fits +# zerofile2 = d0205_0018.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2120 0.017747 2.5412 0.037209 + 2 1 1 B 1.2377 0.0076416 2.5951 0.016022 + 3 4 2 A 1.2042 0.018348 2.5249 0.038470 + 4 3 2 B 1.1985 0.014051 2.5129 0.029461 + 5 6 3 A 1.1873 0.0083574 2.4893 0.017523 + 6 5 3 B 1.2953 0.021348 2.7158 0.044761 + 7 8 4 A 1.2288 0.016836 2.5765 0.035299 + 8 7 4 B 1.2299 0.019779 2.5788 0.041470 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Oct-28.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Oct-28.log new file mode 100644 index 0000000000..702beadbe5 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Oct-28.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2013-Oct-28.log +# datestr = Mon Oct 28 10:30:38 2013 +# mode = Direct +# flatfile1 = d1029_0028.fits +# flatfile2 = d1029_0029.fits +# flatfile3 = d1029_0030.fits +# flatfile4 = d1029_0031.fits +# flatfile5 = d1029_0032.fits +# zerofile1 = d1029_0025.fits +# zerofile2 = d1029_0026.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2116 0.018703 2.5403 0.039215 + 2 1 1 B 1.2291 0.011658 2.5771 0.024444 + 3 4 2 A 1.1976 0.020518 2.5109 0.043019 + 4 3 2 B 1.1872 0.0095606 2.4891 0.020046 + 5 6 3 A 1.1863 0.010872 2.4872 0.022794 + 6 5 3 B 1.2785 0.017024 2.6807 0.035694 + 7 8 4 A 1.2288 0.018051 2.5763 0.037848 + 8 7 4 B 1.2298 0.010007 2.5785 0.020982 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Sep-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Sep-26.log new file mode 100644 index 0000000000..504a3c25d1 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Sep-26.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2013-Sep-26.log +# datestr = Thu Sep 26 17:13:18 2013 +# mode = Direct +# flatfile1 = d0927_0011.fits +# flatfile2 = d0927_0012.fits +# flatfile3 = d0927_0013.fits +# flatfile4 = d0927_0014.fits +# flatfile5 = d0927_0015.fits +# zerofile1 = d0927_0008.fits +# zerofile2 = d0927_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2159 0.010762 2.5494 0.022563 + 2 1 1 B 1.2344 0.0091386 2.5882 0.019160 + 3 4 2 A 1.1994 0.017440 2.5147 0.036565 + 4 3 2 B 1.1972 0.012260 2.5101 0.025705 + 5 6 3 A 1.1881 0.019040 2.4911 0.039920 + 6 5 3 B 1.2892 0.014831 2.7030 0.031096 + 7 8 4 A 1.2381 0.016243 2.5960 0.034057 + 8 7 4 B 1.2314 0.010438 2.5818 0.021886 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2014-Oct-16.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2014-Oct-16.log new file mode 100644 index 0000000000..3082731279 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2014-Oct-16.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2014-Oct-16.log +# datestr = Thu Oct 16 09:14:41 2014 +# mode = Direct +# flatfile1 = d1002_0110.fits +# flatfile2 = d1002_0111.fits +# flatfile3 = d1002_0112.fits +# flatfile4 = d1002_0113.fits +# flatfile5 = d1002_0114.fits +# zerofile1 = d1002_0107.fits +# zerofile2 = d1002_0108.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2113 0.021651 2.5398 0.045396 + 2 1 1 B 1.2293 0.011356 2.5775 0.023809 + 3 4 2 A 1.2018 0.018134 2.5197 0.038021 + 4 3 2 B 1.1962 0.018497 2.5080 0.038781 + 5 6 3 A 0.38248 0.078593 0.0000 0.0000 + 6 5 3 B 1.2907 0.012124 2.7063 0.025420 + 7 8 4 A 1.2242 0.020894 2.5668 0.043808 + 8 7 4 B 1.2264 0.014217 2.5713 0.029809 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Apr-10.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Apr-10.log new file mode 100644 index 0000000000..395ec320fd --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Apr-10.log @@ -0,0 +1,28 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2015-Apr-10.log +# datestr = Fri Apr 10 17:17:57 2015 +# mode = Direct +# flatfile1 = d0411_0019.fits +# flatfile2 = d0411_0024.fits +# flatfile3 = d0411_0025.fits +# flatfile4 = d0411_0026.fits +# flatfile5 = d0411_0027.fits +# flatfile6 = d0411_0028.fits +# zerofile1 = d0411_0016.fits +# zerofile2 = d0411_0017.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2088 0.023120 2.5345 0.048474 + 2 1 1 B 1.2237 0.010551 2.5657 0.022122 + 3 4 2 A 1.1874 0.015721 2.4897 0.032962 + 4 3 2 B 1.1833 0.012724 2.4811 0.026678 + 5 6 3 A 6.6175 3.3380 0.0000 0.0000 + 6 5 3 B 1.2717 0.017433 2.6663 0.036551 + 7 8 4 A 1.2306 0.019980 2.5801 0.041891 + 8 7 4 B 1.2173 0.015029 2.5523 0.031512 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-01.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-01.log new file mode 100644 index 0000000000..827ec1df34 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-01.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2015-Dec-01.log +# datestr = Tue Dec 1 13:13:02 2015 +# mode = Direct +# flatfile1 = d1202_0011.fits +# flatfile2 = d1202_0012.fits +# flatfile3 = d1202_0013.fits +# flatfile4 = d1202_0014.fits +# flatfile5 = d1202_0015.fits +# zerofile1 = d1202_0008.fits +# zerofile2 = d1202_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2029 0.019667 2.5220 0.041235 + 2 1 1 B 1.2224 0.018197 2.5630 0.038153 + 3 4 2 A 1.1753 0.019515 2.4641 0.040917 + 4 3 2 B 1.1812 0.016200 2.4765 0.033966 + 5 6 3 A 1.1378 0.020328 2.3856 0.042621 + 6 5 3 B 1.2443 0.024999 2.6089 0.052415 + 7 8 4 A 1.3916 0.011417 4.3765 0.035906 + 8 7 4 B 1.2168 0.010457 2.5512 0.021926 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-30.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-30.log new file mode 100644 index 0000000000..131911fd1a --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-30.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2015-Dec-30.log +# datestr = Wed Dec 30 12:16:34 2015 +# mode = Direct +# flatfile1 = d1231_0012.fits +# flatfile2 = d1231_0013.fits +# flatfile3 = d1231_0014.fits +# flatfile4 = d1231_0015.fits +# flatfile5 = d1231_0016.fits +# zerofile1 = d1231_0009.fits +# zerofile2 = d1231_0010.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2026 0.013645 2.5215 0.028609 + 2 1 1 B 1.2266 0.016573 2.5717 0.034748 + 3 4 2 A 1.1678 0.011537 2.4486 0.024190 + 4 3 2 B 1.1840 0.015072 2.4825 0.031601 + 5 6 3 A 1.1448 0.014461 2.4003 0.030319 + 6 5 3 B 1.2446 0.016256 2.6095 0.034084 + 7 8 4 A 1.2200 0.012824 2.5580 0.026888 + 8 7 4 B 1.2223 0.010710 2.5627 0.022455 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-02.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-02.log new file mode 100644 index 0000000000..227f036f95 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-02.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2015-Sep-02.log +# datestr = Wed Sep 2 10:13:48 2015 +# mode = Direct +# flatfile1 = d0903_0012.fits +# flatfile2 = d0903_0013.fits +# flatfile3 = d0903_0014.fits +# flatfile4 = d0903_0015.fits +# flatfile5 = d0903_0016.fits +# zerofile1 = d0903_0009.fits +# zerofile2 = d0903_0010.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.1941 0.024805 2.5037 0.052008 + 2 1 1 B 1.2292 0.015682 2.5771 0.032881 + 3 4 2 A 1.1656 0.028640 2.4439 0.060050 + 4 3 2 B 1.1856 0.010798 2.4859 0.022639 + 5 6 3 A 1.1427 0.010662 2.3959 0.022356 + 6 5 3 B 1.2428 0.023203 2.6057 0.048650 + 7 8 4 A 1.2297 0.018093 2.5783 0.037935 + 8 7 4 B 1.2209 0.016638 2.5598 0.034885 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-30.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-30.log new file mode 100644 index 0000000000..ddb0122bcd --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-30.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2015-Sep-30.log +# datestr = Wed Sep 30 14:46:52 2015 +# mode = Direct +# flatfile1 = d0929_0026.fits +# flatfile2 = d0929_0027.fits +# flatfile3 = d0929_0028.fits +# flatfile4 = d0929_0029.fits +# flatfile5 = d0929_0030.fits +# zerofile1 = d0929_0023.fits +# zerofile2 = d0929_0024.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2129 0.016825 2.5431 0.035277 + 2 1 1 B 1.2280 0.016373 2.5747 0.034329 + 3 4 2 A 1.1693 0.022456 2.4516 0.047082 + 4 3 2 B 1.1905 0.013078 2.4962 0.027420 + 5 6 3 A 1.1396 0.018878 2.3893 0.039581 + 6 5 3 B 1.2513 0.015501 2.6235 0.032501 + 7 8 4 A 1.2288 0.0099079 2.5764 0.020774 + 8 7 4 B 1.2205 0.013599 2.5591 0.028512 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Dec-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Dec-17.log new file mode 100644 index 0000000000..c9a7aa9b37 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Dec-17.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2016-Dec-17.log +# datestr = Sat Dec 17 12:43:30 2016 +# mode = Direct +# flatfile1 = d1218_0005.fits +# flatfile2 = d1218_0006.fits +# flatfile3 = d1218_0007.fits +# flatfile4 = d1218_0008.fits +# flatfile5 = d1218_0009.fits +# zerofile1 = d1218_0002.fits +# zerofile2 = d1218_0003.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.1966 0.017150 2.5090 0.035959 + 2 1 1 B 1.2216 0.014696 2.5613 0.030814 + 3 4 2 A 1.1670 0.019917 2.4468 0.041759 + 4 3 2 B 1.1852 0.013697 2.4851 0.028719 + 5 6 3 A 1.1583 0.018198 2.4287 0.038155 + 6 5 3 B 1.2700 0.012915 2.6629 0.027079 + 7 8 4 A 1.2209 0.014479 2.5597 0.030357 + 8 7 4 B 1.2202 0.0095583 2.5583 0.020041 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-May-03.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-May-03.log new file mode 100644 index 0000000000..afaaf5973e --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-May-03.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2016-May-03.log +# datestr = Tue May 3 11:48:18 2016 +# mode = Direct +# flatfile1 = d0504_0011.fits +# flatfile2 = d0504_0012.fits +# flatfile3 = d0504_0013.fits +# flatfile4 = d0504_0014.fits +# flatfile5 = d0504_0015.fits +# zerofile1 = d0504_0008.fits +# zerofile2 = d0504_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2062 0.017440 2.5290 0.036566 + 2 1 1 B 1.2305 0.016079 2.5800 0.033713 + 3 4 2 A 1.1762 0.015266 2.4661 0.032007 + 4 3 2 B 1.1904 0.015536 2.4959 0.032573 + 5 6 3 A 1.1435 0.019940 2.3976 0.041808 + 6 5 3 B 1.2439 0.025989 2.6081 0.054490 + 7 8 4 A 1.2267 0.011221 2.5720 0.023528 + 8 7 4 B 1.2226 0.016266 2.5634 0.034104 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Nov-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Nov-22.log new file mode 100644 index 0000000000..75fe5a8b51 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Nov-22.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2016-Nov-22.log +# datestr = Tue Nov 22 16:05:21 2016 +# mode = Direct +# flatfile1 = d1123_0012.fits +# flatfile2 = d1123_0013.fits +# flatfile3 = d1123_0014.fits +# flatfile4 = d1123_0015.fits +# flatfile5 = d1123_0016.fits +# zerofile1 = d1123_0009.fits +# zerofile2 = d1123_0010.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2037 0.023999 2.5237 0.050319 + 2 1 1 B 1.2204 0.014640 2.5587 0.030696 + 3 4 2 A 1.1761 0.024087 2.4660 0.050502 + 4 3 2 B 1.1876 0.015070 2.4900 0.031597 + 5 6 3 A 1.1586 0.018038 2.4292 0.037819 + 6 5 3 B 1.2594 0.021622 2.6406 0.045335 + 7 8 4 A 1.2283 0.014526 2.5753 0.030456 + 8 7 4 B 1.2185 0.020491 2.5548 0.042963 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-Dec-10.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-Dec-10.log new file mode 100644 index 0000000000..bd92eb2d8e --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-Dec-10.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2017-Dec-10.log +# datestr = Sun Dec 10 12:33:50 2017 +# mode = Direct +# flatfile1 = d1211_0050.fits +# flatfile2 = d1211_0051.fits +# flatfile3 = d1211_0052.fits +# flatfile4 = d1211_0053.fits +# flatfile5 = d1211_0054.fits +# zerofile1 = d1211_0047.fits +# zerofile2 = d1211_0048.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2062 0.015234 2.5291 0.031940 + 2 1 1 B 1.2210 0.016928 2.5601 0.035493 + 3 4 2 A 8.1527 3.4832 -0.0000 0.0000 + 4 3 2 B 1.1877 0.013289 2.4902 0.027862 + 5 6 3 A 1.1668 0.010371 2.4465 0.021744 + 6 5 3 B 1.2499 0.017126 2.6206 0.035907 + 7 8 4 A 1.2169 0.014226 2.5514 0.029827 + 8 7 4 B 1.2276 0.014758 2.5739 0.030942 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-May-16.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-May-16.log new file mode 100644 index 0000000000..43f98b221f --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-May-16.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2017-May-16.log +# datestr = Tue May 16 14:17:38 2017 +# mode = Direct +# flatfile1 = d0517_0032.fits +# flatfile2 = d0517_0033.fits +# flatfile3 = d0517_0034.fits +# flatfile4 = d0517_0035.fits +# flatfile5 = d0517_0036.fits +# zerofile1 = d0517_0029.fits +# zerofile2 = d0517_0030.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.1967 0.023154 2.5091 0.048547 + 2 1 1 B 1.2186 0.013135 2.5551 0.027541 + 3 4 2 A 1.1712 0.022452 2.4557 0.047075 + 4 3 2 B 1.1863 0.015390 2.4872 0.032268 + 5 6 3 A 1.1582 0.012982 2.4284 0.027220 + 6 5 3 B 1.2717 0.022158 2.6663 0.046459 + 7 8 4 A 1.2231 0.018996 2.5644 0.039827 + 8 7 4 B 1.2132 0.018976 2.5436 0.039787 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Apr-19.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Apr-19.log new file mode 100644 index 0000000000..5b53ac384f --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Apr-19.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2018-Apr-19.log +# datestr = Thu Apr 19 20:15:59 2018 +# mode = Direct +# flatfile1 = d0420_0017.fits +# flatfile2 = d0420_0018.fits +# flatfile3 = d0420_0019.fits +# flatfile4 = d0420_0020.fits +# flatfile5 = d0420_0021.fits +# zerofile1 = d0420_0014.fits +# zerofile2 = d0420_0015.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2087 0.015297 2.5342 0.032073 + 2 1 1 B 1.2272 0.014454 2.5731 0.030306 + 3 4 2 A 0.59913 0.39172 -0.0000 0.0000 + 4 3 2 B 1.1865 0.018092 2.4869 0.035813 + 5 6 3 A 1.1743 0.020004 2.4621 0.041942 + 6 5 3 B 1.2492 0.029381 2.6191 0.061603 + 7 8 4 A 1.2210 0.016258 2.5600 0.034089 + 8 7 4 B 1.2174 0.015866 2.5526 0.033266 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-08.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-08.log new file mode 100644 index 0000000000..9b3d7ac21a --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-08.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2018-Aug-08.log +# datestr = Wed Aug 8 18:58:48 2018 +# mode = Direct +# flatfile1 = d0809_0033.fits +# flatfile2 = d0809_0034.fits +# flatfile3 = d0809_0035.fits +# flatfile4 = d0809_0036.fits +# flatfile5 = d0809_0037.fits +# zerofile1 = d0809_0030.fits +# zerofile2 = d0809_0031.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2084 0.020639 2.5336 0.043273 + 2 1 1 B 1.2260 0.020071 2.5706 0.042083 + 3 4 2 A 1.1712 0.022483 2.4556 0.047140 + 4 3 2 B 1.1896 0.013173 2.4942 0.027620 + 5 6 3 A 1.1629 0.013157 2.4383 0.027586 + 6 5 3 B 1.2542 0.019223 2.6296 0.040305 + 7 8 4 A 1.2256 0.014612 2.5696 0.030637 + 8 7 4 B 1.2261 0.012793 2.5707 0.026824 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-17.log new file mode 100644 index 0000000000..dd0d8a43c5 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-17.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2018-Aug-17.log +# datestr = Fri Aug 17 13:45:31 2018 +# mode = Direct +# flatfile1 = d0818_0041.fits +# flatfile2 = d0818_0042.fits +# flatfile3 = d0818_0043.fits +# flatfile4 = d0818_0044.fits +# flatfile5 = d0818_0045.fits +# zerofile1 = d0818_0038.fits +# zerofile2 = d0818_0039.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2543 0.018228 2.6300 0.038218 + 2 1 1 B 1.2221 0.012989 2.5623 0.027235 + 3 4 2 A 1.1697 0.014521 2.4524 0.030445 + 4 3 2 B 1.1861 0.014467 2.4869 0.030332 + 5 6 3 A 1.1648 0.010588 2.4422 0.022199 + 6 5 3 B 1.2526 0.023916 2.6264 0.050145 + 7 8 4 A 1.2341 0.017459 2.5875 0.036606 + 8 7 4 B 1.2207 0.012766 2.5594 0.026766 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-May-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-May-17.log new file mode 100644 index 0000000000..f78c342af6 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-May-17.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2018-May-17.log +# datestr = Thu May 17 14:54:11 2018 +# mode = Direct +# flatfile1 = d0518_0024.fits +# flatfile2 = d0518_0025.fits +# flatfile3 = d0518_0026.fits +# flatfile4 = d0518_0027.fits +# flatfile5 = d0518_0028.fits +# zerofile1 = d0518_0021.fits +# zerofile2 = d0518_0022.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2080 0.019749 2.5328 0.041407 + 2 1 1 B 1.2269 0.017901 2.5724 0.037533 + 3 4 2 A 0.63905 0.42127 -0.0000 0.0000 + 4 3 2 B 1.1866 0.013973 2.4879 0.029297 + 5 6 3 A 1.1672 0.014287 2.4473 0.029956 + 6 5 3 B 1.2478 0.031201 2.6163 0.065418 + 7 8 4 A 1.2277 0.016168 2.5741 0.033898 + 8 7 4 B 1.2200 0.020422 2.5579 0.042818 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Nov-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Nov-26.log new file mode 100644 index 0000000000..ffaab01329 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Nov-26.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2018-Nov-26.log +# datestr = Mon Nov 26 18:00:30 2018 +# mode = Direct +# flatfile1 = d1127_0014.fits +# flatfile2 = d1127_0015.fits +# flatfile3 = d1127_0016.fits +# flatfile4 = d1127_0017.fits +# flatfile5 = d1127_0018.fits +# zerofile1 = d1127_0011.fits +# zerofile2 = d1127_0012.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2735 0.011005 2.6702 0.023074 + 2 1 1 B 1.2007 0.014247 2.5175 0.029870 + 3 4 2 A 1.1747 0.018535 2.4630 0.038862 + 4 3 2 B 1.1866 0.012016 2.4879 0.025195 + 5 6 3 A 1.1681 0.014539 3.6401 0.17244 + 6 5 3 B 1.2536 0.019390 2.6284 0.040655 + 7 8 4 A 1.2299 0.010285 2.5788 0.021565 + 8 7 4 B 1.2175 0.014690 2.5526 0.030799 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Dec-29.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Dec-29.log new file mode 100644 index 0000000000..dcba891348 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Dec-29.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2019-Dec-29.log +# datestr = Sun Dec 29 13:16:14 2019 +# mode = Direct +# flatfile1 = d1230_0012.fits +# flatfile2 = d1230_0013.fits +# flatfile3 = d1230_0014.fits +# flatfile4 = d1230_0015.fits +# flatfile5 = d1230_0016.fits +# zerofile1 = d1230_0009.fits +# zerofile2 = d1230_0010.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2648 0.020301 2.6519 0.042565 + 2 1 1 B 1.1952 0.016133 2.5059 0.033827 + 3 4 2 A 1.1621 0.014632 2.4366 0.030679 + 4 3 2 B 1.1835 0.016240 2.4814 0.034049 + 5 6 3 A 1.1513 0.014941 2.4141 0.037324 + 6 5 3 B 1.2599 0.024316 2.6417 0.050982 + 7 8 4 A 1.2474 0.012977 3.9171 0.056409 + 8 7 4 B 1.2192 0.018556 2.5564 0.038906 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-19.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-19.log new file mode 100644 index 0000000000..441a7f3804 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-19.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2019-Nov-19.log +# datestr = Tue Nov 19 08:15:56 2019 +# mode = Direct +# flatfile1 = d1119_0004.fits +# flatfile2 = d1119_0005.fits +# flatfile3 = d1119_0006.fits +# flatfile4 = d1119_0007.fits +# flatfile5 = d1119_0008.fits +# zerofile1 = d1119_0001.fits +# zerofile2 = d1119_0002.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2611 0.020239 2.6442 0.042434 + 2 1 1 B 1.1931 0.014274 2.5017 0.029927 + 3 4 2 A 1.1634 0.016668 2.4392 0.034946 + 4 3 2 B 1.1765 0.016175 2.4667 0.033913 + 5 6 3 A 1.1584 0.011443 2.4287 0.023991 + 6 5 3 B 1.2621 0.020207 2.6463 0.042368 + 7 8 4 A 1.2450 0.010563 2.6217 0.038713 + 8 7 4 B 1.2111 0.012566 2.5393 0.026347 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-27.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-27.log new file mode 100644 index 0000000000..33b82cac0c --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-27.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2019-Nov-27.log +# datestr = Wed Nov 27 07:15:01 2019 +# mode = Direct +# flatfile1 = d1127_0025.fits +# flatfile2 = d1127_0026.fits +# flatfile3 = d1127_0027.fits +# flatfile4 = d1127_0028.fits +# flatfile5 = d1127_0029.fits +# zerofile1 = d1127_0022.fits +# zerofile2 = d1127_0023.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2673 0.015794 2.6571 0.033116 + 2 1 1 B 1.1956 0.014366 2.5068 0.030121 + 3 4 2 A 1.1723 0.015917 2.4580 0.033373 + 4 3 2 B 1.1809 0.017559 2.4759 0.036815 + 5 6 3 A 1.1549 0.012052 2.4215 0.025268 + 6 5 3 B 1.2643 0.018608 2.6508 0.039015 + 7 8 4 A 1.2407 0.011724 2.6469 0.10091 + 8 7 4 B 1.2115 0.013294 2.5401 0.027873 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Jun-21.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Jun-21.log new file mode 100644 index 0000000000..93a1440e3e --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Jun-21.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2021-Jun-21.log +# datestr = Mon Jun 21 15:43:33 2021 +# mode = Direct +# flatfile1 = /sdata1005/deimos5/2021jun22/d0622_0020.fits +# flatfile2 = /sdata1005/deimos5/2021jun22/d0622_0021.fits +# flatfile3 = /sdata1005/deimos5/2021jun22/d0622_0022.fits +# flatfile4 = /sdata1005/deimos5/2021jun22/d0622_0023.fits +# flatfile5 = /sdata1005/deimos5/2021jun22/d0622_0024.fits +# zerofile1 = /sdata1005/deimos5/2021jun22/d0622_0017.fits +# zerofile2 = /sdata1005/deimos5/2021jun22/d0622_0018.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2320 0.017068 2.5831 0.035785 + 2 1 1 B 1.1796 0.016234 2.4732 0.034038 + 3 4 2 A 1.7137 0.027804 1.7966 0.029148 + 4 3 2 B 1.1728 0.012499 2.4589 0.026207 + 5 6 3 A 1.1609 0.013320 2.4341 0.027928 + 6 5 3 B 1.2614 0.020335 2.6447 0.042635 + 7 8 4 A 1.2460 0.013255 3.9177 0.040593 + 8 7 4 B 1.2156 0.011986 2.5486 0.025132 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-04.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-04.log new file mode 100644 index 0000000000..a4cdebb01f --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-04.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2021-May-04.log +# datestr = Tue May 4 23:23:33 2021 +# mode = Direct +# flatfile1 = /sdata1005/deimos10/2021may05/d0505_0026.fits +# flatfile2 = /sdata1005/deimos10/2021may05/d0505_0027.fits +# flatfile3 = /sdata1005/deimos10/2021may05/d0505_0028.fits +# flatfile4 = /sdata1005/deimos10/2021may05/d0505_0029.fits +# flatfile5 = /sdata1005/deimos10/2021may05/d0505_0030.fits +# zerofile1 = /sdata1005/deimos10/2021may05/d0505_0023.fits +# zerofile2 = /sdata1005/deimos10/2021may05/d0505_0024.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2345 0.016751 2.5885 0.035122 + 2 1 1 B 1.1730 0.012694 2.4594 0.026616 + 3 4 2 A 1.7085 0.033074 1.7911 0.034673 + 4 3 2 B 1.1736 0.012521 2.4606 0.026252 + 5 6 3 A 1.1542 0.016727 2.4201 0.035072 + 6 5 3 B 1.2689 0.017654 2.6605 0.037016 + 7 8 4 A 1.2468 0.013576 2.6293 0.038898 + 8 7 4 B 1.2085 0.016786 2.5338 0.035195 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-05.log new file mode 100644 index 0000000000..b10516f3f2 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-05.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2021-May-05.log +# datestr = Wed May 5 13:36:39 2021 +# mode = Direct +# flatfile1 = /sdata1005/deimos6/2021may06/d0506_0022.fits +# flatfile2 = /sdata1005/deimos6/2021may06/d0506_0023.fits +# flatfile3 = /sdata1005/deimos6/2021may06/d0506_0024.fits +# flatfile4 = /sdata1005/deimos6/2021may06/d0506_0025.fits +# flatfile5 = /sdata1005/deimos6/2021may06/d0506_0026.fits +# zerofile1 = /sdata1005/deimos6/2021may06/d0506_0019.fits +# zerofile2 = /sdata1005/deimos6/2021may06/d0506_0020.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2299 0.019747 2.5787 0.041404 + 2 1 1 B 1.1742 0.017435 2.4619 0.036555 + 3 4 2 A 1.7006 0.033702 1.7828 0.035331 + 4 3 2 B 1.1712 0.013545 2.4556 0.028400 + 5 6 3 A 1.1579 0.017180 2.4277 0.036021 + 6 5 3 B 1.2627 0.017114 2.6475 0.035882 + 7 8 4 A 1.2534 0.013603 3.9364 0.055853 + 8 7 4 B 1.2141 0.014372 2.5455 0.030135 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-31.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-31.log new file mode 100644 index 0000000000..093551d80e --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-31.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2021-May-31.log +# datestr = Mon May 31 17:24:42 2021 +# mode = Direct +# flatfile1 = /sdata1005/deimos6/2021jun01/d0601_0034.fits +# flatfile2 = /sdata1005/deimos6/2021jun01/d0601_0035.fits +# flatfile3 = /sdata1005/deimos6/2021jun01/d0601_0036.fits +# flatfile4 = /sdata1005/deimos6/2021jun01/d0601_0037.fits +# flatfile5 = /sdata1005/deimos6/2021jun01/d0601_0038.fits +# zerofile1 = /sdata1005/deimos6/2021jun01/d0601_0031.fits +# zerofile2 = /sdata1005/deimos6/2021jun01/d0601_0032.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2284 0.020443 2.5755 0.042862 + 2 1 1 B 1.1759 0.014111 2.4654 0.029586 + 3 4 2 A 1.7021 0.033337 1.7843 0.034948 + 4 3 2 B 1.1763 0.014020 2.4663 0.029395 + 5 6 3 A 1.1623 0.014713 2.4373 0.034536 + 6 5 3 B 1.2711 0.017036 2.6651 0.035719 + 7 8 4 A 1.2472 0.016559 3.9223 0.052079 + 8 7 4 B 1.2208 0.0096946 2.5597 0.020327 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Sep-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Sep-05.log new file mode 100644 index 0000000000..66fd4a03d2 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Sep-05.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2021-Sep-05.log +# datestr = Sun Sep 5 21:52:11 2021 +# mode = Direct +# flatfile1 = /sdata1004/deimos5/2021sep06/d0906_0023.fits +# flatfile2 = /sdata1004/deimos5/2021sep06/d0906_0024.fits +# flatfile3 = /sdata1004/deimos5/2021sep06/d0906_0025.fits +# flatfile4 = /sdata1004/deimos5/2021sep06/d0906_0026.fits +# flatfile5 = /sdata1004/deimos5/2021sep06/d0906_0027.fits +# zerofile1 = /sdata1004/deimos5/2021sep06/d0906_0020.fits +# zerofile2 = /sdata1004/deimos5/2021sep06/d0906_0021.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2314 0.020328 2.5818 0.042621 + 2 1 1 B 1.1796 0.017172 2.4732 0.036005 + 3 4 2 A 1.6996 0.030561 1.7818 0.032039 + 4 3 2 B 1.1707 0.015098 2.4545 0.031656 + 5 6 3 A 1.1663 0.015360 2.4454 0.032204 + 6 5 3 B 1.2680 0.014573 2.6587 0.030556 + 7 8 4 A 1.2415 0.015028 3.9047 0.047264 + 8 7 4 B 1.2145 0.013076 2.5463 0.027416 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Apr-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Apr-22.log new file mode 100644 index 0000000000..a8f2a404ec --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Apr-22.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2022-Apr-22.log +# datestr = Fri Apr 22 12:02:48 2022 +# mode = Direct +# flatfile1 = /sdata1005/deimos2/2022apr23/d0423_0007.fits +# flatfile2 = /sdata1005/deimos2/2022apr23/d0423_0008.fits +# flatfile3 = /sdata1005/deimos2/2022apr23/d0423_0009.fits +# flatfile4 = /sdata1005/deimos2/2022apr23/d0423_0010.fits +# flatfile5 = /sdata1005/deimos2/2022apr23/d0423_0011.fits +# zerofile1 = /sdata1005/deimos2/2022apr23/d0423_0004.fits +# zerofile2 = /sdata1005/deimos2/2022apr23/d0423_0005.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2296 0.015107 2.5781 0.031676 + 2 1 1 B 1.1701 0.015093 2.4533 0.031646 + 3 4 2 A 1.6979 0.036449 1.7799 0.038211 + 4 3 2 B 1.1699 0.016766 2.4530 0.035153 + 5 6 3 A 1.1625 0.011392 2.4373 0.023885 + 6 5 3 B 1.2509 0.020297 2.6227 0.042556 + 7 8 4 A 1.2287 0.016077 2.5762 0.033707 + 8 7 4 B 1.2257 0.012202 2.5700 0.025585 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jan-28.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jan-28.log new file mode 100644 index 0000000000..04b488d1ff --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jan-28.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2022-Jan-28.log +# datestr = Fri Jan 28 11:59:35 2022 +# mode = Direct +# flatfile1 = /sdata1005/dmoseng/2022jan29/d0129_0020.fits +# flatfile2 = /sdata1005/dmoseng/2022jan29/d0129_0021.fits +# flatfile3 = /sdata1005/dmoseng/2022jan29/d0129_0022.fits +# flatfile4 = /sdata1005/dmoseng/2022jan29/d0129_0023.fits +# flatfile5 = /sdata1005/dmoseng/2022jan29/d0129_0024.fits +# zerofile1 = /sdata1005/dmoseng/2022jan29/d0129_0017.fits +# zerofile2 = /sdata1005/dmoseng/2022jan29/d0129_0018.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2300 0.014013 2.5790 0.029381 + 2 1 1 B 1.1732 0.013404 2.4599 0.028103 + 3 4 2 A 1.6990 0.025338 1.7811 0.026563 + 4 3 2 B 1.1722 0.0099498 2.4577 0.020861 + 5 6 3 A 1.1477 0.010857 2.4063 0.022763 + 6 5 3 B 1.2589 0.021481 2.6395 0.045038 + 7 8 4 A 1.2485 0.013253 3.9267 0.041681 + 8 7 4 B 1.2103 0.012897 2.5375 0.027041 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-01.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-01.log new file mode 100644 index 0000000000..53b6ff9077 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-01.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2022-Jun-01.log +# datestr = Wed Jun 1 23:21:27 2022 +# mode = Direct +# flatfile1 = /sdata1005/deimos3/2022jun02/d0602_0019.fits +# flatfile2 = /sdata1005/deimos3/2022jun02/d0602_0020.fits +# flatfile3 = /sdata1005/deimos3/2022jun02/d0602_0021.fits +# flatfile4 = /sdata1005/deimos3/2022jun02/d0602_0022.fits +# flatfile5 = /sdata1005/deimos3/2022jun02/d0602_0023.fits +# zerofile1 = /sdata1005/deimos3/2022jun02/d0602_0017.fits +# zerofile2 = /sdata1005/deimos3/2022jun02/d0602_0018.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2344 0.022140 2.5882 0.046421 + 2 1 1 B 1.1725 0.023057 2.4584 0.048343 + 3 4 2 A 1.6891 0.032117 1.7708 0.033669 + 4 3 2 B 1.1657 0.016579 2.4442 0.034760 + 5 6 3 A 1.1581 0.0090475 2.4282 0.018970 + 6 5 3 B 1.2565 0.021344 2.6345 0.044751 + 7 8 4 A 1.2287 0.016826 2.5763 0.035278 + 8 7 4 B 1.2097 0.012957 2.5364 0.027168 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-22.log new file mode 100644 index 0000000000..2e80c06315 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-22.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2022-Jun-22.log +# datestr = Wed Jun 22 18:43:19 2022 +# mode = Direct +# flatfile1 = /sdata1005/deimos4/2022jun23/d0623_0027.fits +# flatfile2 = /sdata1005/deimos4/2022jun23/d0623_0028.fits +# flatfile3 = /sdata1005/deimos4/2022jun23/d0623_0029.fits +# flatfile4 = /sdata1005/deimos4/2022jun23/d0623_0030.fits +# flatfile5 = /sdata1005/deimos4/2022jun23/d0623_0031.fits +# zerofile1 = /sdata1005/deimos4/2022jun23/d0623_0024.fits +# zerofile2 = /sdata1005/deimos4/2022jun23/d0623_0025.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2378 0.012998 2.5953 0.027252 + 2 1 1 B 1.1759 0.014031 2.4656 0.029418 + 3 4 2 A 1.7069 0.042328 1.7894 0.044375 + 4 3 2 B 1.1753 0.018892 2.4643 0.039611 + 5 6 3 A 1.1654 0.018260 2.4435 0.038287 + 6 5 3 B 1.2532 0.020820 2.6275 0.043652 + 7 8 4 A 1.2308 0.012811 2.5806 0.026860 + 8 7 4 B 1.2233 0.0088112 2.5648 0.018474 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Nov-14.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Nov-14.log new file mode 100644 index 0000000000..32782d4831 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Nov-14.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2022-Nov-14.log +# datestr = Mon Nov 14 00:21:39 2022 +# mode = Direct +# flatfile1 = /sdata1004/dmoseng/2022nov14/d1114_0005.fits +# flatfile2 = /sdata1004/dmoseng/2022nov14/d1114_0006.fits +# flatfile3 = /sdata1004/dmoseng/2022nov14/d1114_0007.fits +# flatfile4 = /sdata1004/dmoseng/2022nov14/d1114_0008.fits +# flatfile5 = /sdata1004/dmoseng/2022nov14/d1114_0009.fits +# zerofile1 = /sdata1004/dmoseng/2022nov14/d1114_0002.fits +# zerofile2 = /sdata1004/dmoseng/2022nov14/d1114_0003.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2350 0.019341 2.5894 0.040552 + 2 1 1 B 1.1713 0.013260 2.4559 0.027802 + 3 4 2 A 1.7019 0.038561 1.7842 0.040425 + 4 3 2 B 1.1708 0.014360 2.4548 0.030109 + 5 6 3 A 1.1575 0.014873 2.4269 0.031185 + 6 5 3 B 1.2581 0.025157 2.6379 0.052747 + 7 8 4 A 1.2304 0.018116 2.5798 0.037984 + 8 7 4 B 1.2196 0.012423 2.5572 0.026048 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Oct-11.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Oct-11.log new file mode 100644 index 0000000000..339034a145 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Oct-11.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2022-Oct-11.log +# datestr = Tue Oct 11 15:00:05 2022 +# mode = Direct +# flatfile1 = /sdata1004/dmoseng/2022oct12/d1012_0011.fits +# flatfile2 = /sdata1004/dmoseng/2022oct12/d1012_0012.fits +# flatfile3 = /sdata1004/dmoseng/2022oct12/d1012_0013.fits +# flatfile4 = /sdata1004/dmoseng/2022oct12/d1012_0014.fits +# flatfile5 = /sdata1004/dmoseng/2022oct12/d1012_0015.fits +# zerofile1 = /sdata1004/dmoseng/2022oct12/d1012_0008.fits +# zerofile2 = /sdata1004/dmoseng/2022oct12/d1012_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2304 0.022985 2.5797 0.048192 + 2 1 1 B 1.1774 0.014896 2.4687 0.031232 + 3 4 2 A 1.7045 0.027092 1.7869 0.028402 + 4 3 2 B 1.1813 0.016740 2.4767 0.035098 + 5 6 3 A 1.1603 0.018360 2.4328 0.038495 + 6 5 3 B 1.2589 0.023527 2.6395 0.049329 + 7 8 4 A 1.2265 0.015310 2.5716 0.032099 + 8 7 4 B 1.2167 0.014626 2.5510 0.030665 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Aug-06.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Aug-06.log new file mode 100644 index 0000000000..d956adbf97 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Aug-06.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2023-Aug-06.log +# datestr = Sun Aug 6 20:10:00 2023 +# mode = Direct +# flatfile1 = /sdata1005/dmoseng/2023aug07/d0728_0036.fits +# flatfile2 = /sdata1005/dmoseng/2023aug07/d0728_0037.fits +# flatfile3 = /sdata1005/dmoseng/2023aug07/d0728_0038.fits +# flatfile4 = /sdata1005/dmoseng/2023aug07/d0728_0039.fits +# flatfile5 = /sdata1005/dmoseng/2023aug07/d0728_0040.fits +# zerofile1 = /sdata1005/dmoseng/2023aug07/d0728_0033.fits +# zerofile2 = /sdata1005/dmoseng/2023aug07/d0728_0034.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2301 0.021649 2.5792 0.045392 + 2 1 1 B 1.1708 0.013904 2.4549 0.029152 + 3 4 2 A 1.6935 0.026653 1.7753 0.027941 + 4 3 2 B 1.1726 0.011941 2.4585 0.025037 + 5 6 3 A 1.1530 0.013522 2.4174 0.028350 + 6 5 3 B 1.2652 0.027525 2.6526 0.057711 + 7 8 4 A 1.2423 0.015752 3.9071 0.049542 + 8 7 4 B 1.2170 0.012133 2.5516 0.025440 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Dec-11.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Dec-11.log new file mode 100644 index 0000000000..f1f322d6e8 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Dec-11.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2023-Dec-11.log +# datestr = Mon Dec 11 01:23:00 2023 +# mode = Direct +# flatfile1 = /sdata1005/deimos2/2023dec11/d1211_0014.fits +# flatfile2 = /sdata1005/deimos2/2023dec11/d1211_0015.fits +# flatfile3 = /sdata1005/deimos2/2023dec11/d1211_0016.fits +# flatfile4 = /sdata1005/deimos2/2023dec11/d1211_0017.fits +# flatfile5 = /sdata1005/deimos2/2023dec11/d1211_0018.fits +# zerofile1 = /sdata1005/deimos2/2023dec11/d1211_0011.fits +# zerofile2 = /sdata1005/deimos2/2023dec11/d1211_0012.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2298 0.016408 2.5785 0.034402 + 2 1 1 B 1.1769 0.016055 2.4676 0.033662 + 3 4 2 A 1.7054 0.022554 1.7879 0.023644 + 4 3 2 B 1.1711 0.021072 2.4555 0.044182 + 5 6 3 A 1.1532 0.010453 2.4180 0.021917 + 6 5 3 B 1.2657 0.024084 3.7961 0.49716 + 7 8 4 A 1.2384 0.019650 3.8947 0.061800 + 8 7 4 B 1.2181 0.014519 2.5541 0.030443 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Jan-06.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Jan-06.log new file mode 100644 index 0000000000..2c8539a6ff --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Jan-06.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2023-Jan-06.log +# datestr = Fri Jan 6 22:27:30 2023 +# mode = Direct +# flatfile1 = /sdata1005/deimos2/2023jan07/d0107_0029.fits +# flatfile2 = /sdata1005/deimos2/2023jan07/d0107_0030.fits +# flatfile3 = /sdata1005/deimos2/2023jan07/d0107_0031.fits +# flatfile4 = /sdata1005/deimos2/2023jan07/d0107_0032.fits +# flatfile5 = /sdata1005/deimos2/2023jan07/d0107_0033.fits +# zerofile1 = /sdata1005/deimos2/2023jan07/d0107_0026.fits +# zerofile2 = /sdata1005/deimos2/2023jan07/d0107_0027.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2306 0.016749 2.5802 0.035117 + 2 1 1 B 1.1764 0.013286 2.4666 0.027856 + 3 4 2 A 1.7039 0.028774 1.7863 0.030164 + 4 3 2 B 1.1705 0.016283 2.4542 0.034140 + 5 6 3 A 1.1522 0.019050 2.4157 0.039942 + 6 5 3 B 1.2689 0.021091 2.6604 0.044221 + 7 8 4 A 1.2447 0.013178 3.9145 0.041445 + 8 7 4 B 1.2209 0.015194 2.5599 0.031858 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Oct-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Oct-05.log new file mode 100644 index 0000000000..bffa0eb30f --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Oct-05.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2023-Oct-05.log +# datestr = Thu Oct 5 12:54:46 2023 +# mode = Direct +# flatfile1 = /sdata1004/dmoseng/2023oct05/d1005_0038.fits +# flatfile2 = /sdata1004/dmoseng/2023oct05/d1005_0039.fits +# flatfile3 = /sdata1004/dmoseng/2023oct05/d1005_0040.fits +# flatfile4 = /sdata1004/dmoseng/2023oct05/d1005_0041.fits +# flatfile5 = /sdata1004/dmoseng/2023oct05/d1005_0042.fits +# zerofile1 = /sdata1004/dmoseng/2023oct05/d1005_0035.fits +# zerofile2 = /sdata1004/dmoseng/2023oct05/d1005_0036.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2279 0.015704 2.5745 0.032925 + 2 1 1 B 1.1784 0.014061 2.4708 0.029482 + 3 4 2 A 1.7079 0.036212 1.7904 0.037962 + 4 3 2 B 1.1721 0.012944 2.4575 0.027139 + 5 6 3 A 1.1637 0.0096046 2.4400 0.020138 + 6 5 3 B 1.2597 0.014713 2.6412 0.030848 + 7 8 4 A 1.2471 0.015236 3.9221 0.047917 + 8 7 4 B 1.2130 0.013148 2.5433 0.027566 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2008-Dec-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2008-Dec-17.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2008-Dec-17.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2008-Dec-17.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2008-Jul-09.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2008-Jul-09.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2008-Jul-09.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2008-Jul-09.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2008-Nov-19.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2008-Nov-19.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2008-Nov-19.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2008-Nov-19.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Apr-16.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Apr-16.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Apr-16.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Apr-16.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Dec-08.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Dec-08.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Dec-08.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Dec-08.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Jun-18.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Jun-18.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Jun-18.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Jun-18.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Mar-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Mar-17.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Mar-17.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Mar-17.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Sep-30.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Sep-30.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Sep-30.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Sep-30.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Feb-02.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Feb-02.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Feb-02.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Feb-02.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Jan-07.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Jan-07.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Jan-07.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Jan-07.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Jun-06.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Jun-06.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Jun-06.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Jun-06.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Mar-02.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Mar-02.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Mar-02.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Mar-02.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-May-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-May-05.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-May-05.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-May-05.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Sep-01.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Sep-01.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Sep-01.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Sep-01.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-Feb-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-Feb-26.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-Feb-26.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-Feb-26.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-Mar-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-Mar-22.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-Mar-22.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-Mar-22.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-May-25.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-May-25.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-May-25.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-May-25.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-Oct-25.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-Oct-25.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-Oct-25.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-Oct-25.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2012-Nov-11.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2012-Nov-11.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2012-Nov-11.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2012-Nov-11.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Aug-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Aug-26.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Aug-26.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Aug-26.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Dec-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Dec-22.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Dec-22.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Dec-22.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Feb-04.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Feb-04.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Feb-04.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Feb-04.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Oct-28.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Oct-28.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Oct-28.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Oct-28.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Sep-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Sep-26.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Sep-26.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Sep-26.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2014-Oct-16.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2014-Oct-16.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2014-Oct-16.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2014-Oct-16.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Apr-10.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Apr-10.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Apr-10.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Apr-10.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Dec-01.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Dec-01.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Dec-01.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Dec-01.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Dec-30.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Dec-30.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Dec-30.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Dec-30.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Sep-02.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Sep-02.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Sep-02.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Sep-02.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Sep-30.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Sep-30.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Sep-30.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Sep-30.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2016-Dec-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2016-Dec-17.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2016-Dec-17.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2016-Dec-17.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2016-May-03.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2016-May-03.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2016-May-03.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2016-May-03.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2016-Nov-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2016-Nov-22.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2016-Nov-22.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2016-Nov-22.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2017-Dec-10.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2017-Dec-10.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2017-Dec-10.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2017-Dec-10.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2017-May-16.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2017-May-16.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2017-May-16.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2017-May-16.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2018-Apr-19.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2018-Apr-19.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2018-Apr-19.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2018-Apr-19.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2018-May-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2018-May-17.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2018-May-17.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2018-May-17.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2018-Nov-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2018-Nov-26.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2018-Nov-26.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2018-Nov-26.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2019-Dec-29.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2019-Dec-29.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2019-Dec-29.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2019-Dec-29.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2019-Nov-19.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2019-Nov-19.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2019-Nov-19.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2019-Nov-19.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2019-Nov-27.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2019-Nov-27.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2019-Nov-27.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2019-Nov-27.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-Jun-21.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-Jun-21.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-Jun-21.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-Jun-21.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-May-04.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-May-04.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-May-04.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-May-04.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-May-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-May-05.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-May-05.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-May-05.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-May-31.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-May-31.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-May-31.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-May-31.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-Sep-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-Sep-05.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-Sep-05.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-Sep-05.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2022-Apr-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Apr-22.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2022-Apr-22.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Apr-22.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2022-Jan-28.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jan-28.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2022-Jan-28.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jan-28.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-01.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-01.log new file mode 100644 index 0000000000..5719f819e8 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-01.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2022-Jun-01.log +# datestr = Wed Jun 1 23:22:28 2022 +# mode = Spectral +# flatfile1 = /sdata1005/deimos3/2022jun02/d0602_0027.fits +# flatfile2 = /sdata1005/deimos3/2022jun02/d0602_0028.fits +# flatfile3 = /sdata1005/deimos3/2022jun02/d0602_0029.fits +# flatfile4 = /sdata1005/deimos3/2022jun02/d0602_0030.fits +# flatfile5 = /sdata1005/deimos3/2022jun02/d0602_0031.fits +# zerofile1 = /sdata1005/deimos3/2022jun02/d0602_0024.fits +# zerofile2 = /sdata1005/deimos3/2022jun02/d0602_0025.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1689 0.017343 2.4508 0.036363 + 4 3 2 B 1.1647 0.017516 2.4420 0.036727 + 6 5 3 B 1.2477 0.020783 2.6161 0.043576 + 8 7 4 B 1.2056 0.020708 2.5278 0.043419 + 10 10 5 B 1.2199 0.016986 2.5577 0.035614 + 12 12 6 B 1.2028 0.013458 2.5219 0.028217 + 14 14 7 B 1.1980 0.018849 2.5118 0.039520 + 16 16 8 B 1.2148 0.018771 2.5471 0.039356 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-22.log new file mode 100644 index 0000000000..a38ff21a5f --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-22.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2022-Jun-22.log +# datestr = Wed Jun 22 18:44:36 2022 +# mode = Spectral +# flatfile1 = /sdata1005/deimos4/2022jun23/d0623_0035.fits +# flatfile2 = /sdata1005/deimos4/2022jun23/d0623_0036.fits +# flatfile3 = /sdata1005/deimos4/2022jun23/d0623_0037.fits +# flatfile4 = /sdata1005/deimos4/2022jun23/d0623_0038.fits +# flatfile5 = /sdata1005/deimos4/2022jun23/d0623_0039.fits +# zerofile1 = /sdata1005/deimos4/2022jun23/d0623_0032.fits +# zerofile2 = /sdata1005/deimos4/2022jun23/d0623_0033.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1826 0.016517 2.4796 0.034631 + 4 3 2 B 1.1710 0.014819 2.4553 0.031070 + 6 5 3 B 1.2479 0.021303 2.6165 0.044665 + 8 7 4 B 1.2141 0.014918 2.5457 0.031280 + 10 10 5 B 1.2165 0.015308 3.8256 0.047888 + 12 12 6 B 1.2035 0.018497 2.5234 0.038782 + 14 14 7 B 1.2006 0.019129 2.5172 0.040108 + 16 16 8 B 1.2167 0.022545 2.5510 0.047271 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Nov-14.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Nov-14.log new file mode 100644 index 0000000000..13553d0881 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Nov-14.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2022-Nov-14.log +# datestr = Mon Nov 14 00:22:22 2022 +# mode = Spectral +# flatfile1 = /sdata1004/dmoseng/2022nov14/d1114_0013.fits +# flatfile2 = /sdata1004/dmoseng/2022nov14/d1114_0014.fits +# flatfile3 = /sdata1004/dmoseng/2022nov14/d1114_0015.fits +# flatfile4 = /sdata1004/dmoseng/2022nov14/d1114_0016.fits +# flatfile5 = /sdata1004/dmoseng/2022nov14/d1114_0017.fits +# zerofile1 = /sdata1004/dmoseng/2022nov14/d1114_0010.fits +# zerofile2 = /sdata1004/dmoseng/2022nov14/d1114_0011.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1784 0.016103 2.4708 0.033763 + 4 3 2 B 1.1722 0.015694 2.4578 0.032905 + 6 5 3 B 1.2531 0.015877 2.6275 0.033289 + 8 7 4 B 1.2187 0.015008 2.5552 0.031466 + 10 10 5 B 1.2158 0.013563 2.5492 0.028437 + 12 12 6 B 1.2030 0.018836 2.5223 0.039492 + 14 14 7 B 1.2054 0.014269 2.5274 0.029917 + 16 16 8 B 1.2544 0.022474 2.6302 0.047122 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Oct-11.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Oct-11.log new file mode 100644 index 0000000000..0080d5652f --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Oct-11.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2022-Oct-11.log +# datestr = Tue Oct 11 15:00:40 2022 +# mode = Spectral +# flatfile1 = /sdata1004/dmoseng/2022oct12/d1012_0019.fits +# flatfile2 = /sdata1004/dmoseng/2022oct12/d1012_0020.fits +# flatfile3 = /sdata1004/dmoseng/2022oct12/d1012_0021.fits +# flatfile4 = /sdata1004/dmoseng/2022oct12/d1012_0022.fits +# flatfile5 = /sdata1004/dmoseng/2022oct12/d1012_0023.fits +# zerofile1 = /sdata1004/dmoseng/2022oct12/d1012_0016.fits +# zerofile2 = /sdata1004/dmoseng/2022oct12/d1012_0017.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1797 0.017066 2.4735 0.035782 + 4 3 2 B 1.1797 0.017516 2.4736 0.036725 + 6 5 3 B 1.2532 0.016484 2.6276 0.034562 + 8 7 4 B 1.2138 0.012262 2.5449 0.025710 + 10 10 5 B 1.2171 0.017933 2.5518 0.037600 + 12 12 6 B 1.2034 0.017107 2.5231 0.035869 + 14 14 7 B 1.2060 0.017311 2.5287 0.036296 + 16 16 8 B 1.2179 0.012596 2.5535 0.026409 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Aug-06.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Aug-06.log new file mode 100644 index 0000000000..79cc78b548 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Aug-06.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2023-Aug-06.log +# datestr = Sun Aug 6 20:07:19 2023 +# mode = Spectral +# flatfile1 = /sdata1005/dmoseng/2023aug07/d0728_0044.fits +# flatfile2 = /sdata1005/dmoseng/2023aug07/d0728_0045.fits +# flatfile3 = /sdata1005/dmoseng/2023aug07/d0728_0046.fits +# flatfile4 = /sdata1005/dmoseng/2023aug07/d0728_0047.fits +# flatfile5 = /sdata1005/dmoseng/2023aug07/d0728_0048.fits +# zerofile1 = /sdata1005/dmoseng/2023aug07/d0728_0041.fits +# zerofile2 = /sdata1005/dmoseng/2023aug07/d0728_0042.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1784 0.014323 2.4708 0.030031 + 4 3 2 B 1.1687 0.012534 2.4504 0.026280 + 6 5 3 B 1.2621 0.019232 2.6463 0.040324 + 8 7 4 B 1.2110 0.016884 2.5390 0.035401 + 10 10 5 B 1.2421 0.019382 2.6044 0.040637 + 12 12 6 B 1.2009 0.018459 2.5178 0.038702 + 14 14 7 B 1.1947 0.016139 2.5049 0.033839 + 16 16 8 B 1.2446 0.022439 2.6096 0.047049 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Dec-11.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Dec-11.log new file mode 100644 index 0000000000..8a1ae1af1e --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Dec-11.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2023-Dec-11.log +# datestr = Mon Dec 11 01:24:09 2023 +# mode = Spectral +# flatfile1 = /sdata1005/deimos2/2023dec11/d1211_0022.fits +# flatfile2 = /sdata1005/deimos2/2023dec11/d1211_0023.fits +# flatfile3 = /sdata1005/deimos2/2023dec11/d1211_0024.fits +# flatfile4 = /sdata1005/deimos2/2023dec11/d1211_0025.fits +# flatfile5 = /sdata1005/deimos2/2023dec11/d1211_0026.fits +# zerofile1 = /sdata1005/deimos2/2023dec11/d1211_0019.fits +# zerofile2 = /sdata1005/deimos2/2023dec11/d1211_0020.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1800 0.013007 2.4740 0.027271 + 4 3 2 B 1.1719 0.015773 2.4572 0.033070 + 6 5 3 B 1.2648 0.019444 3.9777 0.061153 + 8 7 4 B 1.2095 0.018859 2.5360 0.039542 + 10 10 5 B 1.2327 0.017411 2.5846 0.036505 + 12 12 6 B 1.1974 0.013983 2.5106 0.029318 + 14 14 7 B 1.1943 0.018010 2.5041 0.037762 + 16 16 8 B 1.2449 0.016423 2.6101 0.034435 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Jan-06.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Jan-06.log new file mode 100644 index 0000000000..52b8822d48 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Jan-06.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2023-Jan-06.log +# datestr = Fri Jan 6 22:28:15 2023 +# mode = Spectral +# flatfile1 = /sdata1005/deimos2/2023jan07/d0107_0037.fits +# flatfile2 = /sdata1005/deimos2/2023jan07/d0107_0038.fits +# flatfile3 = /sdata1005/deimos2/2023jan07/d0107_0039.fits +# flatfile4 = /sdata1005/deimos2/2023jan07/d0107_0040.fits +# flatfile5 = /sdata1005/deimos2/2023jan07/d0107_0041.fits +# zerofile1 = /sdata1005/deimos2/2023jan07/d0107_0034.fits +# zerofile2 = /sdata1005/deimos2/2023jan07/d0107_0035.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1790 0.013500 2.4721 0.028305 + 4 3 2 B 1.1731 0.018365 2.4596 0.038507 + 6 5 3 B 1.2702 0.015348 3.9949 0.048269 + 8 7 4 B 1.2170 0.014134 2.5517 0.029635 + 10 10 5 B 1.2471 0.023396 7.7994 0.26533 + 12 12 6 B 1.1977 0.017206 2.5112 0.036076 + 14 14 7 B 1.1976 0.014123 2.5109 0.029612 + 16 16 8 B 1.2485 0.021585 2.6177 0.045256 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Oct-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Oct-05.log new file mode 100644 index 0000000000..81ae1c4cc8 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Oct-05.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2023-Oct-05.log +# datestr = Thu Oct 5 12:57:16 2023 +# mode = Spectral +# flatfile1 = /sdata1004/dmoseng/2023oct05/d1005_0046.fits +# flatfile2 = /sdata1004/dmoseng/2023oct05/d1005_0047.fits +# flatfile3 = /sdata1004/dmoseng/2023oct05/d1005_0048.fits +# flatfile4 = /sdata1004/dmoseng/2023oct05/d1005_0049.fits +# flatfile5 = /sdata1004/dmoseng/2023oct05/d1005_0050.fits +# zerofile1 = /sdata1004/dmoseng/2023oct05/d1005_0043.fits +# zerofile2 = /sdata1004/dmoseng/2023oct05/d1005_0044.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1799 0.014565 2.4740 0.030537 + 4 3 2 B 1.1741 0.015021 2.4618 0.031494 + 6 5 3 B 1.2625 0.015967 2.6471 0.033478 + 8 7 4 B 1.2105 0.016401 2.5381 0.034388 + 10 10 5 B 1.2344 0.014263 2.5882 0.029905 + 12 12 6 B 1.2006 0.014541 2.5172 0.030488 + 14 14 7 B 1.1942 0.015844 2.5038 0.033219 + 16 16 8 B 1.2538 0.016214 2.6288 0.033996 diff --git a/pypeit/data/utils.py b/pypeit/data/utils.py index cba62b0153..2434625fdd 100644 --- a/pypeit/data/utils.py +++ b/pypeit/data/utils.py @@ -790,3 +790,4 @@ def load_sky_spectrum(sky_file: str) -> xspectrum1d.XSpectrum1D: (`linetools.spectra.xspectrum1d.XSpectrum1D`_): Sky spectrum """ return xspectrum1d.XSpectrum1D.from_file(str(Paths.sky_spec / sky_file)) + diff --git a/pypeit/datamodel.py b/pypeit/datamodel.py index 9101420aa5..b352198af7 100644 --- a/pypeit/datamodel.py +++ b/pypeit/datamodel.py @@ -1129,6 +1129,30 @@ def _parse(cls, hdu, ext=None, ext_pseudo=None, transpose_table_arrays=False, return _d, dm_version_passed and found_data, dm_type_passed and found_data, \ np.unique(parsed_hdus).tolist() + @classmethod + def _check_parsed(cls, version_passed, type_passed, chk_version=True): + """ + Convenience function that issues the warnings/errors caused by parsing a + file into a datamodel. + + Args: + version_passed (:obj:`bool`): + Flag that the datamodel version is correct. + type_passed (:obj:`bool`): + Flag that the datamodel class type is correct. + chk_version (:obj:`bool`, optional): + Flag to impose strict version checking. + """ + if not type_passed: + msgs.error(f'The HDU(s) cannot be parsed by a {cls.__name__} object!', + cls='PypeItDataModelError') + if not version_passed: + msg = f'Current version of {cls.__name__} object in code ({cls.version}) ' \ + 'does not match version used to write your HDU(s)!' + if chk_version: + msgs.error(msg, cls='PypeItDataModelError') + else: + msgs.warn(msg) def __getattr__(self, item): """Maps values to attributes. @@ -1424,18 +1448,10 @@ def from_hdu(cls, hdu, chk_version=True, **kwargs): **kwargs: Passed directly to :func:`_parse`. """ + # Parse the data d, dm_version_passed, dm_type_passed, parsed_hdus = cls._parse(hdu, **kwargs) # Check version and type? - if not dm_type_passed: - msgs.error(f'The HDU(s) cannot be parsed by a {cls.__name__} object!', - cls='PypeItDataModelError') - if not dm_version_passed: - msg = f'Current version of {cls.__name__} object in code ({cls.version}) ' \ - 'does not match version used to write your HDU(s)!' - if chk_version: - msgs.error(msg, cls='PypeItDataModelError') - else: - msgs.warn(msg) + cls._check_parsed(dm_version_passed, dm_type_passed, chk_version=chk_version) # Instantiate # NOTE: We can't use `cls(d)`, where `d` is the dictionary returned by diff --git a/pypeit/deprecated/find_objects.py b/pypeit/deprecated/find_objects.py deleted file mode 100644 index 695ab55698..0000000000 --- a/pypeit/deprecated/find_objects.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python -# -# See top-level LICENSE file for Copyright information -# -# -*- coding: utf-8 -*- -""" -This script enables the viewing of a processed FITS file -with extras. Run above the Science/ folder. -""" - -import os -import argparse -import numpy as np - -from astropy.table import Table - -from pypeit import msgs -from pypeit import io -from pypeit import slittrace -from pypeit.core import gui -from pypeit.core.parse import get_dnum - - -def parse_args(options=None, return_parser=False): - - parser = argparse.ArgumentParser(description='Display sky subtracted, spec2d image in the' - 'interactive object finding GUI. Run above' - 'the Science/ folder', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('file', type = str, default=None, help='PYPEIT spec2d file') - parser.add_argument("--list", default=False, help="List the extensions only?", - action="store_true") - parser.add_argument('--det', default=1, type=int, help="Detector") - parser.add_argument("--old", default=False, action="store_true", help="Used old slit tracing") - - if return_parser: - return parser - - return parser.parse_args() if options is None else parser.parse_args(options) - - -def parse_traces(hdulist_1d, det_nm): - """Extract the relevant trace information - """ - traces = dict(traces=[], fwhm=[]) - pkflux = [] - for hdu in hdulist_1d: - if det_nm in hdu.name: - tbl = Table(hdu.data) - trace = tbl['TRACE_SPAT'] - fwhm = tbl['FWHMFIT'] - obj_id = hdu.name.split('-')[0] - traces['traces'].append(trace.copy()) - traces['fwhm'].append(np.median(fwhm)) - pkflux.append(np.median(tbl['BOX_COUNTS'])) - traces['pkflux'] = np.array(pkflux) - return traces - - -def main(args): - - raise NotImplementedError('This script is currently out of date.') - - # List only? - hdu = io.fits_open(args.file) - head0 = hdu[0].header - if args.list: - hdu.info() - return - - # Init - sdet = get_dnum(args.det, prefix=False) - - # One detector, sky sub for now - names = [hdu[i].name for i in range(len(hdu))] - - try: - exten = names.index('DET{:s}-PROCESSED'.format(sdet)) - except: # Backwards compatability - msgs.error('Requested detector {:s} was not processed.\n' - 'Maybe you chose the wrong one to view?\n' - 'Set with --det= or check file contents with --list'.format(sdet)) - sciimg = hdu[exten].data - try: - exten = names.index('DET{:s}-SKY'.format(sdet)) - except: # Backwards compatability - msgs.error('Requested detector {:s} has no sky model.\n' - 'Maybe you chose the wrong one to view?\n' - 'Set with --det= or check file contents with --list'.format(sdet)) - skymodel = hdu[exten].data - try: - exten = names.index('DET{:s}-MASK'.format(sdet)) - except ValueError: # Backwards compatability - msgs.error('Requested detector {:s} has no bit mask.\n' - 'Maybe you chose the wrong one to view?\n' - 'Set with --det= or check file contents with --list'.format(sdet)) - - mask = hdu[exten].data - frame = (sciimg - skymodel) * (mask == 0) - - mdir = head0['PYPMFDIR'] - mkey = head0['FRAMMKEY'] - mast_key = '{0}_{1:02d}'.format(mkey, args.det) - if not os.path.exists(mdir): - mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir)) - msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base)) - mdir = mdir_base - - # Assumes a MasterSlit file has been written - #slits = slittrace.SlitTraceSet.from_master('{0}_{1:02d}'.format(head0['TRACMKEY'], args.det), - # mdir) - # Load the slits information - slits = slittrace.SlitTraceSet.from_master(mast_key, mdir) - - # Object traces - left, right, mask = slits.select_edges() - msgs.error("You need to choose which slits you care about here") - - # Get object traces - spec1d_file = args.file.replace('spec2d', 'spec1d') - if os.path.isfile(spec1d_file): - hdulist_1d = io.fits_open(spec1d_file) - else: - hdulist_1d = [] - msgs.warn('Could not find spec1d file: {:s}'.format(spec1d_file) + msgs.newline() + - ' No objects were extracted.') - - msgs.error("This code needs to be refactored since tslits_dict was removed...") - import pdb - pdb.set_trace() - tslits_dict['objtrc'] = parse_traces(hdulist_1d, det_nm) - obj_trace = parse_traces(hdulist_1d, 'DET{:s}'.format(sdet)) - - # TODO :: Need to include standard star trace in the spec2d files - std_trace = None - - # Extract some trace models - fwhm = 2 # Start with some default value - # TODO: Dictionaries like this are a pet peeve of mine. I'd prefer - # either individual objects or a class with a well-formed data model. - # TODO: Why do all of these dictionary elements need fwhm? Can they - # be different? - trace_models = dict() - # Brightest object on slit - trace_models['object'] = dict(trace_model=None, fwhm=fwhm) - if len(obj_trace['pkflux']) > 0: - smash_peakflux = obj_trace['pkflux'] - ibri = smash_peakflux.argmax() - trace_models['object']['trace_model'] = obj_trace['traces'][ibri] - trace_models['object']['fwhm'] = obj_trace['fwhm'][ibri] - # Standard star trace - trace_models['std'] = dict(trace_model=std_trace, fwhm=trace_models['object']['fwhm']) - # Trace of the slit edge - # TODO: Any particular reason to use the lefts? - trace_models['slit'] = dict(trace_model=left.copy(), fwhm=trace_models['object']['fwhm']) - - # Finally, initialise the GUI - gui.object_find.initialise(args.det, frame, left, right, obj_trace, trace_models, None, - printout=True, slit_ids=slits.id) - - ofgui = gui_object_find.initialise(args.det, frame, tslits_dict, None, printout=True, slit_ids=slits.id) diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index 684a13d5d8..efc9e1f610 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -103,7 +103,6 @@ """ import os -import time import inspect from pathlib import Path from collections import OrderedDict @@ -118,21 +117,18 @@ from matplotlib import pyplot as plt from matplotlib import ticker, rc -from astropy.stats import sigma_clipped_stats from astropy import table from pypeit import msgs -from pypeit import io from pypeit import utils from pypeit import sampling from pypeit import slittrace from pypeit.datamodel import DataContainer from pypeit import calibframe -from pypeit.images.mosaic import Mosaic from pypeit.bitmask import BitMask from pypeit.display import display from pypeit.par.pypeitpar import EdgeTracePar -from pypeit.core import parse, pydl, procimg, pca, trace, slitdesign_matching +from pypeit.core import parse, procimg, trace, slitdesign_matching from pypeit.core import fitting from pypeit.images.buildimage import TraceImage from pypeit.tracepca import TracePCA @@ -176,7 +172,8 @@ def __init__(self): ('ABNORMALSLIT_SHORT', 'Slit formed by left and right edge is abnormally short'), ('ABNORMALSLIT_LONG', 'Slit formed by left and right edge is abnormally long'), ('USERRMSLIT', 'Slit removed by user'), - ('NOORDER', 'Unable to associate this trace with an echelle order (echelle ' + ('NOORDER', '(DEPRECATED as of version 1.15.0; use ORDERMISMATCH). ' + 'Unable to associate this trace with an echelle order (echelle ' 'spectrographs only)'), ('ORDERMISMATCH', 'Slit traces are not well matched to any echelle order (echelle ' 'spectrographs only)'), @@ -184,7 +181,7 @@ def __init__(self): 'order missed by the automated tracing'), ('LARGELENGTHCHANGE', 'Large difference in the slit length as a function of ' 'wavelength.')]) - super(EdgeTraceBitMask, self).__init__(list(mask.keys()), descr=list(mask.values())) + super().__init__(list(mask.keys()), descr=list(mask.values())) @property def bad_flags(self): @@ -497,7 +494,7 @@ def __init__(self, traceimg, spectrograph, par, qa_path=None, auto=False, debug= show_stages=False): # Instantiate as an empty DataContainer - super(EdgeTraceSet, self).__init__() + super().__init__() # Check input types if not isinstance(traceimg, TraceImage): @@ -716,12 +713,12 @@ def rectify(self, flux, bpm=None, extract_width=None, mask_threshold=0.5, side=' """ if self.pcatype is None: msgs.error('Must first run the PCA analysis for the traces; run build_pca.') - pca = (self.left_pca if side == 'left' else self.right_pca) \ + _pca = (self.left_pca if side == 'left' else self.right_pca) \ if self.par['left_right_pca'] else self.pca # Get the traces that cross the reference spatial position at # the first and last pixels of the image - first_last_trace = pca.predict(np.array([0,self.nspat-1])) + first_last_trace = _pca.predict(np.array([0,self.nspat-1])) # Use these two traces to define the spatial pixel coordinates # to sample start = np.ceil(np.amax(np.amin(first_last_trace, axis=1))).astype(int) @@ -730,7 +727,7 @@ def rectify(self, flux, bpm=None, extract_width=None, mask_threshold=0.5, side=' # Rectify the image # TODO: This has its limitations if the PCA is highly non-linear. ocol = np.arange(self.nspat+buffer)-start - return sampling.rectify_image(flux, pca.predict(ocol), bpm=bpm, ocol=ocol, + return sampling.rectify_image(flux, _pca.predict(ocol), bpm=bpm, ocol=ocol, max_ocol=self.nspat-1, extract_width=extract_width, mask_threshold=mask_threshold) @@ -1057,13 +1054,13 @@ def initial_trace(self, bpm=None): # coordinates self.edge_img = np.zeros((self.nspec, self.ntrace), dtype=int) for i in range(self.ntrace): - row, col = np.where(np.invert(trace_id_img.mask) + row, col = np.where(np.logical_not(trace_id_img.mask) & (trace_id_img.data == self.traceid[i])) self.edge_img[row,i] = col self.edge_msk[row,i] = 0 # Turn-off the mask # Flag any insert traces - row, col = np.where(np.invert(trace_id_img.mask) & inserted_edge + row, col = np.where(np.logical_not(trace_id_img.mask) & inserted_edge & (trace_id_img.data == self.traceid[i])) if len(row) > 0: self.edge_msk[row,i] = self.bitmask.turn_on(self.edge_msk[row,i], 'ORPHANINSERT') @@ -1135,7 +1132,7 @@ def _base_header(self, hdr=None): `astropy.io.fits.Header`_: Header object to include in all HDU extensions. """ - _hdr = super(EdgeTraceSet, self)._base_header(hdr=hdr) + _hdr = super()._base_header(hdr=hdr) _hdr['QAPATH'] = 'None' if self.qa_path is None else str(self.qa_path) self.par.to_header(_hdr) self.bitmask.to_header(_hdr) @@ -1207,7 +1204,7 @@ def to_hdu(self, **kwargs): # Do not need to change the default behavior if the PCA # doesn't exist or there is only one PCA for both left and # right edges. - return super(EdgeTraceSet, self).to_hdu(**kwargs) + return super().to_hdu(**kwargs) # TODO: We need a better solution for multiple levels of nested # DataContainers. Here the commpication is that we're writing @@ -1220,7 +1217,7 @@ def to_hdu(self, **kwargs): self.left_pca, self.right_pca = None, None # Run the default (with add_primary = False) - hdu = super(EdgeTraceSet, self).to_hdu(**kwargs) + hdu = super().to_hdu(**kwargs) # Reset them self.left_pca, self.right_pca = _left_pca, _right_pca @@ -1259,13 +1256,9 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): # parse traceimg because it's not a single-extension # DataContainer. It *will* parse pca, left_pca, and right_pca, # if they exist, but not their model components. - d, version_passed, type_passed, parsed_hdus = super(EdgeTraceSet, cls)._parse(hdu) - if not type_passed: - msgs.error('The HDU(s) cannot be parsed by a {0} object!'.format(cls.__name__)) - if not version_passed: - _f = msgs.error if chk_version else msgs.warn - _f('Current version of {0} object in code (v{1})'.format(cls.__name__, cls.version) - + ' does not match version used to write your HDU(s)!') + d, version_passed, type_passed, parsed_hdus = cls._parse(hdu) + # Check + cls._check_parsed(version_passed, type_passed, chk_version=chk_version) # Instantiate the TraceImage from the header d['traceimg'] = TraceImage.from_hdu(hdu, chk_version=chk_version) @@ -1292,7 +1285,7 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): else cls.datamodel[key]['atype']) # Instantiate - self = super(EdgeTraceSet, cls).from_dict(d=d) + self = super().from_dict(d=d) # Calibration frame attributes # NOTE: If multiple HDUs are parsed, this assumes that the information @@ -1318,7 +1311,8 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): hdr_bitmask = BitMask.from_header(hdu['SOBELSIG'].header) if chk_version and hdr_bitmask.bits != self.bitmask.bits: msgs.error('The bitmask in this fits file appear to be out of date! Recreate this ' - 'file either by rerunning run_pypeit or pypeit_trace_edges.') + 'file by re-running the relevant script or set chk_version=False.', + cls='PypeItBitMaskError') return self @@ -1416,7 +1410,7 @@ def show(self, include_error=False, thin=10, in_ginga=False, include_img=True, fit = self.edge_fit err = np.ma.MaskedArray(self.edge_err, mask=np.ma.getmaskarray(cen).copy()) msk = None if flag is None \ - else np.ma.MaskedArray(self.edge_cen, mask=np.invert( + else np.ma.MaskedArray(self.edge_cen, mask=np.logical_not( self.bitmask.flagged(self.edge_msk, flag=_flag))) is_left = self.is_left is_right = self.is_right @@ -1451,7 +1445,7 @@ def show(self, include_error=False, thin=10, in_ginga=False, include_img=True, nslits = slits.nslits is_left = np.ones(2*nslits, dtype=bool) is_left[nslits:] = False - is_right = np.invert(is_left) + is_right = np.logical_not(is_left) gpm = np.ones(2*nslits, dtype=bool) traceid = np.concatenate((-np.arange(nslits), np.arange(nslits))) synced = True @@ -2082,7 +2076,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp if subset is not None: msgs.info('Tolerance for finding repeat traces: {0:.1f}'.format(self.par['match_tol'])) side = -1 if np.all(self.traceid[indx] < 0) else 1 - compare = (side*self.traceid > 0) & np.invert(indx) + compare = (side*self.traceid > 0) & np.logical_not(indx) if np.any(compare): # Use masked arrays to ease exclusion of masked data _col = np.ma.MaskedArray(np.round(_cen).astype(int), mask=_bpm) @@ -2093,7 +2087,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp # TODO: This tolerance uses the integer image # coordinates, not the floating-point centroid # coordinates.... - repeat[indx] = (mindiff.data < self.par['match_tol']) & np.invert(mindiff.mask) + repeat[indx] = (mindiff.data < self.par['match_tol']) & np.logical_not(mindiff.mask) if np.any(repeat): _msk[:,repeat] = self.bitmask.turn_on(_msk[:,repeat], 'DUPLICATE') msgs.info('Found {0} repeat trace(s).'.format(np.sum(repeat))) @@ -2103,7 +2097,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp if minimum_spec_length is not None: msgs.info('Minimum spectral length of any trace (pixels): {0:.2f}'.format( minimum_spec_length)) - short[indx] = np.sum(np.invert(_bpm[:,indx]), axis=0) < minimum_spec_length + short[indx] = np.sum(np.logical_not(_bpm[:,indx]), axis=0) < minimum_spec_length if np.any(short): _msk[:,short] = self.bitmask.turn_on(_msk[:,short], 'SHORTRANGE') msgs.info('Found {0} short trace(s).'.format(np.sum(short))) @@ -2115,7 +2109,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp hit_min = np.zeros_like(indx, dtype=bool) if min_spatial is not None: hit_min[indx] = (col[self.nspec//2,indx] <= min_spatial) \ - & np.invert(_bpm[self.nspec//2,indx]) + & np.logical_not(_bpm[self.nspec//2,indx]) if np.any(hit_min): _msk[:,hit_min] = self.bitmask.turn_on(_msk[:,hit_min], 'HITMIN') msgs.info('{0} trace(s) hit the minimum centroid value.'.format(np.sum(hit_min))) @@ -2126,7 +2120,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp hit_max = np.zeros_like(indx, dtype=bool) if max_spatial is not None: hit_max[indx] = (col[self.nspec//2,indx] >= max_spatial) \ - & np.invert(_bpm[self.nspec//2,indx]) + & np.logical_not(_bpm[self.nspec//2,indx]) if np.any(hit_max): _msk[:,hit_max] = self.bitmask.turn_on(_msk[:,hit_max], 'HITMAX') msgs.info('{0} trace(s) hit the maximum centroid value.'.format(np.sum(hit_max))) @@ -2143,7 +2137,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp # Good traces bad = indx & (repeat | short | hit_min | hit_max | off_detector) msgs.info('Identified {0} bad trace(s) in all.'.format(np.sum(bad))) - good = indx & np.invert(bad) + good = indx & np.logical_not(bad) return good, bad def merge_traces(self, merge_frac=0.5, refit=True, debug=False): @@ -2214,7 +2208,7 @@ def merge_traces(self, merge_frac=0.5, refit=True, debug=False): for side in ['left', 'right']: # Match traces on the same side and indx = np.where((self.is_left if side == 'left' else self.is_right) - & np.invert(np.all(cen_match.mask, axis=0)))[0] + & np.logical_not(np.all(cen_match.mask, axis=0)))[0] if indx.size == 0: continue @@ -2230,7 +2224,7 @@ def merge_traces(self, merge_frac=0.5, refit=True, debug=False): merge = np.append(indx[i], indx[i+1:][merge]) msgs.info('Merging traces: {0}'.format(self.traceid[merge])) merged_trace = np.ma.mean(cen_merge[:,merge], axis=1) - gpm = np.invert(np.ma.getmaskarray(merged_trace)) + gpm = np.logical_not(np.ma.getmaskarray(merged_trace)) self.edge_cen[gpm,indx[i]] = merged_trace[gpm] self.edge_msk[gpm,indx[i]] = 0 @@ -2403,23 +2397,22 @@ def check_synced(self, rebuild_pca=False): """ Quality check and masking of the synchronized edges. - Before executing this method, the slit edges must be - synchronized (see :func:`sync`) and ordered spatially in - left-right pairs (see :func:`spatial_sort`). The former is - checked explicitly. Any traces fully masked as bad (see - :func:`clean_traces`) are removed, along with its - synchronized partner. + Before executing this method, the slit edges must be synchronized (see + :func:`sync`) and ordered spatially in left-right pairs (see + :func:`spatial_sort`); only the former is checked explicitly. Any traces + fully masked as bad (see :func:`clean_traces`) are removed, along with + its synchronized partner. Used parameters from :attr:`par` - (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are - `minimum_slit_gap`, `minimum_slit_length`, - `minimum_slit_length_sci`, and `length_range`. + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are ``minimum_slit_gap``, + ``minimum_slit_length``, ``minimum_slit_length_sci``, and + ``length_range``. Checks are: - - Any trace falling off the edge of the detector is - masked (see :class:`EdgeTraceBitMask`). This is the - only check performed by default (i.e., when no keyword - arguments are provided). + + - Any trace falling off the edge of the detector is masked (see + :class:`EdgeTraceBitMask`). This is the only check performed by + default (i.e., when no keyword arguments are provided). - Traces that form slit gaps (the median difference between the right and left traces of adjacent slits) that are below an absolute tolerance are removed and @@ -2469,9 +2462,12 @@ def check_synced(self, rebuild_pca=False): """ if self.is_empty: msgs.warn('No traces to check.') + return # Decide if the PCA should be rebuilt _rebuild_pca = rebuild_pca and self.pcatype is not None and self.can_pca() + if rebuild_pca and not _rebuild_pca: + msgs.warn('Rebuilding the PCA was requested but is not possible.') # Remove any fully masked traces and its synced counterpart; # force the removal of traces marked as SYNCERROR, even if @@ -2512,7 +2508,7 @@ def check_synced(self, rebuild_pca=False): msgs.info('Binning: {0}'.format(self.traceimg.detector.binning)) msgs.info('Platescale per binned pixel: {0}'.format(platescale)) if self.par['minimum_slit_dlength'] is not None: - length_atol = self.par['minimum_slit_dlength']/platescale + dlength_atol = self.par['minimum_slit_dlength']/platescale if self.par['minimum_slit_length'] is not None: length_atol = self.par['minimum_slit_length']/platescale if self.par['minimum_slit_length_sci'] is not None: @@ -2545,7 +2541,7 @@ def check_synced(self, rebuild_pca=False): msgs.info('Found {0} slit(s) with gaps below {1} arcsec ({2:.2f} pixels).'.format( np.sum(indx), self.par['minimum_slit_gap'], gap_atol)) rmtrace = np.concatenate(([False],np.repeat(indx,2),[False])) - self.remove_traces(rmtrace, rebuild_pca=rebuild_pca) + self.remove_traces(rmtrace, rebuild_pca=_rebuild_pca) # TODO: This should never happen, but keep this around # until we're sure it doesn't. if self.is_empty: @@ -2607,6 +2603,15 @@ def check_synced(self, rebuild_pca=False): self.edge_msk[:,long] \ = self.bitmask.turn_on(self.edge_msk[:,long], 'ABNORMALSLIT_LONG') + # TODO: Consider removing slits that have large length changes. Like so: +# # Remove traces that have significant changes in their spatial extent +# # along the dispersion direction. +# dl_flag = self.fully_masked_traces(flag='LARGELENGTHCHANGE') +# if np.any(dl_flag): +# msgs.info(f'Removing {np.sum(dl_flag)} traces because of large spatial extent ' +# ' changes along the dispersion direction.') +# self.remove_traces(dl_flag, rebuild_pca=_rebuild_pca) + # Get the slits that have been flagged as abnormally short. This should # be the same as the definition above, it's just redone here to ensure # `short` is defined when `length_rtol` is None. @@ -2952,7 +2957,7 @@ def spatial_sort(self, use_mean=False, use_fit=True): # Reorder the trace numbers indx = self.traceid < 0 self.traceid[indx] = -1-np.arange(np.sum(indx)) - indx = np.invert(indx) + indx = np.logical_not(indx) self.traceid[indx] = 1+np.arange(np.sum(indx)) def _reset_pca(self, rebuild): @@ -3077,7 +3082,7 @@ def fit_refine(self, weighting='uniform', debug=False, idx=None): _sobelsig = self._side_dependent_sobel(side) # Select traces on this side and that are not fully masked indx = (self.is_left if side == 'left' else self.is_right) \ - & np.invert(np.all(edge_bpm, axis=0)) + & np.logical_not(np.all(edge_bpm, axis=0)) if not np.any(indx): continue @@ -3153,7 +3158,7 @@ def can_pca(self): # NOTE: Because of the run of check_traces above, short traces # are fully flagged meaning that we can just check if the # length of the trace is larger than 0. - good = np.sum(np.invert(self.bitmask.flagged(self.edge_msk)), axis=0) > 0 + good = np.sum(np.logical_not(self.bitmask.flagged(self.edge_msk)), axis=0) > 0 # Returned value depends on whether or not the left and right # traces are done separately @@ -3425,33 +3430,36 @@ def pca_refine(self, use_center=False, debug=False, force=False): def peak_refine(self, rebuild_pca=False, debug=False): """ - Refine the trace by isolating peaks and troughs in the - Sobel-filtered image. + Refine the trace by isolating peaks and troughs in the Sobel-filtered + image. This function *requires* that the PCA model exists; see - :func:`build_pca` or :func:`pca_refine`. It is also primarily - a wrapper for :func:`~pypeit.core.trace.peak_trace`. See the - documentation of that function for the explanation of the - algorithm. - - If the left and right traces have separate PCA - decompositions, this function makes one call to - :func:`~pypeit.core.trace.peak_trace` for each side. - Otherwise, a single call is made to - :func:`~pypeit.core.trace.peak_trace` where both the peak and - troughs in :attr:`sobelsig` are detected and traced. - - Note that this effectively reinstantiates much of the object - attributes, including :attr:`traceid` :attr:`edge_cen` - :attr:`edge_err` :attr:`edge_msk` :attr:`edge_img` - :attr:`edge_fit`, and :attr:`fittype`. + :func:`build_pca` or :func:`pca_refine`. It is also primarily a wrapper + for :func:`~pypeit.core.trace.peak_trace`. See the documentation of that + function for the explanation of the algorithm. + + If the left and right traces have separate PCA decompositions, this + function makes one call to :func:`~pypeit.core.trace.peak_trace` for + each side. Otherwise, a single call is made to + :func:`~pypeit.core.trace.peak_trace` where both the peak and troughs in + :attr:`sobelsig` are detected and traced. + + Optionally, the code will match and compare the traces found and fit by + :func:`~pypeit.core.trace.peak_trace` to the original traces. If the + RMS difference between the matched traces is large, they can be removed + (see ``trace_rms_tol`` in :class:`~pypeit.par.pypeitpar.EdgeTracePar`). + + Note that this effectively reinstantiates much of the object attributes, + including :attr:`traceid`, :attr:`edge_cen`, :attr:`edge_err`, + :attr:`edge_msk`, :attr:`edge_img`, :attr:`edge_fit`, and + :attr:`fittype`. Used parameters from :attr:`par` - (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are - ``left_right_pca``, ``edge_thresh``, ``smash_range``, - ``edge_detect_clip``, ``trace_median_frac``, ``trace_thresh``, - ``fit_function``, ``fit_order``, ``fwhm_uniform``, ``fwhm_uniform``, - ``niter_gaussian``, ``niter_gaussian``, ``fit_maxdev``, and + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are ``left_right_pca``, + ``edge_thresh``, ``smash_range``, ``edge_detect_clip``, + ``trace_median_frac``, ``trace_thresh``, ``trace_rms_tol``, + ``fit_function``, ``fit_order``, ``fwhm_uniform``, ``niter_uniform``, + ``fwhm_gaussian``, ``niter_gaussian``, ``fit_maxdev``, and ``fit_maxiter``. Args: @@ -3528,8 +3536,6 @@ def peak_refine(self, rebuild_pca=False, debug=False): # Iterate through each side for side in ['left', 'right']: # Get the image relevant to tracing -# _sobelsig = trace.prepare_sobel_for_trace(self.sobelsig, bpm=self.bpm, boxcar=5, -# side=side) _sobelsig = self._side_dependent_sobel(side) _pca = self.left_pca if side == 'left' else self.right_pca @@ -3575,6 +3581,52 @@ def peak_refine(self, rebuild_pca=False, debug=False): if ntrace < self.ntrace: msgs.warn('Found fewer traces using peak finding than originally available. ' 'May want to reset peak threshold.') + + if self.par['trace_rms_tol'] is not None: + # Get the PCA reference row. The PCA *must* have been defined to get + # this far (see pcatype test at the beginning of the function). + reference_row = self.left_pca.reference_row if self.par['left_right_pca'] \ + else self.pca.reference_row + # Match the new trace positions to the input ones + gpm = np.logical_not(self.bitmask.flagged(self.edge_msk[reference_row])) + # TODO: Include a tolerance here? Needs testing with more datasets. + peak_indx = slitdesign_matching.match_positions_1D( + self.edge_fit[reference_row][gpm], + fit[reference_row]) + + # Determine the RMS difference between the input and output traces. + # This allows us to compare traces that had already been identified + # to their new measurements resulting from peak_trace, and remove + # them if they are too discrepant from their original form. This is + # largely meant to find and remove poorly constrained traces, where + # the polynomial fit goes wonky. + diff = fit - self.edge_fit.T[gpm][peak_indx].T + rms = np.sqrt(np.mean((diff - np.mean(diff, axis=0)[None,:])**2, axis=0)) + + # Report + msgs.info('-'*30) + msgs.info('Matched spatial locations and RMS difference along spectral direction') + msgs.info(f' {"OLD":>8} {"NEW":>8} {"RMS":>8}') + msgs.info(' '+'-'*8+' '+'-'*8+' '+'-'*8) + for i in range(len(peak_indx)): + if peak_indx[i] < 0: + continue + msgs.info(f' {self.edge_fit[reference_row][gpm][peak_indx[i]]:8.1f}' + f' {fit[reference_row][i]:8.1f} {rms[i]:8.3f}') + + # Select traces below the RMS tolerance or that were newly + # identified by peak_trace. I.e., this will *not* catch newly + # identified traces found by peak_trace that are also poorly + # constrained! + indx = (rms < self.par['trace_rms_tol']) | (peak_indx == -1) + if not np.all(indx): + msgs.info(f'Removing {indx.size - np.sum(indx)} trace(s) due to large RMS ' + 'difference with previous trace locations.') + fit = fit[:,indx] + cen = cen[:,indx] + err = err[:,indx] + msk = msk[:,indx] + nleft -= np.sum(np.where(np.logical_not(indx))[0] < nleft) # Reset the trace data self.traceid = np.zeros(ntrace, dtype=int) @@ -3707,7 +3759,7 @@ def _get_reference_locations(self, trace_cen, add_edge): # Build a masked array with the trace positions at that # spectral row, masked where new traces are supposed to go. trace_ref = np.ma.masked_all(add_edge.size) - trace_ref[np.invert(add_edge)] = trace_cen[reference_row,:] + trace_ref[np.logical_not(add_edge)] = trace_cen[reference_row,:] trace_ref = trace_ref.reshape(-1,2) # Get the length and center of each slit in pixels @@ -4777,304 +4829,6 @@ def _fill_objects_table(self, maskdef_id): self.objects['TRACEID'] = utils.index_of_x_eq_y(self.objects['MASKDEF_ID'], self.design['MASKDEF_ID'], strict=True) -# NOTE: I'd like us to keep this commented mask_refine function around -# for the time being. - # def mask_refine(self, design_file=None, allow_resync=False, debug=False): - # """ - # Use the mask design data to refine the edge trace positions. - # - # Use of this method requires: - # - a PCA decomposition is available, - # - the traces are synchronized into left-right pairs, and - # - :attr:`spectrograph` has a viable `get_slitmask` method - # to read slit mask design data from a file. That file is - # either provided directly or pulled from one of the - # files used to construct the trace image; see - # `design_file`. The result of the `get_slitmask` method - # must provide a - # :class:`pypeit.spectrographs.slitmask.SlitMask` object - # with the slit-mask design data. - # - # TODO: Traces don't need to be synchronized... - # - # Also useful, but not required, is for :attr:`spectrograph` to - # have a viable `get_detector_map` method that provides a - # :class:`pypeit.spectrograph.opticalmodel.DetectorMap` object, - # which is used to provide a guess offset between the slit-mask - # focal-plane positions and the trace pixel positions. If no - # such `get_detector_method` exists, the guess offset is:: - # - # this - # - # and the match between expected and traced slit positions may - # be unstable. - # - # The method uses - # :class:`pypeit.spectrographs.slitmask.SlitRegister` to match - # the expected and traced position and identify both missing - # and erroneous trace locations. The former are used to add new - # traces and the latter are removed. The method also constructs - # the :attr:`design` and :attr:`objects` tables, depending on - # the data accessible via the - # :class:`pypeit.spectrographs.slitmask.SlitMask` instance. - # - # Used parameters from :attr:`par` - # (:class:`pypeit.par.pypeitpar.EdgeTracePar`) are - # `left_right_pca`, `mask_reg_maxiter`, `mask_reg_maxsep`, - # `mask_reg_sigrej`, and `ignore_alignment`. - # - # Args: - # design_file (:obj:`str`, optional): - # A file with the mask design data. If None, the method - # will use the first file in :attr:`files`; if - # :attr:`files` is also None, the method will raise an - # exception. - # debug (:obj:`bool`, optional): - # Run in debug mode. - # """ - # # Still not done with this function... - # raise NotImplementedError() - # - # # Check that there are traces to refine! - # if self.is_empty: - # msgs.error('No traces to refine.') - # - # # The PCA decomposition must have already been determined - # if self.pcatype is None: - # msgs.error('Must first run the PCA analysis for the traces; run build_pca.') - # - # # Get the file to use when parsing the mask design information - # _design_file = (None if self.traceimg.files is None else self.traceimg.files[0]) \ - # if design_file is None else design_file - # if _design_file is None or not os.path.isfile(_design_file): - # msgs.error('Slit-mask design file not found or none provided.') - # - # # Get the paramters to use - # maxiter = self.par['mask_reg_maxiter'] - # maxsep = self.par['mask_reg_maxsep'] - # sigma = self.par['mask_reg_sigrej'] - # ignore_alignment = self.par['ignore_alignment'] - # - # # TODO: Set allow_resync and design_file to be a parameters, as - # # well? - # - # # Read the design data - # msgs.info('Reading slit-mask design information from: {0}'.format(_design_file)) - # if self.spectrograph.get_slitmask(_design_file) is None: - # msgs.error('Unable to read design file or no slit-mask design reader ' - # 'defined for {0}.'.format(self.spectrograph.spectrograph)) - # - # # Match both left and right edges simultaneously - # x_design = np.array([self.spectrograph.slitmask.bottom[:, 0], - # self.spectrograph.slitmask.top[:, 0]]).T.ravel() - # reference_row = self.left_pca.reference_row if self.par['left_right_pca'] \ - # else self.pca.reference_row - # x_det = self.edge_fit[reference_row, :] - # - # # Mask traces that are fully masked, except if they were - # # specifically inserted in a previous step - # # TODO: Should the BOXSLITS also be included here? - # x_det_bpm = self.fully_masked_traces(flag=self.bitmask.bad_flags, - # exclude=self.bitmask.insert_flags) - # - # # x_design = np.amin(self.spectrograph.slitmask.corners[:,:,0], axis=1) - # # side = self.traceid < 0 - # # x_det = self.edge_fit[self.pca.reference_row,side] - # - # # x_design = np.amax(self.spectrograph.slitmask.corners[:,:,0], axis=1) - # # side = self.traceid > 0 - # # x_det = self.edge_fit[self.pca.reference_row,side] - # - # # Estimate the scale in pixels/mm as the telescope platescale - # # in arcsec/mm divided by the detector platescale in - # # arcsec/pixel - # pix_per_mm = self.spectrograph.telescope.platescale() \ - # / self.traceimg.detector['platescale'] - # # / self.spectrograph.detector[self.det - 1]['platescale'] - # - # # If the traces are synchronized, use the estimated scale to - # # first mask edges that yeild slits that are too small relative - # # to the range of slit lengths in the mask file. - # if self.is_synced: - # slit_len_det = np.diff(x_det.reshape(-1, 2), axis=1).ravel() - # slit_len_mask = np.diff(x_design.reshape(-1, 2), axis=1).ravel() * pix_per_mm - # indx = (slit_len_det < np.amin(slit_len_mask) / 1.1) \ - # | (slit_len_det > np.amax(slit_len_mask) * 1.1) - # if np.any(indx): - # msgs.info('Removing {0} edges that form (an) '.format(np.sum(indx) * 2) - # + 'errantly small or large slit(s) compared to the mask design data.') - # x_det_bpm[np.repeat(indx, 2)] = True - # - # # Initial guess for the offset - # try: - # raise NotImplementedError() - # # Try using the spectrograph detector map - # self.spectrograph.get_detector_map() - # # Set the offset based on the location of this detector - # offset = self.spectrograph.detector_map.image_coordinates( - # self.spectrograph.detector_map.npix[0] / 2, - # self.spectrograph.detector_map.npix[1] / 2, - # detector=self.traceimg.detector.det, - # in_mm=False)[0][0] - self.spectrograph.detector_map.npix[0] / 2 - # # Set the bounds to some nominal fraction of the detector - # # size and pix/mm scale; allow for a +/- 10% deviation in - # # the pixel scale - # # TODO: Is 10% generally enough (for any instrument)? Make - # # this a (spectrograph-specific) parameter? - # offset_rng = [offset - 0.1 * self.spectrograph.detector_map.npix[0], - # offset + 0.1 * self.spectrograph.detector_map.npix[0]] - # except: - # # No detector map - # msgs.warn('No detector map available for {0}'.format(self.spectrograph.spectrograph) - # + '; attempting to match to slit-mask design anyway.') - # # Set the guess offset such that two sets of coordinates - # # are offset to their mean - # offset = np.mean(x_det) - np.mean(pix_per_mm * x_design) - # # Set the offset range - # offset_rng = [offset - np.absolute(np.amin(x_det) - np.amin(pix_per_mm * x_design)) * 1.1, - # offset + np.absolute(np.amax(pix_per_mm * x_design) - np.amax(x_det)) * 1.1] - # - # # import pdb - # # pdb.set_trace() - # # - # # slitmask.xc_trace(x_det, x_design, pix_per_mm) - # # - # # pdb.set_trace() - # - # # The solution can be highly dependent on the initial guess for - # # the offset, so do an initial grid search to get close to the - # # solution. - # msgs.info('Running a grid search to try to find the best starting offset.') - # # Step by 2 pixels - # off = np.arange(offset_rng[0], offset_rng[1], 2).astype(float) - # rms = np.zeros_like(off, dtype=float) - # scl = np.zeros_like(off, dtype=float) - # par = np.array([0, pix_per_mm]) - # bounds = np.array([offset_rng, [pix_per_mm / 1.1, pix_per_mm * 1.1]]) - # register = slitmask.SlitRegister(x_det, x_design, trace_mask=x_det_bpm) - # - # # NOTE: The commented approach below gets the RMS at each - # # offset point just using the estimated scale. This is faster - # # than the approach taken, but results are sensitive to the - # # accuracy of the estimated scale, which can lead to problems - # # in corner cases. - # # for i in range(off.size): - # # print('Grid point: {0}/{1}'.format(i+1, off.size), end='\r') - # # par[0] = off[i] - # # register.par = par - # # minsep = register.match(unique=True)[1] - # # rms[i] = sigma_clipped_stats(minsep, sigma=5)[2] - # # print('Grid point: {0}/{0}'.format(off.size)) - # - # # For each grid point, keep the offset fixed and find the best - # # scale. No rejection iterations are performed. - # for i in range(off.size): - # print('Grid point: {0}/{1}'.format(i + 1, off.size), end='\r') - # par[0] = off[i] - # register.find_best_match(guess=par, fix=[True, False], bounds=bounds, penalty=False) - # minsep = register.match(unique=True)[1] - # scl[i] = register.par[1] - # rms[i] = sigma_clipped_stats(minsep, sigma=5)[2] - # print('Grid point: {0}/{0}'.format(off.size)) - # - # # Use the grid point with the best RMS - # minindx = np.argmin(rms) - # offset = off[minindx] - # best_rms = rms[minindx] - # msgs.info('Minimum RMS ({0:.2f}) found with offset = {1:.2f}'.format(best_rms, offset)) - # if debug: - # # Plot the result - # ax1 = plt.subplot(211) - # ax1.scatter(off, rms, color='k', marker='.', s=100, lw=0, zorder=0) - # ax1.scatter(offset, best_rms, color='C3', marker='x', s=50, zorder=1) - # ax1.set_xlabel('Trace Offset (pix)') - # ax1.set_ylabel('RMS (det-mask; pix)') - # ax1.set_title('Grid search for initial offset') - # ax2 = plt.subplot(212, sharex=ax1) - # ax2.scatter(off, scl, color='k', marker='.', s=100, lw=0, zorder=0) - # ax2.set_ylabel('Best-fit scale') - # plt.show() - # - # # Do the final fit with some rejection iterations - # register.find_best_match(guess=[offset, pix_per_mm], bounds=bounds, penalty=False, - # maxiter=maxiter, maxsep=maxsep, sigma=sigma, debug=debug) - # - # if debug: - # register.show(minmax=[0, self.nspat], synced=True) - # - # # Find the missing, bad, and masked traces - # missing, bad = register.trace_mismatch(minmax=[0, self.nspat], synced=True) - # # masked_by_registration = np.where(register.trace_mask & np.invert(x_det_bpm))[0] - # # bad = np.append(bad, masked_by_registration) - # bad = np.append(bad, np.where(register.trace_mask | x_det_bpm)[0]) - # - # # Ignore missing alignment boxes - # if ignore_alignment: - # missing = missing[np.invert(self.spectrograph.slitmask.alignment_slit[missing // 2])] - # found_alignment_slits = register.match_index[ - # self.spectrograph.slitmask.alignment_slit[register.match_index // 2]] - # bad = np.append(bad, found_alignment_slits) - # - # # Report - # msgs.info('Best-fitting offset and scale for mask coordinates: {0:.2f} {1:.2f}'.format( - # *register.par)) - # msgs.info('Traces will {0} alignment slits'.format('exclude' if ignore_alignment - # else 'include')) - # msgs.info('Number of missing mask traces to insert: {0}'.format(len(missing))) - # msgs.info('Number of bad or alignment traces to remove: {0}'.format(len(bad))) - # - # if self.is_synced and (len(missing) - len(bad)) % 2 != 0: - # if allow_resync: - # msgs.warning('Difference in added and removed traces is odd; will resync traces.') - # else: - # msgs.error('Difference in added and removed traces desyncronizes traces.') - # - # if len(bad) > 0: - # # Remove the bad traces and rebuild the pca - # rmtrace = np.zeros(self.ntrace, dtype=bool) - # rmtrace[bad] = True - # self.remove_traces(rmtrace, rebuild_pca=True) - # - # if len(missing) > 0: - # # Even indices are lefts, odd indices are rights - # side = missing % 2 * 2 - 1 - # # Predict the traces using the PCA - # missing_traces = self.predict_traces(register.match_coo[missing], side) - # # Insert them - # self.insert_traces(side, missing_traces, mode='mask') - # - # # import pdb - # # pdb.set_trace() - # - # if len(bad) > 0 or len(missing) > 0: - # # Traces were removed and/or inserted, resync or recheck that the edges are synced. - # if (len(missing) - len(bad)) % 2 != 0 and allow_resync: - # self.sync(rebuild_pca=True) - # else: - # self.check_synced(rebuild_pca=True) - # reference_row = self.left_pca.reference_row if self.par['left_right_pca'] \ - # else self.pca.reference_row - # # Reset the match after removing/inserting traces - # x_det = self.edge_fit[reference_row, :] - # # TODO: Should the BOXSLITS also be included here? - # x_det_bpm = self.fully_masked_traces(flag=self.bitmask.bad_flags, - # exclude=self.bitmask.insert_flags) - # register = slitmask.SlitRegister(x_det, x_design, trace_mask=x_det_bpm, - # guess=[offset, pix_per_mm], bounds=bounds, - # penalty=False, maxiter=maxiter, maxsep=maxsep, - # sigma=sigma, debug=debug, fit=True) - # - # # TODO: This fit should *never* result in missing or bad - # # traces! Keep this for a while until we feel like we've - # # vetted the code well enough. - # missing, bad = register.trace_mismatch(minmax=[0, self.nspat], synced=True) - # if len(missing) != 0 or len(bad) != 0: - # msgs.error('CODING ERROR: Should never find missing or bad traces in re-fit!') - # - # # Fill the slit-design and object tables - # self._fill_design_table(register, _design_file) - # self._fill_objects_table(register) - def order_refine(self, debug=False): """ For echelle spectrographs, attempt to add any orders that are not @@ -5095,12 +4849,12 @@ def order_refine(self, debug=False): # Update the PCA self.build_pca() + reference_row = self.left_pca.reference_row if self.par['left_right_pca'] \ + else self.pca.reference_row if self.spectrograph.ech_fixed_format: - add_left, add_right = self.order_refine_fixed_format(debug=debug) + add_left, add_right = self.order_refine_fixed_format(reference_row, debug=debug) rmtraces = None else: - reference_row = self.left_pca.reference_row if self.par['left_right_pca'] \ - else self.pca.reference_row add_left, add_right, rmtraces \ = self.order_refine_free_format(reference_row, debug=debug) @@ -5121,9 +4875,9 @@ def order_refine(self, debug=False): # If fixed-format, rematch the orders if self.spectrograph.ech_fixed_format: - self.match_order() + self.match_order(reference_row=reference_row) - def order_refine_fixed_format(self, debug=False): + def order_refine_fixed_format(self, reference_row, debug=False): """ Refine the order locations for fixed-format Echelles. """ @@ -5139,7 +4893,7 @@ def order_refine_fixed_format(self, debug=False): # This needs to be checked. # First match the expected orders - spat_offset = self.match_order() + spat_offset = self.match_order(reference_row=reference_row) available_orders = self.orderid[1::2] missed_orders = np.setdiff1d(self.spectrograph.orders, available_orders) @@ -5379,16 +5133,15 @@ def slit_spatial_center(self, normalized=True, spec=None, use_center=False, Return coordinates normalized by the size of the detector. spec (:obj:`int`, optional): - Spectral position (row) at which to return the - spatial position. If ``None``, set at the central row - (i.e., ``self.nspat//2``) + Spectral position (row) at which to return the spatial position. + If ``None``, use the PCA reference row if a PCA exists or the + central row (i.e., ``self.nspec//2``), otherwise. use_center (:obj:`bool`, optional): Use the measured centroids to define the slit edges even if the slit edges have been otherwise modeled. include_box (:obj:`bool`, optional): Include box slits in the calculated coordinates. - Returns: `numpy.ma.MaskedArray`_: Spatial coordinates of the slit centers in pixels or in fractions of the detector. Masked @@ -5401,9 +5154,14 @@ def slit_spatial_center(self, normalized=True, spec=None, use_center=False, gpm = self.good_traces(include_box=include_box) good_slit = np.all(gpm.reshape(-1,2), axis=1) - # TODO: Use reference_row by default? Except that it's only - # defined if the PCA is defined. - _spec = self.nspec//2 if spec is None else spec + # Set the spectral position to use as a reference. + _spec = spec + if _spec is None: + if self.pcatype is None: + _spec = self.nspec//2 + else: + _spec = self.left_pca.reference_row if self.par['left_right_pca'] \ + else self.pca.reference_row # Synced, spatially sorted traces are always ordered in left, # right pairs @@ -5414,52 +5172,54 @@ def slit_spatial_center(self, normalized=True, spec=None, use_center=False, slit_c[good_slit] = np.mean(trace_cen[_spec,:].reshape(-1,2), axis=1) return slit_c/self.nspat if normalized else slit_c - def match_order(self): + def match_order(self, reference_row=None): """ Match synchronized slits to the expected echelle orders. This function will fault if called for non-Echelle spectrographs! - For Echelle spectrographs, this finds the best matching order - for each left-right trace pair; the traces must have already - been synchronized into left-right pairs. Currently, this is a - very simple, non-optimized match: - - - The closest order from - ``self.spectrograph.order_spat_pos`` is matched to each - slit. - - Any slit that is not matched to an order (only if the - number of slits is larger than the number of expected - orders) is flagged as ``NOORDER``. - - If multiple slits are matched to the same order, the - slit with the smallest on-detector separation is kept - and the other match is ignored. - - Any slit matched to an order with a separation above - the provided tolerance is flagged as ORDERMISMATCH. - - A warning is issued if the number of valid matches - is not identical to the number of expected orders - (``self.spectrograph.norders``). The warning includes - the list of orders that were not identified. - - The match tolerance is et by the parameter ``order_match``. - An offset can be applied to improve the match via the - parameter ``order_offset``; i.e., this should minimize the - difference between the expected order positions and - ``self.slit_spatial_center() + self.par['order_offset']``. - Both ``order_match`` and ``order_offset`` are given in - fractions of the detector size along the spatial axis. + For Echelle spectrographs, this finds the best matching order for each + left-right trace pair; the traces must have already been synchronized + into left-right pairs. Currently, this is a very simple, non-optimized + match: + + - The closest order from ``self.spectrograph.order_spat_pos`` is + matched to each slit. + - Any slit that cannot be matched to an order -- either because + there are more "slits" than orders or because the separation is + larger than the provided tolerance -- is flagged as + ``ORDERMISMATCH``. + - A warning is issued if the number of valid matches is not + identical to the number of expected orders + (``self.spectrograph.norders``). The warning includes the list of + orders that were not identified. + + The match tolerance is set by the parameter ``order_match``. An offset + can be applied to improve the match via the parameter ``order_offset``; + i.e., this should minimize the difference between the expected order + positions and ``self.slit_spatial_center() + self.par['order_offset']``. + Both ``order_match`` and ``order_offset`` are given in fractions of the + detector size along the spatial axis. The result of this method is to instantiate :attr:`orderid`. + Args: + reference_row (:obj:`int`, optional): + The spectral pixel (row) used to generate spatial positions of + the orders to match against the expected positions. If + ``None``, use the PCA reference row if a PCA exists or the + central row (i.e., ``self.nspec//2``), otherwise. + Returns: - :obj:`float`: The median offset in pixels between the archived order - positions and those measured via the edge tracing. + :obj:`float`: The median offset in the relative of the detector size + between the archived order positions and those measured via the edge + tracing. Raises: PypeItError: - Raised if the number of orders or their spatial - locations are not defined for an Echelle - spectrograph. + Raised if the number of orders, the order number, or their + spatial locations are not defined for an Echelle spectrograph. + Also raised if the edge traces are not synced. """ if self.spectrograph.norders is None: @@ -5471,76 +5231,70 @@ def match_order(self): if self.spectrograph.order_spat_pos is None: msgs.error('Coding error: order_spat_pos not defined for {0}!'.format( self.spectrograph.__class__.__name__)) + if not self.is_synced: + msgs.error('EdgeTraceSet must be synced to match to orders.') offset = self.par['order_offset'] if offset is None: offset = 0.0 # Get the order centers in fractions of the detector width. This - # requires the slits to be synced! Masked elements in slit_cen are for - # bad slits. - slit_cen = self.slit_spatial_center() - - # Calculate the separation between the order and every - sep = self.spectrograph.order_spat_pos[:,None] - slit_cen[None,:] - offset - # Find the smallest offset for each order - slit_indx = np.ma.MaskedArray(np.ma.argmin(np.absolute(sep), axis=1)) + # requires the slits to be synced (checked above)! Masked elements in + # slit_cen are for bad slits or syncing. + slit_cen = self.slit_spatial_center(spec=reference_row) + offset + good_sync = np.logical_not(np.ma.getmaskarray(slit_cen)) + # "slit_indx" matches the "slit" index to the order number. I.e., + # "slit_indx" has one element per expected order, and the value at a + # given position is the index of the paired traces for the relevant + # order. + slit_indx = slitdesign_matching.match_positions_1D( + slit_cen.data[good_sync], # (Good) Measured positions + self.spectrograph.order_spat_pos, # Expected positions + tol=self.par['order_match']) # Matching tolerance + + # Boolean array selecting found orders + fnd = slit_indx > -1 + missed_orders = self.spectrograph.orders[np.logical_not(fnd)] + if not np.all(fnd): + msgs.warn(f'Did not find all orders! Missing orders: {missed_orders}') + + # Flag paired edges that were not matched to a known order + nomatch = np.setdiff1d(np.arange(np.sum(good_sync)), slit_indx[fnd]) + if nomatch.size > 0: + msgs.warn(f'Flagging {nomatch.size} trace pairs as not being matched to an order.') + # Create a vector that selects the appropriate traces. This + # *assumes* that the traces are left-right syncronized and the order + # has not changed between the order of the traces in the relevant + # array and how the centers of the synced traces are computed + # (slit_spatial_center). + flag = np.append(2*nomatch, 2*nomatch+1) + self.edge_msk[:,flag] = self.bitmask.turn_on(self.edge_msk[:,flag], 'ORDERMISMATCH') # Minimum separation between the order and its matching slit; - # keep the signed value for reporting, but used the absolute + # keep the signed value for reporting, but use the absolute # value of the difference for vetting below. - sep = sep[(np.arange(self.spectrograph.norders),slit_indx)] - med_offset = np.median(sep) - min_sep = np.absolute(sep - med_offset) + # NOTE: This includes indices for orders that were not found. This is + # largely for book-keeping purposes in the print statement below. + sep = self.spectrograph.order_spat_pos - slit_cen.data[good_sync][slit_indx] + med_offset = np.median(sep[fnd]) # Report msgs.info(f'Median offset is {med_offset:.3f}.') msgs.info('After offsetting, order-matching separations are:') - msgs.info(' {0:>6} {1:>4} {2:>6}'.format('ORDER', 'PAIR', 'SEP')) - msgs.info(' {0} {1} {2}'.format('-'*6, '-'*4, '-'*6)) + msgs.info(f' {"ORDER":>6} {"PAIR":>4} {"SEP":>6}') + msgs.info(f' {"-"*6} {"-"*4} {"-"*6}') for i in range(self.spectrograph.norders): - msgs.info(' {0:>6} {1:>4} {2:6.3f}'.format(self.spectrograph.orders[i], i+1, sep[i])) - msgs.info(' {0} {1} {2}'.format('-'*6, '-'*4, '-'*6)) - - # Single slit matched to multiple orders - uniq, cnts = np.unique(slit_indx.compressed(), return_counts=True) - for u in uniq[cnts > 1]: - # Find the unmasked and multiply-matched indices - indx = (slit_indx.data == u) & np.logical_not(np.ma.getmaskarray(slit_indx)) - # Keep the one with the smallest separation and mask the rest - slit_indx[np.setdiff1d(np.where(indx), [np.argmin(min_sep[indx])])] = np.ma.masked - - # Flag orders separated by more than the provided threshold - if self.par['order_match'] is not None: - indx = (min_sep > self.par['order_match']) \ - & np.logical_not(np.ma.getmaskarray(min_sep)) - if np.any(indx): - # Flag the associated traces - _indx = np.isin(np.absolute(self.traceid), (slit_indx[indx]).compressed()+1) - self.edge_msk[:,_indx] = self.bitmask.turn_on(self.edge_msk[:,_indx], - 'ORDERMISMATCH') - # Disassociate these orders from any slit - slit_indx[indx] = np.ma.masked - - # Unmatched slits - indx = np.logical_not(np.isin(np.arange(self.nslits), slit_indx.compressed())) - if np.any(indx): - # This works because the traceids are sorted and synced - indx = np.repeat(indx, 2) - self.edge_msk[:,indx] = self.bitmask.turn_on(self.edge_msk[:,indx], 'NOORDER') - - # Warning that there are missing orders - missed_order = np.ma.getmaskarray(slit_indx) - if np.any(missed_order): - msgs.warn('Did not find all orders! Missing orders: {0}'.format( - ', '.join(self.spectrograph.orders[missed_order].astype(str)))) + if fnd[i]: + msgs.info(f' {self.spectrograph.orders[i]:>6} {i+1:>4} {sep[i]-med_offset:6.3f}') + else: + msgs.info(f' {self.spectrograph.orders[i]:>6} {"N/A":>4} {"MISSED":>6}') + msgs.info(f' {"-"*6} {"-"*4} {"-"*6}') # Instantiate the order ID; 0 means the order is unassigned self.orderid = np.zeros(self.nslits*2, dtype=int) - found_orders = self.spectrograph.orders[np.logical_not(missed_order)] - nfound = len(found_orders) - indx = (2*slit_indx.compressed()[:,None] + np.tile(np.array([0,1]), (nfound,1))).ravel() - self.orderid[indx] = (np.array([-1,1])[None,:]*found_orders[:,None]).ravel() + raw_indx = np.arange(self.nslits)[good_sync][slit_indx[fnd]] + indx = (2*raw_indx[:,None] + np.tile(np.array([0,1]), (np.sum(fnd),1))).ravel() + self.orderid[indx] = (np.array([-1,1])[None,:]*self.spectrograph.orders[fnd,None]).ravel() return med_offset diff --git a/pypeit/extraction.py b/pypeit/extraction.py index c50f9d9eb3..461f82b1b9 100644 --- a/pypeit/extraction.py +++ b/pypeit/extraction.py @@ -164,8 +164,9 @@ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_ self.initialize_slits(slits) # Internal bpm mask - self.extract_bpm = (self.slits.mask > 0) & (np.logical_not(self.slits.bitmask.flagged( - self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing))) + self.extract_bpm = self.slits.bitmask.flagged( + self.slits.mask, + and_not=self.slits.bitmask.exclude_for_reducing) self.extract_bpm_init = self.extract_bpm.copy() # These may be None (i.e. COADD2D) @@ -879,7 +880,7 @@ def local_skysub_extract(self, global_sky, sobjs, return self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs -class IFUExtract(MultiSlitExtract): +class SlicerIFUExtract(MultiSlitExtract): """ Child of Extract for IFU reductions @@ -887,5 +888,8 @@ class IFUExtract(MultiSlitExtract): """ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, **kwargs): - # IFU doesn't extract, and there's no need for a super call here. - return + super().__init__(sciImg, slits, sobjs_obj, spectrograph, par, objtype, **kwargs) + + #def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, **kwargs): + # # IFU doesn't extract, and there's no need for a super call here. + # return diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index b93704438d..1ba353e92c 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -166,8 +166,19 @@ def __init__(self, sciImg, slits, spectrograph, par, objtype, wv_calib=None, wav # Internal bpm mask # We want to keep the 'BOXSLIT', which has bpm=2. But we don't want to keep 'BOXSLIT' # with other bad flag (for which bpm>2) - self.reduce_bpm = (self.slits.mask > 2) & (np.invert(self.slits.bitmask.flagged( - self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing))) + # TODO: To my mind, we should never be using the value of the bit to + # check for flags. We should be using the BitMask functions. I *think* + # what you want is this: + self.reduce_bpm = self.slits.bitmask.flagged( + self.slits.mask, + exclude='BOXSLIT', + and_not=self.slits.bitmask.exclude_for_reducing) + # I.e., mask anything *except* slits flagged by only 'BOXSLIT', and also + # make sure any of the `exclude_for_reducing` flags are not on. This + # previous code may also have included slits that were flagged as + # SHORTSLIT. Was that on purpose? +# self.reduce_bpm = (self.slits.mask > 2) & (np.invert(self.slits.bitmask.flagged( +# self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing))) self.reduce_bpm_init = self.reduce_bpm.copy() # Load up other input items @@ -482,7 +493,7 @@ def get_platescale(self, slitord_id=None): pass - def global_skysub(self, skymask=None, update_crmask=True, trim_edg = (0, 0), + def global_skysub(self, skymask=None, update_crmask=True, previous_sky=None, show_fit=False, show=False, show_objs=False, objs_not_masked=False, reinit_bpm:bool=True): @@ -496,9 +507,6 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg = (0, 0), A 2D image indicating sky regions (1=sky) update_crmask (bool, optional): Update the crmask in the science image - trim_edg (tuple, optional): - A two tuple of ints that specify the number of pixels to trim from the slit edges - Only used by the IFU child show_fit (bool, optional): Show the sky fits? show (bool, optional): @@ -537,10 +545,9 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg = (0, 0), sigrej = 3.0 # We use this tmp bpm so that we exclude the BOXSLITS during the global_skysub - tmp_bpm = (self.slits.mask > 0) & \ - (np.invert(self.slits.bitmask.flagged(self.slits.mask, - flag=self.slits.bitmask.exclude_for_reducing))) - gdslits = np.where(np.invert(tmp_bpm))[0] + tmp_bpm = self.slits.bitmask.flagged(self.slits.mask, + and_not=self.slits.bitmask.exclude_for_reducing) + gdslits = np.where(np.logical_not(tmp_bpm))[0] # Mask objects using the skymask? If skymask has been set by objfinding, and masking is requested, then do so skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) @@ -960,173 +967,81 @@ class SlicerIFUFindObjects(MultiSlitFindObjects): """ def __init__(self, sciImg, slits, spectrograph, par, objtype, **kwargs): super().__init__(sciImg, slits, spectrograph, par, objtype, **kwargs) - self.initialize_slits(slits, initial=True) - def find_objects_pypeline(self, image, ivar, std_trace=None, - show_peaks=False, show_fits=False, show_trace=False, - show=False, save_objfindQA=False, neg=False, debug=False, - manual_extract_dict=None): - """ - See MultiSlitReduce for SlicerIFU reductions + def initialize_slits(self, slits, initial=True): """ - if self.par['reduce']['cube']['slit_spec']: - return super().find_objects_pypeline(image, ivar, std_trace=std_trace, - show_peaks=show_peaks, show_fits=show_fits, show_trace=show_trace, - show=show, save_objfindQA=save_objfindQA, neg=neg, - debug=debug, manual_extract_dict=manual_extract_dict) - return None, None, None + Gather all the :class:`~pypeit.slittrace.SlitTraceSet` attributes that + we'll use here in :class:`FindObjects`. Identical to the parent but the + slits are not trimmed. - def apply_relative_scale(self, scaleImg): - """Apply a relative scale to the science frame (and correct the varframe, too) + Args: + slits (:class:`~pypeit.slittrace.SlitTraceSet`): + SlitTraceSet object containing the slit boundaries that will be + initialized. + initial (:obj:`bool`, optional): + Use the initial definition of the slits. If False, + tweaked slits are used. + """ + super().initialize_slits(slits, initial=True) - Args: - scaleImg (`numpy.ndarray`_): - scale image to divide the science frame by + def global_skysub(self, skymask=None, update_crmask=True, + previous_sky=None, show_fit=False, show=False, show_objs=False, objs_not_masked=False, + reinit_bpm: bool = True): """ - # Check that scaleimg is set to the correct shape - if self.scaleimg.size == 1: - self.scaleimg = np.ones_like(self.sciImg.image) - # Correct the relative illumination of the science frame - msgs.info("Correcting science frame for relative illumination") - self.scaleimg *= scaleImg.copy() - self.sciImg.image, _bpm, varImg = flat.flatfield(self.sciImg.image, scaleImg, - varframe=utils.inverse(self.sciImg.ivar)) - if np.any(_bpm): - self.sciImg.update_mask('BADSCALE', indx=_bpm) - self.sciImg.ivar = utils.inverse(varImg) + Perform global sky subtraction. This SlicerIFU-specific routine ensures that the + edges of the slits are not trimmed, and performs a spatial and spectral + correction using the sky spectrum, if requested. See Reduce.global_skysub() + for parameter definitions. - # RJC :: THIS FUNCTION IS NOT CURRENTLY USED, BUT RJC REQUESTS TO KEEP THIS CODE HERE FOR THE TIME BEING. - # def illum_profile_spatial(self, skymask=None, trim_edg=(0, 0), debug=False): - # """ - # Calculate the residual spatial illumination profile using the sky regions. - # - # The residual is calculated using the differential: - # - # .. code-block:: python - # - # correction = amplitude * (1 + spatial_shift * (dy/dx)/y) - # - # where ``y`` is the spatial profile determined from illumflat, and - # spatial_shift is the residual spatial flexure shift in units of pixels. - # - # Args: - # skymask (`numpy.ndarray`_): - # Mask of sky regions where the spatial illumination will be determined - # trim_edg (:obj:`tuple`): - # A tuple of two ints indicated how much of the slit edges should be - # trimmed when fitting to the spatial profile. - # debug (:obj:`bool`): - # Show debugging plots? - # """ - # - # msgs.info("Performing spatial sensitivity correction") - # # Setup some helpful parameters - # skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) - # hist_trim = 0 # Trim the edges of the histogram to take into account edge effects - # gpm = self.sciImg.select_flag(invert=True) - # slitid_img_init = self.slits.slit_img(pad=0, initial=True, flexure=self.spat_flexure_shift) - # spatScaleImg = np.ones_like(self.sciImg.image) - # # For each slit, grab the spatial coordinates and a spline - # # representation of the spatial profile from the illumflat - # rawimg = self.sciImg.image.copy() - # numbins = int(np.max(self.slits.get_slitlengths(initial=True, median=True))) - # spatbins = np.linspace(0.0, 1.0, numbins + 1) - # spat_slit = 0.5 * (spatbins[1:] + spatbins[:-1]) - # slitlength = np.median(self.slits.get_slitlengths(median=True)) - # coeff_fit = np.zeros((self.slits.nslits, 2)) - # for sl, slitnum in enumerate(self.slits.spat_id): - # msgs.info("Deriving spatial correction for slit {0:d}/{1:d}".format(sl + 1, self.slits.spat_id.size)) - # # Get the initial slit locations - # onslit_b_init = (slitid_img_init == slitnum) - # - # # Synthesize ximg, and edgmask from slit boundaries. Doing this outside this - # # routine would save time. But this is pretty fast, so we just do it here to make the interface simpler. - # spatcoord, edgmask = pixels.ximg_and_edgemask(self.slits_left[:, sl], self.slits_right[:, sl], - # onslit_b_init, trim_edg=trim_edg) - # - # # Make the model histogram - # xspl = np.linspace(0.0, 1.0, 10 * int(slitlength)) # Sub sample each pixel with 10 subpixels - # # TODO: caliBrate is no longer a dependency. If you need these b-splines pass them in. - # modspl = self.caliBrate.flatimages.illumflat_spat_bsplines[sl].value(xspl)[0] - # gradspl = interpolate.interp1d(xspl, np.gradient(modspl) / modspl, kind='linear', bounds_error=False, - # fill_value='extrapolate') - # - # # Ignore skymask - # coord_msk = onslit_b_init & gpm - # hist, _ = np.histogram(spatcoord[coord_msk], bins=spatbins, weights=rawimg[coord_msk]) - # cntr, _ = np.histogram(spatcoord[coord_msk], bins=spatbins) - # hist_slit_all = hist / (cntr + (cntr == 0)) - # histmod, _ = np.histogram(spatcoord[coord_msk], bins=spatbins, weights=gradspl(spatcoord[coord_msk])) - # hist_model = histmod / (cntr + (cntr == 0)) - # - # # Repeat with skymask - # coord_msk = onslit_b_init & gpm & skymask_now - # hist, _ = np.histogram(spatcoord[coord_msk], bins=spatbins, weights=rawimg[coord_msk]) - # cntr, _ = np.histogram(spatcoord[coord_msk], bins=spatbins) - # hist_slit = hist / (cntr + (cntr == 0)) - # - # # Prepare for fit - take the non-zero elements and trim slit edges - # if hist_trim == 0: - # ww = (hist_slit != 0) - # xfit = spat_slit[ww] - # yfit = hist_slit_all[ww] - # mfit = hist_model[ww] - # else: - # ww = (hist_slit[hist_trim:-hist_trim] != 0) - # xfit = spat_slit[hist_trim:-hist_trim][ww] - # yfit = hist_slit_all[hist_trim:-hist_trim][ww] - # mfit = hist_model[hist_trim:-hist_trim][ww] - # - # # Fit the function - # spat_func = lambda par, ydata, model: par[0]*(1 + par[1] * model) - ydata - # res_lsq = least_squares(spat_func, [np.median(yfit), 0.0], args=(yfit, mfit)) - # spatnorm = spat_func(res_lsq.x, 0.0, gradspl(spatcoord[onslit_b_init])) - # spatnorm /= spat_func(res_lsq.x, 0.0, gradspl(0.5)) - # # Set the scaling factor - # spatScaleImg[onslit_b_init] = spatnorm - # coeff_fit[sl, :] = res_lsq.x - # - # if debug: - # from matplotlib import pyplot as plt - # xplt = np.arange(24) - # plt.subplot(121) - # plt.plot(xplt[0::2], coeff_fit[::2, 0], 'rx') - # plt.plot(xplt[1::2], coeff_fit[1::2, 0], 'bx') - # plt.subplot(122) - # plt.plot(xplt[0::2], coeff_fit[::2, 1]/10, 'rx') - # plt.plot(xplt[1::2], coeff_fit[1::2, 1]/10, 'bx') - # plt.show() - # plt.imshow(spatScaleImg, vmin=0.99, vmax=1.01) - # plt.show() - # plt.subplot(133) - # plt.plot(xplt[0::2], coeff_fit[::2, 2], 'rx') - # plt.plot(xplt[1::2], coeff_fit[1::2, 2], 'bx') - # plt.show() - # # Apply the relative scale correction - # self.apply_relative_scale(spatScaleImg) - - def illum_profile_spectral(self, global_sky, skymask=None): - """Calculate the residual spectral illumination profile using the sky regions. - This uses the same routine as the flatfield spectral illumination profile. + See base class method for description of parameters. - Args: - global_sky (`numpy.ndarray`_): - Model of the sky - skymask (`numpy.ndarray`_, optional): - Mask of sky regions where the spectral illumination will be determined + Args: + reinit_bpm (:obj:`bool`, optional): + If True (default), the bpm is reinitialized to the initial bpm + Should be False on the final run in case there was a failure + upstream and no sources were found in the slit/order """ - trim = self.par['calibrations']['flatfield']['slit_trim'] - sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] - smooth_npix = self.par['calibrations']['flatfield']['slit_illum_smooth_npix'] - gpm = self.sciImg.select_flag(invert=True) - # Note :: Need to provide wavelength to illum_profile_spectral (not the tilts) so that the - # relative spectral sensitivity is calculated at a given wavelength for all slits simultaneously. - scaleImg = flatfield.illum_profile_spectral(self.sciImg.image.copy(), self.waveimg, self.slits, - slit_illum_ref_idx=sl_ref, model=global_sky, gpmask=gpm, - skymask=skymask, trim=trim, flexure=self.spat_flexure_shift, - smooth_npix=smooth_npix) - # Now apply the correction to the science frame - self.apply_relative_scale(scaleImg) + + global_sky_sep = super().global_skysub(skymask=skymask, update_crmask=update_crmask, + previous_sky=previous_sky, show_fit=show_fit, show=show, + show_objs=show_objs, + objs_not_masked=objs_not_masked, reinit_bpm=reinit_bpm) + + # Check if flexure or a joint fit is requested. If not return this + if not self.par['reduce']['skysub']['joint_fit'] and self.par['flexure']['spec_method'] == 'skip': + return global_sky_sep + + if self.wv_calib is None: + msgs.error("A wavelength calibration is needed (wv_calib) if a joint sky fit is requested.") + msgs.info("Generating wavelength image") + + # Generate the waveimg which is needed if flexure is being computed + self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spat_flexure=self.spat_flexure_shift) + + # Calculate spectral flexure, so that we can align all slices of the IFU. We need to do this on the model + # sky spectrum. It must be performed in this class if the joint sky fit is requested, because all of + # the wavelengths need to be aligned for different slits before the sky is fit. + method = self.par['flexure']['spec_method'] + # TODO :: Perhaps include a new label for IFU flexure correction - e.g. 'slitcen_relative' or 'slitcenIFU' or 'IFU' + # :: If a new label is introduced, change the other instances of 'method' (see below), and in flexure.spec_flexure_qa() + if method in ['slitcen']: + self.slitshift = self.calculate_flexure(global_sky_sep) + # Recalculate the wavelength image, and the global sky taking into account the spectral flexure + msgs.info("Generating wavelength image, accounting for spectral flexure") + self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spec_flexure=self.slitshift, + spat_flexure=self.spat_flexure_shift) + + # If the joint fit or spec/spat sensitivity corrections are not being performed, return the separate slits sky + if not self.par['reduce']['skysub']['joint_fit']: + return global_sky_sep + + # If we reach this point in the code, a joint skysub has been requested. + # Use sky information in all slits to perform a joint sky fit + global_sky = self.joint_skysub(skymask=skymask, update_crmask=update_crmask, + show_fit=show_fit, show=show, show_objs=show_objs, + objs_not_masked=objs_not_masked) + + return global_sky def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), show_fit=False, show=False, show_objs=False, adderr=0.01, objs_not_masked=False): @@ -1246,77 +1161,25 @@ def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), self.show('global', global_sky=_global_sky, slits=True, sobjs=sobjs_show, clear=False) return _global_sky - def global_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), - previous_sky=None, show_fit=False, show=False, show_objs=False, objs_not_masked=False, - reinit_bpm:bool=True): - """ - Perform global sky subtraction. This SlicerIFU-specific routine ensures that the - edges of the slits are not trimmed, and performs a spatial and spectral - correction using the sky spectrum, if requested. See Reduce.global_skysub() - for parameter definitions. - - See base class method for description of parameters. - - Args: - reinit_bpm (:obj:`bool`, optional): - If True (default), the bpm is reinitialized to the initial bpm - Should be False on the final run in case there was a failure - upstream and no sources were found in the slit/order - """ - if self.par['reduce']['findobj']['skip_skysub']: - msgs.info("Skipping global sky sub as per user request") - return np.zeros_like(self.sciImg.image) - - # Generate a global sky sub for all slits separately - global_sky_sep = super().global_skysub(skymask=skymask, update_crmask=update_crmask, - trim_edg=trim_edg, show_fit=show_fit, show=show, - show_objs=show_objs, reinit_bpm=reinit_bpm) - # Check if any slits failed - if np.any(global_sky_sep[self.slitmask >= 0] == 0) and not self.bkg_redux: - # Cannot continue without a sky model for all slits - msgs.error("Global sky subtraction has failed for at least one slit.") - - # Check if flexure or a joint fit is requested - if not self.par['reduce']['skysub']['joint_fit'] and self.par['flexure']['spec_method'] == 'skip': - return global_sky_sep - if self.wv_calib is None: - msgs.error("A wavelength calibration is needed (wv_calib) if a joint sky fit is requested.") - msgs.info("Generating wavelength image") - - self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spat_flexure=self.spat_flexure_shift) - # Calculate spectral flexure - method = self.par['flexure']['spec_method'] - # TODO :: Perhaps include a new label for IFU flexure correction - e.g. 'slitcen_relative' or 'slitcenIFU' or 'IFU' - # :: If a new label is introduced, change the other instances of 'method' (see below), and in flexure.spec_flexure_qa() - if method in ['slitcen']: - self.calculate_flexure(global_sky_sep) - - # If the joint fit or spec/spat sensitivity corrections are not being performed, return the separate slits sky - if not self.par['reduce']['skysub']['joint_fit']: - return global_sky_sep - - # Do the spatial scaling first - # if self.par['scienceframe']['process']['use_illumflat']: - # # Perform the correction - # self.illum_profile_spatial(skymask=skymask) - # # Re-generate a global sky sub for all slits separately - # global_sky_sep = Reduce.global_skysub(self, skymask=skymask, update_crmask=update_crmask, trim_edg=trim_edg, - # show_fit=show_fit, show=show, show_objs=show_objs) - - # Use sky information in all slits to perform a joint sky fit - global_sky = self.joint_skysub(skymask=skymask, update_crmask=update_crmask, trim_edg=trim_edg, - show_fit=show_fit, show=show, show_objs=show_objs, - objs_not_masked=objs_not_masked) - - return global_sky - + # TODO :: This function should be removed from the find_objects() class, once the flexure code has been tidied up. def calculate_flexure(self, global_sky): """ - Convenience function to calculate the flexure of the IFU + Convenience function to calculate the flexure of an IFU. The flexure is calculated by cross-correlating the + sky model of a reference slit with an archival sky spectrum. This gives an "absolute" flexure correction for + the reference slit in pixels. Then, the flexure for all other slits is calculated by cross-correlating the + sky model of each slit with the sky model of the reference slit. This gives a "relative" flexure correction + for each slit in pixels. The relative flexure is then added to the absolute flexure to give the total flexure + correction for each slit in pixels. - Args: - global_sky (`numpy.ndarray`_): - Model of the sky + Parameters + ---------- + global_sky : ndarray + Sky model + + Returns + ------- + new_slitshift: ndarray + The flexure in pixels """ sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] box_rad = self.par['reduce']['extraction']['boxcar_radius'] @@ -1363,22 +1226,37 @@ def calculate_flexure(self, global_sky): # Replace the reference slit with the absolute shift flex_list[sl_ref] = flex_dict_ref.copy() # Add this flexure to the previous flexure correction - self.slitshift += this_slitshift + new_slitshift = self.slitshift + this_slitshift # Now report the flexure values for slit_idx, slit_spat in enumerate(self.slits.spat_id): msgs.info("Flexure correction, slit {0:d} (spat id={1:d}): {2:.3f} pixels".format(1+slit_idx, slit_spat, self.slitshift[slit_idx])) # Save QA - # TODO :: Need to implement QA + # TODO :: Need to implement QA once the flexure code has been tidied up, and this routine has been moved + # out of the find_objects() class. msgs.work("QA is not currently implemented for the flexure correction") if False:#flex_list is not None: basename = f'{self.basename}_global_{self.spectrograph.get_det_name(self.det)}' out_dir = os.path.join(self.par['rdx']['redux_path'], 'QA') slit_bpm = np.zeros(self.slits.nslits, dtype=bool) flexure.spec_flexure_qa(self.slits.slitord_id, slit_bpm, basename, flex_list, out_dir=out_dir) + return new_slitshift - # Recalculate the wavelength image, and the global sky taking into account the spectral flexure - msgs.info("Generating wavelength image, accounting for spectral flexure") - self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spec_flexure=self.slitshift, - spat_flexure=self.spat_flexure_shift) - return + def apply_relative_scale(self, scaleImg): + """Apply a relative scale to the science frame (and correct the varframe, too) + + Args: + scaleImg (`numpy.ndarray`_): + scale image to divide the science frame by + """ + # Check that scaleimg is set to the correct shape + if self.scaleimg.size == 1: + self.scaleimg = np.ones_like(self.sciImg.image) + # Correct the relative illumination of the science frame + msgs.info("Correcting science frame for relative illumination") + self.scaleimg *= scaleImg.copy() + self.sciImg.image, _bpm, varImg = flat.flatfield(self.sciImg.image, scaleImg, + varframe=utils.inverse(self.sciImg.ivar)) + if np.any(_bpm): + self.sciImg.update_mask('BADSCALE', indx=_bpm) + self.sciImg.ivar = utils.inverse(varImg) diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index ea13b2c3c3..e73f8462ae 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -8,7 +8,7 @@ import inspect import numpy as np -from scipy import interpolate +from scipy import interpolate, ndimage from matplotlib import pyplot as plt from matplotlib import gridspec @@ -16,7 +16,7 @@ from IPython import embed from pypeit import msgs -from pypeit.pypmsgs import PypeItError +from pypeit.pypmsgs import PypeItDataModelError from pypeit import utils from pypeit import bspline @@ -28,7 +28,6 @@ from pypeit.core import tracewave from pypeit.core import basis from pypeit.core import fitting -from pypeit.core import coadd from pypeit import slittrace @@ -389,7 +388,7 @@ def fit2illumflat(self, slits, frametype='illum', finecorr=False, initial=False, # TODO -- Update the internal one? Or remove it altogether?? return illumflat - def show(self, frametype='all', slits=None, wcs_match=True): + def show(self, frametype='all', slits=None, wcs_match=True, chk_version=True): """ Simple wrapper to :func:`show_flats`. @@ -403,6 +402,11 @@ def show(self, frametype='all', slits=None, wcs_match=True): wcs_match (:obj:`bool`, optional): (Attempt to) Match the WCS coordinates of the output images in the `ginga`_ viewer. + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict + version checking to ensure a valid file. If False, the code + will try to keep going, but this may lead to faults and quiet + failures. User beware! """ illumflat_pixel, illumflat_illum = None, None pixelflat_finecorr, illumflat_finecorr = None, None @@ -414,8 +418,8 @@ def show(self, frametype='all', slits=None, wcs_match=True): slits_file = slittrace.SlitTraceSet.construct_file_name(self.calib_key, calib_dir=self.calib_dir) try: - slits = slittrace.SlitTraceSet.from_file(slits_file) - except (FileNotFoundError, PypeItError): + slits = slittrace.SlitTraceSet.from_file(slits_file, chk_version=chk_version) + except (FileNotFoundError, PypeItDataModelError): msgs.warn('Could not load slits to include when showing flat-field images. File ' 'was either not provided directly, or it could not be read based on its ' f'expected name: {slits_file}.') @@ -1152,8 +1156,11 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): # Perform a fine correction to the spatial illumination profile spat_illum_fine = 1 # Default value if the fine correction is not performed if exit_status <= 1 and self.flatpar['slit_illum_finecorr']: - spat_illum = spat_bspl.value(spat_coo_final[onslit_tweak])[0] - self.spatial_fit_finecorr(spat_illum, onslit_tweak, slit_idx, slit_spat, gpm, doqa=doqa) + spat_model = np.ones_like(spec_model) + spat_model[onslit_padded] = spat_bspl.value(spat_coo_final[onslit_padded])[0] + specspat_illum = np.fmax(spec_model, 1.0) * spat_model + norm_spatspec = rawflat / specspat_illum + self.spatial_fit_finecorr(norm_spatspec, onslit_tweak, slit_idx, slit_spat, gpm, doqa=doqa) # ---------------------------------------------------------- # Construct the illumination profile with the tweaked edges @@ -1368,7 +1375,7 @@ def spatial_fit(self, norm_spec, spat_coo, median_slit_width, spat_gpm, gpm, deb return exit_status, spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, \ spat_flat_fit, spat_flat_data_raw - def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gpm, slit_trim=3, doqa=False): + def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, slit_trim=3, doqa=False): """ Generate a relative scaling image for a slicer IFU. All slits are scaled relative to a reference slit, specified in @@ -1376,8 +1383,8 @@ def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gp Parameters ---------- - spat_illum : `numpy.ndarray`_ - An image containing the generated spatial illumination profile for all slits. + normed : `numpy.ndarray`_ + Raw flat field image, normalized by the spectral and spatial illuminations. onslit_tweak : `numpy.ndarray`_ mask indicating which pixels are on the slit (True = on slit) slit_idx : int @@ -1406,10 +1413,6 @@ def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gp onslit_tweak_trim = self.slits.slit_img(pad=-slit_trim, slitidx=slit_idx, initial=False) == slit_spat # Setup slitimg = (slit_spat + 1) * onslit_tweak.astype(int) - 1 # Need to +1 and -1 so that slitimg=-1 when off the slit - normed = self.rawflatimg.image.copy() - ivarnrm = self.rawflatimg.ivar.copy() - normed[onslit_tweak] *= utils.inverse(spat_illum) - ivarnrm[onslit_tweak] *= spat_illum**2 left, right, msk = self.slits.select_edges(initial=True, flexure=self.wavetilts.spat_flexure) this_left = left[:, slit_idx] this_right = right[:, slit_idx] @@ -1432,18 +1435,6 @@ def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gp ypos = (this_wave - wave_min) / (wave_max - wave_min) # Need to use the same wave_min and wave_max as the fitting coordinates xpos = xpos_img[this_slit] - # Normalise the image - delta = 0.5/self.slits.nspec # include the endpoints - bins = np.linspace(0.0-delta, 1.0+delta, self.slits.nspec+1) - censpec, _ = np.histogram(ypos_fit, bins=bins, weights=normed[this_slit_trim]) - nrm, _ = np.histogram(ypos_fit, bins=bins) - censpec *= utils.inverse(nrm) - tiltspl = interpolate.interp1d(0.5*(bins[1:]+bins[:-1]), censpec, kind='linear', - bounds_error=False, fill_value='extrapolate') - nrm_vals = tiltspl(ypos_fit) - normed[this_slit_trim] *= utils.inverse(nrm_vals) - ivarnrm[this_slit_trim] *= nrm_vals**2 - # Mask the edges and fit gpmfit = gpm[this_slit_trim] # Trim by 5% of the slit length, or at least slit_trim pixels @@ -1572,7 +1563,7 @@ def spectral_illumination(self, gpm=None, debug=False): msgs.info('Performing a joint fit to the flat-field response') # Grab some parameters trim = self.flatpar['slit_trim'] - rawflat = self.rawflatimg.image.copy() / self.msillumflat.copy() + rawflat = self.rawflatimg.image / (self.msillumflat * self.mspixelflat) # Grab the GPM and the slit images if gpm is None: # TODO: Should this be *any* flag, or just BPM? @@ -1628,6 +1619,9 @@ def spatillum_finecorr_qa(normed, finecorr, left, right, ypos, cut, outfile=None fcor_cut = finecorr[xmn:xmx, ymn:ymx] vmin, vmax = max(0.95, np.min(fcor_cut)), min(1.05, np.max(fcor_cut)) # Show maximum corrections of ~5% + # For display/visual purposes, apply a median filter to the data + norm_cut = ndimage.median_filter(norm_cut, size=(normed.shape[0]//100, 5)) + # Plot fighght = 8.5 cutrat = fighght*norm_cut.shape[1]/norm_cut.shape[0] @@ -1747,7 +1741,7 @@ def detector_structure_qa(det_resp, det_resp_model, outfile=None, title="Detecto # Axes showing the residual of the detector response fit ax_resd = plt.subplot(gs[2]) ax_resd.imshow(det_resp-det_resp_model, origin='lower', vmin=vmin-1, vmax=vmax-1) - ax_resd.set_xlabel("data-model", fontsize='medium') + ax_resd.set_xlabel("1+data-model", fontsize='medium') ax_resd.axes.xaxis.set_ticks([]) ax_resd.axes.yaxis.set_ticks([]) # Add a colorbar @@ -1770,19 +1764,18 @@ def detector_structure_qa(det_resp, det_resp_model, outfile=None, title="Detecto def show_flats(image_list, wcs_match=True, slits=None, waveimg=None): """ - Interface to ginga to show a set of flat images + Show the flat-field images - Args: - pixelflat (`numpy.ndarray`_): - illumflat (`numpy.ndarray`_ or None): - procflat (`numpy.ndarray`_): - flat_model (`numpy.ndarray`_): - spec_illum (`numpy.ndarray`_ or None): - wcs_match (bool, optional): - slits (:class:`~pypeit.slittrace.SlitTraceSet`, optional): - waveimg (`numpy.ndarray`_ or None): - - Returns: + Parameters + ---------- + image_list : list + List of tuples containing the image data, image name and the cut levels + wcs_match : bool, optional + Match the WCS of the images + slits : :class:`pypeit.slittrace.SlitTraceSet`, optional + Slit traces to be overplotted on the images + waveimg : `numpy.ndarray`_, optional + Wavelength image to be overplotted on the images """ display.connect_to_ginga(raise_err=True, allow_new=True) @@ -1806,11 +1799,10 @@ def show_flats(image_list, wcs_match=True, slits=None, waveimg=None): clear = False -def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_npix=None, - model=None, gpmask=None, skymask=None, trim=3, flexure=None): +# TODO :: This could possibly be moved to core.flat +def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_npix=None, polydeg=None, + model=None, gpmask=None, skymask=None, trim=3, flexure=None, maxiter=5): """ - TODO :: This could possibly be moved to core.flat - Determine the relative spectral illumination of all slits. Currently only used for image slicer IFUs. @@ -1826,6 +1818,9 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ Index of slit that is used as the reference. smooth_npix : int, optional smoothing used for determining smoothly varying relative weights by sn_weights + polydeg : int, optional + Degree of polynomial to be used for determining relative spectral sensitivity. If None, + coadd.smooth_weights will be used, with the smoothing length set to smooth_npix. model : `numpy.ndarray`_, None A model of the rawimg data. If None, rawimg will be used. gpmask : `numpy.ndarray`_, None @@ -1837,6 +1832,8 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ when deriving the spectral illumination flexure : float, None Spatial flexure + maxiter : :obj:`int` + Maximum number of iterations to perform Returns ------- @@ -1844,6 +1841,10 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ An image containing the appropriate scaling """ msgs.info("Performing relative spectral sensitivity correction (reference slit = {0:d})".format(slit_illum_ref_idx)) + if polydeg is not None: + msgs.info("Using polynomial of degree {0:d} for relative spectral sensitivity".format(polydeg)) + else: + msgs.info("Using 'smooth_weights' algorithm for relative spectral sensitivity") # Setup some helpful parameters skymask_now = skymask if (skymask is not None) else np.ones_like(rawimg, dtype=bool) gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool) @@ -1879,37 +1880,54 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ sn_smooth_npix = wave_ref.size // smooth_npix if (smooth_npix is not None) else wave_ref.size // 10 # Iterate until convergence - maxiter = 5 lo_prev, hi_prev = 1.0E-32, 1.0E32 for rr in range(maxiter): - # Reset the relative scaling for this iteration - relscl_model = np.ones_like(rawimg) - # Build the relative illumination, by successively finding the slits closest in wavelength to the reference - for ss in range(slits.spat_id.size): - # Calculate the region of overlap - onslit_b = (slitid_img_trim == slits.spat_id[wvsrt[ss]]) - onslit_b_init = (slitid_img_init == slits.spat_id[wvsrt[ss]]) - onslit_b_olap = onslit_b & gpm & (waveimg >= mnmx_wv[wvsrt[ss], 0]) & (waveimg <= mnmx_wv[wvsrt[ss], 1]) & skymask_now - hist, edge = np.histogram(waveimg[onslit_b_olap], bins=wavebins, weights=modelimg_copy[onslit_b_olap]) - cntr, edge = np.histogram(waveimg[onslit_b_olap], bins=wavebins) - cntr = cntr.astype(float) - cntr *= spec_ref - norm = utils.inverse(cntr) - arr = hist * norm - # Calculate a smooth version of the relative response - relscale = coadd.smooth_weights(arr, (arr != 0), sn_smooth_npix) - rescale_model = interpolate.interp1d(wave_ref, relscale, kind='linear', bounds_error=False, + # Perform two iterations: + # (ii=0) dynamically build reference spectrum + # (ii=1) used fixed reference spectrum to calculate the relative illumination. + for ii in range(2): + # Reset the relative scaling for this iteration + relscl_model = np.ones_like(rawimg) + # Build the relative illumination, by successively finding the slits closest in wavelength to the reference + for ss in range(1, slits.spat_id.size): + # Calculate the region of overlap + onslit_b = (slitid_img_trim == slits.spat_id[wvsrt[ss]]) + onslit_b_init = (slitid_img_init == slits.spat_id[wvsrt[ss]]) + onslit_b_olap = onslit_b & gpm & (waveimg >= mnmx_wv[wvsrt[ss], 0]) & (waveimg <= mnmx_wv[wvsrt[ss], 1]) & skymask_now + hist, edge = np.histogram(waveimg[onslit_b_olap], bins=wavebins, weights=modelimg_copy[onslit_b_olap]) + cntr, edge = np.histogram(waveimg[onslit_b_olap], bins=wavebins) + cntr = cntr.astype(float) + # Note, if ii=1 (i.e. the reference spectrum is fixed), we want to make sure that the + # spec_ref will give a result of 1 for all wavelengths. Apply this correction first. + if (ii == 1) and (slits.spat_id[wvsrt[ss]] == slit_illum_ref_idx): + # This must be the first element of the loop by construction, but throw an error just in case + if ss != 0: + msgs.error("CODING ERROR - An error has occurred in the relative spectral illumination." + + msgs.newline() + "Please contact the developers.") + tmp_cntr = cntr * spec_ref + tmp_arr = hist * utils.inverse(tmp_cntr) + # Calculate a smooth version of the relative response + ref_relscale = flat.smooth_scale(tmp_arr, wave_ref=wave_ref, polydeg=polydeg, sn_smooth_npix=sn_smooth_npix) + # Update the reference spectrum + spec_ref /= ref_relscale + # Normalise by the reference spectrum + cntr *= spec_ref + norm = utils.inverse(cntr) + arr = hist * norm + # Calculate a smooth version of the relative response + relscale = flat.smooth_scale(arr, wave_ref=wave_ref, polydeg=polydeg, sn_smooth_npix=sn_smooth_npix) + # Store the result + relscl_model[onslit_b_init] = interpolate.interp1d(wave_ref, relscale, kind='linear', bounds_error=False, fill_value="extrapolate")(waveimg[onslit_b_init]) - # Store the result - relscl_model[onslit_b_init] = rescale_model.copy() - # Build a new reference spectrum to increase wavelength coverage of the reference spectrum (and improve S/N) - onslit_ref_trim = onslit_ref_trim | (onslit_b & gpm & skymask_now) - hist, edge = np.histogram(waveimg[onslit_ref_trim], bins=wavebins, weights=modelimg_copy[onslit_ref_trim]/relscl_model[onslit_ref_trim]) - cntr, edge = np.histogram(waveimg[onslit_ref_trim], bins=wavebins) - cntr = cntr.astype(float) - norm = utils.inverse(cntr) - spec_ref = hist * norm + # Build a new reference spectrum to increase wavelength coverage of the reference spectrum (and improve S/N) + if ii == 0: + onslit_ref_trim |= (onslit_b & gpm & skymask_now) + hist, edge = np.histogram(waveimg[onslit_ref_trim], bins=wavebins, weights=modelimg_copy[onslit_ref_trim]/relscl_model[onslit_ref_trim]) + cntr, edge = np.histogram(waveimg[onslit_ref_trim], bins=wavebins) + cntr = cntr.astype(float) + norm = utils.inverse(cntr) + spec_ref = hist * norm minv, maxv = np.min(relscl_model), np.max(relscl_model) if 1/minv + maxv > lo_prev+hi_prev: # Adding noise, so break diff --git a/pypeit/images/bitmaskarray.py b/pypeit/images/bitmaskarray.py index db9c6898b6..20268e1855 100644 --- a/pypeit/images/bitmaskarray.py +++ b/pypeit/images/bitmaskarray.py @@ -13,7 +13,10 @@ import numpy as np +from astropy.io import fits + from pypeit.datamodel import DataContainer +from pypeit.bitmask import BitMask from pypeit import msgs @@ -58,6 +61,13 @@ def __init__(self, shape, asuint=False): self._set_keys() self.mask = np.zeros(shape, dtype=self.bitmask.minimum_dtype(asuint=asuint)) + @classmethod + def from_array(cls, arr): + # Instantiate using the shape of the provided array + self = cls(arr.shape, asuint=np.issubdtype(arr.dtype, np.unsignedinteger)) + self.mask[...] = arr[...] + return self + def _set_keys(self): """ Set :attr:`lower_keys`, which are needed for the bit access convenience @@ -149,6 +159,42 @@ def _bundle(self): d[0].update(self.bitmask.to_dict()) return d + @classmethod + def from_hdu(cls, hdu, chk_version=True, **kwargs): + """ + Instantiate the object from an HDU extension. + + This overrides the base-class method, only to add checks (or not) for + the bitmask. + + Args: + hdu (`astropy.io.fits.HDUList`_, `astropy.io.fits.ImageHDU`_, `astropy.io.fits.BinTableHDU`_): + The HDU(s) with the data to use for instantiation. + chk_version (:obj:`bool`, optional): + If True, raise an error if the datamodel version or + type check failed. If False, throw a warning only. + **kwargs: + Passed directly to :func:`_parse`. + """ + # Run the default parser + d, version_passed, type_passed, parsed_hdus = cls._parse(hdu, **kwargs) + # Check + cls._check_parsed(version_passed, type_passed, chk_version=chk_version) + + # Instantiate + self = super().from_dict(d=d) + + # Check the bitmasks. Bits should have been written to *any* header + # associated with the object + hdr = hdu[parsed_hdus[0]].header if isinstance(hdu, fits.HDUList) else hdu.header + hdr_bitmask = BitMask.from_header(hdr) + if chk_version and hdr_bitmask.bits != self.bitmask.bits: + msgs.error('The bitmask in this fits file appear to be out of date! Recreate this ' + 'file by re-running the relevant script or set chk_version=False.', + cls='PypeItBitMaskError') + + return self + def copy(self): """Create a deep copy.""" _self = super().__new__(self.__class__) diff --git a/pypeit/images/detector_container.py b/pypeit/images/detector_container.py index d5e624a4fb..dcebd6ccf2 100644 --- a/pypeit/images/detector_container.py +++ b/pypeit/images/detector_container.py @@ -104,7 +104,7 @@ class DetectorContainer(datamodel.DataContainer): 'where the valid data sections can be obtained, one ' 'per amplifier. If defined explicitly should be in ' 'FITS format (e.g., [1:2048,10:4096]).'), - 'det': dict(otype=int, + 'det': dict(otype=(int, np.integer), descr='PypeIt designation for detector number (1-based).'), 'binning': dict(otype=str, descr='Binning in PypeIt orientation (not the original)')} diff --git a/pypeit/images/pypeitimage.py b/pypeit/images/pypeitimage.py index 35bc7d9862..fe123d26f8 100644 --- a/pypeit/images/pypeitimage.py +++ b/pypeit/images/pypeitimage.py @@ -284,13 +284,8 @@ def from_hdu(cls, hdu, chk_version=True, hdu_prefix=None): # Parse everything *but* the mask extension d, version_passed, type_passed, parsed_hdus \ - = super()._parse(hdu, ext=ext, hdu_prefix=_hdu_prefix) - if not type_passed: - msgs.error(f'The HDU(s) cannot be parsed by a {cls.__name__} object!') - if not version_passed: - _f = msgs.error if chk_version else msgs.warn - _f(f'Current version of {cls.__name__} object in code (v{cls.version})' - ' does not match version used to write your HDU(s)!') + = cls._parse(hdu, ext=ext, hdu_prefix=_hdu_prefix) + cls._check_parsed(version_passed, type_passed, chk_version=chk_version) if mask_ext in hdu: # If the mask extension exists, parse it diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 6259672b07..8108c4b828 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -612,6 +612,7 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl self.subtract_bias(bias) # TODO: Checking for count (well-depth) saturation should be done here. + # TODO :: Non-linearity correction should be done here. # - Create the dark current image(s). The dark-current image *always* # includes the tabulated dark current and the call below ensures @@ -831,7 +832,7 @@ def flatfield(self, flatimages, slits=None, force=False, debug=False): viewer, ch = display.show_image(self.image[0], chname='orig_image') display.show_slits(viewer, ch, left, right) # , slits.id) - # Apply the relative spectral illumination + # Retrieve the relative spectral illumination profile spec_illum = flatimages.pixelflat_spec_illum if self.par['use_specillum'] else 1. # Apply flat-field correction @@ -1167,10 +1168,10 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): this_modpar = msscattlight.scattlight_param.copy() this_modpar[8] = 0.0 # This is the zero-level of the scattlight frame. The zero-level is determined by the finecorr # Apply the requested method for the scattered light - do_finecorr = self.par["scattlight"]["finecorr"] + do_finecorr = self.par["scattlight"]["finecorr_method"] is not None if self.par["scattlight"]["method"] == "model": # Use predefined model parameters - scatt_img = scattlight.scattered_light_model(this_modpar, _img) + scatt_img = scattlight.scattered_light_model_pad(this_modpar, _img) if debug: specbin, spatbin = parse.parse_binning(self.detector[0]['binning']) tmp = msscattlight.scattlight_param.copy() @@ -1199,7 +1200,7 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): plt.subplot(223) plt.imshow(scatt_img*offslitmask, vmin=-2*vmax, vmax=2*vmax) plt.subplot(224) - plt.imshow((_frame - scatt_img)*offslitmask, vmin=-2*vmax, vmax=2*vmax) + plt.imshow((_frame - scatt_img)*offslitmask, vmin=-vmax/5, vmax=vmax/5) # plt.imshow((_frame - scatt_img)*offslitmask, vmin=-vmax/5, vmax=vmax/5) plt.show() elif self.par["scattlight"]["method"] == "archive": @@ -1248,8 +1249,9 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): mask_regions=self.par['scattlight']['finecorr_mask']) # Calculate the fine correction to the scattered light image, and add it to the full model scatt_img += scattlight.fine_correction(_img-scatt_img, full_bpm, offslitmask, + method=self.par['scattlight']['finecorr_method'], polyord=self.par['scattlight']['finecorr_order']) - # Subtract the scattered light model from the image + # Subtract the total scattered light model from the image self.image[ii, ...] -= scatt_img self.steps[step] = True diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index 84a44edf66..332822081f 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -935,22 +935,25 @@ def options(self): # Load coordinate offsets for each file. This is "Delta RA cos(dec)" and "Delta Dec" # Get the RA offset of each file - off_ra = self.path_and_files('ra_offset', skip_blank=False, check_exists=False) - if off_ra is None: - opts['ra_offset'] = None - elif len(off_ra) == 1 and len(self.filenames) > 1: - opts['ra_offset'] = off_ra*len(self.filenames) - elif len(off_ra) != 0: - opts['ra_offset'] = off_ra + off_ra, off_dec = None, None + if 'ra_offset' in self.data.keys(): + off_ra = self.data['ra_offset'].tolist() + if len(off_ra) == 1 and len(self.filenames) > 1: + # Convert from arcsec to degrees + opts['ra_offset'] = [off_ra[0]/3600.0 for _ in range(len(self.filenames))] + elif len(off_ra) != 0: + # Convert from arcsec to degrees + opts['ra_offset'] = [ora/3600.0 for ora in off_ra] # Get the DEC offset of each file - off_dec = self.path_and_files('dec_offset', skip_blank=False, check_exists=False) - if off_dec is None: - opts['dec_offset'] = None - elif len(off_dec) == 1 and len(self.filenames) > 1: - opts['dec_offset'] = off_dec*len(self.filenames) - elif len(off_dec) != 0: - opts['dec_offset'] = off_dec - # Check that both have been set + if 'dec_offset' in self.data.keys(): + off_dec = self.data['dec_offset'].tolist() + if len(off_dec) == 1 and len(self.filenames) > 1: + # Convert from arcsec to degrees + opts['dec_offset'] = [off_dec[0]/3600.0 for _ in range(len(self.filenames))] + elif len(off_dec) != 0: + # Convert from arcsec to degrees + opts['dec_offset'] = [odec/3600.0 for odec in off_dec] + # Check that both have been set or both are not set if (off_ra is not None and off_dec is None) or (off_ra is None and off_dec is not None): msgs.error("You must specify both or neither of the following arguments: ra_offset, dec_offset") diff --git a/pypeit/metadata.py b/pypeit/metadata.py index 36d1856e7c..66130515e3 100644 --- a/pypeit/metadata.py +++ b/pypeit/metadata.py @@ -1257,7 +1257,7 @@ def find_frames(self, ftype, calib_ID=None, index=False): if ftype == 'None': return self['framebit'] == 0 # Select frames - indx = self.type_bitmask.flagged(self['framebit'], ftype) + indx = self.type_bitmask.flagged(self['framebit'], flag=ftype) if calib_ID is not None: # Select frames in the same calibration group @@ -1914,7 +1914,7 @@ def write(self, output=None, rows=None, columns=None, sort_col=None, overwrite=F # Always write the table in ascii format with io.StringIO() as ff: - output_tbl.write(ff, format='ascii.fixed_width') + output_tbl.write(ff, format='ascii.fixed_width', bookend=False) data_lines = ff.getvalue().split('\n')[:-1] if ofile is None: @@ -1954,7 +1954,7 @@ def find_calib_group(self, grp): """ if 'calibbit' not in self.keys(): msgs.error('Calibration groups are not set. First run set_calibration_groups.') - return self.calib_bitmask.flagged(self['calibbit'].data, grp) + return self.calib_bitmask.flagged(self['calibbit'].data, flag=grp) def find_frame_calib_groups(self, row): """ diff --git a/pypeit/onespec.py b/pypeit/onespec.py index 61faae4eed..6eb9ef83e0 100644 --- a/pypeit/onespec.py +++ b/pypeit/onespec.py @@ -46,7 +46,7 @@ class OneSpec(datamodel.DataContainer): Build from PYP_SPEC """ - version = '1.0.1' + version = '1.0.2' datamodel = {'wave': dict(otype=np.ndarray, atype=np.floating, # TODO: The "weighted by pixel contributions" part @@ -62,6 +62,8 @@ class OneSpec(datamodel.DataContainer): 'see ``fluxed``'), 'ivar': dict(otype=np.ndarray, atype=np.floating, descr='Inverse variance array (matches units of flux)'), + 'sigma': dict(otype=np.ndarray, atype=np.floating, + descr='One sigma noise array, equivalent to 1/sqrt(ivar) (matches units of flux)'), 'mask': dict(otype=np.ndarray, atype=np.integer, descr='Mask array (1=Good,0=Bad)'), 'telluric': dict(otype=np.ndarray, atype=np.floating, descr='Telluric model'), @@ -83,28 +85,32 @@ class OneSpec(datamodel.DataContainer): 'history'] @classmethod - def from_file(cls, ifile): + def from_file(cls, ifile, verbose=True, chk_version=True, **kwargs): """ - Over-load :func:`pypeit.datamodel.DataContainer.from_file` - to deal with the header + Instantiate the object from an extension in the specified fits file. + Over-load :func:`~pypeit.datamodel.DataContainer.from_file` + to deal with the header + Args: - ifile (str): - Filename holding the object + ifile (:obj:`str`, `Path`_): + Fits file with the data to read + verbose (:obj:`bool`, optional): + Print informational messages (not currently used) + chk_version (:obj:`bool`, optional): + Passed to :func:`from_hdu`. + kwargs (:obj:`dict`, optional): + Arguments passed directly to :func:`from_hdu`. """ - hdul = io.fits_open(ifile) - slf = super().from_hdu(hdul) - - # Internals - slf.filename = ifile - slf.head0 = hdul[0].header - # Meta - slf.spectrograph = load_spectrograph(slf.PYP_SPEC) - slf.spect_meta = slf.spectrograph.parse_spec_header(slf.head0) - # - return slf - - def __init__(self, wave, wave_grid_mid, flux, PYP_SPEC=None, ivar=None, mask=None, telluric=None, + with io.fits_open(ifile) as hdu: + self = cls.from_hdu(hdu, chk_version=chk_version, **kwargs) + self.filename = ifile + self.head0 = hdu[0].header + self.spectrograph = load_spectrograph(self.PYP_SPEC) + self.spect_meta = self.spectrograph.parse_spec_header(self.head0) + return self + + def __init__(self, wave, wave_grid_mid, flux, PYP_SPEC=None, ivar=None, sigma=None, mask=None, telluric=None, obj_model=None, ext_mode=None, fluxed=None): args, _, _, values = inspect.getargvalues(inspect.currentframe()) @@ -118,15 +124,6 @@ def _bundle(self): """ return super()._bundle(ext='SPECTRUM') - @property - def sig(self): - """ Return the 1-sigma array - - Returns: - `numpy.ndarray`_: error array - """ - return np.sqrt(utils.inverse(self.ivar)) - def to_file(self, ofile, primary_hdr=None, history=None, **kwargs): """ Over-load :func:`pypeit.datamodel.DataContainer.to_file` diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 7fd2fadbce..19f918f3a8 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -683,7 +683,7 @@ def __init__(self, method=None, pixelflat_file=None, spec_samp_fine=None, defaults['slit_illum_relative'] = False dtypes['slit_illum_relative'] = bool descr['slit_illum_relative'] = 'Generate an image of the relative spectral illumination ' \ - 'for a multi-slit setup. If you set ``use_slitillum = ' \ + 'for a multi-slit setup. If you set ``use_specillum = ' \ 'True`` for any of the frames that use the flatfield ' \ 'model, this *must* be set to True. Currently, this is ' \ 'only used for SlicerIFU reductions.' @@ -1019,7 +1019,7 @@ class ScatteredLightPar(ParSet): see :ref:`parameters`. """ - def __init__(self, method=None, finecorr=None, finecorr_pad=None, finecorr_order=None, finecorr_mask=None): + def __init__(self, method=None, finecorr_method=None, finecorr_pad=None, finecorr_order=None, finecorr_mask=None): # Grab the parameter names and values from the function # arguments @@ -1040,7 +1040,7 @@ def __init__(self, method=None, finecorr=None, finecorr_pad=None, finecorr_order options['method'] = ScatteredLightPar.valid_scattlight_methods() dtypes['method'] = str descr['method'] = 'Method used to fit the overscan. ' \ - 'Options are: {0}'.format(', '.join(options['method'])) + '.' + \ + 'Options are: {0}'.format(', '.join(options['method'])) + '. ' + \ '\'model\' will the scattered light model parameters derived from a ' \ 'user-specified frame during their reduction (note, you will need to make sure ' \ 'that you set appropriate scattlight frames in your .pypeit file for this option). ' \ @@ -1049,15 +1049,21 @@ def __init__(self, method=None, finecorr=None, finecorr_pad=None, finecorr_order '\'archive\' will use an archival model parameter solution for the scattered ' \ 'light (note that this option is not currently available for all spectrographs).' - defaults['finecorr'] = True - dtypes['finecorr'] = bool - descr['finecorr'] = 'If True, a fine correction to the scattered light will be performed. However, the ' \ - 'fine correction will only be applied if the model/frame/archive correction is performed.' - - defaults['finecorr_pad'] = 2 + defaults['finecorr_method'] = None + options['finecorr_method'] = ScatteredLightPar.valid_finecorr_scattlight_methods() + dtypes['finecorr_method'] = str + descr['finecorr_method'] = 'If None, a fine correction to the scattered light will not be performed. ' \ + 'Otherwise, the allowed methods include: ' \ + '{0}'.format(', '.join(options['finecorr_method'])) + '. ' + \ + '\'median\' will subtract a constant value from an entire CCD row, based on a ' \ + 'median of the pixels that are not on slits (see also, \'finecorr_pad\'). ' \ + '\'poly\' will fit a polynomial to the scattered light in each row, based ' \ + 'on the pixels that are not on slits (see also, \'finecorr_pad\').' + + defaults['finecorr_pad'] = 4 dtypes['finecorr_pad'] = int - descr['finecorr_pad'] = 'Number of unbinned pixels to extend the slit edges by when masking the slits for the' \ - 'fine correction to the scattered light.' + descr['finecorr_pad'] = 'Number of unbinned pixels to extend the slit edges by when masking the slits for ' \ + 'the fine correction to the scattered light.' defaults['finecorr_order'] = 2 dtypes['finecorr_order'] = int @@ -1086,7 +1092,7 @@ def __init__(self, method=None, finecorr=None, finecorr_pad=None, finecorr_order @classmethod def from_dict(cls, cfg): k = np.array([*cfg.keys()]) - parkeys = ['method', 'finecorr', 'finecorr_pad', 'finecorr_order', 'finecorr_mask'] + parkeys = ['method', 'finecorr_method', 'finecorr_pad', 'finecorr_order', 'finecorr_mask'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -1101,7 +1107,10 @@ def validate(self): """ Check the parameters are valid for the provided method. """ - pass + if self.data['method'] is not None and self.data['method'] not in self.valid_scattlight_methods(): + raise ValueError("If 'method' is not None it must be one of:\n"+", ".join(self.valid_scattlight_methods())) + if self.data['finecorr_method'] is not None and self.data['finecorr_method'] not in self.valid_finecorr_scattlight_methods(): + raise ValueError("If 'finecorr_method' is not None it must be one of:\n"+", ".join(self.valid_finecorr_scattlight_methods())) @staticmethod def valid_scattlight_methods(): @@ -1110,6 +1119,13 @@ def valid_scattlight_methods(): """ return ['model', 'frame', 'archive'] + @staticmethod + def valid_finecorr_scattlight_methods(): + """ + Return the valid scattered light methods. + """ + return ['median', 'poly'] + class Coadd1DPar(ParSet): """ @@ -1121,9 +1137,10 @@ class Coadd1DPar(ParSet): def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, sn_smooth_npix=None, sigrej_exp=None, wave_method=None, dv=None, dwave=None, dloglam=None, wave_grid_min=None, wave_grid_max=None, spec_samp_fact=None, ref_percentile=None, maxiter_scale=None, - sigrej_scale=None, scale_method=None, sn_min_medscale=None, sn_min_polyscale=None, maxiter_reject=None, + sigrej_scale=None, scale_method=None, sn_min_medscale=None, sn_min_polyscale=None, + weight_method=None, maxiter_reject=None, lower=None, upper=None, maxrej=None, sn_clip=None, nbests=None, coaddfile=None, - mag_type=None, filter=None, filter_mag=None, filter_mask=None, chk_version=None): + mag_type=None, filter=None, filter_mag=None, filter_mask=None): # Grab the parameter names and values from the function # arguments @@ -1169,6 +1186,7 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, defaults['wave_method'] = 'linear' dtypes['wave_method'] = str + options['wave_method'] = Coadd1DPar.valid_wave_methods() descr['wave_method'] = "Method used to construct wavelength grid for coadding spectra. The routine that creates " \ "the wavelength is :func:`~pypeit.core.wavecal.wvutils.get_wave_grid`. The options are:" \ " "\ @@ -1223,8 +1241,11 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, dtypes['sigrej_scale'] = [int, float] descr['sigrej_scale'] = 'Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec.' + + defaults['scale_method'] = 'auto' dtypes['scale_method'] = str + options['scale_method'] = Coadd1DPar.valid_scale_methods() descr['scale_method'] = "Method used to rescale the spectra prior to coadding. The options are:" \ " "\ "'auto' -- Determine the scaling method automatically based on the S/N ratio which works well. "\ @@ -1233,6 +1254,7 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, "'none' -- Do not rescale. " \ "'hand' -- Pass in hand scaling factors. This option is not well tested." + defaults['sn_min_medscale'] = 0.5 dtypes['sn_min_medscale'] = [int, float] descr['sn_min_medscale'] = "For scale method set to ``auto``, this sets the minimum SNR for which median scaling is attempted." @@ -1241,6 +1263,27 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, dtypes['sn_min_polyscale'] = [int, float] descr['sn_min_polyscale'] = "For scale method set to ``auto``, this sets the minimum SNR for which polynomial scaling is attempted." + defaults['weight_method'] = 'auto' + options['weight_method'] = Coadd1DPar.valid_weight_methods() + dtypes['weight_method'] = str + descr['weight_method'] = "Method used to weight the spectra for coadding. The options are:" \ + " " \ + "'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent." \ + "'constant' -- Constant weights based on rms_sn**2" \ + "'uniform' -- Uniform weighting" \ + "'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_" \ + "sn ratio. This option will not work well at low S/N ratio although it is useful for " \ + "objects where only a small fraction of the spectral coverage has high S/N ratio " \ + "(like high-z quasars)." \ + "'relative' -- Apply relative weights implying one reference exposure will receive unit " \ + "weight at all wavelengths and all others receive relatively wavelength dependent "\ + "weights . Note, relative weighting will only work well " \ + "when there is at least one spectrum with a reasonable S/N, and a continuum. " \ + "This option may only be better when the object being used has a strong " \ + "continuum + emission lines. This is particularly useful if you " \ + "are dealing with highly variable spectra (e.g. emission lines) and" \ + "require a precision better than ~1 per cent." \ + "'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated." defaults['maxiter_reject'] = 5 dtypes['maxiter_reject'] = int @@ -1294,13 +1337,6 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, dtypes['coaddfile'] = str descr['coaddfile'] = 'Output filename' - - # Version checking - defaults['chk_version'] = True - dtypes['chk_version'] = bool - descr['chk_version'] = 'If True enforce strict PypeIt version checking to ensure that spec1d*.fits files were created' \ - 'with the current version of PypeIt' - # Instantiate the parameter set super(Coadd1DPar, self).__init__(list(pars.keys()), values=list(pars.values()), @@ -1315,8 +1351,8 @@ def from_dict(cls, cfg): parkeys = ['ex_value', 'flux_value', 'nmaskedge', 'sn_smooth_npix', 'sigrej_exp', 'wave_method', 'dv', 'dwave', 'dloglam', 'wave_grid_min', 'wave_grid_max', 'spec_samp_fact', 'ref_percentile', 'maxiter_scale', 'sigrej_scale', 'scale_method', - 'sn_min_medscale', 'sn_min_polyscale', 'maxiter_reject', 'lower', 'upper', - 'maxrej', 'sn_clip', 'nbests', 'coaddfile', 'chk_version', + 'sn_min_medscale', 'sn_min_polyscale', 'weight_method', 'maxiter_reject', 'lower', 'upper', + 'maxrej', 'sn_clip', 'nbests', 'coaddfile', 'filter', 'mag_type', 'filter_mag', 'filter_mask'] badkeys = np.array([pk not in parkeys for pk in k]) @@ -1332,7 +1368,23 @@ def validate(self): """ Check the parameters are valid for the provided method. """ - pass + allowed_extensions = self.valid_ex() + if self.data['ex_value'] not in allowed_extensions: + raise ValueError("'ex_value' must be one of:\n" + ", ".join(allowed_extensions)) + + allowed_wave_methods = self.valid_wave_methods() + if self.data['wave_method'] not in allowed_wave_methods: + raise ValueError("'wave_method' must be one of:\n" + ", ".join(allowed_wave_methods)) + + allowed_scale_methods = self.valid_scale_methods() + if self.data['scale_method'] not in allowed_scale_methods: + raise ValueError("'scale_method' must be one of:\n" + ", ".join(allowed_scale_methods)) + + allowed_weight_methods = self.valid_weight_methods() + if self.data['weight_method'] not in allowed_weight_methods: + raise ValueError("'weight_method' must be one of:\n" + ", ".join(allowed_weight_methods)) + + @staticmethod def valid_ex(): @@ -1342,6 +1394,26 @@ def valid_ex(): return ['BOX', 'OPT'] + @staticmethod + def valid_wave_methods(): + """ Return the valid options for the wavelength grid of spectra. """ + + return ['iref', 'velocity', 'log10', 'linear', 'concatenate'] + + @staticmethod + def valid_scale_methods(): + """ Return the valid options for the scaling of spectra. """ + + return ['auto', 'poly', 'median', 'none', 'hand'] + + @staticmethod + def valid_weight_methods(): + """ Return the valid options for the weighting of spectra. """ + + return ['auto', 'constant', 'uniform', 'wave_dependent', 'relative', 'ivar'] + + + class Coadd2DPar(ParSet): """ A parameter set holding the arguments for how to perform 2D coadds @@ -1476,6 +1548,7 @@ def validate(self): + class CubePar(ParSet): """ The parameter set used to hold arguments for functionality relevant @@ -1485,11 +1558,11 @@ class CubePar(ParSet): see :ref:`parameters`. """ - def __init__(self, slit_spec=None, relative_weights=None, align=None, combine=None, output_filename=None, + def __init__(self, slit_spec=None, weight_method=None, align=None, combine=None, output_filename=None, standard_cube=None, reference_image=None, save_whitelight=None, whitelight_range=None, method=None, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, spatial_delta=None, wave_delta=None, astrometric=None, grating_corr=None, scale_corr=None, - skysub_frame=None, spec_subpixel=None, spat_subpixel=None): + skysub_frame=None, spec_subpixel=None, spat_subpixel=None, slice_subpixel=None): # Grab the parameter names and values from the function # arguments @@ -1512,12 +1585,28 @@ def __init__(self, slit_spec=None, relative_weights=None, align=None, combine=No descr['slit_spec'] = 'If the data use slits in one spatial direction, set this to True. ' \ 'If the data uses fibres for all spaxels, set this to False.' - defaults['relative_weights'] = False - dtypes['relative_weights'] = [bool] - descr['relative_weights'] = 'If set to True, the combined frames will use a relative weighting scheme. ' \ - 'This only works well if there is a common continuum source in the field of ' \ - 'view of all input observations, and is generally only required if high ' \ - 'relative precision is desired.' + defaults['weight_method'] = 'auto' + options['weight_method'] = Coadd1DPar.valid_weight_methods() + dtypes['weight_method'] = str + descr['weight_method'] = "Method used to weight the spectra for coadding. The options are:" \ + " " \ + "'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent." \ + "'constant' -- Constant weights based on rms_sn**2" \ + "'uniform' -- Uniform weighting" \ + "'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_" \ + "sn ratio. This option will not work well at low S/N ratio although it is useful for " \ + "objects where only a small fraction of the spectral coverage has high S/N ratio " \ + "(like high-z quasars)." \ + "'relative' -- Apply relative weights implying one reference exposure will receive unit " \ + "weight at all wavelengths and all others receive relatively wavelength dependent "\ + "weights . Note, relative weighting will only work well " \ + "when there is at least one spectrum with a reasonable S/N, and a continuum. " \ + "This option may only be better when the object being used has a strong " \ + "continuum + emission lines. This is particularly useful if you " \ + "are dealing with highly variable spectra (e.g. emission lines) and" \ + "require a precision better than ~1 per cent." \ + "'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated." + defaults['align'] = False dtypes['align'] = [bool] @@ -1597,7 +1686,7 @@ def __init__(self, slit_spec=None, relative_weights=None, align=None, combine=No 'each detector pixel in the spectral direction. The total number of subpixels ' \ 'in each pixel is given by spec_subpixel x spat_subpixel. The default option ' \ 'is to divide each spec2d pixel into 25 subpixels during datacube creation. ' \ - 'See also, spat_subpixel.' + 'See also, spat_subpixel and slice_subpixel.' defaults['spat_subpixel'] = 5 dtypes['spat_subpixel'] = int @@ -1605,7 +1694,13 @@ def __init__(self, slit_spec=None, relative_weights=None, align=None, combine=No 'each detector pixel in the spatial direction. The total number of subpixels ' \ 'in each pixel is given by spec_subpixel x spat_subpixel. The default option ' \ 'is to divide each spec2d pixel into 25 subpixels during datacube creation. ' \ - 'See also, spec_subpixel.' + 'See also, spec_subpixel and slice_subpixel.' + + defaults['slice_subpixel'] = 5 + dtypes['slice_subpixel'] = int + descr['slice_subpixel'] = 'When method=subpixel, slice_subpixel sets the subpixellation scale of ' \ + 'each IFU slice. The default option is to divide each slice into 5 sub-slices ' \ + 'during datacube creation. See also, spec_subpixel and spat_subpixel.' defaults['ra_min'] = None dtypes['ra_min'] = float @@ -1693,8 +1788,8 @@ def from_dict(cls, cfg): # Basic keywords parkeys = ['slit_spec', 'output_filename', 'standard_cube', 'reference_image', 'save_whitelight', - 'method', 'spec_subpixel', 'spat_subpixel', 'ra_min', 'ra_max', 'dec_min', 'dec_max', - 'wave_min', 'wave_max', 'spatial_delta', 'wave_delta', 'relative_weights', 'align', 'combine', + 'method', 'spec_subpixel', 'spat_subpixel', 'slice_subpixel', 'ra_min', 'ra_max', 'dec_min', 'dec_max', + 'wave_min', 'wave_max', 'spatial_delta', 'wave_delta', 'weight_method', 'align', 'combine', 'astrometric', 'grating_corr', 'scale_corr', 'skysub_frame', 'whitelight_range'] badkeys = np.array([pk not in parkeys for pk in k]) @@ -1717,6 +1812,11 @@ def validate(self): if len(self.data['whitelight_range']) != 2: raise ValueError("The 'whitelight_range' must be a two element list of either NoneType or float") + allowed_weight_methods = Coadd1DPar.valid_weight_methods() + if self.data['weight_method'] not in allowed_weight_methods: + raise ValueError("'weight_method' must be one of:\n" + ", ".join(allowed_weight_methods)) + + class FluxCalibratePar(ParSet): """ @@ -2209,7 +2309,7 @@ class TelluricPar(ParSet): """ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_bounds=None, pix_shift_bounds=None, - delta_coeff_bounds=None, minmax_coeff_bounds=None, maxiter=None, + delta_coeff_bounds=None, minmax_coeff_bounds=None, maxiter=None, tell_npca=None, teltype=None, sticky=None, lower=None, upper=None, seed=None, tol=None, popsize=None, recombination=None, polish=None, disp=None, objmodel=None, redshift=None, delta_redshift=None, pca_file=None, npca=None, bal_wv_min_max=None, bounds_norm=None, tell_norm_thresh=None, only_orders=None, pca_lower=None, @@ -2224,6 +2324,7 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ # Initialize the other used specifications for this parameter # set defaults = OrderedDict.fromkeys(pars.keys()) + options = OrderedDict.fromkeys(pars.keys()) dtypes = OrderedDict.fromkeys(pars.keys()) descr = OrderedDict.fromkeys(pars.keys()) @@ -2234,6 +2335,18 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ 'must be downloaded from the GoogleDrive and installed in your PypeIt installation via ' \ 'the pypeit_install_telluric script. NOTE: This parameter no longer includes the full ' \ 'pathname to the Telluric Grid file, but is just the filename of the grid itself.' + + defaults['tell_npca'] = 5 + dtypes['tell_npca'] = int + descr['tell_npca'] = 'Number of telluric PCA components used. Can be set to any number from 1 to 10.' + + defaults['teltype'] = 'pca' + options['teltype'] = TelluricPar.valid_teltype() + dtypes['teltype'] = str + descr['teltype'] = 'Method used to evaluate telluric models, either pca or grid. The grid option uses a ' \ + 'fixed grid of pre-computed HITRAN+LBLRTM atmospheric transmission models for each ' \ + 'observatory, whereas the pca option uses principal components of a larger model grid ' \ + 'to compute an accurate pseudo-telluric model with a much lighter telgridfile.' defaults['sn_clip'] = 30.0 dtypes['sn_clip'] = [int, float] @@ -2254,12 +2367,12 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ pars['resln_frac_bounds'] = tuple_force(pars['resln_frac_bounds']) - defaults['resln_frac_bounds'] = (0.5,1.5) + defaults['resln_frac_bounds'] = (0.6,1.4) dtypes['resln_frac_bounds'] = tuple descr['resln_frac_bounds'] = 'Bounds for the resolution fit optimization which is part of the telluric model. ' \ - 'This range is in units of the resln_guess, so the (0.5, 1.5) would bound the ' \ + 'This range is in units of the resln_guess, so the (0.6, 1.4) would bound the ' \ 'spectral resolution fit to be within the range ' \ - 'bounds_resln = (0.5*resln_guess, 1.5*resln_guess)' + 'bounds_resln = (0.6*resln_guess, 1.4*resln_guess)' pars['pix_shift_bounds'] = tuple_force(pars['pix_shift_bounds']) defaults['pix_shift_bounds'] = (-5.0,5.0) @@ -2462,7 +2575,7 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ @classmethod def from_dict(cls, cfg): k = np.array([*cfg.keys()]) - parkeys = ['telgridfile', 'sn_clip', 'resln_guess', 'resln_frac_bounds', + parkeys = ['telgridfile', 'teltype', 'sn_clip', 'resln_guess', 'resln_frac_bounds', 'tell_npca', 'pix_shift_bounds', 'delta_coeff_bounds', 'minmax_coeff_bounds', 'maxiter', 'sticky', 'lower', 'upper', 'seed', 'tol', 'popsize', 'recombination', 'polish', 'disp', 'objmodel','redshift', 'delta_redshift', @@ -2479,12 +2592,27 @@ def from_dict(cls, cfg): for pk in parkeys: kwargs[pk] = cfg[pk] if pk in k else None return cls(**kwargs) + + @staticmethod + def valid_teltype(): + """ + Return the valid telluric methods. + """ + return ['pca', 'grid'] def validate(self): """ Check the parameters are valid for the provided method. """ - pass + if self.data['tell_npca'] < 1 or self.data['tell_npca'] > 10: + raise ValueError('Invalid value {:d} for tell_npca '.format(self.data['tell_npca'])+ + '(must be between 1 and 10).') + + self.data['teltype'] = self.data['teltype'].lower() + if self.data['teltype'] not in TelluricPar.valid_teltype(): + raise ValueError('Invalid teltype "{}"'.format(self.data['teltype'])+ + ', valid options are: {}.'.format(TelluricPar.valid_teltype())) + # JFH add something in here which checks that the recombination value provided is bewteen 0 and 1, although # scipy.optimize.differential_evoluiton probalby checks this. @@ -2503,7 +2631,7 @@ class ReduxPar(ParSet): """ def __init__(self, spectrograph=None, detnum=None, sortroot=None, calwin=None, scidir=None, qadir=None, redux_path=None, ignore_bad_headers=None, slitspatnum=None, - maskIDs=None, quicklook=None): + maskIDs=None, quicklook=None, chk_version=None): # Grab the parameter names and values from the function # arguments @@ -2583,6 +2711,17 @@ def __init__(self, spectrograph=None, detnum=None, sortroot=None, calwin=None, s descr['redux_path'] = 'Path to folder for performing reductions. Default is the ' \ 'current working directory.' + # Version checking + defaults['chk_version'] = True + dtypes['chk_version'] = bool + descr['chk_version'] = 'If True enforce strict PypeIt version checking to ensure that ' \ + 'all files were created with the current version of PypeIt. If ' \ + 'set to False, the code will attempt to read out-of-date files ' \ + 'and keep going. Beware (!!) that this can lead to unforeseen ' \ + 'bugs that either cause the code to crash or lead to erroneous ' \ + 'results. I.e., you really need to know what you are doing if ' \ + 'you set this to False!' + # Instantiate the parameter set super(ReduxPar, self).__init__(list(pars.keys()), values=list(pars.values()), @@ -2598,7 +2737,7 @@ def from_dict(cls, cfg): # Basic keywords parkeys = [ 'spectrograph', 'quicklook', 'detnum', 'sortroot', 'calwin', 'scidir', 'qadir', - 'redux_path', 'ignore_bad_headers', 'slitspatnum', 'maskIDs'] + 'redux_path', 'ignore_bad_headers', 'slitspatnum', 'maskIDs', 'chk_version'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -2999,20 +3138,20 @@ class EdgeTracePar(ParSet): see :ref:`parameters`. """ prefix = 'ETP' # Prefix for writing parameters to a header is a class attribute - def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enhance=None, exclude_regions=None, - follow_span=None, det_min_spec_length=None, max_shift_abs=None, max_shift_adj=None, - max_spat_error=None, match_tol=None, fit_function=None, fit_order=None, - fit_maxdev=None, fit_maxiter=None, fit_niter=None, fit_min_spec_length=None, - auto_pca=None, left_right_pca=None, pca_min_edges=None, pca_n=None, - pca_var_percent=None, pca_function=None, pca_order=None, pca_sigrej=None, - pca_maxrej=None, pca_maxiter=None, smash_range=None, edge_detect_clip=None, - trace_median_frac=None, trace_thresh=None, fwhm_uniform=None, niter_uniform=None, - fwhm_gaussian=None, niter_gaussian=None, det_buffer=None, max_nudge=None, - sync_predict=None, sync_center=None, gap_offset=None, sync_to_edge=None, - bound_detector=None, minimum_slit_dlength=None, dlength_range=None, - minimum_slit_length=None, minimum_slit_length_sci=None, - length_range=None, minimum_slit_gap=None, clip=None, order_match=None, - order_offset=None, add_missed_orders=None, order_width_poly=None, + def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enhance=None, + exclude_regions=None, follow_span=None, det_min_spec_length=None, + max_shift_abs=None, max_shift_adj=None, max_spat_error=None, match_tol=None, + fit_function=None, fit_order=None, fit_maxdev=None, fit_maxiter=None, + fit_niter=None, fit_min_spec_length=None, auto_pca=None, left_right_pca=None, + pca_min_edges=None, pca_n=None, pca_var_percent=None, pca_function=None, + pca_order=None, pca_sigrej=None, pca_maxrej=None, pca_maxiter=None, + smash_range=None, edge_detect_clip=None, trace_median_frac=None, trace_thresh=None, + trace_rms_tol=None, fwhm_uniform=None, niter_uniform=None, fwhm_gaussian=None, + niter_gaussian=None, det_buffer=None, max_nudge=None, sync_predict=None, + sync_center=None, gap_offset=None, sync_to_edge=None, bound_detector=None, + minimum_slit_dlength=None, dlength_range=None, minimum_slit_length=None, + minimum_slit_length_sci=None, length_range=None, minimum_slit_gap=None, clip=None, + order_match=None, order_offset=None, add_missed_orders=None, order_width_poly=None, order_gap_poly=None, order_spat_range=None, overlap=None, use_maskdesign=None, maskdesign_maxsep=None, maskdesign_step=None, maskdesign_sigrej=None, pad=None, add_slits=None, add_predict=None, rm_slits=None, maskdesign_filename=None): @@ -3212,6 +3351,13 @@ def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enha 'image (see `trace_median_frac`), values in the median-filtered ' \ 'image *below* this threshold are masked in the refitting of ' \ 'the edge trace data. If None, no masking applied.' + + dtypes['trace_rms_tol'] = [int, float] + descr['trace_rms_tol'] = 'After retracing edges using peaks detected in the rectified ' \ + 'and collapsed image, the RMS difference (in pixels) between ' \ + 'the original and refit traces are calculated. This sets the ' \ + 'upper limit of the RMS for traces that will be removed. If ' \ + 'None, no limit is set and all new traces are kept.' defaults['fwhm_uniform'] = 3.0 dtypes['fwhm_uniform'] = [int, float] @@ -3512,19 +3658,19 @@ def from_dict(cls, cfg): # TODO Please provide docs k = np.array([*cfg.keys()]) parkeys = ['filt_iter', 'sobel_mode', 'edge_thresh', 'sobel_enhance', 'exclude_regions', - 'follow_span', 'det_min_spec_length', - 'max_shift_abs', 'max_shift_adj', 'max_spat_error', 'match_tol', 'fit_function', - 'fit_order', 'fit_maxdev', 'fit_maxiter', 'fit_niter', 'fit_min_spec_length', - 'auto_pca', 'left_right_pca', 'pca_min_edges', 'pca_n', 'pca_var_percent', - 'pca_function', 'pca_order', 'pca_sigrej', 'pca_maxrej', 'pca_maxiter', - 'smash_range', 'edge_detect_clip', 'trace_median_frac', 'trace_thresh', - 'fwhm_uniform', 'niter_uniform', 'fwhm_gaussian', 'niter_gaussian', - 'det_buffer', 'max_nudge', 'sync_predict', 'sync_center', 'gap_offset', - 'sync_to_edge', 'bound_detector', 'minimum_slit_dlength', 'dlength_range', - 'minimum_slit_length', 'minimum_slit_length_sci', 'length_range', - 'minimum_slit_gap', 'clip', 'order_match', 'order_offset', 'add_missed_orders', - 'order_width_poly', 'order_gap_poly', 'order_spat_range', 'overlap', - 'use_maskdesign', 'maskdesign_maxsep', 'maskdesign_step', 'maskdesign_sigrej', + 'follow_span', 'det_min_spec_length', 'max_shift_abs', 'max_shift_adj', + 'max_spat_error', 'match_tol', 'fit_function', 'fit_order', 'fit_maxdev', + 'fit_maxiter', 'fit_niter', 'fit_min_spec_length', 'auto_pca', 'left_right_pca', + 'pca_min_edges', 'pca_n', 'pca_var_percent', 'pca_function', 'pca_order', + 'pca_sigrej', 'pca_maxrej', 'pca_maxiter', 'smash_range', 'edge_detect_clip', + 'trace_median_frac', 'trace_thresh', 'trace_rms_tol', 'fwhm_uniform', + 'niter_uniform', 'fwhm_gaussian', 'niter_gaussian', 'det_buffer', 'max_nudge', + 'sync_predict', 'sync_center', 'gap_offset', 'sync_to_edge', 'bound_detector', + 'minimum_slit_dlength', 'dlength_range', 'minimum_slit_length', + 'minimum_slit_length_sci', 'length_range', 'minimum_slit_gap', 'clip', + 'order_match', 'order_offset', 'add_missed_orders', 'order_width_poly', + 'order_gap_poly', 'order_spat_range', 'overlap', 'use_maskdesign', + 'maskdesign_maxsep', 'maskdesign_step', 'maskdesign_sigrej', 'maskdesign_filename', 'pad', 'add_slits', 'add_predict', 'rm_slits'] badkeys = np.array([pk not in parkeys for pk in k]) @@ -5052,7 +5198,9 @@ class Collate1DPar(ParSet): For a table with the current keywords, defaults, and descriptions, see :ref:`parameters`. """ - def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, match_using=None, exclude_slit_trace_bm=[], exclude_serendip=False, wv_rms_thresh=None, outdir=None, spec1d_outdir=None, refframe=None, chk_version=False): + def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, match_using=None, + exclude_slit_trace_bm=[], exclude_serendip=False, wv_rms_thresh=None, outdir=None, + spec1d_outdir=None, refframe=None): # Grab the parameter names and values from the function # arguments @@ -5131,10 +5279,6 @@ def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, ma descr['refframe'] = 'Perform reference frame correction prior to coadding. ' \ 'Options are: {0}'.format(', '.join(options['refframe'])) - defaults['chk_version'] = False - dtypes['chk_version'] = bool - descr['chk_version'] = "Whether to check the data model versions of spec1d files and sensfunc files." - # Instantiate the parameter set super(Collate1DPar, self).__init__(list(pars.keys()), values=list(pars.values()), @@ -5146,7 +5290,9 @@ def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, ma @classmethod def from_dict(cls, cfg): k = [*cfg.keys()] - parkeys = ['tolerance', 'dry_run', 'ignore_flux', 'flux', 'match_using', 'exclude_slit_trace_bm', 'exclude_serendip', 'outdir', 'spec1d_outdir', 'wv_rms_thresh', 'refframe', 'chk_version'] + parkeys = ['tolerance', 'dry_run', 'ignore_flux', 'flux', 'match_using', + 'exclude_slit_trace_bm', 'exclude_serendip', 'outdir', 'spec1d_outdir', + 'wv_rms_thresh', 'refframe'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index 06fdca97d6..cb199c91f5 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -161,7 +161,6 @@ def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_calibs=False, self.det = None self.tstart = None self.basename = None -# self.sciI = None self.obstime = None @property @@ -283,12 +282,13 @@ def calib_all(self): for self.det in detectors: msgs.info(f'Working on detector {self.det}') # Instantiate Calibrations class + user_slits = slittrace.merge_user_slit(self.par['rdx']['slitspatnum'], + self.par['rdx']['maskIDs']) self.caliBrate = calibrations.Calibrations.get_instance( self.fitstbl, self.par['calibrations'], self.spectrograph, self.calibrations_path, qadir=self.qa_path, reuse_calibs=self.reuse_calibs, - show=self.show, - user_slits=slittrace.merge_user_slit(self.par['rdx']['slitspatnum'], - self.par['rdx']['maskIDs'])) + show=self.show, user_slits=user_slits, + chk_version=self.par['rdx']['chk_version']) # Do it # These need to be separate to accommodate COADD2D self.caliBrate.set_config(grp_frames[0], self.det, self.par['calibrations']) @@ -695,13 +695,13 @@ def calib_one(self, frames, det): msgs.info(f'Building/loading calibrations for detector {det}') # Instantiate Calibrations class + user_slits=slittrace.merge_user_slit(self.par['rdx']['slitspatnum'], + self.par['rdx']['maskIDs']) caliBrate = calibrations.Calibrations.get_instance( self.fitstbl, self.par['calibrations'], self.spectrograph, self.calibrations_path, qadir=self.qa_path, - reuse_calibs=self.reuse_calibs, show=self.show, - user_slits=slittrace.merge_user_slit( - self.par['rdx']['slitspatnum'], self.par['rdx']['maskIDs'])) - #slitspat_num=self.par['rdx']['slitspatnum']) + reuse_calibs=self.reuse_calibs, show=self.show, user_slits=user_slits, + chk_version=self.par['rdx']['chk_version']) # These need to be separate to accomodate COADD2D caliBrate.set_config(frames[0], det, self.par['calibrations']) caliBrate.run_the_steps() diff --git a/pypeit/pypmsgs.py b/pypeit/pypmsgs.py index 0755b00fd6..36caf265b4 100644 --- a/pypeit/pypmsgs.py +++ b/pypeit/pypmsgs.py @@ -31,6 +31,9 @@ class PypeItError(Exception): pass +class PypeItBitMaskError(PypeItError): + pass + class PypeItDataModelError(PypeItError): pass diff --git a/pypeit/scattlight.py b/pypeit/scattlight.py index 6ed95d82b4..a6f149a6d4 100644 --- a/pypeit/scattlight.py +++ b/pypeit/scattlight.py @@ -105,7 +105,7 @@ def get_model(self, image): msgs.warn("No scattered light parameters are available") return np.zeros_like(image) # Return the model of the scattered light - return scattlight.scattered_light_model(self.scattlight_param, image) + return scattlight.scattered_light_model_pad(self.scattlight_param, image) def show(self, image=None, slits=None, mask=False, wcs_match=True): """ Display the master scattered light frame, the model, and data-model. @@ -141,3 +141,4 @@ def show(self, image=None, slits=None, mask=False, wcs_match=True): # Display frames display.show_scattered_light(image_list, slits=slits, wcs_match=wcs_match) + diff --git a/pypeit/scripts/__init__.py b/pypeit/scripts/__init__.py index e08966dc63..a16a0f9573 100644 --- a/pypeit/scripts/__init__.py +++ b/pypeit/scripts/__init__.py @@ -31,6 +31,7 @@ from pypeit.scripts import multislit_flexure from pypeit.scripts import obslog from pypeit.scripts import parse_slits +from pypeit.scripts import print_bpm from pypeit.scripts import qa_html from pypeit.scripts import ql #from pypeit.scripts import ql_multislit diff --git a/pypeit/scripts/arxiv_solution.py b/pypeit/scripts/arxiv_solution.py index a608d5e2dd..160a622a1a 100644 --- a/pypeit/scripts/arxiv_solution.py +++ b/pypeit/scripts/arxiv_solution.py @@ -25,6 +25,8 @@ def get_parser(cls, width=None): parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename make_arxiv_solution_YYYYMMDD-HHMM.log') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod @@ -33,6 +35,8 @@ def main(args): from pypeit.wavecalib import WaveCalib from pypeit.core.wavecal import wvutils + chk_version = not args.try_old + # Set the verbosity, and create a logfile if verbosity == 2 msgs.set_logfile_and_verbosity('arxiv_solution', args.verbosity) @@ -43,7 +47,7 @@ def main(args): msgs.error("The following MasterWaveCalib file does not exist:" + msgs.newline() + args.file) # Load the wavelength calibration file - wv_calib = WaveCalib.from_file(args.file) + wv_calib = WaveCalib.from_file(args.file, chk_version=chk_version) # Check if a wavelength solution exists if wv_calib['wv_fits'][args.slit]['wave_soln'] is None: gd_slits = [] diff --git a/pypeit/scripts/chk_alignments.py b/pypeit/scripts/chk_alignments.py index a484d494e7..5882328a32 100644 --- a/pypeit/scripts/chk_alignments.py +++ b/pypeit/scripts/chk_alignments.py @@ -21,14 +21,17 @@ def get_parser(cls, width=None): help='PypeIt Alignment file [e.g. Alignment_A_1_DET01.fits]') parser.add_argument('--chname', default='Alignments', type=str, help='Channel name for image in Ginga') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod - def main(pargs): + def main(args): from pypeit import alignframe # Load - alignments = alignframe.Alignments.from_file(pargs.file) + chk_version = not args.try_old + alignments = alignframe.Alignments.from_file(args.file, chk_version=chk_version) # Show alignments.show() diff --git a/pypeit/scripts/chk_edges.py b/pypeit/scripts/chk_edges.py index 990a607ed9..d043e76e58 100644 --- a/pypeit/scripts/chk_edges.py +++ b/pypeit/scripts/chk_edges.py @@ -6,7 +6,6 @@ """ from pypeit.scripts import scriptbase -from pathlib import Path class ChkEdges(scriptbase.ScriptBase): @@ -30,21 +29,26 @@ def get_parser(cls, width=None): return parser @staticmethod - def main(pargs): + def main(args): + + from pathlib import Path + from pypeit import edgetrace, slittrace, msgs + chk_version = not args.try_old + # Load - edges = edgetrace.EdgeTraceSet.from_file(pargs.trace_file, chk_version=(not pargs.try_old)) + edges = edgetrace.EdgeTraceSet.from_file(args.trace_file, chk_version=chk_version) - if pargs.mpl: + if args.mpl: edges.show(thin=10, include_img=True, idlabel=True) return # Set the Slits file name - slit_filename = pargs.slits_file + slit_filename = args.slits_file if slit_filename is not None: # File provided by user - slit_filename = Path(pargs.slits_file).resolve() + slit_filename = Path(args.slits_file).resolve() if not slit_filename.exists(): # But doesn't exist msgs.warn(f'{slit_filename} does not exist!') @@ -58,7 +62,7 @@ def main(pargs): msgs.warn(f'{slit_filename} does not exist!') # NOTE: At this point, slit_filename *must* be a Path object - slits = slittrace.SlitTraceSet.from_file(slit_filename, chk_version=(not pargs.try_old)) \ + slits = slittrace.SlitTraceSet.from_file(slit_filename, chk_version=chk_version) \ if slit_filename.exists() else None edges.show(thin=10, in_ginga=True, slits=slits) diff --git a/pypeit/scripts/chk_flats.py b/pypeit/scripts/chk_flats.py index 92211ca894..19f557eb9f 100644 --- a/pypeit/scripts/chk_flats.py +++ b/pypeit/scripts/chk_flats.py @@ -27,9 +27,11 @@ def main(args): from pypeit import flatfield + chk_version = not args.try_old + # Load - flatImages = flatfield.FlatImages.from_file(args.file, chk_version=(not args.try_old)) + flatImages = flatfield.FlatImages.from_file(args.file, chk_version=chk_version) # Show - flatImages.show(args.type) + flatImages.show(args.type, chk_version=chk_version) diff --git a/pypeit/scripts/chk_noise_1dspec.py b/pypeit/scripts/chk_noise_1dspec.py index 33123e5533..88577e109b 100644 --- a/pypeit/scripts/chk_noise_1dspec.py +++ b/pypeit/scripts/chk_noise_1dspec.py @@ -168,11 +168,16 @@ def get_parser(cls, width=None): 'save, a folder called spec1d*_noisecheck' ' will be created and all the relevant ' 'plot will be placed there.') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod def main(args): + + chk_version = not args.try_old + # Load em line_names, line_wav = utils.list_of_spectral_lines() @@ -198,8 +203,9 @@ def main(args): head = fits.getheader(file) # I/O spec object - specObjs = [OneSpec.from_file(file)] if args.fileformat == 'coadd1d' else \ - specobjs.SpecObjs.from_fitsfile(file, chk_version=False) + specObjs = [OneSpec.from_file(file, chk_version=chk_version)] \ + if args.fileformat == 'coadd1d' else \ + specobjs.SpecObjs.from_fitsfile(file, chk_version=chk_version) # loop on the spectra for spec in specObjs: diff --git a/pypeit/scripts/chk_noise_2dspec.py b/pypeit/scripts/chk_noise_2dspec.py index 6dc8372af9..95e5287ae9 100644 --- a/pypeit/scripts/chk_noise_2dspec.py +++ b/pypeit/scripts/chk_noise_2dspec.py @@ -171,6 +171,8 @@ def get_parser(cls, width=None): 'noise values to be printed in the terminal.') parser.add_argument('--list', default=False, help='List the extensions only?', action='store_true') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @@ -178,6 +180,8 @@ def get_parser(cls, width=None): @staticmethod def main(args): + chk_version = not args.try_old + # Parse the detector name try: det = int(args.det) @@ -205,7 +209,7 @@ def main(args): if args.list: io.fits_open(file).info() continue - spec2DObj = spec2dobj.Spec2DObj.from_file(file, detname, chk_version=False) + spec2DObj = spec2dobj.Spec2DObj.from_file(file, detname, chk_version=chk_version) # Deal with redshifts if args.z is not None: diff --git a/pypeit/scripts/chk_scattlight.py b/pypeit/scripts/chk_scattlight.py index f7446f17ae..548b9fd536 100644 --- a/pypeit/scripts/chk_scattlight.py +++ b/pypeit/scripts/chk_scattlight.py @@ -6,10 +6,6 @@ """ from pypeit.scripts import scriptbase -from pypeit import msgs -from pypeit.pypmsgs import PypeItError, PypeItDataModelError -from pypeit.images.detector_container import DetectorContainer -from pypeit import io class ChkScattLight(scriptbase.ScriptBase): @@ -31,12 +27,20 @@ def get_parser(cls, width=None): parser.add_argument('--mask', type=bool, default=False, help='If True, the detector pixels that are considered on the slit will be ' 'masked to highlight the scattered light regions') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod def main(args): from pypeit import scattlight, spec2dobj, slittrace + from pypeit import msgs + from pypeit.pypmsgs import PypeItError, PypeItDataModelError + from pypeit.images.detector_container import DetectorContainer + from pypeit import io + + chk_version = not args.try_old # Parse the detector name try: @@ -47,10 +51,10 @@ def main(args): detname = DetectorContainer.get_name(det) # Load scattered light calibration frame - ScattLightImage = scattlight.ScatteredLight.from_file(args.file) + ScattLightImage = scattlight.ScatteredLight.from_file(args.file, chk_version=chk_version) # Load slits information - slits = slittrace.SlitTraceSet.from_file(args.slits) + slits = slittrace.SlitTraceSet.from_file(args.slits, chk_version=chk_version) # Load the alternate file if requested display_frame = None # The default is to display the frame used to calculate the scattered light model @@ -58,7 +62,8 @@ def main(args): msgs.error("displaying the spec2d scattered light is not currently supported") try: # TODO :: the spec2d file may have already had the scattered light removed, so this is not correct. This script only works when the scattered light is turned off for the spec2d file - spec2D = spec2dobj.Spec2DObj.from_file(args.spec2d, detname, chk_version=False) + spec2D = spec2dobj.Spec2DObj.from_file(args.spec2d, detname, + chk_version=chk_version) except PypeItDataModelError: msgs.warn(f"Error loading spec2d file {args.spec2d} - attempting to load science image from fits") spec2D = None diff --git a/pypeit/scripts/chk_tilts.py b/pypeit/scripts/chk_tilts.py index e85bdbac40..95f1d2ded2 100644 --- a/pypeit/scripts/chk_tilts.py +++ b/pypeit/scripts/chk_tilts.py @@ -26,17 +26,20 @@ def get_parser(cls, width=None): parser.add_argument('--show_traces', default=False, action='store_true', help='Show the traced tilts. This slows down the plotting (mostly in Ginga). If not set, ' 'only the fitted, masked and rejected in the fit tilts are shown.') - parser.add_argument('--try_old', default=True, action='store_true', + parser.add_argument('--try_old', default=False, action='store_true', help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod - def main(pargs): + def main(args): from pypeit import wavetilts + chk_version = not args.try_old + # Load - tilts = wavetilts.WaveTilts.from_file(pargs.file, chk_version=(not pargs.try_old)) - tilts.show(in_ginga=np.logical_not(pargs.mpl), show_traces=pargs.show_traces) + tilts = wavetilts.WaveTilts.from_file(args.file, chk_version=chk_version) + tilts.show(in_ginga=np.logical_not(args.mpl), show_traces=args.show_traces, + chk_version=chk_version) diff --git a/pypeit/scripts/chk_wavecalib.py b/pypeit/scripts/chk_wavecalib.py index f06c94b992..9c8e767e1b 100644 --- a/pypeit/scripts/chk_wavecalib.py +++ b/pypeit/scripts/chk_wavecalib.py @@ -18,15 +18,18 @@ def get_parser(cls, width=None): parser.add_argument('input_file', type=str, nargs='+', help='One or more PypeIt WaveCalib file [e.g. WaveCalib_A_1_DET01.fits] or ' 'spec2d file') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod def main(args): from IPython import embed - from pypeit import wavecalib, spec2dobj, msgs - from pypeit.pypmsgs import PypeItError from astropy.io import fits + from pypeit import wavecalib, spec2dobj, msgs + + chk_version = not args.try_old # Loop over the input files for in_file in args.input_file: @@ -44,17 +47,18 @@ def main(args): msgs.error("Bad file type input!") if file_type == 'WaveCalib': - waveCalib = wavecalib.WaveCalib.from_file(in_file, chk_version=False) + waveCalib = wavecalib.WaveCalib.from_file(in_file, chk_version=chk_version) waveCalib.wave_diagnostics(print_diag=True) continue elif file_type == 'AllSpec2D': - allspec2D = spec2dobj.AllSpec2DObj.from_fits(in_file, chk_version=False) + allspec2D = spec2dobj.AllSpec2DObj.from_fits(in_file, chk_version=chk_version) for det in allspec2D.detectors: print('') print('='*50 + f'{det:^7}' + '='*51) wave_diag = allspec2D[det].wavesol - for colname in ['minWave', 'Wave_cen', 'maxWave', 'IDs_Wave_cov(%)', 'mesured_fwhm']: + for colname in ['minWave', 'Wave_cen', 'maxWave', 'IDs_Wave_cov(%)', + 'measured_fwhm']: wave_diag[colname].format = '0.1f' for colname in ['dWave', 'RMS']: wave_diag[colname].format = '0.3f' diff --git a/pypeit/scripts/coadd_1dspec.py b/pypeit/scripts/coadd_1dspec.py index e6e2c604c7..7763f9b905 100644 --- a/pypeit/scripts/coadd_1dspec.py +++ b/pypeit/scripts/coadd_1dspec.py @@ -247,7 +247,6 @@ def main(args): # sensfile = os.path.join(args.test_spec_path, sensfile) # coaddfile = os.path.join(args.test_spec_path, coaddfile) - if coaddfile is None: coaddfile = build_coadd_file_name(coadd1dFile.filenames, spectrograph) @@ -258,7 +257,8 @@ def main(args): par=par['coadd1d'], sensfuncfile=coadd1dFile.sensfiles, setup_id=coadd1dFile.setup_id, - debug=args.debug, show=args.show) + debug=args.debug, show=args.show, + chk_version=par['rdx']['chk_version']) # Run coAdd1d.run() # Save to file diff --git a/pypeit/scripts/coadd_datacube.py b/pypeit/scripts/coadd_datacube.py index 26e12a6bd2..2d4fbe4d71 100644 --- a/pypeit/scripts/coadd_datacube.py +++ b/pypeit/scripts/coadd_datacube.py @@ -13,7 +13,7 @@ from pypeit.coadd3d import CoAdd3D from pypeit.spectrographs.util import load_spectrograph from pypeit.scripts import scriptbase - +from IPython import embed class CoAddDataCube(scriptbase.ScriptBase): @@ -47,6 +47,7 @@ def main(args): spectrograph_def_par = spectrograph.default_pypeit_par() parset = par.PypeItPar.from_cfg_lines(cfg_lines=spectrograph_def_par.to_config(), merge_with=(coadd3dfile.cfg_lines,)) + # If detector was passed as an argument override whatever was in the coadd3d file if args.det is not None: msgs.info("Restricting to detector={}".format(args.det)) @@ -60,8 +61,9 @@ def main(args): # Instantiate CoAdd3d tstart = time.time() - coadd = CoAdd3D.get_instance(coadd3dfile.filenames, parset, skysub_frame=skysub_frame, scale_corr=scale_corr, - ra_offsets=ra_offsets, dec_offsets=dec_offsets, spectrograph=spectrograph, + coadd = CoAdd3D.get_instance(coadd3dfile.filenames, parset, skysub_frame=skysub_frame, + scale_corr=scale_corr, ra_offsets=ra_offsets, + dec_offsets=dec_offsets, spectrograph=spectrograph, det=args.det, overwrite=args.overwrite) # Coadd the files diff --git a/pypeit/scripts/collate_1d.py b/pypeit/scripts/collate_1d.py index 6e99c789ad..572cbf9344 100644 --- a/pypeit/scripts/collate_1d.py +++ b/pypeit/scripts/collate_1d.py @@ -105,7 +105,7 @@ def find_slits_to_exclude(spec2d_files, par): Args: spec2d_files (:obj:`list`): List of spec2d files to build the map from. - par (:class:`~pypeit.par.pypeitpar.Collate1DPar`): + par (:class:`~pypeit.par.pypeitpar.PypeItPar`): Parameters from a ``.collate1d`` file Returns: @@ -124,11 +124,11 @@ def find_slits_to_exclude(spec2d_files, par): exclude_map = dict() for spec2d_file in spec2d_files: - allspec2d = AllSpec2DObj.from_fits(spec2d_file, chk_version=par['collate1d']['chk_version']) + allspec2d = AllSpec2DObj.from_fits(spec2d_file, chk_version=par['rdx']['chk_version']) for sobj2d in [allspec2d[det] for det in allspec2d.detectors]: for (slit_id, mask, slit_mask_id) in sobj2d['slits'].slit_info: for flag in exclude_flags: - if bit_mask.flagged(mask, flag): + if bit_mask.flagged(mask, flag=flag): if slit_mask_id not in exclude_map: exclude_map[slit_mask_id] = {flag} else: @@ -247,7 +247,7 @@ def read_spec1d_files(par, spec1d_files, failure_msgs): good_spec1d_files = [] for spec1d_file in spec1d_files: try: - sobjs = SpecObjs.from_fitsfile(spec1d_file, chk_version = par['collate1d']['chk_version']) + sobjs = SpecObjs.from_fitsfile(spec1d_file, chk_version=par['rdx']['chk_version']) specobjs_list.append(sobjs) good_spec1d_files.append(spec1d_file) except Exception as e: @@ -301,7 +301,8 @@ def flux(par, spectrograph, spec1d_files, failed_fluxing_msgs): # Flux calibrate the spec1d file try: msgs.info(f"Running flux calibrate on {spec1d_file}") - FxCalib = fluxcalibrate.flux_calibrate([spec1d_file], [sens_file], par=par['fluxcalib'], chk_version=par['collate1d']['chk_version']) + FxCalib = fluxcalibrate.flux_calibrate([spec1d_file], [sens_file], par=par['fluxcalib'], + chk_version=par['rdx']['chk_version']) flux_calibrated_files.append(spec1d_file) except Exception: @@ -352,7 +353,7 @@ def refframe_correction(par, spectrograph, spec1d_files, spec1d_failure_msgs): for spec1d in spec1d_files: # Get values from the fits header needed to calculate the correction try: - sobjs = SpecObjs.from_fitsfile(spec1d) + sobjs = SpecObjs.from_fitsfile(spec1d, chk_version=par['rdx']['chk_version']) hdr_ra = sobjs.header['RA'] hdr_dec = sobjs.header['DEC'] hdr_radec = ltu.radec_to_coord((hdr_ra, hdr_dec)) @@ -425,9 +426,6 @@ def coadd(par, coaddfile, source): # Set destination file for coadding par['coadd1d']['coaddfile'] = coaddfile - # Whether to be forgiving of data model versions - par['coadd1d']['chk_version'] = par['collate1d']['chk_version'] - # Determine if we should coadd flux calibrated data flux_key = par['coadd1d']['ex_value'] + "_FLAM" @@ -610,7 +608,7 @@ def build_parameters(args): params['collate1d']['refframe'] = args.refframe if args.chk_version is True: - params['collate1d']['chk_version'] = True + params['rdx']['chk_version'] = True return params, spectrograph, spec1d_files @@ -664,7 +662,8 @@ class Collate1D(scriptbase.ScriptBase): @classmethod def get_parser(cls, width=None): # A blank Colate1DPar to avoid duplicating the help text. - blank_par = pypeitpar.Collate1DPar() + blank_pypar = pypeitpar.PypeItPar() + blank_par = blank_pypar['collate1d'] parser = super().get_parser(description='Flux/Coadd multiple 1d spectra from multiple ' 'nights and prepare a directory for the KOA.', @@ -694,8 +693,6 @@ def get_parser(cls, width=None): 'F| value are skipped, else all wavelength rms values are accepted.\n' 'F| refframe Perform reference frame correction prior to coadding.\n' f'F| Options are {pypeitpar.WavelengthSolutionPar.valid_reference_frames()}. Defaults to None.\n' - 'F| chk_version If true, spec1ds and archival sensfuncs must match the currently\n' - 'F| supported versions. If false (the default) version numbers are not checked.\n' '\n' 'F|spec1d read\n' 'F|\n' @@ -722,7 +719,7 @@ def get_parser(cls, width=None): parser.add_argument("--wv_rms_thresh", type=float, default = None, help=blank_par.descr['wv_rms_thresh']) parser.add_argument("--refframe", type=str, default = None, choices = pypeitpar.WavelengthSolutionPar.valid_reference_frames(), help=blank_par.descr['refframe']) - parser.add_argument('--chk_version', action = 'store_true', help=blank_par.descr['chk_version']) + parser.add_argument('--chk_version', action = 'store_true', help=blank_pypar['rdx'].descr['chk_version']) parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename collate_1d_YYYYMMDD-HHMM.log') diff --git a/pypeit/scripts/edge_inspector.py b/pypeit/scripts/edge_inspector.py index ac942e640b..c0cf4c6d12 100644 --- a/pypeit/scripts/edge_inspector.py +++ b/pypeit/scripts/edge_inspector.py @@ -17,6 +17,8 @@ def get_parser(cls, width=None): width=width) parser.add_argument('trace_file', type=str, default=None, help='PypeIt Edges file [e.g. Edges_A_0_DET01.fits.gz]') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod @@ -27,10 +29,12 @@ def main(args): from pypeit import edgetrace from pypeit.core.gui import edge_inspector + chk_version = not args.try_old + # Set the file name to the full path trace_file = Path(args.trace_file).resolve() # Load - edges = edgetrace.EdgeTraceSet.from_file(trace_file) + edges = edgetrace.EdgeTraceSet.from_file(trace_file, chk_version=chk_version) # Inspector object pointer = edge_inspector.EdgeInspectorGUI(edges) # Run. Ends when window is closed diff --git a/pypeit/scripts/flux_calib.py b/pypeit/scripts/flux_calib.py index 2405fd0cdc..8155da6a51 100644 --- a/pypeit/scripts/flux_calib.py +++ b/pypeit/scripts/flux_calib.py @@ -64,15 +64,19 @@ def get_parser(cls, width=None): parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename flux_calib_YYYYMMDD-HHMM.log') - # parser.add_argument("--plot", default=False, action="store_true", # help="Show the sensitivity function?") + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod def main(args): """ Runs fluxing steps """ + + chk_version = not args.try_old + # Set the verbosity, and create a logfile if verbosity == 2 msgs.set_logfile_and_verbosity('flux_calib', args.verbosity) @@ -109,7 +113,8 @@ def main(args): 'Run pypeit_flux_calib --help for information on the format') # Instantiate - fluxcalibrate.flux_calibrate(fluxFile.filenames, sensfiles, par=par['fluxcalib']) + fluxcalibrate.flux_calibrate(fluxFile.filenames, sensfiles, par=par['fluxcalib'], + chk_version=chk_version) msgs.info('Flux calibration complete') return 0 diff --git a/pypeit/scripts/identify.py b/pypeit/scripts/identify.py index 8102b4f32b..f69cb5f9f1 100644 --- a/pypeit/scripts/identify.py +++ b/pypeit/scripts/identify.py @@ -42,6 +42,8 @@ def get_parser(cls, width=None): parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename identify_YYYYMMDD-HHMM.log') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod @@ -58,11 +60,13 @@ def main(args): from pypeit import slittrace from pypeit.images.buildimage import ArcImage + chk_version = not args.try_old + # Set the verbosity, and create a logfile if verbosity == 2 msgs.set_logfile_and_verbosity('identify', args.verbosity) # Load the Arc file - msarc = ArcImage.from_file(args.arc_file) + msarc = ArcImage.from_file(args.arc_file, chk_version=chk_version) # Load the spectrograph spec = load_spectrograph(msarc.PYP_SPEC) @@ -78,13 +82,13 @@ def main(args): par['lamps'] = lamps # Load the slits - slits = slittrace.SlitTraceSet.from_file(args.slits_file) + slits = slittrace.SlitTraceSet.from_file(args.slits_file, chk_version=chk_version) # Reset the mask slits.mask = slits.mask_init # Check if a solution exists solnname = WaveCalib.construct_file_name(msarc.calib_key, calib_dir=msarc.calib_dir) - wv_calib = WaveCalib.from_file(solnname) \ + wv_calib = WaveCalib.from_file(solnname, chk_version=chk_version) \ if os.path.exists(solnname) and args.solution else None # Load the calibration frame (if it exists and is desired). Bad-pixel mask diff --git a/pypeit/scripts/multislit_flexure.py b/pypeit/scripts/multislit_flexure.py index be36bc47c6..98e836196b 100644 --- a/pypeit/scripts/multislit_flexure.py +++ b/pypeit/scripts/multislit_flexure.py @@ -95,12 +95,12 @@ def get_parser(cls, width=None): return parser @staticmethod - def main(pargs): + def main(args): from astropy.io import fits # Load the file - flexFile = inputfiles.FlexureFile.from_file(pargs.flex_file) + flexFile = inputfiles.FlexureFile.from_file(args.flex_file) # Read in spectrograph from spec1dfile header header = fits.getheader(flexFile.filenames[0]) @@ -141,8 +141,7 @@ def main(pargs): # Write msgs.info("Write to disk") - mdFlex.to_file(pargs.outroot+root+'.fits', - overwrite=pargs.clobber) + mdFlex.to_file(args.outroot+root+'.fits', overwrite=args.clobber) # Apply?? diff --git a/pypeit/scripts/parse_slits.py b/pypeit/scripts/parse_slits.py index e11d1785f2..c150032a20 100644 --- a/pypeit/scripts/parse_slits.py +++ b/pypeit/scripts/parse_slits.py @@ -41,7 +41,7 @@ def print_slits(slits): this_flags = [] if slits.mask[slit_idx] != 0: for key in bitmask.keys(): - if bitmask.flagged(slits.mask[slit_idx], key): + if bitmask.flagged(slits.mask[slit_idx], flag=key): this_flags += [key] allflags[slit_idx] = ', '.join(this_flags) @@ -58,14 +58,18 @@ def get_parser(cls, width=None): parser = super().get_parser(description='Print info on slits from a input file', width=width) parser.add_argument('input_file', type=str, help='Either a spec2D or Slits filename') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod - def main(pargs): + def main(args): + + chk_version = not args.try_old # What kind of file are we?? - hdul = fits.open(pargs.input_file) + hdul = fits.open(args.input_file) head0 = hdul[0].header head1 = hdul[1].header file_type = None @@ -79,12 +83,12 @@ def main(pargs): msgs.error("Bad file type input!") if file_type == 'Slits': - slits = slittrace.SlitTraceSet.from_file(pargs.input_file, chk_version=False) + slits = slittrace.SlitTraceSet.from_file(args.input_file, chk_version=chk_version) print('') print_slits(slits) elif file_type == 'AllSpec2D': - allspec2D = spec2dobj.AllSpec2DObj.from_fits(pargs.input_file, chk_version=False) + allspec2D = spec2dobj.AllSpec2DObj.from_fits(args.input_file, chk_version=chk_version) # Loop on Detectors for det in allspec2D.detectors: print('') diff --git a/pypeit/scripts/print_bpm.py b/pypeit/scripts/print_bpm.py new file mode 100644 index 0000000000..2d9b2f0937 --- /dev/null +++ b/pypeit/scripts/print_bpm.py @@ -0,0 +1,109 @@ +""" +This script prints a user-friendly description of the bad pixel mask +based on a spec2d file. + +.. include common links, assuming primary doc root is up one directory +.. include:: ../include/links.rst +""" +from IPython import embed + +from astropy.io import fits + +from pypeit import __version__ +from pypeit import msgs, spec2dobj +from pypeit.images.detector_container import DetectorContainer +from pypeit.images.imagebitmask import ImageBitMask +from pypeit.pypmsgs import PypeItDataModelError +from pypeit.scripts import scriptbase + + +class PrintBPM(scriptbase.ScriptBase): + + @classmethod + def get_parser(cls, width=None): + parser = super().get_parser(description='Print out an informative description of a ' + 'bad pixel masked value. Usually, you should ' + 'run pypeit_show_2dspec --showmask first to ' + 'see the bad pixel mask values. Then, call this ' + 'script with the BPM value that you want to find' + 'more information about.', + width=width) + + parser.add_argument('bit', type=int, default=None, help='Bad pixel mask value to describe in plain text') + parser.add_argument('--file', type=str, default=None, help='PypeIt spec2d file to use for the description' + '(optional). If provided, the bitmask contained ' + 'in the spec2d file will be used to describe the ' + 'bad pixel mask value. If not provided, the default ' + 'pypeit bad pixel mask will be used.') + parser.add_argument('--det', default='1', type=str, + help='Detector name or number. If a number, the name is constructed ' + 'assuming the reduction is for a single detector. If a string, ' + 'it must match the name of the detector object (e.g., DET01 for ' + 'a detector, MSC01 for a mosaic). This is not required, and the ' + 'value is acceptable. Default is 1.') + return parser + + @staticmethod + def main(args): + + # Convert the integer bitmask value to a list of binary numbers + binvals = [int(x) for x in bin(args.bit)[2:]][::-1] + + if args.file is None: + msgs.info("Using the default PypeIt bad pixel mask.") + # Generate an Image BitMask object + bpm = ImageBitMask() + descr = bpm.descr + else: + # Read the spec2d file + msgs.info("Using the bad pixel mask from the following spec2d file:" + msgs.newline() + f"{args.file}.") + spec2d_file = args.file + + # Parse the detector name + try: + det = int(args.det) + except: + detname = args.det + else: + detname = DetectorContainer.get_name(det) + + # Try to read the Spec2DObj using the current datamodel, but allowing + # for the datamodel version to be different + try: + spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=False) + except PypeItDataModelError: + try: + # Try to get the pypeit version used to write this file + file_pypeit_version = fits.getval(args.file, 'VERSPYP', 0) + except KeyError: + file_pypeit_version = '*unknown*' + msgs.warn(f'Your installed version of PypeIt ({__version__}) cannot be used to parse ' + f'{args.file}, which was reduced using version {file_pypeit_version}. You ' + 'are strongly encouraged to re-reduce your data using this (or, better yet, ' + 'the most recent) version of PypeIt. Script will try to parse only the ' + 'relevant bits from the spec2d file and continue (possibly with more ' + 'limited functionality).') + # Generate an Image BitMask object + msgs.info("Using the default PypeIt bad pixel mask.") + bpm = ImageBitMask() + descr = bpm.descr + else: + bpm = spec2DObj.bpmmask + descr = bpm.bitmask.descr + + # Print the description of the bad pixel mask value + outstr = f"The bad pixel mask value ({args.bit}) corresponds to the following:" \ + + msgs.newline() + msgs.newline() + bitkeys = list(bpm.bits.keys()) + # Pad the bit keys with spaces so that they all have the same length + bitlen = len(max(bitkeys, key=len)) + for i in range(len(binvals)): + if binvals[i] == 1: + outstr += f"* {bitkeys[i].ljust(bitlen)} : {descr[i]}" + msgs.newline() + + # Print the message to the user + msgs.info(outstr) + + # Finally, print out a message to point users to the online documentation + msgs.info("Please see the following website for more information:" + msgs.newline() + + "https://pypeit.readthedocs.io/en/release/out_masks.html") diff --git a/pypeit/scripts/ql.py b/pypeit/scripts/ql.py index f1a2224d3b..2049632882 100644 --- a/pypeit/scripts/ql.py +++ b/pypeit/scripts/ql.py @@ -211,7 +211,8 @@ def generate_sci_pypeitfile(redux_path:str, slitspatnum:str=None, maskID:str=None, boxcar_radius:float=None, - snr_thresh:float=None): + snr_thresh:float=None, + chk_version:bool=True): """ Prepare to reduce the science frames: @@ -350,7 +351,7 @@ def generate_sci_pypeitfile(redux_path:str, # Iterate through each file to find the one with the relevant mask ID. detname = None for sliittrace_file in slittrace_files: - slitTrace = SlitTraceSet.from_file(sliittrace_file) + slitTrace = SlitTraceSet.from_file(sliittrace_file, chk_version=chk_version) if maskID in slitTrace.maskdef_id: detname = slitTrace.detname # Mosaic? @@ -743,6 +744,8 @@ def get_parser(cls, width=None): help='If coadding, adjust the spatial grid sampling by this ' 'factor. For a finer grid, set value to <1.0; for coarser ' 'sampling, set value to >1.0).') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @@ -752,6 +755,8 @@ def main(args): tstart = time.perf_counter() + chk_version = not args.try_old + # Parse the raw files files = get_files(args.raw_files, args.raw_path) if len(files) == 0: @@ -960,7 +965,8 @@ def main(args): slitspatnum=args.slitspatnum, maskID=args.maskID, boxcar_radius=args.boxcar_radius, - snr_thresh=args.snr_thresh) + snr_thresh=args.snr_thresh, + chk_version=chk_version) # Run it pypeIt = pypeit.PypeIt(sci_pypeit_file, reuse_calibs=True) diff --git a/pypeit/scripts/sensfunc.py b/pypeit/scripts/sensfunc.py index d6325f7a03..bc4406711c 100644 --- a/pypeit/scripts/sensfunc.py +++ b/pypeit/scripts/sensfunc.py @@ -179,7 +179,8 @@ def main(args): if args.outfile is None else args.outfile # Instantiate the relevant class for the requested algorithm sensobj = sensfunc.SensFunc.get_instance(args.spec1dfile, outfile, par['sensfunc'], - debug=args.debug) + debug=args.debug, + chk_version=par['rdx']['chk_version']) # Generate the sensfunc sensobj.run() # Write it out to a file, including the new primary FITS header diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index 5a9c13029b..6c54be71f0 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -20,7 +20,7 @@ from pypeit import io from pypeit import utils from pypeit import __version__ -from pypeit.pypmsgs import PypeItError, PypeItDataModelError +from pypeit.pypmsgs import PypeItDataModelError, PypeItBitMaskError from pypeit.display import display from pypeit.images.imagebitmask import ImageBitMask @@ -88,8 +88,15 @@ def get_parser(cls, width=None): help='Restrict plotting to this slit (PypeIt ID notation)') parser.add_argument('--maskID', type=int, default=None, help='Restrict plotting to this maskID') - parser.add_argument('--showmask', default=False, help='Overplot masked pixels', - action='store_true') + parser.add_argument('--showmask', nargs='*', default=None, + help='Include a channel showing the mask. If no arguments are ' + 'provided, the mask bit values are provided directly. You can ' + 'also specify one or more mask flags used to construct an ' + 'image identifying which pixels are flagged by any of these ' + 'issues. E.g., to show pixels flagged by the instrument ' + 'specific bad-pixel mask or cosmic arrays, use --showmask BPM CR ' + '. See https://pypeit.readthedocs.io/en/release/out_masks.html ' + 'for the list of flags.') parser.add_argument('--removetrace', default=False, action='store_true', help='Do not overplot traces in the skysub, sky_resid, and resid ' 'channels') @@ -109,11 +116,15 @@ def get_parser(cls, width=None): help='Do *not* clear all existing tabs') parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod def main(args): + chk_version = not args.try_old + # List only? if args.list: io.fits_open(args.file).info() @@ -134,22 +145,38 @@ def main(args): show_channels = [0,1,2,3] if args.channels is None \ else [int(item) for item in args.channels.split(',')] + # Need to update clear throughout in case only some channels are being displayed + _clear = args.clear + # Try to read the Spec2DObj using the current datamodel, but allowing # for the datamodel version to be different try: - spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=False) - except PypeItDataModelError: + spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=chk_version) + except (PypeItDataModelError, PypeItBitMaskError): try: # Try to get the pypeit version used to write this file file_pypeit_version = fits.getval(args.file, 'VERSPYP', 0) except KeyError: file_pypeit_version = '*unknown*' - msgs.warn(f'Your installed version of PypeIt ({__version__}) cannot be used to parse ' + if chk_version: + msgs_func = msgs.error + addendum = 'To allow the script to attempt to read the data anyway, use the ' \ + '--try_old command-line option. This will first try to simply ' \ + 'ignore the version number. If the datamodels are incompatible ' \ + '(e.g., the new datamodel contains components not in a previous ' \ + 'version), this may not be enough and the script will continue by ' \ + 'trying to parse only the components necessary for use by this ' \ + 'script. In either case, BEWARE that the displayed data may be in ' \ + 'error!' + else: + msgs_func = msgs.warn + addendum = 'The datamodels are sufficiently different that the script will now ' \ + 'try to parse only the components necessary for use by this ' \ + 'script. BEWARE that the displayed data may be in error!' + msgs_func(f'Your installed version of PypeIt ({__version__}) cannot be used to parse ' f'{args.file}, which was reduced using version {file_pypeit_version}. You ' 'are strongly encouraged to re-reduce your data using this (or, better yet, ' - 'the most recent) version of PypeIt. Script will try to parse only the ' - 'relevant bits from the spec2d file and continue (possibly with more ' - 'limited functionality).') + 'the most recent) version of PypeIt. ' + addendum) spec2DObj = None if spec2DObj is None: @@ -243,7 +270,7 @@ def main(args): sobjs = None msgs.warn('Could not find spec1d file: {:s}'.format(spec1d_file) + msgs.newline() + ' No objects were extracted.') - + # TODO: This may be too restrictive, i.e. ignore BADFLTCALIB?? slit_gpm = slit_mask == 0 @@ -265,9 +292,12 @@ def main(args): display.connect_to_ginga(raise_err=True, allow_new=True) # Show the bitmask? - if args.showmask and bpmmask is not None: - viewer, ch_mask = display.show_image(bpmmask, chname='MASK', waveimg=waveimg, - clear=args.clear) + if args.showmask is not None and bpmmask is not None: + _mask = bpmmask.mask if len(args.showmask) == 0 \ + else bpmmask.flagged(flag=args.showmask) + viewer, ch_mask = display.show_image(_mask, chname='MASK', waveimg=waveimg, + clear=_clear) + _clear = False channel_names = [] # SCIIMG @@ -278,7 +308,8 @@ def main(args): chname_sci = args.prefix+f'sciimg-{detname}' # Clear all channels at the beginning viewer, ch_sci = display.show_image(sciimg, chname=chname_sci, waveimg=waveimg, - clear=args.clear, cuts=(cut_min, cut_max)) + clear=_clear, cuts=(cut_min, cut_max)) + _clear=False if sobjs is not None: show_trace(sobjs, detname, viewer, ch_sci) display.show_slits(viewer, ch_sci, left, right, slit_ids=slid_IDs, @@ -294,8 +325,9 @@ def main(args): cut_max = mean + 4.0 * sigma chname_skysub = args.prefix+f'skysub-{detname}' viewer, ch_skysub = display.show_image(image, chname=chname_skysub, - waveimg=waveimg, cuts=(cut_min, cut_max), + waveimg=waveimg, clear=_clear, cuts=(cut_min, cut_max), wcs_match=True) + _clear = False if not args.removetrace and sobjs is not None: show_trace(sobjs, detname, viewer, ch_skysub) display.show_slits(viewer, ch_skysub, left, right, slit_ids=slid_IDs, @@ -328,7 +360,8 @@ def main(args): chname_skyresids = args.prefix+f'sky_resid-{detname}' image = (sciimg - skymodel) * np.sqrt(ivarmodel) * model_gpm.astype(float) viewer, ch_sky_resids = display.show_image(image, chname_skyresids, waveimg=waveimg, - cuts=(-5.0, 5.0)) + clear=_clear, cuts=(-5.0, 5.0)) + _clear = False if not args.removetrace and sobjs is not None: show_trace(sobjs, detname, viewer, ch_sky_resids) display.show_slits(viewer, ch_sky_resids, left, right, slit_ids=slid_IDs, @@ -341,7 +374,8 @@ def main(args): # full model residual map image = (sciimg - skymodel - objmodel) * np.sqrt(ivarmodel) * img_gpm.astype(float) viewer, ch_resids = display.show_image(image, chname=chname_resids, waveimg=waveimg, - cuts=(-5.0, 5.0), wcs_match=True) + clear=_clear, cuts=(-5.0, 5.0), wcs_match=True) + _clear = False if not args.removetrace and sobjs is not None: show_trace(sobjs, detname, viewer, ch_resids) display.show_slits(viewer, ch_resids, left, right, slit_ids=slid_IDs, diff --git a/pypeit/scripts/show_wvcalib.py b/pypeit/scripts/show_wvcalib.py index 78d484b1ac..712234c5e2 100644 --- a/pypeit/scripts/show_wvcalib.py +++ b/pypeit/scripts/show_wvcalib.py @@ -26,25 +26,29 @@ def get_parser(cls, width=None): parser.add_argument("--slit_file", type=str, help="Slit file") parser.add_argument("--is_order", default=False, action="store_true", help="Input slit/order is an order") + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod - def main(pargs, unit_test=False): + def main(args, unit_test=False): """ Shows the spectrum """ from matplotlib import pyplot as plt + chk_version = not args.try_old + # Load - wvcalib = wavecalib.WaveCalib.from_file(pargs.file) - if pargs.slit_file is not None: - slits = slittrace.SlitTraceSet.from_file(pargs.slit_file) + wvcalib = wavecalib.WaveCalib.from_file(args.file, chk_version=chk_version) + if args.slit_file is not None: + slits = slittrace.SlitTraceSet.from_file(args.slit_file, chk_version=chk_version) # Parse - if pargs.is_order: - idx = np.where(slits.ech_order == pargs.slit_order)[0][0] + if args.is_order: + idx = np.where(slits.ech_order == args.slit_order)[0][0] else: - idx = np.where(wvcalib.spat_ids == pargs.slit_order)[0][0] + idx = np.where(wvcalib.spat_ids == args.slit_order)[0][0] # Grab it spec = wvcalib.arc_spectra[:,idx] diff --git a/pypeit/scripts/skysub_regions.py b/pypeit/scripts/skysub_regions.py index f1347f40c8..516d24a691 100644 --- a/pypeit/scripts/skysub_regions.py +++ b/pypeit/scripts/skysub_regions.py @@ -31,6 +31,8 @@ def get_parser(cls, width=None): parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename skysub_regions_YYYYMMDD-HHMM.log') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod @@ -46,6 +48,8 @@ def main(args): from pypeit.images.detector_container import DetectorContainer from pypeit.edgetrace import EdgeTraceSet + chk_version = not args.try_old + # Parse the detector name try: det = int(args.det) @@ -55,7 +59,7 @@ def main(args): detname = DetectorContainer.get_name(det) # Load it up - spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=True) + spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=chk_version) frame = spec2DObj.sciimg hdr = fits.open(args.file)[0].header fname = hdr['FILENAME'] diff --git a/pypeit/scripts/tellfit.py b/pypeit/scripts/tellfit.py index 20ee30ae75..7b908af6d8 100644 --- a/pypeit/scripts/tellfit.py +++ b/pypeit/scripts/tellfit.py @@ -72,6 +72,10 @@ def get_parser(cls, width=None): parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename tellfit_YYYYMMDD-HHMM.log') + parser.add_argument('--chk_version', default=False, action='store_true', + help='Ensure the datamodels are from the current PypeIt version. ' + 'By default (consistent with previous functionality) this is ' + 'not enforced and crashes may ensue ...') return parser @staticmethod @@ -126,8 +130,8 @@ def main(args): if par['sensfunc']['IR']['telgridfile'] is not None: par['telluric']['telgridfile'] = par['sensfunc']['IR']['telgridfile'] else: - par['telluric']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' - msgs.warn(f"No telluric grid file given. Using {par['telluric']['telgridfile']}.") + par['telluric']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' + msgs.warn(f"No telluric file given. Using PCA method with {par['telluric']['telgridfile']}.") # Checks if par['telluric']['telgridfile'] is None: @@ -155,15 +159,20 @@ def main(args): par['telluric']['pca_file'], par['telluric']['redshift'], modelfile, outfile, npca=par['telluric']['npca'], + teltype=par['telluric']['teltype'], tell_npca=par['telluric']['tell_npca'], pca_lower=par['telluric']['pca_lower'], pca_upper=par['telluric']['pca_upper'], bounds_norm=par['telluric']['bounds_norm'], tell_norm_thresh=par['telluric']['tell_norm_thresh'], only_orders=par['telluric']['only_orders'], bal_wv_min_max=par['telluric']['bal_wv_min_max'], + pix_shift_bounds=par['telluric']['pix_shift_bounds'], maxiter=par['telluric']['maxiter'], + popsize=par['telluric']['popsize'], + tol=par['telluric']['tol'], debug_init=args.debug, disp=args.debug, - debug=args.debug, show=args.plot) + debug=args.debug, show=args.plot, + chk_version=args.chk_version) elif par['telluric']['objmodel']=='star': TelStar = telluric.star_telluric(args.spec1dfile, par['telluric']['telgridfile'], modelfile, outfile, @@ -175,14 +184,19 @@ def main(args): model=par['telluric']['model'], polyorder=par['telluric']['polyorder'], only_orders=par['telluric']['only_orders'], + teltype=par['telluric']['teltype'], tell_npca=par['telluric']['tell_npca'], mask_hydrogen_lines=par['sensfunc']['mask_hydrogen_lines'], mask_helium_lines=par['sensfunc']['mask_helium_lines'], hydrogen_mask_wid=par['sensfunc']['hydrogen_mask_wid'], delta_coeff_bounds=par['telluric']['delta_coeff_bounds'], minmax_coeff_bounds=par['telluric']['minmax_coeff_bounds'], + pix_shift_bounds=par['telluric']['pix_shift_bounds'], maxiter=par['telluric']['maxiter'], + popsize=par['telluric']['popsize'], + tol=par['telluric']['tol'], debug_init=args.debug, disp=args.debug, - debug=args.debug, show=args.plot) + debug=args.debug, show=args.plot, + chk_version=args.chk_version) elif par['telluric']['objmodel']=='poly': TelPoly = telluric.poly_telluric(args.spec1dfile, par['telluric']['telgridfile'], modelfile, outfile, @@ -190,14 +204,19 @@ def main(args): func=par['telluric']['func'], model=par['telluric']['model'], polyorder=par['telluric']['polyorder'], + teltype=par['telluric']['teltype'], tell_npca=par['telluric']['tell_npca'], fit_wv_min_max=par['telluric']['fit_wv_min_max'], mask_lyman_a=par['telluric']['mask_lyman_a'], delta_coeff_bounds=par['telluric']['delta_coeff_bounds'], minmax_coeff_bounds=par['telluric']['minmax_coeff_bounds'], only_orders=par['telluric']['only_orders'], + pix_shift_bounds=par['telluric']['pix_shift_bounds'], maxiter=par['telluric']['maxiter'], + popsize=par['telluric']['popsize'], + tol=par['telluric']['tol'], debug_init=args.debug, disp=args.debug, - debug=args.debug, show=args.plot) + debug=args.debug, show=args.plot, + chk_version=args.chk_version) else: msgs.error("Object model is not supported yet. Must be 'qso', 'star', or 'poly'.") diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index 33937efba5..e7ccadad1b 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -128,7 +128,8 @@ class SensFunc(datamodel.DataContainer): 'steps', 'splice_multi_det', 'meta_spec', - 'std_dict' + 'std_dict', + 'chk_version' ] _algorithm = None @@ -189,20 +190,21 @@ def empty_sensfunc_table(norders, nspec, nspec_in, ncoeff=1): table.Column(name='SENS_FLUXED_STD_MASK', dtype=bool, length=norders, shape=(nspec_in,), description='The good pixel mask for the fluxed standard star spectrum ')]) - - # Superclass factory method generates the subclass instance @classmethod - def get_instance(cls, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): + def get_instance(cls, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False, + chk_version=True): """ Instantiate the relevant subclass based on the algorithm provided in ``par``. """ return next(c for c in cls.__subclasses__() - if c.__name__ == f"{par['algorithm']}SensFunc")(spec1dfile, sensfile, par, - par_fluxcalib=par_fluxcalib, debug=debug) + if c.__name__ == f"{par['algorithm']}SensFunc")( + spec1dfile, sensfile, par, par_fluxcalib=par_fluxcalib, debug=debug, + chk_version=chk_version) - def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): + def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False, + chk_version=True): # Instantiate as an empty DataContainer super().__init__() @@ -211,6 +213,7 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): self.spec1df = spec1dfile self.sensfile = sensfile self.par = par + self.chk_version = chk_version # Spectrograph header = fits.getheader(self.spec1df) self.PYP_SPEC = header['PYP_SPEC'] @@ -220,7 +223,8 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): # spectrograph objects with configuration specific information from # spec1d files. self.spectrograph.dispname = header['DISPNAME'] - self.par_fluxcalib = self.spectrograph.default_pypeit_par()['fluxcalib'] if par_fluxcalib is None else par_fluxcalib + self.par_fluxcalib = self.spectrograph.default_pypeit_par()['fluxcalib'] \ + if par_fluxcalib is None else par_fluxcalib # Set the algorithm in the datamodel self.algorithm = self.__class__._algorithm @@ -238,17 +242,21 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): self.splice_multi_det = True if self.par['multi_spec_det'] is not None else False # Read in the Standard star data - self.sobjs_std = specobjs.SpecObjs.from_fitsfile(self.spec1df).get_std(multi_spec_det=self.par['multi_spec_det']) + self.sobjs_std = specobjs.SpecObjs.from_fitsfile( + self.spec1df, chk_version=self.chk_version).get_std( + multi_spec_det=self.par['multi_spec_det']) if self.sobjs_std is None: - msgs.error('There is a problem with your standard star spec1d file: {:s}'.format(self.spec1df)) + msgs.error(f'There is a problem with your standard star spec1d file: {self.spec1df}') # Unpack standard - wave, counts, counts_ivar, counts_mask, trace_spec, trace_spat, self.meta_spec, header = self.sobjs_std.unpack_object(ret_flam=False) + wave, counts, counts_ivar, counts_mask, trace_spec, trace_spat, self.meta_spec, header \ + = self.sobjs_std.unpack_object(ret_flam=False) # Compute the blaze function # TODO Make the blaze function optional - log10_blaze_function = self.compute_blaze(wave, trace_spec, trace_spat, par['flatfile']) if par['flatfile'] is not None else None + log10_blaze_function = self.compute_blaze(wave, trace_spec, trace_spat, par['flatfile']) \ + if par['flatfile'] is not None else None # Perform any instrument tweaks wave_twk, counts_twk, counts_ivar_twk, counts_mask_twk, log10_blaze_function_twk = \ @@ -269,7 +277,8 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): star_mag=self.par['star_mag'], ra=star_ra, dec=star_dec) - def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, min_blaze_value=1e-3, debug=False): + def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, + min_blaze_value=1e-3, debug=False): """ Compute the blaze function from a flat field image. @@ -293,9 +302,7 @@ def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, `numpy.ndarray`_: The log10 blaze function. Shape = (nspec, norddet) if norddet > 1, else shape = (nspec,) """ - - - flatImages = flatfield.FlatImages.from_file(flatfile) + flatImages = flatfield.FlatImages.from_file(flatfile, chk_version=self.chk_version) pixelflat_raw = flatImages.pixelflat_raw pixelflat_norm = flatImages.pixelflat_norm @@ -309,7 +316,8 @@ def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, mask_box = (pixmsk != pixtot) & np.isfinite(wave) & (wave > 0.0) # TODO This is ugly and redundant with spec_atleast_2d, but the order of operations compels me to do it this way - blaze_function = (np.clip(flux_box*mask_box, 1e-3, 1e9)).reshape(-1,1) if flux_box.ndim == 1 else flux_box*mask_box + blaze_function = (np.clip(flux_box*mask_box, 1e-3, 1e9)).reshape(-1,1) \ + if flux_box.ndim == 1 else flux_box*mask_box wave_debug = wave.reshape(-1,1) if wave.ndim == 1 else wave log10_blaze_function = np.zeros_like(blaze_function) norddet = log10_blaze_function.shape[1] @@ -410,13 +418,9 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): """ # Run the default parser to get most of the data. This correctly parses # everything except for the Telluric.model data table. - d, version_passed, type_passed, parsed_hdus = super()._parse(hdu, allow_subclasses=True) - if not type_passed: - msgs.error('The HDU(s) cannot be parsed by a {0} object!'.format(cls.__name__)) - if not version_passed: - _f = msgs.error if chk_version else msgs.warn - _f('Current version of {0} object in code (v{1})'.format(cls.__name__, cls.version) - + ' does not match version used to write your HDU(s)!') + d, version_passed, type_passed, parsed_hdus = cls._parse(hdu, allow_subclasses=True) + # Check + cls._check_parsed(version_passed, type_passed, chk_version=chk_version) # Load the telluric model, if it exists if 'TELLURIC' in [h.name for h in hdu]: @@ -464,9 +468,11 @@ def flux_std(self): # TODO assign this to the data model # Unpack the fluxed standard - _wave, _flam, _flam_ivar, _flam_mask, _, _, _, _ = self.sobjs_std.unpack_object(ret_flam=True) + _wave, _flam, _flam_ivar, _flam_mask, _, _, _, _ \ + = self.sobjs_std.unpack_object(ret_flam=True) # Reshape to 2d arrays - wave, flam, flam_ivar, flam_mask, _, _, _ = utils.spec_atleast_2d(_wave, _flam, _flam_ivar, _flam_mask) + wave, flam, flam_ivar, flam_mask, _, _, _ \ + = utils.spec_atleast_2d(_wave, _flam, _flam_ivar, _flam_mask) # Store in the sens table self.sens['SENS_FLUXED_STD_WAVE'] = wave.T self.sens['SENS_FLUXED_STD_FLAM'] = flam.T @@ -792,7 +798,7 @@ def write_QA(self): @classmethod - def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): + def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True, chk_version=True): """ Get the weights based on the sensfunc @@ -804,13 +810,17 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): norders, nexp) or (nspec, norders). debug (bool): default=False show the weights QA + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict + version checking to ensure a valid file. If False, the code + will try to keep going, but this may lead to faults and quiet + failures. User beware! Returns: `numpy.ndarray`_: sensfunc weights evaluated on the input waves - wavelength grid + wavelength grid, shape = same as waves """ - sens = cls.from_file(sensfile) - # wave, zeropoint, meta_table, out_table, header_sens = sensfunc.SensFunc.load(sensfile) + sens = cls.from_file(sensfile, chk_version=chk_version) if waves.ndim == 2: nspec, norder = waves.shape @@ -819,6 +829,10 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): elif waves.ndim == 3: nspec, norder, nexp = waves.shape waves_stack = waves + elif waves.ndim == 1: + nspec = waves.size + norder, nexp = 1, 1 + waves_stack = np.reshape(waves, (nspec, 1, 1)) else: msgs.error('Unrecognized dimensionality for waves') @@ -839,8 +853,11 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): coadd.weights_qa(utils.echarr_to_echlist(waves_stack)[0], utils.echarr_to_echlist(weights_stack)[0], utils.echarr_to_echlist(waves_stack > 1.0)[0], title='sensfunc_weights') + # Reshape to be the same size/dimension as the input spectrum if waves.ndim == 2: weights_stack = np.reshape(weights_stack, (nspec, norder)) + elif waves.ndim == 1: + weights_stack = np.reshape(weights_stack, (nspec)) return weights_stack @@ -885,7 +902,10 @@ def compute_zeropoint(self): ech_orders=self.meta_spec['ECH_ORDERS'], resln_guess=self.par['IR']['resln_guess'], resln_frac_bounds=self.par['IR']['resln_frac_bounds'], + pix_shift_bounds=self.par['IR']['pix_shift_bounds'], sn_clip=self.par['IR']['sn_clip'], + teltype=self.par['IR']['teltype'], + tell_npca=self.par['IR']['tell_npca'], mask_hydrogen_lines=self.par['mask_hydrogen_lines'], maxiter=self.par['IR']['maxiter'], lower=self.par['IR']['lower'], @@ -1009,8 +1029,10 @@ class UVISSensFunc(SensFunc): _algorithm = 'UVIS' """Algorithm used for the sensitivity calculation.""" - def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): - super().__init__(spec1dfile, sensfile, par, par_fluxcalib=par_fluxcalib, debug=debug) + def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False, + chk_version=True): + super().__init__(spec1dfile, sensfile, par, par_fluxcalib=par_fluxcalib, debug=debug, + chk_version=chk_version) # Add some cards to the meta spec. These should maybe just be added # already in unpack object diff --git a/pypeit/setup_gui/dialog_helpers.py b/pypeit/setup_gui/dialog_helpers.py index f257c7a7c2..54e6c2749d 100644 --- a/pypeit/setup_gui/dialog_helpers.py +++ b/pypeit/setup_gui/dialog_helpers.py @@ -2,7 +2,7 @@ from __future__ import annotations import enum -from typing import Optional +from typing import Optional,Union from pathlib import Path from dataclasses import dataclass @@ -93,7 +93,7 @@ def __init__(self, caption : str, file_mode : QFileDialog.FileMode, file_type : Optional[FileType] =None, - default_file : Optional[str|Path] = None, + default_file : Optional[Union[str,Path]] = None, history : Optional[QStringListModel] = None, save : bool = False, ask_for_all : bool = False): diff --git a/pypeit/setup_gui/model.py b/pypeit/setup_gui/model.py index 639613e1ba..ca423ce50c 100644 --- a/pypeit/setup_gui/model.py +++ b/pypeit/setup_gui/model.py @@ -14,6 +14,7 @@ import numpy as np import astropy.table import io +import typing from pathlib import Path from functools import partial from qtpy.QtCore import QAbstractTableModel, QAbstractProxyModel, QAbstractItemModel, QAbstractListModel, QModelIndex, Qt, Signal, QObject, QThread, QStringListModel @@ -218,7 +219,7 @@ class PypeItMetadataModel(QAbstractTableModel): metadata: The PypeItMetaData object being wrapped. If this is None, the model is in a "NEW" state. """ - def __init__(self, metadata : PypeItMetaData | None): + def __init__(self, metadata : typing.Union[PypeItMetaData, None]): super().__init__() self.metadata = metadata @@ -448,7 +449,7 @@ def getDefaultColumns(self): else: return ['filename', 'frametype', 'ra', 'dec', 'target', 'dispname', 'decker', 'binning', 'mjd', 'airmass', 'exptime'] - def getStringColumnSize(self, colname: str) -> int | None: + def getStringColumnSize(self, colname: str) -> typing.Union[int,None]: """ Return the maximum size of a string column. diff --git a/pypeit/setup_gui/text_viewer.py b/pypeit/setup_gui/text_viewer.py index dc823e61d1..1f527f4328 100644 --- a/pypeit/setup_gui/text_viewer.py +++ b/pypeit/setup_gui/text_viewer.py @@ -1,6 +1,6 @@ from pathlib import Path import io -from typing import Optional +from typing import Optional,Union from qtpy.QtWidgets import QHBoxLayout, QVBoxLayout, QFileDialog, QWidget, QPlainTextEdit, QPushButton @@ -29,7 +29,7 @@ class TextViewerWindow(QWidget): """Signal sent when the window is closed.""" - def __init__(self, title : str, width : int, height : int, text_stream : io.TextIOBase, start_at_top : bool, filename: Optional[str|Path] = None, file_type : FileType = FileType("Text Files", ".txt")): + def __init__(self, title : str, width : int, height : int, text_stream : io.TextIOBase, start_at_top : bool, filename: Optional[Union[str,Path]] = None, file_type : FileType = FileType("Text Files", ".txt")): super().__init__() self._text_stream = text_stream self._file_type = file_type diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 5554ac753b..4cec5b5916 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -12,11 +12,12 @@ import numpy as np from astropy.table import Table -from astropy.coordinates import SkyCoord, Angle +from astropy.coordinates import SkyCoord from astropy import units from astropy.stats import sigma_clipped_stats -from scipy.interpolate import RegularGridInterpolator, interp1d +from astropy.io import fits +from pypeit.pypmsgs import PypeItBitMaskError from pypeit import msgs from pypeit import datamodel from pypeit import calibframe @@ -31,8 +32,17 @@ class SlitTraceBitMask(BitMask): """ version = '1.0.1' + # TODO: Consider using a unique bit prefix for when the bits are written to + # the header, like so: + # + # prefix = 'SLITB' + # + # It's not necessary, though. + def __init__(self): - # Only ever append new bits (and don't remove old ones) + # !!!!!!!!!! + # IMPORTANT: Only ever *append* new bits, and don't remove old ones + # !!!!!!!!!! mask = dict([ ('SHORTSLIT', 'Slit formed by left and right edge is too short. Not ignored for flexure'), ('BOXSLIT', 'Slit formed by left and right edge is valid (large enough to be a valid ' @@ -40,6 +50,7 @@ def __init__(self): ('USERIGNORE', 'User has specified to ignore this slit. Not ignored for flexure.'), ('BADWVCALIB', 'Wavelength calibration failed for this slit'), ('BADTILTCALIB', 'Tilts analysis failed for this slit'), + ('BADALIGNCALIB', 'Alignment analysis failed for this slit'), ('SKIPFLATCALIB', 'Flat field generation failed for this slit. Skip flat fielding'), ('BADFLATCALIB', 'Flat field generation failed for this slit. Ignore it fully.'), ('BADREDUCE', 'Reduction failed for this slit'), # THIS IS DEPRECATED (we may remove in v1.13) BUT STAYS HERE TO ALLOW FOR BACKWARDS COMPATIBILITY @@ -57,7 +68,7 @@ def exclude_for_reducing(self): def exclude_for_flexure(self): # Ignore these flags when performing a flexure calculation # Currently they are *all* of the flags.. - return ['SHORTSLIT', 'USERIGNORE', 'BADWVCALIB', 'BADTILTCALIB', + return ['SHORTSLIT', 'USERIGNORE', 'BADWVCALIB', 'BADTILTCALIB', 'BADALIGNCALIB', 'SKIPFLATCALIB', 'BADFLATCALIB', 'BADSKYSUB', 'BADEXTRACT'] @@ -83,7 +94,7 @@ class SlitTraceSet(calibframe.CalibFrame): calib_file_format = 'fits.gz' """File format for the calibration frame file.""" - version = '1.1.4' + version = '1.1.5' """SlitTraceSet data model version.""" bitmask = SlitTraceBitMask() @@ -114,13 +125,18 @@ class SlitTraceSet(calibframe.CalibFrame): descr='Slit ID number from SPAT measured at half way point.'), 'maskdef_id': dict(otype=np.ndarray, atype=(int,np.integer), descr='Slit ID number slitmask'), - 'maskdef_designtab': dict(otype=Table, descr='Table with slitmask design and object info'), + 'maskdef_designtab': dict(otype=Table, + descr='Table with slitmask design and object info'), 'maskfile': dict(otype=str, descr='Data file that yielded the slitmask info'), - 'maskdef_posx_pa': dict(otype=float, descr='PA that aligns with spatial dimension of the detector'), - 'maskdef_offset': dict(otype=float, descr='Slitmask offset (pixels) from position expected ' - 'by the slitmask design'), + 'maskdef_posx_pa': dict(otype=float, + descr='PA that aligns with spatial dimension of the ' + 'detector'), + 'maskdef_offset': dict(otype=float, + descr='Slitmask offset (pixels) from position expected ' + 'by the slitmask design'), 'maskdef_objpos': dict(otype=np.ndarray, atype=np.floating, - descr='Object positions expected by the slitmask design [relative pixels]'), + descr='Object positions expected by the slitmask design ' + '[relative pixels]'), 'maskdef_slitcen': dict(otype=np.ndarray, atype=np.floating, descr='Slit centers expected by the slitmask design'), 'ech_order': dict(otype=np.ndarray, atype=(int,np.integer), @@ -151,13 +167,12 @@ class SlitTraceSet(calibframe.CalibFrame): 'mask': dict(otype=np.ndarray, atype=np.integer, descr='Bit mask for slits (fully good slits have 0 value). Shape ' 'is Nslits.'), - 'slitbitm': dict(otype=str, descr='List of BITMASK keys from SlitTraceBitMask'), 'specmin': dict(otype=np.ndarray, atype=np.floating, - descr='Minimum spectral position (pixel units) allowed for each slit/order. ' - 'Shape is Nslits.'), + descr='Minimum spectral position (pixel units) allowed for each ' + 'slit/order. Shape is Nslits.'), 'specmax': dict(otype=np.ndarray, atype=np.floating, - descr='Maximum spectral position (pixel units) allowed for each slit/order. ' - 'Shape is Nslits.')} + descr='Maximum spectral position (pixel units) allowed for each ' + 'slit/order. Shape is Nslits.')} """Provides the class data model.""" # TODO: Allow tweaked edges to be arguments? @@ -168,7 +183,7 @@ def __init__(self, left_init, right_init, pypeline, detname=None, nspec=None, ns pad=0, spat_id=None, maskdef_id=None, maskdef_designtab=None, maskfile=None, maskdef_posx_pa=None, maskdef_offset=None, maskdef_objpos=None, maskdef_slitcen=None, ech_order=None, nslits=None, left_tweak=None, - right_tweak=None, center=None, mask=None, slitbitm=None): + right_tweak=None, center=None, mask=None): # Instantiate the DataContainer args, _, _, values = inspect.getargvalues(inspect.currentframe()) @@ -240,21 +255,29 @@ def _validate(self): self.mask_init = np.atleast_1d(self.mask_init) self.specmin = np.atleast_1d(self.specmin) self.specmax = np.atleast_1d(self.specmax) - if self.slitbitm is None: - self.slitbitm = ','.join(list(self.bitmask.keys())) - else: - # Validate -- All of the keys must be present and in current order, but new ones can exist - bitms = self.slitbitm.split(',') - curbitm = list(self.bitmask.keys()) - for kk, bit in enumerate(bitms): - if curbitm[kk] != bit: - msgs.error("Input BITMASK keys differ from current data model!") - # Update to current, no matter what - self.slitbitm = ','.join(list(self.bitmask.keys())) # Mask if self.mask is None: self.mask = self.mask_init.copy() + def _base_header(self, hdr=None): + """ + Construct the baseline header for all HDU extensions. + + This appends the :class:`SlitTraceBitMask` data to the supplied header. + + Args: + hdr (`astropy.io.fits.Header`_, optional): + Baseline header for additional data. If None, set by + :func:`pypeit.io.initialize_header()`. + + Returns: + `astropy.io.fits.Header`_: Header object to include in + all HDU extensions. + """ + _hdr = super()._base_header(hdr=hdr) + self.bitmask.to_header(_hdr) + return _hdr + def _bundle(self): """ Bundle the data in preparation for writing to a fits file. @@ -298,6 +321,48 @@ def _parse(cls, hdu, hdu_prefix=None, **kwargs): except KeyError: return super()._parse(hdu, ext='SLITS', transpose_table_arrays=True) + @classmethod + def from_hdu(cls, hdu, chk_version=True, **kwargs): + """ + Instantiate the object from an HDU extension. + + This overrides the base-class method, only to add checks (or not) for + the bitmask. + + Args: + hdu (`astropy.io.fits.HDUList`_, `astropy.io.fits.ImageHDU`_, `astropy.io.fits.BinTableHDU`_): + The HDU(s) with the data to use for instantiation. + chk_version (:obj:`bool`, optional): + If True, raise an error if the datamodel version or + type check failed. If False, throw a warning only. + **kwargs: + Passed directly to :func:`_parse`. + """ + # Run the default parser + d, version_passed, type_passed, parsed_hdus = cls._parse(hdu, **kwargs) + # Check + cls._check_parsed(version_passed, type_passed, chk_version=chk_version) + + # Instantiate + self = super().from_dict(d=d) + + # Calibration frame attributes + # NOTE: If multiple HDUs are parsed, this assumes that the information + # necessary to set all the calib internals is always in *every* header. + # BEWARE! + self.calib_keys_from_header(hdu[parsed_hdus[0]].header) + + # Check the bitmasks. Bits should have been written to *any* header + # associated with the object + hdr = hdu[parsed_hdus[0]].header if isinstance(hdu, fits.HDUList) else hdu.header + hdr_bitmask = BitMask.from_header(hdr) + if chk_version and hdr_bitmask.bits != self.bitmask.bits: + msgs.error('The bitmask in this fits file appear to be out of date! Recreate this ' + 'file by re-running the relevant script or set chk_version=False.', + cls='PypeItBitMaskError') + + return self + def init_tweaked(self): """ Initialize the tweaked slits. @@ -345,7 +410,8 @@ def slitord_id(self): @property def slitord_txt(self): """ - Return string indicating if the logs/QA should use "slit" (MultiSlit, SlicerIFU) or "order" (Echelle) + Return string indicating if the logs/QA should use "slit" (MultiSlit, + SlicerIFU) or "order" (Echelle). Returns: str: Either 'slit' or 'order' @@ -383,10 +449,9 @@ def slitord_to_zero(self, slitord): """ if self.pypeline in ['MultiSlit', 'SlicerIFU']: return np.where(self.spat_id == slitord)[0][0] - elif self.pypeline in ['Echelle']: + if self.pypeline == 'Echelle': return np.where(self.ech_order == slitord)[0][0] - else: - msgs.error('Unrecognized Pypeline {:}'.format(self.pypeline)) + msgs.error('Unrecognized Pypeline {:}'.format(self.pypeline)) def get_slitlengths(self, initial=False, median=False): """ @@ -412,11 +477,9 @@ def get_slitlengths(self, initial=False, median=False): """ left, right, _ = self.select_edges(initial=initial) slitlen = right - left - if median is True: - slitlen = np.median(slitlen, axis=1) - return slitlen + return np.median(slitlen, axis=1) if median else slitlen - def get_radec_image(self, wcs, alignSplines, tilts, initial=True, flexure=None): + def get_radec_image(self, wcs, alignSplines, tilts, slice_offset=None, initial=True, flexure=None): """Generate an RA and DEC image for every pixel in the frame NOTE: This function is currently only used for SlicerIFU reductions. @@ -429,6 +492,11 @@ def get_radec_image(self, wcs, alignSplines, tilts, initial=True, flexure=None): transform between detector pixel coordinates and WCS pixel coordinates. tilts : `numpy.ndarray`_ Spectral tilts. + slice_offset : float, optional + Offset to apply to the slice positions. A value of 0.0 means that the + slice positions are the centre of the slits. A value of +/-0.5 means that + the slice positions are at the edges of the slits. If None, the slice_offset + is set to 0.0. initial : bool Select the initial slit edges? flexure : float, optional @@ -447,7 +515,13 @@ def get_radec_image(self, wcs, alignSplines, tilts, initial=True, flexure=None): reference (usually the centre of the slit) and the edges of the slits. Shape is (nslits, 2). """ - msgs.info("Generating an RA/DEC image") + substring = '' if slice_offset is None else f' with slice_offset={slice_offset:.3f}' + msgs.info("Generating an RA/DEC image"+substring) + # Check the input + if slice_offset is None: + slice_offset = 0.0 + if slice_offset < -0.5 or slice_offset > 0.5: + msgs.error(f"Slice offset must be between -0.5 and 0.5. slice_offset={slice_offset}") # Initialise the output raimg = np.zeros((self.nspec, self.nspat)) decimg = np.zeros((self.nspec, self.nspat)) @@ -458,13 +532,14 @@ def get_radec_image(self, wcs, alignSplines, tilts, initial=True, flexure=None): onslit = (slitid_img_init == spatid) onslit_init = np.where(onslit) if self.mask[slit_idx] != 0: - msgs.error(f"Slit {spatid} ({slit_idx+1}/{self.spat_id.size}) is masked. Cannot generate RA/DEC image.") + msgs.error(f'Slit {spatid} ({slit_idx+1}/{self.spat_id.size}) is masked. Cannot ' + 'generate RA/DEC image.') # Retrieve the pixel offset from the central trace evalpos = alignSplines.transform(slit_idx, onslit_init[1], onslit_init[0]) minmax[slit_idx, 0] = np.min(evalpos) minmax[slit_idx, 1] = np.max(evalpos) # Calculate the WCS from the pixel positions - slitID = np.ones(evalpos.size) * slit_idx - wcs.wcs.crpix[0] + slitID = np.ones(evalpos.size) * slit_idx + slice_offset - wcs.wcs.crpix[0] world_ra, world_dec, _ = wcs.wcs_pix2world(slitID, evalpos, tilts[onslit_init]*(self.nspec-1), 0) # Set the RA first and DEC next raimg[onslit] = world_ra.copy() @@ -509,9 +584,8 @@ def select_edges(self, initial=False, flexure=None): # Return return left.copy(), right.copy(), self.mask.copy() - def slit_img(self, pad=None, slitidx=None, initial=False, - flexure=None, - exclude_flag=None, use_spatial=True): + def slit_img(self, pad=None, slitidx=None, initial=False, flexure=None, exclude_flag=None, + use_spatial=True): r""" Construct an image identifying each pixel with its associated slit. @@ -555,8 +629,8 @@ def slit_img(self, pad=None, slitidx=None, initial=False, :attr:`right_init`) are used. To use the nominal edges regardless of the presence of the tweaked edges, set this to True. See :func:`select_edges`. - exclude_flag (:obj:`str`, optional): - Bitmask flag to ignore when masking + exclude_flag (:obj:`str`, :obj:`list`, optional): + One or more bitmask flag names to ignore when masking Warning -- This could conflict with input slitids, i.e. avoid using both use_spatial (bool, optional): If True, use self.spat_id value instead of 0-based indices @@ -588,10 +662,11 @@ def slit_img(self, pad=None, slitidx=None, initial=False, if slitidx is not None: slitidx = np.atleast_1d(slitidx).ravel() else: - bpm = self.mask.astype(bool) - if exclude_flag: - bpm &= np.invert(self.bitmask.flagged(self.mask, flag=exclude_flag)) - slitidx = np.where(np.invert(bpm))[0] + bpm = self.bitmask.flagged(self.mask, and_not=exclude_flag) +# bpm = self.mask.astype(bool) +# if exclude_flag: +# bpm &= np.invert(self.bitmask.flagged(self.mask, flag=exclude_flag)) + slitidx = np.where(np.logical_not(bpm))[0] # TODO: When specific slits are chosen, need to check that the # padding doesn't lead to slit overlap. @@ -749,10 +824,11 @@ def slit_spat_pos(left, right, nspat): def mask_add_missing_obj(self, sobjs, spat_flexure, fwhm, boxcar_rad): """ - Generate new SpecObj and add them into the SpecObjs object for any slits missing the targeted source. + Generate new SpecObj and add them into the SpecObjs object for any slits + missing the targeted source. Args: - sobjs (:class:`pypeit.specobjs.SpecObjs`): + sobjs (:class:`~pypeit.specobjs.SpecObjs`): List of SpecObj that have been found and traced spat_flexure (:obj:`float`): Shifts, in spatial pixels, between this image and SlitTrace @@ -762,16 +838,19 @@ def mask_add_missing_obj(self, sobjs, spat_flexure, fwhm, boxcar_rad): BOX_RADIUS in pixels to be used in the boxcar extraction Returns: - :class:`pypeit.specobjs.SpecObjs`: Updated list of SpecObj that have been found and traced + :class:`~pypeit.specobjs.SpecObjs`: Updated list of SpecObj that have + been found and traced """ msgs.info('Add undetected objects at the expected location from slitmask design.') if fwhm is None: - msgs.error('A FWHM for the optimal extraction must be provided. See `find_fwhm` in `FindObjPar`.') + msgs.error('A FWHM for the optimal extraction must be provided. See `find_fwhm` in ' + '`FindObjPar`.') if self.maskdef_objpos is None: - msgs.error('An array with the object positions expected from slitmask design is missing.') + msgs.error('An array with the object positions expected from slitmask design is ' + 'missing.') if self.maskdef_offset is None: msgs.error('A value for the slitmask offset must be provided.') @@ -783,10 +862,10 @@ def mask_add_missing_obj(self, sobjs, spat_flexure, fwhm, boxcar_rad): else: cut_sobjs = sobjs - # get slits edges init - left_init, _, _ = self.select_edges(initial=True, flexure=spat_flexure) # includes flexure - # get slits edges tweaked - left_tweak, right_tweak, _ = self.select_edges(initial=False, flexure=spat_flexure) # includes flexure + # get slits edges init; includes flexure + left_init, _, _ = self.select_edges(initial=True, flexure=spat_flexure) + # get slits edges tweaked; includes flexure + left_tweak, right_tweak, _ = self.select_edges(initial=False, flexure=spat_flexure) # midpoint in the spectral direction specmid = left_init[:,0].size//2 @@ -1398,7 +1477,7 @@ def mask_flats(self, flatImages): """ # Loop on all the FLATFIELD BPM keys for flag in ['SKIPFLATCALIB', 'BADFLATCALIB']: - bad_flats = self.bitmask.flagged(flatImages.get_bpmflats(), flag) + bad_flats = self.bitmask.flagged(flatImages.get_bpmflats(), flag=flag) if np.any(bad_flats): self.mask[bad_flats] = self.bitmask.turn_on(self.mask[bad_flats], flag) diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index fd6906288d..505fcbb764 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -109,33 +109,30 @@ class Spec2DObj(datamodel.DataContainer): 'head0' # Raw header ] - # TODO: Allow for **kwargs here? @classmethod - def from_file(cls, file, detname, chk_version=True): + def from_file(cls, ifile, detname, chk_version=True): """ - Override base-class :func:`~pypeit.datamodel.DataContainer.from_file` to - specify detector to read. + Instantiate the object from an extension in the specified fits file. + Over-load :func:`~pypeit.datamodel.DataContainer.from_file` + to specify detector to read. + Args: - file (:obj:`str`): - File name to read. + ifile (:obj:`str`, `Path`_): + Fits file with the data to read detname (:obj:`str`): The string identifier for the detector or mosaic used to select the data that is read. chk_version (:obj:`bool`, optional): - If False, allow a mismatch in datamodel to proceed - - Returns: - :class:`~pypeit.spec2dobj.Spec2DObj`: 2D spectra object. + Passed to :func:`from_hdu`. """ - with io.fits_open(file) as hdu: + with io.fits_open(ifile) as hdu: # Check detname is valid detnames = np.unique([h.name.split('-')[0] for h in hdu[1:]]) if detname not in detnames: msgs.error(f'Your --det={detname} is not available. \n Choose from: {detnames}') return cls.from_hdu(hdu, detname, chk_version=chk_version) - # TODO: Allow for **kwargs here? @classmethod def from_hdu(cls, hdu, detname, chk_version=True): """ @@ -160,7 +157,7 @@ def from_hdu(cls, hdu, detname, chk_version=True): if len(ext) == 0: # No relevant extensions! - msgs.error(f'{detname} not available in any extension of {file}') + msgs.error(f'{detname} not available in any extension of the input HDUList.') mask_ext = f'{detname}-BPMMASK' has_mask = mask_ext in ext @@ -319,10 +316,14 @@ def update_slits(self, spec2DObj): msgs.error("SPAT_IDs are not in sync!") # Find the good ones on the input object - bpm = spec2DObj.slits.mask.astype(bool) - exc_reduce = np.invert(spec2DObj.slits.bitmask.flagged( - spec2DObj.slits.mask, flag=spec2DObj.slits.bitmask.exclude_for_reducing)) - gpm = np.invert(bpm & exc_reduce) +# bpm = spec2DObj.slits.mask.astype(bool) +# exc_reduce = np.invert(spec2DObj.slits.bitmask.flagged( +# spec2DObj.slits.mask, flag=spec2DObj.slits.bitmask.exclude_for_reducing)) +# gpm = np.invert(bpm & exc_reduce) + bpm = spec2DObj.slits.bitmask.flagged( + spec2DObj.slits.mask, + and_not=spec2DObj.slits.bitmask.exclude_for_reducing) + gpm = np.logical_not(bpm) # Update slits.mask self.slits.mask[gpm] = spec2DObj.slits.mask[gpm] diff --git a/pypeit/specobj.py b/pypeit/specobj.py index 530dfef9a1..4644f11d2a 100644 --- a/pypeit/specobj.py +++ b/pypeit/specobj.py @@ -578,8 +578,10 @@ def to_arrays(self, extraction='OPT', fluxed=True): Convert spectrum into np.ndarray arrays Args: - extraction (str): Extraction method to convert + extraction (str): + Extraction method to convert fluxed: + Use the fluxed tags Returns: tuple: wave, flux, ivar, mask arrays @@ -599,7 +601,7 @@ def to_arrays(self, extraction='OPT', fluxed=True): # Return return self[swave], self[sflux], self[sivar], self[smask] - def to_xspec1d(self, masked=False, **kwargs): + def to_xspec1d(self, masked=True, extraction='OPT', fluxed=True): """ Create an `XSpectrum1D `_ using this spectrum. @@ -607,18 +609,22 @@ def to_xspec1d(self, masked=False, **kwargs): Args: masked (:obj:`bool`, optional): If True, only unmasked data are included. - kwargs (:obj:`dict`, optional): - Passed directly to :func:`to_arrays`. + extraction (str): + Extraction method to convert + fluxed: + Use the fluxed tags Returns: `linetools.spectra.xspectrum1d.XSpectrum1D`_: Spectrum object """ - wave, flux, ivar, gpm = self.to_arrays(**kwargs) + wave, flux, ivar, gpm = self.to_arrays(extraction=extraction, fluxed=fluxed) sig = np.sqrt(utils.inverse(ivar)) + wave_gpm = wave > 1.0 + wave, flux, sig, gpm = wave[wave_gpm], flux[wave_gpm], sig[wave_gpm], gpm[wave_gpm] if masked: - wave = wave[gpm] - flux = flux[gpm] - sig = sig[gpm] + flux = flux*gpm + sig = sig*gpm + # Create return xspectrum1d.XSpectrum1D.from_tuple((wave, flux, sig)) diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index e45c155203..3d96b8f2d5 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -24,10 +24,13 @@ from pypeit.spectrographs.util import load_spectrograph from pypeit.core import parse from pypeit.images.detector_container import DetectorContainer +# NOTE: Mosaic cannot be found in this module explicitly, but it is used in +# statements like: dmodcls = eval(hdu.header['DMODCLS']) from pypeit.images.mosaic import Mosaic -from pypeit import slittrace from pypeit import utils + +# TODO: Make this a DataContainer class SpecObjs: """ Object to hold a set of :class:`~pypeit.specobj.SpecObj` objects @@ -572,7 +575,9 @@ def apply_flux_calib(self, par, spectrograph, sens): _extinct_correct = (True if sens.algorithm == 'UVIS' else False) \ if par['extinct_correct'] is None else par['extinct_correct'] - if spectrograph.pypeline == 'MultiSlit': + # TODO enbaling this for now in case someone wants to treat the IFU as a slit spectrograph + # (not recommnneded but useful for quick reductions where you don't want to construct cubes and don't care about DAR). + if spectrograph.pypeline in ['MultiSlit','SlicerIFU']: for ii, sci_obj in enumerate(self.specobjs): if sens.wave.shape[1] == 1: sci_obj.apply_flux_calib(sens.wave[:, 0], sens.zeropoint[:, 0], @@ -1035,6 +1040,8 @@ def get_std_trace(detname, std_outfile, chk_version=True): std_trace = std_trace.flatten() elif 'Echelle' in pypeline: std_trace = std_trace.T + elif 'SlicerIFU' in pypeline: + std_trace = None else: msgs.error('Unrecognized pypeline') else: diff --git a/pypeit/spectrographs/gemini_flamingos.py b/pypeit/spectrographs/gemini_flamingos.py index 6873fb4631..9e9d459404 100644 --- a/pypeit/spectrographs/gemini_flamingos.py +++ b/pypeit/spectrographs/gemini_flamingos.py @@ -143,7 +143,7 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 # TODO: replace the telluric grid file for Gemini-S site. - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par diff --git a/pypeit/spectrographs/gemini_gmos.py b/pypeit/spectrographs/gemini_gmos.py index b16d279c50..7f810c2aab 100644 --- a/pypeit/spectrographs/gemini_gmos.py +++ b/pypeit/spectrographs/gemini_gmos.py @@ -10,7 +10,7 @@ from astropy.coordinates import SkyCoord from astropy import units from astropy.wcs import wcs -from astropy.io import fits +from astropy.io import fits from pypeit import msgs from pypeit.spectrographs import spectrograph @@ -39,7 +39,7 @@ class GeminiGMOSMosaicLookUp: .. code-block:: python from geminidr.gmos.lookups.geometry_conf import geometry - + Updating to any changes made to the DRAGONS version requires by-hand editing of the PypeIt code. """ @@ -248,7 +248,7 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): def default_pypeit_par(cls): """ Return the default parameters to use for this instrument. - + Returns: :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by all of PypeIt methods. @@ -530,48 +530,11 @@ def allowed_mosaics(self): """ return [(1,2,3)] - def list_detectors(self, mosaic=False): - """ - List the *names* of the detectors in this spectrograph. - - This is primarily used :func:`~pypeit.slittrace.average_maskdef_offset` - to measure the mean offset between the measured and expected slit - locations. - - Detectors separated along the dispersion direction should be ordered - along the first axis of the returned array. For example, Keck/DEIMOS - returns: - - .. code-block:: python - - dets = np.array([['DET01', 'DET02', 'DET03', 'DET04'], - ['DET05', 'DET06', 'DET07', 'DET08']]) - - such that all the bluest detectors are in ``dets[0]``, and the slits - found in detectors 1 and 5 are just from the blue and red counterparts - of the same slit. - - Args: - mosaic (:obj:`bool`, optional): - Is this a mosaic reduction? - It is used to determine how to list the detector, i.e., 'DET' or 'MSC'. - - Returns: - `numpy.ndarray`_: The list of detectors in a `numpy.ndarray`_. If - the array is 2D, there are detectors separated along the dispersion - axis. - """ - if mosaic: - return np.array([self.get_det_name(_det) for _det in self.allowed_mosaics]) - - return np.array([detector_container.DetectorContainer.get_name(i+1) - for i in range(self.ndet)]).reshape(2,-1) - @property def default_mosaic(self): return self.allowed_mosaics[0] - + def get_slitmask(self, filename): """ Parse the slitmask data from a MOSFIRE file into :attr:`slitmask`, a @@ -596,15 +559,15 @@ def get_slitmask(self, filename): # Projected distance (in arcsec) of the object from the left and right (top and bot) edges of the slit slit_length = mask_tbl['slitsize_y'].to('arcsec').value # arcsec - topdist = np.round(slit_length/2. - + topdist = np.round(slit_length/2. - mask_tbl['slitpos_y'].to('arcsec').value, 3) - botdist = np.round(slit_length/2. + + botdist = np.round(slit_length/2. + mask_tbl['slitpos_y'].to('arcsec').value, 3) # Coordinates # WARNING -- GMOS TABLE IS ONLY IN FLOAT32!!! - obj_ra = mask_tbl['RA'].value * 15. - obj_dec = mask_tbl['DEC'].value + obj_ra = mask_tbl['RA'].value * 15. + obj_dec = mask_tbl['DEC'].value objname = mask_tbl['ID'].value.astype(str) slitID = mask_tbl['ID'].value # Slit and objects are the same @@ -620,7 +583,7 @@ def get_slitmask(self, filename): topdist, botdist]).T # Mask pointing - mask_coord = SkyCoord(mask_tbl.meta['RA_IMAG'], mask_tbl.meta['DEC_IMAG'], + mask_coord = SkyCoord(mask_tbl.meta['RA_IMAG'], mask_tbl.meta['DEC_IMAG'], unit=("hourangle", "deg")) # PA corresponding to positive x on detector (spatial) @@ -631,8 +594,8 @@ def get_slitmask(self, filename): # Slit positions obj_coord = SkyCoord(ra=obj_ra, dec=obj_dec, unit='deg') offsets = np.sqrt( - mask_tbl['slitpos_x'].to('arcsec').value**2 + - mask_tbl['slitpos_y'].to('arcsec').value**2) + mask_tbl['slitpos_x'].to('arcsec').value**2 + + mask_tbl['slitpos_y'].to('arcsec').value**2) # NOT READY FOR TILTS if np.any(np.invert(np.isclose(mask_tbl['slittilt'].value, 0.))): msgs.error('NOT READY FOR TILTED SLITS') @@ -648,7 +611,7 @@ def get_slitmask(self, filename): slit_pa*units.deg, off_sign*offset*units.arcsec) slit_ra.append(slit_coord.ra.deg) slit_dec.append(slit_coord.dec.deg) - + # Instantiate the slit mask object and return it self.slitmask = SlitMask( @@ -660,12 +623,12 @@ def get_slitmask(self, filename): np.zeros(slitID.size), np.zeros(slitID.size), np.zeros(slitID.size), - np.zeros(slitID.size)]).T.reshape(-1,4,2), + np.zeros(slitID.size)]).T.reshape(-1,4,2), slitid=np.array(slitID, dtype=int), align=mask_tbl['priority'].value == b'0', science=mask_tbl['priority'].value != b'0', onsky=np.array([ - slit_ra, slit_dec, + slit_ra, slit_dec, np.array(mask_tbl['slitsize_y'].to('arcsec').value, dtype=float), np.array(mask_tbl['slitsize_x'].to('arcsec').value, dtype=float), slit_pas]).T, @@ -684,7 +647,7 @@ def get_maskdef_slitedges(self, ccdnum=None, filename=None, debug=None, Args: binning (_type_, optional): _description_. Defaults to None. binning(str, optional): spec,spat binning of the flat field image - filename (:obj:`list`, optional): Names + filename (:obj:`list`, optional): Names the mask design info and wcs_file in that order debug (:obj:`bool`, optional): Debug ccdnum (:obj:`int`, optional): detector number @@ -710,10 +673,10 @@ def get_maskdef_slitedges(self, ccdnum=None, filename=None, debug=None, self.get_slitmask(maskfile) # Binning of flat - _, bin_spat= parse.parse_binning(binning) + _, bin_spat= parse.parse_binning(binning) # Slit center - slit_coords = SkyCoord(ra=self.slitmask.onsky[:,0], + slit_coords = SkyCoord(ra=self.slitmask.onsky[:,0], dec=self.slitmask.onsky[:,1], unit='deg') mask_coord = SkyCoord(ra=self.slitmask.mask_radec[0], dec=self.slitmask.mask_radec[1], unit='deg') @@ -721,7 +684,7 @@ def get_maskdef_slitedges(self, ccdnum=None, filename=None, debug=None, # Load up the acquisition image (usually a sciframe) hdul_acq = fits.open(wcs_file) acq_binning = self.get_meta_value(self.get_headarr(hdul_acq), 'binning') - _, bin_spat_acq = parse.parse_binning(acq_binning) + _, bin_spat_acq = parse.parse_binning(acq_binning) wcss = [wcs.WCS(hdul_acq[i].header) for i in range(1, len(hdul_acq))] left_edges = [] @@ -736,7 +699,7 @@ def get_maskdef_slitedges(self, ccdnum=None, filename=None, debug=None, right_coord = slit_coords[islit].directional_offset_by( self.slitmask.onsky[islit,4]*units.deg, self.slitmask.onsky[islit,2]*units.arcsec/2.) - + got_it = False for kk, iwcs in enumerate(wcss): pix_xy = iwcs.world_to_pixel(left_coord) @@ -753,8 +716,8 @@ def get_maskdef_slitedges(self, ccdnum=None, filename=None, debug=None, # DEBUGGING # tbl = Table() -# tbl['left'] = left_edges -# tbl['right'] = right_edges +# tbl['left'] = left_edges +# tbl['right'] = right_edges # tbl['ID'] = self.slitmask.slitid # tbl.sort('left') # embed(header='641 of gemini_gmos') @@ -866,14 +829,14 @@ def get_detector_par(self, det, hdu=None): def default_pypeit_par(cls): """ Return the default parameters to use for this instrument. - + Returns: :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by all of PypeIt methods. """ par = super().default_pypeit_par() par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' # Bound the detector with slit edges if no edges are found. These data are often trimmed # so we implement this here as the default. par['calibrations']['slitedges']['bound_detector'] = True @@ -922,7 +885,7 @@ def bpm(self, filename, det, shape=None, msbias=None): hdrs = self.get_headarr(filename) binning = self.get_meta_value(hdrs, 'binning') obs_epoch = self.get_meta_value(hdrs, 'mjd') - bin_spec, bin_spat= parse.parse_binning(binning) + bin_spec, bin_spat= parse.parse_binning(binning) # Add the detector-specific, hard-coded bad columns if 1 in _det: @@ -944,7 +907,7 @@ def bpm(self, filename, det, shape=None, msbias=None): # Bad amp as of January 28, 2022 # https://gemini.edu/sciops/instruments/gmos/GMOS-S_badamp5_ops_3.pdf if obs_epoch > 2022.07: - badr = (768*2)//bin_spec + badr = (768*2)//bin_spec _bpm_img[i,badr:,:] = 1 if 3 in _det: msgs.info("Using hard-coded BPM for det=3 on GMOSs") @@ -983,7 +946,7 @@ def config_specific_par(self, scifile, inp_par=None): # The bad amp needs a larger follow_span for slit edge tracing obs_epoch = self.get_meta_value(scifile, 'mjd') binning = self.get_meta_value(scifile, 'binning') - bin_spec, bin_spat= parse.parse_binning(binning) + bin_spec, bin_spat= parse.parse_binning(binning) if obs_epoch > 2022.07: par['calibrations']['slitedges']['follow_span'] = 290*bin_spec # @@ -1349,7 +1312,7 @@ def config_specific_par(self, scifile, inp_par=None): par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gmos_r400_e2v_mosaic.fits' # The blue wavelengths are *faint* # But redder observations may prefer something closer to the default - par['calibrations']['wavelengths']['sigdetect'] = 1. + par['calibrations']['wavelengths']['sigdetect'] = 1. # Return return par diff --git a/pypeit/spectrographs/gemini_gnirs.py b/pypeit/spectrographs/gemini_gnirs.py index 921ecccaea..148464d7d1 100644 --- a/pypeit/spectrographs/gemini_gnirs.py +++ b/pypeit/spectrographs/gemini_gnirs.py @@ -297,7 +297,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 6 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par def config_specific_par(self, scifile, inp_par=None): diff --git a/pypeit/spectrographs/gtc_osiris.py b/pypeit/spectrographs/gtc_osiris.py index daa644241c..f622ad99de 100644 --- a/pypeit/spectrographs/gtc_osiris.py +++ b/pypeit/spectrographs/gtc_osiris.py @@ -110,7 +110,7 @@ def default_pypeit_par(cls): par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['arcframe']['exprng'] = [None, None] # Long arc exposures par['calibrations']['arcframe']['process']['clip'] = False - par['calibrations']['standardframe']['exprng'] = [None, 180] + par['calibrations']['standardframe']['exprng'] = [None, 300] # Multiple arcs with different lamps, so can't median combine nor clip, also need to remove continuum par['calibrations']['arcframe']['process']['combine'] = 'mean' par['calibrations']['arcframe']['process']['subtract_continuum'] = True @@ -384,7 +384,7 @@ def config_specific_par(self, scifile, inp_par=None): par['calibrations']['wavelengths']['lamps'] = ['ArI','XeI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2500I.fits' par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['IR']['telgridfile'] = "TelFit_MaunaKea_3100_26100_R20000.fits" + par['sensfunc']['IR']['telgridfile'] = "TellPCA_3000_26000_R10000.fits" else: msgs.warn('gtc_osiris.py: template arc missing for this grism! Trying holy-grail...') par['calibrations']['wavelengths']['method'] = 'holy-grail' @@ -955,7 +955,7 @@ def config_specific_par(self, scifile, inp_par=None): par['calibrations']['wavelengths']['lamps'] = ['ArI','XeI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2500I.fits' par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['IR']['telgridfile'] = "TelFit_MaunaKea_3100_26100_R20000.fits" + par['sensfunc']['IR']['telgridfile'] = "TellPCA_3000_26000_R10000.fits" else: msgs.warn('gtc_osiris.py: template arc missing for this grism! Trying holy-grail...') par['calibrations']['wavelengths']['method'] = 'holy-grail' diff --git a/pypeit/spectrographs/keck_deimos.py b/pypeit/spectrographs/keck_deimos.py index b7c711af6c..cff433892e 100644 --- a/pypeit/spectrographs/keck_deimos.py +++ b/pypeit/spectrographs/keck_deimos.py @@ -210,10 +210,14 @@ def get_detector_par(self, det, hdu=None): )) if hdu is not None: + amp = self.get_meta_value(self.get_headarr(hdu), 'amp') + if amp == 'DUAL:A+B': + msgs.error('PypeIt can only reduce images with AMPMODE == SINGLE:B or AMPMODE == SINGLE:A.') + amp_folder = "ampA" if amp == 'SINGLE:A' else "ampB" # raw frame date in mjd date = time.Time(self.get_meta_value(self.get_headarr(hdu), 'mjd'), format='mjd').value # get the measurements files - measure_files = sorted((data.Paths.spectrographs / "keck_deimos" / "gain_ronoise").glob("*")) + measure_files = sorted((data.Paths.spectrographs / "keck_deimos" / "gain_ronoise" / amp_folder).glob("*")) # Parse the dates recorded in the name of the files measure_dates = np.array([f.name.split('.')[2] for f in measure_files]) # convert into datetime format @@ -225,28 +229,43 @@ def get_detector_par(self, det, hdu=None): # get measurements tab_measure = Table.read(measure_files[close_idx], format='ascii') measured_det = tab_measure['col3'] + measured_amptype = tab_measure['col4'] measured_gain = tab_measure['col5'] # [e-/DN] measured_ronoise = tab_measure['col7'] # [e-] - msgs.info(f"We are using DEIMOS gain/RN values based on WMKO estimates on {measure_dates[close_idx]}.") + msgs.info(f"We are using DEIMOS gain/RN values for AMPMODE = {amp} " + f"based on WMKO estimates on {measure_dates[close_idx]}.") + # find values for this amp and each detector + this_amp = measured_amptype == 'A' if amp == 'SINGLE:A' else measured_amptype == 'B' + det_1 = this_amp & (measured_det == 1) + det_2 = this_amp & (measured_det == 2) + det_3 = this_amp & (measured_det == 3) + det_4 = this_amp & (measured_det == 4) + # we don't have measurements for the red detectors when amp = 'SINGLE:A', + # therefore we use the values for the blue detectors + det_5 = this_amp & (measured_det == 5 if amp == 'SINGLE:B' else measured_det == 1) + det_6 = this_amp & (measured_det == 6 if amp == 'SINGLE:B' else measured_det == 2) + det_7 = this_amp & (measured_det == 7 if amp == 'SINGLE:B' else measured_det == 3) + det_8 = this_amp & (measured_det == 8 if amp == 'SINGLE:B' else measured_det == 4) + # get gain - detector_dict1['gain'] = measured_gain[measured_det == 1] - detector_dict2['gain'] = measured_gain[measured_det == 2] - detector_dict3['gain'] = measured_gain[measured_det == 3] - detector_dict4['gain'] = measured_gain[measured_det == 4] - detector_dict5['gain'] = measured_gain[measured_det == 5] - detector_dict6['gain'] = measured_gain[measured_det == 6] - detector_dict7['gain'] = measured_gain[measured_det == 7] - detector_dict8['gain'] = measured_gain[measured_det == 8] + detector_dict1['gain'] = measured_gain[det_1].data + detector_dict2['gain'] = measured_gain[det_2].data + detector_dict3['gain'] = measured_gain[det_3].data + detector_dict4['gain'] = measured_gain[det_4].data + detector_dict5['gain'] = measured_gain[det_5].data + detector_dict6['gain'] = measured_gain[det_6].data + detector_dict7['gain'] = measured_gain[det_7].data + detector_dict8['gain'] = measured_gain[det_8].data # get ronoise - detector_dict1['ronoise'] = measured_ronoise[measured_det == 1] - detector_dict2['ronoise'] = measured_ronoise[measured_det == 2] - detector_dict3['ronoise'] = measured_ronoise[measured_det == 3] - detector_dict4['ronoise'] = measured_ronoise[measured_det == 4] - detector_dict5['ronoise'] = measured_ronoise[measured_det == 5] - detector_dict6['ronoise'] = measured_ronoise[measured_det == 6] - detector_dict7['ronoise'] = measured_ronoise[measured_det == 7] - detector_dict8['ronoise'] = measured_ronoise[measured_det == 8] + detector_dict1['ronoise'] = measured_ronoise[det_1].data + detector_dict2['ronoise'] = measured_ronoise[det_2].data + detector_dict3['ronoise'] = measured_ronoise[det_3].data + detector_dict4['ronoise'] = measured_ronoise[det_4].data + detector_dict5['ronoise'] = measured_ronoise[det_5].data + detector_dict6['ronoise'] = measured_ronoise[det_6].data + detector_dict7['ronoise'] = measured_ronoise[det_7].data + detector_dict8['ronoise'] = measured_ronoise[det_8].data detectors = [detector_dict1, detector_dict2, detector_dict3, detector_dict4, detector_dict5, detector_dict6, detector_dict7, detector_dict8] @@ -301,7 +320,7 @@ def default_pypeit_par(cls): par['scienceframe']['process']['objlim'] = 1.5 # If telluric is triggered - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R15000.fits' return par def config_specific_par(self, scifile, inp_par=None): @@ -325,9 +344,14 @@ def config_specific_par(self, scifile, inp_par=None): headarr = self.get_headarr(scifile) - # When using LVM mask reduce only detectors 3,7 - if 'LVMslit' in self.get_meta_value(headarr, 'decker'): - par['rdx']['detnum'] = [(3,7)] + # When using LVM mask or AMPMODE = SINGLE:A reduce only detectors 3,7 + if ('LVMslit' in self.get_meta_value(headarr, 'decker') or + self.get_meta_value(headarr, 'amp') == 'SINGLE:A'): + # give an info message if AMPMODE = SINGLE:A + if self.get_meta_value(headarr, 'amp') == 'SINGLE:A': + msgs.info('Data taken with AMPMODE = SINGLE:A. Only detectors 3,7 will be reduced. To change this,' + ' modify the detnum parameter in the pypeit file.') + par['rdx']['detnum'] = [(3, 7)] # Turn PCA off for long slits # TODO: I'm a bit worried that this won't catch all @@ -542,7 +566,7 @@ def valid_configuration_values(self): and their associated discrete set of valid values. If there are no restrictions on configuration values, None is returned. """ - return {'amp': ['SINGLE:B'], 'mode':['Spectral']} + return {'amp': ['SINGLE:B', 'SINGLE:A'], 'mode':['Spectral']} def config_independent_frames(self): """ @@ -696,9 +720,9 @@ def get_rawimage(self, raw_file, det): .. warning:: PypeIt currently *cannot* reduce images produced by - reading the DEIMOS CCDs with the A amplifier or those + reading the DEIMOS CCDs with the A+B amplifier or those taken in imaging mode. All image handling assumes DEIMOS - images have been read with the B amplifier in the + images have been read with the B or A amplifier in the "Spectral" observing mode. This method will fault if this is not true based on the header keywords MOSMODE and AMPMODE. @@ -741,8 +765,9 @@ def get_rawimage(self, raw_file, det): mosaic = None if nimg == 1 else self.get_mosaic_par(det, hdu=hdu) detectors = [self.get_detector_par(det, hdu=hdu)] if nimg == 1 else mosaic.detectors - if hdu[0].header['AMPMODE'] != 'SINGLE:B': - msgs.error('PypeIt can only reduce images with AMPMODE == SINGLE:B.') + # TODO check that that read noise and gain are the same for this amplifier mode?? + if hdu[0].header['AMPMODE'] not in ['SINGLE:B', 'SINGLE:A']: + msgs.error('PypeIt can only reduce images with AMPMODE == SINGLE:B or AMPMODE == SINGLE:A.') if hdu[0].header['MOSMODE'] != 'Spectral': msgs.error('PypeIt can only reduce images with MOSMODE == Spectral.') diff --git a/pypeit/spectrographs/keck_hires.py b/pypeit/spectrographs/keck_hires.py index 2757803548..b03fd2359a 100644 --- a/pypeit/spectrographs/keck_hires.py +++ b/pypeit/spectrographs/keck_hires.py @@ -157,7 +157,14 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 5 #[9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7] - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_10500_R120000.fits' + par['sensfunc']['IR']['pix_shift_bounds'] = (-40.0,40.0) + + # Telluric parameters + # HIRES is usually oversampled, so the helio shift can be large + par['telluric']['pix_shift_bounds'] = (-40.0,40.0) + # Similarly, the resolution guess is higher than it should be + par['telluric']['resln_frac_bounds'] = (0.25,1.25) # Coadding par['coadd1d']['wave_method'] = 'log10' diff --git a/pypeit/spectrographs/keck_kcwi.py b/pypeit/spectrographs/keck_kcwi.py index 191d158701..5743eaa6eb 100644 --- a/pypeit/spectrographs/keck_kcwi.py +++ b/pypeit/spectrographs/keck_kcwi.py @@ -313,6 +313,9 @@ def default_pypeit_par(cls): # Flux calibration parameters par['sensfunc']['UVIS']['extinct_correct'] = False # This must be False - the extinction correction is performed when making the datacube + # If telluric is triggered + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R15000.fits' + return par def pypeit_file_keys(self): @@ -871,7 +874,6 @@ def init_meta(self): self.meta['dispangle'] = dict(ext=0, card='BGRANGLE', rtol=0.01) self.meta['cenwave'] = dict(ext=0, card='BCWAVE', rtol=0.01) - def raw_header_cards(self): """ Return additional raw header cards to be propagated in @@ -919,9 +921,10 @@ def default_pypeit_par(cls): par['calibrations']['pixelflatframe']['process']['subtract_scattlight'] = True par['calibrations']['illumflatframe']['process']['subtract_scattlight'] = True par['scienceframe']['process']['subtract_scattlight'] = True - par['scienceframe']['process']['scattlight']['finecorr_pad'] = 2 + par['scienceframe']['process']['scattlight']['finecorr_method'] = 'median' + par['scienceframe']['process']['scattlight']['finecorr_pad'] = 4 # This is the unbinned number of pixels to pad par['scienceframe']['process']['scattlight']['finecorr_order'] = 2 - par['scienceframe']['process']['scattlight']['finecorr_mask'] = 12 # Mask the middle inter-slit region. It contains a strange scattered light feature that doesn't appear to affect any other inter-slit regions + # par['scienceframe']['process']['scattlight']['finecorr_mask'] = 12 # Mask the middle inter-slit region. It contains a strange scattered light feature that doesn't appear to affect any other inter-slit regions # Correct the illumflat for pixel-to-pixel sensitivity variations par['calibrations']['illumflatframe']['process']['use_pixelflat'] = True @@ -941,6 +944,7 @@ def default_pypeit_par(cls): # Alter the method used to combine pixel flats par['calibrations']['pixelflatframe']['process']['combine'] = 'median' par['calibrations']['flatfield']['spec_samp_coarse'] = 20.0 + par['calibrations']['flatfield']['spat_samp'] = 1.0 # This should give 1% accuracy in the spatial illumination correction for 2x2 binning, and <0.5% accuracy for 1x1 binning #par['calibrations']['flatfield']['tweak_slits'] = False # Do not tweak the slit edges (we want to use the full slit) par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.0 # Make sure the full slit is used (i.e. when the illumination fraction is > 0.5) par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.0 # Make sure the full slit is used (i.e. no padding) @@ -1167,13 +1171,13 @@ def fit_2d_det_response(self, det_resp, gpmask): msgs.info("Performing a 2D fit to the detector response") # Define a 2D sine function, which is a good description of KCWI data - def sinfunc2d(x, amp, scl, phase, wavelength, angle): + def sinfunc2d(x, amp, scl, quad, phase, wavelength, angle): """ 2D Sine function """ xx, yy = x angle *= np.pi / 180.0 - return 1 + (amp + xx * scl) * np.sin( + return 1 + (amp + xx*scl + xx*xx*quad) * np.sin( 2 * np.pi * (xx * np.cos(angle) + yy * np.sin(angle)) / wavelength + phase) x = np.arange(det_resp.shape[0]) @@ -1181,11 +1185,12 @@ def sinfunc2d(x, amp, scl, phase, wavelength, angle): xx, yy = np.meshgrid(x, y, indexing='ij') # Prepare the starting parameters amp = 0.02 # Roughly a 2% effect - scale = 0.0 # Assume the amplitude is constant over the detector + scale, quad = 0.0, 0.0 # Assume the amplitude is constant over the detector wavelength = np.sqrt(det_resp.shape[0] ** 2 + det_resp.shape[1] ** 2) / 31.5 # 31-32 cycles of the pattern from corner to corner phase, angle = 0.0, -45.34 # No phase, and a decent guess at the angle - p0 = [amp, scale, phase, wavelength, angle] - popt, pcov = curve_fit(sinfunc2d, (xx[gpmask], yy[gpmask]), det_resp[gpmask], p0=p0) + p0 = [amp, scale, quad, phase, wavelength, angle] + this_bpm = gpmask & (np.abs(det_resp-1) < 0.1) # Only expect this to be a 5% effect + popt, pcov = curve_fit(sinfunc2d, (xx[this_bpm], yy[this_bpm]), det_resp[this_bpm], p0=p0) return sinfunc2d((xx, yy), *popt) diff --git a/pypeit/spectrographs/keck_lris.py b/pypeit/spectrographs/keck_lris.py index 5172bd92c8..43b5f53aa5 100644 --- a/pypeit/spectrographs/keck_lris.py +++ b/pypeit/spectrographs/keck_lris.py @@ -101,7 +101,7 @@ def default_pypeit_par(cls): # If telluric is triggered - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par diff --git a/pypeit/spectrographs/keck_mosfire.py b/pypeit/spectrographs/keck_mosfire.py index 9059901347..e874baea83 100644 --- a/pypeit/spectrographs/keck_mosfire.py +++ b/pypeit/spectrographs/keck_mosfire.py @@ -127,7 +127,7 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 13 par['sensfunc']['IR']['maxiter'] = 2 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par # NOTE: This function is used by the dev-suite diff --git a/pypeit/spectrographs/keck_nires.py b/pypeit/spectrographs/keck_nires.py index bd4b3b1138..761fcb7253 100644 --- a/pypeit/spectrographs/keck_nires.py +++ b/pypeit/spectrographs/keck_nires.py @@ -140,7 +140,7 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 par['sensfunc']['IR']['maxiter'] = 2 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' # Coadding par['coadd1d']['wave_method'] = 'log10' diff --git a/pypeit/spectrographs/keck_nirspec.py b/pypeit/spectrographs/keck_nirspec.py index eab7cafc91..60e84e0df7 100644 --- a/pypeit/spectrographs/keck_nirspec.py +++ b/pypeit/spectrographs/keck_nirspec.py @@ -129,7 +129,12 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' + par['sensfunc']['IR']['pix_shift_bounds'] = (-8.0,8.0) + + # Telluric parameters + par['telluric']['pix_shift_bounds'] = (-8.0,8.0) + return par def init_meta(self): diff --git a/pypeit/spectrographs/magellan_fire.py b/pypeit/spectrographs/magellan_fire.py index 2762b10d86..95d4dcb617 100644 --- a/pypeit/spectrographs/magellan_fire.py +++ b/pypeit/spectrographs/magellan_fire.py @@ -201,7 +201,7 @@ def default_pypeit_par(cls): par['sensfunc']['polyorder'] = 5 par['sensfunc']['IR']['maxiter'] = 2 # place holder for telgrid file - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R15000.fits' # Coadding. I'm not sure what this should be for PRISM mode? par['coadd1d']['wave_method'] = 'log10' @@ -418,7 +418,7 @@ def default_pypeit_par(cls): par['reduce']['findobj']['find_trim_edge'] = [50,50] par['flexure']['spec_method'] = 'skip' - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' # Set the default exposure time ranges for the frame typing par['calibrations']['standardframe']['exprng'] = [None, 60] diff --git a/pypeit/spectrographs/mmt_binospec.py b/pypeit/spectrographs/mmt_binospec.py index 1b3998f1f4..90e178b658 100644 --- a/pypeit/spectrographs/mmt_binospec.py +++ b/pypeit/spectrographs/mmt_binospec.py @@ -214,7 +214,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['polyorder'] = 7 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par diff --git a/pypeit/spectrographs/mmt_mmirs.py b/pypeit/spectrographs/mmt_mmirs.py index d9341cd176..4425543c29 100644 --- a/pypeit/spectrographs/mmt_mmirs.py +++ b/pypeit/spectrographs/mmt_mmirs.py @@ -202,7 +202,7 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 # ToDo: replace the telluric grid file for MMT site. - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par diff --git a/pypeit/spectrographs/p200_dbsp.py b/pypeit/spectrographs/p200_dbsp.py index caecea72f0..0ea3032236 100644 --- a/pypeit/spectrographs/p200_dbsp.py +++ b/pypeit/spectrographs/p200_dbsp.py @@ -525,7 +525,7 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'UVIS' par['sensfunc']['UVIS']['polycorrect'] = False - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Lick_3100_11100_R10000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par def config_specific_par(self, scifile, inp_par=None): diff --git a/pypeit/spectrographs/p200_tspec.py b/pypeit/spectrographs/p200_tspec.py index 1d4000d047..641dc2d60e 100644 --- a/pypeit/spectrographs/p200_tspec.py +++ b/pypeit/spectrographs/p200_tspec.py @@ -215,7 +215,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' # Coadding par['coadd1d']['wave_method'] = 'log10' diff --git a/pypeit/spectrographs/shane_kast.py b/pypeit/spectrographs/shane_kast.py index e58691757f..0f667a44c0 100644 --- a/pypeit/spectrographs/shane_kast.py +++ b/pypeit/spectrographs/shane_kast.py @@ -57,7 +57,7 @@ def default_pypeit_par(cls): par['calibrations']['standardframe']['exprng'] = [1, 61] # par['scienceframe']['exprng'] = [61, None] - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Lick_3100_11100_R10000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par def init_meta(self): @@ -460,7 +460,7 @@ def default_pypeit_par(cls): # TODO In case someone wants to use the IR algorithm for shane kast this is the telluric file. Note the IR # algorithm is not the default. - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Lick_3100_11100_R10000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par def init_meta(self): @@ -684,7 +684,7 @@ def default_pypeit_par(cls): par['calibrations']['wavelengths']['rms_thresh_frac_fwhm'] = 0.09 par['calibrations']['wavelengths']['sigdetect'] = 5. par['calibrations']['wavelengths']['use_instr_flag'] = True - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Lick_3100_11100_R10000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par diff --git a/pypeit/spectrographs/slitmask.py b/pypeit/spectrographs/slitmask.py index 47fa898b6b..cd45452741 100644 --- a/pypeit/spectrographs/slitmask.py +++ b/pypeit/spectrographs/slitmask.py @@ -254,20 +254,20 @@ def nslits(self): @property def alignment_slit(self): """Boolean array selecting the alignment slits.""" - return self.bitmask.flagged(self.mask, 'ALIGN') + return self.bitmask.flagged(self.mask, flag='ALIGN') def is_alignment(self, i): """Check if specific slit is an alignment slit.""" - return self.bitmask.flagged(self.mask[i], 'ALIGN') + return self.bitmask.flagged(self.mask[i], flag='ALIGN') @property def science_slit(self): """Boolean array selecting the slits with science targets.""" - return self.bitmask.flagged(self.mask, 'SCIENCE') + return self.bitmask.flagged(self.mask, flag='SCIENCE') def is_science(self, i): """Check if specific slit should have a science target.""" - return self.bitmask.flagged(self.mask[i], 'SCIENCE') + return self.bitmask.flagged(self.mask[i], flag='SCIENCE') class SlitRegister: diff --git a/pypeit/spectrographs/soar_goodman.py b/pypeit/spectrographs/soar_goodman.py index 1f1f9c5bc4..293bcaf025 100644 --- a/pypeit/spectrographs/soar_goodman.py +++ b/pypeit/spectrographs/soar_goodman.py @@ -330,7 +330,7 @@ def default_pypeit_par(cls): par['scienceframe']['exprng'] = [90, None] #par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R15000.fits' # TODO: Temporary fix for failure mode. Remove once Ryan provides a # fix. @@ -527,7 +527,7 @@ def default_pypeit_par(cls): par['scienceframe']['exprng'] = [90, None] # par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R15000.fits' return par diff --git a/pypeit/spectrographs/vlt_fors.py b/pypeit/spectrographs/vlt_fors.py index 254245ff44..aff71e4f86 100644 --- a/pypeit/spectrographs/vlt_fors.py +++ b/pypeit/spectrographs/vlt_fors.py @@ -69,7 +69,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 5 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Paranal_VIS_9800_25000_R25000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' @@ -457,4 +457,4 @@ def parse_dither_pattern(self, file_list, ext=None): # dither_id.append(hdr['FRAMEID']) # offset_arcsec[ifile] = hdr['YOFFSET'] - return dither_pattern, dither_id, offset_arcsec \ No newline at end of file + return dither_pattern, dither_id, offset_arcsec diff --git a/pypeit/spectrographs/vlt_sinfoni.py b/pypeit/spectrographs/vlt_sinfoni.py index 68f40af6b3..df29ac8653 100644 --- a/pypeit/spectrographs/vlt_sinfoni.py +++ b/pypeit/spectrographs/vlt_sinfoni.py @@ -157,7 +157,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 7 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Paranal_NIR_9800_25000_R25000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par diff --git a/pypeit/spectrographs/vlt_xshooter.py b/pypeit/spectrographs/vlt_xshooter.py index b2ffa58ceb..01cc8fc088 100644 --- a/pypeit/spectrographs/vlt_xshooter.py +++ b/pypeit/spectrographs/vlt_xshooter.py @@ -344,7 +344,12 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Paranal_NIR_9800_25000_R25000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' + par['sensfunc']['IR']['pix_shift_bounds'] = (-10.0,10.0) + + # Telluric parameters + par['telluric']['pix_shift_bounds'] = (-10.0,10.0) + par['telluric']['resln_frac_bounds'] = (0.4,2.0) # Coadding par['coadd1d']['wave_method'] = 'log10' @@ -719,7 +724,12 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 #[9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7] - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Paranal_VIS_4900_11100_R25000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' + par['sensfunc']['IR']['pix_shift_bounds'] = (-10.0,10.0) + + # Telluric parameters + par['telluric']['pix_shift_bounds'] = (-10.0,10.0) + par['telluric']['resln_frac_bounds'] = (0.4,2.0) # Coadding par['coadd1d']['wave_method'] = 'log10' @@ -1007,8 +1017,11 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' - # This is a hack until we we have a Paranal file generated that covers the UVB wavelength range. + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' + par['sensfunc']['IR']['pix_shift_bounds'] = (-8.0,8.0) + + # Telluric parameters + par['telluric']['pix_shift_bounds'] = (-8.0,8.0) # Coadding par['coadd1d']['wave_method'] = 'log10' diff --git a/pypeit/specutils/pypeit_loaders.py b/pypeit/specutils/pypeit_loaders.py index c23e134620..3b4aed3df8 100644 --- a/pypeit/specutils/pypeit_loaders.py +++ b/pypeit/specutils/pypeit_loaders.py @@ -36,6 +36,7 @@ from pypeit import msgs from pypeit import specobjs from pypeit import onespec +from pypeit import utils def _enforce_monotonic_wavelengths(wave, flux, ivar, strict=True): @@ -49,7 +50,9 @@ def _enforce_monotonic_wavelengths(wave, flux, ivar, strict=True): flux : `numpy.ndarray`_ Spectrum flux ivar : `numpy.ndarray`_ - Spectrum inverse variance. Can be None. + Spectrum inverse variance. Can be None or the standard deviation. The + only operation on this and the ``flux`` vector is to downselect the + monotonically increasing values. strict : bool, optional Check that the wavelength vector is monotonically increasing. If not, raise an error (as would be done by the `specutils.SpectrumList`_ class). @@ -147,7 +150,8 @@ def identify_pypeit_onespec(origin, *args, **kwargs): priority=10, dtype=SpectrumList, autogenerate_spectrumlist=False) -def pypeit_spec1d_loader(filename, extract=None, fluxed=True, strict=True, **kwargs): +def pypeit_spec1d_loader(filename, extract=None, fluxed=True, strict=True, chk_version=True, + **kwargs): """ Load spectra from a PypeIt spec1d file into a SpectrumList. @@ -169,6 +173,10 @@ def pypeit_spec1d_loader(filename, extract=None, fluxed=True, strict=True, **kwa raise an error (as would be done by the `specutils.SpectrumList`_ class). If False, wavelengths that are *not* monotonically increasing are masked in the construction of the returned `specutils.SpectrumList`_ object. + chk_version : :obj:`bool`, optional + When reading in existing files written by PypeIt, perform strict version + checking to ensure a valid file. If False, the code will try to keep + going, but this may lead to faults and quiet failures. User beware! kwargs : dict, optional **Ignored!** Used to catch spurious arguments passed to the base class that are ignored by this function. @@ -180,7 +188,7 @@ def pypeit_spec1d_loader(filename, extract=None, fluxed=True, strict=True, **kwa """ # Try to load the file and ignoring any version mismatch try: - sobjs = specobjs.SpecObjs.from_fitsfile(filename, chk_version=False) + sobjs = specobjs.SpecObjs.from_fitsfile(filename, chk_version=chk_version) except PypeItError: file_pypeit_version = astropy.io.fits.getval(filename, 'VERSPYP', 'PRIMARY') msgs.error(f'Unable to ingest {filename.name} using pypeit.specobjs module from your version ' @@ -202,10 +210,11 @@ def pypeit_spec1d_loader(filename, extract=None, fluxed=True, strict=True, **kwa continue _wave, _flux, _ivar = _enforce_monotonic_wavelengths(_wave[_gpm], _flux[_gpm], _ivar[_gpm], strict=strict) + _sigma = np.sqrt(utils.inverse(_ivar)) flux_unit = astropy.units.Unit("1e-17 erg/(s cm^2 Angstrom)" if _cal else "electron") spec += \ [Spectrum1D(flux=astropy.units.Quantity(_flux * flux_unit), - uncertainty=astropy.nddata.InverseVariance(_ivar / flux_unit**2), + uncertainty=astropy.nddata.StdDevUncertainty(_sigma * flux_unit), meta={'name': sobj.NAME, 'extract': _ext, 'fluxed': _cal}, spectral_axis=astropy.units.Quantity(_wave * astropy.units.angstrom), velocity_convention="doppler_optical", @@ -218,7 +227,7 @@ def pypeit_spec1d_loader(filename, extract=None, fluxed=True, strict=True, **kwa extensions=["fits"], priority=10, dtype=Spectrum1D) -def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): +def pypeit_onespec_loader(filename, grid=False, strict=True, chk_version=True, **kwargs): """ Load a spectrum from a PypeIt OneSpec file into a Spectrum1D object. @@ -234,6 +243,10 @@ def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): raise an error (as would be done by the `specutils.Spectrum1D`_ class). If False, wavelengths that are *not* monotonically increasing are masked in the construction of the returned `specutils.Spectrum1D`_ object. + chk_version : :obj:`bool`, optional + When reading in existing files written by PypeIt, perform strict version + checking to ensure a valid file. If False, the code will try to keep + going, but this may lead to faults and quiet failures. User beware! kwargs : dict, optional **Ignored!** Used to catch spurious arguments passed to the base class that are ignored by this function. @@ -245,7 +258,7 @@ def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): """ # Try to load the file and ignoring any version mismatch try: - spec = onespec.OneSpec.from_file(filename) + spec = onespec.OneSpec.from_file(filename, chk_version=chk_version) except PypeItError: file_pypeit_version = astropy.io.fits.getval(filename, 'VERSPYP', 'PRIMARY') msgs.error(f'Unable to ingest {filename.name} using pypeit.specobjs module from your version ' @@ -256,7 +269,7 @@ def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): flux_unit = astropy.units.Unit("1e-17 erg/(s cm^2 Angstrom)" if spec.fluxed else "ct/s") wave = spec.wave_grid_mid if grid else spec.wave - wave, flux, ivar = _enforce_monotonic_wavelengths(wave, spec.flux, spec.ivar, strict=strict) + wave, flux, sigma = _enforce_monotonic_wavelengths(wave, spec.flux, spec.sigma, strict=strict) # If the input filename is actually a string, assign it as the spectrum # name. Otherwise, try assuming it's a _io.FileIO object, and if that @@ -269,9 +282,10 @@ def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): except AttributeError: name = '' + # TODO We should be dealing with masking here. return Spectrum1D(flux=astropy.units.Quantity(flux * flux_unit), - uncertainty=None if ivar is None - else astropy.nddata.InverseVariance(ivar / flux_unit**2), + uncertainty=None if spec.sigma is None + else astropy.nddata.StdDevUncertainty(sigma * flux_unit), meta={'name': name, 'extract': spec.ext_mode, 'fluxed': spec.fluxed, 'grid': grid}, spectral_axis=astropy.units.Quantity(wave * astropy.units.angstrom), diff --git a/pypeit/tests/test_bitmask.py b/pypeit/tests/test_bitmask.py index 2b870dd02a..b8dcc8f220 100644 --- a/pypeit/tests/test_bitmask.py +++ b/pypeit/tests/test_bitmask.py @@ -7,6 +7,7 @@ from astropy.io import fits from pypeit.bitmask import BitMask +from pypeit.slittrace import SlitTraceBitMask #----------------------------------------------------------------------------- @@ -107,3 +108,64 @@ def test_wrong_bits(): assert numpy.sum(image_bm.flagged(mask, flag='COSMIC')) == numpy.sum(cosmics_indx) +def test_flag_order(): + + bm = ImageBitMask() + + flags = bm.keys() + assert bm.correct_flag_order(flags), 'Flags should not be mismatched' + + flags += ['NEWBIT'] + assert bm.correct_flag_order(flags), 'Appending flags should be fine' + + flags = bm.keys()[:-1] + assert bm.correct_flag_order(flags), 'Checking a subset of the flags should be fine' + + flags = bm.keys()[::-1] + assert not bm.correct_flag_order(flags), 'Reordering the flags is not okay' + + +def test_exclude_and_not(): + + n = 1024 + shape = (n,n) + + rng = numpy.random.default_rng(99) + + image_bm = ImageBitMask() + mask = numpy.zeros(shape, dtype=image_bm.minimum_dtype()) + + cosmics_indx = numpy.zeros(shape, dtype=bool) + cosmics_indx[rng.integers(0,high=n,size=9000), rng.integers(0,high=n,size=9000)] = True + mask[cosmics_indx] = image_bm.turn_on(mask[cosmics_indx], 'COSMIC') + + saturated_indx = numpy.zeros(shape, dtype=bool) + saturated_indx[rng.integers(0,high=n,size=9000), rng.integers(0,high=n,size=9000)] = True + mask[saturated_indx] = image_bm.turn_on(mask[saturated_indx], 'SATURATED') + + # NOTE: Want to make sure there are pixels flagged as both COSMIC and + # SATURATED. Otherwise the `and_not` test is not useful. + assert numpy.sum(cosmics_indx & saturated_indx) > 0, 'Bad test setup' + + assert numpy.array_equal(image_bm.flagged(mask), cosmics_indx | saturated_indx), \ + 'Mask incorrect' + assert numpy.array_equal(image_bm.flagged(mask, exclude='SATURATED'), cosmics_indx), \ + 'Exclude incorrect' + + assert numpy.array_equal(image_bm.flagged(mask, and_not='SATURATED'), + cosmics_indx & numpy.logical_not(saturated_indx)), 'Expunge incorrect' + +def test_boxslit(): + """ + Tests old vs. new bpm after adding `and_not` functionality. + """ + bm = SlitTraceBitMask() + v = numpy.array([10,0,2]) + + desired_bpm = (v > 2) & (numpy.invert(bm.flagged(v, flag=bm.exclude_for_reducing))) + + new_bpm = bm.flagged(v, exclude='BOXSLIT', and_not=bm.exclude_for_reducing) + + assert numpy.all(new_bpm == desired_bpm) + + diff --git a/pypeit/tests/test_collate_1d.py b/pypeit/tests/test_collate_1d.py index 3a4db0ae1f..de5a097fab 100644 --- a/pypeit/tests/test_collate_1d.py +++ b/pypeit/tests/test_collate_1d.py @@ -623,7 +623,9 @@ def mock_geomotion_correct(*args, **kwargs): # Test where one VEL_CORR is already set, and the SpecObj objects have no RA/DEC so the header RA/DEC must be used instead sobjs = MockSpecObjs("spec1d_file4") - monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", lambda x: sobjs) + def mock_from_fitsfile(*args, **kwargs): + return sobjs + monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_from_fitsfile) refframe_correction(par, spectrograph, spec1d_files, spec1d_failure_msgs) assert len(spec1d_failure_msgs) == 1 diff --git a/pypeit/tests/test_onespec.py b/pypeit/tests/test_onespec.py index 02c51747c1..4e9d479fd4 100644 --- a/pypeit/tests/test_onespec.py +++ b/pypeit/tests/test_onespec.py @@ -21,7 +21,7 @@ def test_init(): assert spec.spectrograph is None, 'Spectrograph should not be set' spec = onespec.OneSpec(wave, wave, flux, ivar=2*np.ones_like(flux)) - assert np.allclose(spec.sig, 1/np.sqrt(2)), 'Conversion to sigma is wrong' + #assert np.allclose(spec.sigma, 1/np.sqrt(2)), 'Conversion to sigma is wrong' def test_io(): diff --git a/pypeit/tests/test_slitmatch.py b/pypeit/tests/test_slitmatch.py new file mode 100644 index 0000000000..fe03404c68 --- /dev/null +++ b/pypeit/tests/test_slitmatch.py @@ -0,0 +1,65 @@ + +from IPython import embed + +import numpy as np +from pypeit.core import slitdesign_matching + +def test_match_positions(): + + # Random number generator + rng = np.random.default_rng(99) + + # Fake nominal positions + nominal = np.linspace(0, 1, 5) + d = np.mean(np.diff(nominal)) + + #---------- + # Test 1: Same number of measurements + + # Vectors used to rearrange and then recover a set of "measured" positions + isrt = np.arange(nominal.size) + rng.shuffle(isrt) + srt = np.argsort(isrt) + + # Generate fake data with no matching ambiguity + measured = rng.uniform(low=-d/3, high=d/3, size=nominal.size) + nominal[isrt] + + match = slitdesign_matching.match_positions_1D(measured, nominal) + + assert np.array_equal(match, srt), 'Bad match' + + #---------- + # Test 2: Add ambiguity by appending another measurement that is a better + # match than the existing one + measured = np.append(measured, [nominal[2]]) + + match = slitdesign_matching.match_positions_1D(measured, nominal) + assert match[2] == nominal.size, 'Should use new measurement' + assert np.sum(np.absolute(nominal - measured[match])) \ + < np.sum(np.absolute(nominal - measured[srt])), 'Should get a better match' + + #---------- + # Test 3: Fake a missing measurement + measured = measured[1:5] + match = slitdesign_matching.match_positions_1D(measured, nominal) + + assert match.size == nominal.size, 'Match vector should be the same size as nominal' + assert match[isrt[0]] == -1, 'Should not match missing element' + + #---------- + # Test 4: Missing a measurement and effectively a repeat of an existing one + measured = np.append(measured, [nominal[1]]) + match = slitdesign_matching.match_positions_1D(measured, nominal) + + assert match.size == nominal.size, 'Match vector should be the same size as nominal' + assert np.all(match > -1), 'All should match' + assert np.sum(np.absolute(nominal - measured[match]) > d/2) == 1, 'Should be one large outlier' + + #---------- + # Test 5: Same as test 4, but include a tolerance that should remove the outlier + match = slitdesign_matching.match_positions_1D(measured, nominal, tol=d/2) + + assert match.size == nominal.size, 'Match vector should be the same size as nominal' + assert match[isrt[0]] == -1, 'Should not match missing element' + + diff --git a/pypeit/tests/test_slittrace.py b/pypeit/tests/test_slittrace.py index bf105dd9ca..8d4b3571c7 100644 --- a/pypeit/tests/test_slittrace.py +++ b/pypeit/tests/test_slittrace.py @@ -14,8 +14,17 @@ def test_bits(): # Make sure bits are correct bm = SlitTraceBitMask() + assert bm.bits['SHORTSLIT'] == 0, 'Bits changed' + assert bm.bits['BOXSLIT'] == 1, 'Bits changed' assert bm.bits['USERIGNORE'] == 2, 'Bits changed' - assert bm.bits['BADFLATCALIB'] == 6, 'Bits changed' + assert bm.bits['BADWVCALIB'] == 3, 'Bits changed' + assert bm.bits['BADTILTCALIB'] == 4, 'Bits changed' + assert bm.bits['BADALIGNCALIB'] == 5, 'Bits changed' + assert bm.bits['SKIPFLATCALIB'] == 6, 'Bits changed' + assert bm.bits['BADFLATCALIB'] == 7, 'Bits changed' + assert bm.bits['BADREDUCE'] == 8, 'Bits changed' + assert bm.bits['BADSKYSUB'] == 9, 'Bits changed' + assert bm.bits['BADEXTRACT'] == 10, 'Bits changed' def test_init(): diff --git a/pypeit/tests/test_wvcalib.py b/pypeit/tests/test_wvcalib.py index 4e85aef2a4..d49100fb95 100644 --- a/pypeit/tests/test_wvcalib.py +++ b/pypeit/tests/test_wvcalib.py @@ -17,6 +17,13 @@ from pypeit.tests.tstutils import data_path +def test_wavefit_hduprefix(): + spat_id = 175 + prefix = wv_fitting.WaveFit.hduext_prefix_from_spatid(spat_id) + _spat_id = wv_fitting.WaveFit.parse_spatid_from_hduext(prefix) + assert spat_id == _spat_id, 'Bad parse' + + def test_wavefit(): "Fuss with the WaveFit DataContainer" out_file = Path(data_path('test_wavefit.fits')).resolve() diff --git a/pypeit/tests/tstutils.py b/pypeit/tests/tstutils.py index 7222d9d3b9..68d1c71028 100644 --- a/pypeit/tests/tstutils.py +++ b/pypeit/tests/tstutils.py @@ -36,8 +36,7 @@ # Tests require the Telluric file (Mauna Kea) par = Spectrograph.default_pypeit_par() -tell_test_grid = data.get_telgrid_filepath('TelFit_MaunaKea_3100_26100_R20000.fits') -#tell_test_grid = data.Paths.telgrid / 'TelFit_MaunaKea_3100_26100_R20000.fits' +tell_test_grid = data.get_telgrid_filepath('TellPCA_3000_26000_R25000.fits') telluric_required = pytest.mark.skipif(not tell_test_grid.is_file(), reason='no Mauna Kea telluric file') diff --git a/pypeit/tracepca.py b/pypeit/tracepca.py index 2f59ff6e69..14ef7a3a44 100644 --- a/pypeit/tracepca.py +++ b/pypeit/tracepca.py @@ -112,7 +112,7 @@ def __init__(self, trace_cen=None, npca=None, pca_explained_var=99.0, reference_ coo=None): # Instantiate as an empty DataContainer - super(TracePCA, self).__init__() + super().__init__() self.is_empty = True # Only do the decomposition if the trace coordinates are provided. @@ -240,7 +240,7 @@ def predict(self, x): def _bundle(self, ext='PCA'): """Bundle the data for writing.""" - d = super(TracePCA, self)._bundle(ext=ext) + d = super()._bundle(ext=ext) if self.pca_coeffs_model is None: return d @@ -264,8 +264,7 @@ def _parse(cls, hdu, hdu_prefix=None, **kwargs): argument descriptions. """ # Run the default parser to get most of the data - d, version_passed, type_passed, parsed_hdus \ - = super(TracePCA, cls)._parse(hdu, hdu_prefix=hdu_prefix) + d, version_passed, type_passed, parsed_hdus = super()._parse(hdu, hdu_prefix=hdu_prefix) # This should only ever read one hdu! if len(parsed_hdus) > 1: @@ -291,7 +290,7 @@ def from_dict(cls, d=None): :class:`~pypeit.datamodel.DataContainer.from_dict` that appropriately toggles :attr:`is_empty`. """ - self = super(TracePCA, cls).from_dict(d=d) + self = super().from_dict(d=d) self.is_empty = False return self diff --git a/pypeit/wavecalib.py b/pypeit/wavecalib.py index 3c5a3be0d1..1902aead36 100644 --- a/pypeit/wavecalib.py +++ b/pypeit/wavecalib.py @@ -12,6 +12,7 @@ from linetools.utils import jsonify from astropy.table import Table +from astropy.io import fits from pypeit import msgs from pypeit.core import arc, qa @@ -159,62 +160,85 @@ def _bundle(self): return _d @classmethod - def _parse(cls, hdu, **kwargs): + def from_hdu(cls, hdu, chk_version=True, **kwargs): """ - See :func:`~pypeit.datamodel.DataContainer._parse` for description and - list of returned objects. All keyword arguments are ignored by this - function! + Instantiate the object from an HDU extension. + + This overrides the base-class method. Overriding this method is + preferrable to overriding the ``_parse`` method because it makes it + easier to deal with the :class:`~pypeit.datamodel.DataContainer` nesting + of this object. + + Args: + hdu (`astropy.io.fits.HDUList`_, `astropy.io.fits.ImageHDU`_, `astropy.io.fits.BinTableHDU`_): + The HDU(s) with the data to use for instantiation. + chk_version (:obj:`bool`, optional): + If True, raise an error if the datamodel version or + type check failed. If False, throw a warning only. + kwargs (:obj:`dict`, optional): + Used for consistency with base class. Ignored. """ - # Grab everything but the bsplines - _d, dm_version_passed, dm_type_passed, parsed_hdus = super()._parse(hdu) - # Now the wave_fits - list_of_wave_fits = [] - list_of_wave2d_fits = [] - list_of_fwhm_fits = [] - spat_ids = [] - for ihdu in hdu: - if 'WAVEFIT' in ihdu.name: - # Allow for empty - if len(ihdu.data) == 0: - # TODO: This is a hack. We shouldn't be writing empty HDUs, - # except for the primary HDU. - iwavefit = wv_fitting.WaveFit(ihdu.header['SPAT_ID'], ech_order=ihdu.header.get('ECH_ORDER')) - else: - # TODO -- Replace the following with WaveFit._parse() and pass that back!! - iwavefit = wv_fitting.WaveFit.from_hdu(ihdu)# , chk_version=False) - parsed_hdus += ihdu.name - if iwavefit.version != wv_fitting.WaveFit.version: - msgs.warn("Your WaveFit is out of date!!") - # Grab PypeItFit (if it exists) - hdname = ihdu.name.replace('WAVEFIT', 'PYPEITFIT') - if hdname in [khdu.name for khdu in hdu]: - iwavefit.pypeitfit = fitting.PypeItFit.from_hdu(hdu[hdname]) - parsed_hdus += hdname - list_of_wave_fits.append(iwavefit) - # Grab SPAT_ID for checking - spat_ids.append(iwavefit.spat_id) - elif 'WAVE2DFIT' in ihdu.name: - iwave2dfit = fitting.PypeItFit.from_hdu(ihdu) - list_of_wave2d_fits.append(iwave2dfit) - parsed_hdus += ihdu.name - elif 'FWHMFIT' in ihdu.name: - # TODO: This is a hack. We shouldn't be writing empty HDUs, - # except for the primary HDU. - ifwhmfit = fitting.PypeItFit() if len(ihdu.data) == 0 \ - else fitting.PypeItFit.from_hdu(ihdu) - list_of_fwhm_fits.append(ifwhmfit) - parsed_hdus += ihdu.name + # Run the default parser to get most of the data. This won't parse the + # extensions with the WAVEFIT, WAVE2DFIT, or FWHMFIT results. + d, version_passed, type_passed, parsed_hdus = super()._parse(hdu) # Check - if spat_ids != _d['spat_ids'].tolist(): - #embed(header="198 of wavecalib.py") - msgs.error("Bad parsing of WaveCalib") - # Finish - _d['wv_fits'] = np.asarray(list_of_wave_fits) - if len(list_of_wave2d_fits) > 0: - _d['wv_fit2d'] = np.asarray(list_of_wave2d_fits) - if len(list_of_fwhm_fits) > 0: - _d['fwhm_map'] = np.asarray(list_of_fwhm_fits) - return _d, dm_version_passed, dm_type_passed, parsed_hdus + cls._check_parsed(version_passed, type_passed, chk_version=chk_version) + + # Get the list of all extensions + ext = [h.name for h in hdu] if isinstance(hdu, fits.HDUList) else [hdu.name] + + # Get the SPAT_IDs + if 'SPAT_IDS' in parsed_hdus: + # Use the ones parsed above + spat_ids = d['spat_ids'] + else: + # This line parses all the spat_ids from the extension names, + # filters out any None values from the list, and gets the unique set + # of integers + spat_ids = np.unique(list(filter(None.__ne__, + [wv_fitting.WaveFit.parse_spatid_from_hduext(e) for e in ext]))) + + # Parse all the WAVEFIT extensions + wave_fits = [] + for spat_id in spat_ids: + _ext = wv_fitting.WaveFit.hduext_prefix_from_spatid(spat_id)+'WAVEFIT' + if _ext not in ext: + continue + # TODO: I (KBW) don't think we should be writing empty HDUs + if len(hdu[_ext].data) == 0: + wave_fits += [wv_fitting.WaveFit(hdu[_ext].header['SPAT_ID'], + ech_order=hdu[_ext].header.get('ECH_ORDER'))] + else: + wave_fits += [wv_fitting.WaveFit.from_hdu(hdu, spat_id=spat_id, + chk_version=chk_version)] + if len(wave_fits) > 0: + d['wv_fits'] = np.asarray(wave_fits) + + # Parse all the WAVE2DFIT extensions + # TODO: It would be good to have the WAVE2DFIT extensions follow the + # same naming convention as the WAVEFIT extensions... + wave2d_fits = [fitting.PypeItFit.from_hdu(hdu[e], chk_version=chk_version) + for e in ext if 'WAVE2DFIT' in e] + if len(wave2d_fits) > 0: + d['wv_fit2d'] = np.asarray(wave2d_fits) + + # Parse all the FWHMFIT extensions + fwhm_fits = [] + for _ext in ext: + if 'FWHMFIT' not in _ext: + continue + # TODO: I (KBW) don't think we should be writing empty HDUs + fwhm_fits += [fitting.PypeItFit() if len(hdu[_ext].data) == 0 \ + else fitting.PypeItFit.from_hdu(hdu[_ext], chk_version=chk_version)] + if len(fwhm_fits) > 0: + d['fwhm_map'] = np.asarray(fwhm_fits) + + # Instantiate the object + self = cls.from_dict(d=d) + # This is a CalibFrame, so parse the relevant keys for the naming system + self.calib_keys_from_header(hdu[parsed_hdus[0]].header) + # Return the constructed object + return self @property def par(self): @@ -304,8 +328,9 @@ def build_waveimg(self, tilts, slits, spat_flexure=None, spec_flexure=None): # Setup #ok_slits = slits.mask == 0 - bpm = slits.mask.astype(bool) - bpm &= np.logical_not(slits.bitmask.flagged(slits.mask, flag=slits.bitmask.exclude_for_reducing)) +# bpm = slits.mask.astype(bool) +# bpm &= np.logical_not(slits.bitmask.flagged(slits.mask, flag=slits.bitmask.exclude_for_reducing)) + bpm = slits.bitmask.flagged(slits.mask, and_not=slits.bitmask.exclude_for_reducing) ok_slits = np.logical_not(bpm) # image = np.zeros_like(tilts) @@ -401,8 +426,8 @@ def wave_diagnostics(self, print_diag=False): diag['IDs_Wave_cov(%)'] = lines_cov diag['IDs_Wave_cov(%)'].format = '0.1f' # FWHM - diag['mesured_fwhm'] = [0. if wvfit.fwhm is None else wvfit.fwhm for wvfit in self.wv_fits] - diag['mesured_fwhm'].format = '0.1f' + diag['measured_fwhm'] = [0. if wvfit.fwhm is None else wvfit.fwhm for wvfit in self.wv_fits] + diag['measured_fwhm'].format = '0.1f' # RMS diag['RMS'] = [0 if wvfit.rms is None else wvfit.rms for wvfit in self.wv_fits] diag['RMS'].format = '0.3f' diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index 7b4e805db9..8d760cf0cc 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -173,7 +173,8 @@ def spatid_to_zero(self, spat_id): mtch = self.spat_id == spat_id return np.where(mtch)[0][0] - def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False): + def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False, + chk_version=True): """ Show in ginga or mpl Tiltimg with the tilts traced and fitted overlaid @@ -187,18 +188,24 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False): If True, show the image in ginga. Otherwise, use matplotlib. show_traces (bool, optional): If True, show the traces of the tilts on the image. - + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict + version checking to ensure a valid file. If False, the code + will try to keep going, but this may lead to faults and quiet + failures. User beware! """ # get tilt_img_dict if (Path(self.calib_dir).resolve() / self.tiltimg_filename).exists(): - tilt_img_dict = buildimage.TiltImage.from_file(Path(self.calib_dir).resolve() / self.tiltimg_filename) + cal_file = Path(self.calib_dir).resolve() / self.tiltimg_filename + tilt_img_dict = buildimage.TiltImage.from_file(cal_file, chk_version=chk_version) else: msgs.error(f'Tilt image {str((Path(self.calib_dir).resolve() / self.tiltimg_filename))} NOT FOUND.') # get slits slitmask = None if (Path(self.calib_dir).resolve() / self.slits_filename).exists(): - slits = slittrace.SlitTraceSet.from_file(Path(self.calib_dir).resolve() / self.slits_filename) + cal_file = Path(self.calib_dir).resolve() / self.slits_filename + slits = slittrace.SlitTraceSet.from_file(cal_file, chk_version=chk_version) _slitmask = slits.slit_img(initial=True, flexure=self.spat_flexure) _left, _right, _mask = slits.select_edges(flexure=self.spat_flexure) gpm = _mask == 0 @@ -215,7 +222,7 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False): if waveimg is None and slits is not None and same_size and in_ginga: wv_calib_name = wavecalib.WaveCalib.construct_file_name(self.calib_key, calib_dir=self.calib_dir) if Path(wv_calib_name).resolve().exists(): - wv_calib = wavecalib.WaveCalib.from_file(wv_calib_name) + wv_calib = wavecalib.WaveCalib.from_file(wv_calib_name, chk_version=chk_version) tilts = self.fit2tiltimg(slitmask, flexure=self.spat_flexure) waveimg = wv_calib.build_waveimg(tilts, slits, spat_flexure=self.spat_flexure) else: @@ -780,7 +787,7 @@ def run(self, doqa=True, debug=False, show=False): # Record the Mask bpmtilts = np.zeros_like(self.slits.mask, dtype=self.slits.bitmask.minimum_dtype()) for flag in ['BADTILTCALIB']: - bpm = self.slits.bitmask.flagged(self.slits.mask, flag) + bpm = self.slits.bitmask.flagged(self.slits.mask, flag=flag) if np.any(bpm): bpmtilts[bpm] = self.slits.bitmask.turn_on(bpmtilts[bpm], flag) diff --git a/setup.cfg b/setup.cfg index 969670b068..5e57b0d31d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -118,6 +118,7 @@ console_scripts = pypeit_obslog = pypeit.scripts.obslog:ObsLog.entry_point #pypeit_parse_calib_id = pypeit.scripts.parse_calib_id:ParseCalibID.entry_point pypeit_parse_slits = pypeit.scripts.parse_slits:ParseSlits.entry_point + pypeit_print_bpm = pypeit.scripts.print_bpm:PrintBPM.entry_point pypeit_qa_html = pypeit.scripts.qa_html:QAHtml.entry_point pypeit_ql = pypeit.scripts.ql:QL.entry_point #pypeit_ql_jfh_multislit = pypeit.scripts.ql_multislit:QL_Multislit.entry_point