diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml index b0c16d58a7..ae23b4affe 100644 --- a/.github/workflows/ci_cron.yml +++ b/.github/workflows/ci_cron.yml @@ -16,7 +16,7 @@ jobs: matrix: os: [ubuntu-latest] python: ['3.9', '3.10', '3.11'] - toxenv: [test-alldeps, test-numpydev, test-linetoolsdev, test-gingadev, test-astropydev, conda] + toxenv: [test-alldeps, test-numpydev, test-linetoolsdev, test-gingadev, test-astropydev] steps: - name: Check out repository uses: actions/checkout@v3 @@ -31,4 +31,4 @@ jobs: python -m pip install --upgrade pip tox - name: Test with tox run: | - tox -e ${{ matrix.toxenv }} + tox -e ${{ matrix.python }}-${{ matrix.toxenv }} diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 7f8526535d..6e735c802f 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -18,7 +18,7 @@ jobs: matrix: os: [ubuntu-latest] python: ['3.9', '3.10', '3.11'] - toxenv: [test, test-alldeps-cov, test-linetoolsdev, test-gingadev, test-astropydev, conda] + toxenv: [test, test-alldeps-cov, test-linetoolsdev, test-gingadev, test-astropydev] steps: - name: Check out repository uses: actions/checkout@v3 @@ -31,7 +31,7 @@ jobs: python -m pip install --upgrade pip tox - name: Test with tox run: | - tox -e ${{ matrix.toxenv }} + tox -e ${{ matrix.python }}-${{ matrix.toxenv }} - name: Upload coverage to codecov if: "contains(matrix.toxenv, '-cov')" uses: codecov/codecov-action@v3 @@ -62,7 +62,22 @@ jobs: python -m pip install --upgrade pip tox - name: Test with tox run: | - tox -e ${{ matrix.toxenv }} + tox -e ${{ matrix.python }}-${{ matrix.toxenv }} + + conda: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Conda environment check + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install base dependencies + run: | + python -m pip install --upgrade pip tox + - name: Run pypeit tests from environment built with conda + run: | + tox -e conda codestyle: runs-on: ubuntu-latest diff --git a/CHANGES.rst b/CHANGES.rst index fcfe167030..60293c93ee 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,58 @@ +1.14.0 (18 Sep 2023) +-------------------- + +- Add support for Gemini/GNIRS (IFU) +- Add support for Keck/KCRM +- Added a script to convert a wavelength solution into something that can be placed in the reid archive. +- Hotfix for GTC/OSIRIS lamp list +- Hotfix for Arc1D stats annotations on the QA +- Hotfix for metadata (correctly set config_independent frames when multiple configurations are being setup) +- Hotfix for metadata (support lists in ``config_independent_frames()``) +- Hotfix for rebin (speed-up and conserves flux) +- Hotfix for skysub regions GUI that used np.bool +- Hotfix to stop pypeit_setup from crashing on data from lbt_luci1, lbt_luci2, magellan_fire, + magellan_fire_long, p200_tspec, or vlt_sinfoni. +- Hotfix to set BPM for each type of calibration file. +- Adds Keck/ESI to PypeIt +- Instrumental FWHM map is calculated and output in ``Calibrations`` and ``spec1d`` files. +- Adds Keck/ESI to PypeIt +- Add MDM/Modspec spectrograph +- Store user-generated wavelength solution in pypeit cache +- Improvements to wavelength grids and masking in coadd routines. +- Fixed a bug in echelle coadding where the wrong coadded spectra were being + used in final stacks. +- Sensitivity function models can now be computed relative to the blaze + spectrum. +- Refactored coadding routines to work with lists to support coadding data from + different setups. +- Changes to how masking is dealt with in extraction to fix a bug in how masks + were being treated for echelle data +- Various fixes and changes required to add more support for Keck/HIRES and JWST +- Fix a bug in ``spectrograph.select_detectors``, where a list of ``slitspatnum`` could not be used. +- Improvements in 2D coaddition + - Fix a bug in `pypeit_setup_coadd2d` for the output file name of the .coadd2d file + - Added possibility to specify more than one Science folder in `pypeit_setup_coadd2d` + - Now ``only_slits`` parameter in `pypeit_coadd_2dspec` includes the detector number (similar to ``slitspatnum``) + - Added ``exclude_slits`` parameter in `pypeit_coadd_2dspec` to exclude specific slits + - Fix wrong RA & Dec for 2D coadded serendips +- Changed calibration frame naming as an attempt to avoid very long names for + files with many calibration groups. Sequential numbers are reduced to a + range; e.g., ``'0-1-2-3-4'`` becomes ``'0+4'`` and + ``'3-5-6-10-11-12-15-18-19'`` becomes ``'3-5+6-10+12-15-18+19'`` +- HIRES wavelength solution improvements galor +- Added `redo_slits` option +- Refactored ``load_line_lists()`` yet again! +- Improvements for keck/LRIS + - Generated wavelength templates for all the LRIS grism & grating + - Added FeAr line list + - Improved calibration association and frame typing + - Improved and added documentation + - Changes to ``metadata.py`` including commenting out, in the pypeit file, + files that have frametype None (this prevent ``run_pypeit`` to crash) + - Added a function ``check_spectrograph()`` (currently only defined for LRIS), + that checks (during ``pypeit_setup``) if the selected spectrograph is the + corrected one for the data used. + 1.13.0 (2 June 2023) -------------------- @@ -7,7 +62,6 @@ - Allow user control of the local sky subtraction window - Deprecate use of python 3.8 with PypeIt, allow python 3.11 - Make pypeit_show_2dspec (somewhat) backwards compatible. -- Added the option to disable strict version checking for 1d coadds. - Hotfix for KCWI when using alignment (aka ContBars) frames for the astrometric correction. - Sensitivity function masking and output updates - Fixed a bug in the `variance_model` calculation for combined images. @@ -27,7 +81,8 @@ - The ``'calib'`` column is now always added to the pypeit file, regardless of whether or not you also request the ``'comb_id'`` and ``'bkg_id'`` columns. - Names of associated calibration frames now written to ``spec2d`` file headers. -- Major quicklook updates. ql_multislit.py deprecated. +- Added the option to disable strict version checking for 1d coadds. +- Major quicklook updates. ql_multislit.py temporarily deprecated. - Improve speed in ginga visualization of traces and added `pypeit_chk_tilts`. Note that this script uses an update of the tilts datamodel, so it will not work on older reductions. @@ -1523,7 +1578,7 @@ better with 2d coadds. - Other odds and ends including code flow doc - Introduce pypit/par and pypit/config directories - Introduce PypitPar as an initial step toward refactoring the front end -- Move spectrograph specific code into spectographs/ folder +- Move spectrograph specific code into spectrographs/ folder - Introduces the Spectrographs class - Introduces the Calibrations class with Notebook - Bug fix in view_fits script diff --git a/LICENSE.rst b/LICENSE.rst index 48b41e9629..3466086125 100644 --- a/LICENSE.rst +++ b/LICENSE.rst @@ -1,4 +1,4 @@ -Copyright (c) 2018-2019, PypeIt Developers +Copyright (c) 2018-2023, PypeIt Developers All rights reserved. Redistribution and use in source and binary forms, with or without modification, diff --git a/README.rst b/README.rst index cb46f3a511..1bfea2284e 100644 --- a/README.rst +++ b/README.rst @@ -97,7 +97,7 @@ follow our `Code of Conduct Along with our extensive `online documentation `__, we encourage the PypeIt user -base to communicate via our `PypeIt Users Slack `__. +base to communicate via our `PypeIt Users Slack `__. All are welcome to join using `this invitation link `__. If you find a bug (particularly one that is experienced by others in the Users @@ -170,4 +170,6 @@ development of PypeIt. * Milan Roberson * Timothy Pickering * Timothy Ellsworth-Bowers +* Gregory Simonian +* Heather Martin diff --git a/doc/Makefile b/doc/Makefile index 3ec6d18a2e..c71afd4a43 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -84,7 +84,7 @@ htmlonly: picky: make apirst - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) -n $(BUILDDIR)/html + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) -n -vvv -T $(BUILDDIR)/html -W @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." diff --git a/doc/api/pypeit.scripts.arxiv_solution.rst b/doc/api/pypeit.scripts.arxiv_solution.rst new file mode 100644 index 0000000000..edae4df2cf --- /dev/null +++ b/doc/api/pypeit.scripts.arxiv_solution.rst @@ -0,0 +1,8 @@ +pypeit.scripts.arxiv\_solution module +===================================== + +.. automodule:: pypeit.scripts.arxiv_solution + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.scripts.ql_multislit.rst b/doc/api/pypeit.scripts.ql_multislit.rst deleted file mode 100644 index 88a8b4b553..0000000000 --- a/doc/api/pypeit.scripts.ql_multislit.rst +++ /dev/null @@ -1,8 +0,0 @@ -pypeit.scripts.ql\_multislit module -=================================== - -.. automodule:: pypeit.scripts.ql_multislit - :members: - :private-members: - :undoc-members: - :show-inheritance: diff --git a/doc/api/pypeit.scripts.rst b/doc/api/pypeit.scripts.rst index f6765004a8..5d312d0e49 100644 --- a/doc/api/pypeit.scripts.rst +++ b/doc/api/pypeit.scripts.rst @@ -7,6 +7,7 @@ Submodules .. toctree:: :maxdepth: 4 + pypeit.scripts.arxiv_solution pypeit.scripts.cache_github_data pypeit.scripts.chk_alignments pypeit.scripts.chk_edges @@ -34,7 +35,6 @@ Submodules pypeit.scripts.parse_slits pypeit.scripts.qa_html pypeit.scripts.ql - pypeit.scripts.ql_multislit pypeit.scripts.run_pypeit pypeit.scripts.scriptbase pypeit.scripts.sensfunc diff --git a/doc/api/pypeit.spectrographs.keck_esi.rst b/doc/api/pypeit.spectrographs.keck_esi.rst new file mode 100644 index 0000000000..b87b500d22 --- /dev/null +++ b/doc/api/pypeit.spectrographs.keck_esi.rst @@ -0,0 +1,8 @@ +pypeit.spectrographs.keck\_esi module +===================================== + +.. automodule:: pypeit.spectrographs.keck_esi + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.spectrographs.mdm_modspec.rst b/doc/api/pypeit.spectrographs.mdm_modspec.rst new file mode 100644 index 0000000000..5e669a872b --- /dev/null +++ b/doc/api/pypeit.spectrographs.mdm_modspec.rst @@ -0,0 +1,8 @@ +pypeit.spectrographs.mdm\_modspec module +======================================== + +.. automodule:: pypeit.spectrographs.mdm_modspec + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.spectrographs.rst b/doc/api/pypeit.spectrographs.rst index dd574769a5..c53968bcbb 100644 --- a/doc/api/pypeit.spectrographs.rst +++ b/doc/api/pypeit.spectrographs.rst @@ -15,6 +15,7 @@ Submodules pypeit.spectrographs.jwst_nircam pypeit.spectrographs.jwst_nirspec pypeit.spectrographs.keck_deimos + pypeit.spectrographs.keck_esi pypeit.spectrographs.keck_hires pypeit.spectrographs.keck_kcwi pypeit.spectrographs.keck_lris @@ -26,6 +27,7 @@ Submodules pypeit.spectrographs.ldt_deveny pypeit.spectrographs.magellan_fire pypeit.spectrographs.magellan_mage + pypeit.spectrographs.mdm_modspec pypeit.spectrographs.mdm_osmos pypeit.spectrographs.mmt_binospec pypeit.spectrographs.mmt_bluechannel diff --git a/doc/calibrations/calibrations.rst b/doc/calibrations/calibrations.rst index bc7d175a08..d27c68f43c 100644 --- a/doc/calibrations/calibrations.rst +++ b/doc/calibrations/calibrations.rst @@ -1,4 +1,6 @@ +.. include:: ../include/links.rst + .. _calibrations: ============ @@ -175,15 +177,22 @@ All reduced calibration frames are named according to their primary calibration type (e.g., ``Arc``). They are also assigned a unique identifier that is a combination of: - - the instrument configuration (setup) identifier (e.g., ``A``), + #. the instrument configuration (setup) identifier (e.g., ``A``), - - the list of associated calibration groups (e.g., ``1-2`` or ``all``), and + #. a compressed list of associated calibration groups (e.g., ``1+2`` or ``all``), and - - the detector or mosaic identifier (e.g., ``DET01`` or ``MSC01``). + #. the detector or mosaic identifier (e.g., ``DET01`` or ``MSC01``). -.. note:: +For the second component, sequential numbers are reduced to a range; e.g., +``'0-1-2-3-4'`` becomes ``'0+4'`` and ``'3-5-6-10-11-12-15-18-19'`` becomes +``'3-5+6-10+12-15-18+19'``. + +.. warning:: If you have a lot of calibration groups in your pypeit file, you may end up - with very long file names! + with very long file names! This may cause a fault when the file name is + included in the header of the output fits files. If using the calibration + group ``all`` doesn't solve the problem or isn't possible given your + application, please `Submit an issue`_. diff --git a/doc/calibrations/image_proc.rst b/doc/calibrations/image_proc.rst index e034cb5231..efd4159d68 100644 --- a/doc/calibrations/image_proc.rst +++ b/doc/calibrations/image_proc.rst @@ -198,7 +198,7 @@ in our documentation as ``(nspec,nspat)``. The operations required to flip/transpose the image arrays to match the PypeIt convention are dictated by instrument-specific :class:`~pypeit.images.detector_container.DetectorContainer` parameters and performed by -:func:`~pypeit.spectrograph.spectrographs.Spectrograph.orient_image`. Image +:func:`~pypeit.spectrographs.spectrograph.Spectrograph.orient_image`. Image orientation will be performed if the ``orient`` parameter is true. .. warning:: diff --git a/doc/calibrations/slit_tracing.rst b/doc/calibrations/slit_tracing.rst index 9e444c8a35..5eefda7ea8 100644 --- a/doc/calibrations/slit_tracing.rst +++ b/doc/calibrations/slit_tracing.rst @@ -19,12 +19,40 @@ task owing to the wide variety in: Developing a single algorithm to handle all of these edge cases (pun intended) is challenging if not impossible. Therefore, there are a number of user-input parameters that one may need to consider when running PypeIt (see -below). +below :ref:`slit_tracing_issues` and :ref:`slit_tracing_customizing`). Underlying the effort is the :class:`~pypeit.edgetrace.EdgeTraceSet` class; see :func:`~pypeit.edgetrace.EdgeTraceSet.auto_trace` for a description of the algorithm. +Slit-mask design matching +------------------------- + +PypeIt can incorporate information about the slit-mask design into the +slit-tracing process. This is primarily performed by +:func:`pypeit.edgetrace.EdgeTraceSet.maskdesign_matching`, which matches +the slit edges traced by PypeIt to the slit edges predicted +using the slit-mask design information stored in the observations. +Moreover, :func:`~pypeit.edgetrace.EdgeTraceSet.maskdesign_matching` +uses the predicted slit edges positions to add slit traces that have +not been detected in the image. + + +This functionality at the moment is implemented only for these +:ref:`slitmask_info_instruments` and is switched on by setting +``use_maskdesign`` flag in :ref:`edgetracepar` to True. Other parameters +may need to be adjusted as well, depending on the instrument (see +:ref:`slitmask_ids_report` and the relevant instrument documentation pages). + +.. _slitmask_info_instruments: + +Slit-mask design Spectrographs +++++++++++++++++++++++++++++++ +- :doc:`../spectrographs/deimos` +- :doc:`../spectrographs/mosfire` +- :doc:`../spectrographs/lris` (limited) +- :doc:`../spectrographs/gemini_gmos` (limited) + Viewing ======= @@ -61,6 +89,8 @@ additional output that can be used to diagnose the parameterized fits to the edge traces and the PCA decomposition. Fair warning that, for images with many slits, these plots can be laborious to wade through... +.. _slit_tracing_issues: + Known Slit Tracing Issues ========================= @@ -189,6 +219,8 @@ For example: This will remove any slit on detector 2 that contains ``x_spat=2121`` at ``y_spec=2000`` and similarly for the slit on ``det=3``. +.. _slit_tracing_customizing: + Slit Tracing Customizing ======================== diff --git a/doc/calibrations/slits.rst b/doc/calibrations/slits.rst index 2ea786986c..e331640198 100644 --- a/doc/calibrations/slits.rst +++ b/doc/calibrations/slits.rst @@ -45,10 +45,10 @@ This will show a table that looks like this: 322 699091 248.21048545837402 .. 278.352352142334 345.8145122528076 .. 376.4430561065674 251.7797389884793 .. 281.9216056724392 345.8145122528076 .. 376.4430561065674 297.0124988555908 .. 327.3977041244507 0 0 -inf inf 457 699080 350.76593017578125 .. 379.9788398742676 514.4993572235107 .. 544.0507583618164 355.0206685112119 .. 384.23357820969824 513.3126446738094 .. 542.864045812115 432.632643699646 .. 462.014799118042 0 0 -inf inf -In addition, if reducing :doc:`../spectrographs/deimos` or -:doc:`../spectrographs/mosfire` data and slit-mask design matching is performed -(see :ref:`deimos-mask-matching` for DEIMOS and :ref:`mosfire-edge-tracing` for -MOSFIRE), a second `astropy.io.fits.BinTableHDU`_ is written to disk. +In addition, if reducing data from these :ref:`slitmask_info_instruments` +and slit-mask design matching is performed (see e.g., :ref:`deimos-mask-matching` +for DEIMOS and :ref:`mosfire-edge-tracing` for MOSFIRE), a second +`astropy.io.fits.BinTableHDU`_ is written to disk. .. code-block:: console diff --git a/doc/calibrations/tilt.rst b/doc/calibrations/tilt.rst index 12724b924b..614d4be345 100644 --- a/doc/calibrations/tilt.rst +++ b/doc/calibrations/tilt.rst @@ -58,7 +58,7 @@ Current TiltImage Data Model ============================ Internally, the image is held in -:class:`pypeit.tiltimage.TiltImage` +:class:`pypeit.images.buildimage.tiltimage.TiltImage` which subclasses from :class:`pypeit.images.pypeitimage.PypeItImage` and :class:`pypeit.datamodel.DataContainer`. diff --git a/doc/calibrations/wave_calib.rst b/doc/calibrations/wave_calib.rst index 01130132b5..f11dcca81c 100644 --- a/doc/calibrations/wave_calib.rst +++ b/doc/calibrations/wave_calib.rst @@ -81,8 +81,9 @@ HgI 2900-12000 28 February 2022 KrI 4000-10000 3 May 2018 NeI 5000-12000 3 May 2018 XeI 4000-12000 3 May 2018 -ZnI 3000-5000 21 December 2016 +ZnI 3000-5000 6 Sep 2023 ThAr 3000-11000 9 January 2018 +FeAr 3000-9000 6 Sep 2023 ====== ========== ================ In the case of the ThAr list, all of the lines are taken from the NIST database, @@ -259,6 +260,63 @@ observations, long-slit observations where wavelengths vary (*e.g.*, grating tilts). We are likely to implement this for echelle observations (*e.g.*, HIRES). +.. _wvcalib-echelle: + +Echelle Spectrographs +===================== + +Echelle spectrographs are a special case for wavelength +solutions, primarily because the orders follow the +grating equation. + +In general, the approach is: + + #. Identify the arc lines in each order + + #. Fit the arc lines in each order to a polynomial, individually + + #. Fit a 2D solution to the lines using the order number as a basis + + #. Reject orders where the RMS of the fit (measured in binned pixels) + exceeds ``rms_threshold`` + + #. Attempt to recover the missing orders using the 2D fit and a higher RMS + threshold + + #. Refit the 2D solution + +One should always inspect the outputs, especially the 2D solution +(global and orders). One may then need to modify the ``rms_threshold`` +parameter and/or hand-fit a few of the orders to improve the solution. + +.. _wvcalib-rms-threshold: + +rms_threshold +------------- + +All of the echelle spectrographs have a default ``rms_threshold`` +matched to a default ``FWHM`` parameter (also measured in binned pixels). +The ``rms_threshold`` adopted in the analysis is one +scaled by the measured FWHM from the arc lines +(again, binned pixels) of the adopted calibration files + +That is, each order must satisfy the following: + +.. code-block:: ini + + RMS < rms_threshold * (measured_FWHM/default_FWHM) + +Note: in a future release, we will re-define ``rms_threshold`` to be +in units of the measured FWHM. + +Mosaics +------- + +For echelle spectrographs with multiple detectors *per* camera +that are mosaiced (e.g. Keck/HIRES), we fit the 2D solutions on a *per* detector +basis. Ths is because we have found the mosaic solutions to be +too difficult to make sufficiently accurate. + .. _wvcalib-byhand: By-Hand Approach diff --git a/doc/coadd2d.rst b/doc/coadd2d.rst index 90abc82804..eae4d1d110 100644 --- a/doc/coadd2d.rst +++ b/doc/coadd2d.rst @@ -19,6 +19,32 @@ part of the data reduction process, although it can combine (without weighting) multiple exposures during reductions (see :ref:`2d_combine`). +.. note:: + + Because the flux of the single reduced science frames is expressed in ``counts``, + coadding frames with different exposure times is not recommended. If the user still + wishes to do so, the fluxes of the individual frames are rescaled by the median + exposure time. For example, if we have four frames with exposure times of + ``1800``, ``1800``, ``1800``, and ``1200`` seconds, the exposure + time of the coadded frame will be: + + .. code-block:: python + + coadd_exptime = np.percentile([1800,1800,1800,1200],50, method='higher') + + and the flux of the individual frames will be rescaled by: + + .. code-block:: python + + rescale_factor = coadd_exptime / exptime + + where ``exptime`` is the exposure time of the individual frames. ``coadd_exptime`` is saved + in the header of the coadded frame as ``ALLSPEC2D_EFFECTIVE_EXPTIME``, so that the user can + easily convert the flux of the coadded frame from ``counts`` to ``counts/s``. + + Note, also, that the combination (without weighting) of multiple exposures during main reduction + (i.e, :ref:`2d_combine`) does not perform this rescaling. + .. _coadd2d_file: coadd2d file diff --git a/doc/coadd3d.rst b/doc/coadd3d.rst index 4bb5e8e1d7..d0b2454297 100644 --- a/doc/coadd3d.rst +++ b/doc/coadd3d.rst @@ -179,7 +179,9 @@ frames, but add the following keyword arguments at the top of your [reduce] [[skysub]] joint_fit = True - user_regions = :50,50: + user_regions = : + [flexure] + spec_method = slitcen This ensures that all pixels in the slit are used to generate a complete model of the sky. @@ -266,15 +268,51 @@ cubes covering different wavelength range, but it can coadd multiple spec2D files into a single datacube if the wavelength setup overlaps, and the spatial positions are very similar. -Difficulties with combining multiple datacubes -============================================== +Combining multiple datacubes +============================ PypeIt is able to combine standard star frames for flux calibration, and should not have any difficulty with this. If your science observations are designed so that there is very little overlap between exposures, you should -not expect the automatic combination algorithm to perform well. Instead, you -should output individual data cubes and manually combine the cubes with some -other purpose-built software. +not assume that the automatic combination algorithm will perform well. Instead, +you may prefer to output individual data cubes and manually combine the cubes +with some other purpose-built software. If you know the relative offsets very +well, then you can specify these, and PypeIt can combine all frames into a +single combined datacube. This is the recommended approach, provided that you +know the relative offsets of each frame. In the following example, the first +cube is assumed to be the reference cube (0.0 offset in both RA and Dec), and +the second science frame is offset relative to the first by: + +.. code-block:: ini + + Delta RA x cos(Dec) = 1.0" W + Delta Dec = 2.0" N + +The offset convention used in PypeIt is that positive offsets translate the RA and Dec +of a frame to higher RA (i.e. more East) and higher Dec (i.e. more North). In the above +example, frame 2 is 1" to the West of frame 1, meaning that we need to move frame 2 by +1" to the East (i.e. a correction of +1"). Similarly, we need to more frame 2 by 2" South +(i.e. a correction of -2"). Therefore, in the above example, the coadd3d file would look +like the following: + +.. code-block:: ini + + # User-defined execution parameters + [rdx] + spectrograph = keck_kcwi + detnum = 1 + [reduce] + [[cube]] + combine = True + output_filename = BB1245p4238_datacube.fits + align = True + + # Read in the data + spec2d read + filename | ra_offset | dec_offset + Science/spec2d_scienceframe_01.fits | 0.0 | 0.0 + Science/spec2d_scienceframe_02.fits | 1.0 | -2.0 + spec2d end .. _coadd3d_datamodel: diff --git a/doc/collate1d.rst b/doc/collate1d.rst index b77b3fc3e8..be44fe5526 100644 --- a/doc/collate1d.rst +++ b/doc/collate1d.rst @@ -144,6 +144,11 @@ followed by a list of spec1d files. An example configuration file is shown below # current directory. #outdir = /work/output + # Whether to check that spec1d files and archival sensfunc files have an + # up to date datamodel version. If false (the default) version numbers are + # not checked. + #chk_version = True + # A list of the spec1d files. Wildcards are allowed. # This follows the input file data block format. spec1d read diff --git a/doc/conf.py b/doc/conf.py index 4da58a95b8..50a0c7a93d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -120,9 +120,16 @@ # When nit-picking, ignore these warnings: nitpick_ignore = [ ('py:class', 'optional'), + ('py:class', 'iterable'), + ('py:class', 'ndarray'), + ('py:class', 'dict-like'), ('py:class', 'array-like'), + ('py:class', 'table-like'), + ('py:class', 'float-like'), ('py:class', 'scalar-like'), - ('py:class', 'default') ] + ('py:class', 'default'), + ('py:class', 'callable'), + ] # The reST default role (used for this markup: `text`) to use for all # documents. @@ -342,5 +349,5 @@ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} diff --git a/doc/cookbook.rst b/doc/cookbook.rst index ce8a1be670..915aed4865 100644 --- a/doc/cookbook.rst +++ b/doc/cookbook.rst @@ -13,6 +13,10 @@ should adhere to the following approach. Note that: + - Essentially all PypeIt reduction steps are done via executing command-line + scripts using a terminal window. We provide specific commands below and + throughout our documentation. + - This cookbook provides minimal detail, but serves as a basic introduction to the primary steps used to reduce your data with PypeIt. We guide you to other parts of our documentation that explain specific functionality in more @@ -27,8 +31,9 @@ Note that: - Specific advice on :doc:`spectrographs/spectrographs` is provided in their own doc page (although not every supported spectrograph has stand-alone documentation). - - Invariably something will be out of date. When you see an egregious - example, please holler on GitHub or Slack. + - Invariably something will be out of date in our doc pages. When you see an + egregious example, please holler on `GitHub + `__ or `Slack `__. Finally, note that before you keep going, you should have already done the following: diff --git a/doc/dev/add_missing_obj.rst b/doc/dev/add_missing_obj.rst index 3eb70e3e1b..7e3c0b2f80 100644 --- a/doc/dev/add_missing_obj.rst +++ b/doc/dev/add_missing_obj.rst @@ -15,6 +15,7 @@ Version History 1.1 Debora Pelliccia 28 Jul 2021 1.4.3dev 1.3 Debora Pelliccia 21 Oct 2021 1.6.1dev 1.4 J. Xavier Prochaska 25 Jan 2022 1.7.1dev +1.5 Debora Pelliccia 6 Sep 2023 1.13.1dev ========= =================== =========== =========== ---- @@ -23,7 +24,7 @@ Basics ------ The procedure to determine the location on the slit of undetected objects -using the slitmask design information is currently available for Keck/DEIMOS and Keck/MOSFIRE only +using the slitmask design information is currently available for these :ref:`slitmask_info_instruments` only and it is performed right after the object finding (see :ref:`object_finding`) and the RA, Dec and object name assignment procedures (see :ref:`radec_object_report`) have been completed. diff --git a/doc/dev/development.rst b/doc/dev/development.rst index f27edd3287..ea790be111 100644 --- a/doc/dev/development.rst +++ b/doc/dev/development.rst @@ -404,36 +404,49 @@ tagging process is as follows: core development team will decide to merge the ``develop`` branch into ``release``. - * A branch is created off of ``develop`` (typically called ``staged``) - and then a `PR `_ is issued - to merge ``staged`` into ``release``. This merge must meet the same - `Pull Request Acceptance Requirements`_ when merging new branches - into ``develop``. However, typically the tests do not need to be run because - ``develop`` has already passed these tests and ``staged`` will not make any - substantive changes to the code. - - * **Before being merged into release**, the code is tagged as follows: - + * A branch is created off of ``develop`` (typically called ``staged``) and then + a `PR `_ is issued to merge + ``staged`` into ``release``. This ``release...staged`` PR must meet the same + `Pull Request Acceptance Requirements`_ when merging new branches into + ``develop``. Code review is expected to be limited (because all code changes + will have been reviewed before pulling into ``develop``), but the result of + the dev-suite tests must be shown and approved. The reason for creating the + new branch instead of a direct ``release...develop`` PR is to allow for the + following updates to ``staged`` before merging (``develop`` is a protected + branch and cannot be directly edited): + + * Update the documentation by executing ``cd doc ; make clean ; make + html``, add any updated files, and correct any issued errors/warnings. + + * Fix any test failures. As necessary, an accompanying :ref:`dev-suite` + PR may be issued that includes test fixes required code changes. If + no code changes are required, a :ref:`dev-suite` PR should be issued + that merges its ``develop`` branch directly into its ``main`` branch. + + * Make any final updates to ``CHANGES.rst`` and reset the relevant + version header to be the intended tag number. + + * Add a new release doc to the ``doc/releases`` directory, parsing + changes from the ``CHANGES.rst`` file, and add include it in the + ``whatsnew.rst`` doc. + + * Once the ``release`` branch and the :ref:`dev-suite` ``main`` branch are + updated, the dev-suite tests are re-run using these two branches. These + tests must pass before tagging. Once they pass, the code is tagged as + follows: + .. code-block:: bash - cd $PYPEIT_DIR - - # Edit CHANGES.rst to reflect the ending of a development phase - vi CHANGES.rst - git add -u - git commit -m 'tag CHANGES' - - # Create a tag of the form X.Y.Z (using 1.12.1 here as an example). + # Create a tag of the form X.Y.Z (using 1.14.0 here as an example). # The current autogenerated version is found in pypeit/version.py. - git tag 1.12.1 + cd $PYPEIT_DIR + git checkout release + git pull + git tag 1.14.0 - # Push the changes and the new tag - git push + # Push the new tag git push --tags - * The PR is accepted and ``staged`` is merged into ``release``. Hotfix - branches are deleted, but the ``develop`` branch should not be. - * The tag is released for `pip`_ installation. .. code-block:: bash @@ -476,13 +489,12 @@ tagging process is as follows: DOI --- -If we wish to generate a new DOI for the code, it is as simple -as +If we wish to generate a new DOI for the code, it is as simple as * Generate a `new release on GitHub `_. - * Update the DOI in the README.md + * Update the DOI in the ``README.rst`` ---- @@ -490,7 +502,7 @@ as This document was developed and mutually agreed upon by: Kyle Westfall, J. Xavier Prochaska, Joseph Hennawi. -*Last Modified: 28 Feb 2023* +*Last Modified: 07 Sep 2023* ---- diff --git a/doc/dev/lrisconfig.rst b/doc/dev/lrisconfig.rst new file mode 100644 index 0000000000..3e73cedcb5 --- /dev/null +++ b/doc/dev/lrisconfig.rst @@ -0,0 +1,135 @@ +.. include:: ../include/links.rst + +.. _lris_config_report: + +Automated sorting of LRIS frames by instrument configuration +============================================================== + +Version History +--------------- + + +========= ================ =========== =========== +*Version* *Author* *Date* ``PypeIt`` +========= ================ =========== =========== +1.0 Debora Pelliccia 6 Sep 2023 1.13.1.dev +========= ================ =========== =========== + +---- + +Basics +------ + +To prepare for the data reduction, PypeIt, first, automatically associates fits +files to specific :ref:`frame_types` (see :ref:`lris_frames_report`) and, then, +collects groups of frames in unique instrument configurations (see below). This is performed +by the :ref:`pypeit_setup` script, which sorts the frames and writes a +:ref:`pypeit_file` for each unique configuration. See :ref:`setup_doc`. + + +LRIS configuration identification +--------------------------------- + +The LRIS instrument configurations are determined, in the same +way for ``keck_lris_red``, ``keck_lris_red_orig``, ``keck_lris_red_mark4``, +``keck_lris_blue``, and ``keck_lris_blue_orig`` (unless otherwise noted), +by the function :func:`pypeit.metadata.PypeItMetaData.unique_configurations`, +which finds unique combinations of the following keywords: + +=============== ==================================================================== +``fitstbl`` key Header Key +=============== ==================================================================== +``dispname`` ``GRANAME`` (LRIS RED) or ``GRISNAME`` (LRIS BLUE) +``dichroic`` ``DICHNAME`` +``decker`` ``SLITNAME`` +``binning`` ``BINNING`` +``amp`` ``NUMAMPS`` (``TAPLINES`` for ``keck_lris_red_mark4``) +``dispangle`` ``GRANGLE`` (only LRIS RED) +``cenwave`` ``WAVELEN`` (only LRIS RED) +=============== ==================================================================== + +The unique configurations are determined by collating the relevant metadata from the headers +of all frames found by a run of :ref:`pypeit_setup`, *except* those that are designated as +bias frames. The reason is that bias frames can have header data (e.g., ``dispangle``) +that do not match the instrument configuration that an observer intended for their use; +e.g., the frames were taken before the instrument was fully configured for the night's +observations. Therefore, PypeIt uses the ``dateobs``, ``binning``, ``amp`` keys to match +the bias frames to the configurations with frames taken on the same date, with +the same binning and on the same amplifier. + +After that, :func:`pypeit.metadata.PypeItMetaData.set_configurations` associates each frame +to the relevant unique configuration ("setup"), by assigning a setup identifier +(e.g., A,B,C,D...) to every frames for which the values of the above keywords match the +values of the specific unique configuration. + +LRIS calibration groups +----------------------- + +PypeIt uses the concept of a "calibration group" to define a complete set of +calibration frames (e.g., arcs, flats) and the science frames to which these calibration +frames should be applied. + +By default, :ref:`pypeit_setup` uses the setup identifier to assign frames to a single +calibration group. Frames that are in the same calibration group will have the same PypeIt +keyword ``calib``. No automated procedure exists to do anything except this. +However, the user can edit the :ref:`pypeit_file` to, within a given configuration, assign +specific calibration frames to specific science frames using the data in the ``calib`` column +of the :ref:`data_block`. + +Testing +------- + +Requirement PLL-17 states: "As a user, I expect the pipeline to automatically +and correctly associate calibrations with science frames." + +PypeIt meets this requirement in the majority of use cases, as shown by the tests below. + +Note that the tests described in :ref:`lris_frames_report` are also relevant here +since they show that PypeIt correctly identifies LRIS data and associates +them with a single configuration, all written to a single pypeit file. + +To test that PypeIt can successfully identify multiple +configurations among a set of files, we have added five tests +``${PYPEIT_DEV}/unit_tests/test_setups.py``. They are: + +- ``test_setup_keck_lris_blue_multiconfig()``, +- ``test_setup_keck_lris_blue_orig_multiconfig()`` +- ``test_setup_keck_lris_red_multiconfig()`` +- ``test_setup_keck_lris_red_orig_multiconfig()`` +- ``test_setup_keck_lris_red_mark4_multiconfig()`` + +Here is an example of how to run the tests: + +.. code-block:: bash + + cd ${PYPEIT_DEV}/unit_tests + pytest test_setup.py::test_setup_keck_lris_blue_multiconfig -W ignore + +The tests require that you have downloaded the PypeIt +:ref:`dev-suite` and defined the ``PYPEIT_DEV`` environmental +variable that points to the relevant directory. + +The algorithm for all these tests is the same and is as follows: + + 1. Collect the names of all files in selected LRIS directories + (separately for ``keck_lris_blue``, ``keck_lris_blue_orig``, ``keck_lris_red``, + ``keck_lris_red_orig``, ``keck_lris_red_mark4``). + + 2. Use :class:`~pypeit.pypeitsetup.PypeItSetup` to automatically + identify the configurations for these files. + + 3. Check that the code found two configurations and wrote the + pypeit files for each. + + 4. For each configuration: + + a. Read the pypeit file + + b. Check that the name for the setup is correct ('A' or 'B') + + c. Check that the calibration group is the same for all frames ('0' or '1') + + +Because these tests are now included in the PypeIt +:ref:`unit-tests`, these configuration checks are performed by the +developers for every new version of the code. diff --git a/doc/dev/lrisframes.rst b/doc/dev/lrisframes.rst new file mode 100644 index 0000000000..ef7c845273 --- /dev/null +++ b/doc/dev/lrisframes.rst @@ -0,0 +1,136 @@ +.. include:: ../include/links.rst + +.. _lris_frames_report: + +Automated typing of LRIS (BLUE and RED) frames +============================================== + +Version History +--------------- + + +========= ================ =========== =========== +*Version* *Author* *Date* ``PypeIt`` +========= ================ =========== =========== +1.0 Debora Pelliccia 6 Sep 2023 1.13.1.dev +========= ================ =========== =========== + +---- + +Basics +------ + +The general procedure used to assign frames a given type is described +here: :ref:`frame_types`. + +LRIS frame typing +----------------- + +The primary typing of LRIS frames is performed by +:func:`pypeit.spectrographs.keck_lris.KeckLRISSpectrograph.check_frame_type`. +This function checks the values of various header keywords against a +set of criteria used to classify the frame type. The same criteria are used for +``keck_lris_red``, ``keck_lris_red_orig``, ``keck_lris_red_mark4``, ``keck_lris_blue``, +and ``keck_lris_blue_orig``, unless otherwise noted. +The header cards required for the frame-typing and their associated keyword in the +:class:`~pypeit.metadata.PypeItMetaData` object are: + +=============== ====================================================== +``fitstbl`` key Header Key +=============== ====================================================== +``exptime`` ``ELAPTIME`` (``TELAPSE`` for ``keck_lris_red_mark4``) +``hatch`` ``TRAPDOOR`` +``lampstat01`` See below +=============== ====================================================== + +``lampstat01`` is defined using a combination of header keywords, which include +``LAMPS``, ``MERCURY``, ``NEON``, ``ARGON``, ``CADMIUM``, ``ZINC``, ``HALOGEN``, +``KRYPTON``, ``XENON``, ``FEARGON``, ``DEUTERI``, ``FLAMP1``, ``FLAMP2``, ``FLIMAGIN``, +``FLSPECTR``. Since LRIS header keywords have changed over time, the exact combination +of keywords used to define ``lampstat01`` varies depending on the available header keywords. + +The criteria used to select each frame type are as follows: + +============= ============ ====================================== =========================================== +Frame ``hatch`` ``lampstat01`` ``exptime`` +============= ============ ====================================== =========================================== +``science`` ``'open'`` ``'off'`` ``>61s`` +``standard`` ``'open'`` ``'off'`` ``>1s`` & ``<61s`` +``bias`` ``'closed'`` ``'off'`` ``<0.001s`` +``pixelflat`` ``'closed'`` ``'Halogen' or '2H'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) +``pixelflat`` ``'open'`` ``'on'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) +``trace`` ``'closed'`` ``'Halogen'`` or ``'2H'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) +``trace`` ``'open'`` ``'on'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) +``illumflat`` ``'closed'`` ``'Halogen'`` or ``'2H'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) +``illumflat`` ``'open'`` ``'on'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) +``arc`` ``'closed'`` ``!= 'Halogen', '2H', 'on', 'off'`` Not used +``tilt`` ``'closed'`` ``!= 'Halogen', '2H', 'on', 'off'`` Not used +============= ============ ====================================== =========================================== + +Note that PypeIt employs commonly used value of ``exptime`` to distinguish frame type; +however, if needed, the user can specify a different value by +using the ``exprng`` parameter in the :ref:`pypeit_file`; see also :ref:`frame_types`. + +The criteria used to select ``arc`` and ``tilt`` frames are identical; the same is true for +``pixelflat``, ``trace``, and ``illumflat`` frames. However, it's important to note that +PypeIt is able to correctly assign the ``pixelflat``, ``trace``, and ``illumflat`` types +to the internal and dome flat frames, but the twilight flats will generally have the +``science`` or ``standard`` type. Therefore, the user should manually change their frame type +in the :ref:`pypeit_file`. + +Finally, note that a LRIS frame is never given a ``pinhole`` or ``dark`` type. + + +Testing +------- + +Requirement PLL-16 states: "As a user, I expect the pipeline to automatically classify my data." + +``PypeIt`` meets this requirement as demonstrated by the tests at +``${PYPEIT_DEV}/unit_tests/test_frametype.py``. There is one test +per spectrograph: + +- ``test_lris_blue()`` +- ``test_lris_blue_orig()`` +- ``test_lris_red()`` +- ``test_lris_red_orig()`` +- ``test_lris_red_mark4()`` + +Here is an example of how to run the tests: + +.. code-block:: bash + + cd ${PYPEIT_DEV}/unit_tests + pytest test_frametype.py::test_lris_blue -W ignore + +The tests requires that you have downloaded the PypeIt +:ref:`dev-suite` and defined the ``PYPEIT_DEV`` environmental +variable that points to the relevant directory. The algorithm for +all these tests is the same and is as follows: + + 1. Find the directories in the :ref:`dev-suite` with Keck + LRIS data (separately for ``keck_lris_blue``, + ``keck_lris_blue_orig``, ``keck_lris_red``, + ``keck_lris_red_orig``, ``keck_lris_red_mark4``). + + 2. For each directory (i.e., instrument setup): + + a. Make sure there is a "by-hand" version of the pypeit file + for this setup where a human (one of the pypeit + developers) has ensured the frame types are correct. + + b. Effectively run :ref:`pypeit_setup` on each of the + instrument setups to construct a new pypeit file with the + automatically generated frame types. + + c. Read both the by-hand and automatically generated frame + types from these two pypeit files and check that they are + identical. This check is *only* performed for the + calibration frames, not any ``science`` or ``standard`` + frames. + +Because this test is now included in the ``PypeIt`` +:ref:`unit-tests`, this frame-typing check is performed by the +developers for every new version of the code. + + diff --git a/doc/dev/radec_object.rst b/doc/dev/radec_object.rst index 354a3bd246..8ce43422f4 100644 --- a/doc/dev/radec_object.rst +++ b/doc/dev/radec_object.rst @@ -15,6 +15,7 @@ Version History 1.1 Debora Pelliccia 02 Apr 2021 1.3.4dev 1.2 Debora Pelliccia 28 Jul 2021 1.4.3dev 1.3 Debora Pelliccia 21 Oct 2021 1.6.1dev +1.4 Debora Pelliccia 6 Sep 2023 1.13.1dev ========= ================ =========== =========== ---- @@ -23,7 +24,7 @@ Basics ------ The procedure used to assign RA, Dec and object name to each 1D extracted spectrum -is currently available for Keck/DEIMOS and Keck/MOSFIRE only and is performed right after +is currently available for these :ref:`slitmask_info_instruments` only and is performed right after the object finding procedure described in :ref:`object_finding`. @@ -83,7 +84,7 @@ and one is for the RA, Dec and object name assignment. They are the following. compute the slitmask offset. This is the default behaviour for DEIMOS unless **slitmask_offset**, **bright_maskdef_id** or **use_alignbox** is set. Default value is **snr_thrshd=50**. -- **bright_maskdef_id**: ``maskdef_id`` (corresponding to ``dSlitId`` and ``Slit_Number`` in the DEIMOS +- **bright_maskdef_id**: ``maskdef_id`` (corresponding to ``dSlitId`` and ``Slit_Number`` in the DEIMOS/LRIS and MOSFIRE slitmask design, respectively) of a slit containing a bright object that will be used to compute the slitmask offset. This parameter is optional (default value is **bright_maskdef_id=None**) and is ignored if **slitmask_offset** is provided or **use_dither_offset = True**. However, diff --git a/doc/dev/slitmask_ids.rst b/doc/dev/slitmask_ids.rst index e64e4bf763..02b63e8aaf 100644 --- a/doc/dev/slitmask_ids.rst +++ b/doc/dev/slitmask_ids.rst @@ -11,12 +11,13 @@ Version History ========= =================== =========== =========== *Version* *Author* *Date* ``PypeIt`` ========= =================== =========== =========== -1.0 Debora Pelliccia 9 Oct 2020 1.1.2dev -1.1 Debora Pelliccia 5 Nov 2020 1.2.1dev +1.0 Debora Pelliccia 9 Oct 2020 1.1.2dev +1.1 Debora Pelliccia 5 Nov 2020 1.2.1dev 1.2 Debora Pelliccia 26 Jan 2021 1.3.1dev 1.3 Debora Pelliccia 21 Oct 2021 1.6.1dev 1.4 J. Xavier Prochaska 11 Jan 2022 1.7.1dev 1.5 Kyle Westfall 23 Mar 2023 1.12.2dev +1.6 Debora Pelliccia 6 Sep 2023 1.13.1dev ========= =================== =========== =========== ---- @@ -25,7 +26,7 @@ Basics ------ The procedure used to assign slitmask ID to each slit is currently available for -Keck/DEIMOS, Keck/MOSFIRE, and Keck/LRIS (limited) only, +these :ref:`slitmask_info_instruments` only, and is part of the more general slit tracing procedure described in :ref:`slit_tracing`. @@ -55,8 +56,7 @@ form longer slits. For LRIS, the procedure is similar to the one for MOSFIRE where edges are predicted based on the RA, DEC of each slit and the platescale of the detector. -It has been developed and tested for the previous LRISr detector (not Mark4) -and the current LRISb detector. +It has been developed and tested for ``keck_lris_red``, ``keck_lris_red_mark4``, and ``keck_lris_blue``. The function :func:`~pypeit.edgetrace.EdgeTraceSet.maskdesign_matching` assigns to each slit a ``maskdef_id``, which corresponds to `dSlitId` and `Slit_Number` in the DEIMOS/LRIS and @@ -105,8 +105,8 @@ Access The ``maskdef_id`` is recorded for each slits in the :class:`~pypeit.slittrace.SlitTraceSet` datamodel, which is written to disk as a multi-extension FITS file prefixed by Slits. -In addition, for DEIMOS and MOSFIRE a second `astropy.io.fits.BinTableHDU`_ is written to disk and contains -more slitmask design information. See :ref:`slits` for a description of the provided information +In addition, for these :ref:`slitmask_info_instruments` a second `astropy.io.fits.BinTableHDU`_ is written +to disk and contains more slitmask design information. See :ref:`slits` for a description of the provided information and for a way to visualize them. Moreover, the ``maskdef_id`` assigned to each slit can be found, after a full reduction with ``PypeIt`` diff --git a/doc/figures/kastb_ginga_tilts.png b/doc/figures/kastb_ginga_tilts.png new file mode 100644 index 0000000000..facf3e99ba Binary files /dev/null and b/doc/figures/kastb_ginga_tilts.png differ diff --git a/doc/figures/kastb_sens.png b/doc/figures/kastb_sens.png new file mode 100644 index 0000000000..a7cdb42675 Binary files /dev/null and b/doc/figures/kastb_sens.png differ diff --git a/doc/fluxing.rst b/doc/fluxing.rst index ea4ea90ed0..06ef1e9aa3 100644 --- a/doc/fluxing.rst +++ b/doc/fluxing.rst @@ -589,6 +589,5 @@ FluxSpec Class ============== The guts of the flux algorithms are guided by the -:class:`~pypeit.fluxcalibrate.FluxCalibrate` -class. +:func:`~pypeit.fluxcalibrate.apply_flux_calib` class. diff --git a/doc/help/pypeit_cache_github_data.rst b/doc/help/pypeit_cache_github_data.rst index 5948f93d00..ebde464cc9 100644 --- a/doc/help/pypeit_cache_github_data.rst +++ b/doc/help/pypeit_cache_github_data.rst @@ -10,13 +10,14 @@ spectrograph A valid spectrograph identifier: bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, - gemini_gmos_south_ham, gemini_gnirs, gtc_maat, gtc_osiris, - gtc_osiris_plus, jwst_nircam, jwst_nirspec, keck_deimos, - keck_hires, keck_kcwi, keck_lris_blue, keck_lris_blue_orig, - keck_lris_red, keck_lris_red_mark4, keck_lris_red_orig, - keck_mosfire, keck_nires, keck_nirspec_low, lbt_luci1, - lbt_luci2, lbt_mods1b, lbt_mods1r, lbt_mods2b, lbt_mods2r, - ldt_deveny, magellan_fire, magellan_fire_long, magellan_mage, + gemini_gmos_south_ham, gemini_gnirs_echelle, gemini_gnirs_ifu, + gtc_maat, gtc_osiris, gtc_osiris_plus, jwst_nircam, + jwst_nirspec, keck_deimos, keck_esi, keck_hires, keck_kcrm, + keck_kcwi, keck_lris_blue, keck_lris_blue_orig, keck_lris_red, + keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, + keck_nires, keck_nirspec_low, lbt_luci1, lbt_luci2, + lbt_mods1b, lbt_mods1r, lbt_mods2b, lbt_mods2r, ldt_deveny, + magellan_fire, magellan_fire_long, magellan_mage, mdm_modspec, mdm_osmos_mdm4k, mmt_binospec, mmt_bluechannel, mmt_mmirs, not_alfosc, not_alfosc_vert, ntt_efosc2, p200_dbsp_blue, p200_dbsp_red, p200_tspec, shane_kast_blue, shane_kast_red, diff --git a/doc/help/pypeit_chk_for_calibs.rst b/doc/help/pypeit_chk_for_calibs.rst index 34b2c930bc..a683278d1d 100644 --- a/doc/help/pypeit_chk_for_calibs.rst +++ b/doc/help/pypeit_chk_for_calibs.rst @@ -17,14 +17,15 @@ gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, - gemini_gnirs, gtc_maat, gtc_osiris, gtc_osiris_plus, - jwst_nircam, jwst_nirspec, keck_deimos, keck_hires, - keck_kcwi, keck_lris_blue, keck_lris_blue_orig, - keck_lris_red, keck_lris_red_mark4, keck_lris_red_orig, - keck_mosfire, keck_nires, keck_nirspec_low, lbt_luci1, - lbt_luci2, lbt_mods1b, lbt_mods1r, lbt_mods2b, - lbt_mods2r, ldt_deveny, magellan_fire, - magellan_fire_long, magellan_mage, mdm_osmos_mdm4k, + gemini_gnirs_echelle, gemini_gnirs_ifu, gtc_maat, + gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, + keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, + keck_lris_blue, keck_lris_blue_orig, keck_lris_red, + keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, + keck_nires, keck_nirspec_low, lbt_luci1, lbt_luci2, + lbt_mods1b, lbt_mods1r, lbt_mods2b, lbt_mods2r, + ldt_deveny, magellan_fire, magellan_fire_long, + magellan_mage, mdm_modspec, mdm_osmos_mdm4k, mmt_binospec, mmt_bluechannel, mmt_mmirs, not_alfosc, not_alfosc_vert, ntt_efosc2, p200_dbsp_blue, p200_dbsp_red, p200_tspec, shane_kast_blue, diff --git a/doc/help/pypeit_coadd_1dspec.rst b/doc/help/pypeit_coadd_1dspec.rst index 70e09065cf..d363703531 100644 --- a/doc/help/pypeit_coadd_1dspec.rst +++ b/doc/help/pypeit_coadd_1dspec.rst @@ -8,13 +8,18 @@ Coadd 1D spectra produced by PypeIt positional arguments: - coadd1d_file File to guide coadding process. This file must have the + coadd1d_file File to guide coadding process. + + ------------------------ + MultiSlit + ------------------------ + + For coadding Multislit spectra the file must have the following format (see docs for further details including the use of paths): [coadd1d] coaddfile='output_filename.fits' # Optional - sensfuncfile = 'sensfunc.fits' # Required only for Echelle coadd1d read filename | obj_id @@ -46,6 +51,52 @@ inspect the spec1d_*.txt files or run pypeit_show_1dspec spec1dfile --list + ------------------------ + Echelle + ------------------------ + + For coadding Echelle spectra the file must have the + following format (see docs for further details): + + [coadd1d] + coaddfile='output_filename.fits' # Optional + + coadd1d read + filename | obj_id | sensfile | setup_id + spec1dfile1 | objid1 | sensfile1 | setup_id1 + spec1dfile2 | objid2 | sensfile2 | setup_id2 + spec1dfile3 | objid3 | sensfile3 | setup_id3 + ... + coadd1d end + + OR the coadd1d read/end block can look like + + coadd1d read + filename | obj_id | sensfile | setup_id + spec1dfile1 | objid1 | sensfile | setup_id + spec1dfile2 | | | + spec1dfile3 | | | + ... + coadd1d end + + That is the coadd1d block is a four column list of + spec1dfiles, objids, sensitivity function files, and + setup_ids, but you can specify only a single objid, + sensfile, and setup_id for all spec1dfiles on the first + line + + Here: + + spec1dfile: full path to a PypeIt spec1dfile + + objid: the object identifier (see details above) + + sensfile: full path to a PypeIt sensitivity function + file for the echelle setup in question + + setup_id: string identifier for the echelle setup in + question, i.e. 'VIS', 'NIR', or 'UVB' + If the coaddfile is not given the output file will be placed in the same directory as the first spec1d file. diff --git a/doc/help/pypeit_collate_1d.rst b/doc/help/pypeit_collate_1d.rst index c8ae8adc92..d61fa3a918 100644 --- a/doc/help/pypeit_collate_1d.rst +++ b/doc/help/pypeit_collate_1d.rst @@ -8,7 +8,7 @@ [--flux] [--exclude_slit_bm EXCLUDE_SLIT_BM] [--exclude_serendip] [--wv_rms_thresh WV_RMS_THRESH] [--refframe {observed,heliocentric,barycentric}] - [-v VERBOSITY] + [--chk_version] [-v VERBOSITY] [input_file] Flux/Coadd multiple 1d spectra from multiple nights and prepare a directory for @@ -36,6 +36,8 @@ value are skipped, else all wavelength rms values are accepted. refframe Perform reference frame correction prior to coadding. Options are ['observed', 'heliocentric', 'barycentric']. Defaults to None. + chk_version If true, spec1ds and archival sensfuncs must match the currently + supported versions. If false (the default) version numbers are not checked. spec1d read @@ -86,6 +88,8 @@ --refframe {observed,heliocentric,barycentric} Perform reference frame correction prior to coadding. Options are: observed, heliocentric, barycentric + --chk_version Whether to check the data model versions of spec1d files + and sensfunc files. -v VERBOSITY, --verbosity VERBOSITY Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename diff --git a/doc/help/pypeit_compare_sky.rst b/doc/help/pypeit_compare_sky.rst index 9e2f02f7d1..3b0f4b2bf9 100644 --- a/doc/help/pypeit_compare_sky.rst +++ b/doc/help/pypeit_compare_sky.rst @@ -2,7 +2,7 @@ $ pypeit_compare_sky -h usage: pypeit_compare_sky [-h] [--exten EXTEN] [--optimal] - [--scale_user SCALE_USER] [--test] + [--scale_user SCALE_USER] file skyfile Compare the extracted sky spectrum against an archived sky model maintained by @@ -18,5 +18,4 @@ --optimal Show Optimal? Default is boxcar (default: False) --scale_user SCALE_USER Scale user spectrum by a factor (default: 1.0) - --test Load files but do not show plot (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_flux_calib.rst b/doc/help/pypeit_flux_calib.rst index 2794ddf7f3..9bdb6d363f 100644 --- a/doc/help/pypeit_flux_calib.rst +++ b/doc/help/pypeit_flux_calib.rst @@ -1,7 +1,7 @@ .. code-block:: console $ pypeit_flux_calib -h - usage: pypeit_flux_calib [-h] [--debug] [--par_outfile] [-v VERBOSITY] flux_file + usage: pypeit_flux_calib [-h] [--par_outfile] [-v VERBOSITY] flux_file Flux calibrate 1D spectra produced by PypeIt @@ -48,7 +48,6 @@ options: -h, --help show this help message and exit - --debug show debug plots? --par_outfile Output to save the parameters -v VERBOSITY, --verbosity VERBOSITY Verbosity level between 0 [none] and 2 [all]. Default: diff --git a/doc/help/pypeit_flux_setup.rst b/doc/help/pypeit_flux_setup.rst index 83b3fbb379..7f42af20a6 100644 --- a/doc/help/pypeit_flux_setup.rst +++ b/doc/help/pypeit_flux_setup.rst @@ -1,15 +1,21 @@ .. code-block:: console $ pypeit_flux_setup -h - usage: pypeit_flux_setup [-h] [--objmodel {qso,star,poly}] sci_path + usage: pypeit_flux_setup [-h] [--name NAME] [--objmodel {qso,star,poly}] + paths [paths ...] - Setup to perform flux calibration + Setup configuration files to perform flux calibration, 1D coadding, and telluric + correction. positional arguments: - sci_path Path for Science folder + paths One or more paths for Science folders or sensitivity + functions. Sensitivity functions must start with 'sens_' + to be detected. options: -h, --help show this help message and exit + --name NAME The base name to use for the output files. Defaults to + the instrument name is used. --objmodel {qso,star,poly} science object model used in the telluric fitting. The options are: diff --git a/doc/help/pypeit_identify.rst b/doc/help/pypeit_identify.rst index b209265a07..da9e7a8bab 100644 --- a/doc/help/pypeit_identify.rst +++ b/doc/help/pypeit_identify.rst @@ -4,7 +4,7 @@ usage: pypeit_identify [-h] [--lamps LAMPS] [-s] [--wmin WMIN] [--wmax WMAX] [--slit SLIT] [--det DET] [--rmstol RMSTOL] [--fwhm FWHM] [--sigdetect SIGDETECT] [--pixtol PIXTOL] [--linear] - [--force_save] [--rescale_resid] + [--force_save] [--rescale_resid] [-v VERBOSITY] arc_file slits_file Launch PypeIt identify tool, display extracted Arc, and load linelist. @@ -34,4 +34,8 @@ --force_save Save the solutions, despite the RMS (default: False) --rescale_resid Rescale the residual plot to include all points? (default: False) + -v VERBOSITY, --verbosity VERBOSITY + Verbosity level between 0 [none] and 2 [all]. Default: + 1. Level 2 writes a log with filename identify_YYYYMMDD- + HHMM.log (default: 1) \ No newline at end of file diff --git a/doc/help/pypeit_obslog.rst b/doc/help/pypeit_obslog.rst index 1f6e2b5290..2bc08ce369 100644 --- a/doc/help/pypeit_obslog.rst +++ b/doc/help/pypeit_obslog.rst @@ -14,14 +14,15 @@ gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, - gemini_gnirs, gtc_maat, gtc_osiris, gtc_osiris_plus, - jwst_nircam, jwst_nirspec, keck_deimos, keck_hires, - keck_kcwi, keck_lris_blue, keck_lris_blue_orig, - keck_lris_red, keck_lris_red_mark4, keck_lris_red_orig, - keck_mosfire, keck_nires, keck_nirspec_low, lbt_luci1, - lbt_luci2, lbt_mods1b, lbt_mods1r, lbt_mods2b, - lbt_mods2r, ldt_deveny, magellan_fire, - magellan_fire_long, magellan_mage, mdm_osmos_mdm4k, + gemini_gnirs_echelle, gemini_gnirs_ifu, gtc_maat, + gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, + keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, + keck_lris_blue, keck_lris_blue_orig, keck_lris_red, + keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, + keck_nires, keck_nirspec_low, lbt_luci1, lbt_luci2, + lbt_mods1b, lbt_mods1r, lbt_mods2b, lbt_mods2r, + ldt_deveny, magellan_fire, magellan_fire_long, + magellan_mage, mdm_modspec, mdm_osmos_mdm4k, mmt_binospec, mmt_bluechannel, mmt_mmirs, not_alfosc, not_alfosc_vert, ntt_efosc2, p200_dbsp_blue, p200_dbsp_red, p200_tspec, shane_kast_blue, diff --git a/doc/help/pypeit_ql.rst b/doc/help/pypeit_ql.rst index f02e25d753..6344eb9b72 100644 --- a/doc/help/pypeit_ql.rst +++ b/doc/help/pypeit_ql.rst @@ -21,14 +21,15 @@ gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, - gemini_gnirs, gtc_maat, gtc_osiris, gtc_osiris_plus, - jwst_nircam, jwst_nirspec, keck_deimos, keck_hires, - keck_kcwi, keck_lris_blue, keck_lris_blue_orig, - keck_lris_red, keck_lris_red_mark4, keck_lris_red_orig, - keck_mosfire, keck_nires, keck_nirspec_low, lbt_luci1, - lbt_luci2, lbt_mods1b, lbt_mods1r, lbt_mods2b, - lbt_mods2r, ldt_deveny, magellan_fire, - magellan_fire_long, magellan_mage, mdm_osmos_mdm4k, + gemini_gnirs_echelle, gemini_gnirs_ifu, gtc_maat, + gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, + keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, + keck_lris_blue, keck_lris_blue_orig, keck_lris_red, + keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, + keck_nires, keck_nirspec_low, lbt_luci1, lbt_luci2, + lbt_mods1b, lbt_mods1r, lbt_mods2b, lbt_mods2r, + ldt_deveny, magellan_fire, magellan_fire_long, + magellan_mage, mdm_modspec, mdm_osmos_mdm4k, mmt_binospec, mmt_bluechannel, mmt_mmirs, not_alfosc, not_alfosc_vert, ntt_efosc2, p200_dbsp_blue, p200_dbsp_red, p200_tspec, shane_kast_blue, diff --git a/doc/help/pypeit_ql_multislit.rst b/doc/help/pypeit_ql_multislit.rst deleted file mode 100644 index ce83350892..0000000000 --- a/doc/help/pypeit_ql_multislit.rst +++ /dev/null @@ -1,77 +0,0 @@ -.. code-block:: console - - $ pypeit_ql_multislit -h - usage: pypeit_ql_multislit [-h] [--spec_samp_fact SPEC_SAMP_FACT] - [--spat_samp_fact SPAT_SAMP_FACT] [--bkg_redux] - [--flux] [--mask_cr] [--writefits] [--no_gui] - [--box_radius BOX_RADIUS] [--offset OFFSET] - [--redux_path REDUX_PATH] [--master_dir MASTER_DIR] - [--embed] [--show] [--det [DET ...]] - spectrograph full_rawpath files [files ...] - - Script to produce quick-look multislit PypeIt reductions - - positional arguments: - spectrograph A valid spectrograph identifier: bok_bc, - gemini_flamingos1, gemini_flamingos2, - gemini_gmos_north_e2v, gemini_gmos_north_ham, - gemini_gmos_north_ham_ns, gemini_gmos_south_ham, - gemini_gnirs, gtc_maat, gtc_osiris, gtc_osiris_plus, - jwst_nircam, jwst_nirspec, keck_deimos, keck_hires, - keck_kcwi, keck_lris_blue, keck_lris_blue_orig, - keck_lris_red, keck_lris_red_mark4, keck_lris_red_orig, - keck_mosfire, keck_nires, keck_nirspec_low, lbt_luci1, - lbt_luci2, lbt_mods1b, lbt_mods1r, lbt_mods2b, - lbt_mods2r, ldt_deveny, magellan_fire, - magellan_fire_long, magellan_mage, mdm_osmos_mdm4k, - mmt_binospec, mmt_bluechannel, mmt_mmirs, not_alfosc, - not_alfosc_vert, ntt_efosc2, p200_dbsp_blue, - p200_dbsp_red, p200_tspec, shane_kast_blue, - shane_kast_red, shane_kast_red_ret, soar_goodman_blue, - soar_goodman_red, tng_dolores, vlt_fors2, vlt_sinfoni, - vlt_xshooter_nir, vlt_xshooter_uvb, vlt_xshooter_vis, - wht_isis_blue, wht_isis_red - full_rawpath Full path to the raw files - files list of frames i.e. img1.fits img2.fits - - options: - -h, --help show this help message and exit - --spec_samp_fact SPEC_SAMP_FACT - Make the wavelength grid finer (spec_samp_fact < 1.0) or - coarser (spec_samp_fact > 1.0) by this sampling factor, - i.e. units of spec_samp_fact are pixels. (default: 1.0) - --spat_samp_fact SPAT_SAMP_FACT - Make the spatial grid finer (spat_samp_fact < 1.0) or - coarser (spat_samp_fact > 1.0) by this sampling factor, - i.e. units of spat_samp_fact are pixels. (default: 1.0) - --bkg_redux If set the script will perform difference imaging - quicklook. Namely it will identify sequences of AB pairs - based on the dither pattern and perform difference - imaging sky subtraction and fit for residuals (default: - False) - --flux This option will multiply in sensitivity function to - obtain a flux calibrated 2d spectrum (default: False) - --mask_cr This option turns on cosmic ray rejection. This improves - the reduction but doubles runtime. (default: False) - --writefits Write the ouputs to a fits file (default: False) - --no_gui Do not display the results in a GUI (default: False) - --box_radius BOX_RADIUS - Set the radius for the boxcar extraction (default: None) - --offset OFFSET Override the automatic offsets determined from the - headers. Offset is in pixels. This option is useful if a - standard dither pattern was not executed. The offset - convention is such that a negative offset will move the - (negative) B image to the left. (default: None) - --redux_path REDUX_PATH - Location where reduction outputs should be stored. - (default: current working directory) - --master_dir MASTER_DIR - Location of PypeIt Master files used for the reduction. - (default: None) - --embed Upon completion embed in ipython shell (default: False) - --show Show the reduction steps. Equivalent to the -s option - when running pypeit. (default: False) - --det [DET ...] Detector(s) to show. If more than one, the list of - detectors must be one of the allowed mosaics hard-coded - for the selected spectrograph. (default: 1) - \ No newline at end of file diff --git a/doc/help/pypeit_sensfunc.rst b/doc/help/pypeit_sensfunc.rst index f7fbd5a38b..3a5b008e28 100644 --- a/doc/help/pypeit_sensfunc.rst +++ b/doc/help/pypeit_sensfunc.rst @@ -2,8 +2,8 @@ $ pypeit_sensfunc -h usage: pypeit_sensfunc [-h] [--algorithm {UVIS,IR}] [--multi MULTI] [-o OUTFILE] - [-s SENS_FILE] [--debug] [--par_outfile PAR_OUTFILE] - [-v VERBOSITY] + [-s SENS_FILE] [-f FLATFILE] [--debug] + [--par_outfile PAR_OUTFILE] [-v VERBOSITY] spec1dfile Compute a sensitivity function @@ -61,6 +61,18 @@ in the filename. -s SENS_FILE, --sens_file SENS_FILE Configuration file with sensitivity function parameters + -f FLATFILE, --flatfile FLATFILE + Use the flat file for computing the sensitivity + function. Note that it is not possible to set + --flatfile and simultaneously use a .sens file with the + --sens_file option. If you are using a .sens file, set + the flatfile there via e.g.: + + [sensfunc] + flatfile = Calibrations/Flat_A_0_DET01.fits + + Where Flat_A_0_DET01.fits is the flat file in your + Calibrations directory --debug show debug plots? --par_outfile PAR_OUTFILE Name of output file to save the parameters used by the diff --git a/doc/help/pypeit_setup.rst b/doc/help/pypeit_setup.rst index 40dc9ceb1b..1cb8fed6ba 100644 --- a/doc/help/pypeit_setup.rst +++ b/doc/help/pypeit_setup.rst @@ -15,14 +15,15 @@ gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, - gemini_gnirs, gtc_maat, gtc_osiris, gtc_osiris_plus, - jwst_nircam, jwst_nirspec, keck_deimos, keck_hires, - keck_kcwi, keck_lris_blue, keck_lris_blue_orig, - keck_lris_red, keck_lris_red_mark4, keck_lris_red_orig, - keck_mosfire, keck_nires, keck_nirspec_low, lbt_luci1, - lbt_luci2, lbt_mods1b, lbt_mods1r, lbt_mods2b, - lbt_mods2r, ldt_deveny, magellan_fire, - magellan_fire_long, magellan_mage, mdm_osmos_mdm4k, + gemini_gnirs_echelle, gemini_gnirs_ifu, gtc_maat, + gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, + keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, + keck_lris_blue, keck_lris_blue_orig, keck_lris_red, + keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, + keck_nires, keck_nirspec_low, lbt_luci1, lbt_luci2, + lbt_mods1b, lbt_mods1r, lbt_mods2b, lbt_mods2r, + ldt_deveny, magellan_fire, magellan_fire_long, + magellan_mage, mdm_modspec, mdm_osmos_mdm4k, mmt_binospec, mmt_bluechannel, mmt_mmirs, not_alfosc, not_alfosc_vert, ntt_efosc2, p200_dbsp_blue, p200_dbsp_red, p200_tspec, shane_kast_blue, @@ -56,7 +57,9 @@ Include the manual extraction column for the user to edit (default: False) -v VERBOSITY, --verbosity VERBOSITY - Level of verbosity from 0 to 2. (default: 2) + Verbosity level between 0 [none] and 2 [all]. Default: + 1. Level 2 writes a log with filename setup_YYYYMMDD- + HHMM.log (default: 1) -k, --keep_bad_frames Keep all frames, even if they are identified as having bad/unrecognized configurations that cannot be reduced diff --git a/doc/help/pypeit_setup_coadd2d.rst b/doc/help/pypeit_setup_coadd2d.rst index 505042b1ae..b9d76bd679 100644 --- a/doc/help/pypeit_setup_coadd2d.rst +++ b/doc/help/pypeit_setup_coadd2d.rst @@ -1,10 +1,14 @@ .. code-block:: console $ pypeit_setup_coadd2d -h - usage: pypeit_setup_coadd2d [-h] (-f PYPEIT_FILE | -d SCIENCE_DIR) [--keep_par] - [--obj OBJ [OBJ ...]] [--det DET [DET ...]] + usage: pypeit_setup_coadd2d [-h] + (-f PYPEIT_FILE | -d SCIENCE_DIR [SCIENCE_DIR ...]) + [--keep_par] [--obj OBJ [OBJ ...]] + [--det DET [DET ...]] [--only_slits ONLY_SLITS [ONLY_SLITS ...]] - [--offsets OFFSETS] [--weights WEIGHTS] + [--exclude_slits EXCLUDE_SLITS [EXCLUDE_SLITS ...]] + [--spat_toler SPAT_TOLER] [--offsets OFFSETS] + [--weights WEIGHTS] Prepare a configuration file for performing 2D coadds @@ -12,8 +16,10 @@ -h, --help show this help message and exit -f PYPEIT_FILE, --pypeit_file PYPEIT_FILE PypeIt reduction file (default: None) - -d SCIENCE_DIR, --science_dir SCIENCE_DIR - Directory with spec2d files to stack (default: None) + -d SCIENCE_DIR [SCIENCE_DIR ...], --science_dir SCIENCE_DIR [SCIENCE_DIR ...] + One or more directories with spec2d files to stack (use + wildcard to specify multiple directories). (default: + None) --keep_par Propagate all parameters from the pypeit file to the coadd2d file(s). If not set, only the required parameters and their default values are included in the @@ -35,8 +41,22 @@ mosaics made up of detectors 1,5 and 3,7, you would use --det 1,5 3,7 (default: None) --only_slits ONLY_SLITS [ONLY_SLITS ...] - A space-separated set of slits to coadd. If not - provided, all slits are coadded. (default: None) + A space-separated set of slits to coadd. Example syntax + for argument is DET01:175,DET02:205 or MSC02:2234. If + not provided, all slits are coadded. If both --det and + --only_slits are provided, --det will be ignored. This + and --exclude_slits are mutually exclusive. If both are + provided, --only_slits takes precedence. (default: None) + --exclude_slits EXCLUDE_SLITS [EXCLUDE_SLITS ...] + A space-separated set of slits to exclude in the + coaddition. This and --only_slits are mutually + exclusive. If both are provided, --only_slits takes + precedence. (default: None) + --spat_toler SPAT_TOLER + Desired tolerance in spatial pixel used to identify + slits in different exposures. If not provided, the + default value for the specific instrument/configuration + is used. (default: None) --offsets OFFSETS Spatial offsets to apply to each image; see the [coadd2d][offsets] parameter. Options are restricted here to either maskdef_offsets or auto. If not diff --git a/doc/help/pypeit_skysub_regions.rst b/doc/help/pypeit_skysub_regions.rst index d5e887fde1..92b9b0ff8b 100644 --- a/doc/help/pypeit_skysub_regions.rst +++ b/doc/help/pypeit_skysub_regions.rst @@ -4,8 +4,8 @@ usage: pypeit_skysub_regions [-h] [--det DET] [-o] [-i] [-f] [-s] [-v VERBOSITY] file - Display a Raw science image and interactively define the sky regions using a - GUI. Run in the same folder as your .pypeit file + Display a spec2d frame and interactively define the sky regions using a GUI. Run + in the same folder as your .pypeit file positional arguments: file spec2d file diff --git a/doc/help/pypeit_tellfit.rst b/doc/help/pypeit_tellfit.rst index 28ae52089d..28158d4068 100644 --- a/doc/help/pypeit_tellfit.rst +++ b/doc/help/pypeit_tellfit.rst @@ -9,7 +9,8 @@ Telluric correct a spectrum positional arguments: - spec1dfile spec1d file that will be used for telluric correction. + spec1dfile spec1d or coadd file that will be used for telluric + correction. options: -h, --help show this help message and exit diff --git a/doc/help/pypeit_trace_edges.rst b/doc/help/pypeit_trace_edges.rst index 8a15d76819..7f79e8a8ca 100644 --- a/doc/help/pypeit_trace_edges.rst +++ b/doc/help/pypeit_trace_edges.rst @@ -29,17 +29,18 @@ providing files directly: bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, - gemini_gmos_south_ham, gemini_gnirs, gtc_maat, - gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, - keck_deimos, keck_hires, keck_kcwi, keck_lris_blue, + gemini_gmos_south_ham, gemini_gnirs_echelle, + gemini_gnirs_ifu, gtc_maat, gtc_osiris, gtc_osiris_plus, + jwst_nircam, jwst_nirspec, keck_deimos, keck_esi, + keck_hires, keck_kcrm, keck_kcwi, keck_lris_blue, keck_lris_blue_orig, keck_lris_red, keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, keck_nires, keck_nirspec_low, lbt_luci1, lbt_luci2, lbt_mods1b, lbt_mods1r, lbt_mods2b, lbt_mods2r, ldt_deveny, magellan_fire, magellan_fire_long, magellan_mage, - mdm_osmos_mdm4k, mmt_binospec, mmt_bluechannel, - mmt_mmirs, not_alfosc, not_alfosc_vert, ntt_efosc2, - p200_dbsp_blue, p200_dbsp_red, p200_tspec, + mdm_modspec, mdm_osmos_mdm4k, mmt_binospec, + mmt_bluechannel, mmt_mmirs, not_alfosc, not_alfosc_vert, + ntt_efosc2, p200_dbsp_blue, p200_dbsp_red, p200_tspec, shane_kast_blue, shane_kast_red, shane_kast_red_ret, soar_goodman_blue, soar_goodman_red, tng_dolores, vlt_fors2, vlt_sinfoni, vlt_xshooter_nir, diff --git a/doc/help/pypeit_view_fits.rst b/doc/help/pypeit_view_fits.rst index 406eb833da..c08ae16e7f 100644 --- a/doc/help/pypeit_view_fits.rst +++ b/doc/help/pypeit_view_fits.rst @@ -3,7 +3,7 @@ $ pypeit_view_fits -h usage: pypeit_view_fits [-h] [--list] [--proc] [--bkg_file BKG_FILE] [--exten EXTEN] [--det [DET ...]] [--chname CHNAME] - [--embed] + [--showmask] [--embed] spectrograph file View FITS files with ginga @@ -13,14 +13,15 @@ gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, - gemini_gnirs, gtc_maat, gtc_osiris, gtc_osiris_plus, - jwst_nircam, jwst_nirspec, keck_deimos, keck_hires, - keck_kcwi, keck_lris_blue, keck_lris_blue_orig, - keck_lris_red, keck_lris_red_mark4, keck_lris_red_orig, - keck_mosfire, keck_nires, keck_nirspec_low, lbt_luci1, - lbt_luci2, lbt_mods1b, lbt_mods1r, lbt_mods2b, - lbt_mods2r, ldt_deveny, magellan_fire, - magellan_fire_long, magellan_mage, mdm_osmos_mdm4k, + gemini_gnirs_echelle, gemini_gnirs_ifu, gtc_maat, + gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, + keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, + keck_lris_blue, keck_lris_blue_orig, keck_lris_red, + keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, + keck_nires, keck_nirspec_low, lbt_luci1, lbt_luci2, + lbt_mods1b, lbt_mods1r, lbt_mods2b, lbt_mods2r, + ldt_deveny, magellan_fire, magellan_fire_long, + magellan_mage, mdm_modspec, mdm_osmos_mdm4k, mmt_binospec, mmt_bluechannel, mmt_mmirs, not_alfosc, not_alfosc_vert, ntt_efosc2, p200_dbsp_blue, p200_dbsp_red, p200_tspec, shane_kast_blue, @@ -34,9 +35,8 @@ -h, --help show this help message and exit --list List the extensions only? (default: False) --proc Process the image (i.e. orient, overscan subtract, - multiply by gain) using pypeit.images.buildimage. Note - det=mosaic will not work with this option (default: - False) + multiply by gain) using pypeit.images.buildimage. + (default: False) --bkg_file BKG_FILE FITS file to be subtracted from the image in file.--proc must be set in order for this option to work. (default: None) @@ -49,5 +49,6 @@ gemini_gmos, keck_deimos, or keck_lris will show the mosaic of all detectors. (default: 1) --chname CHNAME Name of Ginga tab (default: Image) + --showmask Overplot masked pixels (default: False) --embed Upon completion embed in ipython shell (default: False) \ No newline at end of file diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index 5cdfda7e56..a418bf5071 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -4,23 +4,25 @@ usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c] pypeit_file - ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.12.3.dev337+g096680b06.d20230503 + ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.13.1.dev859+gf736f5f47 ## ## Available spectrographs include: ## bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, ## gemini_gmos_north_ham, gemini_gmos_north_ham_ns, - ## gemini_gmos_south_ham, gemini_gnirs, gtc_maat, gtc_osiris, - ## gtc_osiris_plus, jwst_nircam, jwst_nirspec, keck_deimos, keck_hires, - ## keck_kcwi, keck_lris_blue, keck_lris_blue_orig, keck_lris_red, + ## gemini_gmos_south_ham, gemini_gnirs_echelle, gemini_gnirs_ifu, + ## gtc_maat, gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, + ## keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, + ## keck_lris_blue, keck_lris_blue_orig, keck_lris_red, ## keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, keck_nires, ## keck_nirspec_low, lbt_luci1, lbt_luci2, lbt_mods1b, lbt_mods1r, ## lbt_mods2b, lbt_mods2r, ldt_deveny, magellan_fire, magellan_fire_long, - ## magellan_mage, mdm_osmos_mdm4k, mmt_binospec, mmt_bluechannel, - ## mmt_mmirs, not_alfosc, not_alfosc_vert, ntt_efosc2, p200_dbsp_blue, - ## p200_dbsp_red, p200_tspec, shane_kast_blue, shane_kast_red, - ## shane_kast_red_ret, soar_goodman_blue, soar_goodman_red, tng_dolores, - ## vlt_fors2, vlt_sinfoni, vlt_xshooter_nir, vlt_xshooter_uvb, - ## vlt_xshooter_vis, wht_isis_blue, wht_isis_red + ## magellan_mage, mdm_modspec, mdm_osmos_mdm4k, mmt_binospec, + ## mmt_bluechannel, mmt_mmirs, not_alfosc, not_alfosc_vert, ntt_efosc2, + ## p200_dbsp_blue, p200_dbsp_red, p200_tspec, shane_kast_blue, + ## shane_kast_red, shane_kast_red_ret, soar_goodman_blue, + ## soar_goodman_red, tng_dolores, vlt_fors2, vlt_sinfoni, + ## vlt_xshooter_nir, vlt_xshooter_uvb, vlt_xshooter_vis, wht_isis_blue, + ## wht_isis_red positional arguments: pypeit_file PypeIt reduction file (must have .pypeit extension) diff --git a/doc/include/bitmaskarray_usage.rst b/doc/include/bitmaskarray_usage.rst index 374f66b86e..c61b119730 100644 --- a/doc/include/bitmaskarray_usage.rst +++ b/doc/include/bitmaskarray_usage.rst @@ -38,7 +38,7 @@ You can access the bit flag names using: Bit access ++++++++++ -You can flag bits using :func:`~pypeit.bitmaskarray.BitMaskArray.turn_on`. For +You can flag bits using :func:`~pypeit.images.bitmaskarray.BitMaskArray.turn_on`. For example, the following code flags the center column of the image as being part of the detector bad-pixel mask: diff --git a/doc/include/class_datamodel_datacube.rst b/doc/include/class_datamodel_datacube.rst index 2a2ac61bf3..6449104d1b 100644 --- a/doc/include/class_datamodel_datacube.rst +++ b/doc/include/class_datamodel_datacube.rst @@ -1,15 +1,15 @@ **Version**: 1.1.0 -============== ================ ================= ================================================================================ -Attribute Type Array Type Description -============== ================ ================= ================================================================================ -``PYP_SPEC`` str PypeIt: Spectrograph name -``blaze_spec`` `numpy.ndarray`_ `numpy.floating`_ The spectral blaze function -``blaze_wave`` `numpy.ndarray`_ `numpy.floating`_ Wavelength array of the spectral blaze function -``bpm`` `numpy.ndarray`_ `numpy.uint8`_ Bad pixel mask of the datacube (0=good, 1=bad) -``flux`` `numpy.ndarray`_ `numpy.floating`_ Flux datacube in units of counts/s/Ang/arcsec^2or 10^-17 erg/s/cm^2/Ang/arcsec^2 -``fluxed`` bool Boolean indicating if the datacube is fluxed. -``sensfunc`` `numpy.ndarray`_ `numpy.floating`_ Sensitivity function 10^-17 erg/(counts/cm^2) -``sig`` `numpy.ndarray`_ `numpy.floating`_ Error datacube (matches units of flux) -============== ================ ================= ================================================================================ +============== ================ ================= ================================================================================= +Attribute Type Array Type Description +============== ================ ================= ================================================================================= +``PYP_SPEC`` str PypeIt: Spectrograph name +``blaze_spec`` `numpy.ndarray`_ `numpy.floating`_ The spectral blaze function +``blaze_wave`` `numpy.ndarray`_ `numpy.floating`_ Wavelength array of the spectral blaze function +``bpm`` `numpy.ndarray`_ `numpy.uint8`_ Bad pixel mask of the datacube (0=good, 1=bad) +``flux`` `numpy.ndarray`_ `numpy.floating`_ Flux datacube in units of counts/s/Ang/arcsec^2 or 10^-17 erg/s/cm^2/Ang/arcsec^2 +``fluxed`` bool Boolean indicating if the datacube is fluxed. +``sensfunc`` `numpy.ndarray`_ `numpy.floating`_ Sensitivity function 10^-17 erg/(counts/cm^2) +``sig`` `numpy.ndarray`_ `numpy.floating`_ Error datacube (matches units of flux) +============== ================ ================= ================================================================================= diff --git a/doc/include/class_datamodel_specobj.rst b/doc/include/class_datamodel_specobj.rst index c44849c9f0..11492ddf32 100644 --- a/doc/include/class_datamodel_specobj.rst +++ b/doc/include/class_datamodel_specobj.rst @@ -1,5 +1,5 @@ -**Version**: 1.1.8 +**Version**: 1.1.10 ======================= =================================================================================================== ===================== ==================================================================================================================================================================================== Attribute Type Array Type Description @@ -15,6 +15,7 @@ Attribute Type ``BOX_FLAM_IVAR`` `numpy.ndarray`_ float Boxcar flux inverse variance (1e-17 erg/s/cm^2/Ang)^-2 ``BOX_FLAM_SIG`` `numpy.ndarray`_ float Boxcar flux uncertainty (1e-17 erg/s/cm^2/Ang) ``BOX_FRAC_USE`` `numpy.ndarray`_ float Fraction of pixels in the object profile subimage used for this extraction +``BOX_FWHM`` `numpy.ndarray`_ float Spectral FWHM (in Angstroms) at every pixel of the boxcar extracted flux. ``BOX_MASK`` `numpy.ndarray`_ `numpy.bool`_ Mask for boxcar extracted flux. True=good ``BOX_NPIX`` `numpy.ndarray`_ float Number of pixels used for the boxcar extraction; can be fractional ``BOX_RADIUS`` float Size of boxcar radius (pixels) @@ -31,7 +32,7 @@ Attribute Type ``FLEX_SHIFT_LOCAL`` float Local shift of the spectrum to correct for spectralflexure (pixels). This should be a small correction tothe global value, and is based on the sky spectrumextracted near the object ``FLEX_SHIFT_TOTAL`` float Total shift of the spectrum to correct for spectralflexure (pixels). This is the sum of the global andlocal FLEX_SHIFT ``FWHM`` float Spatial FWHM of the object (pixels) -``FWHMFIT`` `numpy.ndarray`_ Spatial FWHM across the detector (pixels) +``FWHMFIT`` `numpy.ndarray`_ float Spatial FWHM across the detector (pixels) ``MASKDEF_EXTRACT`` bool Boolean indicating if this is a forced extraction at the expected location from slitmask design. ``MASKDEF_ID`` int, `numpy.integer`_ Slitmask definition ID ``MASKDEF_OBJMAG`` float Magnitude of the object from the slitmask definition @@ -51,6 +52,7 @@ Attribute Type ``OPT_FLAM_IVAR`` `numpy.ndarray`_ float Optimal flux inverse variance (1e-17 erg/s/cm^2/Ang)^-2 ``OPT_FLAM_SIG`` `numpy.ndarray`_ float Optimal flux uncertainty (1e-17 erg/s/cm^2/Ang) ``OPT_FRAC_USE`` `numpy.ndarray`_ float Fraction of pixels in the object profile subimage used for this extraction +``OPT_FWHM`` `numpy.ndarray`_ float Spectral FWHM (in Angstroms) at every pixel of the optimally extracted flux. ``OPT_MASK`` `numpy.ndarray`_ `numpy.bool`_ Mask for optimally extracted flux. True=good ``OPT_WAVE`` `numpy.ndarray`_ float Optimal Wavelengths in vacuum (Angstroms) ``PYPELINE`` str Name of the PypeIt pipeline mode @@ -58,13 +60,14 @@ Attribute Type ``S2N`` float Median signal to noise ratio of the extracted spectrum(OPT if available, otherwise BOX) ``SLITID`` int, `numpy.integer`_ PypeIt slit ID (aka SPAT_ID). ``SPAT_FRACPOS`` float, `numpy.floating`_ Fractional location of the object on the slit +``SPAT_FWHM`` float Spatial FWHM of the object (arcsec) ``SPAT_PIXPOS`` float, `numpy.floating`_ Spatial location of the trace on detector (pixel) at half-way ``TRACE_SPAT`` `numpy.ndarray`_ float Object trace along the spec (spatial pixel) ``VEL_CORR`` float Relativistic velocity correction for wavelengths ``VEL_TYPE`` str Type of heliocentric correction (if any) ``WAVE_RMS`` float, `numpy.floating`_ RMS (pix) for the wavelength solution for this slit. ``hand_extract_flag`` bool Boolean indicating if this is a forced extraction at the location provided by the user. -``maskwidth`` float, `numpy.floating`_ Size (in units of fwhm) of the region used for local sky subtraction +``maskwidth`` float, `numpy.floating`_ Size (in units of spatial fwhm) of the region used for local sky subtraction ``smash_peakflux`` float Peak value of the spectral direction collapsed spatial profile ``smash_snr`` float Peak S/N ratio of the spectral direction collapsed patial profile ``trace_spec`` `numpy.ndarray`_ int, `numpy.integer`_ Array of pixels along the spectral direction diff --git a/doc/include/class_datamodel_wavecalib.rst b/doc/include/class_datamodel_wavecalib.rst index abe9120c0c..eba69b1591 100644 --- a/doc/include/class_datamodel_wavecalib.rst +++ b/doc/include/class_datamodel_wavecalib.rst @@ -1,5 +1,5 @@ -**Version**: 1.1.0 +**Version**: 1.1.1 =============== ================ ================================================ =================================================================================================================================================================== Attribute Type Array Type Description @@ -7,6 +7,7 @@ Attribute Type Array Type ``PYP_SPEC`` str PypeIt spectrograph name ``arc_spectra`` `numpy.ndarray`_ `numpy.floating`_ 2D array: 1D extracted spectra, slit by slit (nspec, nslits) ``det_img`` `numpy.ndarray`_ `numpy.integer`_ Detector image which indicates which pixel in the mosaic corresponds to which detector; used occasionally by echelle. Currently only saved if ech_separate_2d=True +``fwhm_map`` `numpy.ndarray`_ :class:`~pypeit.core.fitting.PypeItFit` A fit that determines the spectral FWHM at every location of every slit ``lamps`` str List of arc lamps used for the wavelength calibration ``nslits`` int Total number of slits. This can include masked slits ``spat_ids`` `numpy.ndarray`_ `numpy.integer`_ Slit spat_ids. Named distinctly from that in WaveFit diff --git a/doc/include/class_datamodel_wavetilts.rst b/doc/include/class_datamodel_wavetilts.rst index aa5f30354d..f071e3f17d 100644 --- a/doc/include/class_datamodel_wavetilts.rst +++ b/doc/include/class_datamodel_wavetilts.rst @@ -1,19 +1,19 @@ **Version**: 1.2.0 -==================== ============================ ================= ============================================================================================================================== -Attribute Type Array Type Description -==================== ============================ ================= ============================================================================================================================== -``PYP_SPEC`` str PypeIt spectrograph name -``bpmtilts`` `numpy.ndarray`_ `numpy.integer`_ Bad pixel mask for tilt solutions. Keys are taken from SlitTraceSetBitmask -``coeffs`` `numpy.ndarray`_ `numpy.floating`_ 2D coefficents for the fit on the initial slits. One set per slit/order (3D array). -``func2d`` str Function used for the 2D fit -``nslit`` int Total number of slits. This can include masked slits -``slits_filename`` str Path to SlitTraceSet file. This helps to find the Slits calibration file when running pypeit_chk_tilts() -``spat_flexure`` float Flexure shift from the input TiltImage -``spat_id`` `numpy.ndarray`_ `numpy.integer`_ Slit spat_id -``spat_order`` `numpy.ndarray`_ `numpy.integer`_ Order for spatial fit (nslit) -``spec_order`` `numpy.ndarray`_ `numpy.integer`_ Order for spectral fit (nslit) -``tilt_traces`` `astropy.table.table.Table`_ Table with the positions of the traced and fitted tilts for all the slits. see :func:`make_tbl_tilt_traces` for more details. -``tiltimg_filename`` str Path to Tiltimg file. This helps to find Tiltimg file when running pypeit_chk_tilts() -==================== ============================ ================= ============================================================================================================================== +==================== ============================ ================= =============================================================================================================================================================== +Attribute Type Array Type Description +==================== ============================ ================= =============================================================================================================================================================== +``PYP_SPEC`` str PypeIt spectrograph name +``bpmtilts`` `numpy.ndarray`_ `numpy.integer`_ Bad pixel mask for tilt solutions. Keys are taken from SlitTraceSetBitmask +``coeffs`` `numpy.ndarray`_ `numpy.floating`_ 2D coefficents for the fit on the initial slits. One set per slit/order (3D array). +``func2d`` str Function used for the 2D fit +``nslit`` int Total number of slits. This can include masked slits +``slits_filename`` str Path to SlitTraceSet file. This helps to find the Slits calibration file when running pypeit_chk_tilts() +``spat_flexure`` float Flexure shift from the input TiltImage +``spat_id`` `numpy.ndarray`_ `numpy.integer`_ Slit spat_id +``spat_order`` `numpy.ndarray`_ `numpy.integer`_ Order for spatial fit (nslit) +``spec_order`` `numpy.ndarray`_ `numpy.integer`_ Order for spectral fit (nslit) +``tilt_traces`` `astropy.table.table.Table`_ Table with the positions of the traced and fitted tilts for all the slits. see :func:`~pypeit.wavetilts.BuildWaveTilts.make_tbl_tilt_traces` for more details. +``tiltimg_filename`` str Path to Tiltimg file. This helps to find Tiltimg file when running pypeit_chk_tilts() +==================== ============================ ================= =============================================================================================================================================================== diff --git a/doc/include/datamodel_sensfunc.rst b/doc/include/datamodel_sensfunc.rst index 43578bde7e..2d9e9b5bf2 100644 --- a/doc/include/datamodel_sensfunc.rst +++ b/doc/include/datamodel_sensfunc.rst @@ -44,18 +44,23 @@ Column Data Type Description SENS table -========================== ========= ====================================================== -Column Data Type Description -========================== ========= ====================================================== -``SENS_WAVE`` float64 Wavelength vector -``SENS_COUNTS_PER_ANG`` float64 Flux in counts per angstrom -``SENS_ZEROPOINT`` float64 Measured sensitivity zero-point data -``SENS_ZEROPOINT_GPM`` bool Good-pixel mask for the measured zero points -``SENS_ZEROPOINT_FIT`` float64 Best-fit smooth model to the zero points -``SENS_ZEROPOINT_FIT_GPM`` bool Good-pixel mask for the model zero points -``SENS_COEFF`` float64 Coefficients of smooth model fit to zero points -``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) -``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) -``WAVE_MIN`` float64 Minimum wavelength included in the fit -``WAVE_MAX`` float64 Maximum wavelength included in the fit -========================== ========= ====================================================== +============================= ========= ====================================================================== +Column Data Type Description +============================= ========= ====================================================================== +``SENS_WAVE`` float64 Wavelength vector +``SENS_COUNTS_PER_ANG`` float64 Flux in counts per angstrom +``SENS_LOG10_BLAZE_FUNCTION`` float64 Log10 of the blaze function for each slit/order +``SENS_ZEROPOINT`` float64 Measured sensitivity zero-point data +``SENS_ZEROPOINT_GPM`` bool Good-pixel mask for the measured zero points +``SENS_ZEROPOINT_FIT`` float64 Best-fit smooth model to the zero points +``SENS_ZEROPOINT_FIT_GPM`` bool Good-pixel mask for the model zero points +``SENS_COEFF`` float64 Coefficients of smooth model fit to zero points +``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) +``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) +``WAVE_MIN`` float64 Minimum wavelength included in the fit +``WAVE_MAX`` float64 Maximum wavelength included in the fit +``SENS_FLUXED_STD_WAVE`` float64 The wavelength array for the fluxed standard star spectrum +``SENS_FLUXED_STD_FLAM`` float64 The F_lambda for the fluxed standard star spectrum +``SENS_FLUXED_STD_FLAM_IVAR`` float64 The inverse variance of F_lambda for the fluxed standard star spectrum +``SENS_FLUXED_STD_MASK`` bool The good pixel mask for the fluxed standard star spectrum +============================= ========= ====================================================================== diff --git a/doc/include/datamodel_specobj.rst b/doc/include/datamodel_specobj.rst index eea96330c5..deb7cb2c19 100644 --- a/doc/include/datamodel_specobj.rst +++ b/doc/include/datamodel_specobj.rst @@ -1,6 +1,6 @@ -Version: 1.1.8 +Version: 1.1.10 ======================= ========================= ================= ==================================================================================================================================================================================== Obj Key Obj Type Array Type Description @@ -16,6 +16,7 @@ Obj Key Obj Type Array Type Descripti ``BOX_FLAM_IVAR`` ndarray float Boxcar flux inverse variance (1e-17 erg/s/cm^2/Ang)^-2 ``BOX_FLAM_SIG`` ndarray float Boxcar flux uncertainty (1e-17 erg/s/cm^2/Ang) ``BOX_FRAC_USE`` ndarray float Fraction of pixels in the object profile subimage used for this extraction +``BOX_FWHM`` ndarray float Spectral FWHM (in Angstroms) at every pixel of the boxcar extracted flux. ``BOX_MASK`` ndarray bool Mask for boxcar extracted flux. True=good ``BOX_NPIX`` ndarray float Number of pixels used for the boxcar extraction; can be fractional ``BOX_RADIUS`` float Size of boxcar radius (pixels) @@ -32,7 +33,7 @@ Obj Key Obj Type Array Type Descripti ``FLEX_SHIFT_LOCAL`` float Local shift of the spectrum to correct for spectralflexure (pixels). This should be a small correction tothe global value, and is based on the sky spectrumextracted near the object ``FLEX_SHIFT_TOTAL`` float Total shift of the spectrum to correct for spectralflexure (pixels). This is the sum of the global andlocal FLEX_SHIFT ``FWHM`` float Spatial FWHM of the object (pixels) -``FWHMFIT`` ndarray Spatial FWHM across the detector (pixels) +``FWHMFIT`` ndarray float Spatial FWHM across the detector (pixels) ``MASKDEF_EXTRACT`` bool Boolean indicating if this is a forced extraction at the expected location from slitmask design. ``MASKDEF_ID`` int, integer Slitmask definition ID ``MASKDEF_OBJMAG`` float Magnitude of the object from the slitmask definition @@ -52,6 +53,7 @@ Obj Key Obj Type Array Type Descripti ``OPT_FLAM_IVAR`` ndarray float Optimal flux inverse variance (1e-17 erg/s/cm^2/Ang)^-2 ``OPT_FLAM_SIG`` ndarray float Optimal flux uncertainty (1e-17 erg/s/cm^2/Ang) ``OPT_FRAC_USE`` ndarray float Fraction of pixels in the object profile subimage used for this extraction +``OPT_FWHM`` ndarray float Spectral FWHM (in Angstroms) at every pixel of the optimally extracted flux. ``OPT_MASK`` ndarray bool Mask for optimally extracted flux. True=good ``OPT_WAVE`` ndarray float Optimal Wavelengths in vacuum (Angstroms) ``PYPELINE`` str Name of the PypeIt pipeline mode @@ -59,13 +61,14 @@ Obj Key Obj Type Array Type Descripti ``S2N`` float Median signal to noise ratio of the extracted spectrum(OPT if available, otherwise BOX) ``SLITID`` int, integer PypeIt slit ID (aka SPAT_ID). ``SPAT_FRACPOS`` float, floating Fractional location of the object on the slit +``SPAT_FWHM`` float Spatial FWHM of the object (arcsec) ``SPAT_PIXPOS`` float, floating Spatial location of the trace on detector (pixel) at half-way ``TRACE_SPAT`` ndarray float Object trace along the spec (spatial pixel) ``VEL_CORR`` float Relativistic velocity correction for wavelengths ``VEL_TYPE`` str Type of heliocentric correction (if any) ``WAVE_RMS`` float, floating RMS (pix) for the wavelength solution for this slit. ``hand_extract_flag`` bool Boolean indicating if this is a forced extraction at the location provided by the user. -``maskwidth`` float, floating Size (in units of fwhm) of the region used for local sky subtraction +``maskwidth`` float, floating Size (in units of spatial fwhm) of the region used for local sky subtraction ``smash_peakflux`` float Peak value of the spectral direction collapsed spatial profile ``smash_snr`` float Peak S/N ratio of the spectral direction collapsed patial profile ``trace_spec`` ndarray int,numpy.integer Array of pixels along the spectral direction diff --git a/doc/include/datamodel_wavecalib.rst b/doc/include/datamodel_wavecalib.rst index b8e7eaef22..e9eea37072 100644 --- a/doc/include/datamodel_wavecalib.rst +++ b/doc/include/datamodel_wavecalib.rst @@ -1,5 +1,5 @@ -Version 1.1.0 +Version 1.1.1 ======================== ============================== ========= =================================================================================================================================== HDU Name HDU Type Data Type Description diff --git a/doc/include/dependencies_table.rst b/doc/include/dependencies_table.rst index 5412b694ef..407b7af4f9 100644 --- a/doc/include/dependencies_table.rst +++ b/doc/include/dependencies_table.rst @@ -1,5 +1,5 @@ -======================= ================================================================================================================================================================================================================================================================================================= -Python Version ``>=3.9,<3.12`` -Required for users ``IPython>=7.10.0``, ``PyYAML>=5.1``, ``astropy>=4.3``, ``bottleneck``, ``configobj>=5.0.6``, ``extension-helpers>=0.1``, ``ginga>=3.2``, ``linetools``, ``matplotlib>=3.3``, ``numpy>=1.21``, ``packaging>=0.19``, ``pygithub``, ``pyqt6``, ``qtpy>=1.9``, ``scikit-learn>=1.0``, ``scipy>=1.7`` -Required for developers ``coverage``, ``docutils<0.18``, ``pytest-astropy``, ``pytest-cov``, ``pytest>=6.0.0``, ``sphinx-automodapi``, ``sphinx<6,>=1.6``, ``sphinx_rtd_theme==1.1.1``, ``tox`` -======================= ================================================================================================================================================================================================================================================================================================= +======================= ===================================================================================================================================================================================================================================================================================================== +Python Version ``>=3.9,<3.12`` +Required for users ``IPython>=7.10.0``, ``PyYAML>=5.1``, ``astropy>=4.3``, ``bottleneck``, ``configobj>=5.0.6``, ``extension-helpers>=0.1``, ``ginga>=4.1.1``, ``linetools``, ``matplotlib>=3.7``, ``numpy>=1.22``, ``packaging>=0.19``, ``pygithub``, ``pyqt6``, ``qtpy>=2.0.1``, ``scikit-learn>=1.0``, ``scipy>=1.7`` +Required for developers ``coverage``, ``docutils<0.19``, ``pytest-astropy``, ``pytest-cov``, ``pytest>=6.0.0``, ``scikit-image``, ``specutils``, ``sphinx-automodapi``, ``sphinx<7,>=1.6``, ``sphinx_rtd_theme==1.2.2``, ``tox`` +======================= ===================================================================================================================================================================================================================================================================================================== diff --git a/doc/include/dev_suite_readme.rst b/doc/include/dev_suite_readme.rst index 57491ad4d6..f996477749 100644 --- a/doc/include/dev_suite_readme.rst +++ b/doc/include/dev_suite_readme.rst @@ -459,8 +459,8 @@ Additional Options --debug Debug using only blue setups (default: False) -p, --prep_only Only prepare to execute run_pypeit, but do not actually run it. (default: False) - -m, --do_not_reuse_masters - run pypeit without using any existing masters + -m, --do_not_reuse_calibs + run pypeit without using any existing calibrations (default: False) -t THREADS, --threads THREADS Run THREADS number of parallel tests. (default: 1) diff --git a/doc/include/gemini_gnirs_A.pypeit.rst b/doc/include/gemini_gnirs_echelle_A.pypeit.rst similarity index 98% rename from doc/include/gemini_gnirs_A.pypeit.rst rename to doc/include/gemini_gnirs_echelle_A.pypeit.rst index 06ca5ad954..71e0008845 100644 --- a/doc/include/gemini_gnirs_A.pypeit.rst +++ b/doc/include/gemini_gnirs_echelle_A.pypeit.rst @@ -5,7 +5,7 @@ # User-defined execution parameters [rdx] - spectrograph = gemini_gnirs + spectrograph = gemini_gnirs_echelle # Setup setup read @@ -17,7 +17,7 @@ # Data block data read - path /Users/westfall/Work/packages/PypeIt-development-suite/RAW_DATA/gemini_gnirs/32_SB_SXD + path /Users/westfall/Work/packages/PypeIt-development-suite/RAW_DATA/gemini_gnirs_echelle/32_SB_SXD filename | frametype | ra | dec | target | dispname | decker | binning | mjd | airmass | exptime | dispangle | dithoff | calib | comb_id | bkg_id cN20170331S0216.fits | arc,science,tilt | 205.53380833 | 9.47733611 | pisco | 32/mmSB_G5533 | 0.68arcsec_G5530 | 1,1 | 57843.3709743134 | 1.077 | 300.0 | 6.1887 | -0.34225501721318 | 0 | 5 | -1 cN20170331S0217.fits | arc,science,tilt | 205.53380833 | 9.47733611 | pisco | 32/mmSB_G5533 | 0.68arcsec_G5530 | 1,1 | 57843.3746886267 | 1.068 | 300.0 | 6.1887 | 2.65774498278682 | 0 | 6 | -1 diff --git a/doc/include/gemini_gnirs_A_corrected.pypeit.rst b/doc/include/gemini_gnirs_echelle_A_corrected.pypeit.rst similarity index 95% rename from doc/include/gemini_gnirs_A_corrected.pypeit.rst rename to doc/include/gemini_gnirs_echelle_A_corrected.pypeit.rst index e9b8cf8af5..54821e864c 100644 --- a/doc/include/gemini_gnirs_A_corrected.pypeit.rst +++ b/doc/include/gemini_gnirs_echelle_A_corrected.pypeit.rst @@ -5,11 +5,11 @@ # User-defined execution parameters [rdx] - spectrograph = gemini_gnirs - [calibrations] - [[wavelengths]] - rms_threshold = 0.8, 0.4, 0.8, 0.8, 0.5, 0.9 - sigdetect = 5.,5.,6.,6.,5.,7. + spectrograph = gemini_gnirs_echelle + #[calibrations] + # [[wavelengths]] + # rms_threshold = 0.8, 0.4, 0.8, 0.8, 0.5, 0.9 + # sigdetect = 5.,5.,6.,6.,5.,7. # Setup setup read @@ -18,7 +18,7 @@ # Read in the data data read - path /Users/joe/python/PypeIt-development-suite/RAW_DATA/gemini_gnirs/32_SB_SXD/ + path /Users/joe/python/PypeIt-development-suite/RAW_DATA/gemini_gnirs_echelle/32_SB_SXD/ | filename | date | frametype | target | exptime | dispname | decker | setup | calib | dispangle | idname | comb_id | bkg_id | | cN20170331S0206.fits | 2017-03-31 | standard | HIP62745 | 10.0 | 32/mmSB_G5533 | SCXD_G5531 | A | 0 | 6.1887 | OBJECT | 5 | 6 | | cN20170331S0207.fits | 2017-03-31 | standard | HIP62745 | 10.0 | 32/mmSB_G5533 | SCXD_G5531 | A | 0 | 6.1887 | OBJECT | 6 | 5 | diff --git a/doc/include/inst_detector_table.rst b/doc/include/inst_detector_table.rst index 5f18ef4af5..f7f6c8ad74 100644 --- a/doc/include/inst_detector_table.rst +++ b/doc/include/inst_detector_table.rst @@ -2,8 +2,8 @@ Instrument Det specaxis specflip spatflip namp gain RN darkcurr min sat nonlinear platescale ============================ === ======== ======== ======== ======== ========================== ====================== ======== ======== ============ ========= ========== ``bok_bc`` 1 1 False False 1 1.5 3.0 5.4 -1.0e+10 65535.0 1.0000 0.2000 -``gemini_flamingos1`` 1 0 False False 1 3.8 6.0 0.01 -1.0e+10 320000.0 0.8750 0.1500 -``gemini_flamingos2`` 1 0 True False 1 4.44 5.0 0.5 -1.0e+10 700000.0 1.0000 0.1787 +``gemini_flamingos1`` 1 0 False False 1 3.8 6.0 1080.0 -1.0e+10 320000.0 0.8750 0.1500 +``gemini_flamingos2`` 1 0 True False 1 4.44 5.0 1800.0 -1.0e+10 700000.0 1.0000 0.1787 ``gemini_gmos_north_e2v`` 1 1 False False 2 2.27, 2.27 3.32, 3.32 0.0 -1.0e+10 110900.0 0.9500 0.0728 ... 2 1 False False 2 2.27, 2.27 3.32, 3.32 0.0 -1.0e+10 115500.0 0.9500 0.0728 ... 3 1 False False 2 2.27, 2.27 3.32, 3.32 0.0 -1.0e+10 116700.0 0.9500 0.0728 @@ -16,15 +16,16 @@ Instrument Det specaxis specflip spatflip namp gain ``gemini_gmos_south_ham`` 1 1 False False 4 1.83, 1.83, 1.83, 1.83 3.98, 3.98, 3.98, 3.98 0.0 -1.0e+10 129000.0 0.9500 0.0800 ... 2 1 False False 4 1.83, 1.83, 1.83, 1.83 3.98, 3.98, 3.98, 3.98 0.0 -1.0e+10 123000.0 0.9500 0.0800 ... 3 1 False False 4 1.83, 1.83, 1.83, 1.83 3.98, 3.98, 3.98, 3.98 0.0 -1.0e+10 125000.0 0.9500 0.0800 -``gemini_gnirs`` 1 0 True True 1 13.5 7.0 0.15 -1.0e+10 150000.0 0.7100 0.1500 +``gemini_gnirs_echelle`` 1 0 True True 1 13.5 7.0 540.0 -1.0e+10 150000.0 0.7100 0.1500 +``gemini_gnirs_ifu`` 1 0 True True 1 13.5 7.0 540.0 -1.0e+10 150000.0 0.7100 0.1500 ``gtc_maat`` 1 1 True False 1 1.9 4.3 5.0 0.0e+00 65535.0 0.9500 0.1250 ``gtc_osiris`` 1 0 False False 1 0.95 4.5 0.0 0.0e+00 65535.0 0.9500 0.1270 ... 2 0 False False 1 0.95 4.5 0.0 0.0e+00 65535.0 0.9500 0.1270 ``gtc_osiris_plus`` 1 1 True False 1 1.9 4.3 5.0 0.0e+00 65535.0 0.9500 0.1250 -``jwst_nircam`` 1 1 False False 1 1.84 8.55 0.0335 -1.0e+10 59200.0 0.9500 0.0630 -... 2 1 False False 1 1.8 8.57 0.035 -1.0e+10 58500.0 0.9500 0.0630 -``jwst_nirspec`` 1 1 False False 1 0.996 5.17 0.0092 -1.0e+10 55100.0 0.9500 0.1000 -... 2 1 False False 1 1.137 6.6 0.0057 -1.0e+10 60400.0 0.9500 0.1000 +``jwst_nircam`` 1 1 False False 1 1.84 8.55 120.6 -1.0e+10 59200.0 0.9500 0.0630 +... 2 1 False False 1 1.8 8.57 126.0 -1.0e+10 58500.0 0.9500 0.0630 +``jwst_nirspec`` 1 1 False False 1 0.996 5.17 33.12 -1.0e+10 55100.0 0.9500 0.1000 +... 2 1 False False 1 1.137 6.6 20.52 -1.0e+10 60400.0 0.9500 0.1000 ``keck_deimos`` 1 0 False False 1 1.226 2.57 3.3 -1.0e+10 65535.0 0.9500 0.1185 ... 2 0 False False 1 1.188 2.491 3.6 -1.0e+10 65535.0 0.9500 0.1185 ... 3 0 False False 1 1.248 2.618 3.5 -1.0e+10 65535.0 0.9500 0.1185 @@ -33,21 +34,23 @@ Instrument Det specaxis specflip spatflip namp gain ... 6 0 False False 1 1.177 2.469 3.8 -1.0e+10 65535.0 0.9500 0.1185 ... 7 0 False False 1 1.201 2.518 3.3 -1.0e+10 65535.0 0.9500 0.1185 ... 8 0 False False 1 1.23 2.58 3.7 -1.0e+10 65535.0 0.9500 0.1185 +``keck_esi`` 1 0 False False 2 1.3, 1.3 2.5, 2.5 2.1 -1.0e+10 65535.0 0.9900 0.1542 ``keck_hires`` 1 0 False False 1 1.9 2.8 0.0 -1.0e+10 65535.0 0.7000 0.1350 ... 2 0 False False 1 2.1 3.1 0.0 -1.0e+10 65535.0 0.7000 0.1350 ... 3 0 False False 1 2.1 3.1 0.0 -1.0e+10 65535.0 0.7000 0.1350 -``keck_kcwi`` 1 0 None False ``None`` ``None`` ``None`` 0.0 -1.0e+10 65535.0 0.9500 0.1457 +``keck_kcrm`` 1 0 None False ``None`` ``None`` ``None`` 0.0 -1.0e+10 65535.0 0.9500 0.1457 +``keck_kcwi`` 1 0 None False ``None`` ``None`` ``None`` 1.0 -1.0e+10 65535.0 0.9500 0.1457 ``keck_lris_blue`` 1 0 False False 2 1.55, 1.56 3.9, 4.2 0.0 -1.0e+10 65535.0 0.8600 0.1350 ... 2 0 False False 2 1.63, 1.7 3.6, 3.6 0.0 -1.0e+10 65535.0 0.8600 0.1350 ``keck_lris_blue_orig`` 1 0 True False 2 1.55, 1.56 3.9, 4.2 0.0 -1.0e+10 65535.0 0.8600 0.1350 ... 2 0 True False 2 1.63, 1.7 3.6, 3.6 0.0 -1.0e+10 65535.0 0.8600 0.1350 ``keck_lris_red`` 1 0 False False 2 1.255, 1.18 4.64, 4.76 0.0 -1.0e+10 65535.0 0.7600 0.1350 ... 2 0 False False 2 1.191, 1.162 4.54, 4.62 0.0 -1.0e+10 65535.0 0.7600 0.1350 -``keck_lris_red_mark4`` 1 0 True False 2 1.61, 1.60153 3.65, 3.52 0.0 -1.0e+10 65535.0 0.7600 0.1230 -``keck_lris_red_orig`` 1 1 False False 2 1.98, 2.17 6.1, 6.3 0.0 -1.0e+10 65535.0 0.7600 0.2100 -``keck_mosfire`` 1 1 False False 1 2.15 5.8 0.8 -1.0e+10 1000000000.0 1.0000 0.1798 -``keck_nires`` 1 1 True False 1 3.8 5.0 0.01 -1.0e+10 1000000.0 0.7600 0.1500 -``keck_nirspec_low`` 1 0 False False 1 5.8 23.0 0.8 -1.0e+10 100000.0 1.0000 0.1930 +``keck_lris_red_mark4`` 1 0 True True 2 1.61, 1.60153 3.65, 3.52 0.0 -1.0e+10 65535.0 0.7600 0.1350 +``keck_lris_red_orig`` 1 1 False False 2 1.98, 2.17 6.1, 6.3 0.0 -1.0e+10 65535.0 0.7600 0.2110 +``keck_mosfire`` 1 1 False False 1 2.15 5.8 28.8 -1.0e+10 1000000000.0 1.0000 0.1798 +``keck_nires`` 1 1 True False 1 3.8 5.0 468.0 -1.0e+10 1000000.0 0.7600 0.1500 +``keck_nirspec_low`` 1 0 False False 1 5.8 23.0 2520.0 -1.0e+10 100000.0 1.0000 0.1930 ``lbt_luci1`` 1 1 False False 1 2.0 4.61 0.0 -1.0e+10 100000000.0 0.8000 0.2500 ``lbt_luci2`` 1 1 False False 1 2.0 4.47 0.0 -1.0e+10 100000000.0 0.8000 0.2500 ``lbt_mods1b`` 1 0 True False 4 2.55, 1.91, 2.09, 2.02 3.41, 2.93, 2.92, 2.76 0.5 -1.0e+10 65535.0 0.9900 0.1200 @@ -55,29 +58,30 @@ Instrument Det specaxis specflip spatflip namp gain ``lbt_mods2b`` 1 0 True False 4 1.99, 2.06, 1.96, 2.01 3.66, 3.62, 3.72, 3.64 0.5 -1.0e+10 65535.0 0.9900 0.1200 ``lbt_mods2r`` 1 0 False False 4 1.7, 1.67, 1.66, 1.66 2.95, 2.65, 2.78, 2.87 0.4 -1.0e+10 65535.0 0.9900 0.1230 ``ldt_deveny`` 1 1 True False 1 1.52 4.9 4.5 -1.0e+10 65535.0 0.9700 0.3400 -``magellan_fire`` 1 1 True False 1 1.2 5.0 0.01 -1.0e+10 100000.0 1.0000 0.1800 -``magellan_fire_long`` 1 0 False False 1 3.8 6.0 0.01 -1.0e+10 320000.0 0.8750 0.1500 +``magellan_fire`` 1 1 True False 1 1.2 5.0 3.06 -1.0e+10 100000.0 1.0000 0.1800 +``magellan_fire_long`` 1 0 False False 1 3.8 6.0 3.06 -1.0e+10 320000.0 0.8750 0.1500 ``magellan_mage`` 1 1 True False 1 1.02 2.9 1.0 -1.0e+10 65535.0 0.9900 0.3000 +``mdm_modspec`` 1 0 True False 1 1.3 7.9 0.0 -1.0e+10 65535.0 0.9700 0.2800 ``mdm_osmos_mdm4k`` 1 1 True False 4 2.2, 2.2, 2.2, 2.2 5.0, 5.0, 5.0, 5.0 0.0 -1.0e+10 65535.0 0.8600 0.2730 -``mmt_binospec`` 1 0 False False 4 1.085, 1.046, 1.042, 0.975 3.2, 3.2, 3.2, 3.2 3.0 -1.0e+10 65535.0 0.9500 0.2400 -... 2 0 False False 4 1.028, 1.115, 1.047, 1.045 3.6, 3.6, 3.6, 3.6 3.0 -1.0e+10 65535.0 0.9500 0.2400 +``mmt_binospec`` 1 0 False False 4 1.085, 1.046, 1.042, 0.975 3.2, 3.2, 3.2, 3.2 3.6 -1.0e+10 65535.0 0.9500 0.2400 +... 2 0 False False 4 1.028, 1.115, 1.047, 1.045 3.6, 3.6, 3.6, 3.6 3.6 -1.0e+10 65535.0 0.9500 0.2400 ``mmt_bluechannel`` 1 0 False False 1 ``None`` ``None`` 0.0 -1.0e+10 65535.0 0.9500 0.3000 -``mmt_mmirs`` 1 0 False False 1 0.95 3.14 0.01 -1.0e+10 700000.0 1.0000 0.2012 +``mmt_mmirs`` 1 0 False False 1 0.95 3.14 36.0 -1.0e+10 700000.0 1.0000 0.2012 ``not_alfosc`` 1 0 True False 1 ``None`` ``None`` 1.3 -1.0e+10 700000.0 0.8600 0.2138 ``not_alfosc_vert`` 1 1 False False 1 ``None`` ``None`` 1.3 -1.0e+10 700000.0 0.8600 0.2138 ``ntt_efosc2`` 1 0 False False 1 0.91 10.0 0.0 -1.0e+10 65535 0.8000 0.1200 ``p200_dbsp_blue`` 1 0 True False 1 0.72 2.5 0.0 -1.0e+10 65000.0 0.9538 0.3890 ``p200_dbsp_red`` 1 1 False False 1 2.8 8.5 0.0 -1.0e+10 45000.0 0.8889 0.2930 -``p200_tspec`` 1 1 True False 1 3.8 3.5 0.085 -1.0e+10 28000 0.9000 0.3700 +``p200_tspec`` 1 1 True False 1 3.8 3.5 306.0 -1.0e+10 28000 0.9000 0.3700 ``shane_kast_blue`` 1 1 False False 2 1.2, 1.2 3.7, 3.7 0.0 -1.0e+10 65535.0 0.7600 0.4300 ``shane_kast_red`` 1 0 False False 2 1.9, 1.9 3.8, 3.8 0.0 -1.0e+10 65535.0 0.7600 0.4300 ``shane_kast_red_ret`` 1 1 False False 1 3.0 12.5 0.0 -1.0e+10 120000.0 0.7600 0.7740 -``soar_goodman_blue`` 1 1 False False 1 ``None`` ``None`` 8e-05 -1.0e+10 65535.0 1.0000 0.1500 -``soar_goodman_red`` 1 1 False False 1 ``None`` ``None`` 8e-05 -1.0e+10 65535.0 1.0000 0.1500 +``soar_goodman_blue`` 1 1 False False 1 ``None`` ``None`` 0.0 -1.0e+10 65535.0 1.0000 0.1500 +``soar_goodman_red`` 1 1 False False 1 ``None`` ``None`` 0.0 -1.0e+10 65535.0 1.0000 0.1500 ``tng_dolores`` 1 1 False False 1 0.97 9.0 0.0 -1.0e+10 65500.0 0.9900 0.2520 ``vlt_fors2`` 1 1 False False 1 0.7 2.9 2.1 -1.0e+10 200000.0 0.8000 0.1260 ... 2 1 False False 1 0.7 3.15 1.4 -1.0e+10 200000.0 0.8000 0.1260 -``vlt_sinfoni`` 1 0 True False 1 2.42 7.0 0.15 -1.0e+10 1000000000.0 1.0000 0.0125 +``vlt_sinfoni`` 1 0 True False 1 2.42 7.0 540.0 -1.0e+10 1000000000.0 1.0000 0.0125 ``vlt_xshooter_nir`` 1 1 False False 1 2.12 8.0 0.0 -1.0e+10 200000.0 0.8600 0.1970 ``vlt_xshooter_uvb`` 1 0 True True 1 1.61 2.6 0.0 -1.0e+10 65000.0 0.8600 0.1610 ``vlt_xshooter_vis`` 1 0 False False 1 0.595 3.1 0.0 -1.0e+10 65535.0 0.8600 0.1600 diff --git a/doc/include/keck_deimos.sorted.rst b/doc/include/keck_deimos.sorted.rst index 655eedb3a6..82f3eaa4b4 100644 --- a/doc/include/keck_deimos.sorted.rst +++ b/doc/include/keck_deimos.sorted.rst @@ -10,14 +10,14 @@ filter1: OG550 #--------------------------------------------------------- filename | frametype | ra | dec | target | dispname | decker | binning | mjd | airmass | exptime | dispangle | amp | filter1 | lampstat01 | dateobs | utc | frameno | calib - DE.20170527.06713.fits | arc,tilt | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.077631 | 1.41291034 | 1.0 | 8099.98291016 | SINGLE:B | OG550 | Kr Xe Ar Ne | 2017-05-27 | 01:51:53.87 | 30 | 0 d0527_0030.fits.gz | arc,tilt | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.077631 | 1.41291034 | 1.0 | 8099.98291016 | SINGLE:B | OG550 | Kr Xe Ar Ne | 2017-05-27 | 01:51:53.87 | 30 | 0 - d0527_0031.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.07851 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:53:10.93 | 31 | 0 + DE.20170527.06713.fits | arc,tilt | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.077631 | 1.41291034 | 1.0 | 8099.98291016 | SINGLE:B | OG550 | Kr Xe Ar Ne | 2017-05-27 | 01:51:53.87 | 30 | 0 DE.20170527.06790.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.07851 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:53:10.93 | 31 | 0 + d0527_0031.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.07851 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:53:10.93 | 31 | 0 d0527_0032.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.079356 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:54:24.03 | 32 | 0 DE.20170527.06864.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.079356 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:54:24.03 | 32 | 0 - DE.20170527.06936.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.080211 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:55:36.93 | 33 | 0 d0527_0033.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.080211 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:55:36.93 | 33 | 0 + DE.20170527.06936.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.080211 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:55:36.93 | 33 | 0 DE.20170527.37601.fits | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.435131 | 1.03078874 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:26:41.61 | 80 | 0 DE.20170527.38872.fits | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.449842 | 1.01267696 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:47:52.92 | 81 | 0 d0527_0081.fits.gz | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.449842 | 1.01267696 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:47:52.92 | 81 | 0 diff --git a/doc/include/links.rst b/doc/include/links.rst index ef5b046d3f..dddaf2ac0d 100644 --- a/doc/include/links.rst +++ b/doc/include/links.rst @@ -13,6 +13,7 @@ .. _str.splitlines: https://docs.python.org/3/library/stdtypes.html#str.splitlines .. _textwrap.wrap: https://docs.python.org/3/library/textwrap.html#textwrap.wrap .. _Path: https://docs.python.org/3/library/pathlib.html +.. _io.TextIOWrapper: https://docs.python.org/3/library/io.html#io.TextIOWrapper .. numpy .. _numpy: https://numpy.org/ @@ -34,7 +35,6 @@ .. scipy .. _scipy.optimize.curve_fit: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.curve_fit.html .. _scipy.optimize.leastsq: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.leastsq.html -.. _scipy.interpolate.interp1d: https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.interp1d.html .. _scipy.optimize.least_squares: http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.least_squares.html .. _scipy.optimize.OptimizeResult: http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.OptimizeResult.html .. _scipy.optimize.differential_evolution: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.differential_evolution.html @@ -54,12 +54,14 @@ .. _matplotlib: https://matplotlib.org/ .. _matplotlib.pyplot.imshow: http://matplotlib.org/api/pyplot_api.html#matplotlib.pyplot.imshow .. _matplotlib.axes.Axes: https://matplotlib.org/api/axes_api.html#matplotlib.axes.Axes +.. _matplotlib.backend_bases.Event: https://matplotlib.org/stable/api/backend_bases_api.html#matplotlib.backend_bases.Event .. configobj .. _configobj: http://configobj.readthedocs.io/en/latest/ .. astropy .. _astropy.io.fits: http://docs.astropy.org/en/stable/io/fits/index.html +.. _astropy.io.fits.getheader: https://docs.astropy.org/en/stable/io/fits/api/files.html#astropy.io.fits.getheader .. _astropy.io.fits.open: http://docs.astropy.org/en/stable/io/fits/api/files.html#astropy.io.fits.open .. _astropy.io.fits.HDUList: http://docs.astropy.org/en/stable/io/fits/api/hdulists.html .. _astropy.io.fits.HDUList.writeto: http://docs.astropy.org/en/stable/io/fits/api/hdulists.html#astropy.io.fits.HDUList.writeto @@ -73,7 +75,7 @@ .. _astropy.table.Table.read: https://docs.astropy.org/en/stable/api/astropy.table.Table.html#astropy.table.Table.read .. _astropy.table.Table.remove_rows: https://docs.astropy.org/en/stable/api/astropy.table.Table.html#astropy.table.Table.remove_rows .. _astropy.table.Row: https://docs.astropy.org/en/stable/api/astropy.table.Row.html -.. _astropy.wcs.wcs.WCS: http://docs.astropy.org/en/stable/api/astropy.wcs.WCS.html +.. _astropy.wcs.WCS: http://docs.astropy.org/en/stable/api/astropy.wcs.WCS.html .. _astropy.modeling: http://docs.astropy.org/en/stable/modeling/index.html .. _astropy.modeling.polynomial.Legendre1D: http://docs.astropy.org/en/stable/api/astropy.modeling.polynomial.Legendre1D.html .. _astropy.modeling.models.CompoundModel: http://docs.astropy.org/en/stable/modeling/compound-models.html @@ -81,11 +83,13 @@ .. _astropy.cosmology.FlatLambdaCDM: http://docs.astropy.org/en/stable/api/astropy.cosmology.FlatLambdaCDM.html#flatlambdacdm .. _astropy.constants: http://docs.astropy.org/en/stable/constants/index.html .. _astropy.time.Time: https://docs.astropy.org/en/stable/time/ +.. _astropy.coordinates.Angle: https://docs.astropy.org/en/stable/api/astropy.coordinates.Angle.html .. _astropy.coordinates.SkyCoord: https://docs.astropy.org/en/stable/api/astropy.coordinates.SkyCoord.html .. _astropy.coordinates.EarthLocation: https://docs.astropy.org/en/stable/api/astropy.coordinates.EarthLocation.html .. _astropy.coordinates.SkyCoord.radial_velocity_correction: https://docs.astropy.org/en/stable/api/astropy.coordinates.SkyCoord.html#astropy.coordinates.SkyCoord.radial_velocity_correction .. _astropy.stats.SigmaClip: https://docs.astropy.org/en/stable/api/astropy.stats.SigmaClip.html .. _astropy.stats.sigma_clipped_stats: https://docs.astropy.org/en/stable/api/astropy.stats.sigma_clipped_stats.html +.. _astropy.units.Unit: https://docs.astropy.org/en/stable/api/astropy.units.Unit.html .. _astropy.units.Quantity: https://docs.astropy.org/en/stable/api/astropy.units.Quantity.html .. scikit @@ -122,11 +126,14 @@ .. _bottleneck: https://bottleneck.readthedocs.io/en/latest/ .. _specutils: https://specutils.readthedocs.io/en/stable/ .. _specutils.Spectrum1D: https://specutils.readthedocs.io/en/stable/api/specutils.Spectrum1D.html +.. _specutils.SpectrumList: https://specutils.readthedocs.io/en/stable/api/specutils.SpectrumList.html .. _jdaviz: https://jdaviz.readthedocs.io/en/latest/ .. _jupyter notebook: https://jupyter.org/ +.. _ppxf: https://pypi.org/project/ppxf/ .. ginga .. _ginga: https://ginga.readthedocs.io/en/stable/ +.. _ginga.RemoteClient: https://ginga.readthedocs.io/en/stable/manual/plugins_global/rc.html .. _AstroImage: https://ginga.readthedocs.io/en/stable/dev_manual/image_wrappers.html#astroimage .. _ginga GlobalPlugin: https://ginga.readthedocs.io/en/stable/dev_manual/developers.html#writing-a-global-plugin @@ -147,3 +154,5 @@ .. _L.A.Cosmic: http://www.astro.yale.edu/dokkum/lacosmic/ .. _van Dokkum (2001, PASP, 113, 1420): https://ui.adsabs.harvard.edu/abs/2001PASP..113.1420V/abstract .. _LowRedux: http://www.ucolick.org/~xavier/LowRedux/ + + diff --git a/doc/include/shane_kast_blue_A.pypeit.rst b/doc/include/shane_kast_blue_A.pypeit.rst index 260bb5d36f..b206d00094 100644 --- a/doc/include/shane_kast_blue_A.pypeit.rst +++ b/doc/include/shane_kast_blue_A.pypeit.rst @@ -33,7 +33,6 @@ b11.fits.gz | pixelflat,illumflat,trace | 144.955 | 37.43222222222222 | Dome Flat | 600/4310 | 2.0 arcsec | 1,1 | 57162.07897476852 | 1.0 | 15.0 | d55 | 0 b12.fits.gz | pixelflat,illumflat,trace | 145.0908333333333 | 37.43222222222222 | Dome Flat | 600/4310 | 2.0 arcsec | 1,1 | 57162.079351388886 | 1.0 | 15.0 | d55 | 0 b13.fits.gz | pixelflat,illumflat,trace | 145.22791666666666 | 37.43222222222222 | Dome Flat | 600/4310 | 2.0 arcsec | 1,1 | 57162.079728240744 | 1.0 | 15.0 | d55 | 0 - b2.fits.gz | pixelflat,illumflat,trace | 143.36208333333335 | 37.43222222222222 | Dome Flat | 600/4310 | 2.0 arcsec | 1,1 | 57162.07473645834 | 1.0 | 30.0 | d55 | 0 b3.fits.gz | pixelflat,illumflat,trace | 143.86791666666667 | 37.43222222222222 | Dome Flat | 600/4310 | 2.0 arcsec | 1,1 | 57162.07596400463 | 1.0 | 15.0 | d55 | 0 b4.fits.gz | pixelflat,illumflat,trace | 144.00458333333333 | 37.43222222222222 | Dome Flat | 600/4310 | 2.0 arcsec | 1,1 | 57162.076341782406 | 1.0 | 15.0 | d55 | 0 b5.fits.gz | pixelflat,illumflat,trace | 144.14041666666665 | 37.43222222222222 | Dome Flat | 600/4310 | 2.0 arcsec | 1,1 | 57162.07671956019 | 1.0 | 15.0 | d55 | 0 diff --git a/doc/include/spectrographs_table.rst b/doc/include/spectrographs_table.rst index f1b86b6607..a1f4674edc 100644 --- a/doc/include/spectrographs_table.rst +++ b/doc/include/spectrographs_table.rst @@ -1,61 +1,65 @@ -======================== ============================================================================ ========= ============ =============================================================================================================================== ========= ========= ========= ============================================================================================= -``PypeIt`` Name ``PypeIt`` Class Telescope Camera URL Pipeline Supported QL Tested Comments -======================== ============================================================================ ========= ============ =============================================================================================================================== ========= ========= ========= ============================================================================================= -bok_bc :class:`~pypeit.spectrographs.bok_bc.BokBCSpectrograph` BOK BC `Link `__ MultiSlit True False Bok B&C spectrometer -gemini_flamingos1 :class:`~pypeit.spectrographs.gemini_flamingos.GeminiFLAMINGOS1Spectrograph` GEMINI-S FLAMINGOS `Link `__ MultiSlit False False -gemini_flamingos2 :class:`~pypeit.spectrographs.gemini_flamingos.GeminiFLAMINGOS2Spectrograph` GEMINI-S FLAMINGOS `Link `__ MultiSlit True False Flamingos-2 NIR spectrograph -gemini_gmos_north_e2v :class:`~pypeit.spectrographs.gemini_gmos.GeminiGMOSNE2VSpectrograph` GEMINI-N GMOS-N `Link `__ MultiSlit True False E2V detector; see :doc:`gemini_gmos` -gemini_gmos_north_ham :class:`~pypeit.spectrographs.gemini_gmos.GeminiGMOSNHamSpectrograph` GEMINI-N GMOS-N `Link `__ MultiSlit True False Hamamatsu detector (R400, B600, R831); Used since Feb 2017; see :doc:`gemini_gmos` -gemini_gmos_north_ham_ns :class:`~pypeit.spectrographs.gemini_gmos.GeminiGMOSNHamNSSpectrograph` GEMINI-N GMOS-N `Link `__ MultiSlit True False Same as gemini_gmos_north_ham when used in nod-and-shuffle mode; see :doc:`gemini_gmos` -gemini_gmos_south_ham :class:`~pypeit.spectrographs.gemini_gmos.GeminiGMOSSHamSpectrograph` GEMINI-S GMOS-S `Link `__ MultiSlit True False Hamamatsu detector (R400, B600, R831); see :doc:`gemini_gmos` -gemini_gnirs :class:`~pypeit.spectrographs.gemini_gnirs.GeminiGNIRSSpectrograph` GEMINI-N GNIRS `Link `__ Echelle True False -gtc_maat :class:`~pypeit.spectrographs.gtc_osiris.GTCMAATSpectrograph` GTC OSIRIS `Link `__ IFU True False See :doc:`gtc_osiris` -gtc_osiris :class:`~pypeit.spectrographs.gtc_osiris.GTCOSIRISSpectrograph` GTC OSIRIS `Link `__ MultiSlit True False See :doc:`gtc_osiris` -gtc_osiris_plus :class:`~pypeit.spectrographs.gtc_osiris.GTCOSIRISPlusSpectrograph` GTC OSIRIS `Link `__ MultiSlit True False See :doc:`gtc_osiris` -jwst_nircam :class:`~pypeit.spectrographs.jwst_nircam.JWSTNIRCamSpectrograph` JWST NIRCAM `Link `__ MultiSlit False False -jwst_nirspec :class:`~pypeit.spectrographs.jwst_nirspec.JWSTNIRSpecSpectrograph` JWST NIRSPEC `Link `__ MultiSlit True False -keck_deimos :class:`~pypeit.spectrographs.keck_deimos.KeckDEIMOSSpectrograph` KECK DEIMOS `Link `__ MultiSlit True True Supported gratings: 600ZD, 830G, 900ZD, 1200B, 1200G; see :doc:`deimos` -keck_hires :class:`~pypeit.spectrographs.keck_hires.KECKHIRESSpectrograph` KECK HIRES `Link `__ Echelle False False -keck_kcwi :class:`~pypeit.spectrographs.keck_kcwi.KeckKCWISpectrograph` KECK KCWI `Link `__ IFU True False Supported setups: BM, BH2; see :doc:`keck_kcwi` -keck_lris_blue :class:`~pypeit.spectrographs.keck_lris.KeckLRISBSpectrograph` KECK LRISb `Link `__ MultiSlit True False Blue camera; see :doc:`lris` -keck_lris_blue_orig :class:`~pypeit.spectrographs.keck_lris.KeckLRISBOrigSpectrograph` KECK LRISb `Link `__ MultiSlit True False Original detector; replaced in 20??; see :doc:`lris` -keck_lris_red :class:`~pypeit.spectrographs.keck_lris.KeckLRISRSpectrograph` KECK LRISr `Link `__ MultiSlit True True Red camera; LBNL detector, 2kx4k; see :doc:`lris` -keck_lris_red_mark4 :class:`~pypeit.spectrographs.keck_lris.KeckLRISRMark4Spectrograph` KECK LRISr `Link `__ MultiSlit True True New Mark4 detector, circa Spring 2021; Supported setups = R400 -keck_lris_red_orig :class:`~pypeit.spectrographs.keck_lris.KeckLRISROrigSpectrograph` KECK LRISr `Link `__ MultiSlit True True Original detector; replaced in 2009; see :doc:`lris` -keck_mosfire :class:`~pypeit.spectrographs.keck_mosfire.KeckMOSFIRESpectrograph` KECK MOSFIRE `Link `__ MultiSlit True False Gratings tested: Y, J, J2, H, K; see :doc:`mosfire` -keck_nires :class:`~pypeit.spectrographs.keck_nires.KeckNIRESSpectrograph` KECK NIRES `Link `__ Echelle True False see :doc:`keck_nires` -keck_nirspec_low :class:`~pypeit.spectrographs.keck_nirspec.KeckNIRSPECLowSpectrograph` KECK NIRSPEC `Link `__ MultiSlit True False Low-dispersion grating -lbt_luci1 :class:`~pypeit.spectrographs.lbt_luci.LBTLUCI1Spectrograph` LBT LUCI1 `Link `__ MultiSlit True False -lbt_luci2 :class:`~pypeit.spectrographs.lbt_luci.LBTLUCI2Spectrograph` LBT LUCI2 `Link `__ MultiSlit True False -lbt_mods1b :class:`~pypeit.spectrographs.lbt_mods.LBTMODS1BSpectrograph` LBT MODS1B `Link `__ MultiSlit True False MODS-I blue spectrometer -lbt_mods1r :class:`~pypeit.spectrographs.lbt_mods.LBTMODS1RSpectrograph` LBT MODS1R `Link `__ MultiSlit True False MODS-I red spectrometer -lbt_mods2b :class:`~pypeit.spectrographs.lbt_mods.LBTMODS2BSpectrograph` LBT MODS2B `Link `__ MultiSlit True False MODS-II blue spectrometer -lbt_mods2r :class:`~pypeit.spectrographs.lbt_mods.LBTMODS2RSpectrograph` LBT MODS2R `Link `__ MultiSlit True False MODS-II red spectrometer -ldt_deveny :class:`~pypeit.spectrographs.ldt_deveny.LDTDeVenySpectrograph` LDT DeVeny `Link `__ MultiSlit True False LDT DeVeny Optical Spectrograph, 2015 - present -magellan_fire :class:`~pypeit.spectrographs.magellan_fire.MagellanFIREEchelleSpectrograph` MAGELLAN FIRE `Link `__ Echelle True False Magellan/FIRE in echelle mode -magellan_fire_long :class:`~pypeit.spectrographs.magellan_fire.MagellanFIRELONGSpectrograph` MAGELLAN FIRE `Link `__ MultiSlit True False Magellan/FIRE in long-slit/high-throughput mode -magellan_mage :class:`~pypeit.spectrographs.magellan_mage.MagellanMAGESpectrograph` MAGELLAN MagE `Link `__ Echelle True False See :doc:`mage` -mdm_osmos_mdm4k :class:`~pypeit.spectrographs.mdm_osmos.MDMOSMOSMDM4KSpectrograph` KPNO MDM4K `Link `__ MultiSlit True False MDM OSMOS spectrometer -mmt_binospec :class:`~pypeit.spectrographs.mmt_binospec.MMTBINOSPECSpectrograph` MMT BINOSPEC `Link `__ MultiSlit True False -mmt_bluechannel :class:`~pypeit.spectrographs.mmt_bluechannel.MMTBlueChannelSpectrograph` MMT Blue_Channel `Link `__ MultiSlit True False -mmt_mmirs :class:`~pypeit.spectrographs.mmt_mmirs.MMTMMIRSSpectrograph` MMT MMIRS `Link `__ MultiSlit True False -not_alfosc :class:`~pypeit.spectrographs.not_alfosc.NOTALFOSCSpectrograph` NOT ALFOSC `Link `__ MultiSlit True False For use with the standard horizontal slits only. Grisms 3, 4, 5, 7, 8, 10, 11, 17, 18, 19, 20 -not_alfosc_vert :class:`~pypeit.spectrographs.not_alfosc.NOTALFOSCSpectrographVert` NOT ALFOSC `Link `__ MultiSlit True False Grisms 3, 4, 5, 7, 8, 10, 11, 17, 18, 19, 20. For vertical slits only -ntt_efosc2 :class:`~pypeit.spectrographs.ntt_efosc2.NTTEFOSC2Spectrograph` NTT EFOSC2 `Link `__ MultiSlit True False The ESO Faint Object Spectrograph and Camera version 2 -p200_dbsp_blue :class:`~pypeit.spectrographs.p200_dbsp.P200DBSPBlueSpectrograph` P200 DBSPb `Link `__ MultiSlit True False Blue camera -p200_dbsp_red :class:`~pypeit.spectrographs.p200_dbsp.P200DBSPRedSpectrograph` P200 DBSPr `Link `__ MultiSlit True False Red camera -p200_tspec :class:`~pypeit.spectrographs.p200_tspec.P200TSPECSpectrograph` P200 TSPEC `Link `__ Echelle True False TripleSpec spectrograph -shane_kast_blue :class:`~pypeit.spectrographs.shane_kast.ShaneKastBlueSpectrograph` SHANE KASTb `Link `__ MultiSlit True True -shane_kast_red :class:`~pypeit.spectrographs.shane_kast.ShaneKastRedSpectrograph` SHANE KASTr `Link `__ MultiSlit True True -shane_kast_red_ret :class:`~pypeit.spectrographs.shane_kast.ShaneKastRedRetSpectrograph` SHANE KASTr `Link `__ MultiSlit True True Red reticon -soar_goodman_blue :class:`~pypeit.spectrographs.soar_goodman.SOARGoodmanBlueSpectrograph` SOAR blue `Link `__ MultiSlit True False Supported gratings: 400_SYZY at M1 tilt -soar_goodman_red :class:`~pypeit.spectrographs.soar_goodman.SOARGoodmanRedSpectrograph` SOAR red `Link `__ MultiSlit True False Supported gratings: 400_SYZY at M1 and M2 tilts -tng_dolores :class:`~pypeit.spectrographs.tng_dolores.TNGDoloresSpectrograph` TNG DOLORES `Link `__ MultiSlit False False DOLORES (LRS) spectrograph; LR-R -vlt_fors2 :class:`~pypeit.spectrographs.vlt_fors.VLTFORS2Spectrograph` VLT FORS2 `Link `__ MultiSlit True False 300I, 300V gratings -vlt_sinfoni :class:`~pypeit.spectrographs.vlt_sinfoni.VLTSINFONISpectrograph` VLT SINFONI `Link `__ MultiSlit True False Gratings tested: K -vlt_xshooter_nir :class:`~pypeit.spectrographs.vlt_xshooter.VLTXShooterNIRSpectrograph` VLT XShooter_NIR `Link `__ Echelle True False See :doc:`xshooter` -vlt_xshooter_uvb :class:`~pypeit.spectrographs.vlt_xshooter.VLTXShooterUVBSpectrograph` VLT XShooter_UVB `Link `__ Echelle True False See :doc:`xshooter` -vlt_xshooter_vis :class:`~pypeit.spectrographs.vlt_xshooter.VLTXShooterVISSpectrograph` VLT XShooter_VIS `Link `__ Echelle True False See :doc:`xshooter` -wht_isis_blue :class:`~pypeit.spectrographs.wht_isis.WHTISISBlueSpectrograph` WHT ISISb `Link `__ MultiSlit False False Blue camera -wht_isis_red :class:`~pypeit.spectrographs.wht_isis.WHTISISRedSpectrograph` WHT ISISr `Link `__ MultiSlit False False Red camera -======================== ============================================================================ ========= ============ =============================================================================================================================== ========= ========= ========= ============================================================================================= +======================== ============================================================================ ========= ============ =============================================================================================================================== ========= ========= ========= =============================================================================================== +``PypeIt`` Name ``PypeIt`` Class Telescope Camera URL Pipeline Supported QL Tested Comments +======================== ============================================================================ ========= ============ =============================================================================================================================== ========= ========= ========= =============================================================================================== +bok_bc :class:`~pypeit.spectrographs.bok_bc.BokBCSpectrograph` BOK BC `Link `__ MultiSlit True False Bok B&C spectrometer +gemini_flamingos1 :class:`~pypeit.spectrographs.gemini_flamingos.GeminiFLAMINGOS1Spectrograph` GEMINI-S FLAMINGOS `Link `__ MultiSlit False False +gemini_flamingos2 :class:`~pypeit.spectrographs.gemini_flamingos.GeminiFLAMINGOS2Spectrograph` GEMINI-S FLAMINGOS `Link `__ MultiSlit True False Flamingos-2 NIR spectrograph +gemini_gmos_north_e2v :class:`~pypeit.spectrographs.gemini_gmos.GeminiGMOSNE2VSpectrograph` GEMINI-N GMOS-N `Link `__ MultiSlit True False E2V detector; see :doc:`gemini_gmos` +gemini_gmos_north_ham :class:`~pypeit.spectrographs.gemini_gmos.GeminiGMOSNHamSpectrograph` GEMINI-N GMOS-N `Link `__ MultiSlit True False Hamamatsu detector (R400, B600, R831); Used since Feb 2017; see :doc:`gemini_gmos` +gemini_gmos_north_ham_ns :class:`~pypeit.spectrographs.gemini_gmos.GeminiGMOSNHamNSSpectrograph` GEMINI-N GMOS-N `Link `__ MultiSlit True False Same as gemini_gmos_north_ham when used in nod-and-shuffle mode; see :doc:`gemini_gmos` +gemini_gmos_south_ham :class:`~pypeit.spectrographs.gemini_gmos.GeminiGMOSSHamSpectrograph` GEMINI-S GMOS-S `Link `__ MultiSlit True False Hamamatsu detector (R400, B600, R831); see :doc:`gemini_gmos` +gemini_gnirs_echelle :class:`~pypeit.spectrographs.gemini_gnirs.GeminiGNIRSEchelleSpectrograph` GEMINI-N GNIRS `Link `__ Echelle False False +gemini_gnirs_ifu :class:`~pypeit.spectrographs.gemini_gnirs.GNIRSIFUSpectrograph` GEMINI-N GNIRS `Link `__ IFU False False +gtc_maat :class:`~pypeit.spectrographs.gtc_osiris.GTCMAATSpectrograph` GTC OSIRIS `Link `__ IFU True False See :doc:`gtc_osiris` +gtc_osiris :class:`~pypeit.spectrographs.gtc_osiris.GTCOSIRISSpectrograph` GTC OSIRIS `Link `__ MultiSlit True False See :doc:`gtc_osiris` +gtc_osiris_plus :class:`~pypeit.spectrographs.gtc_osiris.GTCOSIRISPlusSpectrograph` GTC OSIRIS `Link `__ MultiSlit True False See :doc:`gtc_osiris` +jwst_nircam :class:`~pypeit.spectrographs.jwst_nircam.JWSTNIRCamSpectrograph` JWST NIRCAM `Link `__ MultiSlit False False +jwst_nirspec :class:`~pypeit.spectrographs.jwst_nirspec.JWSTNIRSpecSpectrograph` JWST NIRSPEC `Link `__ MultiSlit True False +keck_deimos :class:`~pypeit.spectrographs.keck_deimos.KeckDEIMOSSpectrograph` KECK DEIMOS `Link `__ MultiSlit True True Supported gratings: 600ZD, 830G, 900ZD, 1200B, 1200G; see :doc:`deimos` +keck_esi :class:`~pypeit.spectrographs.keck_esi.KeckESISpectrograph` KECK ESI Echelle True False +keck_hires :class:`~pypeit.spectrographs.keck_hires.KECKHIRESSpectrograph` KECK HIRES `Link `__ Echelle False False +keck_kcrm :class:`~pypeit.spectrographs.keck_kcwi.KeckKCRMSpectrograph` KECK KCRM `Link `__ IFU True False Supported setups: RM1, RM2, RH3; see :doc:`keck_kcwi` +keck_kcwi :class:`~pypeit.spectrographs.keck_kcwi.KeckKCWISpectrograph` KECK KCWI `Link `__ IFU True False Supported setups: BL, BM, BH2; see :doc:`keck_kcwi` +keck_lris_blue :class:`~pypeit.spectrographs.keck_lris.KeckLRISBSpectrograph` KECK LRISb `Link `__ MultiSlit True False Blue camera; Current FITS file format; used from May 2009, see :doc:`lris` +keck_lris_blue_orig :class:`~pypeit.spectrographs.keck_lris.KeckLRISBOrigSpectrograph` KECK LRISb `Link `__ MultiSlit True False Blue camera; Original FITS file format; used until April 2009; see :doc:`lris` +keck_lris_red :class:`~pypeit.spectrographs.keck_lris.KeckLRISRSpectrograph` KECK LRISr `Link `__ MultiSlit True True Red camera; Current FITS file format; LBNL detector, 2kx4k; used from May 2009, see :doc:`lris` +keck_lris_red_mark4 :class:`~pypeit.spectrographs.keck_lris.KeckLRISRMark4Spectrograph` KECK LRISr `Link `__ MultiSlit True True Red camera; New Mark4 detector, in operation since May 2021; see :doc:`lris` +keck_lris_red_orig :class:`~pypeit.spectrographs.keck_lris.KeckLRISROrigSpectrograph` KECK LRISr `Link `__ MultiSlit True True Red camera; Original FITS file format; used until April 2009; see :doc:`lris` +keck_mosfire :class:`~pypeit.spectrographs.keck_mosfire.KeckMOSFIRESpectrograph` KECK MOSFIRE `Link `__ MultiSlit True False Gratings tested: Y, J, J2, H, K; see :doc:`mosfire` +keck_nires :class:`~pypeit.spectrographs.keck_nires.KeckNIRESSpectrograph` KECK NIRES `Link `__ Echelle True False see :doc:`keck_nires` +keck_nirspec_low :class:`~pypeit.spectrographs.keck_nirspec.KeckNIRSPECLowSpectrograph` KECK NIRSPEC `Link `__ MultiSlit True False Low-dispersion grating +lbt_luci1 :class:`~pypeit.spectrographs.lbt_luci.LBTLUCI1Spectrograph` LBT LUCI1 `Link `__ MultiSlit True False +lbt_luci2 :class:`~pypeit.spectrographs.lbt_luci.LBTLUCI2Spectrograph` LBT LUCI2 `Link `__ MultiSlit True False +lbt_mods1b :class:`~pypeit.spectrographs.lbt_mods.LBTMODS1BSpectrograph` LBT MODS1B `Link `__ MultiSlit True False MODS-I blue spectrometer +lbt_mods1r :class:`~pypeit.spectrographs.lbt_mods.LBTMODS1RSpectrograph` LBT MODS1R `Link `__ MultiSlit True False MODS-I red spectrometer +lbt_mods2b :class:`~pypeit.spectrographs.lbt_mods.LBTMODS2BSpectrograph` LBT MODS2B `Link `__ MultiSlit True False MODS-II blue spectrometer +lbt_mods2r :class:`~pypeit.spectrographs.lbt_mods.LBTMODS2RSpectrograph` LBT MODS2R `Link `__ MultiSlit True False MODS-II red spectrometer +ldt_deveny :class:`~pypeit.spectrographs.ldt_deveny.LDTDeVenySpectrograph` LDT DeVeny `Link `__ MultiSlit True False LDT DeVeny Optical Spectrograph, 2015 - present +magellan_fire :class:`~pypeit.spectrographs.magellan_fire.MagellanFIREEchelleSpectrograph` MAGELLAN FIRE `Link `__ Echelle True False Magellan/FIRE in echelle mode +magellan_fire_long :class:`~pypeit.spectrographs.magellan_fire.MagellanFIRELONGSpectrograph` MAGELLAN FIRE `Link `__ MultiSlit True False Magellan/FIRE in long-slit/high-throughput mode +magellan_mage :class:`~pypeit.spectrographs.magellan_mage.MagellanMAGESpectrograph` MAGELLAN MagE `Link `__ Echelle True False See :doc:`mage` +mdm_modspec :class:`~pypeit.spectrographs.mdm_modspec.MDMModspecEchelleSpectrograph` HILTNER Echelle MultiSlit True False MDM Modspec spectrometer; Only 1200l/mm disperser (so far) +mdm_osmos_mdm4k :class:`~pypeit.spectrographs.mdm_osmos.MDMOSMOSMDM4KSpectrograph` HILTNER MDM4K `Link `__ MultiSlit True False MDM OSMOS spectrometer +mmt_binospec :class:`~pypeit.spectrographs.mmt_binospec.MMTBINOSPECSpectrograph` MMT BINOSPEC `Link `__ MultiSlit True False +mmt_bluechannel :class:`~pypeit.spectrographs.mmt_bluechannel.MMTBlueChannelSpectrograph` MMT Blue_Channel `Link `__ MultiSlit True False +mmt_mmirs :class:`~pypeit.spectrographs.mmt_mmirs.MMTMMIRSSpectrograph` MMT MMIRS `Link `__ MultiSlit True False +not_alfosc :class:`~pypeit.spectrographs.not_alfosc.NOTALFOSCSpectrograph` NOT ALFOSC `Link `__ MultiSlit True False For use with the standard horizontal slits only. Grisms 3, 4, 5, 7, 8, 10, 11, 17, 18, 19, 20 +not_alfosc_vert :class:`~pypeit.spectrographs.not_alfosc.NOTALFOSCSpectrographVert` NOT ALFOSC `Link `__ MultiSlit True False Grisms 3, 4, 5, 7, 8, 10, 11, 17, 18, 19, 20. For vertical slits only +ntt_efosc2 :class:`~pypeit.spectrographs.ntt_efosc2.NTTEFOSC2Spectrograph` NTT EFOSC2 `Link `__ MultiSlit True False The ESO Faint Object Spectrograph and Camera version 2 +p200_dbsp_blue :class:`~pypeit.spectrographs.p200_dbsp.P200DBSPBlueSpectrograph` P200 DBSPb `Link `__ MultiSlit True False Blue camera +p200_dbsp_red :class:`~pypeit.spectrographs.p200_dbsp.P200DBSPRedSpectrograph` P200 DBSPr `Link `__ MultiSlit True False Red camera +p200_tspec :class:`~pypeit.spectrographs.p200_tspec.P200TSPECSpectrograph` P200 TSPEC `Link `__ Echelle True False TripleSpec spectrograph +shane_kast_blue :class:`~pypeit.spectrographs.shane_kast.ShaneKastBlueSpectrograph` SHANE KASTb `Link `__ MultiSlit True True +shane_kast_red :class:`~pypeit.spectrographs.shane_kast.ShaneKastRedSpectrograph` SHANE KASTr `Link `__ MultiSlit True True +shane_kast_red_ret :class:`~pypeit.spectrographs.shane_kast.ShaneKastRedRetSpectrograph` SHANE KASTr `Link `__ MultiSlit True True Red reticon +soar_goodman_blue :class:`~pypeit.spectrographs.soar_goodman.SOARGoodmanBlueSpectrograph` SOAR blue `Link `__ MultiSlit True False Supported gratings: 400_SYZY at M1 tilt +soar_goodman_red :class:`~pypeit.spectrographs.soar_goodman.SOARGoodmanRedSpectrograph` SOAR red `Link `__ MultiSlit True False Supported gratings: 400_SYZY at M1 and M2 tilts +tng_dolores :class:`~pypeit.spectrographs.tng_dolores.TNGDoloresSpectrograph` TNG DOLORES `Link `__ MultiSlit False False DOLORES (LRS) spectrograph; LR-R +vlt_fors2 :class:`~pypeit.spectrographs.vlt_fors.VLTFORS2Spectrograph` VLT FORS2 `Link `__ MultiSlit True False 300I, 300V gratings +vlt_sinfoni :class:`~pypeit.spectrographs.vlt_sinfoni.VLTSINFONISpectrograph` VLT SINFONI `Link `__ MultiSlit True False Gratings tested: K +vlt_xshooter_nir :class:`~pypeit.spectrographs.vlt_xshooter.VLTXShooterNIRSpectrograph` VLT XShooter_NIR `Link `__ Echelle True False See :doc:`xshooter` +vlt_xshooter_uvb :class:`~pypeit.spectrographs.vlt_xshooter.VLTXShooterUVBSpectrograph` VLT XShooter_UVB `Link `__ Echelle True False See :doc:`xshooter` +vlt_xshooter_vis :class:`~pypeit.spectrographs.vlt_xshooter.VLTXShooterVISSpectrograph` VLT XShooter_VIS `Link `__ Echelle True False See :doc:`xshooter` +wht_isis_blue :class:`~pypeit.spectrographs.wht_isis.WHTISISBlueSpectrograph` WHT ISISb `Link `__ MultiSlit False False Blue camera +wht_isis_red :class:`~pypeit.spectrographs.wht_isis.WHTISISRedSpectrograph` WHT ISISr `Link `__ MultiSlit False False Red camera +======================== ============================================================================ ========= ============ =============================================================================================================================== ========= ========= ========= =============================================================================================== diff --git a/doc/installing.rst b/doc/installing.rst index 559cf1a9fa..30a648f98c 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -38,7 +38,7 @@ Setup a clean python environment Both methods discussed below for installing PypeIt (via `pip`_ or `conda`_) also install or upgrade its :ref:`dependencies`. For this reason, we highly -(!!) recommended you first set up a clean python environment in which to install +(!!) recommend you first set up a clean python environment in which to install PypeIt. This mitigates any possible dependency conflicts with other packages you use. @@ -130,8 +130,8 @@ and then reinstalling. version after upgrading to a new version. **The best approach is to always re-reduce data you're still working with anytime you update PypeIt.** -Install via ``conda`` (recommended overall) -------------------------------------------- +Install via ``conda`` +--------------------- `conda`_ is a popular and widely-used package and environment manager. We provide a yaml file that can be used to setup a conda environment called diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index b1607074e3..c0451af10d 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -54,7 +54,7 @@ the precedence order is as follows: configurations via its ``config_specific_par`` method. This allows the code to automatically define, e.g., the archived arc spectrum used for wavelength calibration given the grating used. For example, see - :func:`~pypeit.spectrographs.shane_kast.ShaneKastSpectrograph.config_specific_par` + :func:`~pypeit.spectrographs.shane_kast.ShaneKastBlueSpectrograph.config_specific_par` for Shane/Kast. These configuration-specific parameters are currently not documented here; however, they can be viewed by looking at the source code display in the API documentation. @@ -236,30 +236,30 @@ CalibrationsPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.CalibrationsPar` -===================== ==================================================== ======= ================================= =========================================================================================================================================================================================================================== -Key Type Options Default Description -===================== ==================================================== ======= ================================= =========================================================================================================================================================================================================================== -``alignframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the align frames -``alignment`` :class:`~pypeit.par.pypeitpar.AlignPar` .. `AlignPar Keywords`_ Define the procedure for the alignment of traces -``arcframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the wavelength calibration -``biasframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the bias correction -``bpm_usebias`` bool .. False Make a bad pixel mask from bias frames? Bias frames must be provided. -``calib_dir`` str .. ``Calibrations`` The name of the directory for the processed calibration frames. The host path for the directory is set by the redux_path (see :class:`ReduxPar`). Beware that success when changing the default value is not well tested! -``darkframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the dark-current correction -``flatfield`` :class:`~pypeit.par.pypeitpar.FlatFieldPar` .. `FlatFieldPar Keywords`_ Parameters used to set the flat-field procedure -``illumflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the illumination flat -``lampoffflatsframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the lamp off flats -``pinholeframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the pinholes -``pixelflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the pixel flat -``raise_chk_error`` bool .. True Raise an error if the calibration check fails -``skyframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the sky background observations -``slitedges`` :class:`~pypeit.par.pypeitpar.EdgeTracePar` .. `EdgeTracePar Keywords`_ Slit-edge tracing parameters -``standardframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the spectrophotometric standard observations -``tiltframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the wavelength tilts -``tilts`` :class:`~pypeit.par.pypeitpar.WaveTiltsPar` .. `WaveTiltsPar Keywords`_ Define how to trace the slit tilts using the trace frames -``traceframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for images used for slit tracing -``wavelengths`` :class:`~pypeit.par.pypeitpar.WavelengthSolutionPar` .. `WavelengthSolutionPar Keywords`_ Parameters used to derive the wavelength solution -===================== ==================================================== ======= ================================= =========================================================================================================================================================================================================================== +===================== ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= +Key Type Options Default Description +===================== ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= +``alignframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the align frames +``alignment`` :class:`~pypeit.par.pypeitpar.AlignPar` .. `AlignPar Keywords`_ Define the procedure for the alignment of traces +``arcframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the wavelength calibration +``biasframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the bias correction +``bpm_usebias`` bool .. False Make a bad pixel mask from bias frames? Bias frames must be provided. +``calib_dir`` str .. ``Calibrations`` The name of the directory for the processed calibration frames. The host path for the directory is set by the redux_path (see :class:`~pypeit.par.pypeitpar.ReduxPar`). Beware that success when changing the default value is not well tested! +``darkframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the dark-current correction +``flatfield`` :class:`~pypeit.par.pypeitpar.FlatFieldPar` .. `FlatFieldPar Keywords`_ Parameters used to set the flat-field procedure +``illumflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the illumination flat +``lampoffflatsframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the lamp off flats +``pinholeframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the pinholes +``pixelflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the pixel flat +``raise_chk_error`` bool .. True Raise an error if the calibration check fails +``skyframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the sky background observations +``slitedges`` :class:`~pypeit.par.pypeitpar.EdgeTracePar` .. `EdgeTracePar Keywords`_ Slit-edge tracing parameters +``standardframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the spectrophotometric standard observations +``tiltframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the wavelength tilts +``tilts`` :class:`~pypeit.par.pypeitpar.WaveTiltsPar` .. `WaveTiltsPar Keywords`_ Define how to trace the slit tilts using the trace frames +``traceframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for images used for slit tracing +``wavelengths`` :class:`~pypeit.par.pypeitpar.WavelengthSolutionPar` .. `WavelengthSolutionPar Keywords`_ Parameters used to derive the wavelength solution +===================== ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= ---- @@ -271,14 +271,14 @@ AlignPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.AlignPar` -=============== ============= ======= ============= ================================================================================================================================================================================================================================================================================ -Key Type Options Default Description -=============== ============= ======= ============= ================================================================================================================================================================================================================================================================================ -``locations`` list, ndarray .. 0.0, 0.5, 1.0 Locations of the bars, in a list, specified as a fraction of the slit width -``snr_thresh`` int, float .. 1.0 S/N ratio threshold for finding an alignment trace. This should be a low number to ensure that the algorithm finds all bars. The algorithm will then only use the N most significant detections, where N is the number of elements specified in the "locations" keyword argument -``trace_npoly`` int .. 4 Order of the polynomial to use when fitting the trace of a single bar -``trim_edge`` list .. 0, 0 Trim the slit by this number of pixels left/right before finding alignment bars -=============== ============= ======= ============= ================================================================================================================================================================================================================================================================================ +=============== ============= ======= ======== ================================================================================================================================================================================================================================================================================ +Key Type Options Default Description +=============== ============= ======= ======== ================================================================================================================================================================================================================================================================================ +``locations`` list, ndarray .. 0.0, 1.0 Locations of the bars, in a list, specified as a fraction of the slit width +``snr_thresh`` int, float .. 1.0 S/N ratio threshold for finding an alignment trace. This should be a low number to ensure that the algorithm finds all bars. The algorithm will then only use the N most significant detections, where N is the number of elements specified in the "locations" keyword argument +``trace_npoly`` int .. 4 Order of the polynomial to use when fitting the trace of a single bar +``trim_edge`` list .. 0, 0 Trim the slit by this number of pixels left/right before finding alignment bars +=============== ============= ======= ======== ================================================================================================================================================================================================================================================================================ ---- @@ -330,6 +330,7 @@ Class Instantiation: :class:`~pypeit.par.pypeitpar.EdgeTracePar` =========================== ================ =========================================== ============== ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== Key Type Options Default Description =========================== ================ =========================================== ============== ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== +``add_missed_orders`` bool .. False If orders are not detected by the automated edge tracing, attempt to add them based on their expected positions on on the detector. Echelle spectrographs only. ``add_predict`` str .. ``nearest`` Sets the method used to predict the shape of the left and right traces for a user-defined slit inserted. Options are (1) ``straight`` inserts traces with a constant spatial pixels position, (2) ``nearest`` inserts traces with a form identical to the automatically identified trace at the nearest spatial position to the inserted slit, or (3) ``pca`` uses the PCA decomposition to predict the shape of the traces. ``add_slits`` str, list .. .. Add one or more user-defined slits. The syntax to define a slit to add is: 'det:spec:spat_left:spat_right' where det=detector, spec=spectral pixel, spat_left=spatial pixel of left slit boundary, and spat_righ=spatial pixel of right slit boundary. For example, '2:2000:2121:2322,3:2000:1201:1500' will add a slit to detector 2 passing through spec=2000 extending spatially from 2121 to 2322 and another on detector 3 at spec=2000 extending from 1201 to 1500. ``auto_pca`` bool .. True During automated tracing, attempt to construct a PCA decomposition of the traces. When True, the edge traces resulting from the initial detection, centroid refinement, and polynomial fitting must meet a set of criteria for performing the pca; see :func:`pypeit.edgetrace.EdgeTraceSet.can_pca`. If False, the ``sync_predict`` parameter *cannot* be set to ``pca``; if it is not, the value is set to ``nearest`` and a warning is issued when validating the parameter set. @@ -346,11 +347,11 @@ Key Type Options ``fit_maxdev`` int, float .. 5.0 Maximum deviation between the fitted and measured edge position for rejection in spatial pixels. ``fit_maxiter`` int .. 25 Maximum number of rejection iterations during edge fitting. ``fit_min_spec_length`` float .. 0.6 Minimum unmasked spectral length of a traced slit edge to use in any modeling procedure (polynomial fitting or PCA decomposition). -``fit_niter`` int .. 1 Number of iterations of re-measuring and re-fitting the edge data; see :func:`pypeit.core.trace.fit_trace`. +``fit_niter`` int .. 1 Number of iterations of re-measuring and re-fitting the edge data; see :func:`~pypeit.core.trace.fit_trace`. ``fit_order`` int .. 5 Order of the function fit to edge measurements. ``follow_span`` int .. 20 In the initial connection of spectrally adjacent edge detections, this sets the number of previous spectral rows to consider when following slits forward. -``fwhm_gaussian`` int, float .. 3.0 The `fwhm` parameter to use when using Gaussian weighting in :func:`pypeit.core.trace.fit_trace` when refining the PCA predictions of edges. See description :func:`pypeit.core.trace.peak_trace`. -``fwhm_uniform`` int, float .. 3.0 The `fwhm` parameter to use when using uniform weighting in :func:`pypeit.core.trace.fit_trace` when refining the PCA predictions of edges. See description of :func:`pypeit.core.trace.peak_trace`. +``fwhm_gaussian`` int, float .. 3.0 The `fwhm` parameter to use when using Gaussian weighting in :func:`~pypeit.core.trace.fit_trace` when refining the PCA predictions of edges. See description :func:`~pypeit.core.trace.peak_trace`. +``fwhm_uniform`` int, float .. 3.0 The `fwhm` parameter to use when using uniform weighting in :func:`~pypeit.core.trace.fit_trace` when refining the PCA predictions of edges. See description of :func:`~pypeit.core.trace.peak_trace`. ``gap_offset`` int, float .. 5.0 Offset (pixels) used for the slit edge gap width when inserting slit edges (see `sync_center`) or when nudging predicted slit edges to avoid slit overlaps. This should be larger than `minimum_slit_gap` when converted to arcseconds. ``left_right_pca`` bool .. False Construct a PCA decomposition for the left and right traces separately. This can be important for cross-dispersed echelle spectrographs (e.g., Keck-NIRES) ``length_range`` int, float .. .. Allowed range in slit length compared to the median slit length. For example, a value of 0.3 means that slit lengths should not vary more than 30%. Relatively shorter or longer slits are masked or clipped. Most useful for echelle or multi-slit data where the slits should have similar or identical lengths. @@ -367,8 +368,8 @@ Key Type Options ``minimum_slit_gap`` int, float .. .. Minimum slit gap in arcsec. Gaps between slits are determined by the median difference between the right and left edge locations of adjacent slits. Slits with small gaps are merged by removing the intervening traces.If None, no minimum slit gap is applied. This should be smaller than `gap_offset` when converted to pixels. ``minimum_slit_length`` int, float .. .. Minimum slit length in arcsec. Slit lengths are determined by the median difference between the left and right edge locations for the unmasked trace locations. This is used to identify traces that are *erroneously* matched together to form slits. Short slits are expected to be ignored or removed (see ``clip``). If None, no minimum slit length applied. ``minimum_slit_length_sci`` int, float .. .. Minimum slit length in arcsec for a science slit. Slit lengths are determined by the median difference between the left and right edge locations for the unmasked trace locations. Used in combination with ``minimum_slit_length``, this parameter is used to identify box or alignment slits; i.e., those slits that are shorter than ``minimum_slit_length_sci`` but larger than ``minimum_slit_length`` are box/alignment slits. Box slits are *never* removed (see ``clip``), but no spectra are extracted from them. If None, no minimum science slit length is applied. -``niter_gaussian`` int .. 6 The number of iterations of :func:`pypeit.core.trace.fit_trace` to use when using Gaussian weighting. -``niter_uniform`` int .. 9 The number of iterations of :func:`pypeit.core.trace.fit_trace` to use when using uniform weighting. +``niter_gaussian`` int .. 6 The number of iterations of :func:`~pypeit.core.trace.fit_trace` to use when using Gaussian weighting. +``niter_uniform`` int .. 9 The number of iterations of :func:`~pypeit.core.trace.fit_trace` to use when using uniform weighting. ``order_match`` int, float .. .. For echelle spectrographs, this is the tolerance allowed for matching identified "slits" to echelle orders. Must be in the fraction of the detector spatial scale (i.e., a value of 0.05 means that the order locations must be within 5% of the expected value). If None, no limit is used. ``order_offset`` int, float .. .. Offset to introduce to the expected order positions to improve the match for this specific data. This is an additive offset to the measured slit positions; i.e., this should minimize the difference between the expected order positions and ``self.slit_spatial_center() + offset``. Must be in the fraction of the detector spatial scale. If None, no offset is applied. ``overlap`` bool .. False Assume slits identified as abnormally short are actually due to overlaps between adjacent slits/orders. If set to True, you *must* have also used ``length_range`` to identify left-right edge pairs that have an abnormally short separation. For those short slits, the code attempts to convert the short slits into slit gaps. This is particularly useful for blue orders in Keck-HIRES data. @@ -417,7 +418,7 @@ Key Type Options Default Descrip ``sig_neigh`` int, float .. 10.0 Significance threshold for arcs to be used in line identification for the purpose of identifying neighboring lines. The tracethresh parameter above determines the significance threshold of lines that will be traced, but these lines must be at least nfwhm_neigh fwhm away from neighboring lines. This parameter determines the significance above which a line must be to be considered a possible colliding neighbor. A low value of sig_neigh will result in an overall larger number of lines, which will result in more lines above tracethresh getting rejected ``sigrej2d`` int, float .. 3.0 Outlier rejection significance determining which pixels on a fit to an arc line tilt are rejected by the global 2D fit ``sigrej_trace`` int, float .. 3.0 Outlier rejection significance to determine which traced arc lines should be included in the global fit -``spat_order`` int, float, list, ndarray .. 3 Order of the legendre polynomial to be fit to the the tilt of an arc line. This parameter determines both the orer of the *individual* arc line tilts, as well as the order of the spatial direction of the 2d legendre polynomial (spatial, spectral) that is fit to obtain a global solution for the tilts across the slit/order. This can be a single number or a list/array providing the value for each slit +``spat_order`` int, float, list, ndarray .. 3 Order of the legendre polynomial to be fit to the tilt of an arc line. This parameter determines both the order of the *individual* arc line tilts, as well as the order of the spatial direction of the 2d legendre polynomial (spatial, spectral) that is fit to obtain a global solution for the tilts across the slit/order. This can be a single number or a list/array providing the value for each slit ``spec_order`` int, float, list, ndarray .. 4 Order of the spectral direction of the 2d legendre polynomial (spatial, spectral) that is fit to obtain a global solution for the tilts across the slit/order. This can be a single number or a list/array providing the value for each slit ``tracethresh`` int, float, list, ndarray .. 20.0 Significance threshold for arcs to be used in tracing wavelength tilts. This can be a single number or a list/array providing the value for each slit/order. =================== ========================= ======= ============== ============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= @@ -445,6 +446,8 @@ Key Type Options ``func`` str .. ``legendre`` Function used for wavelength solution fits ``fwhm`` int, float .. 4.0 Spectral sampling of the arc lines. This is the FWHM of an arcline in binned pixels of the input arc image ``fwhm_fromlines`` bool .. False Estimate spectral resolution in each slit using the arc lines. If True, the estimated FWHM will override ``fwhm`` only in the determination of the wavelength solution (`i.e.`, not in WaveTilts). +``fwhm_spat_order`` int .. 0 This parameter determines the spatial polynomial order to use in the 2D polynomial fit to the FWHM of the arc lines. See also, fwhm_spec_order. +``fwhm_spec_order`` int .. 1 This parameter determines the spectral polynomial order to use in the 2D polynomial fit to the FWHM of the arc lines. See also, fwhm_spat_order. ``lamps`` list .. .. Name of one or more ions used for the wavelength calibration. Use ``None`` for no calibration. Choose ``use_header`` to use the list of lamps recorded in the header of the arc frames (this is currently available only for Keck DEIMOS and LDT DeVeny). ``match_toler`` float .. 2.0 Matching tolerance in pixels when searching for new lines. This is the difference in pixels between the wavlength assigned to an arc line by an iteration of the wavelength solution to the wavelength in the line list. This parameter is also used as the matching tolerance in pixels for a line reidentification. A good line match must match within this tolerance to the shifted and stretched archive spectrum, and the archive wavelength solution at this match must be within match_toler dispersion elements from the line in line list. ``method`` str ``holy-grail``, ``identify``, ``reidentify``, ``echelle``, ``full_template`` ``holy-grail`` Method to use to fit the individual arc lines. Note that some of the available methods should not be used; they are unstable and require significant parameter tweaking to succeed. You should use one of 'holy-grail', 'reidentify', or 'full_template'. 'holy-grail' attempts to get a first guess at line IDs by looking for patterns in the line locations. It is fully automated. When it works, it works well; however, it can fail catastrophically. Instead, 'reidentify' and 'full_template' are the preferred methods. They require an archived wavelength solution for your specific instrument/grating combination as a reference. This is used to anchor the wavelength solution for the data being reduced. All options are: holy-grail, identify, reidentify, echelle, full_template. @@ -455,10 +458,12 @@ Key Type Options ``nreid_min`` int .. 1 Minimum number of times that a given candidate reidentified line must be properly matched with a line in the arxiv to be considered a good reidentification. If there is a lot of duplication in the arxiv of the spectra in question (i.e. multislit) set this to a number like 1-4. For echelle this depends on the number of solutions in the arxiv. Set this to 1 for fixed format echelle spectrographs. For an echelle with a tiltable grating, this will depend on the number of solutions in the arxiv. ``nsnippet`` int .. 2 Number of spectra to chop the arc spectrum into when ``method`` is 'full_template' ``numsearch`` int .. 20 Number of brightest arc lines to search for in preliminary identification +``qa_log`` bool .. True Governs whether the wavelength solution arc line QA plots will have log or linear scalingIf True, the scaling will be log, if False linear +``redo_slits`` int, list .. .. Redo the input slit(s) [multislit] or order(s) [echelle] ``reference`` str ``arc``, ``sky``, ``pixel`` ``arc`` Perform wavelength calibration with an arc, sky frame. Use 'pixel' for no wavelength solution. ``refframe`` str ``observed``, ``heliocentric``, ``barycentric`` ``heliocentric`` Frame of reference for the wavelength calibration. Options are: observed, heliocentric, barycentric ``reid_arxiv`` str .. .. Name of the archival wavelength solution file that will be used for the wavelength reidentification. Only used if ``method`` is 'reidentify' or 'full_template'. -``rms_threshold`` float, list, ndarray .. 0.15 Minimum RMS for keeping a slit/order solution. This can be a single number or a list/array providing the value for each slit. Only used if ``method`` is either 'holy-grail' or 'reidentify' +``rms_threshold`` float .. 0.15 Maximum RMS (in binned pixels) for keeping a slit/order solution. Used for echelle spectrographs, the 'reidentify' method, and when re-analyzing a slit with the redo_slits parameter.In a future PR, we will refactor the code to always scale this threshold off the measured FWHM of the arc lines. ``sigdetect`` int, float, list, ndarray .. 5.0 Sigma threshold above fluctuations for arc-line detection. Arcs are continuum subtracted and the fluctuations are computed after continuum subtraction. This can be a single number or a vector (list or numpy array) that provides the detection threshold for each slit. ``sigrej_final`` float .. 3.0 Number of sigma for rejection for the final guess to the wavelength solution. ``sigrej_first`` float .. 2.0 Number of sigma for rejection for the first guess to the wavelength solution. @@ -481,7 +486,9 @@ Key Type Options Default Description ==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= ``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that spec1d*.fits files were createdwith the current version of PypeIt ``coaddfile`` str .. .. Output filename +``dloglam`` int, float .. .. Dispersion in units of log10(wave) in case you want to specify it in the get_wave_grid (for the 'velocity' or 'log10' options), otherwise a median value is computed from the data. ``dv`` int, float .. .. Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), otherwise a median value is computed from the data. +``dwave`` int, float .. .. Dispersion in Angstroms in case you want to specify it in the get_wave_grid (for the 'linear' option), otherwise a median value is computed from the data. ``ex_value`` str .. ``OPT`` The extraction to coadd, i.e. optimal or boxcar. Must be either 'OPT' or 'BOX' ``filter`` str .. ``none`` Filter for scaling. See flux_calib.load_fitler_file() for naming. Ignore if none ``filter_mag`` float .. .. Magnitude of the source in the given filter @@ -492,11 +499,11 @@ Key Type Options Default Description ``maxiter_reject`` int .. 5 Maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen successive iterations or when maxiter_reject is reached. ``maxiter_scale`` int .. 5 Maximum number of iterations performed for rescaling spectra. ``maxrej`` int .. .. Coadding performs iterative rejection by comparing each exposure to a preliminary stack of all the exposures. If this parameter is set then it will not reject more than maxrej pixels per iteration of this rejection. The default is None, which means no maximum on rejected pixels. -``nbest`` int .. .. Number of orders to use for estimating the per exposure weights. Default is None, which will just use one fourth of the total number of orders. This is only used for Echelle. +``nbests`` list, int .. .. Number of orders to use for estimating the per exposure weights. Default is None, which will just use one fourth of the total number of orders. This is only used for Echelle ``nmaskedge`` int .. 2 Number of edge pixels to mask. This should be removed/fixed. ``ref_percentile`` int, float .. 70.0 Percentile used for selecting the minimum SNR cut from a reference spectrum used to robustly determine the median ratio between spectra. This parameter is used by coadd1d.robust_median_ratio as part of the automatic rescaling procedure. Pixels above this percentile cut are deemed the "good" pixels and are used to compute the ratio of two spectra. This must be a number between 0 and 100. ``scale_method`` str .. ``auto`` Method used to rescale the spectra prior to coadding. The options are: 'auto' -- Determine the scaling method automatically based on the S/N ratio which works well. 'poly' -- Polynomial rescaling. 'median' -- Median rescaling 'none' -- Do not rescale. 'hand' -- Pass in hand scaling factors. This option is not well tested. -``sensfuncfile`` str .. .. File containing sensitivity function which is a requirement for echelle coadds. This is only used for Echelle. +``sigrej_exp`` int, float .. .. Rejection threshold used for rejecting exposures with S/N more than sigrej_exp*sigma above the median S/N. If None (the default), no rejection is performed. Currently, only available for multi-slit observations. ``sigrej_scale`` int, float .. 3.0 Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. ``sn_clip`` int, float .. 30.0 Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectrum which neverthless differ at a level greater than the formal S/N due to systematics. ``sn_min_medscale`` int, float .. 0.5 For scale method set to ``auto``, this sets the minimum SNR for which median scaling is attempted. @@ -519,17 +526,19 @@ Coadd2DPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.Coadd2DPar` -==================== ========= ======= ======== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -==================== ========= ======= ======== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -``manual`` str .. .. Manual extraction parameters. det:spat:spec:fwhm:boxcar_radius. Multiple manual extractions are semi-colon separated, and spat,spec are in the pseudo-image generated by COADD2D.boxcar_radius is optional and in pixels (not arcsec!). -``offsets`` str, list .. ``auto`` Offsets for the images being combined (spat pixels). Options are: ``maskdef_offsets``, ``header``, ``auto``, and a list of offsets. Use ``maskdef_offsets`` to use the offsets computed during the slitmask design matching (currently available for DEIMOS and MOSFIRE only). If equal to ``header``, the dither offsets recorded in the header, when available, will be used. If ``auto`` is chosen, PypeIt will try to compute the offsets using a reference object with the highest S/N, or an object selected by the user (see ``user_obj``). If a list of offsets is provided, PypeIt will use it. -``only_slits`` int, list .. .. Slit ID, or list of slit IDs that the user want to restrict the coadd to. I.e., only this/these slit/s will be coadded. -``spat_toler`` int .. 5 This parameter provides the desired tolerance in spatial pixel used to identify slits in different exposures -``use_slits4wvgrid`` bool .. False If True, use the slits to set the trace down the center -``user_obj`` int, list .. .. Object that the user wants to use to compute the weights and/or the offsets for coadding images. For slit spectroscopy, provide the ``SLITID`` and the ``OBJID``, separated by comma, of the selected object. For echelle spectroscopy, provide the ``ECH_OBJID`` of the selected object. See :doc:`out_spec1D` for more info about ``SLITID``, ``OBJID`` and ``ECH_OBJID``. If this parameter is not ``None``, it will be used to compute the offsets only if ``offsets = auto``, and it will used to compute the weights only if ``weights = auto``. -``weights`` str, list .. ``auto`` Mode for the weights used to coadd images. Options are: ``auto``, ``uniform``, or a list of weights. If ``auto`` is used, PypeIt will try to compute the weights using a reference object with the highest S/N, or an object selected by the user (see ``user_obj``), if ``uniform`` is used, uniform weights will be applied. If a list of weights is provided, PypeIt will use it. -==================== ========= ======= ======== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== +==================== ========= ======= ======== ============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ +Key Type Options Default Description +==================== ========= ======= ======== ============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ +``exclude_slits`` str, list .. .. Exclude one or more slits from the coaddition. Example syntax -- DET01:175,DET02:205 or MSC02:2234. This and ``only_slits`` are mutually exclusive. If both are provided, ``only_slits`` takes precedence. +``manual`` str .. .. Manual extraction parameters. det:spat:spec:fwhm:boxcar_radius. Multiple manual extractions are semi-colon separated, and spat,spec are in the pseudo-image generated by COADD2D.boxcar_radius is optional and in pixels (not arcsec!). +``offsets`` str, list .. ``auto`` Offsets for the images being combined (spat pixels). Options are: ``maskdef_offsets``, ``header``, ``auto``, and a list of offsets. Use ``maskdef_offsets`` to use the offsets computed during the slitmask design matching (currently available for these :ref:`slitmask_info_instruments` only). If equal to ``header``, the dither offsets recorded in the header, when available, will be used. If ``auto`` is chosen, PypeIt will try to compute the offsets using a reference object with the highest S/N, or an object selected by the user (see ``user_obj``). If a list of offsets is provided, PypeIt will use it. +``only_slits`` str, list .. .. Restrict coaddition to one or more of slits. Example syntax -- DET01:175,DET02:205 or MSC02:2234. This and ``exclude_slits`` are mutually exclusive. If both are provided, ``only_slits`` takes precedence. +``spat_toler`` int .. 5 This parameter provides the desired tolerance in spatial pixel used to identify slits in different exposures +``use_slits4wvgrid`` bool .. False If True, use the slits to set the trace down the center +``user_obj`` int, list .. .. Object that the user wants to use to compute the weights and/or the offsets for coadding images. For longslit/multislit spectroscopy, provide the ``SLITID`` and the ``OBJID``, separated by comma, of the selected object. For echelle spectroscopy, provide the ``ECH_OBJID`` of the selected object. See :doc:`out_spec1D` for more info about ``SLITID``, ``OBJID`` and ``ECH_OBJID``. If this parameter is not ``None``, it will be used to compute the offsets only if ``offsets = auto``, and it will used to compute the weights only if ``weights = auto``. +``wave_method`` str .. .. Argument to :func:`~pypeit.core.wavecal.wvutils.get_wave_grid` method, which determines how the 2d coadd wavelength grid is constructed. The default is None, which will use a linear gridfor longslit/multislit coadds and a log10 grid for echelle coadds. Currently supported options with 2d coadding are:* 'iref' -- Use one of the exposures (the first) as the reference for the wavelength grid * 'velocity' -- Grid is uniform in velocity* 'log10' -- Grid is uniform in log10(wave). This is the same as velocity.* 'linear' -- Grid is uniform in wavelength +``weights`` str, list .. ``auto`` Mode for the weights used to coadd images. Options are: ``auto``, ``uniform``, or a list of weights. If ``auto`` is used, PypeIt will try to compute the weights using a reference object with the highest S/N, or an object selected by the user (see ``user_obj``), if ``uniform`` is used, uniform weights will be applied. If a list of weights is provided, PypeIt will use it. +==================== ========= ======= ======== ============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ ---- @@ -544,6 +553,7 @@ Class Instantiation: :class:`~pypeit.par.pypeitpar.Collate1DPar` ========================= ========== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== Key Type Options Default Description ========================= ========== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== +``chk_version`` bool .. False Whether to check the data model versions of spec1d files and sensfunc files. ``dry_run`` bool .. False If set, the script will display the matching File and Object Ids but will not flux, coadd or archive. ``exclude_serendip`` bool .. False Whether to exclude SERENDIP objects from collating. ``exclude_slit_trace_bm`` list, str .. [] A list of slit trace bitmask bits that should be excluded. @@ -656,20 +666,21 @@ Class Instantiation: :class:`~pypeit.par.pypeitpar.CubePar` ==================== ===== ======= ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== Key Type Options Default Description ==================== ===== ======= ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== +``align`` bool .. False If set to True, the input frames will be spatially aligned by cross-correlating the whitelight images with either a reference image (see ``reference_image``) or the whitelight image that is generated using the first spec2d listed in the coadd3d file. Alternatively, the user can specify the offsets (i.e. Delta RA x cos(dec) and Delta Dec, both in arcsec) in the spec2d block of the coadd3d file. See the documentation for examples of this usage. ``astrometric`` bool .. True If true, an astrometric correction will be applied using the alignment frames. ``combine`` bool .. False If set to True, the input frames will be combined. Otherwise, a separate datacube will be generated for each input spec2d file, and will be saved as a spec3d file. ``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. ``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. ``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. ``method`` str .. ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "NGP" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). -``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with "spec3d_*" +``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` ``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. ``ra_min`` float .. .. Minimum RA to use when generating the WCS. If None, the default is minimum RA based on the WCS of all spaxels. Units should be degrees. ``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). ``relative_weights`` bool .. False If set to True, the combined frames will use a relative weighting scheme. This only works well if there is a common continuum source in the field of view of all input observations, and is generally only required if high relative precision is desired. -``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. If combine=False, the individual spec3d files will have a suffix "_whitelight". +``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". ``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. -``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time. +``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. ``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. ``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel. ``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. @@ -678,6 +689,7 @@ Key Type Options Default Description ``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. ``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. ``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. +``whitelight_range`` list .. None, None A two element list specifying the wavelength range over which to generate the white light image. The first (second) element is the minimum (maximum) wavelength to use. If either of these elements are None, PypeIt will automatically use a wavelength range that ensures all spaxels have the same wavelength coverage. Note, if you are using a reference_image to align all frames, it is preferable to use the same white light wavelength range for all white light images. For example, you may wish to use an emission line map to register two frames. ==================== ===== ======= ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== @@ -723,7 +735,7 @@ Key Type Options Default Description ``find_extrap_npoly`` int .. 3 Polynomial order used for trace extrapolation ``find_fwhm`` int, float .. 5.0 Indicates roughly the fwhm of objects in pixels for object finding ``find_maxdev`` int, float .. 2.0 Maximum deviation of pixels from polynomial fit to trace used to reject bad pixels in trace fitting. -``find_min_max`` list .. .. It defines the minimum and maximum of your object in the spectral direction on the detector. It only used for object finding. This parameter is helpful if your object only has emission lines or at high redshift and the trace only shows in part of the detector. +``find_min_max`` list .. .. It defines the minimum and maximum of your object in pixels in the spectral direction on the detector. It only used for object finding. This parameter is helpful if your object only has emission lines or at high redshift and the trace only shows in part of the detector. ``find_negative`` bool .. .. Identify negative objects in object finding for spectra that are differenced. This is used to manually override the default behavior in PypeIt for object finding by setting this parameter to something other than None The default behavior is that PypeIt will search for negative object traces if background frames are present in the PypeIt file that are classified as "science" (i.e. via pypeit_setup -b, and setting bkg_id in the PypeIt file). If background frames are present that are classified as "sky", then PypeIt will NOT search for negative object traces. If one wishes to explicitly override this default behavior, set this parameter to True to find negative objects or False to ignore them. ``find_trim_edge`` list .. 5, 5 Trim the slit by this number of pixels left/right before finding objects ``maxnumber_sci`` int .. 10 Maximum number of objects to extract in a science frame. Use None for no limit. This parameter can be useful in situations where systematics lead to spurious extra objects. Setting this parameter means they will be trimmed. For mulitslit maxnumber applies per slit, for echelle observations this applies per order. Note that objects on a slit/order impact the sky-modeling and so maxnumber should never be lower than the true number of detectable objects on your slit. For image differenced observations with positive and negative object traces, maxnumber applies to the number of positive (or negative) traces individually. In other words, if you had two positive objects and one negative object, then you would set maxnumber to be equal to two (not three). Note that if manually extracted apertures are explicitly requested, they do not count against this maxnumber. If more than maxnumber objects are detected, then highest S/N ratio objects will be the ones that are kept. For multislit observations the choice here depends on the slit length. For echelle observations with short slits we set the default to be 1 @@ -751,7 +763,7 @@ Key Type Options Default Description =================== ========== ======= ======= =================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== ``bspline_spacing`` int, float .. 0.6 Break-point spacing for the bspline sky subtraction fits. ``global_sky_std`` bool .. True Global sky subtraction will be performed on standard stars. This should be turned off for example for near-IR reductions with narrow slits, since bright standards can fill the slit causing global sky-subtraction to fail. In these situations we go straight to local sky-subtraction since it is designed to deal with such situations -``joint_fit`` bool .. False Perform a simultaneous joint fit to sky regions using all available slits. Currently, this parameter is only used for IFU data reduction. +``joint_fit`` bool .. False Perform a simultaneous joint fit to sky regions using all available slits. Currently, this parameter is only used for IFU data reduction. Note that the current implementation does not account for variations in the instrument FWHM in different slits. This will be addressed by Issue #1660. ``local_maskwidth`` float .. 4.0 Initial width of the region in units of FWHM that will be used for local sky subtraction ``mask_by_boxcar`` bool .. False In global sky evaluation, mask the sky region around the object by the boxcar radius (set in ExtractionPar). ``max_mask_frac`` float .. 0.8 Maximum fraction of total pixels on a slit that can be masked by the input masks. If more than this threshold is masked the code will return zeros and throw a warning. @@ -775,7 +787,7 @@ Class Instantiation: :class:`~pypeit.par.pypeitpar.SlitMaskPar` Key Type Options Default Description =========================== ========== ======= ======= ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== ``assign_obj`` bool .. False If SlitMask object was generated, assign RA,DEC,name to detected objects -``bright_maskdef_id`` int .. .. `maskdef_id` (corresponding to `dSlitId` and `Slit_Number` in the DEIMOS and MOSFIRE slitmask design, respectively) of a slit containing a bright object that will be used to compute the slitmask offset. This parameter is optional and is ignored if ``slitmask_offset`` is provided. +``bright_maskdef_id`` int .. .. `maskdef_id` (corresponding e.g., to `dSlitId` and `Slit_Number` in the DEIMOS/LRIS and MOSFIRE slitmask design, respectively) of a slit containing a bright object that will be used to compute the slitmask offset. This parameter is optional and is ignored if ``slitmask_offset`` is provided. ``extract_missing_objs`` bool .. False Force extraction of undetected objects at the location expected from the slitmask design. ``missing_objs_boxcar_rad`` int, float .. 1.0 Indicates the boxcar radius in arcsec for the force extraction of undetected objects. ``missing_objs_fwhm`` int, float .. .. Indicates the FWHM in arcsec for the force extraction of undetected objects. PypeIt will try to determine the FWHM from the flux profile (by using ``missing_objs_fwhm`` as initial guess). If the FWHM cannot be determined, ``missing_objs_fwhm`` will be assumed. If you do not want PypeIt to try to determine the FWHM set the parameter ``use_user_fwhm`` in ``ExtractionPar`` to True. If ``missing_objs_fwhm`` is ``None`` (which is the default) PypeIt will use the median FWHM of all the detected objects. @@ -868,6 +880,7 @@ Key Type Options ``algorithm`` str ``UVIS``, ``IR`` ``UVIS`` Specify the algorithm for computing the sensitivity function. The options are: (1) UVIS = Should be used for data with :math:`\lambda < 7000` A. No detailed model of telluric absorption but corrects for atmospheric extinction. (2) IR = Should be used for data with :math:`\lambda > 7000` A. Peforms joint fit for sensitivity function and telluric absorption using HITRAN models. ``extrap_blu`` float .. 0.1 Fraction of minimum wavelength coverage to grow the wavelength coverage of the sensitivitity function in the blue direction (`i.e.`, if the standard star spectrum cuts off at ``wave_min``) the sensfunc will be extrapolated to cover down to (1.0 - ``extrap_blu``) * ``wave_min`` ``extrap_red`` float .. 0.1 Fraction of maximum wavelength coverage to grow the wavelength coverage of the sensitivitity function in the red direction (`i.e.`, if the standard star spectrumcuts off at ``wave_max``) the sensfunc will be extrapolated to cover up to (1.0 + ``extrap_red``) * ``wave_max`` +``flatfile`` str .. .. Flat field file to be used if the sensitivity function model will utilize the blaze function computed from a flat field file in the Calibrations directory, e.g.Calibrations/Flat_A_0_DET01.fits ``hydrogen_mask_wid`` float .. 10.0 Mask width from line center for hydrogen recombination lines in Angstroms (total mask width is 2x this value). ``mask_helium_lines`` bool .. False Mask certain ``HeII`` recombination lines prominent in O-type stars in the sensitivity function fit A region equal to 0.5 * ``hydrogen_mask_wid`` on either side of the line center is masked. ``mask_hydrogen_lines`` bool .. True Mask hydrogen Balmer, Paschen, Brackett, and Pfund recombination lines in the sensitivity function fit. A region equal to ``hydrogen_mask_wid`` on either side of the line center is masked. @@ -930,7 +943,7 @@ Key Type Options Default ``lower`` int, float .. 3.0 Lower rejection threshold in units of sigma_corr*sigma, where sigma is the formal noise of the spectrum, and sigma_corr is an empirically determined correction to the formal error. The distribution of input chi (defined by chi = (data - model)/sigma) values is analyzed, and a correction factor to the formal error sigma_corr is returned which is multiplied into the formal errors. In this way, a rejection threshold of i.e. 3-sigma, will always correspond to roughly the same percentile. This renormalization is performed with coadd1d.renormalize_errors function, and guarantees that rejection is not too agressive in cases where the empirical errors determined from the chi-distribution differ significantly from the formal noise which is used to determine chi. ``mask_lyman_a`` bool .. True Mask the blueward of Lyman-alpha line during the fitting? ``maxiter`` int .. 2 Maximum number of iterations for the telluric + object model fitting. The code performs multiple iterations rejecting outliers at each step. The fit is then performed anew to the remaining good pixels. For this reason if you run with the disp=True option, you will see that the f(x) loss function gets progressively better during the iterations. -``minmax_coeff_bounds`` tuple .. (-5.0, 5.0) Parameters setting the polynomial coefficient bounds for sensfunc optimization. Bounds are currently determined as follows. We compute an initial fit to the sensfunc in the pypeit.core.telluric.init_sensfunc_model function. That deterines a set of coefficients. The bounds are then determined according to: [(np.fmin(np.abs(this_coeff)*obj_params['delta_coeff_bounds'][0], obj_params['minmax_coeff_bounds'][0]), np.fmax(np.abs(this_coeff)*obj_params['delta_coeff_bounds'][1], obj_params['minmax_coeff_bounds'][1]))] +``minmax_coeff_bounds`` tuple .. (-5.0, 5.0) Parameters setting the polynomial coefficient bounds for sensfunc optimization. Bounds are currently determined as follows. We compute an initial fit to the sensfunc in the :func:`~pypeit.core.telluric.init_sensfunc_model` function. That deterines a set of coefficients. The bounds are then determined according to: [(np.fmin(np.abs(this_coeff)*obj_params['delta_coeff_bounds'][0], obj_params['minmax_coeff_bounds'][0]), np.fmax(np.abs(this_coeff)*obj_params['delta_coeff_bounds'][1], obj_params['minmax_coeff_bounds'][1]))] ``model`` str .. ``exp`` Types of polynomial model. Options are poly, square, exp corresponding to normal polynomial, squared polynomial, or exponentiated polynomial ``npca`` int .. 8 Number of pca for the objmodel=qso qso PCA fit ``objmodel`` str .. .. The object model to be used for telluric fitting. Currently the options are: qso, star, and poly @@ -984,7 +997,7 @@ Alterations to the default parameters are: spectrograph = bok_bc [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -993,7 +1006,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_biasimage = False @@ -1001,7 +1014,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] use_biasimage = False use_overscan = False @@ -1022,7 +1035,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] use_biasimage = False use_overscan = False @@ -1062,7 +1075,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[standardframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] mask_cr = True use_biasimage = False @@ -1070,14 +1083,14 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[wavelengths]] - lamps = NeI, ArI, ArII, HeI + lamps = NeI, ArI, ArII, HeI, fwhm = 5.0 rms_threshold = 0.5 [[slitedges]] edge_thresh = 50.0 sync_predict = nearest [scienceframe] - exprng = 90, None + exprng = 90, None, [[process]] mask_cr = True sigclip = 5.0 @@ -1116,7 +1129,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] mask_cr = True use_biasimage = False @@ -1124,7 +1137,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 1, 50 + exprng = 1, 50, [[[process]]] use_biasimage = False use_overscan = False @@ -1183,7 +1196,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[standardframe]] - exprng = None, 60 + exprng = None, 60, [[[process]]] mask_cr = True use_biasimage = False @@ -1192,7 +1205,7 @@ Alterations to the default parameters are: use_illumflat = False [[wavelengths]] method = full_template - lamps = ArI, ArII, ThAr, NeI + lamps = ArI, ArII, ThAr, NeI, sigdetect = 3 fwhm = 20 reid_arxiv = magellan_fire_long.fits @@ -1204,7 +1217,7 @@ Alterations to the default parameters are: [[tilts]] tracethresh = 5 [scienceframe] - exprng = 20, None + exprng = 20, None, [[process]] mask_cr = True use_biasimage = False @@ -1214,7 +1227,7 @@ Alterations to the default parameters are: [reduce] [[findobj]] snr_thresh = 5.0 - find_trim_edge = 50, 50 + find_trim_edge = 50, 50, .. _instr_par-gemini_flamingos2: @@ -1236,7 +1249,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] mask_cr = True use_biasimage = False @@ -1244,14 +1257,14 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 50, None + exprng = 50, None, [[[process]]] use_biasimage = False use_overscan = False use_pixelflat = False use_illumflat = False [[tiltframe]] - exprng = 50, None + exprng = 50, None, [[[process]]] use_biasimage = False use_overscan = False @@ -1304,7 +1317,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[standardframe]] - exprng = None, 30 + exprng = None, 30, [[[process]]] mask_cr = True use_biasimage = False @@ -1312,7 +1325,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[wavelengths]] - lamps = OH_NIRES + lamps = OH_NIRES, fwhm = 5 rms_threshold = 0.5 match_toler = 5.0 @@ -1325,7 +1338,7 @@ Alterations to the default parameters are: tracethresh = 5 spat_order = 4 [scienceframe] - exprng = 20, None + exprng = 20, None, [[process]] mask_cr = True use_biasimage = False @@ -1335,7 +1348,7 @@ Alterations to the default parameters are: [reduce] [[findobj]] snr_thresh = 5.0 - find_trim_edge = 10, 10 + find_trim_edge = 10, 10, [[skysub]] sky_sigrej = 5.0 [sensfunc] @@ -1354,7 +1367,7 @@ Alterations to the default parameters are: [rdx] spectrograph = gemini_gmos_north_e2v - detnum = (1, 2, 3) + detnum = (1, 2, 3), [calibrations] [[biasframe]] [[[process]]] @@ -1411,7 +1424,7 @@ Alterations to the default parameters are: noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = CuI, ArI, ArII + lamps = CuI, ArI, ArII, rms_threshold = 0.4 nsnippet = 1 [[slitedges]] @@ -1437,7 +1450,7 @@ Alterations to the default parameters are: [rdx] spectrograph = gemini_gmos_north_ham - detnum = (1, 2, 3) + detnum = (1, 2, 3), [calibrations] [[biasframe]] [[[process]]] @@ -1494,7 +1507,7 @@ Alterations to the default parameters are: noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = CuI, ArI, ArII + lamps = CuI, ArI, ArII, rms_threshold = 0.4 nsnippet = 1 [[slitedges]] @@ -1520,7 +1533,7 @@ Alterations to the default parameters are: [rdx] spectrograph = gemini_gmos_north_ham_ns - detnum = (1, 2, 3) + detnum = (1, 2, 3), [calibrations] [[biasframe]] [[[process]]] @@ -1577,7 +1590,7 @@ Alterations to the default parameters are: noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = CuI, ArI, ArII + lamps = CuI, ArI, ArII, rms_threshold = 0.4 nsnippet = 1 [[slitedges]] @@ -1603,7 +1616,7 @@ Alterations to the default parameters are: [rdx] spectrograph = gemini_gmos_south_ham - detnum = (1, 2, 3) + detnum = (1, 2, 3), [calibrations] [[biasframe]] [[[process]]] @@ -1660,7 +1673,7 @@ Alterations to the default parameters are: noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = CuI, ArI, ArII + lamps = CuI, ArI, ArII, rms_threshold = 0.4 nsnippet = 1 [[slitedges]] @@ -1681,16 +1694,16 @@ Alterations to the default parameters are: [[IR]] telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits -.. _instr_par-gemini_gnirs: +.. _instr_par-gemini_gnirs_echelle: -GEMINI-N GNIRS (``gemini_gnirs``) ---------------------------------- +GEMINI-N GNIRS (``gemini_gnirs_echelle``) +----------------------------------------- Alterations to the default parameters are: .. code-block:: ini [rdx] - spectrograph = gemini_gnirs + spectrograph = gemini_gnirs_echelle [calibrations] [[biasframe]] [[[process]]] @@ -1720,7 +1733,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 30 + exprng = None, 30, [[[process]]] satpix = nothing use_biasimage = False @@ -1740,7 +1753,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 30 + exprng = None, 30, [[[process]]] use_biasimage = False use_overscan = False @@ -1768,7 +1781,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[standardframe]] - exprng = None, 30 + exprng = None, 30, [[[process]]] use_biasimage = False use_overscan = False @@ -1776,8 +1789,10 @@ Alterations to the default parameters are: use_illumflat = False [[flatfield]] tweak_slits_thresh = 0.9 + [[tilts]] + spat_order = 1 [scienceframe] - exprng = 30, None + exprng = 30, None, [[process]] mask_cr = True use_biasimage = False @@ -1786,7 +1801,7 @@ Alterations to the default parameters are: use_illumflat = False [reduce] [[findobj]] - find_trim_edge = 2, 2 + find_trim_edge = 2, 2, maxnumber_sci = 2 maxnumber_std = 1 [[skysub]] @@ -1801,6 +1816,142 @@ Alterations to the default parameters are: [[IR]] telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits +.. _instr_par-gemini_gnirs_ifu: + +GEMINI-N GNIRS (``gemini_gnirs_ifu``) +------------------------------------- +Alterations to the default parameters are: + +.. code-block:: ini + + [rdx] + spectrograph = gemini_gnirs_ifu + [calibrations] + [[biasframe]] + [[[process]]] + combine = median + use_biasimage = False + use_overscan = False + shot_noise = False + use_pixelflat = False + use_illumflat = False + [[darkframe]] + [[[process]]] + mask_cr = True + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False + [[arcframe]] + [[[process]]] + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False + [[tiltframe]] + [[[process]]] + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False + [[pixelflatframe]] + exprng = None, 30, + [[[process]]] + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False + [[pinholeframe]] + [[[process]]] + use_biasimage = False + use_overscan = False + use_illumflat = False + [[alignframe]] + [[[process]]] + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False + [[traceframe]] + exprng = None, 30, + [[[process]]] + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False + [[illumflatframe]] + [[[process]]] + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False + [[lampoffflatsframe]] + [[[process]]] + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False + [[skyframe]] + [[[process]]] + mask_cr = True + use_biasimage = False + use_overscan = False + noise_floor = 0.01 + use_illumflat = False + [[standardframe]] + exprng = None, 30, + [[[process]]] + use_biasimage = False + use_overscan = False + noise_floor = 0.01 + use_illumflat = False + [[flatfield]] + tweak_slits_thresh = 0.0 + tweak_slits_maxfrac = 0.0 + slit_trim = 2 + slit_illum_finecorr = False + [[slitedges]] + pad = 2 + [[tilts]] + spat_order = 1 + spec_order = 1 + [scienceframe] + exprng = 30, None, + [[process]] + mask_cr = True + sigclip = 4.0 + objlim = 1.5 + use_biasimage = False + use_overscan = False + noise_floor = 0.01 + use_illumflat = False + [reduce] + [[findobj]] + find_trim_edge = 2, 2, + maxnumber_sci = 2 + maxnumber_std = 1 + [[skysub]] + global_sky_std = False + no_poly = True + [[extraction]] + model_full_slit = True + skip_extraction = True + [[cube]] + grating_corr = False + [flexure] + spec_maxshift = 0 + [sensfunc] + algorithm = IR + polyorder = 6 + [[UVIS]] + extinct_correct = False + [[IR]] + telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + .. _instr_par-gtc_maat: GTC OSIRIS (``gtc_maat``) @@ -1813,7 +1964,7 @@ Alterations to the default parameters are: spectrograph = gtc_maat [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -1821,7 +1972,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -1845,7 +1996,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing @@ -1870,7 +2021,7 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 180 + exprng = None, 180, [[[process]]] mask_cr = True noise_floor = 0.01 @@ -1878,7 +2029,7 @@ Alterations to the default parameters are: slit_illum_finecorr = False [[wavelengths]] method = full_template - lamps = XeI,HgI,NeI,ArI + lamps = XeI, HgI, NeI, ArI, [[slitedges]] sync_predict = nearest bound_detector = True @@ -1886,7 +2037,7 @@ Alterations to the default parameters are: spat_order = 1 spec_order = 1 [scienceframe] - exprng = 90, None + exprng = 90, None, [[process]] mask_cr = True sigclip = 4.0 @@ -1921,7 +2072,7 @@ Alterations to the default parameters are: spectrograph = gtc_osiris [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -1929,7 +2080,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -1953,7 +2104,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing @@ -1978,13 +2129,13 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 180 + exprng = None, 180, [[[process]]] mask_cr = True noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = XeI,HgI,NeI,ArI + lamps = XeI, HgI, NeI, ArI, [[slitedges]] sync_predict = nearest bound_detector = True @@ -1992,7 +2143,7 @@ Alterations to the default parameters are: spat_order = 5 spec_order = 5 [scienceframe] - exprng = 90, None + exprng = 90, None, [[process]] mask_cr = True noise_floor = 0.01 @@ -2014,7 +2165,7 @@ Alterations to the default parameters are: spectrograph = gtc_osiris_plus [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -2022,7 +2173,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -2046,7 +2197,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing @@ -2071,13 +2222,13 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 180 + exprng = None, 180, [[[process]]] mask_cr = True noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = XeI,HgI,NeI,ArI + lamps = XeI, HgI, NeI, ArI, [[slitedges]] sync_predict = nearest bound_detector = True @@ -2085,7 +2236,7 @@ Alterations to the default parameters are: spat_order = 5 spec_order = 5 [scienceframe] - exprng = 90, None + exprng = 90, None, [[process]] mask_cr = True noise_floor = 0.01 @@ -2166,9 +2317,9 @@ Alterations to the default parameters are: objlim = 2.0 noise_floor = 0.01 [reduce] - trim_edge = 0, 0 + trim_edge = 0, 0, [[findobj]] - find_trim_edge = 0, 0 + find_trim_edge = 0, 0, maxnumber_sci = 2 find_fwhm = 2.0 [[skysub]] @@ -2252,9 +2403,9 @@ Alterations to the default parameters are: objlim = 2.0 noise_floor = 0.01 [reduce] - trim_edge = 0, 0 + trim_edge = 0, 0, [[findobj]] - find_trim_edge = 0, 0 + find_trim_edge = 0, 0, maxnumber_sci = 2 find_fwhm = 2.0 [[skysub]] @@ -2264,7 +2415,7 @@ Alterations to the default parameters are: max_mask_frac = 0.95 [[extraction]] boxcar_radius = 0.2 - sn_gauss = 6.0 + sn_gauss = 5.0 model_full_slit = True use_2dmodel_mask = False @@ -2278,7 +2429,7 @@ Alterations to the default parameters are: [rdx] spectrograph = keck_deimos - detnum = (1, 5), (2, 6), (3, 7), (4, 8) + detnum = (1, 5), (2, 6), (3, 7), (4, 8), [calibrations] [[biasframe]] [[[process]]] @@ -2350,7 +2501,7 @@ Alterations to the default parameters are: use_biasimage = False noise_floor = 0.01 [[wavelengths]] - lamps = ArI, NeI, KrI, XeI + lamps = ArI, NeI, KrI, XeI, match_toler = 2.5 n_first = 3 [[slitedges]] @@ -2374,6 +2525,109 @@ Alterations to the default parameters are: [[IR]] telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits +.. _instr_par-keck_esi: + +KECK ESI (``keck_esi``) +----------------------- +Alterations to the default parameters are: + +.. code-block:: ini + + [rdx] + spectrograph = keck_esi + [calibrations] + [[biasframe]] + [[[process]]] + combine = median + use_biasimage = False + shot_noise = False + use_pixelflat = False + use_illumflat = False + [[darkframe]] + exprng = 1, None, + [[[process]]] + mask_cr = True + use_pixelflat = False + use_illumflat = False + [[arcframe]] + exprng = 300, None, + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[tiltframe]] + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[pixelflatframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[alignframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[traceframe]] + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[illumflatframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[lampoffflatsframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[skyframe]] + [[[process]]] + mask_cr = True + noise_floor = 0.01 + [[standardframe]] + exprng = None, 60, + [[[process]]] + mask_cr = True + noise_floor = 0.01 + [[wavelengths]] + method = reidentify + echelle = True + ech_sigrej = 3.0 + lamps = CuI, ArI, NeI, HgI, XeI, ArII, + fwhm = 2.9 + fwhm_fromlines = True + reid_arxiv = keck_esi_ECH.fits + cc_thresh = 0.5 + cc_local_thresh = 0.5 + rms_threshold = 0.3 + [[slitedges]] + edge_thresh = 5.0 + det_min_spec_length = 0.2 + max_shift_adj = 3.0 + fit_min_spec_length = 0.4 + left_right_pca = True + pca_order = 3 + pca_sigrej = 1.5 + add_missed_orders = True + [[tilts]] + tracethresh = 10.0 + [scienceframe] + exprng = 60, None, + [[process]] + satpix = nothing + mask_cr = True + sigclip = 20.0 + noise_floor = 0.01 + [reduce] + [[findobj]] + find_trim_edge = 4, 4, + maxnumber_sci = 2 + maxnumber_std = 1 + [[extraction]] + model_full_slit = True + .. _instr_par-keck_hires: KECK HIRES (``keck_hires``) @@ -2384,7 +2638,7 @@ Alterations to the default parameters are: [rdx] spectrograph = keck_hires - detnum = (1, 2, 3) + detnum = (1, 2, 3), [calibrations] [[biasframe]] [[[process]]] @@ -2475,7 +2729,7 @@ Alterations to the default parameters are: method = echelle echelle = True ech_sigrej = 3.0 - lamps = ThAr + lamps = ThAr, fwhm = 8.0 cc_thresh = 0.5 cc_local_thresh = 0.5 @@ -2504,17 +2758,118 @@ Alterations to the default parameters are: use_illumflat = False [reduce] [[findobj]] - find_trim_edge = 3, 3 + find_trim_edge = 3, 3, [[skysub]] global_sky_std = False [[extraction]] model_full_slit = True + [coadd1d] + wave_method = log10 [sensfunc] algorithm = IR - polyorder = 11 [[IR]] telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits +.. _instr_par-keck_kcrm: + +KECK KCRM (``keck_kcrm``) +------------------------- +Alterations to the default parameters are: + +.. code-block:: ini + + [rdx] + spectrograph = keck_kcrm + [calibrations] + [[biasframe]] + exprng = None, 0.001, + [[[process]]] + combine = median + use_biasimage = False + shot_noise = False + use_pixelflat = False + use_illumflat = False + [[darkframe]] + exprng = 0.01, None, + [[[process]]] + mask_cr = True + use_pixelflat = False + use_illumflat = False + [[arcframe]] + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[tiltframe]] + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[pixelflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[alignframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[alignment]] + locations = 0.1, 0.3, 0.5, 0.7, 0.9, + [[traceframe]] + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[illumflatframe]] + [[[process]]] + satpix = nothing + use_illumflat = False + [[lampoffflatsframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[skyframe]] + [[[process]]] + mask_cr = True + noise_floor = 0.01 + [[standardframe]] + [[[process]]] + mask_cr = True + noise_floor = 0.01 + [[flatfield]] + spec_samp_coarse = 20.0 + tweak_slits_thresh = 0.0 + tweak_slits_maxfrac = 0.0 + slit_illum_relative = True + slit_illum_ref_idx = 14 + slit_illum_smooth_npix = 5 + fit_2d_det_response = True + [[wavelengths]] + fwhm_spat_order = 2 + [[slitedges]] + edge_thresh = 5 + fit_order = 4 + pad = 2 + [scienceframe] + [[process]] + mask_cr = True + sigclip = 4.0 + objlim = 1.5 + use_biasimage = False + noise_floor = 0.01 + use_specillum = True + [reduce] + [[skysub]] + no_poly = True + [[extraction]] + skip_extraction = True + [flexure] + spec_maxshift = 3 + [sensfunc] + [[UVIS]] + extinct_correct = False + .. _instr_par-keck_kcwi: KECK KCWI (``keck_kcwi``) @@ -2527,7 +2882,7 @@ Alterations to the default parameters are: spectrograph = keck_kcwi [calibrations] [[biasframe]] - exprng = None, 0.01 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -2536,7 +2891,7 @@ Alterations to the default parameters are: use_illumflat = False use_pattern = True [[darkframe]] - exprng = 0.01, None + exprng = 0.01, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -2562,7 +2917,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[alignment]] - locations = 0.1, 0.3, 0.5, 0.7, 0.9 + locations = 0.1, 0.3, 0.5, 0.7, 0.9, [[traceframe]] [[[process]]] use_pixelflat = False @@ -2594,7 +2949,10 @@ Alterations to the default parameters are: slit_illum_ref_idx = 14 slit_illum_smooth_npix = 5 fit_2d_det_response = True + [[wavelengths]] + fwhm_spat_order = 2 [[slitedges]] + edge_thresh = 5 fit_order = 4 pad = 2 [scienceframe] @@ -2604,6 +2962,7 @@ Alterations to the default parameters are: objlim = 1.5 use_biasimage = False noise_floor = 0.01 + use_specillum = True use_pattern = True [reduce] [[skysub]] @@ -2628,7 +2987,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_blue [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -2636,7 +2995,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -2650,24 +3009,25 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 300 + exprng = None, 300, [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 300 + exprng = None, 300, [[[process]]] use_pixelflat = False use_illumflat = False [[illumflatframe]] + exprng = None, 300, [[[process]]] satpix = nothing use_pixelflat = False @@ -2682,17 +3042,15 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 30 + exprng = 1, 61, [[[process]]] mask_cr = True noise_floor = 0.01 spat_flexure_correct = True [[wavelengths]] method = full_template - lamps = NeI, ArI, CdI, KrI, XeI, ZnI, HgI sigdetect = 10.0 rms_threshold = 0.2 - match_toler = 2.5 n_first = 3 [[slitedges]] edge_thresh = 15.0 @@ -2700,10 +3058,10 @@ Alterations to the default parameters are: fit_order = 3 fit_min_spec_length = 0.2 sync_center = gap - minimum_slit_length = 4.0 - minimum_slit_length_sci = 6 + minimum_slit_length = 3.0 + minimum_slit_length_sci = 5.0 [scienceframe] - exprng = 60, None + exprng = 61, None, [[process]] mask_cr = True noise_floor = 0.01 @@ -2726,7 +3084,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_blue_orig [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -2734,7 +3092,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -2748,24 +3106,25 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 300 + exprng = None, 300, [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 300 + exprng = None, 300, [[[process]]] use_pixelflat = False use_illumflat = False [[illumflatframe]] + exprng = None, 300, [[[process]]] satpix = nothing use_pixelflat = False @@ -2780,17 +3139,15 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 30 + exprng = 1, 61, [[[process]]] mask_cr = True noise_floor = 0.01 spat_flexure_correct = True [[wavelengths]] method = full_template - lamps = NeI, ArI, CdI, KrI, XeI, ZnI, HgI sigdetect = 10.0 rms_threshold = 0.2 - match_toler = 2.5 n_first = 3 [[slitedges]] edge_thresh = 15.0 @@ -2798,10 +3155,10 @@ Alterations to the default parameters are: fit_order = 3 fit_min_spec_length = 0.2 sync_center = gap - minimum_slit_length = 4.0 - minimum_slit_length_sci = 6 + minimum_slit_length = 3.0 + minimum_slit_length_sci = 5.0 [scienceframe] - exprng = 60, None + exprng = 61, None, [[process]] mask_cr = True noise_floor = 0.01 @@ -2824,7 +3181,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_red [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -2832,7 +3189,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -2846,24 +3203,25 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 60 + exprng = None, 60, [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 60 + exprng = None, 60, [[[process]]] use_pixelflat = False use_illumflat = False [[illumflatframe]] + exprng = None, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -2878,20 +3236,19 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 30 + exprng = 1, 61, [[[process]]] mask_cr = True noise_floor = 0.01 spat_flexure_correct = True [[wavelengths]] - lamps = NeI, ArI, CdI, KrI, XeI, ZnI, HgI sigdetect = 10.0 rms_threshold = 0.2 [[slitedges]] fit_order = 3 sync_center = gap - minimum_slit_length = 4.0 - minimum_slit_length_sci = 6 + minimum_slit_length = 3.0 + minimum_slit_length_sci = 5.0 [[tilts]] tracethresh = 25 maxdev_tracefit = 1.0 @@ -2900,7 +3257,7 @@ Alterations to the default parameters are: maxdev2d = 1.0 sigrej2d = 5.0 [scienceframe] - exprng = 60, None + exprng = 61, None, [[process]] mask_cr = True sigclip = 5.0 @@ -2930,7 +3287,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_red_mark4 [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -2938,7 +3295,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -2952,24 +3309,25 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 60 + exprng = None, 60, [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 60 + exprng = None, 60, [[[process]]] use_pixelflat = False use_illumflat = False [[illumflatframe]] + exprng = None, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -2984,20 +3342,19 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 30 + exprng = 1, 61, [[[process]]] mask_cr = True noise_floor = 0.01 spat_flexure_correct = True [[wavelengths]] - lamps = NeI, ArI, CdI, KrI, XeI, ZnI, HgI sigdetect = 10.0 rms_threshold = 0.2 [[slitedges]] fit_order = 3 sync_center = gap - minimum_slit_length = 4.0 - minimum_slit_length_sci = 6 + minimum_slit_length = 3.0 + minimum_slit_length_sci = 5.0 [[tilts]] tracethresh = 25 maxdev_tracefit = 1.0 @@ -3006,7 +3363,7 @@ Alterations to the default parameters are: maxdev2d = 1.0 sigrej2d = 5.0 [scienceframe] - exprng = 60, None + exprng = 61, None, [[process]] mask_cr = True sigclip = 5.0 @@ -3036,7 +3393,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_red_orig [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -3044,7 +3401,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -3058,24 +3415,25 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 60 + exprng = None, 60, [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 60 + exprng = None, 60, [[[process]]] use_pixelflat = False use_illumflat = False [[illumflatframe]] + exprng = None, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3090,20 +3448,19 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 30 + exprng = 1, 61, [[[process]]] mask_cr = True noise_floor = 0.01 spat_flexure_correct = True [[wavelengths]] - lamps = NeI, ArI, KrI, XeI, HgI sigdetect = 10.0 rms_threshold = 0.2 [[slitedges]] fit_order = 3 sync_center = gap - minimum_slit_length = 4.0 - minimum_slit_length_sci = 6 + minimum_slit_length = 3.0 + minimum_slit_length_sci = 5.0 [[tilts]] tracethresh = 25 maxdev_tracefit = 1.0 @@ -3112,7 +3469,7 @@ Alterations to the default parameters are: maxdev2d = 1.0 sigrej2d = 5.0 [scienceframe] - exprng = 60, None + exprng = 61, None, [[process]] mask_cr = True sigclip = 5.0 @@ -3150,7 +3507,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 1, None + exprng = 1, None, [[[process]]] mask_cr = True use_biasimage = False @@ -3158,7 +3515,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 1, None + exprng = 1, None, [[[process]]] use_biasimage = False use_overscan = False @@ -3215,21 +3572,21 @@ Alterations to the default parameters are: use_overscan = False noise_floor = 0.01 [[standardframe]] - exprng = None, 20 + exprng = None, 20, [[[process]]] mask_cr = True use_biasimage = False use_overscan = False noise_floor = 0.01 [[wavelengths]] - lamps = OH_NIRES + lamps = OH_NIRES, fwhm = 5.0 rms_threshold = 0.3 [[slitedges]] edge_thresh = 50.0 sync_predict = nearest [scienceframe] - exprng = 20, None + exprng = 20, None, [[process]] satpix = nothing mask_cr = True @@ -3277,14 +3634,14 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 61, None + exprng = 61, None, [[[process]]] use_biasimage = False use_overscan = False use_pixelflat = False use_illumflat = False [[tiltframe]] - exprng = 61, None + exprng = 61, None, [[[process]]] use_biasimage = False use_overscan = False @@ -3337,7 +3694,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[standardframe]] - exprng = None, 60 + exprng = None, 60, [[[process]]] mask_cr = True use_biasimage = False @@ -3349,11 +3706,12 @@ Alterations to the default parameters are: echelle = True ech_norder_coeff = 6 ech_sigrej = 3.0 - lamps = OH_NIRES - fwhm = 5.0 + lamps = OH_NIRES, + fwhm = 2.2 + fwhm_fromlines = True reid_arxiv = keck_nires.fits - rms_threshold = 0.2 - n_final = 3, 4, 4, 4, 4 + rms_threshold = 0.3 + n_final = 3, 4, 4, 4, 4, [[slitedges]] fit_min_spec_length = 0.4 left_right_pca = True @@ -3362,7 +3720,7 @@ Alterations to the default parameters are: [[tilts]] tracethresh = 10.0 [scienceframe] - exprng = 61, None + exprng = 61, None, [[process]] satpix = nothing mask_cr = True @@ -3376,6 +3734,8 @@ Alterations to the default parameters are: bspline_spacing = 0.8 [[extraction]] boxcar_radius = 0.75 + [coadd1d] + wave_method = log10 [coadd2d] offsets = header [sensfunc] @@ -3404,7 +3764,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] mask_cr = True use_biasimage = False @@ -3412,7 +3772,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] use_biasimage = False use_overscan = False @@ -3471,7 +3831,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[standardframe]] - exprng = None, 20 + exprng = None, 20, [[[process]]] mask_cr = True use_biasimage = False @@ -3481,14 +3841,14 @@ Alterations to the default parameters are: [[flatfield]] tweak_slits_thresh = 0.8 [[wavelengths]] - lamps = OH_NIRES + lamps = OH_NIRES, fwhm = 5.0 rms_threshold = 0.2 [[slitedges]] edge_thresh = 200.0 sync_predict = nearest [scienceframe] - exprng = 20, None + exprng = 20, None, [[process]] satpix = nothing mask_cr = True @@ -3598,7 +3958,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[wavelengths]] - lamps = OH_NIRES + lamps = OH_NIRES, fwhm = 5.0 rms_threshold = 0.2 [[slitedges]] @@ -3711,7 +4071,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[wavelengths]] - lamps = OH_NIRES + lamps = OH_NIRES, fwhm = 5.0 rms_threshold = 0.2 [[slitedges]] @@ -3747,7 +4107,7 @@ Alterations to the default parameters are: spectrograph = lbt_mods1b [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -3755,7 +4115,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -3771,20 +4131,20 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] use_pixelflat = False use_illumflat = False @@ -3803,12 +4163,12 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = 1, 200 + exprng = 1, 200, [[[process]]] mask_cr = True noise_floor = 0.01 [[wavelengths]] - lamps = XeI, KrI, ArI, HgI + lamps = XeI, KrI, ArI, HgI, sigdetect = 10.0 rms_threshold = 0.4 [[slitedges]] @@ -3820,7 +4180,7 @@ Alterations to the default parameters are: spec_order = 5 maxdev2d = 0.02 [scienceframe] - exprng = 200, None + exprng = 200, None, [[process]] mask_cr = True noise_floor = 0.01 @@ -3839,7 +4199,7 @@ Alterations to the default parameters are: spectrograph = lbt_mods1r [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -3847,7 +4207,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -3863,20 +4223,20 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] use_pixelflat = False use_illumflat = False @@ -3895,12 +4255,12 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = 1, 200 + exprng = 1, 200, [[[process]]] mask_cr = True noise_floor = 0.01 [[wavelengths]] - lamps = ArI, NeI, KrI, XeI + lamps = ArI, NeI, KrI, XeI, fwhm = 10.0 rms_threshold = 0.4 match_toler = 2.5 @@ -3914,7 +4274,7 @@ Alterations to the default parameters are: spec_order = 5 maxdev2d = 0.02 [scienceframe] - exprng = 200, None + exprng = 200, None, [[process]] mask_cr = True noise_floor = 0.01 @@ -3933,7 +4293,7 @@ Alterations to the default parameters are: spectrograph = lbt_mods2b [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -3941,7 +4301,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -3957,20 +4317,20 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] use_pixelflat = False use_illumflat = False @@ -3989,12 +4349,12 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = 1, 200 + exprng = 1, 200, [[[process]]] mask_cr = True noise_floor = 0.01 [[wavelengths]] - lamps = XeI, KrI, ArI, HgI + lamps = XeI, KrI, ArI, HgI, sigdetect = 10.0 rms_threshold = 0.4 [[slitedges]] @@ -4006,7 +4366,7 @@ Alterations to the default parameters are: spec_order = 5 maxdev2d = 0.02 [scienceframe] - exprng = 200, None + exprng = 200, None, [[process]] mask_cr = True noise_floor = 0.01 @@ -4025,7 +4385,7 @@ Alterations to the default parameters are: spectrograph = lbt_mods2r [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -4033,7 +4393,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -4049,20 +4409,20 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] use_pixelflat = False use_illumflat = False @@ -4081,12 +4441,12 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = 1, 200 + exprng = 1, 200, [[[process]]] mask_cr = True noise_floor = 0.01 [[wavelengths]] - lamps = ArI, NeI, KrI, XeI + lamps = ArI, NeI, KrI, XeI, fwhm = 10.0 rms_threshold = 1.0 match_toler = 2.5 @@ -4100,7 +4460,7 @@ Alterations to the default parameters are: spec_order = 5 maxdev2d = 0.02 [scienceframe] - exprng = 200, None + exprng = 200, None, [[process]] mask_cr = True noise_floor = 0.01 @@ -4186,7 +4546,7 @@ Alterations to the default parameters are: slit_illum_finecorr = False [[wavelengths]] method = full_template - lamps = use_header + lamps = use_header, fwhm_fromlines = True n_first = 3 n_final = 5 @@ -4245,7 +4605,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] mask_cr = True use_biasimage = False @@ -4253,7 +4613,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] use_biasimage = False use_overscan = False @@ -4312,7 +4672,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[standardframe]] - exprng = None, 60 + exprng = None, 60, [[[process]]] mask_cr = True use_biasimage = False @@ -4324,13 +4684,13 @@ Alterations to the default parameters are: echelle = True ech_norder_coeff = 6 ech_sigrej = 3.0 - lamps = OH_FIRE_Echelle - sigdetect = 5, 10, 10, 10, 10, 20, 30, 30, 30, 30, 30, 10, 30, 30, 60, 30, 30, 10, 20, 30, 10 + lamps = OH_FIRE_Echelle, + sigdetect = 5, 10, 10, 10, 10, 20, 30, 30, 30, 30, 30, 10, 30, 30, 60, 30, 30, 10, 20, 30, 10, reid_arxiv = magellan_fire_echelle.fits cc_thresh = 0.35 rms_threshold = 1.0 match_toler = 30.0 - n_final = 3, 3, 3, 2, 4, 4, 4, 3, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 6, 6, 4 + n_final = 3, 3, 3, 2, 4, 4, 4, 3, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 6, 6, 4, [[slitedges]] edge_thresh = 3.0 max_shift_adj = 0.5 @@ -4341,7 +4701,7 @@ Alterations to the default parameters are: [[tilts]] tracethresh = 5 [scienceframe] - exprng = 20, None + exprng = 20, None, [[process]] satpix = nothing mask_cr = True @@ -4356,6 +4716,8 @@ Alterations to the default parameters are: maxnumber_std = 1 [[extraction]] model_full_slit = True + [coadd1d] + wave_method = log10 [sensfunc] algorithm = IR [[IR]] @@ -4381,7 +4743,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] mask_cr = True use_biasimage = False @@ -4389,7 +4751,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 1, 50 + exprng = 1, 50, [[[process]]] use_biasimage = False use_overscan = False @@ -4448,7 +4810,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[standardframe]] - exprng = None, 60 + exprng = None, 60, [[[process]]] mask_cr = True use_biasimage = False @@ -4457,7 +4819,7 @@ Alterations to the default parameters are: use_illumflat = False [[wavelengths]] method = full_template - lamps = ArI, ArII, ThAr, NeI + lamps = ArI, ArII, ThAr, NeI, sigdetect = 3 fwhm = 20 reid_arxiv = magellan_fire_long.fits @@ -4469,7 +4831,7 @@ Alterations to the default parameters are: [[tilts]] tracethresh = 5 [scienceframe] - exprng = 20, None + exprng = 20, None, [[process]] mask_cr = True use_biasimage = False @@ -4479,7 +4841,7 @@ Alterations to the default parameters are: [reduce] [[findobj]] snr_thresh = 5 - find_trim_edge = 50, 50 + find_trim_edge = 50, 50, [sensfunc] [[IR]] telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits @@ -4503,13 +4865,13 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] mask_cr = True use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] use_pixelflat = False use_illumflat = False @@ -4546,7 +4908,7 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 20 + exprng = None, 20, [[[process]]] mask_cr = True noise_floor = 0.01 @@ -4554,11 +4916,13 @@ Alterations to the default parameters are: method = reidentify echelle = True ech_sigrej = 3.0 - lamps = ThAr_MagE + lamps = ThAr_MagE, + fwhm = 3.0 + fwhm_fromlines = True reid_arxiv = magellan_mage.fits cc_thresh = 0.5 cc_local_thresh = 0.5 - rms_threshold = 0.2 + rms_threshold = 0.4 [[slitedges]] edge_thresh = 10.0 max_shift_adj = 3.0 @@ -4567,7 +4931,7 @@ Alterations to the default parameters are: [[tilts]] tracethresh = 10.0 [scienceframe] - exprng = 20, None + exprng = 20, None, [[process]] satpix = nothing mask_cr = True @@ -4575,16 +4939,108 @@ Alterations to the default parameters are: noise_floor = 0.01 [reduce] [[findobj]] - find_trim_edge = 4, 4 + find_trim_edge = 4, 4, maxnumber_sci = 2 maxnumber_std = 1 [[extraction]] model_full_slit = True + [coadd1d] + wave_method = log10 + +.. _instr_par-mdm_modspec: + +HILTNER Echelle (``mdm_modspec``) +--------------------------------- +Alterations to the default parameters are: + +.. code-block:: ini + + [rdx] + spectrograph = mdm_modspec + [calibrations] + [[biasframe]] + exprng = None, 0.001, + [[[process]]] + overscan_method = median + combine = median + use_biasimage = False + shot_noise = False + use_pixelflat = False + use_illumflat = False + [[darkframe]] + exprng = 999999, None, + [[[process]]] + mask_cr = True + use_pixelflat = False + use_illumflat = False + [[arcframe]] + [[[process]]] + clip = False + use_pixelflat = False + use_illumflat = False + subtract_continuum = True + [[tiltframe]] + [[[process]]] + clip = False + use_pixelflat = False + use_illumflat = False + subtract_continuum = True + [[pixelflatframe]] + [[[process]]] + satpix = nothing + n_lohi = 1, 1, + comb_sigrej = 3.0 + use_pixelflat = False + use_illumflat = False + [[pinholeframe]] + exprng = 999999, None, + [[alignframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[traceframe]] + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[illumflatframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[lampoffflatsframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[skyframe]] + [[[process]]] + mask_cr = True + noise_floor = 0.01 + [[standardframe]] + [[[process]]] + mask_cr = True + noise_floor = 0.01 + [[flatfield]] + slit_illum_finecorr = False + [[wavelengths]] + method = full_template + lamps = ArI, XeI, NeI, + reid_arxiv = mdm_modspec_1200_5100.fits + n_final = 9 + [[slitedges]] + sync_predict = nearest + bound_detector = True + [scienceframe] + exprng = 10, 600, + [[process]] + mask_cr = True + noise_floor = 0.01 .. _instr_par-mdm_osmos_mdm4k: -KPNO MDM4K (``mdm_osmos_mdm4k``) --------------------------------- +HILTNER MDM4K (``mdm_osmos_mdm4k``) +----------------------------------- Alterations to the default parameters are: .. code-block:: ini @@ -4593,7 +5049,7 @@ Alterations to the default parameters are: spectrograph = mdm_osmos_mdm4k [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -4601,7 +5057,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -4621,7 +5077,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing @@ -4646,19 +5102,19 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] mask_cr = True noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = ArI, XeI + lamps = ArI, XeI, sigdetect = 10.0 reid_arxiv = mdm_osmos_mdm4k.fits [[slitedges]] sync_predict = nearest [scienceframe] - exprng = 90, None + exprng = 90, None, [[process]] mask_cr = True noise_floor = 0.01 @@ -4682,14 +5138,14 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] mask_cr = True use_biasimage = False use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] use_biasimage = False use_pixelflat = False @@ -4737,14 +5193,14 @@ Alterations to the default parameters are: use_biasimage = False noise_floor = 0.01 [[standardframe]] - exprng = None, 100 + exprng = None, 100, [[[process]]] mask_cr = True use_biasimage = False noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = HeI, NeI, ArI, ArII + lamps = HeI, NeI, ArI, ArII, fwhm = 5.0 rms_threshold = 0.5 [[slitedges]] @@ -4754,7 +5210,7 @@ Alterations to the default parameters are: spat_order = 6 spec_order = 6 [scienceframe] - exprng = 20, None + exprng = 20, None, [[process]] mask_cr = True sigclip = 5.0 @@ -4791,14 +5247,14 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 300, None + exprng = 300, None, [[[process]]] mask_cr = True use_biasimage = False use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 1, None + exprng = 1, None, [[[process]]] use_biasimage = False use_pixelflat = False @@ -4809,7 +5265,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 600 + exprng = None, 600, [[[process]]] satpix = nothing use_biasimage = False @@ -4826,13 +5282,13 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 600 + exprng = None, 600, [[[process]]] use_biasimage = False use_pixelflat = False use_illumflat = False [[illumflatframe]] - exprng = 1, None + exprng = 1, None, [[[process]]] satpix = nothing use_biasimage = False @@ -4851,14 +5307,14 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[standardframe]] - exprng = None, 600 + exprng = None, 600, [[[process]]] mask_cr = True use_biasimage = False noise_floor = 0.01 use_illumflat = False [[wavelengths]] - lamps = use_header + lamps = use_header, fwhm_fromlines = True rms_threshold = 0.5 [[slitedges]] @@ -4899,7 +5355,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 30, None + exprng = 30, None, [[[process]]] mask_cr = True use_biasimage = False @@ -4907,14 +5363,14 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 60, None + exprng = 60, None, [[[process]]] use_biasimage = False use_overscan = False use_pixelflat = False use_illumflat = False [[tiltframe]] - exprng = 60, None + exprng = 60, None, [[[process]]] use_biasimage = False use_overscan = False @@ -4967,7 +5423,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[standardframe]] - exprng = None, 60 + exprng = None, 60, [[[process]]] mask_cr = True use_biasimage = False @@ -4975,7 +5431,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[wavelengths]] - lamps = OH_NIRES + lamps = OH_NIRES, fwhm = 5 rms_threshold = 0.5 match_toler = 5.0 @@ -4990,7 +5446,7 @@ Alterations to the default parameters are: spat_order = 7 spec_order = 5 [scienceframe] - exprng = 30, None + exprng = 30, None, [[process]] mask_cr = True grow = 0.5 @@ -5022,7 +5478,7 @@ Alterations to the default parameters are: spectrograph = not_alfosc [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 1, [[[process]]] combine = median use_biasimage = False @@ -5031,7 +5487,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_overscan = False @@ -5059,7 +5515,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] use_overscan = False [[alignframe]] @@ -5091,14 +5547,14 @@ Alterations to the default parameters are: use_overscan = False noise_floor = 0.01 [[standardframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] mask_cr = True use_overscan = False noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = HeI, NeI, ArI + lamps = HeI, NeI, ArI, sigdetect = 10.0 [[slitedges]] edge_thresh = 30 @@ -5106,7 +5562,7 @@ Alterations to the default parameters are: bound_detector = True minimum_slit_gap = 15 [scienceframe] - exprng = 10, None + exprng = 10, None, [[process]] mask_cr = True use_overscan = False @@ -5124,7 +5580,7 @@ Alterations to the default parameters are: spectrograph = not_alfosc_vert [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 1, [[[process]]] combine = median use_biasimage = False @@ -5133,7 +5589,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_overscan = False @@ -5161,7 +5617,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] use_overscan = False [[alignframe]] @@ -5193,14 +5649,14 @@ Alterations to the default parameters are: use_overscan = False noise_floor = 0.01 [[standardframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] mask_cr = True use_overscan = False noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = HeI, NeI, ArI + lamps = HeI, NeI, ArI, sigdetect = 10.0 [[slitedges]] edge_thresh = 30 @@ -5208,7 +5664,7 @@ Alterations to the default parameters are: bound_detector = True minimum_slit_gap = 15 [scienceframe] - exprng = 10, None + exprng = 10, None, [[process]] mask_cr = True use_overscan = False @@ -5281,7 +5737,7 @@ Alterations to the default parameters are: tweak_slits_thresh = 0.9 [[wavelengths]] method = full_template - lamps = HeI, ArI + lamps = HeI, ArI, sigdetect = 10.0 rms_threshold = 0.25 [[slitedges]] @@ -5314,7 +5770,7 @@ Alterations to the default parameters are: [calibrations] bpm_usebias = True [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -5322,13 +5778,13 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] use_pixelflat = False use_illumflat = False @@ -5343,7 +5799,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing @@ -5368,19 +5824,19 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] combine = median mask_cr = True noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = FeI, ArI, ArII + lamps = FeI, ArI, ArII, [[slitedges]] fit_min_spec_length = 0.55 sync_predict = nearest [scienceframe] - exprng = 90, None + exprng = 90, None, [[process]] combine = median mask_cr = True @@ -5402,7 +5858,7 @@ Alterations to the default parameters are: [calibrations] bpm_usebias = True [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -5410,13 +5866,13 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] use_pixelflat = False use_illumflat = False @@ -5431,7 +5887,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing @@ -5456,18 +5912,18 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] combine = median mask_cr = True noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = ArI, ArII, NeI, HeI + lamps = ArI, ArII, NeI, HeI, [[slitedges]] sync_predict = nearest [scienceframe] - exprng = 90, None + exprng = 90, None, [[process]] combine = median mask_cr = True @@ -5500,7 +5956,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] mask_cr = True use_biasimage = False @@ -5508,14 +5964,14 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 100, None + exprng = 100, None, [[[process]]] use_biasimage = False use_overscan = False use_pixelflat = False use_illumflat = False [[tiltframe]] - exprng = 100, None + exprng = 100, None, [[[process]]] use_biasimage = False use_overscan = False @@ -5568,7 +6024,7 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[standardframe]] - exprng = None, 60 + exprng = None, 60, [[[process]]] mask_cr = True use_biasimage = False @@ -5580,11 +6036,12 @@ Alterations to the default parameters are: echelle = True ech_norder_coeff = 6 ech_sigrej = 3.0 - lamps = OH_NIRES - fwhm = 5.0 + lamps = OH_NIRES, + fwhm = 2.9 + fwhm_fromlines = True reid_arxiv = p200_triplespec.fits rms_threshold = 0.3 - n_final = 3, 4, 4, 4, 4 + n_final = 3, 4, 4, 4, 4, [[slitedges]] fit_min_spec_length = 0.3 left_right_pca = True @@ -5593,7 +6050,7 @@ Alterations to the default parameters are: [[tilts]] tracethresh = 10.0 [scienceframe] - exprng = 60, None + exprng = 60, None, [[process]] satpix = nothing mask_cr = True @@ -5611,6 +6068,8 @@ Alterations to the default parameters are: [[extraction]] boxcar_radius = 0.75 model_full_slit = True + [coadd1d] + wave_method = log10 [sensfunc] algorithm = IR polyorder = 8 @@ -5629,7 +6088,7 @@ Alterations to the default parameters are: spectrograph = shane_kast_blue [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -5637,13 +6096,13 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = None, 61 + exprng = None, 61, [[[process]]] use_pixelflat = False use_illumflat = False @@ -5652,20 +6111,20 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] use_pixelflat = False use_illumflat = False @@ -5684,13 +6143,13 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = 1, 61 + exprng = 1, 61, [[[process]]] mask_cr = True noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = CdI, HgI, HeI + lamps = CdI, HgI, HeI, rms_threshold = 0.2 match_toler = 2.5 n_first = 3 @@ -5702,13 +6161,16 @@ Alterations to the default parameters are: spec_order = 5 maxdev2d = 0.02 [scienceframe] - exprng = 61, None + exprng = 61, None, [[process]] mask_cr = True noise_floor = 0.01 [flexure] spec_method = boxcar spectrum = sky_kastb_600.fits + [sensfunc] + [[IR]] + telgridfile = TelFit_Lick_3100_11100_R10000.fits .. _instr_par-shane_kast_red: @@ -5722,7 +6184,7 @@ Alterations to the default parameters are: spectrograph = shane_kast_red [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -5730,13 +6192,13 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = None, 61 + exprng = None, 61, [[[process]]] use_pixelflat = False use_illumflat = False @@ -5745,20 +6207,20 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] use_pixelflat = False use_illumflat = False @@ -5777,17 +6239,17 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = 1, 61 + exprng = 1, 61, [[[process]]] mask_cr = True noise_floor = 0.01 [[wavelengths]] - lamps = NeI, HgI, HeI, ArI + lamps = NeI, HgI, HeI, ArI, [[slitedges]] sync_predict = nearest bound_detector = True [scienceframe] - exprng = 61, None + exprng = 61, None, [[process]] mask_cr = True noise_floor = 0.01 @@ -5809,7 +6271,7 @@ Alterations to the default parameters are: spectrograph = shane_kast_red_ret [calibrations] [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -5817,13 +6279,13 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = None, 61 + exprng = None, 61, [[[process]]] use_pixelflat = False use_illumflat = False @@ -5832,20 +6294,20 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = 0, None + exprng = 0, None, [[[process]]] use_pixelflat = False use_illumflat = False @@ -5864,24 +6326,27 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = 1, 61 + exprng = 1, 61, [[[process]]] mask_cr = True noise_floor = 0.01 [[wavelengths]] - lamps = NeI, HgI, HeI, ArI + lamps = NeI, HgI, HeI, ArI, rms_threshold = 0.2 use_instr_flag = True [[slitedges]] sync_predict = nearest bound_detector = True [scienceframe] - exprng = 61, None + exprng = 61, None, [[process]] mask_cr = True noise_floor = 0.01 [flexure] spec_method = boxcar + [sensfunc] + [[IR]] + telgridfile = TelFit_Lick_3100_11100_R10000.fits .. _instr_par-soar_goodman_blue: @@ -5908,7 +6373,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = None, 30 + exprng = None, 30, [[[process]]] use_biasimage = False use_pixelflat = False @@ -5956,20 +6421,20 @@ Alterations to the default parameters are: use_biasimage = False noise_floor = 0.01 [[standardframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] mask_cr = True use_biasimage = False noise_floor = 0.01 [[wavelengths]] - lamps = NeI, ArI, HgI + lamps = NeI, ArI, HgI, fwhm = 5.0 rms_threshold = 0.5 [[slitedges]] sync_predict = nearest bound_detector = True [scienceframe] - exprng = 90, None + exprng = 90, None, [[process]] mask_cr = True use_biasimage = False @@ -6005,7 +6470,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = None, 30 + exprng = None, 30, [[[process]]] use_biasimage = False use_pixelflat = False @@ -6053,7 +6518,7 @@ Alterations to the default parameters are: use_biasimage = False noise_floor = 0.01 [[standardframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] mask_cr = True use_biasimage = False @@ -6061,14 +6526,14 @@ Alterations to the default parameters are: [[flatfield]] slit_illum_finecorr = False [[wavelengths]] - lamps = NeI, ArI, HgI + lamps = NeI, ArI, HgI, fwhm = 5.0 rms_threshold = 0.5 [[slitedges]] sync_predict = nearest bound_detector = True [scienceframe] - exprng = 90, None + exprng = 90, None, [[process]] mask_cr = True use_biasimage = False @@ -6091,7 +6556,7 @@ Alterations to the default parameters are: spectrograph = tng_dolores [calibrations] [[biasframe]] - exprng = None, 0.1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -6099,7 +6564,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_pixelflat = False @@ -6122,7 +6587,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[alignframe]] [[[process]]] satpix = nothing @@ -6153,7 +6618,7 @@ Alterations to the default parameters are: [[slitedges]] sync_predict = nearest [scienceframe] - exprng = 1, None + exprng = 1, None, [[process]] mask_cr = True noise_floor = 0.01 @@ -6238,7 +6703,7 @@ Alterations to the default parameters are: [[flatfield]] tweak_slits_thresh = 0.9 [[wavelengths]] - lamps = HeI, ArI + lamps = HeI, ArI, sigdetect = 10.0 rms_threshold = 0.25 [[slitedges]] @@ -6278,7 +6743,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] mask_cr = True use_biasimage = False @@ -6286,7 +6751,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = 20, None + exprng = 20, None, [[[process]]] mask_cr = True sigclip = 20.0 @@ -6348,7 +6813,7 @@ Alterations to the default parameters are: use_overscan = False noise_floor = 0.01 [[standardframe]] - exprng = None, 20 + exprng = None, 20, [[[process]]] mask_cr = True use_biasimage = False @@ -6356,7 +6821,7 @@ Alterations to the default parameters are: noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = OH_FIRE_Echelle + lamps = OH_FIRE_Echelle, fwhm = 5.0 reid_arxiv = vlt_sinfoni_K.fits rms_threshold = 0.3 @@ -6368,7 +6833,7 @@ Alterations to the default parameters are: [[tilts]] tracethresh = 5.0 [scienceframe] - exprng = 20, None + exprng = 20, None, [[process]] satpix = nothing mask_cr = True @@ -6490,13 +6955,14 @@ Alterations to the default parameters are: ech_nspec_coeff = 5 ech_norder_coeff = 5 ech_sigrej = 3.0 - lamps = OH_XSHOOTER + lamps = OH_XSHOOTER, sigdetect = 10.0 fwhm = 5.0 reid_arxiv = vlt_xshooter_nir.fits cc_thresh = 0.5 cc_local_thresh = 0.5 - rms_threshold = 0.25 + rms_threshold = 0.6 + qa_log = False [[slitedges]] edge_thresh = 50.0 max_shift_adj = 0.5 @@ -6529,6 +6995,8 @@ Alterations to the default parameters are: global_sky_std = False [[extraction]] model_full_slit = True + [coadd1d] + wave_method = log10 [sensfunc] algorithm = IR polyorder = 8 @@ -6627,17 +7095,21 @@ Alterations to the default parameters are: mask_cr = True use_biasimage = False noise_floor = 0.01 + [[flatfield]] + tweak_slits_thresh = 0.9 [[wavelengths]] method = reidentify echelle = True ech_sigrej = 3.0 - lamps = ThAr_XSHOOTER_UVB + lamps = ThAr_XSHOOTER_UVB, sigdetect = 3.0 + fwhm = 3.8 + fwhm_fromlines = True reid_arxiv = vlt_xshooter_uvb1x1.fits cc_thresh = 0.5 cc_local_thresh = 0.5 - rms_threshold = 0.6 - n_final = 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 + rms_threshold = 0.7 + n_final = 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, [[slitedges]] edge_thresh = 8.0 max_shift_adj = 0.5 @@ -6651,7 +7123,7 @@ Alterations to the default parameters are: noise_floor = 0.01 [reduce] [[findobj]] - find_trim_edge = 3, 3 + find_trim_edge = 3, 3, maxnumber_sci = 2 maxnumber_std = 1 [[skysub]] @@ -6659,6 +7131,13 @@ Alterations to the default parameters are: global_sky_std = False [[extraction]] model_full_slit = True + [coadd1d] + wave_method = log10 + [sensfunc] + algorithm = IR + polyorder = 8 + [[IR]] + telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits .. _instr_par-vlt_xshooter_vis: @@ -6758,13 +7237,14 @@ Alterations to the default parameters are: method = reidentify echelle = True ech_sigrej = 3.0 - lamps = ThAr_XSHOOTER_VIS - fwhm = 11.0 + lamps = ThAr_XSHOOTER_VIS, + fwhm = 8.0 + fwhm_fromlines = True reid_arxiv = vlt_xshooter_vis1x1.fits cc_thresh = 0.5 cc_local_thresh = 0.5 - rms_threshold = 0.5 - n_final = 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3 + rms_threshold = 1.2 + n_final = 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, [[slitedges]] edge_thresh = 8.0 max_shift_adj = 0.5 @@ -6782,7 +7262,7 @@ Alterations to the default parameters are: noise_floor = 0.01 [reduce] [[findobj]] - find_trim_edge = 3, 3 + find_trim_edge = 3, 3, maxnumber_sci = 2 maxnumber_std = 1 [[skysub]] @@ -6790,9 +7270,11 @@ Alterations to the default parameters are: global_sky_std = False [[extraction]] model_full_slit = True + [coadd1d] + wave_method = log10 [sensfunc] algorithm = IR - polyorder = 9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7 + polyorder = 8 [[IR]] telgridfile = TelFit_Paranal_VIS_4900_11100_R25000.fits @@ -6809,7 +7291,7 @@ Alterations to the default parameters are: [calibrations] bpm_usebias = True [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -6818,14 +7300,14 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_overscan = False use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] use_overscan = False use_pixelflat = False @@ -6843,7 +7325,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] use_overscan = False [[alignframe]] @@ -6875,21 +7357,21 @@ Alterations to the default parameters are: use_overscan = False noise_floor = 0.01 [[standardframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] mask_cr = True use_overscan = False noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = NeI, ArI, ArII, CuI + lamps = NeI, ArI, ArII, CuI, sigdetect = 10.0 n_first = 3 n_final = 5 [[slitedges]] sync_predict = nearest [scienceframe] - exprng = 90, None + exprng = 90, None, [[process]] mask_cr = True use_overscan = False @@ -6908,7 +7390,7 @@ Alterations to the default parameters are: [calibrations] bpm_usebias = True [[biasframe]] - exprng = None, 1 + exprng = None, 0.001, [[[process]]] combine = median use_biasimage = False @@ -6917,14 +7399,14 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[darkframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] mask_cr = True use_overscan = False use_pixelflat = False use_illumflat = False [[arcframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] use_overscan = False use_pixelflat = False @@ -6942,7 +7424,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] - exprng = 999999, None + exprng = 999999, None, [[[process]]] use_overscan = False [[alignframe]] @@ -6974,19 +7456,19 @@ Alterations to the default parameters are: use_overscan = False noise_floor = 0.01 [[standardframe]] - exprng = None, 120 + exprng = None, 120, [[[process]]] mask_cr = True use_overscan = False noise_floor = 0.01 [[wavelengths]] method = full_template - lamps = NeI, ArI, ArII, CuI + lamps = NeI, ArI, ArII, CuI, sigdetect = 10.0 [[slitedges]] sync_predict = nearest [scienceframe] - exprng = 90, None + exprng = 90, None, [[process]] mask_cr = True use_overscan = False diff --git a/doc/releases/1.13.0.rst b/doc/releases/1.13.0.rst index 78b4da1267..84bf7775fc 100644 --- a/doc/releases/1.13.0.rst +++ b/doc/releases/1.13.0.rst @@ -2,8 +2,8 @@ Version 1.13.0 ============== -Funtionality/Performance Improvements and Additions ---------------------------------------------------- +Functionality/Performance Improvements and Additions +---------------------------------------------------- - Allow user control of the local sky subtraction window - Implemented a resample algorithm when generating datacubes diff --git a/doc/releases/1.14.0.rst b/doc/releases/1.14.0.rst new file mode 100644 index 0000000000..ecd0eafd0f --- /dev/null +++ b/doc/releases/1.14.0.rst @@ -0,0 +1,100 @@ + +Version 1.14.0 +============== + +Dependency Changes +------------------ + +- Main dependency bumps: numpy>=1.22, matplotlib>=3.7, ginga>=4.1.1, + qtpy>=2.0.1 + +Functionality/Performance Improvements and Additions +---------------------------------------------------- + +- Improvements to wavelength grids and masking in coadd routines. +- Refactored coadding routines to work with lists to support coadding + data from different setups. +- Sensitivity function models can now be computed relative to the + flat-field spectrum. +- Improvements in 2D coaddition + + - Fix a bug in ``pypeit_setup_coadd2d`` for the output file name of + the ``.coadd2d`` file + - Added possibility to specify more than one Science folder in + ``pypeit_setup_coadd2d`` + - Now ``only_slits`` parameter in ``pypeit_coadd_2dspec`` includes + the detector number (similar to ``slitspatnum``) + - Added ``exclude_slits`` parameter in ``pypeit_coadd_2dspec`` to + exclude specific slits + - Fix wrong RA and Dec for 2D coadded serendips + +- Allow wavelength calibrations for specific slits/orders to be redone + (instead of adopting the value from a processed calibration frame); + see new ``redo_slits`` parameter. + +Instrument-specific Updates +--------------------------- + +- Adds/Improves support for Gemini/GNIRS (IFU), Keck/KCRM, Keck/ESI, + MDM/Modspec, Keck/HIRES, JWST +- HIRES wavelength solution improvements galore +- Improvements for Keck/LRIS + + - Generated wavelength templates for all the LRIS grism & grating + - Added FeAr line list + - Improved calibration association and frame typing + - Improved and added documentation + - Changes to ``metadata.py`` including commenting out, in the pypeit + file, files that have frametype None (this prevent ``run_pypeit`` + to crash) + - Added a function ``check_spectrograph()`` (currently only defined + for LRIS), that checks (during ``pypeit_setup``) if the selected + spectrograph is the corrected one for the data used. + +Script Changes +-------------- + +- Added a script to convert a wavelength solution into something that + can be placed in the reid archive. +- Store user-generated wavelength solution in pypeit cache + +Datamodel Changes +----------------- + +- Changed calibration frame naming as an attempt to avoid very long + names for files with many calibration groups. Sequential numbers are + reduced to a range; e.g., ``'0-1-2-3-4'`` becomes ``'0+4'`` and + ``'3-5-6-10-11-12-15-18-19'`` becomes ``'3-5+6-10+12-15-18+19'`` +- Instrumental FWHM map is calculated and output in ``Calibrations`` and + ``spec1d`` files. + +Under-the-hood Improvements +--------------------------- + +- Change how masking is dealt with in extraction to fix a bug in how + masks were being treated for echelle data +- Refactored function that loads wavelength calibration line lists + +Bug Fixes +--------- + +- Hotfix for GTC/OSIRIS lamp list +- Hotfix for Arc1D stats annotations on the QA +- Hotfix for metadata: + + - correctly set ``config_independent_frames`` when multiple + configurations are being setup + - support lists in ``config_independent_frames`` + +- Hotfix for rebin (speed-up and conserves flux) +- Hotfix for skysub regions GUI that used ``np.bool`` +- Hotfix to stop pypeit_setup from crashing on data from lbt_luci1, + lbt_luci2, magellan_fire, magellan_fire_long, p200_tspec, or + vlt_sinfoni. +- Hotfix to set BPM for each type of calibration file. +- Fixed a bug in echelle coadding where the wrong coadded spectra were + being used in final stacks. +- Fix a bug in ``spectrograph.select_detectors``, where a list of + ``slitspatnum`` could not be used. + + diff --git a/doc/scripts.rst b/doc/scripts.rst index 0b0a72cefd..f7ba29a9f8 100644 --- a/doc/scripts.rst +++ b/doc/scripts.rst @@ -269,21 +269,31 @@ The script usage can be displayed by calling the script with the pypeit_flux_setup ----------------- -.. TODO: Can someone please check/edit this? - Once you have a set of 1D object spectra and a sensitivity function, this script -helps you setup the necessary input file to perform the flux calibration and -telluric correction. See :ref:`fluxing` (specifically, +helps you create the necessary input file to perform the flux calibration, 1d coadding, +and telluric correction. See :ref:`fluxing` (specifically, :ref:`apply_fluxcal`), :doc:`coadd1d`, and :doc:`telluric` for details. Note you will need to hand edit the files generated by this script: - - Give sensfunc file name in the fluxing pypeit file - - Give sensfunc file name in the coadding pypeit file - - The coadding pypeit file includes all objects extracted from - your main reduction, so you need to pick up the one you are - interested in and remove all others in the coadding pypeit file - (between coadd1d read and coadd1d end) + - Double check the fluxing pypeit file (ending in ``.flux``) to make sure + that the correct sensitivity function files were found by the script, and + were matched with the right spec1d files. This is in the section between + ``flux read`` and ``flux end``. + + - Remove unwanted spec1d files from the fluxing file. + + - The coadding pypeit file (ending in ``.coadd1d``) includes all objects + extracted from your main reduction, so you need to pick the ones you are + interested in and remove all others in the coadding pypeit file (between + ``coadd1d read`` and ``coadd1d end``). + + - For echelle spectrographs, double check that the coadding pypeit file has + the sensitivity function files matched to the correct spec1d files, and + that the files have been correctly separated into different setups. + + - Add any additional configuration parameters if needed; see + :doc:`pypeit_par`. The script usage can be displayed by calling the script with the ``-h`` option: diff --git a/doc/scripts/base_par.rst b/doc/scripts/base_par.rst index e5fb84b0fe..810ba81ed6 100644 --- a/doc/scripts/base_par.rst +++ b/doc/scripts/base_par.rst @@ -54,7 +54,7 @@ the precedence order is as follows: configurations via its ``config_specific_par`` method. This allows the code to automatically define, e.g., the archived arc spectrum used for wavelength calibration given the grating used. For example, see - :func:`~pypeit.spectrographs.shane_kast.ShaneKastSpectrograph.config_specific_par` + :func:`~pypeit.spectrographs.shane_kast.ShaneKastBlueSpectrograph.config_specific_par` for Shane/Kast. These configuration-specific parameters are currently not documented here; however, they can be viewed by looking at the source code display in the API documentation. diff --git a/doc/scripts/build_afterburn_datamodels_rst.py b/doc/scripts/build_afterburn_datamodels_rst.py index d7a5e90b7c..86d33487e1 100644 --- a/doc/scripts/build_afterburn_datamodels_rst.py +++ b/doc/scripts/build_afterburn_datamodels_rst.py @@ -90,7 +90,7 @@ def sens_datamodel(output_root): for i,key in enumerate(telluric.keys()): tell_table[i+1,:] = [f'``{key}``', column_type(telluric[key]), telluric[key].description] - sens = SensFunc.empty_sensfunc_table(1, 1) + sens = SensFunc.empty_sensfunc_table(1, 1, 1) ncol = len(sens.keys()) sens_table = numpy.empty((ncol+1, 3), dtype=object) sens_table[0,:] = ['Column', 'Data Type', 'Description'] diff --git a/doc/scripts/make_example_files.py b/doc/scripts/make_example_files.py index 015381c8e9..c39996e411 100644 --- a/doc/scripts/make_example_files.py +++ b/doc/scripts/make_example_files.py @@ -75,17 +75,18 @@ def make_example_gnirs_pypeit_files(version, date): oroot = Path(resource_filename('pypeit', '')).resolve().parent / 'doc' / 'include' # Create the default pypeit file - droot = Path(os.getenv('PYPEIT_DEV')).resolve() / 'RAW_DATA' / 'gemini_gnirs' / '32_SB_SXD' + droot = Path(os.getenv('PYPEIT_DEV')).resolve() / 'RAW_DATA' / 'gemini_gnirs_echelle' \ + / '32_SB_SXD' - pargs = setup.Setup.parse_args(['-r', str(droot), '-s', 'gemini_gnirs', '-b', '-c', 'A', - '-d', str(oroot), + pargs = setup.Setup.parse_args(['-r', str(droot), '-s', 'gemini_gnirs_echelle', '-b', + '-c', 'A', '-d', str(oroot), '--version_override', version, '--date_override', date]) setup.Setup.main(pargs) - ofile = oroot / 'gemini_gnirs_A.pypeit.rst' + ofile = oroot / 'gemini_gnirs_echelle_A.pypeit.rst' with open(ofile, 'w') as f: - with open(oroot / 'gemini_gnirs_A' / 'gemini_gnirs_A.pypeit', 'r') as p: + with open(oroot / 'gemini_gnirs_echelle_A' / 'gemini_gnirs_echelle_A.pypeit', 'r') as p: lines = p.readlines() f.write('.. code-block:: console\n') f.write('\n') @@ -93,13 +94,13 @@ def make_example_gnirs_pypeit_files(version, date): f.write(' '+l) f.write('\n\n') - shutil.rmtree(oroot / 'gemini_gnirs_A') + shutil.rmtree(oroot / 'gemini_gnirs_echelle_A') # Copy over the one that is actually used by the dev-suite dev = Path(os.getenv('PYPEIT_DEV')).resolve() \ - / 'pypeit_files' / 'gemini_gnirs_32_sb_sxd.pypeit' + / 'pypeit_files' / 'gemini_gnirs_echelle_32_sb_sxd.pypeit' - ofile = oroot / 'gemini_gnirs_A_corrected.pypeit.rst' + ofile = oroot / 'gemini_gnirs_echelle_A_corrected.pypeit.rst' with open(ofile, 'w') as f: with open(dev, 'r') as p: lines = p.readlines() diff --git a/doc/setup.rst b/doc/setup.rst index 904e700e4c..2a046b17e5 100644 --- a/doc/setup.rst +++ b/doc/setup.rst @@ -216,7 +216,7 @@ this setup as determined using the relevant metadata. Each "setup block" is followed by a table listing the files and relevant metadata for all files matched to that instrument configuration. The data provided is specific to each instrument, as defined by, e.g., -:func:`~pypeit.spectrographs.keck_deimos.pypeit_file_keys`. +:func:`~pypeit.spectrographs.keck_deimos.KeckDEIMOSSpectrograph.pypeit_file_keys`. The ``sorted`` file is only provided as a means of assessing the automated setup identification and file sorting, and we encourage you diff --git a/doc/spectrographs/deimos.rst b/doc/spectrographs/deimos.rst index 332182ae25..a6e4d8bc81 100644 --- a/doc/spectrographs/deimos.rst +++ b/doc/spectrographs/deimos.rst @@ -101,8 +101,8 @@ Slit-mask design matching PypeIt is able to match the traced slit to the slit-mask design information contained as meta data in the DEIMOS observations. This functionality at the moment is -implemented only for DEIMOS and MOSFIRE and is switched on by setting ``use_maskdesign`` flag in -:ref:`edgetracepar` to True. This is, already, the default for DEIMOS, +implemented only for these :ref:`slitmask_info_instruments` and is switched on by setting +``use_maskdesign`` flag in :ref:`edgetracepar` to True. This is, already, the default for DEIMOS, except when the ``LongMirr`` or the ``LVM`` mask is used. PypeIt also assigns to each extracted 1D spectrum the corresponding RA, Dec and object name diff --git a/doc/spectrographs/gemini_gmos.rst b/doc/spectrographs/gemini_gmos.rst index bcf44a1b72..4a64176f10 100644 --- a/doc/spectrographs/gemini_gmos.rst +++ b/doc/spectrographs/gemini_gmos.rst @@ -163,4 +163,5 @@ The modifications to the :ref:`pypeit_file` will look like: The two files provided must be located either: (1) in the path(s) of the raw files provided in the :ref:`data_block`, (2) the current working data, and/or - (3) be named with the full path. \ No newline at end of file + (3) be named with the full path. + diff --git a/doc/spectrographs/keck_hires.rst b/doc/spectrographs/keck_hires.rst new file mode 100644 index 0000000000..f8d3277554 --- /dev/null +++ b/doc/spectrographs/keck_hires.rst @@ -0,0 +1,18 @@ +========== +Keck HIRES +========== + +Overview +======== + +This file summarizes several instrument specific settings that are related to the Keck/HIRES spectrograph. + + +Wavelengths +=========== + +See :ref:`wvcalib-echelle` for details on the wavelength calibration. + +We also note that several Orders from 40-45 are +frequently flagged as bad in the wavelength solution. +This is due, in part, to very bright ThAr line contamination. \ No newline at end of file diff --git a/doc/spectrographs/keck_kcwi.rst b/doc/spectrographs/keck_kcwi.rst index 72bbbfb735..c4850dadff 100644 --- a/doc/spectrographs/keck_kcwi.rst +++ b/doc/spectrographs/keck_kcwi.rst @@ -161,7 +161,14 @@ Sky subtraction --------------- See :doc:`../skysub` for useful hints to define the sky regions -using an interactive GUI. +using an interactive GUI. You can use the joint_fit parameter (see above) +to jointly fit the sky in all slits (and compute the relative spectral +sensitivity variation for each slice). However, note that some modes of +KCWI and KCRM have significant variation of the instrument FWHM across +the field of view. The current implementation of this joint sky subtraction +does not account for the variation of the FWHM across the field of view. +This will be addressed in the future (refer to Issue #1660 for any updates +regarding this). Flexure corrections ------------------- diff --git a/doc/spectrographs/lris.rst b/doc/spectrographs/lris.rst index 9660cd7949..46e9beca35 100644 --- a/doc/spectrographs/lris.rst +++ b/doc/spectrographs/lris.rst @@ -7,10 +7,127 @@ Overview ======== This file summarizes several instrument specific -settings that are related to the Keck/LRIS spectrograph. +settings that are related to the Keck/LRIS (RED and BLUE) spectrograph. -Common Items -============ +Common Items to LRISb and LRISr +=============================== + +.. _lris_FITS_format: + +FITS format ++++++++++++ + +Before May 2009, both LRISb and LRISr observations were stored in standard simple +`FITS `__ format consisting of a +single primary HDU without any extensions. Subsequently, the data were stored in +multi-extension FITS files, including four extensions, one for each amplifier. +To handle both formats, PypeIt defined 2 sets of spectrographs for each LRIS +camera, one for the pre-May 2009 data (``keck_lris_blue_orig`` and ``keck_lris_red_orig``) +and one for the post-May 2009 data (``keck_lris_blue`` and ``keck_lris_red``). +Note that this change in FITS format coincided with the installation of the LBNL +detectors (2kx4k) in LRISr (see :ref:`keck-lris-red`). + +Slit-masks +++++++++++ + +PypeIt can now incorporate slitmask information in the reduction +routine for ``keck_lris_red``, ``keck_lris_red_mark4``, and ``keck_lris_blue`` +similar to its DEIMOS capabilities (see :ref:`deimos-mask-matching`). +That is, if the trace calibration files with mask information are used, PypeIt is +capable of using said information to match the traced slit edges to those predicted +by the slitmask design, assign to each extracted spectrum the corresponding RA, Dec and +object name information, force the extraction of undetected objects at the location +expected from the slitmask design, identify serendipitous sources and, subsequently, +collate by RA/Dec. See `Additional Reading`_ for more information. + +Unfortunately, LRIS raw frames do not come ready with slitmask +data and thus this information needs to be inserted by the user before +processing with PypeIt if they are desirous of incorporating +the above mentioned features into their reduction. To do so, the user +must first obtain the mask design files and then process them with +the software `TILSOTUA `__. +Here are the steps to follow: + +#. Obtain the mask design files, which include: + + #. Output files produced by `autoslit `__. + + - One file with the ``".file3"`` extension containing milling information. + + - One file with the ``".file1"`` extension containing the object catalog matched to the slitmask slits. + + #. The ASCII object list file fed as input to + `autoslit `__ to generate the files above. + + .. note:: + + ``".file3"`` is mandatory while the other two files can be optionally excluded to + debug `TILSOTUA `__. + +#. Process the design files with `TILSOTUA `__ : + The design files contain the milling blueprint (the ``BluSlits`` table). + When using the ``".file3"`` design file, TILSOTUA creates a FITS file with the ``BluSlits`` table + following the UCO/Lick template. If the ``".file1"`` file and the object list are provided, + the FITS mask design file will also includes the ``DesiSlits``, ``ObjectCat`` and ``SlitObjMap`` + binary tables, otherwise they will be empty. These tables include information on the + slitmask design, the object catalog and the mapping between the two, similar to the + binary tables in DEIMOS raw frames. TILSOTUA populates these tables using its ``xytowcs`` + function (in `LRIS_Mask_Coords_to_WCS.py + `__). + This function can be run by providing two parameters: + + - the ``input_file_name``, which is either the FITS or ``".file3"`` mask + design file (be sure the name includes the extension); + + - the ``output_file``, which is the name of the output file that + TILSOTUA will generate. Do not include any extension such as ``.fits``. + + If only the ``".file3"`` file is provided, the calling sequence is: + + .. code-block:: python + + from tilsotua import xytowcs + + xytowcs(input_file_name="yourmaskname.file3",output_file="yourmaskname_output") + + Although the other parameters are optional for `xytowcs` as a standalone code, users interested in applying the slitmask information to their PypeIt reduction **must provide the `obj_file` and `file1` files to ensure that object names are assigned to the extracted spectra**. + + If the ``".file1"`` file and the object list are provided, the calling sequence is: + + .. code-block:: python + + from tilsotua import xytowcs + + xytowcs(input_file_name="yourmaskname.file3",output_file="yourmaskname_output", + obj_file="yourtargets.obj", file1="yourmaskname.file1") + + It is assumed that the entries in ``file1`` and ``obj_file`` have unique ``Name`` values, i.e., make + sure you have a unique identifier for each object. Without this, it is not possible to correctly + reconcile the two tables. + +#. Add the TILSOTUA-generated slitmask design information to your raw trace FITS files: + The user must first verify that TILSOTUA has indeed processed the files correctly. This implies: + + - TILSOTUA has correctly identified the alignment stars (see the QA plot it generates). + + - TILSOTUA has estimated the values of the ``TopDist`` and ``BotDist`` columns in the ``SlitObjMap`` table correctly. + + Once satisfied with the processed FITS file from TILSOTUA, the user can append the binary tables + populated by TILSOTUA to the LRIS trace FITS files as additional HDUs, e.g.: + + .. code-block:: python + + from astropy.io import fits + + tracehdus = fits.open("trace_rawframe.fits") + autoslithdus = fits.open("yourmaskname_output.fits") + + for hdu in autoslithdus[1:]: + tracehdus.append(hdu) + tracehdus.writeto("trace_with_maskinfo.fits") + +If processed correctly, PypeIt should now fully utilize its arsenal of slitmask processing tools +to reduce and coadd spectra with the WCS information incorporated. Flexure +++++++ @@ -24,32 +141,65 @@ to turn either of these off. .. _lrisb: -keck_lris_blue -============== +LRIS BLUE +========= + +This section provides information on both ``keck_lris_blue_orig`` and ``keck_lris_blue``. +When not specified, the information applies to both. + +Default Settings +++++++++++++++++ -LRISb Default Settings -++++++++++++++++++++++ +See :ref:`instr_par-keck_lris_blue` and :ref:`instr_par-keck_lris_blue_orig` for +a listing of modifications to the default settings. +*You do not have to add these changes to your PypeIt reduction file!* This is just a listing of +how the parameters used for LRISb differ from the defaults listed in the preceding tables on that page. +Moreover, additional modifications may have been made for specific setups, e.g, for different grisms, +or different slitmasks, etc. You can see a list of all the used parameters in the ``keck_lris_blue_XXX.par`` +file generated by PypeIt at the beginning of the reduction. -See :ref:`instr_par-keck_lris_blue` for -a listing of modifications to the default settings. *You do not have to add these changes to -your PypeIt reduction file!* This is just a listing of how the parameters used -for Keck/LRIS differ from the defaults listed in the preceding tables on -that page. +Calibrations +++++++++++++ -Taking Calibrations for LRISb -+++++++++++++++++++++++++++++ +Wavelength calibration +^^^^^^^^^^^^^^^^^^^^^^ Arcs ---- -We recommend that you turn on *all* of the standard -arc lamps, including those slated for the red side. +We recommend that you turn on *most* of the standard +arc lamps, including those slated for the red side. These are:: Ne,Ar,Cd,Kr,Xe,Zn,Hg -The archived solutions expect all of these lamps. +The archived solutions expect most of these lamps. FeAr lamp can also be used, +since a FeAr line list is available for reduction process, although it appears +to be less useful to obtain good wavelength solutions. + +Wavelength Solution +------------------- + +As default, the wavelength calibration is performed using the :ref:`wvcalib-fulltemplate` +algorithm. The templates are created in the same way as done for Keck/DEIMOS +(see :ref:`deimos_wavecalib`) and kept in the ``data/arc_lines/reid_arxiv`` directory. +There are four templates, one per each LRISb grism: + +=========== ======================================================= + GRISM template +=========== ======================================================= + 300/5000 keck_lris_blue_B300_5000_d680_ArCdHgKrNeXeZnFeAr.fits + 400/3400 keck_lris_blue_B400_3400_d560_ArCdHgNeZnFeAr.fits + 600/4000 keck_lris_blue_B600_4000_d560_ArCdHgKrNeXeZn.fits + 1200/3400 keck_lris_blue_B1200_3400_d560_ArCdHgNeZn.fits +=========== ======================================================= + +PypeIt will automatically choose the right template according to the specific dataset. +These templates work for both ``keck_lris_blue_orig`` and ``keck_lris_blue``. + +Flat Fielding +^^^^^^^^^^^^^ Pixel Flat ---------- @@ -84,11 +234,14 @@ of sunset/sunrise. Internal/dome flats are likely to be too faint in the very blue. -.. _400-3400-grism: +.. TODO: Can somebody comment on this. This warning says that the internal flats + are too faint in the very blue for tracing, while the warning above says that they may be + too bright for pixelflat. Is it true? +.. _400-3400-grism: 400/3400 grism -++++++++++++++ +************** If you are using this grism, you are likely aware there are strong ghosts. We have found these complicate edge tracing @@ -104,29 +257,86 @@ need to increase the ``edge_thresh`` parameter to .. _keck-lris-red: -keck_lris_red -============= +LRIS RED +======== + +This section provides information on ``keck_lris_red_orig``, ``keck_lris_red``, +and ``keck_lris_red_mark4``. When not specified, the information applies to all. + +.. _lrisr-detectors: Detectors +++++++++ -There have been 3 (or is it 4?!) generations of detectors -in the LRISr camera. In PypeIt parlance, the original is named ``keck_lris_red_orig``, -the LBNL detectors (2kx4k) are ``keck_lris_red``, and the newest -Mark4 detector is ``keck_lris_red_mark4``. +There have been 3 (or is it 4?!) generations of detectors in the LRISr camera. +The first detector change (from Tektronik 2kx2k to LBNL 2kx4k) happened in May 2009, +concurrently with the change in the FITS file format (see :ref:`lris_FITS_format`). +To reduce the data taken before May 2009, the user should use the +``keck_lris_red_orig`` spectrograph. For data taken with the LBNL detectors, +after May 2009, the user should use the ``keck_lris_red`` spectrograph. +The newest detector was installed around May 2022 and is referred to as +the Mark4 detector. To reduce the data taken with this detector, +the user should use the ``keck_lris_red_mark4`` spectrograph. + + +Default Settings +++++++++++++++++ + +See :ref:`instr_par-keck_lris_red`, :ref:`instr_par-keck_lris_red_orig`, +and :ref:`instr_par-keck_lris_red_mark4` for a listing of modifications to the default settings. +*You do not have to add these changes to your PypeIt reduction file!* This is just a listing of how +the parameters used for LRISr differ from the defaults listed in the preceding tables on that page. +Moreover, additional modifications may have been made for specific setups, e.g, for different gratings, +or different slitmasks, etc. You can see a list of all the used parameters in the ``keck_lris_red_XXX.par`` +file generated by PypeIt at the beginning of the reduction. + +Calibrations +++++++++++++ -For the latter (Mark4), the wavelengths have been incorporated for the -R400 grating only so far but the arxiv solutions from the LBNL detector -may work ok. Check the outputs! +Wavelength calibration +^^^^^^^^^^^^^^^^^^^^^^ -LRISr Default Settings -++++++++++++++++++++++ +Arcs +---- + +We recommend that you turn on *most* of the standard +arc lamps:: + + Ne,Ar,Cd,Kr,Xe,Zn,Hg -See :ref:`instr_par-keck_lris_red` for -a listing of modifications to the default settings. *You do not have to add these changes to -your PypeIt reduction file!* This is just a listing of how the parameters used -for Keck/LRIS differ from the defaults listed in the preceding tables on -that page. +The archived solutions expect most (or all) of these lamps. + + +Wavelength Solution +------------------- +As default, the wavelength calibration is performed using the :ref:`wvcalib-fulltemplate` +algorithm. The templates are created in the same way as done for Keck/DEIMOS +(see :ref:`deimos_wavecalib`) and kept in the ``data/arc_lines/reid_arxiv`` directory. +When the first detector change happened (from Tektronik 2kx2k to LBNL 2kx4k) in May 2009 +(see :ref:`lrisr-detectors`), the pixel size changed from 24 to 15 microns. Because of this, +different templates are used for ``keck_lris_red_orig`` and ``keck_lris_red``. Luckily, +the pixel size did not change when the Mark4 detector was installed, so the same templates +are used for ``keck_lris_red`` and ``keck_lris_red_mark4``. +These are the templates available, one per each LRISr grating: + +============ ================================================ ============================================================= + GRATING template for ``keck_lris_red_orig`` template for ``keck_lris_red`` and ``keck_lris_red_mark4`` +============ ================================================ ============================================================= + 150/7500 keck_lris_red_orig_R150_7500_ArHgNe.fits keck_lris_red_R150_7500_ArCdHgNeZn.fits + 300/5000 keck_lris_red_orig_R300_5000_ArCdHgNeZn.fits keck_lris_red_R300_5000_ArCdHgKrNeXeZn.fits + 400/8500 keck_lris_red_orig_R400_8500_ArCdHgNeZn.fits keck_lris_red_R400_8500_ArCdHgKrNeXeZn.fits + 600/5000 keck_lris_red_orig_R600_5000_ArCdHgNeZn.fits keck_lris_red_R600_5000_ArCdHgKrNeXeZn.fits + 600/7500 keck_lris_red_orig_R600_7500_ArCdHgNeZn.fits keck_lris_red_R600_7500_ArCdHgKrNeXeZn.fits + 600/10000 keck_lris_red_orig_R600_10000_ArCdHgNeZn.fits keck_lris_red_R600_10000_ArCdHgKrNeXeZn.fits + 831/8200 keck_lris_red_orig_R831_8200_ArCdHgNeZn.fits keck_lris_red_R831_8200_ArCdHgKrNeXeZn.fits + 900/5500 keck_lris_red_orig_R900_5500_ArCdHgNeZn.fits keck_lris_red_R900_5500_ArCdHgNeZn.fits + 1200/7500 keck_lris_red_orig_R1200_7500_ArCdHgNeZn.fits keck_lris_red_R1200_7500_ArCdHgKrNeXeZn.fits + 1200/9000 keck_lris_red_R1200_9000.fits +============ ================================================ ============================================================= + +PypeIt will automatically choose the right template according to the specific dataset. +Note that the 1200/9000 grating was first released in 2013, so it was not available for +``keck_lris_red_orig``. Known issues ============ @@ -164,94 +374,21 @@ The code may identify a 'ghost' slit in empty detector real estate if your mask does not fill most of the field. Be prepared to ignore it. -Slit-masks -++++++++++ - -PypeIt can now incorporate slitmask information in the reduction -routine for LRIS similar to its DEIMOS capabilities. That is, if the trace -calibrations files with mask information are fed to PypeIt, it is -capable of using said information to determine object coordinates, -identify targeted and serendipitous source and subsequently, collate by -ra/dec. -Unfortunately, LRIS raw frames do not come ready with slitmask -data and thus this information needs to be inserted by the user before -processing with PypeIt if they are desirous of incorporating -the abovementioned features into their reduction. -Here are the steps to do so: +Additional Reading +================== -#. Obtain the mask design files. The design files include: - - #. The AUTOSLIT-generated mask design files. - - #. One file with the ".file3" extension containing milling information. - - #. One file with the ".file1" extension containing the object catalog - corresponding to the mask slits. - - #. The ASCII object list file fed as input to AUTOSLIT to generate the files - above. - - ".file3" is mandatory while the other two files can be optionally excluded to - debug `TILSOTUA `__. - -#. Process the design files with `TILSOTUA - `__ : The design files contain the - milling blueprint (the `BluSlits` table). When using the ".file3" design - files, TILSOTUA creates FITS files based on the UCO/Lick template. The FITS - mask design files have empty `DesiSlits`, `ObjectCat` and `SlitObjMap` binary - tables. DEIMOS users may be familiar with these tables from their raw frames. - TILSOTUA populates these tables using its ``xytowcs`` function (in - ``LRIS_Mask_Coords_to_WCS.py``). One provides the code with two parameters: - ``input_file_name`` is either the FITS or ".file3" mask design file (be sure - the name includes the extension), and ``output_file_base`` is the prefix for - the the four files that get created by the code. The calling sequence is: - - .. code-block:: python - - xytowcs(input_file_name="yourmaskname.file3",output_file_base="yourmaskname_output.fits") - -#. The `ObjectCat` and `SlitObjMap` are only populated if ".file1" and the object list are provided. - e.g. - - .. code-block:: python - - xytowcs(input_file_name="yourmaskname.file3",output_file_base="yourmaskname_output.fits", - obj_file="yourtargets.obj", file1="yourmaskname.file1") - - It is assumed that the entries in `file1` and `obj_file` have unique `Name` values. i.e. Make - sure you have a unique identifier for each object. Without this, it is not possible to correctly - reconcile the two tables. - -#. Append TILSOTUA's output to your raw trace files: Once the user is satisfied - with the processed FITS file from TILSOTUA, append the binary tables to the - trace FITS files. The user must first verify that TILSOTUA has indeed - processed the files correctly. This implies: - - #. TILSOTUA has correctly identified the alignment stars (see the QA plot it generates). - - #. TILSOTUA has estimated the `TopDist` and `BotDist` in the `SlitObjMap` table correctly. - -One may append the binary tables from the outputs as additional `HDUs` in the LRIS trace files. e.g. - - .. code-block:: python - - from astropy.io import fits - - tracehdus = fits.open("trace_rawframe.fits") - autoslithdus = fits.open("yourmaskname_output.fits") - - for hdu in autoslithdus[1:]: - tracehdus.append(hdu) - tracehdus.writeto("trace_with_maskinfo.fits") - -If processed correctly, PypeIt should now fully utilize its -arsenal of slitmask processing tools to reduce and coadd spectra -with the WCS information incorporated. +Here are additional docs related to Keck/LRIS. Note many of them are related +to the development of PypeIt for use with LRIS data: -.. TODO: be specific about what you mean by "append the binary tables" +.. TODO: Generally useful information in these dev docs should be moved into +.. user-level doc pages, even if that means repeating information. -.. TODO: Does the above mean that LRIS should be included in lists of - instruments that use mask design information. Most relevant places claim we - can only do this for DEIMOS and MOSFIRE. +.. toctree:: + :maxdepth: 1 + ../dev/lrisframes + ../dev/lrisconfig + ../dev/slitmask_ids + ../dev/radec_object + ../dev/add_missing_obj diff --git a/doc/spectrographs/mmt_bluechannel.rst b/doc/spectrographs/mmt_bluechannel.rst index 65f86da6ba..b865873649 100644 --- a/doc/spectrographs/mmt_bluechannel.rst +++ b/doc/spectrographs/mmt_bluechannel.rst @@ -43,4 +43,5 @@ A few notes on setting up observations to work optimally with ``pypeit`` and wha this is not the case or otherwise define which observations are standard stars. * Images taken as part of spectrograph focus runs are automatically identified and configured as ``tilt`` images, but not ``arc``. This - is because their line widths can vary by quite a bit and thus shouldn't be coadded to/averaged with in-focus ``arc`` images. \ No newline at end of file + is because their line widths can vary by quite a bit and thus shouldn't be coadded to/averaged with in-focus ``arc`` images. + diff --git a/doc/spectrographs/mosfire.rst b/doc/spectrographs/mosfire.rst index 348e864f4a..ff20168c89 100644 --- a/doc/spectrographs/mosfire.rst +++ b/doc/spectrographs/mosfire.rst @@ -58,7 +58,7 @@ Multi-slits PypeIt is able to match the traced slit to the slit-mask design information contained as metadata in the MOSFIRE observations. This functionality at the moment is -implemented only for MOSFIRE and DEIMOS and is switched on by setting ``use_maskdesign`` flag in +implemented only for these :ref:`slitmask_info_instruments` and is switched on by setting ``use_maskdesign`` flag in :ref:`edgetracepar` to True. This is, already, the default for MOSFIRE, except when the ``LONGSLIT`` mask is used. diff --git a/doc/spectrographs/soar_goodman.rst b/doc/spectrographs/soar_goodman.rst index b12175048f..d8f6f34726 100644 --- a/doc/spectrographs/soar_goodman.rst +++ b/doc/spectrographs/soar_goodman.rst @@ -15,3 +15,4 @@ archive: PypeIt will only work with the .fz format, so please ensure that you are only using data in this format. + diff --git a/doc/spectrographs/spectrographs.rst b/doc/spectrographs/spectrographs.rst index 7f278b754f..2ab94922ce 100644 --- a/doc/spectrographs/spectrographs.rst +++ b/doc/spectrographs/spectrographs.rst @@ -38,6 +38,7 @@ instrument-specific details for running PypeIt. gemini_gnirs gtc_osiris deimos + keck_hires keck_kcwi lris keck_nires @@ -53,3 +54,5 @@ instrument-specific details for running PypeIt. shane_kast soar_goodman xshooter + + diff --git a/doc/spectrographs/xshooter.rst b/doc/spectrographs/xshooter.rst index 06409ce672..8e4420b399 100644 --- a/doc/spectrographs/xshooter.rst +++ b/doc/spectrographs/xshooter.rst @@ -10,3 +10,51 @@ This file summarizes several instrument specific settings that are related to the VLT/XShooter spectrograph. +Wavelengths +=========== + +As it is common for ESO to obtain calibrations with different +slit widths and binning, this can lead to various challenges +for PypeIt. + +As regards wavelengths, the varying binning and slit widths lead +to differing FWHM of the arc lines. And because the RMS threshold +for a good solution is scaled to FWHM, the default is to measure +the FWHM from the lines themselves. + +If too many orders are being rejected, you may wish to adjust things +in one or more ways. + +FWHM +---- + +For the UVB or the VIS, you may turn off measuring the FWHM (in units +of binned pixdels) from the arc lines +by adding this to your :ref:`pypeit_file`: + + +.. code-block:: ini + + [calibrations] + [[wavelengths]] + fwhm_fromlines = False + +This will set the FWHM to the default value for UVB/VIS which +may yield a better set of discovered arc lines. + +RMS +--- + +Another option is to increase the RMS threshold for a good solution. +This may be done in the :ref:`pypeit_file` as well: + +.. code-block:: ini + + [calibrations] + [[wavelengths]] + rms_threshold = 1.5 + + +Note that this is scaled by the ratio of the measured FWHM value +to the default value. See :ref:`wvcalib-echelle` for +further details. diff --git a/doc/tutorials/coadd2d_howto.rst b/doc/tutorials/coadd2d_howto.rst index ad10a44488..f42c942c99 100644 --- a/doc/tutorials/coadd2d_howto.rst +++ b/doc/tutorials/coadd2d_howto.rst @@ -116,7 +116,7 @@ Here are the options: .. note:: The ``offsets = maskdef_offsets`` option is only available for multi-slit observations and - currently only for Keck/DEIMOS and Keck/MOSFIRE. + currently only for these :ref:`slitmask_info_instruments`. Set the parameters :ref:`slitmaskpar` in the :ref:`pypeit_file` during the main PypeIt run to determine how ``maskdef_offsets`` are computed. See :ref:`radec_object_report` for more info. diff --git a/doc/tutorials/deimos_howto.rst b/doc/tutorials/deimos_howto.rst index b7eb53f273..560f73d947 100644 --- a/doc/tutorials/deimos_howto.rst +++ b/doc/tutorials/deimos_howto.rst @@ -329,7 +329,7 @@ Last, here is a screen shot from the GUI showing the .. image:: ../figures/deimos_spec1d.png This uses the -`XSpecGUI `_ +`XSpecGUI `__ from the `linetools`_ package. The black line is the flux and the red line is the estimated error. diff --git a/doc/tutorials/gnirs_howto.rst b/doc/tutorials/gnirs_howto.rst index 4a8e49d1a5..7f4cc376fa 100644 --- a/doc/tutorials/gnirs_howto.rst +++ b/doc/tutorials/gnirs_howto.rst @@ -24,14 +24,14 @@ To setup the pypeit file, first run :ref:`pypeit_setup`: .. code-block:: bash - pypeit_setup -r absolute_path -s gemini_gnirs -b -c A + pypeit_setup -r absolute_path -s gemini_gnirs_echelle -b -c A where ``-b`` indicates that the data uses background images and includes the ``calib``, ``comb_id``, ``bkg_id`` in the pypeit file. The resulting pypeit file looks like: -.. include:: ../include/gemini_gnirs_A.pypeit.rst +.. include:: ../include/gemini_gnirs_echelle_A.pypeit.rst Reliable image typing and sequence generation based on header cards is not yet implemented for GNIRS. Hence, several modifications to the PypeIt file need to be made before executing :ref:`run-pypeit`. @@ -93,7 +93,7 @@ entire ABBA sequence into set of calibration frames. For this reason, we set the The edited pypeit file (exactly the one to reduce our example Gemini/GNIRS data set in the `PypeIt Development Suite`_) is: -.. include:: ../include/gemini_gnirs_A_corrected.pypeit.rst +.. include:: ../include/gemini_gnirs_echelle_A_corrected.pypeit.rst Note that the telluric standard has its ``calib`` IDs set to all 0s, which corresponds to the ``calib`` ID of the nearest science ABBA sequence in time. diff --git a/doc/tutorials/kast_howto.rst b/doc/tutorials/kast_howto.rst index f0e593b481..443862f182 100644 --- a/doc/tutorials/kast_howto.rst +++ b/doc/tutorials/kast_howto.rst @@ -11,7 +11,7 @@ Overview ======== This doc goes through a full run of PypeIt on one of the Shane Kast *blue* -datasets in the `PypeIt Development Suite`_. +datasets in the :ref:`dev-suite`. Setup ===== @@ -26,11 +26,11 @@ Place all of the files in a single folder. Mine is named .. code-block:: bash $ ls - b10.fits.gz b15.fits.gz b1.fits.gz b24.fits.gz b4.fits.gz b9.fits.gz - b11.fits.gz b16.fits.gz b20.fits.gz b27.fits.gz b5.fits.gz - b12.fits.gz b17.fits.gz b21.fits.gz b28.fits.gz b6.fits.gz - b13.fits.gz b18.fits.gz b22.fits.gz b2.fits.gz b7.fits.gz - b14.fits.gz b19.fits.gz b23.fits.gz b3.fits.gz b8.fits.gz + b1.fits.gz b14.fits.gz b19.fits.gz b24.fits.gz b5.fits.gz + b10.fits.gz b15.fits.gz b20.fits.gz b27.fits.gz b6.fits.gz + b11.fits.gz b16.fits.gz b21.fits.gz b28.fits.gz b7.fits.gz + b12.fits.gz b17.fits.gz b22.fits.gz b3.fits.gz b8.fits.gz + b13.fits.gz b18.fits.gz b23.fits.gz b4.fits.gz b9.fits.gz Run ``pypeit_setup`` -------------------- @@ -48,10 +48,12 @@ Here is my call for these data: cd folder_for_reducing # this is usually *not* the raw data folder pypeit_setup -r ${RAW_PATH}/b -s shane_kast_blue -c A -This creates a :doc:`../pypeit_file` in the folder named -``shane_kast_blue_A`` beneath where the script was run. -Note that ``$RAW_PATH`` should be the *full* path, i.e. including a / -at the start. +This creates a :doc:`../pypeit_file` in the folder named ``shane_kast_blue_A`` +beneath where the script was run. Note that ``$RAW_PATH`` should be the *full* +path (i.e., as given in the example above). Note that I have selected a single +configuration (using the ``-c A``) option. There is only one instrument +configuration for this dataset, meaning using ``--c A`` and ``-c all`` are +equivalent; see :doc:`../setup`. The ``shane_kast_blue_A.pypeit`` files looks like this: @@ -60,7 +62,15 @@ The ``shane_kast_blue_A.pypeit`` files looks like this: For some instruments (especially Kast), it is common for frametypes to be incorrectly assigned owing to limited or erroneous headers. However, in this example, all of the frametypes were accurately assigned in the -:doc:`../pypeit_file`, so there are no edits to be made. +:doc:`../pypeit_file`. + +.. note:: + + This is the rare case when the observation of a standard star is correctly + typed. Generally, it will be difficult for the automatic frame-typing code + to distinguish standard-star observations from science targets, meaning that + you'll need to edit the pypeit file directly to designate standard-star + observations as such. Main Run ======== @@ -71,10 +81,13 @@ simply: .. code-block:: bash cd shane_kast_blue_A - run_pypeit shane_kast_blue_A.pypeit -o + run_pypeit shane_kast_blue_A.pypeit -The ``-o`` indicates that any existing output files should be overwritten. As -there are none, it is superfluous but we recommend (almost) always using it. +If you find you need to re-run the code, you can use the ``-o`` option to ensure +the code overwrites any existing output files (excluding processed calibration +frames). If you find you need to re-build the calibrations, it's best to remove +the relevant (or all) files from the ``Calibrations/`` directory **instead** of +using the ``-m`` option. The :doc:`../running` doc describes the process in some more detail. @@ -100,12 +113,13 @@ with `ginga`_: .. code-block:: bash - ginga Calibrations/Bias_A_1_01.fits + ginga Calibrations/Bias_A_0_DET01.fits As typical of most bias images, it is featureless (effectively noise from the readout). -.. image:: ../figures/kastb_bias_image.png +.. figure:: ../figures/kastb_bias_image.png + :width: 40% See :doc:`../calibrations/bias` for further details. @@ -117,12 +131,13 @@ with `ginga`_: .. code-block:: bash - ginga Calibrations/Arc_A_1_01.fits + ginga Calibrations/Arc_A_0_DET01.fits As typical of most arc images, one sees a series of arc lines, here oriented horizontally (as always in PypeIt). -.. image:: ../figures/kastb_arc_image.png +.. figure:: ../figures/kastb_arc_image.png + :width: 30% See :doc:`../calibrations/arc` for further details. @@ -140,9 +155,10 @@ the :ref:`pypeit_chk_edges` script, with this explicit call: .. code-block:: bash - pypeit_chk_edges Calibrations/Edges_A_1_01.fits.gz + pypeit_chk_edges Calibrations/Edges_A_0_DET01.fits.gz -.. image:: ../figures/kastb_edges_image.png +.. figure:: ../figures/kastb_edges_image.png + :width: 40% The data is the combined flat images and the green/red lines indicate the left/right slit edges (green/magenta in more recent versions). The S174 label @@ -161,13 +177,15 @@ calibration. These are PNGs in the ``QA/PNG/`` folder. :: Here is an example of the 1D fits, written to -the ``QA/PNGs/Arc_1dfit_A_1_01_S0175.png`` file: +the ``QA/PNGs/Arc_1dfit_A_0_DET01_S0175.png`` file: -.. image:: ../figures/kastb_arc1d.png +.. figure:: ../figures/kastb_arc1d.png + :width: 90% What you hope to see in this QA is: - - On the left, many of the blue arc lines marked with green IDs + - On the left, many of the blue arc lines marked with *green* IDs + - That the green IDs span the full spectral range. - In the upper right, an RMS < 0.1 pixels - In the lower right, a random scatter about 0 residuals @@ -177,9 +195,10 @@ See :doc:`../calibrations/wvcalib` for further details. :: There are several QA files written for the 2D fits. -Here is ``QA/PNGs/Arc_tilts_2d_A_1_01_S0175.png``: +Here is ``QA/PNGs/Arc_tilts_2d_A_0_DET01_S0175.png``: -.. image:: ../figures/kastb_arc2d.png +.. figure:: ../figures/kastb_arc2d.png + :width: 50% Each horizontal line of circles traces the arc line centroid as a function of spatial position along the slit length. These data are used to fit the tilt in @@ -187,6 +206,21 @@ the spectral position. "Good" measurements included in the parametric trace are shown as black points; rejected points are shown in red. Provided most were not rejected, the fit should be good. An RMS<0.1 is also desired. +We also provide a script so that the arcline traces can be assessed against the +image using `ginga`_, similar to checking the slit edge tracing. + +.. code-block:: bash + + pypeit_chk_tilts Calibrations/Tilts_A_0_DET01.fits.gz + +.. figure:: ../figures/kastb_ginga_tilts.png + :width: 40% + + Main `ginga`_ window produced by ``pypeit_chk_tilts``. The arc image is + shown in gray scale, the slit edges are shown in green/magenta, masked pixels + are highlighted in red, good centroids are shown in blue, and centroids + rejected during the fit are shown in yellow. + See :doc:`../calibrations/wvcalib` for further details. Flatfield @@ -201,25 +235,29 @@ window (``pixflat_norm``) after using .. code-block:: bash - pypeit_chk_flats Calibrations/Flat_A_1_01.fits + pypeit_chk_flats Calibrations/Flat_A_0_DET01.fits -.. image:: ../figures/kastb_flat.png +.. figure:: ../figures/kastb_flat.png + :width: 40% One notes the pixel-to-pixel variations; these are at the percent level. The slit edges defined by the code are also plotted (green/red lines; green/magenta in more recent versions). The region of the detector beyond these images -has been set to unit value. +has been set to unity. See :doc:`../calibrations/flat` for further details. Spectra ------- -Eventually (be patient), the code will start -generating 2D and 1D spectra outputs. One per standard -and science frame, located in the ``Science/`` folder. +Eventually (be patient), the code will start generating 2D and 1D spectra +outputs. One per standard and science frame, located in the ``Science/`` +folder. + +For reference, full processing of this dataset on my circa 2020 Macbook +Pro took a little more than 2 minutes. Spec2D ++++++ @@ -230,9 +268,10 @@ window (``sky_resid-det01``) after using .. code-block:: bash - pypeit_show_2dspec Science/spec2d_b27-J1217p3905_KASTb_2015may20T045733.560.fits + pypeit_show_2dspec Science/spec2d_b27-J1217p3905_KASTb_20150520T045733.560.fits -.. image:: ../figures/kastb_spec2d.png +.. figure:: ../figures/kastb_spec2d.png + :width: 40% The green/red lines are the slit edges (green/magenta in more recent versions). The brighter pixels down the center of the slit is the object. The orange line @@ -250,26 +289,46 @@ Here is a screen shot from the GUI showing the .. code-block:: bash - pypeit_show_1dspec Science/spec1d_b27-J1217p3905_KASTb_2015may20T045733.560.fits + pypeit_show_1dspec Science/spec1d_b27-J1217p3905_KASTb_20150520T045733.560.fits -.. image:: ../figures/kastb_spec1d.png +.. figure:: ../figures/kastb_spec1d.png + :width: 70% -This uses the -`XSpecGUI `_ -from the `linetools`_ package. +This uses the `XSpecGUI +`__ from the +`linetools`_ package. With your mouse hovering over the window, type ``?`` to +open a webpage with the set of available commands used to interact with the +plot. The spectrum can also be ingested into a `specutils.Spectrum1D`_ object +using our :ref:`spec1D-specutils`. See :doc:`../out_spec1D` for further details. Fluxing ======= +This dataset includes observations of a spectrophotometric standard, Feige 66, +which is reduced alongside the science target observations. + Now that we have a reduced standard star spectrum, we can use that to generate a sensitivity file. Here is the -call for this example, which I run in the ``Science/`` folder: +call for this example: .. code-block:: bash - pypeit_sensfunc spec1d_b24-Feige66_KASTb_2015may20T041246.960.fits -o Kastb_feige66_sens.fits + pypeit_sensfunc Science/spec1d_b24-Feige66_KASTb_20150520T041246.960.fits -o Kastb_feige66_sens.fits + +This produces the sensitivity function (saved to ``Kastb_feige66_sens.fits``) +and three QA (pdf) plots. The main QA plot looks like this: + +.. figure:: ../figures/kastb_sens.png + :width: 60% + + QA plot from the sensitivity calculation. Black is the observed zeropoints, + red is the best-fit model, orange are points masked *before* fitting, blue + are points masked *during* fitting. + +The other two plots show the flux-calibrated standard-star spectrum against the +archived spectrum and the full system (top of atmosphere) throughput. See :doc:`../fluxing` for further details. diff --git a/doc/tutorials/tutorials.rst b/doc/tutorials/tutorials.rst index d706d328eb..585b52e6f4 100644 --- a/doc/tutorials/tutorials.rst +++ b/doc/tutorials/tutorials.rst @@ -5,14 +5,16 @@ Tutorials ========= +If you've landed here without first reading through the :ref:`cookbook`, you're +encouraged to start there and come back. + If this is your **first time using PypeIt**, you're encouraged to read through the :doc:`Shane Kast` tutorial as a general example of how to use -PypeIt; see also the :ref:`cookbook`. **You are also encouraged to pull example -data from the DevSuite for your instrument when learning how to use the -software**; see :ref:`dev-suite`. +PypeIt. **You are also encouraged to pull example data from the DevSuite for +your instrument when learning how to use the software**; see :ref:`dev-suite`. -For examples of reductions for different types of data (long-slit, echelle, -etc), we recommend the following starting points: +For examples of reductions for different types of data, we recommend the +following starting points: - **Long-slit data**: :doc:`Shane Kast` diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index eda198e910..b27d56fa64 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -15,6 +15,10 @@ For a detailed log of code edits (including releases earlier than version ---- +.. include:: releases/1.14.0.rst + +---- + .. include:: releases/1.13.0.rst ---- diff --git a/environment.yml b/environment.yml index 4d634f5f8d..882905717b 100644 --- a/environment.yml +++ b/environment.yml @@ -7,12 +7,12 @@ dependencies: - pyyaml>=5.4 - requests>=2.26 - packaging>=21.0 - - numpy>=1.21 + - numpy>=1.22 - scipy>=1.7 - scikit-learn>=1.0 - astropy>=4.3 - setuptools>=46.4 - - matplotlib>=3.4 + - matplotlib>=3.7 - configobj>=5.0.6 - ipython>=7.27 - extension-helpers>=0.1 diff --git a/proposals/NSF/POSE2023/py/figs_pypeit_nsf_pose2023.py b/proposals/NSF/POSE2023/py/figs_pypeit_nsf_pose2023.py new file mode 100644 index 0000000000..e4724c3b72 --- /dev/null +++ b/proposals/NSF/POSE2023/py/figs_pypeit_nsf_pose2023.py @@ -0,0 +1,526 @@ +""" Figures for NSF Cyber Proposal """ +from datetime import datetime +import os, sys +import numpy as np +import scipy +from scipy import stats +import datetime +import geopy + +import argparse + +import matplotlib as mpl +import matplotlib.gridspec as gridspec +from matplotlib import pyplot as plt +from matplotlib.transforms import Affine2D, offset_copy + + +from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER +import cartopy.crs as ccrs +import cartopy + +mpl.rcParams['font.family'] = 'stixgeneral' + +#mpl.rcParams.update({'text.usetex': True}) +#mpl.rcParams['text.latex.preamble'] = r'\usepackage{color}' + + +import h5py +from linetools import utils as ltu + +from pypeit.spectrographs import spectrograph_classes + + +from IPython import embed + +# User Institutions + +user_inst = [ + 'Harvard University', + 'MPIA', + 'UC Berkeley', + 'Princeton University', + 'Cidade University, Brazil', + 'Ohio State University', + 'Vanderbilt University', + 'AIP, Germany', + 'University of Copenhagen', + 'Washington State University', + 'UC Santa Cruz', + 'University of Southampton, UK', + 'UC Los Angeles', + 'University of Illinois', + 'University of Melbourne', + 'University of Hong Kong', + 'University of Toronto', + 'Carnegie Observatories', + 'Northwestern University', + 'University of Dublin', + 'Johns Hopkins University', + 'STScI', + 'Las Cumbres Observatory', + 'UC Santa Barbara', + 'University of Hawaii', + 'Caltech', + 'University of Southern Queensland, Australia', + 'University of Texas, Austin', + 'Flatiron Institute, NYC', + 'Weizmann Institute', + 'Tel Aviv University', + 'Observatoire de Paris', + 'MIT', + 'Stockholm University', + 'University of Cambridge', + 'University of Maryland', + #'NASA Goddard Space Flight Center, Greenbelt, Maryland', + 'Greenbelt, Maryland', + 'University of Washington', + 'University of Portsmouth, UK', + 'Humboldt University, Berlin', + 'Universite Lyon', + 'Konkoly Observatory, Budapest, Hungary', + 'University of Arizona', + 'ESO Garching', + 'Gemini Observatory, AZ', + 'Leiden University', + 'University of Birmingham', + 'Technical University of Denmark', + 'Warsaw University', + 'University of Turku, Finland', + 'Radboud University', + 'SRON, Netherlands', + 'Stockholm University', + 'University of Edinburgh', + 'INAF Rome', + 'Queens University', + 'UC Davis', + 'UC Riverside', + 'York University', + 'Tufts University', + 'UC Irvine', + 'CSIC Madrid', + 'INAF Trieste', + 'INAF Napoli, Italy', + 'Universidad Andres Bello, Santiago', + 'University of Wisconsin', + 'INAF Padova, Italy', + 'San Jose State', + 'University of Waterloo', + 'University of Oulu', + 'Michigan State University', + 'Swinburne University', + 'RIT', + 'IAS, Princeton', + 'Queens University', + 'IAC, Canary Islands', + 'University of North Carolina', + 'Yale University', + 'CSIC Granada', + 'University of Manchester', + 'National Astronomical Observatory of Japan', + 'Monash University', + 'Universidad de Zaragoza, Spain', + 'European University Cyprus, Spain', + 'Jet Propulsion Laboratory, Pasadena, CA', + 'Macquarie University', + 'UNIST, South Korea', + 'University of Firenze', + 'INFN Fiorentino', + 'University of Oslo', + 'INAF Bologna', + 'GEPI Meudon', + 'University of Pisa, Italy', + 'University of Minnesota', + 'LBNL', + 'Royal Observatory, Edinburgh', + 'UCL, London', + 'University of Tokyo', + 'University of Hertfordshire', + 'ASTRON', + 'Max Planck, Bonn', + 'CSIRO, Australia', + 'Curtin University', + 'SKA Observatory, UK', + 'University of Sydney', + 'Pontificia Universidad Catolica de Valparaiso', + 'University of Oxford', + 'University of Chicago', + 'INAF Naples, Italy', + 'CNRS Marseille', + 'Peking University', + 'Kyungpook National University, South Korea', + 'Pusan National University, South Korea', + 'Kyung Hee University, South Korea', + 'Korea Institute for Advanced Study', + 'SRI, Moscow', + 'MPA, Garching', + 'University of Michigan', + 'Karl Remeis-Observatory and Erlangen Centre for Astroparticle Physics', + 'South African Radio Astronomy Observatory, Cape Town', + 'IUCAA, Pune', + 'NRAO Socorro', + 'University of Geneva', + 'IAP, Paris', + 'Universidad de Chile', + 'University of the Western Cape', + 'University of New South Wales', + 'Arkansas Tech', + 'Japan Aerospace Exploration Agency', + 'Australian National University', + 'Maria Mitchell Observatory', + 'Universita degli Studi di Milano Bicocca', + 'Universita di Firenze', + 'INAF Florence', + 'San Diego State University', + 'W.M. Keck Observatory', + ] + +def set_fontsize(ax, fsz): + """ + Set the fontsize throughout an Axis + + Args: + ax (Matplotlib Axis): + fsz (float): Font size + + Returns: + + """ + for item in ([ax.title, ax.xaxis.label, ax.yaxis.label] + + ax.get_xticklabels() + ax.get_yticklabels()): + item.set_fontsize(fsz) + +def rainbow_text(x, y, strings, colors, orientation='vertical', + ax=None, direction='down', **kwargs): + """ + Take a list of *strings* and *colors* and place them next to each + other, with text strings[i] being shown in colors[i]. + + Parameters + ---------- + x, y : float + Text position in data coordinates. + strings : list of str + The strings to draw. + colors : list of color + The colors to use. + orientation : {'horizontal', 'vertical'} + ax : Axes, optional + The Axes to draw into. If None, the current axes will be used. + **kwargs + All other keyword arguments are passed to plt.text(), so you can + set the font size, family, etc. + """ + if ax is None: + ax = plt.gca() + t = ax.transData + fig = ax.figure + canvas = fig.canvas + + assert orientation in ['horizontal', 'vertical'] + #if orientation == 'vertical': + # kwargs.update(rotation=90, verticalalignment='bottom') + + for s, c in zip(strings, colors): + text = ax.text(x, y, s + " ", color=c, transform=t, **kwargs) + + # Need to draw to update the text position. + text.draw(canvas.get_renderer()) + ex = text.get_window_extent() + # Convert window extent from pixels to inches + # to avoid issues displaying at different dpi + ex = fig.dpi_scale_trans.inverted().transform_bbox(ex) + + if orientation == 'horizontal': + t = text.get_transform() + \ + offset_copy(Affine2D(), fig=fig, x=ex.width, y=0) + else: + if direction == 'down': + ydir = -1 + else: + ydir = 1 + t = text.get_transform() + \ + offset_copy(Affine2D(), fig=fig, x=0, y=ydir*ex.height) + #offset_copy(Affine2D(), fig=fig, x=0, y=ex.height) + + +def fig_geo_spectrographs(outfile:str='fig_geo_spectrographs.png', + debug=False): + """ Global geographic plot of PypeIt spectrographs + + Args: + outfile (str): + debug (bool, optional): _description_. Defaults to False. + """ + support_dict = { + 'green': ['keck_deimos', 'keck_mosfire', + 'keck_nires', 'keck_esi', 'keck_hires'], + 'blue': [ 'ldt_deveny', + 'p200_dbsp_blue', 'p200_dbsp_red', + 'shane_kast_blue', 'shane_kast_red', + 'mmt_binospec', 'mmt_bluechannel', 'mmt_mmirs', + 'lbt_mods1r', 'lbt_mods1b', 'lbt_mods2r', 'lbt_mods2b', + 'lbt_luci1', 'lbt_luci2', + 'keck_lris_blue', 'keck_lris_red_mark4', + 'keck_nirspec_low', + ], + 'orange': ['magellan_fire', 'magellan_mage', + 'mdm_modspec', 'mdm_osmos_mdm4k', + 'magellan_fire_long', 'not_alfosc', + 'gtc_maat', 'gtc_osiris', 'gtc_osiris_plus', + 'wht_isis_blue', 'wht_isis_red', + 'tng_dolores'], + 'black': ['rest'], + } + participants = ['keck'] + + # Load up spectrographs + spectrographs = spectrograph_classes() + + # Grab the locations + geo_dict = {} + for key in spectrographs.keys(): + spectrograph = spectrographs[key] + geo_dict[key] = spectrograph.telescope['longitude'], spectrograph.telescope['latitude'] + # Reset JWST + if 'jwst' in key: + geo_dict[key] = -76.615278, 39.289444 + all_lats = np.array([geo_dict[key][1] for key in geo_dict.keys()]) + specs = np.array(list(geo_dict.keys())) + print(f"PypeIt serves {len(specs)} spectrographs") + + if debug: + embed(header='70 of fig_geo_spectrographs') + + # Figure + fig = plt.figure(figsize=(12,8)) + plt.clf() + + tformP = ccrs.PlateCarree() + + ax = plt.axes(projection=tformP) + + + #cm = plt.get_cmap(color) + uni_lats = np.unique(all_lats) + + # Plot em + for uni_lat in uni_lats: + idx = all_lats == uni_lat + if np.abs(uni_lat-19.8283333333333) < 1e-5: + # Gemini + idx = idx | (np.abs(all_lats - 19.82380144722) < 1e-5) + # Magellan/CTIO + if (np.abs(uni_lat+29.00333333333) < 1e-5): + # + idx = idx | (np.abs(all_lats + 30.240741666666672) < 1e-5) + idx = idx | (np.abs(all_lats + 29.256666666666) < 1e-5) + # Arziona + if np.abs(uni_lat-31.6809444444) < 1e-5: + idx = idx | (np.abs(all_lats - 32.7015999999) < 1e-5) # LBT + idx = idx | (np.abs(all_lats - 34.744305000) < 1e-5) # + idx = idx | (np.abs(all_lats - 31.963333333) < 1e-5) + idx = idx | (np.abs(all_lats - 31.94999999) < 1e-5) # MDM + lon, lat = geo_dict[specs[idx][0]][0], geo_dict[specs[idx][0]][1], + # Plot + #if specs[idx][0] in ['gemini_gmos_north_e2v', + # 'shane_kast_blue', 'p200_dbsp_blue', + # 'ntt_efosc2', + # 'ldt_deveny', 'mdm_modspec', 'lbt_luci1']: + # psym = '*' + #else: + # psym = 'o' + psym = 'o' + #print(specs[idx][0]) + plt.plot(lon, lat, psym, transform=tformP) + + if (np.abs(uni_lat-32.) < 3.) & ( + np.abs(uni_lat-31.68094444) > 1e-5) & ( + np.abs(uni_lat-33.356000000) > 1e-5): # Arizona + continue + if np.abs(uni_lat+30.240741666666672) < 1e-5: + continue + if np.abs(uni_lat+29.256666666666) < 1e-5: + continue + if np.abs(uni_lat-19.82380144722) < 1e-5: + continue + # Label + lbl = '' + strings, colors = [], [] + for spec in specs[idx]: + #lbl += spec + '\n' + found_it = False + for color in support_dict.keys(): + if spec in support_dict[color]: + lbl += r'\textcolor{'+color+'}{'+spec+'}\n' + strings.append(spec) + colors.append(color) + found_it = True + break + if not found_it: + lbl += r'\textcolor{'+color+'}{'+spec+'}\n' + strings.append(spec) + colors.append('black') + # Trim off last \n + lbl = lbl[:-3] + if 'vlt' in lbl or 'shane' in lbl: + va = 'bottom' + direction = 'up' + else: + va = 'top' + direction = 'down' + if 'p200' in lbl: + ha = 'right' + else: + ha = 'left' + #lbl = specs[idx][0] + if np.abs(uni_lat-31.6809444) < 1e-5: + lon += 2. + # Color + #ax.text(lon, lat, lbl, transform=tformP, + # fontsize=15, ha=ha, va=va) + rainbow_text(lon, lat, strings, colors, ax=ax, fontsize=14., + ha=ha, va=va, direction=direction) + + + # Zoom in + ax.set_extent([-170, 10, -60, 60], + crs=ccrs.PlateCarree()) + + # Coast lines + ax.coastlines(zorder=10) + ax.add_feature(cartopy.feature.LAND, + facecolor='lightgray', edgecolor='black') + + gl = ax.gridlines(crs=ccrs.PlateCarree(), linewidth=1, + color='black', alpha=0.5, linestyle=':', draw_labels=True) + #gl.xlabels_top = False + #gl.ylabels_left = True + #gl.ylabels_right=False + #gl.xlines = True + #gl.xformatter = LONGITUDE_FORMATTER + #gl.yformatter = LATITUDE_FORMATTER + #gl.xlabel_style = {'color': 'black'}# 'weight': 'bold'} + #gl.ylabel_style = {'color': 'black'}# 'weight': 'bold'} + + set_fontsize(ax, 19.) + plt.savefig(outfile, dpi=300) + plt.close() + print('Wrote {:s}'.format(outfile)) + + +def fig_geo_users(outfile:str='fig_geo_users.png', + load_from_disk=False, + debug=False): + """ Global geographic plot of PypeIt users + + Args: + outfile (str): + debug (bool, optional): _description_. Defaults to False. + """ + # Init + geoloc = geopy.geocoders.Nominatim(user_agent='GoogleV3') + + # Grab the locations + if load_from_disk: + geo_dict = ltu.loadjson('geo_dict.json') + else: + geo_dict = {} + for institute in user_inst: + loc = geoloc.geocode(institute) + if loc is None: + print(f'No location for {institute}') + continue + geo_dict[institute] = loc.longitude, loc.latitude + # Write + ltu.savejson('geo_dict.json', geo_dict, easy_to_read=True) + + all_lats = np.array([geo_dict[key][1] for key in geo_dict.keys()]) + all_lons = np.array([geo_dict[key][0] for key in geo_dict.keys()]) + specs = np.array(list(geo_dict.keys())) + + if debug: + embed(header='70 of fig_geo_spectrographs') + + # Figure + fig = plt.figure(figsize=(12,5)) + plt.clf() + + tformP = ccrs.PlateCarree() + + ax = plt.axes(projection=tformP) + + # Plot + img = plt.scatter(x=all_lons, + y=all_lats, c='r', s=5, + transform=tformP) + + # Zoom in + ax.set_extent([-165, -50, 10, 60], + crs=ccrs.PlateCarree()) + + # Coast lines + ax.coastlines(zorder=10) + ax.add_feature(cartopy.feature.LAND, + facecolor='lightgray', edgecolor='black') + + gl = ax.gridlines(crs=ccrs.PlateCarree(), linewidth=1, + color='black', alpha=0.5, linestyle=':', + draw_labels=True) + #gl.xlabels_top = False + #gl.ylabels_left = True + #gl.ylabels_right=False + #gl.xlines = True + #gl.xformatter = LONGITUDE_FORMATTER + #gl.yformatter = LATITUDE_FORMATTER + #gl.xlabel_style = {'color': 'black'}# 'weight': 'bold'} + #gl.ylabel_style = {'color': 'black'}# 'weight': 'bold'} + + set_fontsize(ax, 23.) + plt.savefig(outfile, dpi=300) + plt.close() + print('Wrote {:s}'.format(outfile)) + + + +#### ########################## ######################### +def main(pargs): + + # Geographical location of spectrographs + if pargs.figure == 'geo_spec': + fig_geo_spectrographs(debug=pargs.debug) + + # Geographical location of authors + if pargs.figure == 'geo_users': + fig_geo_users(debug=pargs.debug, load_from_disk=pargs.load) + + + +def parse_option(): + """ + This is a function used to parse the arguments in the training. + + Returns: + args: (dict) dictionary of the arguments. + """ + parser = argparse.ArgumentParser("SSL Figures") + parser.add_argument("figure", type=str, + help="function to execute: 'geo_spec'") + parser.add_argument('--load', default=False, action='store_true', + help='Load files from disk?') + parser.add_argument('--debug', default=False, action='store_true', + help='Debug?') + args = parser.parse_args() + + return args + +# Command line execution +if __name__ == '__main__': + + pargs = parse_option() + main(pargs) + +# Figures + +# Geographic location of Spectrographs +# python py/figs_pypeit_nsf_pose2023.py geo_spec diff --git a/pypeit/alignframe.py b/pypeit/alignframe.py index ffe2f2353d..598a27855c 100644 --- a/pypeit/alignframe.py +++ b/pypeit/alignframe.py @@ -184,7 +184,7 @@ def build_traces(self, show_peaks=False, debug=False): for slit_idx, slit_spat in enumerate(self.slits.spat_id): specobj_dict = {'SLITID': slit_idx, 'DET': self.rawalignimg.detector.name, 'OBJTYPE': "align_profile", 'PYPELINE': self.spectrograph.pypeline} - msgs.info("Fitting alignment traces in slit {0:d}".format(slit_idx)) + msgs.info("Fitting alignment traces in slit {0:d}/{1:d}".format(slit_idx+1, self.slits.nslits)) align_traces = findobj_skymask.objs_in_slit( self.rawalignimg.image, self.rawalignimg.ivar, slitid_img_init == slit_spat, left[:, slit_idx], right[:, slit_idx], @@ -195,7 +195,7 @@ def build_traces(self, show_peaks=False, debug=False): nperslit=len(self.alignpar['locations'])) if len(align_traces) != len(self.alignpar['locations']): # Align tracing has failed for this slit - msgs.error("Alignment tracing has failed on slit {0:d}".format(slit_idx)) + msgs.error("Alignment tracing has failed on slit {0:d}/{1:d}".format(slit_idx+1,self.slits.nslits)) align_prof['{0:d}'.format(slit_idx)] = align_traces.copy() # Steps diff --git a/pypeit/archive.py b/pypeit/archive.py index 0eaa69c132..888b7f49bf 100755 --- a/pypeit/archive.py +++ b/pypeit/archive.py @@ -77,7 +77,7 @@ class can be used on its own for saving metadata or passed to an col_names (list of str): The column names of the metadata - get_metadata_func (func): + get_metadata_func (callable): Function that reads metadata and file information from the objects being archived. diff --git a/pypeit/bitmask.py b/pypeit/bitmask.py index 40cb4c95ba..caa92c96d5 100644 --- a/pypeit/bitmask.py +++ b/pypeit/bitmask.py @@ -8,8 +8,6 @@ .. include:: ../include/bitmask_usage.rst ----- - .. include common links, assuming primary doc root is up one directory .. include:: ../include/links.rst @@ -400,8 +398,9 @@ def unpack(self, value, flag=None): flag (:obj:`str`, :obj:`list`, optional): The specific bits to unpack. If None, all values are unpacked. + Returns: - tuple: A tuple of boolean numpy.ndarrays flagged according + tuple: A tuple of boolean `numpy.ndarray`_ objects flagged according to each bit. """ _flag = self._prep_flags(flag) diff --git a/pypeit/bspline/bspline.py b/pypeit/bspline/bspline.py index e539416217..df32506847 100644 --- a/pypeit/bspline/bspline.py +++ b/pypeit/bspline/bspline.py @@ -700,7 +700,7 @@ def uniq(x, index=None): Returns ------- - `np.ndarray` + result : `numpy.ndarray`_ The indices of the last occurence in `x` of its unique values. diff --git a/pypeit/calibframe.py b/pypeit/calibframe.py index 7b648a8bdc..30826d8683 100644 --- a/pypeit/calibframe.py +++ b/pypeit/calibframe.py @@ -107,7 +107,7 @@ def set_paths(self, odir, setup, calib_id, detname): detname (:obj:`str`): The identifier used for the detector or detector mosaic for the relevant instrument; see - :func:`~pypeit.spectrograph.spectrograph.Spectrograph.get_det_name`. + :func:`~pypeit.spectrographs.spectrograph.Spectrograph.get_det_name`. """ self.calib_dir = Path(odir).resolve() # TODO: Keep this, or throw an error if the directory doesn't exist instead? @@ -302,6 +302,74 @@ def ingest_calib_id(calib_id): msgs.error(f'Invalid calibration group {c}; must be convertible to an integer.') return _calib_id.tolist() + @staticmethod + def construct_calib_id(calib_id, ingested=False): + """ + Use the calibration ID to construct a unique identifying string included + in output file names. + + Args: + calib_id (:obj:`str`, :obj:`list`, :obj:`int`): + Identifiers for one or more calibration groups for this + calibration frame. Strings (either as individually entered or + as elements of a provided list) can be single or comma-separated + integers. Otherwise, all strings must be convertible to + integers; the only exception is the string 'all'. + ingested (:obj:`bool`, optional): + Indicates that the ``calib_id`` object has already been + "ingested" (see :func:`ingest_calib_id`). If True, this will + skip the ingestion step. + + Returns: + :obj:`str`: A string identifier to include in output file names. + """ + # Ingest the calibration IDs, if necessary + _calib_id = calib_id if ingested else CalibFrame.ingest_calib_id(calib_id) + if len(_calib_id) == 1: + # There's only one calibration ID, so return it. This works both + # for 'all' and for single-integer calibration groupings. + return _calib_id[0] + + # Convert the IDs to integers and sort them + calibs = np.sort(np.array(_calib_id).astype(int)) + + # Find where the list is non-sequential + indx = np.diff(calibs) != 1 + if not np.any(indx): + # The full list is sequential, so give the starting and ending points + return f'{calibs[0]}+{calibs[-1]}' + + # Split the array into sequential subarrays (or single elements) and + # combine them into a single string + split_calibs = np.split(calibs, np.where(indx)[0]+1) + return '-'.join([f'{s[0]}+{s[-1]}' if len(s) > 1 else f'{s[0]}' for s in split_calibs]) + + @staticmethod + def parse_calib_id(calib_id_name): + """ + Parse the calibration ID(s) from the unique string identifier used in + file naming. I.e., this is the inverse of :func:`construct_calib_id`. + + Args: + calib_id_name (:obj:`str`): + The string identifier used in file naming constructed from a + list of calibration IDs using :func:`construct_calib_id`. + + Returns: + :obj:`list`: List of string representations of single calibration + group integer identifiers. + """ + # Name is all, so we're done + if calib_id_name == 'all': + return ['all'] + # Parse the name into slices and enumerate them + calib_id = [] + for slc in calib_id_name.split('-'): + split_slc = slc.split('+') + calib_id += split_slc if len(split_slc) == 1 \ + else np.arange(int(split_slc[0]), int(split_slc[1])+1).astype(str).tolist() + return calib_id + @staticmethod def construct_calib_key(setup, calib_id, detname): """ @@ -325,12 +393,12 @@ def construct_calib_key(setup, calib_id, detname): detname (:obj:`str`): The identifier used for the detector or detector mosaic for the relevant instrument; see - :func:`~pypeit.spectrograph.spectrograph.Spectrograph.get_det_name`. + :func:`~pypeit.spectrographs.spectrograph.Spectrograph.get_det_name`. Returns: :obj:`str`: Calibration identifier. """ - return f'{setup}_{"-".join(CalibFrame.ingest_calib_id(calib_id))}_{detname}' + return f'{setup}_{CalibFrame.construct_calib_id(calib_id)}_{detname}' @staticmethod def parse_calib_key(calib_key): @@ -346,8 +414,8 @@ def parse_calib_key(calib_key): Returns: :obj:`tuple`: The three components of the calibration key. """ - setup, calib_id, detname = calib_key.split('_') - return setup, ','.join(calib_id.split('-')), detname + setup, calib_id_name, detname = calib_key.split('_') + return setup, ','.join(CalibFrame.parse_calib_id(calib_id_name)), detname @classmethod def construct_file_name(cls, calib_key, calib_dir=None): diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index 3e958b479b..8291fe1d9d 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -92,9 +92,9 @@ class Calibrations: Tilt calibration frame alignments (:class:`~pypeit.alignframe.Alignments`): Alignment calibration frame - msbias (:class:`~pypeit.buildimage.BiasImage`): + msbias (:class:`~pypeit.images.buildimage.BiasImage`): Bias calibration frame - msdark (:class:`~pypeit.buildimage.DarkImage`): + msdark (:class:`~pypeit.images.buildimage.DarkImage`): Dark calibration frame msbpm (`numpy.ndarray`_): Boolean array with the bad-pixel mask (pixels that should masked are @@ -313,6 +313,9 @@ def get_arc(self): self.msarc = frame['class'].from_file(cal_file) return self.msarc + # Reset the BPM + self.get_bpm(frame=raw_files[0]) + # Otherwise, create the processed file. msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.') self.msarc = buildimage.buildimage_fromlist(self.spectrograph, self.det, @@ -353,6 +356,9 @@ def get_tiltimg(self): self.mstilt = frame['class'].from_file(cal_file) return self.mstilt + # Reset the BPM + self.get_bpm(frame=raw_files[0]) + # Otherwise, create the processed file. msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.') self.mstilt = buildimage.buildimage_fromlist(self.spectrograph, self.det, @@ -399,6 +405,9 @@ def get_align(self): self.alignments.is_synced(self.slits) return self.alignments + # Reset the BPM + self.get_bpm(frame=raw_files[0]) + # Otherwise, create the processed file. msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.') msalign = buildimage.buildimage_fromlist(self.spectrograph, self.det, @@ -506,7 +515,7 @@ def get_dark(self): # Return it return self.msdark - def get_bpm(self): + def get_bpm(self, frame=None): """ Load or generate the bad pixel mask. @@ -519,8 +528,11 @@ def get_bpm(self): """ # Check internals self._chk_set(['par', 'det']) + # Set the frame to use for the BPM + if frame is None: + frame = self.fitstbl.frame_paths(self.frame) # Build it - self.msbpm = self.spectrograph.bpm(self.fitstbl.frame_paths(self.frame), self.det, + self.msbpm = self.spectrograph.bpm(frame, self.det, msbias=self.msbias if self.par['bpm_usebias'] else None) # Return return self.msbpm @@ -602,6 +614,8 @@ def get_flats(self): # Check if the image files are the same pix_is_illum = Counter(raw_illum_files) == Counter(raw_pixel_files) if len(raw_pixel_files) > 0: + # Reset the BPM + self.get_bpm(frame=raw_pixel_files[0]) msgs.info('Creating pixel-flat calibration frame using files: ') for f in raw_pixel_files: msgs.prindent(f'{Path(f).name}') @@ -610,6 +624,8 @@ def get_flats(self): raw_pixel_files, dark=self.msdark, bias=self.msbias, bpm=self.msbpm) if len(raw_lampoff_files) > 0: + # Reset the BPM + self.get_bpm(frame=raw_lampoff_files[0]) msgs.info('Subtracting lamp off flats using files: ') for f in raw_lampoff_files: msgs.prindent(f'{Path(f).name}') @@ -633,6 +649,8 @@ def get_flats(self): # Only build illum_flat if the input files are different from the pixel flat if not pix_is_illum and len(raw_illum_files) > 0: + # Reset the BPM + self.get_bpm(frame=raw_illum_files[0]) msgs.info('Creating slit-illumination flat calibration frame using files: ') for f in raw_illum_files: msgs.prindent(f'{Path(f).name}') @@ -730,7 +748,7 @@ def get_slits(self): self.slits = frame['class'].from_file(cal_file) self.slits.mask = self.slits.mask_init.copy() if self.user_slits is not None: - self.slits.user_mask(detname, self.user_slits) + self.slits.user_mask(detname, self.user_slits) return self.slits # Slits don't exist or we're not resusing them. See if the Edges @@ -744,13 +762,17 @@ def get_slits(self): # Write the slits calibration file self.slits.to_file() if self.user_slits is not None: - self.slits.user_mask(detname, self.user_slits) + self.slits.user_mask(detname, self.user_slits) return self.slits # Need to build everything from scratch. Start with the trace image. msgs.info('Creating edge tracing calibration frame using files: ') for f in raw_trace_files: msgs.prindent(f'{Path(f).name}') + + # Reset the BPM + self.get_bpm(frame=raw_trace_files[0]) + traceImage = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['traceframe'], raw_trace_files, bias=self.msbias, bpm=self.msbpm, @@ -760,6 +782,10 @@ def get_slits(self): msgs.info('Subtracting lamp off flats using files: ') for f in raw_lampoff_files: msgs.prindent(f'{Path(f).name}') + + # Reset the BPM + self.get_bpm(frame=raw_trace_files[0]) + lampoff_flat = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['lampoffflatsframe'], raw_lampoff_files, dark=self.msdark, @@ -791,7 +817,7 @@ def get_slits(self): edges = None self.slits.to_file() if self.user_slits is not None: - self.slits.user_mask(detname, self.user_slits) + self.slits.user_mask(detname, self.user_slits) return self.slits def get_wv_calib(self): @@ -829,13 +855,16 @@ def get_wv_calib(self): self.wv_calib = None return self.wv_calib - # If a processed calibration frame exists and we want to reuse it, do - # so: - if cal_file.exists() and self.reuse_calibs: + # If a processed calibration frame exists and + # we want to reuse it, do so (or just load it): + if cal_file.exists() and self.reuse_calibs: + # Load the file self.wv_calib = wavecalib.WaveCalib.from_file(cal_file) self.wv_calib.chk_synced(self.slits) self.slits.mask_wvcalib(self.wv_calib) - return self.wv_calib + # Return + if self.par['wavelengths']['redo_slits'] is None: + return self.wv_calib # Determine lamp list to use for wavecalib # Find all the arc frames in this calibration group @@ -857,9 +886,12 @@ def get_wv_calib(self): waveCalib = wavecalib.BuildWaveCalib(self.msarc, self.slits, self.spectrograph, self.par['wavelengths'], lamps, meta_dict=meta_dict, det=self.det, qa_path=self.qa_path) - self.wv_calib = waveCalib.run(skip_QA=(not self.write_qa)) + self.wv_calib = waveCalib.run(skip_QA=(not self.write_qa), + prev_wvcalib=self.wv_calib) # If orders were found, save slits to disk - if self.spectrograph.pypeline == 'Echelle' and not self.spectrograph.ech_fixed_format: + # or if redo_slits + if (self.par['wavelengths']['redo_slits'] is not None) or ( + self.spectrograph.pypeline == 'Echelle' and not self.spectrograph.ech_fixed_format): self.slits.to_file() # Save calibration frame self.wv_calib.to_file() @@ -1039,7 +1071,7 @@ def get_association(fitstbl, spectrograph, caldir, setup, calib_ID, det, must_ex 'dark': [buildimage.DarkImage], 'pixelflat': [flatfield.FlatImages], 'illumflat': [flatfield.FlatImages], - 'lampoffflats': [flatfield.FlatImages], + 'lampoffflats': [flatfield.FlatImages], 'trace': [edgetrace.EdgeTraceSet, slittrace.SlitTraceSet], 'tilt': [buildimage.TiltImage, wavetilts.WaveTilts] } diff --git a/pypeit/coadd1d.py b/pypeit/coadd1d.py index a64bc4a836..34b03974a5 100644 --- a/pypeit/coadd1d.py +++ b/pypeit/coadd1d.py @@ -12,6 +12,7 @@ import numpy as np from astropy.io import fits +from astropy import stats from pypeit.spectrographs.util import load_spectrograph from pypeit.onespec import OneSpec @@ -25,31 +26,44 @@ class CoAdd1D: @classmethod - def get_instance(cls, spec1dfiles, objids, spectrograph=None, par=None, sensfile=None, debug=False, show=False): + def get_instance(cls, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, + debug=False, show=False): """ - Superclass factory method which generates the subclass instance. See __init__ docs for arguments. + Superclass factory method which generates the subclass instance. See :class:`CoAdd1D` instantiation for + argument descriptions. """ pypeline = fits.getheader(spec1dfiles[0])['PYPELINE'] + 'CoAdd1D' return next(c for c in cls.__subclasses__() if c.__name__ == pypeline)( - spec1dfiles, objids, spectrograph=spectrograph, par=par, sensfile=sensfile, debug=debug, show=show) + spec1dfiles, objids, spectrograph=spectrograph, par=par, sensfuncfile=sensfuncfile, setup_id=setup_id, + debug=debug, show=show) - def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfile=None, debug=False, show=False): + def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, + debug=False, show=False): """ Args: spec1dfiles (list): - List of strings which are the spec1dfiles + List of strings which are the spec1dfiles objids (list): - List of strings which are the objids for the object in each spec1d file that you want to coadd + List of strings which are the objids for the object in each + spec1d file that you want to coadd. spectrograph (:class:`pypeit.spectrographs.spectrograph.Spectrograph`, optional): + Spectrograph object. par (:class:`pypeit.par.pypeitpar.Coadd1DPar`, optional): - Pypeit parameter set object for Coadd1D - sensfile (str, optional): - File holding the sensitivity function. This is required for echelle coadds only. + PypeIt parameter set object for Coadd1D + sensfuncile (str or list of strings, optional): + File or list of files holding the sensitivity function. This is + required for echelle coadds only. + setup_id (str or list of strings, optional): + A string or list of strings identifiying the setup IDs to coadd. + This is only used for echelle coadds where a loop over the + different echelle setups is performed. If None, it will be + assumed that all the input files, objids, and sensfuncfiles + correspond to the same setup. debug (bool, optional) - Debug. Default = False + Debug. Default = False show (bool, optional): - Debug. Default = True + Debug. Default = True """ # Instantiate attributes self.spec1dfiles = spec1dfiles @@ -66,19 +80,17 @@ def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfile=No else: self.par = par # - self.sensfile = sensfile self.debug = debug self.show = show self.nexp = len(self.spec1dfiles) # Number of exposures self.coaddfile = None + self.gpm_exp = np.ones(self.nexp, dtype=bool).tolist() # list of bool indicating the exposures that have been coadded def run(self): """ Runs the coadding """ - # Load the data - self.waves, self.fluxes, self.ivars, self.gpms, self.header = self.load_arrays() # Coadd the data self.wave_grid_mid, self.wave_coadd, self.flux_coadd, self.ivar_coadd, self.gpm_coadd = self.coadd() # Scale to a filter magnitude? @@ -87,61 +99,27 @@ def run(self): self.flux_coadd *= scale self.ivar_coadd = self.ivar_coadd / scale**2 - def load_arrays(self): - """ - Load the arrays we need for performing coadds. - Returns: - tuple: - - waves, fluxes, ivars, gpms, header + + def load(self): """ - for iexp in range(self.nexp): - sobjs = specobjs.SpecObjs.from_fitsfile(self.spec1dfiles[iexp], chk_version=self.par['chk_version']) - indx = sobjs.name_indices(self.objids[iexp]) - if not np.any(indx): - msgs.error("No matching objects for {:s}. Odds are you input the wrong OBJID".format(self.objids[iexp])) - wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, meta_spec, header = \ - sobjs[indx].unpack_object(ret_flam=self.par['flux_value'], extract_type=self.par['ex_value']) - # Allocate arrays on first iteration - # TODO :: We should refactor to use a list of numpy arrays, instead of a 2D numpy array. - if iexp == 0: - waves = np.zeros(wave_iexp.shape + (self.nexp,)) - fluxes = np.zeros_like(waves) - ivars = np.zeros_like(waves) - gpms = np.zeros_like(waves, dtype=bool) - header_out = header - if 'RA' in sobjs[indx][0].keys() and 'DEC' in sobjs[indx][0].keys(): - header_out['RA_OBJ'] = sobjs[indx][0]['RA'] - header_out['DEC_OBJ'] = sobjs[indx][0]['DEC'] - # Check if the arrays need to be padded - # TODO :: Remove the if/elif statement below once these 2D arrays have been converted to a list of 1D arrays - if wave_iexp.shape[0] > waves.shape[0]: - padv = [(0, wave_iexp.shape[0]-waves.shape[0]), (0, 0)] - waves = np.pad(waves, padv, mode='constant', constant_values=(0, 0)) - fluxes = np.pad(fluxes, padv, mode='constant', constant_values=(0, 0)) - ivars = np.pad(ivars, padv, mode='constant', constant_values=(0, 1)) - gpms = np.pad(gpms, padv, mode='constant', constant_values=(False, False)) - elif wave_iexp.shape[0] < waves.shape[0]: - padv = [0, waves.shape[0]-wave_iexp.shape[0]] - wave_iexp = np.pad(wave_iexp, padv, mode='constant', constant_values=(0, 0)) - flux_iexp = np.pad(flux_iexp, padv, mode='constant', constant_values=(0, 0)) - ivar_iexp = np.pad(ivar_iexp, padv, mode='constant', constant_values=(0, 1)) - gpm_iexp = np.pad(gpm_iexp, padv, mode='constant', constant_values=(False, False)) - # Store the information - waves[...,iexp], fluxes[...,iexp], ivars[..., iexp], gpms[...,iexp] \ - = wave_iexp, flux_iexp, ivar_iexp, gpm_iexp - return waves, fluxes, ivars, gpms, header_out + Load the arrays we need for performing coadds. Dummy method overloaded by children. + """ + msgs.error('This method is undefined in the base classes and should only be called by the subclasses') + def save(self, coaddfile, telluric=None, obj_model=None, overwrite=True): """ - Generate a :class:`OneSpec` object and write it to disk. + Generate a :class:`~pypeit.onespec.OneSpec` object and write it to disk. Args: coaddfile (str): - File to output coadded spectrum to. - telluric (`numpy.ndarray`_): - obj_model (str): - overwrite (bool): + File to output coadded spectrum to. + telluric (`numpy.ndarray`_, optional): + Telluric model. + obj_model (str, optional): + Name of the object model + overwrite (bool, optional): Overwrite existing file? """ self.coaddfile = coaddfile @@ -152,11 +130,12 @@ def save(self, coaddfile, telluric=None, obj_model=None, overwrite=True): mask=self.gpm_coadd[wave_gpm].astype(int), ext_mode=self.par['ex_value'], fluxed=self.par['flux_value']) - onespec.head0 = self.header + # TODO This is a hack, not sure how to merge the headers at present + onespec.head0 = self.headers[0] # Add history entries for coadding. history = History() - history.add_coadd1d(self.spec1dfiles, self.objids) + history.add_coadd1d(self.spec1dfiles, self.objids, gpm_exp=self.gpm_exp) # Add on others if telluric is not None: @@ -169,43 +148,175 @@ def save(self, coaddfile, telluric=None, obj_model=None, overwrite=True): def coadd(self): """ Dummy method overloaded by sub-classes - - Returns: - :obj:`tuple`: four items - - wave - - flux - - ivar - - gpm - """ - return (None,)*4 + raise NotImplementedError(f'coadding function not defined for {self.__class__.__name__}!') class MultiSlitCoAdd1D(CoAdd1D): """ - Child of CoAdd1d for Multislit and Longslit reductions + Child of CoAdd1d for Multislit and Longslit reductions. """ - def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfile=None, debug=False, show=False): + def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, debug=False, show=False): + """ + See :class:`CoAdd1D` instantiation for argument descriptions. + """ + super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par = par, sensfuncfile = sensfuncfile, + setup_id=setup_id, debug = debug, show = show) + + + def load(self): + """ + Load the arrays we need for performing coadds. + + Returns + ------- + waves : list of float `numpy.ndarray`_ + List of wavelength arrays. The length of the list is nexp. The + arrays can have different shapes. + fluxes : list of float `numpy.ndarray`_ + List of flux arrays. The arrays can have different shapes, but all + are aligned with what is in waves. + ivars : list of float `numpy.ndarray`_ + List of inverse variance arrays. The arrays can have different + shapes, but all are aligned with what is in waves. + gpms : list of bool `numpy.ndarray`_ + List of good pixel mask variance arrays. The arrays can have + different shapes, but all are aligned with what is in waves. + headers : list of header objects + List of headers of length nexp """ - See `CoAdd1D` doc string + waves, fluxes, ivars, gpms, headers = [], [], [], [], [] + for iexp in range(self.nexp): + sobjs = specobjs.SpecObjs.from_fitsfile(self.spec1dfiles[iexp], chk_version=self.par['chk_version']) + indx = sobjs.name_indices(self.objids[iexp]) + if not np.any(indx): + msgs.error( + "No matching objects for {:s}. Odds are you input the wrong OBJID".format(self.objids[iexp])) + if np.sum(indx) > 1: + msgs.error("Error in spec1d file for exposure {:d}: " + "More than one object was identified with the OBJID={:s} in file={:s}".format( + iexp, self.objids[iexp], self.spec1dfiles[iexp])) + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, trace_spec, trace_spat, meta_spec, header = \ + sobjs[indx].unpack_object(ret_flam=self.par['flux_value'], extract_type=self.par['ex_value']) + waves.append(wave_iexp) + fluxes.append(flux_iexp) + ivars.append(ivar_iexp) + gpms.append(gpm_iexp) + header_out = header.copy() + if 'RA' in sobjs[indx][0].keys() and 'DEC' in sobjs[indx][0].keys(): + header_out['RA_OBJ'] = sobjs[indx][0]['RA'] + header_out['DEC_OBJ'] = sobjs[indx][0]['DEC'] + headers.append(header_out) + + return waves, fluxes, ivars, gpms, headers + + def check_exposures(self): + """ + Check if there are bad exposures. + Exposures with flux masked everywhere are always removed. + Exposures that are considered bad based on their S/N compared to the + average S/N among all the exposures, are removed only if self.par['sigrej_exp'] is set. + The attributes self.waves, self.fluxes, self.ivars, self.gpms need to be defined. + + Returns + ------- + gpm_exp: list of bool + List of boolean that indicates which exposures + have been coadded. The length of the list is nexp. + _waves : list of float `numpy.ndarray`_ + Updated list of wavelength arrays. + _fluxes : list of float `numpy.ndarray`_ + Updated list of flux arrays. + _ivars : list of float `numpy.ndarray`_ + Updated list of inverse variance arrays. + _gpms : list of bool `numpy.ndarray`_ + Updated list of good pixel mask variance arrays. """ - super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par = par, sensfile = sensfile, - debug = debug, show = show) + + # initialize the exposures lists + _waves = [wave for wave in self.waves] + _fluxes = [flux for flux in self.fluxes] + _ivars = [ivar for ivar in self.ivars] + _gpms = [gpm for gpm in self.gpms] + _spec1dfiles = [spec1dfile for spec1dfile in self.spec1dfiles] + _objids = [objid for objid in self.objids] + + # good exposures index + goodindx_exp = np.arange(self.nexp) + + # check if there are exposures that are completely masked out, i.e., gpms = False for all spectral pixels + masked_exps = [np.all(np.logical_not(gpm)) for gpm in _gpms] + if np.any(masked_exps): + msgs.warn(f'The following exposure(s) is/are completely masked out. It/They will not be coadded.') + [msgs.warn(f"Exposure {i}: {fname.split('/')[-1]} {obj}") + for i, (fname, obj, masked_exp) in enumerate(zip(_spec1dfiles, _objids, masked_exps)) if masked_exp] + # remove masked out exposure + _waves = [wave for (wave, masked_exp) in zip(_waves, masked_exps) if not masked_exp] + _fluxes = [flux for (flux, masked_exp) in zip(_fluxes, masked_exps) if not masked_exp] + _ivars = [ivar for (ivar, masked_exp) in zip(_ivars, masked_exps) if not masked_exp] + _gpms = [gpm for (gpm, masked_exp) in zip(_gpms, masked_exps) if not masked_exp] + _spec1dfiles = [spec1dfile for (spec1dfile, masked_exp) in zip(_spec1dfiles, masked_exps) if not masked_exp] + _objids = [objid for (objid, masked_exp) in zip(_objids, masked_exps) if not masked_exp] + # update good exposures index + goodindx_exp = goodindx_exp[np.logical_not(masked_exps)] + + # check if there is still more than 1 exposure left + if len(_fluxes) < 2: + msgs.error('At least 2 unmasked exposures are required for coadding.') + + # check if there is any bad exposure by comparing the rms_sn with the median rms_sn among all exposures + if len(_fluxes) > 2: + # Evaluate the sn_weights. + rms_sn, weights = coadd.sn_weights(_fluxes, _ivars, _gpms, const_weights=True) + # some stats + mean, med, sigma = stats.sigma_clipped_stats(rms_sn, sigma_lower=2., sigma_upper=2.) + _sigrej = self.par['sigrej_exp'] if self.par['sigrej_exp'] is not None else 10.0 + # we set thresh_value to never be less than 0.2 + thresh_value = round(0.2 + med + _sigrej * sigma, 2) + bad_exps = rms_sn > thresh_value + if np.any(bad_exps): + warn_msg = f'The following exposure(s) has/have S/N > {thresh_value:.2f} ' \ + f'({_sigrej} sigma above the median S/N in the stack).' + if self.par['sigrej_exp'] is not None: + warn_msg += ' It/They WILL NOT BE COADDED.' + msgs.warn(warn_msg) + [msgs.warn(f"Exposure {i}: {fname.split('/')[-1]} {obj}") + for i, (fname, obj, bad_exp) in enumerate(zip(_spec1dfiles, _objids, bad_exps)) if bad_exp] + if self.par['sigrej_exp'] is not None: + # remove bad exposure + _waves = [wave for (wave, bad_exp) in zip(_waves, bad_exps) if not bad_exp] + _fluxes = [flux for (flux, bad_exp) in zip(_fluxes, bad_exps) if not bad_exp] + _ivars = [ivar for (ivar, bad_exp) in zip(_ivars, bad_exps) if not bad_exp] + _gpms = [gpm for (gpm, bad_exp) in zip(_gpms, bad_exps) if not bad_exp] + _spec1dfiles = [spec1dfile for (spec1dfile, bad_exp) in zip(_spec1dfiles, bad_exps) if not bad_exp] + _objids = [objid for (objid, bad_exp) in zip(_objids, bad_exps) if not bad_exp] + # update good exposures index + goodindx_exp = goodindx_exp[np.logical_not(bad_exps)] + + # gpm for the exposures, i.e., which exposures have been coadded + gpm_exp = np.zeros(self.nexp, dtype=bool) + gpm_exp[goodindx_exp] = True + + return gpm_exp.tolist(), _waves, _fluxes, _ivars, _gpms def coadd(self): """ Perform coadd for for Multi/Longslit data using multi_combspec Returns: - tuple - - wave_grid_mid, wave, flux, ivar, gpm - + tuple: see objects returned by + :func:`~pypeit.core.coadd.multi_combspec`. """ - return coadd.multi_combspec( - self.waves, self.fluxes, self.ivars, self.gpms, + # Load the data + self.waves, self.fluxes, self.ivars, self.gpms, self.headers = self.load() + # check if there are bad exposures and remove them + self.gpm_exp, _waves, _fluxes, _ivars, _gpms = self.check_exposures() + # Perform and return the coadd + return coadd.multi_combspec(_waves, _fluxes, _ivars, _gpms, sn_smooth_npix=self.par['sn_smooth_npix'], wave_method=self.par['wave_method'], - dv=self.par['dv'], wave_grid_min=self.par['wave_grid_min'], wave_grid_max=self.par['wave_grid_max'], + dv=self.par['dv'], dwave=self.par['dwave'], dloglam=self.par['dloglam'], + wave_grid_min=self.par['wave_grid_min'], wave_grid_max=self.par['wave_grid_max'], spec_samp_fact=self.par['spec_samp_fact'], ref_percentile=self.par['ref_percentile'], maxiter_scale=self.par['maxiter_scale'], sigrej_scale=self.par['sigrej_scale'], scale_method=self.par['scale_method'], sn_min_medscale=self.par['sn_min_medscale'], @@ -215,37 +326,79 @@ def coadd(self): - - class EchelleCoAdd1D(CoAdd1D): """ - Child of CoAdd1d for Echelle reductions + Child of CoAdd1d for Echelle reductions. """ - def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfile=None, debug=False, show=False): + def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, + debug=False, show=False): """ - See `CoAdd1D` doc string + See :class:`CoAdd1D` instantiation for argument descriptions. - """ - super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par = par, sensfile = sensfile, - debug = debug, show = show) - def coadd(self): """ - Perform coadd for for echelle data using ech_combspec + super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par = par, sensfuncfile = sensfuncfile, + setup_id=setup_id, debug = debug, show = show) + + if sensfuncfile is None: + msgs.error('sensfuncfile is a required argument for echelle coadding') + + self.sensfuncfile = self.nexp * [sensfuncfile] if isinstance(sensfuncfile, str) else sensfuncfile + nsens = len(self.sensfuncfile) + if nsens == 1: + self.sensfuncfile = self.nexp * [self.sensfuncfile[0]] + nsens = self.nexp + if nsens != self.nexp: + msgs.error('Must enter either one sensfunc file for all exposures or one sensfunc file for ' + f'each exposure. Entered {nsens} files for {self.nexp} exposures.') + + if setup_id is None: + self.setup_id = self.nexp*['A'] + else: + self.setup_id = self.nexp*[setup_id] if isinstance(setup_id, str) else setup_id + nsetup = len(self.setup_id) + if nsetup == 1: + self.setup_id = self.nexp * [self.setup_id[0]] + nsetup = self.nexp + if nsetup != self.nexp: + msgs.error('Must enter either a single setup_id for all exposures or one setup_id for ' + f'each exposure. Entered {nsetup} files for {self.nexp} exposures.') - Returns: - tuple - - wave_grid_mid, wave, flux, ivar, gpm + self.unique_setups = np.unique(self.setup_id).tolist() + self.nsetups_unique = len(self.unique_setups) + + + def coadd(self): """ - weights_sens = sensfunc.SensFunc.sensfunc_weights(self.sensfile, self.waves, - debug=self.debug) + Perform coadd for echelle data using ech_combspec + + Returns + ------- + wave_grid_mid : something + describe + wave_coadd : something + describe + flux_coadd : something + describe + ivar_coadd : something + describe + gpm_coadd : something + describe + """ + + # Load the data + self.waves, self.fluxes, self.ivars, self.gpms, self.weights_sens, self.headers = self.load() wave_grid_mid, (wave_coadd, flux_coadd, ivar_coadd, gpm_coadd), order_stacks \ - = coadd.ech_combspec(self.waves, self.fluxes, self.ivars, self.gpms, weights_sens, - nbest=self.par['nbest'], + = coadd.ech_combspec(self.waves, self.fluxes, self.ivars, self.gpms, self.weights_sens, + setup_ids=self.unique_setups, + nbests=self.par['nbests'], sn_smooth_npix=self.par['sn_smooth_npix'], wave_method=self.par['wave_method'], + dv=self.par['dv'], dwave=self.par['dwave'], dloglam=self.par['dloglam'], + wave_grid_min=self.par['wave_grid_min'], + wave_grid_max=self.par['wave_grid_max'], spec_samp_fact=self.par['spec_samp_fact'], ref_percentile=self.par['ref_percentile'], maxiter_scale=self.par['maxiter_scale'], @@ -256,8 +409,103 @@ def coadd(self): maxiter_reject=self.par['maxiter_reject'], lower=self.par['lower'], upper=self.par['upper'], maxrej=self.par['maxrej'], sn_clip=self.par['sn_clip'], - debug=self.debug, show=self.show) + debug=self.debug, show=self.show, show_exp=self.show) + return wave_grid_mid, wave_coadd, flux_coadd, ivar_coadd, gpm_coadd + def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): + """ + Load the arrays we need for performing coadds for a single setup. + + Args: + spec1dfiles (list): + List of spec1d files for this setup. + objids (list): + List of objids. This is aligned with spec1dfiles + sensfuncfile (list): + List of sensfuncfiles. This is aligned with spec1dfiles and objids + + Returns: + tuple: waves, fluxes, ivars, gpms, header. Each array has shape = + (nspec, norders, nexp) + + """ + nexp = len(spec1dfiles) + for iexp in range(nexp): + sobjs = specobjs.SpecObjs.from_fitsfile(spec1dfiles[iexp], chk_version=self.par['chk_version']) + indx = sobjs.name_indices(objids[iexp]) + if not np.any(indx): + msgs.error("No matching objects for {:s}. Odds are you input the wrong OBJID".format(objids[iexp])) + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, trace_spec, trace_spat, meta_spec, header = \ + sobjs[indx].unpack_object(ret_flam=self.par['flux_value'], extract_type=self.par['ex_value']) + weights_sens_iexp = sensfunc.SensFunc.sensfunc_weights(sensfuncfiles[iexp], wave_iexp, debug=self.debug) + # Allocate arrays on first iteration + # TODO :: We should refactor to use a list of numpy arrays, instead of a 2D numpy array. + if iexp == 0: + waves = np.zeros(wave_iexp.shape + (nexp,)) + fluxes = np.zeros_like(waves) + ivars = np.zeros_like(waves) + weights_sens = np.zeros_like(waves) + gpms = np.zeros_like(waves, dtype=bool) + header_out = header + if 'RA' in sobjs[indx][0].keys() and 'DEC' in sobjs[indx][0].keys(): + header_out['RA_OBJ'] = sobjs[indx][0]['RA'] + header_out['DEC_OBJ'] = sobjs[indx][0]['DEC'] + + # Store the information + waves[...,iexp], fluxes[...,iexp], ivars[..., iexp], gpms[...,iexp], weights_sens[...,iexp] \ + = wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, weights_sens_iexp + return waves, fluxes, ivars, gpms, weights_sens, header_out + + + def load(self): + """ + Load the arrays we need for performing echelle coadds. + + Returns + ------- + waves : list + List of arrays with the wavelength arrays for each setup. The length + of the list equals the number of unique setups and each arrays in + the list has shape = (nspec, norders, nexp) + fluxes : list + List of arrays with the flux arrays for each setup. The length of + the list equals the number of unique setups and each arrays in the + list has shape = (nspec, norders, nexp) + ivars : list + List of arrays with the ivar arrays for each setup. The length of + the list equals the number of unique setups and each arrays in the + list has shape = (nspec, norders, nexp) + gpms : list + List of arrays with the gpm arrays for each setup. The length of the + list equals the number of unique setups and each arrays in the list + has shape = (nspec, norders, nexp) + weights_sens : list + List of arrays with the sensfunc weights for each setup. The length + of the list equals the number of unique setups and each arrays in + the list has shape = (nspec, norders, nexp) + headers : list + List of headers for each setup. The length of the list is the number + of unique setups. + + """ + + _setup = np.asarray(self.setup_id) + _sensfuncfiles = np.asarray(self.sensfuncfile) + _spec1dfiles = np.asarray(self.spec1dfiles) + _objids = np.asarray(self.objids) + waves, fluxes, ivars, gpms, weights_sens, headers = [], [], [], [], [], [] + combined = [waves, fluxes, ivars, gpms, weights_sens, headers] + for uniq_setup in self.unique_setups: + setup_indx = _setup == uniq_setup + loaded = self.load_ech_arrays(_spec1dfiles[setup_indx], _objids[setup_indx], _sensfuncfiles[setup_indx]) + for c, l in zip(combined, loaded): + c.append(l) + + + + return waves, fluxes, ivars, gpms, weights_sens, headers + + diff --git a/pypeit/coadd2d.py b/pypeit/coadd2d.py index 15da1ed341..16e19ed909 100644 --- a/pypeit/coadd2d.py +++ b/pypeit/coadd2d.py @@ -49,9 +49,10 @@ class CoAdd2D: # Superclass factory method generates the subclass instance @classmethod def get_instance(cls, spec2dfiles, spectrograph, par, det=1, offsets=None, weights='auto', + only_slits=None, exclude_slits=None, spec_samp_fact=1.0, spat_samp_fact=1.0, sn_smooth_npix=None, bkg_redux=False, find_negative=False, show=False, - show_peaks=False, debug_offsets=False, debug=False, **kwargs_wave): + show_peaks=False, debug_offsets=False, debug=False): """ Instantiate the subclass appropriate for the provided spectrograph. @@ -65,18 +66,18 @@ def get_instance(cls, spec2dfiles, spectrograph, par, det=1, offsets=None, weigh :class:`CoAdd2D` as its base. """ - return next(c for c in cls.__subclasses__() + return next(c for c in cls.__subclasses__() if c.__name__ == (spectrograph.pypeline + 'CoAdd2D'))( spec2dfiles, spectrograph, par, det=det, offsets=offsets, weights=weights, + only_slits=only_slits, exclude_slits=exclude_slits, spec_samp_fact=spec_samp_fact, spat_samp_fact=spat_samp_fact, sn_smooth_npix=sn_smooth_npix, bkg_redux=bkg_redux, find_negative=find_negative, - show=show, show_peaks=show_peaks, debug_offsets=debug_offsets, debug=debug, - **kwargs_wave) + show=show, show_peaks=show_peaks, debug_offsets=debug_offsets, debug=debug) def __init__(self, spec2d, spectrograph, par, det=1, offsets=None, weights='auto', - spec_samp_fact=1.0, spat_samp_fact=1.0, + only_slits=None, exclude_slits=None, spec_samp_fact=1.0, spat_samp_fact=1.0, sn_smooth_npix=None, bkg_redux=False, find_negative=False, show=False, - show_peaks=False, debug_offsets=False, debug=False, **kwargs_wave): + show_peaks=False, debug_offsets=False, debug=False): """ Args: @@ -92,16 +93,41 @@ def __init__(self, spec2d, spectrograph, par, det=1, offsets=None, weights='auto must include detectors viable as a mosaic for the provided spectrograph; see :func:`~pypeit.spectrographs.spectrograph.Spectrograph.allowed_mosaics`. - offsets (`numpy.ndarray`_, optional): - Spatial offsets to be applied to each image before coadding. For - the default mode of None, images are registered automatically - using the trace of the brightest object. Input offsets are not - yet supported. + offsets (`numpy.ndarray`_ or string, optional): + Spatial offsets to be applied to each image before coadding. + Here are the possible usage cases: + + #. If ``offsets = 'auto'``, auto-compute offsets from brightest + object (if exists) + + #. If ``offsets`` is something other than ``'auto'``, such + as a list, these are the offsets applied to each image. + + + #. (*for MultiSlit only*) If ``offsets = + 'maskdef_offsets'``, use ``maskdef_offset`` saved by the + :class:`~pypeit.slittrace.SlitTraceSet` + + #. (*for MultiSlit only*) If ``offsets = 'header'``, use the + dither offsets recorded in the header + weights (:obj:`str`, :obj:`list`, `numpy.ndarray`_): - Mode for the weights used to coadd images. Options are - ``'auto'`` (default), ``'uniform'``, or a list/array of weights - with ``shape = (nexp,)`` to apply to the image. Note ``'auto'`` - is not allowed if offsets are input. + Mode for the weights used to coadd images. Options are: + + #. ``'auto'``: If brightest object exists, auto-compute the + weights, otherwise use uniform weights. ``'auto'`` is + not allowed if offsets are input. + + #. ``'uniform'``: If brightest object exists, auto-compute + the weights, otherwise use uniform weights. + + #. :obj:`list`, `numpy.ndarray`_: This provides a set of + ``nexp`` weights to use for each exposure. + + only_slits (:obj:`list`, optional): + List of slits to coadd. It must be `slitord_id`. + exclude_slits (:obj:`list`, optional): + List of slits to exclude from coaddition. It must be `slitord_id`. spec_samp_fact (:obj:`float`, optional): Make the wavelength grid sampling finer (``spec_samp_fact`` less than 1.0) or coarser (``spec_samp_fact`` greater than 1.0) by @@ -139,10 +165,7 @@ def __init__(self, spec2d, spectrograph, par, det=1, offsets=None, weights='auto the screen. debug (:obj:`bool`, optional): Show QA for debugging. - **kwargs_wave - Keyword arguments passed to - :func:`pypeit.core.coadd.get_wave_grid`, which determine how the - wavelength grid is created for the 2d coadding. + """ # Use Cases: @@ -187,12 +210,11 @@ def __init__(self, spec2d, spectrograph, par, det=1, offsets=None, weights='auto self.objid_bri = None self.slitidx_bri = None self.snr_bar_bri = None - self.use_weights = None + self.use_weights = None # This is a list of length self.nexp that is assigned by the compute_weights method self.wave_grid = None self.good_slits = None self.maskdef_offset = None - # Load the stack_dict self.stack_dict = self.load_coadd2d_stacks(self.spec2d) self.pypeline = self.spectrograph.pypeline @@ -225,8 +247,18 @@ def __init__(self, spec2d, spectrograph, par, det=1, offsets=None, weights='auto # If smoothing is not input, smooth by 10% of the maximum spectral dimension self.sn_smooth_npix = sn_smooth_npix if sn_smooth_npix is not None else 0.1*self.nspec_max + # coadded frame parameters + + # get slit index that indicates which slits are good for coadding + self.good_slits = self.good_slitindx(only_slits=only_slits, exclude_slits=exclude_slits) + # get the number of slits that are going to be coadded + self.nslits_coadded = self.good_slits.size + + # effective exposure time + self.exptime_coadd = self.stack_dict['exptime_coadd'] + @staticmethod - def default_par(spectrograph, inp_cfg=None, det=None, slits=None): + def default_par(spectrograph, inp_cfg=None, det=None, only_slits=None, exclude_slits=None): """ Get the default 2D coadding parameters. @@ -238,8 +270,12 @@ def default_par(spectrograph, inp_cfg=None, det=None, slits=None): An existing set of parameters to add to. det (:obj:`list`, :obj:`str`, :obj:`tuple`, optional): Limit the coadding to this (set of) detector(s)/detector mosaic(s) - slits (:obj:`list`, :obj:`str`, optional): - Limit the coadding to this (set of) slit(s) + only_slits (:obj:`list`, :obj:`str`, optional): + Limit the coadding to this (set of) slit(s). Only_slits and exclude_slits + are mutually exclusive. If both are set, only_slits takes precedence. + exclude_slits (:obj:`list`, :obj:`str`, optional): + Exclude this (set of) slit(s) from the coadding. Only_slits and exclude_slits + are mutually exclusive. If both are set, only_slits takes precedence. Returns: :obj:`dict`: The default set of parameters. @@ -247,11 +283,27 @@ def default_par(spectrograph, inp_cfg=None, det=None, slits=None): cfg = dict(rdx=dict(spectrograph=spectrograph)) if inp_cfg is not None: cfg = utils.recursive_update(cfg, dict(inp_cfg)) + if only_slits is not None and det is not None: + msgs.warn('only_slits and det are mutually exclusive. Ignoring det.') + _det = None + else: + _det = det + if det is not None: - cfg['rdx']['detnum'] = det - if slits is not None: + cfg['rdx']['detnum'] = _det + + if only_slits is not None and exclude_slits is not None: + msgs.warn('only_slits and exclude_slits are mutually exclusive. Ignoring exclude_slits.') + _exclude_slits = None + else: + _exclude_slits = exclude_slits + + if only_slits is not None: utils.add_sub_dict(cfg, 'coadd2d') - cfg['coadd2d']['only_slits'] = slits + cfg['coadd2d']['only_slits'] = only_slits + if _exclude_slits is not None: + utils.add_sub_dict(cfg, 'coadd2d') + cfg['coadd2d']['exclude_slits'] = _exclude_slits # TODO: Heliocentric for coadd2d needs to be thought through. Currently # turning it off. utils.add_sub_dict(cfg, 'calibrations') @@ -297,7 +349,7 @@ def default_basename(spec2d_files): f"{io.remove_suffix(lasthdr['FILENAME'])}-{frsthdr['TARGET']}" @staticmethod - def output_paths(spec2d_files, par): + def output_paths(spec2d_files, par, coadd_dir=None): """ Construct the names and ensure the existence of the science and QA output directories. @@ -314,6 +366,9 @@ def output_paths(spec2d_files, par): Full set of parameters. The only used parameters are ``par['rdx']['scidir']`` and ``par['rdx']['qadir']``. WARNING: This also *alters* the value of ``par['rdx']['qadir']``!! + coadd_dir (:obj:`str`, optional): + Path to the directory to use for the coadd2d output. + If None, the parent of the science directory is used. Returns: :obj:`tuple`: Two strings with the names of (1) the science output @@ -321,7 +376,10 @@ def output_paths(spec2d_files, par): creates both directories if they do not exist. """ # Science output directory - pypeit_scidir = Path(spec2d_files[0]).parent + if coadd_dir is not None: + pypeit_scidir = Path(coadd_dir).resolve() / 'Science' + else: + pypeit_scidir = Path(spec2d_files[0]).parent coadd_scidir = pypeit_scidir.parent / f"{par['rdx']['scidir']}_coadd" if not coadd_scidir.exists(): coadd_scidir.mkdir(parents=True) @@ -332,7 +390,7 @@ def output_paths(spec2d_files, par): qa_path.mkdir(parents=True) return str(coadd_scidir), str(qa_path) - def good_slitindx(self, only_slits=None): + def good_slitindx(self, only_slits=None, exclude_slits=None): """ This provides an array of index of slits in the un-coadded frames that are considered good for 2d coadding. A bitmask common to all the un-coadded frames is used to determine which slits are good. Also, @@ -340,14 +398,24 @@ def good_slitindx(self, only_slits=None): Args: only_slits (:obj:`list`, optional): - List of slits to combine. It must be `slitord_id` + List of slits to combine. It must be `slitord_id`. + Only_slits and exclude_slits are mutually exclusive. + If both are provided, only_slits takes precedence. + exclude_slits (:obj:`list`, optional): + List of slits to exclude. It must be `slitord_id`. + Only_slits and exclude_slits are mutually exclusive. + If both are provided, only_slits takes precedence. Returns: `numpy.ndarray`_: array of index of good slits in the un-coadded frames """ - only_slits = [only_slits] if (only_slits is not None and - isinstance(only_slits, (int, np.integer))) else only_slits + if exclude_slits is not None and only_slits is not None: + msgs.warn('Both `only_slits` and `exclude_slits` are provided. They are mutually exclusive. ' + 'Using `only_slits` and ignoring `exclude_slits`') + _exclude_slits = None + else: + _exclude_slits = exclude_slits # This creates a unified bpm common to all frames slits0 = self.stack_dict['slits_list'][0] @@ -359,26 +427,43 @@ def good_slitindx(self, only_slits=None): slits = self.stack_dict['slits_list'][i] reduce_bpm |= (slits.mask > 0) & (np.invert(slits.bitmask.flagged(slits.mask, flag=slits.bitmask.exclude_for_reducing))) - # this are the good slit index according to the bpm mask + # these are the good slit index according to the bpm mask good_slitindx = np.where(np.logical_not(reduce_bpm))[0] # If we want to coadd all the good slits - if only_slits is None: + if only_slits is None and _exclude_slits is None: return good_slitindx # If instead we want to coadd only a selected (by the user) number of slits - # this are the `slitord_id` of the slits that we want to coadd - only_slits = np.atleast_1d(only_slits) - # create an array of slit index that are selected by the user and are also good slits - good_onlyslits = np.array([], dtype=int) - for islit in only_slits: - if islit not in slits0.slitord_id[good_slitindx]: - # Warnings for the slits that are selected by the user but NOT good slits - msgs.warn('Slit {} cannot be coadd because masked'.format(islit)) - else: - indx = np.where(slits0.slitord_id[good_slitindx] == islit)[0] - good_onlyslits = np.append(good_onlyslits, good_slitindx[indx]) - return good_onlyslits + if only_slits is not None: + # these are the `slitord_id` of the slits that we want to coadd + _only_slits = np.atleast_1d(only_slits) + # create an array of slit index that are selected by the user and are also good slits + good_onlyslits = np.array([], dtype=int) + msgs.info('Coadding only the following slits:') + for islit in _only_slits: + if islit not in slits0.slitord_id[good_slitindx]: + # Warnings for the slits that are selected by the user but NOT good slits + msgs.warn('Slit {} cannot be coadd because masked'.format(islit)) + else: + msgs.info(f'Slit {islit}') + indx = np.where(slits0.slitord_id[good_slitindx] == islit)[0] + good_onlyslits = np.append(good_onlyslits, good_slitindx[indx]) + return good_onlyslits + + # if we want to exclude some slits (selected by the user) from coadding + # these are the `slitord_id` of the slits that we want to exclude + _exclude_slits = np.atleast_1d(_exclude_slits) + # create an array of slit index that are excluded by the user + exclude_slitindx = np.array([], dtype=int) + msgs.info('Excluding the following slits:') + for islit in _exclude_slits: + if islit in slits0.slitord_id[good_slitindx]: + msgs.info(f'Slit {islit}') + exclude_slitindx = np.append(exclude_slitindx, + np.where(slits0.slitord_id[good_slitindx] == islit)[0][0]) + # these are the good slit index excluding the slits that are selected by the user + return np.delete(good_slitindx, exclude_slitindx) def optimal_weights(self, slitorderid, objid, const_weights=False): """ @@ -411,21 +496,26 @@ def optimal_weights(self, slitorderid, objid, const_weights=False): nexp = len(self.stack_dict['specobjs_list']) nspec = self.stack_dict['specobjs_list'][0][0].TRACE_SPAT.shape[0] # Grab the traces, flux, wavelength and noise for this slit and objid. - flux_stack = np.zeros((nspec, nexp), dtype=float) - ivar_stack = np.zeros((nspec, nexp), dtype=float) - wave_stack = np.zeros((nspec, nexp), dtype=float) - mask_stack = np.zeros((nspec, nexp), dtype=bool) + waves, fluxes, ivars, gpms = [], [], [], [] for iexp, sobjs in enumerate(self.stack_dict['specobjs_list']): - ithis = sobjs.slitorder_objid_indices(slitorderid, objid[iexp]) + ithis = sobjs.slitorder_objid_indices(slitorderid, objid[iexp], toler=self.par['coadd2d']['spat_toler']) if not np.any(ithis): msgs.error('Slit/order or OBJID provided not valid. Optimal weights cannot be determined.') # check if OPT_COUNTS is available - if sobjs[ithis][0].has_opt_ext(): - wave_stack[:, iexp], flux_stack[:, iexp], ivar_stack[:, iexp], mask_stack[:, iexp] = sobjs[ithis][0].get_opt_ext() + if sobjs[ithis][0].has_opt_ext() and np.any(sobjs[ithis][0].OPT_MASK): + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp = sobjs[ithis][0].get_opt_ext() + waves.append(wave_iexp) + fluxes.append(flux_iexp) + ivars.append(ivar_iexp) + gpms.append(gpm_iexp) # check if BOX_COUNTS is available - elif sobjs[ithis][0].has_box_ext(): - wave_stack[:, iexp], flux_stack[:, iexp], ivar_stack[:, iexp], mask_stack[:, iexp] = sobjs[ithis][0].get_box_ext() + elif sobjs[ithis][0].has_box_ext() and np.any(sobjs[ithis][0].BOX_MASK): + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp = sobjs[ithis][0].get_box_ext() + waves.append(wave_iexp) + fluxes.append(flux_iexp) + ivars.append(ivar_iexp) + gpms.append(gpm_iexp) msgs.warn(f'Optimal extraction not available for object ' f'{objid[iexp]} of slit/order {slitorderid} in exp {iexp}. Using box extraction.') else: @@ -433,11 +523,10 @@ def optimal_weights(self, slitorderid, objid, const_weights=False): f'flux not available in slit/order = {slitorderid}') # TODO For now just use the zero as the reference for the wavelengths? Perhaps we should be rebinning the data though? - rms_sn, weights = coadd.sn_weights(wave_stack, flux_stack, ivar_stack, mask_stack, self.sn_smooth_npix, - const_weights=const_weights) - return rms_sn, weights.T + rms_sn, weights = coadd.sn_weights(fluxes, ivars, gpms, self.sn_smooth_npix, const_weights=const_weights) + return rms_sn, weights - def coadd(self, only_slits=None, interp_dspat=True): + def coadd(self, interp_dspat=True): """ Construct a 2d co-add of a stack of PypeIt spec2d reduction outputs. This method calls loops over slits/orders and performs the 2d-coadd by @@ -446,10 +535,6 @@ def coadd(self, only_slits=None, interp_dspat=True): Parameters ---------- - only_slits : list, optional - List of slits to operate on. Not currently supported, i.e. the code - can currently only stack everything because the slit/reduction - bitmask checking is not yet implemented. Default = None interp_dspat : bool, optional Interpolate in the spatial coordinate image to faciliate running through core.extract.local_skysub_extract. Default=True @@ -462,26 +547,33 @@ def coadd(self, only_slits=None, interp_dspat=True): # TODO Make this a PypeIt object, with data model yada-yada. """ - # get slit index that indicates which slits are good for coadding - self.good_slits = self.good_slitindx(only_slits=only_slits) - # get the number of slits that are going to be coadded - self.nslits_coadded = self.good_slits.size coadd_list = [] for slit_idx in self.good_slits: - msgs.info('Performing 2d coadd for slit: {:d}/{:d}'.format(slit_idx, self.nslits_single - 1)) - ref_trace_stack = self.reference_trace_stack(slit_idx, offsets=self.offsets, - objid=self.objid_bri) + msgs.info(f'Performing 2D coadd for slit {self.spat_ids[slit_idx]} ({slit_idx + 1}/{self.nslits_single})') + + # mask identifying the current slit in each exposure + thismask_stack = [np.abs(slitmask - self.spat_ids[slit_idx]) <= self.par['coadd2d']['spat_toler'] + for slitmask in self.stack_dict['slitmask_stack']] - thismask_stack = [np.abs(slitmask - self.stack_dict['slits_list'][0].spat_id[slit_idx]) <= self.par['coadd2d']['spat_toler'] for slitmask in self.stack_dict['slitmask_stack']] + # check if the slit is found in every exposure + if not np.all([np.any(thismask) for thismask in thismask_stack]): + msgs.warn(f'Slit {self.spat_ids[slit_idx]} was not found in every exposures. ' + f'2D coadd cannot be performed on this slit. Try increasing the parameter spat_toler') + continue + + # reference trace + ref_trace_stack = self.reference_trace_stack(slit_idx, offsets=self.offsets, objid=self.objid_bri) # maskdef info maskdef_dict = self.get_maskdef_dict(slit_idx, ref_trace_stack) # weights - if not isinstance(self.use_weights, str) and self.use_weights.ndim > 2: - weights = self.use_weights[slit_idx] - else: - weights = self.use_weights + # if this is echelle data and the parset 'weights' is set to 'auto', + # then the weights are computed per order, i.e., every order has a + # different set of weights in each exposure (len(self.use_weights[slit_idx]) = nexp) + _weights = self.use_weights[slit_idx] if self.pypeline == 'Echelle' and self.weights == 'auto' else self.use_weights + # TODO: Create a method here in the child clases? It is not great to do pypeline specific things in the parent + # Perform the 2d coadd # NOTE: mask_stack is a gpm, and this is called inmask_stack in # compute_coadd2d, and outmask in coadd_dict is also a gpm @@ -490,14 +582,16 @@ def coadd(self, only_slits=None, interp_dspat=True): self.stack_dict['sciivar_stack'], self.stack_dict['skymodel_stack'], mask_stack, -# self.stack_dict['tilts_stack'], thismask_stack, self.stack_dict['waveimg_stack'], self.wave_grid, self.spat_samp_fact, maskdef_dict=maskdef_dict, - weights=weights, interp_dspat=interp_dspat) + weights=_weights, interp_dspat=interp_dspat) coadd_list.append(coadd_dict) + if len(coadd_list) == 0: + msgs.error("All the slits were missing in one or more exposures. 2D coadd cannot be performed") + return coadd_list def create_pseudo_image(self, coadd_list): @@ -603,7 +697,7 @@ def create_pseudo_image(self, coadd_list): if None not in all_maskdef_ids: maskdef_id[islit] = coadd_dict['maskdef_id'] maskdef_objpos[islit] = coadd_dict['maskdef_objpos'] - maskdef_slitcen[:, islit] = np.full(nspec_pseudo, coadd_dict['maskdef_slitcen']) + maskdef_slitcen[:, islit] = np.full(nspec_pseudo, coadd_dict['maskdef_slitcen'] + slit_left[:,islit]) if coadd_dict['maskdef_designtab'] is not None: maskdef_designtab = vstack([maskdef_designtab, coadd_dict['maskdef_designtab']]) @@ -731,7 +825,7 @@ def reduce(self, pseudo_dict, show=False, clear_ginga=True, show_peaks=False, sh waveimg=pseudo_dict['waveimg'], bkg_redux=self.bkg_redux, basename=basename, show=show) - skymodel_pseudo, objmodel_pseudo, ivarmodel_pseudo, outmask_pseudo, sobjs, _, _ = exTract.run( + skymodel_pseudo, objmodel_pseudo, ivarmodel_pseudo, outmask_pseudo, sobjs, _, _, _ = exTract.run( model_noise=False, spat_pix=pseudo_dict['spat_img']) @@ -810,29 +904,28 @@ def offset_slit_cen(self, slitid, offsets): :obj:`list`: A list of reference traces for the 2d coadding that have been offset. """ - return [slits.center[:,slitid] - offsets[iexp] + return [slits.center[:,slitid] - offsets[iexp] for iexp, slits in enumerate(self.stack_dict['slits_list'])] # ref_trace_stack = [] # for iexp, slits in enumerate(self.stack_dict['slits_list']): # ref_trace_stack.append(slits.center[:,slitid] - offsets[iexp]) # return ref_trace_stack - def get_wave_grid(self, **kwargs_wave): + def get_wave_grid(self, wave_method): """ Routine to create a wavelength grid for 2d coadds using all of the wavelengths of the extracted objects. Calls :func:`~pypeit.core.wavecal.wvutils.get_wave_grid`. Args: - **kwargs_wave (dict): - Optional argumments for - :func:`~pypeit.core.wavecal.wvutils.get_wave_grid`. + wave_method (str): + The method to use to create the wavelength grid passed to :func:`~pypeit.core.wavecal.wvutils.get_wave_grid` Returns: tuple: Returns the following: - - wave_grid (np.ndarray): New wavelength grid, not + - wave_grid (`numpy.ndarray`_): New wavelength grid, not masked - - wave_grid_mid (np.ndarray): New wavelength grid + - wave_grid_mid (`numpy.ndarray`_): New wavelength grid evaluated at the centers of the wavelength bins, that is this grid is simply offset from wave_grid by dsamp/2.0, in either linear space or log10 depending @@ -847,10 +940,9 @@ def get_wave_grid(self, **kwargs_wave): # This all seems a bit hacky if self.par['coadd2d']['use_slits4wvgrid'] or nobjs_tot==0: nslits_tot = np.sum([slits.nslits for slits in self.stack_dict['slits_list']]) - waves = np.zeros((self.nspec_max, nslits_tot*3)) - gpm = np.zeros_like(waves, dtype=bool) + waves, gpms = [], [] box_radius = 3. - indx = 0 + #indx = 0 # Loop on the exposures for iexp, (waveimg, slitmask, slits) in enumerate(zip(self.stack_dict['waveimg_stack'], self.stack_dict['slitmask_stack'], @@ -866,29 +958,26 @@ def get_wave_grid(self, **kwargs_wave): box_denom = moment1d(waveimg * mask > 0.0, trace_spat, 2 * box_radius, row=row)[0] wave_box = moment1d(waveimg * mask, trace_spat, 2 * box_radius, row=row)[0] / (box_denom + (box_denom == 0.0)) - waves[:self.nspec_array[iexp], indx:indx+3] = wave_box - # TODO -- This looks a bit risky - gpm[:self.nspec_array[iexp], indx: indx+3] = wave_box > 0. - indx += 3 + gpm_box = box_denom > 0. + waves += [wave for (wave, gpm) in zip(wave_box.T, gpm_box.T) if np.any(gpm)] + gpms += [(wave > 0.) & gpm for (wave, gpm) in zip(wave_box.T, gpm_box.T) if np.any(gpm)] + else: - waves = np.zeros((self.nspec_max, nobjs_tot)) - gpm = np.zeros_like(waves, dtype=bool) - indx = 0 + waves, gpms = [], [] for iexp, spec_this in enumerate(self.stack_dict['specobjs_list']): for spec in spec_this: # NOTE: BOX extraction usage needed for quicklook - waves[:self.nspec_array[iexp], indx] \ - = spec.OPT_WAVE if spec.OPT_WAVE is not None else spec.BOX_WAVE - # TODO -- OPT_MASK is likely to become a bpm with int values - gpm[:self.nspec_array[iexp], indx] \ - = spec.OPT_MASK if spec.OPT_MASK is not None else spec.BOX_MASK - indx += 1 - - wave_grid, wave_grid_mid, dsamp = wvutils.get_wave_grid(waves=waves, masks=gpm, - spec_samp_fact=self.spec_samp_fact, - **kwargs_wave) - - return wave_grid, wave_grid_mid, dsamp + good_opt_ext = spec.has_opt_ext() and np.any(spec.OPT_MASK) + good_box_ext = spec.has_box_ext() and np.any(spec.BOX_MASK) + if good_opt_ext or good_box_ext: + waves.append(spec.OPT_WAVE if good_opt_ext else spec.BOX_WAVE) + gpms.append(spec.OPT_MASK if good_opt_ext else spec.BOX_MASK) + # TODO -- OPT_MASK is likely to become a bpm with int values + #gpm[:self.nspec_array[iexp], indx] = spec.OPT_MASK + #indx += 1 + + return wvutils.get_wave_grid(waves=waves, gpms=gpms, wave_method=wave_method, + spec_samp_fact=self.spec_samp_fact) def load_coadd2d_stacks(self, spec2d, chk_version=False): """ @@ -916,6 +1005,7 @@ def load_coadd2d_stacks(self, spec2d, chk_version=False): sciivar_stack = [] mask_stack = [] slitmask_stack = [] + exptime_stack = [] #tilts_stack = [] # Object stacks specobjs_list = [] @@ -943,17 +1033,34 @@ def load_coadd2d_stacks(self, spec2d, chk_version=False): spat_flexure_list.append(s2dobj.sci_spat_flexure) sciimg_stack.append(s2dobj.sciimg) + exptime_stack.append(s2dobj.head0['EXPTIME']) waveimg_stack.append(s2dobj.waveimg) skymodel_stack.append(s2dobj.skymodel) sciivar_stack.append(s2dobj.ivarmodel) mask_stack.append(s2dobj.bpmmask.mask) slitmask_stack.append(s2dobj.slits.slit_img(flexure=s2dobj.sci_spat_flexure)) + # check if exptime is consistent for all images + exptime_coadd = np.percentile(exptime_stack, 50., method='higher') + isclose_exptime = np.isclose(exptime_stack, exptime_coadd, atol=1.) + if not np.all(isclose_exptime): + msgs.warn('Exposure time is not consistent (within 1 sec) for all frames being coadded! ' + f'Scaling each image by the median exposure time ({exptime_coadd} s) before coadding.') + exp_scale = exptime_coadd / exptime_stack + for iexp in range(nfiles): + if not isclose_exptime[iexp]: + sciimg_stack[iexp] *= exp_scale[iexp] + skymodel_stack[iexp] *= exp_scale[iexp] + sciivar_stack[iexp] /= exp_scale[iexp]**2 + return dict(specobjs_list=specobjs_list, slits_list=slits_list, slitmask_stack=slitmask_stack, - sciimg_stack=sciimg_stack, sciivar_stack=sciivar_stack, + sciimg_stack=sciimg_stack, + sciivar_stack=sciivar_stack, skymodel_stack=skymodel_stack, mask_stack=mask_stack, waveimg_stack=waveimg_stack, + exptime_stack=exptime_stack, + exptime_coadd=exptime_coadd, redux_path=redux_path, detectors=detectors_list, spectrograph=self.spectrograph.name, @@ -962,7 +1069,7 @@ def load_coadd2d_stacks(self, spec2d, chk_version=False): spat_flexure_list=spat_flexure_list) # tilts_stack=tilts_stack, waveimg_stack=waveimg_stack, - def check_input(self, input, type='weights'): + def check_input(self, input, type): """ Check that the number of input values (weights or offsets) is the same as the number of exposures Args: @@ -972,10 +1079,12 @@ def check_input(self, input, type='weights'): Returns: :obj:`list` or `numpy.ndarray`_: User input values """ + if type != 'weights' and type != 'offsets': + msgs.error('Unrecognized type for check_input') if isinstance(input, (list, np.ndarray)): if len(input) != self.nexp: msgs.error(f'If {type} are input it must be a list/array with same number of elements as exposures') - return np.atleast_1d(input) + return np.atleast_1d(input).tolist() if type == 'weights' else np.atleast_1d(input) msgs.error(f'Unrecognized format for {type}') def compute_offsets(self, offsets): @@ -1010,7 +1119,7 @@ def compute_offsets(self, offsets): elif isinstance(offsets, (list, np.ndarray)): msgs.info('Using user input offsets') # use them - self.offsets = self.check_input(offsets, type='offsets') + self.offsets = self.check_input(offsets, 'offsets') self.offsets_report(self.offsets, 'user input') # 3) parset `offsets` is = 'maskdef_offsets' (no matter if we have a bright object or not) @@ -1033,36 +1142,39 @@ def compute_offsets(self, offsets): def compute_weights(self, weights): """ - Determine self.use_weights, the weights to be used in the coadd2d. + Determine the weights to be used in the coadd2d. This is partially overloaded by the child methods. - Args: - weights (:obj:`list` or :obj:`str`): - Value that guides the determination of the weights. - It could be a list of weights or a string. If 'auto' the weight will be computed using - the brightest trace, if 'uniform' uniform weights will be used. + This method sets the internal :attr:`use_weights`. Documentation on the + form of self.use_weights needs to be written. + Args: + weights (:obj:`list`, :obj:`str`): + Value that guides the determination of the weights. It could be + a list of weights or a string. If 'auto' the weight will be + computed using the brightest trace, if 'uniform' uniform weights + will be used. """ msgs.info('Get Weights') # 1) User input weight if isinstance(weights, (list, np.ndarray)): # use those inputs - self.use_weights = self.check_input(weights, type='weights') + self.use_weights = self.check_input(weights, 'weights') msgs.info('Using user input weights') # 2) No bright object and parset `weights` is 'auto' or 'uniform', # or Yes bright object but the user wants still to use uniform weights elif ((self.objid_bri is None) and (weights in ['auto', 'uniform'])) or \ ((self.objid_bri is not None) and (weights == 'uniform')): - # use uniform weights - self.use_weights = 'uniform' if weights == 'auto': # warn if the user had put `auto` in the parset msgs.warn('Weights cannot be computed because no unique reference object ' 'with the highest S/N was found. Using uniform weights instead.') elif weights == 'uniform': msgs.info('Using uniform weights') + # use uniform weights + self.use_weights = (np.ones(self.nexp) / float(self.nexp)).tolist() # 3) Bright object exists and parset `weights` is equal to 'auto' elif (self.objid_bri is not None) and (weights == 'auto'): @@ -1071,6 +1183,7 @@ def compute_weights(self, weights): else: msgs.error('Invalid value for `weights`') + def get_brightest_object(self, specobjs_list, spat_ids): """ Dummy method to identify the brightest object. Overloaded by child methods. @@ -1105,11 +1218,11 @@ def get_maskdef_dict(self, slit_idx, ref_trace_stack): Dummy method to get maskdef info. Overloaded by child methods. Args: - slit_idx: - ref_trace_stack: + slit_idx (?): + ref_trace_stack (?): Returns: - + dict: ? """ return dict(maskdef_id=None, maskdef_objpos=None, maskdef_designtab=None) @@ -1135,20 +1248,22 @@ class MultiSlitCoAdd2D(CoAdd2D): """ def __init__(self, spec2d_files, spectrograph, par, det=1, offsets=None, weights='auto', + only_slits=None, exclude_slits=None, spec_samp_fact=1.0, spat_samp_fact=1.0, sn_smooth_npix=None, - bkg_redux=False, find_negative=False, show=False, show_peaks=False, debug_offsets=False, debug=False, **kwargs_wave): + bkg_redux=False, find_negative=False, show=False, show_peaks=False, debug_offsets=False, debug=False): super().__init__(spec2d_files, spectrograph, det=det, offsets=offsets, weights=weights, - spec_samp_fact=spec_samp_fact, spat_samp_fact=spat_samp_fact, - sn_smooth_npix=sn_smooth_npix, bkg_redux=bkg_redux, find_negative=find_negative, par=par, - show=show, show_peaks=show_peaks, debug_offsets=debug_offsets, - debug=debug, **kwargs_wave) + only_slits=only_slits, exclude_slits=exclude_slits, + spec_samp_fact=spec_samp_fact, spat_samp_fact=spat_samp_fact, + sn_smooth_npix=sn_smooth_npix, bkg_redux=bkg_redux, find_negative=find_negative, par=par, + show=show, show_peaks=show_peaks, debug_offsets=debug_offsets, + debug=debug) # maskdef offset self.maskdef_offset = np.array([slits.maskdef_offset for slits in self.stack_dict['slits_list']]) # Default wave_method for Multislit is linear - kwargs_wave['wave_method'] = 'linear' if 'wave_method' not in kwargs_wave else kwargs_wave['wave_method'] - self.wave_grid, self.wave_grid_mid, self.dsamp = self.get_wave_grid(**kwargs_wave) + wave_method = 'linear' if self.par['coadd2d']['wave_method'] is None else self.par['coadd2d']['wave_method'] + self.wave_grid, self.wave_grid_mid, self.dsamp = self.get_wave_grid(wave_method) # Check if the user-input object to compute offsets and weights exists if self.par['coadd2d']['user_obj'] is not None: @@ -1159,13 +1274,14 @@ def __init__(self, spec2d_files, spectrograph, par, det=1, offsets=None, weights # does it exists? user_obj_exist = [] for sobjs in self.stack_dict['specobjs_list']: - user_obj_exist.append(np.any(sobjs.slitorder_objid_indices(user_slit, user_objid))) + user_obj_exist.append(np.any(sobjs.slitorder_objid_indices(user_slit, user_objid, + toler=self.par['coadd2d']['spat_toler']))) if not np.all(user_obj_exist): msgs.error('Object provided through `user_obj` does not exist in all the exposures.') # find if there is a bright object we could use if len(self.stack_dict['specobjs_list']) > 0 and self.par['coadd2d']['user_obj'] is not None: - _slitidx_bri = np.where(self.spat_ids == user_slit)[0][0] + _slitidx_bri = np.where(np.abs(self.spat_ids - user_slit) <= self.par['coadd2d']['spat_toler'])[0][0] self.objid_bri, self.slitidx_bri, self.spatid_bri, self.snr_bar_bri = \ np.repeat(user_objid, self.nexp), _slitidx_bri, user_slit, None elif len(self.stack_dict['specobjs_list']) > 0 and (offsets == 'auto' or weights == 'auto'): @@ -1210,7 +1326,7 @@ def compute_offsets(self, offsets): thismask_stack = [np.abs(slitmask - self.spatid_bri) <= self.par['coadd2d']['spat_toler'] for slitmask in self.stack_dict['slitmask_stack']] # TODO Need to think abbout whether we have multiple tslits_dict for each exposure or a single one - trace_stack_bri = [slits.center[:, self.slitidx_bri] + trace_stack_bri = [slits.center[:, self.slitidx_bri] for slits in self.stack_dict['slits_list']] # trace_stack_bri = [] # for slits in self.stack_dict['slits_list']: @@ -1263,14 +1379,17 @@ def compute_offsets(self, offsets): def compute_weights(self, weights): """ - Determine self.use_weights, the weights to be used in the coadd2d + Determine the weights to be used in the coadd2d. - Args: - weights (:obj:`list` or :obj:`str`): - Value that guides the determination of the weights. - It could be a list of weights or a string. If equal to 'auto', the weight will be computed - using the brightest trace, if 'uniform' uniform weights will be used. + This method sets the internal :attr:`use_weights`. Documentation on the + form of self.use_weights needs to be written. + Args: + weights (:obj:`list`, :obj:`str`): + Value that guides the determination of the weights. It could be + a list of weights or a string. If equal to 'auto', the weight + will be computed using the brightest trace, if 'uniform' uniform + weights will be used. """ super().compute_weights(weights) @@ -1313,52 +1432,55 @@ def get_brightest_obj(self, specobjs_list, spat_ids): slit_snr_max = np.zeros((nslits, nexp), dtype=float) bpm = np.ones(slit_snr_max.shape, dtype=bool) objid_max = np.zeros((nslits, nexp), dtype=int) - # Loop over each exposure, slit, find the brighest object on that slit for every exposure + # Loop over each exposure, slit, find the brightest object on that slit for every exposure for iexp, sobjs in enumerate(specobjs_list): msgs.info("Working on exposure {}".format(iexp)) - nspec_now = self.nspec_array[iexp] for islit, spat_id in enumerate(spat_ids): + if len(sobjs) == 0: + continue ithis = np.abs(sobjs.SLITID - spat_id) <= self.par['coadd2d']['spat_toler'] - nobj_slit = np.sum(ithis) if np.any(ithis): objid_this = sobjs[ithis].OBJID - flux = np.zeros((nspec_now, nobj_slit)) - ivar = np.zeros((nspec_now, nobj_slit)) - wave = np.zeros((nspec_now, nobj_slit)) - mask = np.zeros((nspec_now, nobj_slit), dtype=bool) - remove_indx = [] + fluxes, ivars, gpms = [], [], [] for iobj, spec in enumerate(sobjs[ithis]): # check if OPT_COUNTS is available - if spec.has_opt_ext(): - wave[:, iobj], flux[:, iobj], ivar[:, iobj], mask[:, iobj] = spec.get_opt_ext() + if spec.has_opt_ext() and np.any(spec.OPT_MASK): + _, flux_iobj, ivar_iobj, gpm_iobj = spec.get_opt_ext() + fluxes.append(flux_iobj) + ivars.append(ivar_iobj) + gpms.append(gpm_iobj) # check if BOX_COUNTS is available - elif spec.has_box_ext(): - wave[:, iobj], flux[:, iobj], ivar[:, iobj], mask[:, iobj] = spec.get_box_ext() + elif spec.has_box_ext() and np.any(spec.BOX_MASK): + _, flux_iobj, ivar_iobj, gpm_iobj = spec.get_box_ext() + fluxes.append(flux_iobj) + ivars.append(ivar_iobj) + gpms.append(gpm_iobj) msgs.warn(f'Optimal extraction not available for obj {spec.OBJID} ' f'in slit {spat_id}. Using box extraction.') # if both are not available, we remove the object in this slit, # because otherwise coadd.sn_weights will crash else: msgs.warn(f'Optimal and Boxcar extraction not available for obj {spec.OBJID} in slit {spat_id}.') - remove_indx.append(iobj) - # if the number of removed objects is less than the total number of objects in this slit, - # i.e., we still have some objects left, we can proced with computing rms_sn - if len(remove_indx) < nobj_slit: - flux = np.delete(flux, remove_indx,1) - ivar = np.delete(ivar, remove_indx,1) - wave = np.delete(wave, remove_indx,1) - mask = np.delete(mask, remove_indx,1) - - rms_sn, weights = coadd.sn_weights(wave, flux, ivar, mask, None, const_weights=True) + #remove_indx.append(iobj) + # if there are objects on this slit left, we can proceed with computing rms_sn + if len(fluxes) > 0: + rms_sn, weights = coadd.sn_weights(fluxes, ivars, gpms, const_weights=True) imax = np.argmax(rms_sn) slit_snr_max[islit, iexp] = rms_sn[imax] objid_max[islit, iexp] = objid_this[imax] bpm[islit, iexp] = False + # If a slit has bpm = True for some exposures and not for others, set bpm = True for all exposures + # Find the rows where any of the bpm values are True + bpm_true_idx = np.array([np.any(b) for b in bpm]) + if np.any(bpm_true_idx): + # Flag all exposures in those rows + bpm[bpm_true_idx, :] = True + # Find the highest snr object among all the slits if np.all(bpm): msgs.warn('You do not appear to have a unique reference object that was traced as the highest S/N ' - 'ratio on the same slit of every exposure') + 'ratio on the same slit of every exposure. Try increasing the parameter `spat_toler`') return None, None, None, None else: # mask the bpm @@ -1368,6 +1490,7 @@ def get_brightest_obj(self, specobjs_list, spat_ids): snr_bar_mean = slit_snr[slitid] snr_bar = slit_snr_max[slitid, :] objid = objid_max[slitid, :] + return objid, slitid, spat_ids[slitid], snr_bar # TODO add an option here to actually use the reference trace for cases where they are on the same slit and it is @@ -1391,15 +1514,18 @@ def get_maskdef_dict(self, slit_idx, ref_trace_stack): """ Args: - slit_idx (:obj:`int`): index of a slit in the uncoadded frames - ref_trace_stack(`numpy.ndarray`_): Stack of reference traces about - which the images are rectified and coadded. It is the slitcen appropriately - shifted according the frames offsets. Shape is (nspec, nimgs). + slit_idx (:obj:`int`): + index of a slit in the uncoadded frames + ref_trace_stack (`numpy.ndarray`_): + Stack of reference traces about which the images are rectified + and coadded. It is the slitcen appropriately shifted according + the frames offsets. Shape is (nspec, nimgs). Returns: - :obj:`dict`: Dictionary containing all the maskdef info. The quantities saved - are: maskdef_id, maskdef_objpos, maskdef_slitcen, maskdef_designtab. To learn what - they are see :class:`~pypeit.slittrace.SlitTraceSet` datamodel. + :obj:`dict`: Dictionary containing all the maskdef info. The + quantities saved are: maskdef_id, maskdef_objpos, maskdef_slitcen, + maskdef_designtab. To learn what they are see + :class:`~pypeit.slittrace.SlitTraceSet` datamodel. """ # maskdef info @@ -1408,9 +1534,9 @@ def get_maskdef_dict(self, slit_idx, ref_trace_stack): self.stack_dict['slits_list'][0].maskdef_objpos is not None and \ self.stack_dict['maskdef_designtab_list'][0] is not None: # maskdef_designtab info for only this slit - this_idx = self.stack_dict['maskdef_designtab_list'][0]['SPAT_ID'] == self.stack_dict['slits_list'][0].spat_id[slit_idx] + this_idx = self.stack_dict['maskdef_designtab_list'][0]['SPAT_ID'] == self.spat_ids[slit_idx] this_maskdef_designtab = self.stack_dict['maskdef_designtab_list'][0][this_idx] - # remove columns that that are irrelevant in the coadd2d frames + # remove columns that are irrelevant in the coadd2d frames this_maskdef_designtab.remove_columns(['TRACEID', 'TRACESROW', 'TRACELPIX', 'TRACERPIX', 'SLITLMASKDEF', 'SLITRMASKDEF']) this_maskdef_designtab.meta['MASKRMSL'] = 0. @@ -1419,36 +1545,37 @@ def get_maskdef_dict(self, slit_idx, ref_trace_stack): # maskdef_id for this slit imaskdef_id = self.stack_dict['slits_list'][0].maskdef_id[slit_idx] - # maskdef_slitcenters. This trace the slit center along the spectral direction. - # But here we take only the value at the mid point - - maskdef_slitcen_pixpos = self.stack_dict['slits_list'][0].maskdef_slitcen[self.nspec_array[0]//2, slit_idx] + self.maskdef_offset - # binned maskdef_slitcenters position with respect to the center of the slit in ref_trace_stack - # this value should be the same for each exposure, but in case there are differences we take the mean value + # maskdef_slitcen (slit center along the spectral direction) and + # maskdef_objpos (expected position of the target, as distance from left slit edge) for this slit + # These are the binned maskdef_slitcen positions w.r.t. the center of the slit in ref_trace_stack slit_cen_dspat_vec = np.zeros(self.nexp) - for iexp, (ref_trace, maskdef_slitcen) in enumerate(zip(ref_trace_stack, maskdef_slitcen_pixpos)): - nspec_this = ref_trace.shape[0] - slit_cen_dspat_vec[iexp] = (maskdef_slitcen - ref_trace[nspec_this//2])/self.spat_samp_fact - - imaskdef_slitcen_dspat = np.mean(slit_cen_dspat_vec) - - # expected position of the targeted object from slitmask design (as distance from left slit edge) - imaskdef_objpos = self.stack_dict['slits_list'][0].maskdef_objpos[slit_idx] - - # find left edge - slits_left, _, _ = self.stack_dict['slits_list'][0].select_edges(flexure=self.stack_dict['spat_flexure_list'][0]) - # targeted object spat pix - nspec_this = slits_left.shape[0] - maskdef_obj_pixpos = imaskdef_objpos + self.maskdef_offset + slits_left[nspec_this//2, slit_idx] - # binned expected object position with respect to the center of the slit in ref_trace_stack - # this value should be the same for each exposure, but in case there are differences we take the mean value - + # These are the binned maskdef_objpos positions w.r.t. the center of the slit in ref_trace_stack objpos_dspat_vec = np.zeros(self.nexp) - for iexp, (ref_trace, maskdef_obj) in enumerate(zip(ref_trace_stack, maskdef_obj_pixpos)): + for iexp in range(self.nexp): + # get maskdef_slitcen + mslitcen_pixpos = self.stack_dict['slits_list'][iexp].maskdef_slitcen + if mslitcen_pixpos.ndim < 2: + mslitcen_pixpos = mslitcen_pixpos[:, None] + maskdef_slitcen_pixpos = mslitcen_pixpos[self.nspec_array[0]//2, slit_idx] + self.maskdef_offset[iexp] + + # get maskdef_objpos + # find left edge + slits_left, _, _ = \ + self.stack_dict['slits_list'][iexp].select_edges(flexure=self.stack_dict['spat_flexure_list'][iexp]) + # targeted object spat pix + maskdef_obj_pixpos = \ + self.stack_dict['slits_list'][iexp].maskdef_objpos[slit_idx] + self.maskdef_offset[iexp] \ + + slits_left[slits_left.shape[0]//2, slit_idx] + + # change reference system + ref_trace = ref_trace_stack[iexp] nspec_this = ref_trace.shape[0] - objpos_dspat_vec[iexp] = (maskdef_obj - ref_trace[nspec_this//2])/self.spat_samp_fact + slit_cen_dspat_vec[iexp] = (maskdef_slitcen_pixpos - ref_trace[nspec_this // 2]) / self.spat_samp_fact + + objpos_dspat_vec[iexp] = (maskdef_obj_pixpos - ref_trace[nspec_this // 2]) / self.spat_samp_fact + imaskdef_slitcen_dspat = np.mean(slit_cen_dspat_vec) imaskdef_objpos_dspat = np.mean(objpos_dspat_vec) else: @@ -1480,18 +1607,19 @@ class EchelleCoAdd2D(CoAdd2D): """ def __init__(self, spec2d_files, spectrograph, par, det=1, offsets=None, weights='auto', + only_slits=None, exclude_slits=None, spec_samp_fact=1.0, spat_samp_fact=1.0, sn_smooth_npix=None, - bkg_redux=False, find_negative=False, show=False, show_peaks=False, debug_offsets=False, debug=False, - **kwargs_wave): + bkg_redux=False, find_negative=False, show=False, show_peaks=False, debug_offsets=False, debug=False): super().__init__(spec2d_files, spectrograph, det=det, offsets=offsets, weights=weights, - spec_samp_fact=spec_samp_fact, spat_samp_fact=spat_samp_fact, - sn_smooth_npix=sn_smooth_npix, bkg_redux=bkg_redux, find_negative=find_negative, - par=par, show=show, show_peaks=show_peaks, debug_offsets=debug_offsets, - debug=debug, **kwargs_wave) + only_slits=only_slits, exclude_slits=exclude_slits, + spec_samp_fact=spec_samp_fact, spat_samp_fact=spat_samp_fact, + sn_smooth_npix=sn_smooth_npix, bkg_redux=bkg_redux, find_negative=find_negative, + par=par, show=show, show_peaks=show_peaks, debug_offsets=debug_offsets, + debug=debug) # Default wave_method for Echelle is log10 - kwargs_wave['wave_method'] = 'log10' if 'wave_method' not in kwargs_wave else kwargs_wave['wave_method'] - self.wave_grid, self.wave_grid_mid, self.dsamp = self.get_wave_grid(**kwargs_wave) + wave_method = 'log10' if self.par['coadd2d']['wave_method'] is None else self.par['coadd2d']['wave_method'] + self.wave_grid, self.wave_grid_mid, self.dsamp = self.get_wave_grid(wave_method) # Check if the user-input object to compute offsets and weights exists if self.par['coadd2d']['user_obj'] is not None: @@ -1565,11 +1693,10 @@ def compute_weights(self, weights): if (self.objid_bri is not None) and (weights == 'auto'): # computing a list of weights for all the slitord_ids that we than parse in coadd slitord_ids = self.stack_dict['slits_list'][0].slitord_id - use_weights = [] - for id in slitord_ids: - _, iweights = self.optimal_weights(id, self.objid_bri) - use_weights.append(iweights) - self.use_weights = np.array(use_weights) + self.use_weights = [] + for idx in slitord_ids: + _, iweights = self.optimal_weights(idx, self.objid_bri) + self.use_weights.append(iweights) if self.par['coadd2d']['user_obj'] is not None: msgs.info('Weights computed using a unique reference object provided by the user') else: @@ -1598,7 +1725,6 @@ def get_brightest_obj(self, specobjs_list, nslits): objid = np.zeros(nexp, dtype=int) snr_bar = np.zeros(nexp) - # norders = specobjs_list[0].ech_orderindx.max() + 1 for iexp, sobjs in enumerate(specobjs_list): msgs.info("Working on exposure {}".format(iexp)) uni_objid = np.unique(sobjs.ECH_OBJID) @@ -1607,25 +1733,32 @@ def get_brightest_obj(self, specobjs_list, nslits): bpm = np.ones((nslits, nobjs), dtype=bool) for iord in range(nslits): for iobj in range(nobjs): + flux = None ind = (sobjs.ECH_ORDERINDX == iord) & (sobjs.ECH_OBJID == uni_objid[iobj]) # check if OPT_COUNTS is available - if sobjs[ind][0].has_opt_ext(): - wave, flux, ivar, mask = sobjs[ind][0].get_opt_ext() + if sobjs[ind][0].has_opt_ext() and np.any(sobjs[ind][0].OPT_MASK): + _, flux, ivar, mask = sobjs[ind][0].get_opt_ext() # check if BOX_COUNTS is available - elif sobjs[ind][0].has_box_ext(): - wave, flux, ivar, mask = sobjs[ind][0].get_box_ext() + elif sobjs[ind][0].has_box_ext() and np.any(sobjs[ind][0].BOX_MASK): + _, flux, ivar, mask = sobjs[ind][0].get_box_ext() msgs.warn(f'Optimal extraction not available for object {sobjs[ind][0].ECH_OBJID} ' f'in order {sobjs[ind][0].ECH_ORDER}. Using box extraction.') else: - flux = None msgs.warn(f'Optimal and Boxcar extraction not available for ' f'object {sobjs[ind][0].ECH_OBJID} in order {sobjs[ind][0].ECH_ORDER}.') continue if flux is not None: - rms_sn, weights = coadd.sn_weights(wave, flux, ivar, mask, self.sn_smooth_npix, const_weights=True) + rms_sn, weights = coadd.sn_weights([flux], [ivar], [mask], const_weights=True) order_snr[iord, iobj] = rms_sn bpm[iord, iobj] = False + # If there are orders that have bpm = True for some objs and not for others, set bpm = True for all objs + # Find the rows where any of the bpm values are True + bpm_true_idx = np.array([np.any(b) for b in bpm]) + if np.any(bpm_true_idx): + # Flag all objs in those rows + bpm[bpm_true_idx, :] = True + # Compute the average SNR and find the brightest object if not np.all(bpm): # mask the bpm diff --git a/pypeit/core/arc.py b/pypeit/core/arc.py index 787c58942a..7f5ab21b2d 100644 --- a/pypeit/core/arc.py +++ b/pypeit/core/arc.py @@ -18,8 +18,6 @@ from pypeit import msgs from pypeit import utils -from pypeit.core.wavecal import wvutils -from pypeit.core.wavecal import wv_fitting from pypeit.core import fitting from IPython import embed @@ -319,11 +317,9 @@ def fit2darc_orders_qa(pypeitFit, nspec, outfile=None): plt.show() -# JFH CAn we replace reasize with this simpler function: rebin_factor -# https://scipy-cookbook.readthedocs.io/items/Rebinning.html def resize_mask2arc(shape_arc, slitmask_orig): """ - Resizes the input slitmask to a new shape. Generally used + Rebin the input slitmask to a new shape. Generally used for cases where the arc image has a different binning than the science image. @@ -350,9 +346,7 @@ def resize_mask2arc(shape_arc, slitmask_orig): msgs.error('Problem with images sizes. arcimg size and calibration size need to be integer multiples of each other') else: msgs.info('Calibration images have different binning than the arcimg. Resizing calibs for arc spectrum extraction.') - slitmask = utils.rebin(slitmask_orig, (nspec, nspat)) - # Previous line using skimage - #slitmask = ((np.round(resize(slitmask_orig.astype(np.integer), (nspec, nspat), preserve_range=True, order=0))).astype(np.integer)).astype(slitmask_orig.dtype) + slitmask = utils.rebin_slice(slitmask_orig, (nspec, nspat)) else: slitmask = slitmask_orig @@ -423,8 +417,8 @@ def resize_spec(spec_from, nspec_to): return spec_to -def get_censpec(slit_cen, slitmask, arcimg, gpm=None, box_rad=3.0, nonlinear_counts=1e10, - slit_bpm=None, slitIDs=None): +def get_censpec(slit_cen, slitmask, arcimg, gpm=None, box_rad=3.0, + nonlinear_counts=1e10, slit_bpm=None, slitIDs=None, verbose=True): """ Extract a boxcar spectrum with radius `box_rad` (pixels) from the input image using the input trace. By default, outliers within the box are clipped @@ -434,25 +428,29 @@ def get_censpec(slit_cen, slitmask, arcimg, gpm=None, box_rad=3.0, nonlinear_cou Parameters ---------- slit_cen : `numpy.ndarray`_ - Trace down the center of the slit + Trace down the center of the slit. Shape (nspec, nslits) slitmask : `numpy.ndarray`_ - Image where pixel values identify its parent slit, - starting with 0. Pixels with -1 are not part of any slit. - Shape must match `arcimg`. + Image where pixel values identify its parent slit, starting with 0. + Pixels with -1 are not part of any slit. Shape must match `arcimg`. arcimg : `numpy.ndarray`_ - Image to extract the arc from. This should be an arcimage - or perhaps a frame with night sky lines. + Image to extract the arc from. This should be an arcimage or perhaps a + frame with night sky lines. Shape (nspec, nspat) gpm : `numpy.ndarray`_, optional - Input mask image with same shape as arcimg. Convention - True = good and False = bad. If None, all pixels are - considered good. + Input mask image with same shape as arcimg. Convention True = good and + False = bad. If None, all pixels are considered good. Shape must match + arcimg. box_rad : :obj:`float`, optional - Half-width of the boxcar (floating-point pixels) in the - spatial direction used to extract the arc. + Half-width of the boxcar (floating-point pixels) in the spatial + direction used to extract the arc. nonlinear_counts : :obj:`float`, optional - Values exceeding this input value are masked as bad. + Values exceeding this input value are masked as bad. slitIDs : :obj:`list`, optional - A list of the slit IDs to extract (if None, all slits will be extracted) + A list of the slit IDs to extract (if None, all slits will be extracted) + slit_bpm: `numpy.ndarray`_, bool, optional + Bad pixel mask for the slits. True = bad. Shape must be (nslits,). Arc + spectra are filled with np.nan for masked slits. + verbose : :obj:`bool`, optional + Print out verbose information? Returns ------- @@ -484,11 +482,12 @@ def get_censpec(slit_cen, slitmask, arcimg, gpm=None, box_rad=3.0, nonlinear_cou continue # Check if this slit is masked if slit_bpm is not None and slit_bpm[islit]: - msgs.info('Ignoring masked slit {}'.format(islit)) + msgs.info('Ignoring masked slit {}'.format(islit+1)) # TODO -- Avoid using NaNs arc_spec[:,islit] = np.nan continue - msgs.info('Extracting approximate arc spectrum along the center of slit {0}'.format(islit)) + if verbose: + msgs.info(f'Extracting approximate arc spectrum of slit {islit+1}/{nslits}') # Create a mask for the pixels that will contribue to the arc arcmask = _gpm & (np.absolute(spat[None,:] - slit_cen[:,islit,None]) < box_rad) # Trimming the image makes this much faster @@ -957,7 +956,7 @@ def detect_lines(censpec, sigdetect=5.0, fwhm=4.0, fit_frac_fwhm=1.25, input_thr The centroids of the line detections twid : `numpy.ndarray`_ The 1sigma Gaussian widths of the line detections - centerr : `numpy.ndarray`_ + center : `numpy.ndarray`_ The variance on tcent w : `numpy.ndarray`_ An index array indicating which detections are the most reliable. diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index 6a4dd41db9..e3158372ab 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -8,6 +8,8 @@ import os import sys +import copy +import string from IPython import embed @@ -217,7 +219,7 @@ def poly_ratio_fitfunc_chi2(theta, gpm, arg_dict): # constrain the flux-corrrection vectors from going too small (or negative), or too large. ## Schlegel's version here vmult = np.fmax(ymult,1e-4)*(ymult <= 1.0) + np.sqrt(ymult)*(ymult > 1.0) - ivarfit = mask_both/(1.0/(ivar_med + np.invert(mask_both)) + np.square(vmult)/(ivar_ref_med + np.invert(mask_both))) + ivarfit = mask_both/(1.0/(ivar_med + np.logical_not(mask_both)) + np.square(vmult)/(ivar_ref_med + np.logical_not(mask_both))) chi_vec = mask_both * (flux_ref_med - flux_scale) * np.sqrt(ivarfit) # Changing the Huber loss parameter from step to step results in instability during optimization --MSR. # Robustly characterize the dispersion of this distribution @@ -488,7 +490,6 @@ def solve_poly_ratio(wave, flux, ivar, flux_ref, ivar_ref, norder, mask = None, ymult = np.fmin(np.fmax(ymult1, scale_min), scale_max) flux_rescale = ymult*flux ivar_rescale = ivar/ymult**2 - if debug: # Determine the y-range for the QA plots scale_spec_qa(wave, flux_med, ivar_med, wave, flux_ref_med, ivar_ref_med, ymult, 'poly', mask = mask, mask_ref=mask_ref, @@ -497,7 +498,7 @@ def solve_poly_ratio(wave, flux, ivar, flux_ref, ivar_ref, norder, mask = None, return ymult, (result.x, wave_min, wave_max), flux_rescale, ivar_rescale, outmask -def interp_oned(wave_new, wave_old, flux_old, ivar_old, gpm_old, sensfunc=False): +def interp_oned(wave_new, wave_old, flux_old, ivar_old, gpm_old, log10_blaze_function=None, sensfunc=False, kind='cubic'): """ Interpolate a 1D spectrum onto a new wavelength grid. @@ -524,6 +525,9 @@ def interp_oned(wave_new, wave_old, flux_old, ivar_old, gpm_old, sensfunc=False) gpm_old (`numpy.ndarray`_): Old good-pixel mask (True=Good) on the wave_old grid. Shape must match ``wave_old``. + log10_blaze_function: `numpy.ndarray`_ or None, optional + Log10 of the blaze function. Shape must match ``wave_old``. Default=None. + sensfunc (:obj:`bool`, optional): If True, the quantities ``flux*delta_wave`` and the corresponding ``ivar/delta_wave**2`` will be interpolated and returned instead of @@ -531,11 +535,22 @@ def interp_oned(wave_new, wave_old, flux_old, ivar_old, gpm_old, sensfunc=False) computation where we need flux*(wavelength bin width). Beacause delta_wave is a difference of the wavelength grid, interpolating in the presence of masked data requires special care. - + kind (:obj:`int`, :obj:`str`, optional): + Specifies the kind of interpolation as a string or as an integer + specifying the order of the spline interpolator to use following the convention of + scipy.interpolate.interp1d. The string has to be one of 'linear', 'nearest', + 'nearest-up', 'zero', 'slinear', 'quadratic', 'cubic', 'previous', or 'next'. 'zero', + 'slinear', 'quadratic' and 'cubic' refer to a spline interpolation of + zeroth, first, second or third order; 'previous' and 'next' simply + return the previous or next value of the point; 'nearest-up' and + 'nearest' differ when interpolating half-integers (e.g. 0.5, 1.5) + in that 'nearest-up' rounds up and 'nearest' rounds down. Default is 'cubic'. Returns: - :obj:`tuple`: Returns three `numpy.ndarray`_ objects with the + :obj:`tuple`: Returns four objects flux_new, ivar_new, gpm_new, log10_blaze_new interpolated flux, inverse variance, and good-pixel mask arrays with - the length matching the new wavelength grid. + the length matching the new wavelength grid. They are all + `numpy.ndarray`_ objects with except if log10_blaze_function is None in which case log10_blaze_new is None + """ # Check input if wave_new.ndim != 1 or wave_old.ndim != 1: @@ -545,7 +560,7 @@ def interp_oned(wave_new, wave_old, flux_old, ivar_old, gpm_old, sensfunc=False) msgs.error('All vectors to interpolate must have the same size.') # Do not interpolate if the wavelength is exactly same with wave_new - if np.array_equal(wave_new, wave_old): + if np.array_equal(wave_new, wave_old) and not sensfunc: return flux_old, ivar_old, gpm_old wave_gpm = wave_old > 1.0 # Deal with the zero wavelengths @@ -553,28 +568,37 @@ def interp_oned(wave_new, wave_old, flux_old, ivar_old, gpm_old, sensfunc=False) delta_wave_interp = wvutils.get_delta_wave(wave_old, wave_gpm) flux_interp = flux_old[wave_gpm]/delta_wave_interp[wave_gpm] ivar_interp = ivar_old[wave_gpm]*delta_wave_interp[wave_gpm]**2 + if log10_blaze_function is not None: + log10_blaze_interp = log10_blaze_function[wave_gpm] - np.log10(delta_wave_interp[wave_gpm]) else: flux_interp = flux_old[wave_gpm] ivar_interp = ivar_old[wave_gpm] + if log10_blaze_function is not None: + log10_blaze_interp = log10_blaze_function[wave_gpm] - flux_new = scipy.interpolate.interp1d(wave_old[wave_gpm], flux_interp, kind='cubic', + flux_new = scipy.interpolate.interp1d(wave_old[wave_gpm], flux_interp, kind=kind, bounds_error=False, fill_value=np.nan)(wave_new) - ivar_new = scipy.interpolate.interp1d(wave_old[wave_gpm], ivar_interp, kind='cubic', + ivar_new = scipy.interpolate.interp1d(wave_old[wave_gpm], ivar_interp, kind=kind, bounds_error=False, fill_value=np.nan)(wave_new) - # Interpolate a floating-point version of the mask + log10_blaze_new = scipy.interpolate.interp1d(wave_old[wave_gpm], log10_blaze_interp, kind=kind, + bounds_error=False, fill_value=np.nan)(wave_new) \ + if log10_blaze_function is not None else None + + # Interpolate a floating-point version of the mask. Use linear interpolation here gpm_new_tmp = scipy.interpolate.interp1d(wave_old[wave_gpm], gpm_old.astype(float)[wave_gpm], - kind='cubic', bounds_error=False, + kind='linear', bounds_error=False, fill_value=np.nan)(wave_new) # Don't allow the ivar to be ever be less than zero ivar_new = (ivar_new > 0.0)*ivar_new gpm_new = (gpm_new_tmp > 0.8) & (ivar_new > 0.0) & np.isfinite(flux_new) & np.isfinite(ivar_new) - return flux_new, ivar_new, gpm_new + return flux_new, ivar_new, gpm_new, log10_blaze_new + # TODO: ``sensfunc`` should be something like "conserve_flux". It would be # useful to compare these resampling routines against # `pypeit.sampling.Resample`. -def interp_spec(wave_new, waves, fluxes, ivars, gpms, sensfunc=False): +def interp_spec(wave_new, waves, fluxes, ivars, gpms, log10_blaze_function=None, sensfunc=False, kind='cubic'): """ Interpolate a set of spectra onto a new wavelength grid. @@ -612,6 +636,16 @@ def interp_spec(wave_new, waves, fluxes, ivars, gpms, sensfunc=False): computation where we need flux*(wavelength bin width). Beacause delta_wave is a difference of the wavelength grid, interpolating in the presence of masked data requires special care. + kind : str or int, optional + Specifies the kind of interpolation as a string or as an integer + specifying the order of the spline interpolator to use following the convention of + scipy.interpolate.interp1d. The string has to be one of 'linear', 'nearest', + 'nearest-up', 'zero', 'slinear', 'quadratic', 'cubic', 'previous', or 'next'. 'zero', + 'slinear', 'quadratic' and 'cubic' refer to a spline interpolation of + zeroth, first, second or third order; 'previous' and 'next' simply + return the previous or next value of the point; 'nearest-up' and + 'nearest' differ when interpolating half-integers (e.g. 0.5, 1.5) + in that 'nearest-up' rounds up and 'nearest' rounds down. Default is 'cubic'. Returns ------- @@ -623,6 +657,10 @@ def interp_spec(wave_new, waves, fluxes, ivars, gpms, sensfunc=False): gpms_inter : `numpy.ndarray`_ interpolated good-pixel mask with size and shape matching the new wavelength grid. + log10_blazes_inter : `numpy.ndarray`_ or None + interpolated log10 blaze function with size and shape matching the new + wavelength grid. If ``log10_blaze_function`` is not provided as an input, + this will be None. """ # Check input if wave_new.ndim > 2: @@ -636,7 +674,8 @@ def interp_spec(wave_new, waves, fluxes, ivars, gpms, sensfunc=False): # single wavelength grid if wave_new.ndim == 1: if fluxes.ndim == 1: - return interp_oned(wave_new, waves, fluxes, ivars, gpms, sensfunc=sensfunc) + return interp_oned(wave_new, waves, fluxes, ivars, gpms, log10_blaze_function=log10_blaze_function, + sensfunc=sensfunc, kind=kind) nexp = fluxes.shape[1] # Interpolate spectra to have the same wave grid with the iexp spectrum. @@ -644,21 +683,38 @@ def interp_spec(wave_new, waves, fluxes, ivars, gpms, sensfunc=False): fluxes_inter = np.zeros((wave_new.size, nexp), dtype=float) ivars_inter = np.zeros((wave_new.size, nexp), dtype=float) gpms_inter = np.zeros((wave_new.size, nexp), dtype=bool) - for ii in range(nexp): - fluxes_inter[:,ii], ivars_inter[:,ii], gpms_inter[:,ii] \ + if log10_blaze_function is not None: + log10_blazes_inter = np.zeros((wave_new.size, nexp), dtype=float) + for ii in range(nexp): + fluxes_inter[:,ii], ivars_inter[:,ii], gpms_inter[:,ii], log10_blazes_inter[:,ii] \ = interp_oned(wave_new, waves[:,ii], fluxes[:,ii], ivars[:,ii], gpms[:,ii], - sensfunc=sensfunc) - return fluxes_inter, ivars_inter, gpms_inter + log10_blaze_function = log10_blaze_function[:, ii], sensfunc=sensfunc, kind=kind) + else: + for ii in range(nexp): + fluxes_inter[:,ii], ivars_inter[:,ii], gpms_inter[:,ii], _ \ + = interp_oned(wave_new, waves[:,ii], fluxes[:,ii], ivars[:,ii], gpms[:,ii], sensfunc=sensfunc, kind=kind) + log10_blazes_inter=None + + return fluxes_inter, ivars_inter, gpms_inter, log10_blazes_inter + # Second case: interpolate a single spectrum onto an (nspec, nexp) array of # wavelengths. To make it here, wave_new.ndim must be 2. fluxes_inter = np.zeros_like(wave_new, dtype=float) ivars_inter = np.zeros_like(wave_new, dtype=float) gpms_inter = np.zeros_like(wave_new, dtype=bool) - for ii in range(wave_new.shape[1]): - fluxes_inter[:,ii], ivars_inter[:,ii], gpms_inter[:,ii] \ - = interp_oned(wave_new[:,ii], waves, fluxes, ivars, gpms, sensfunc=sensfunc) - return fluxes_inter, ivars_inter, gpms_inter + if log10_blaze_function is not None: + log10_blazes_inter = np.zeros_like(wave_new, dtype=float) + for ii in range(wave_new.shape[1]): + fluxes_inter[:,ii], ivars_inter[:,ii], gpms_inter[:,ii], log10_blazes_inter[:, ii] \ + = interp_oned(wave_new[:,ii], waves, fluxes, ivars, gpms, log10_blaze_function=log10_blaze_function, sensfunc=sensfunc) + else: + for ii in range(wave_new.shape[1]): + fluxes_inter[:,ii], ivars_inter[:,ii], gpms_inter[:,ii], _ \ + = interp_oned(wave_new[:,ii], waves, fluxes, ivars, gpms, log10_blaze_function=log10_blaze_function, sensfunc=sensfunc) + log10_blazes_inter=None + + return fluxes_inter, ivars_inter, gpms_inter, log10_blazes_inter def smooth_weights(inarr, gdmsk, sn_smooth_npix): @@ -693,8 +749,7 @@ def smooth_weights(inarr, gdmsk, sn_smooth_npix): sn_conv = convolution.convolve(sn_med2, gauss_kernel, boundary='extend') return sn_conv - -def sn_weights(waves, fluxes, ivars, masks, sn_smooth_npix, const_weights=False, +def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, const_weights=False, ivar_weights=False, relative_weights=False, verbose=False): """ @@ -703,83 +758,74 @@ def sn_weights(waves, fluxes, ivars, masks, sn_smooth_npix, const_weights=False, Parameters ---------- - waves : `numpy.ndarray`_ - Reference wavelength grid for all the spectra. If wave is a - 1d array the routine will assume that all spectra are on the - same wavelength grid. If wave is a 2-d array, it will use - the individual. shape = (nspec,) or (nspec, nexp) - fluxes : `numpy.ndarray`_ - Stack of (nspec, nexp) spectra where nexp = number of - exposures, and nspec is the length of the spectrum. - ivars : `numpy.ndarray`_ - Inverse variance noise vectors for the spectra; shape = (nspec, nexp) - masks : `numpy.ndarray`_ - Mask for stack of spectra. True=Good, False=Bad; shape = (nspec, nexp) - sn_smooth_npix : float - Number of pixels used for determining smoothly varying S/N ratio weights. + fluxes : list + List of len(nexp) containing the `numpy.ndarray`_ 1d float spectra. The + shapes in the list can be different. + ivars : list + List of len(nexp) containing the `numpy.ndarray`_ 1d float inverse + variances of the spectra. + gpms : list + List of len(nexp) containing the `numpy.ndarray`_ 1d float boolean good + pixel masks of the spectra. + sn_smooth_npix : float, optional + Number of pixels used for determining smoothly varying S/N ratio + weights. This can be set to None if const_weights is True, since then + wavelength dependent weights are not used. const_weights : bool, optional - Use a constant weights for each spectrum? + Use a constant weights for each spectrum? ivar_weights : bool, optional - Use inverse variance weighted scheme? + Use inverse variance weighted scheme? relative_weights : bool, optional - Calculate weights by fitting to the ratio of spectra? Note, relative weighting will - only work well when there is at least one spectrum with a reasonable S/N, and a continuum. - RJC note - This argument may only be better when the object being used has a strong - continuum + emission lines. The reference spectrum is assigned a value of 1 for all - wavelengths, and the weights of all other spectra will be determined relative to the - reference spectrum. This is particularly useful if you are dealing with highly variable - spectra (e.g. emission lines) and require a precision better than ~1 per cent. + Calculate weights by fitting to the ratio of spectra? Note, relative + weighting will only work well when there is at least one spectrum with a + reasonable S/N, and a continuum. RJC note - This argument may only be + better when the object being used has a strong continuum + emission + lines. The reference spectrum is assigned a value of 1 for all + wavelengths, and the weights of all other spectra will be determined + relative to the reference spectrum. This is particularly useful if you + are dealing with highly variable spectra (e.g. emission lines) and + require a precision better than ~1 per cent. verbose : bool, optional - Verbosity of print out. + Verbosity of print out. Returns ------- rms_sn : `numpy.ndarray`_ - Root mean square S/N value for each input spectra; shape (nexp,) - weights : `numpy.ndarray`_ - Weights to be applied to the spectra. These are - signal-to-noise squared weights. shape = (nspec, nexp) + Array of root-mean-square S/N value for each input spectra. Shape = (nexp,) + weights : list + List of len(nexp) containing the signal-to-noise squared weights to be + applied to the spectra. This output is aligned with the vector (or + vectors) provided in waves, i.e. it is a list of arrays of type + `numpy.ndarray`_ with the same shape as those in waves. """ + nexp = len(fluxes) + # Check that all the input lists have the same length + if len(ivars) != nexp or len(gpms) != nexp: + msgs.error("Input lists of spectra must have the same length") + + # Check sn_smooth_npix is set if const_weights=False + if sn_smooth_npix is None and not const_weights: + msgs.error('sn_smooth_npix cannot be None if const_weights=False') + # Give preference to ivar_weights if ivar_weights and relative_weights: msgs.warn("Performing inverse variance weights instead of relative weighting") relative_weights = False - # Check input - if fluxes.ndim == 1: - nstack = 1 - nspec = fluxes.shape[0] - wave_stack = waves.reshape((nspec, nstack)) - flux_stack = fluxes.reshape((nspec, nstack)) - ivar_stack = ivars.reshape((nspec, nstack)) - mask_stack = masks.reshape((nspec, nstack)) - elif fluxes.ndim == 2: - nspec, nstack = fluxes.shape - wave_stack = waves - flux_stack = fluxes - ivar_stack = ivars - mask_stack = masks - elif fluxes.ndim == 3: - nspec, norder, nexp = fluxes.shape - wave_stack = np.reshape(waves, (nspec, norder * nexp), order='F') - flux_stack = np.reshape(fluxes, (nspec, norder * nexp), order='F') - ivar_stack = np.reshape(ivars, (nspec, norder * nexp), order='F') - mask_stack = np.reshape(masks, (nspec, norder * nexp), order='F') - nstack = norder*nexp - else: - msgs.error('Unrecognized dimensionality for flux') - # Calculate S/N - sn_val = flux_stack*np.sqrt(ivar_stack) - sn_val_ma = np.ma.array(sn_val, mask=np.logical_not(mask_stack)) - sn_sigclip = stats.sigma_clip(sn_val_ma, sigma=3, maxiters=5) - # TODO: Update with sigma_clipped stats with our new cenfunc and std_func = mad_std - sn2 = (sn_sigclip.mean(axis=0).compressed())**2 #S/N^2 value for each spectrum - if sn2.shape[0] != nstack: - msgs.error('No unmasked value in one of the exposures. Check inputs.') - - rms_sn = np.sqrt(sn2) # Root Mean S/N**2 value for all spectra + sn_val, rms_sn, sn2 = [], [], [] + for iexp in range(nexp): + sn_val_iexp = fluxes[iexp]*np.sqrt(ivars[iexp]) + sn_val.append(sn_val_iexp) + sn_val_ma = np.ma.array(sn_val_iexp, mask=np.logical_not(gpms[iexp])) + sn_sigclip = stats.sigma_clip(sn_val_ma, sigma=3, maxiters=5) + sn2_iexp = sn_sigclip.mean()**2 # S/N^2 value for each spectrum + if sn2_iexp is np.ma.masked: + msgs.error(f'No unmasked value in iexp={iexp+1}/{nexp}. Check inputs.') + else: + sn2.append(sn2_iexp) + rms_sn.append(np.sqrt(sn2_iexp)) # Root Mean S/N**2 value for all spectra # Check if relative weights input if relative_weights: @@ -789,46 +835,47 @@ def sn_weights(waves, fluxes, ivars, masks, sn_smooth_npix, const_weights=False, msgs.info( "The reference spectrum (ref_spec={0:d}) has a typical S/N = {1:.3f}".format(ref_spec, sn2[ref_spec])) # Adjust the arrays to be relative - refscale = (sn_val[:, ref_spec] > 0) / (sn_val[:, ref_spec] + (sn_val[:, ref_spec] == 0)) - for iexp in range(nstack): + refscale = utils.inverse(sn_val[ref_spec]) + gpms_rel = [] + for iexp in range(nexp): # Compute the relative (S/N)^2 and update the mask sn2[iexp] /= sn2[ref_spec] - mask_stack[:, iexp] = mask_stack[:, iexp] & ((mask_stack[:, ref_spec]) | (sn_val[:, ref_spec] != 0)) - sn_val[:, iexp] *= refscale + gpms_rel.append(gpms[iexp] & ((gpms[ref_spec]) | (sn_val[ref_spec] != 0))) + sn_val[iexp] *= refscale + + sn_gpms = gpms_rel + else: + sn_gpms = gpms - # TODO: ivar weights is better than SN**2 or const_weights for merging orders. Eventually, we will change it to # TODO: Should ivar weights be deprecated?? # Initialise weights - weights = np.zeros_like(flux_stack) + weights = [] if ivar_weights: if verbose: msgs.info("Using ivar weights for merging orders") - for iexp in range(nstack): - weights[:, iexp] = smooth_weights(ivar_stack[:, iexp], mask_stack[:, iexp], sn_smooth_npix) + for ivar, mask in zip(ivars, gpms): + weights.append(smooth_weights(ivar, mask, sn_smooth_npix)) else: - for iexp in range(nstack): + for iexp in range(nexp): # Now if (rms_sn[iexp] < 3.0) or const_weights: weight_method = 'constant' - weights[:, iexp] = np.full(nspec, np.fmax(sn2[iexp], 1e-2)) # set the minimum to be 1e-2 to avoid zeros + weights.append(np.full(fluxes[iexp].size, np.fmax(sn2[iexp], 1e-2))) # set the minimum to be 1e-2 to avoid zeros else: weight_method = 'wavelength dependent' # JFH THis line is experimental but it deals with cases where the spectrum drops to zero. We thus # transition to using ivar_weights. This needs more work because the spectra are not rescaled at this point. # RJC - also note that nothing should be changed to sn_val is relative_weights=True #sn_val[sn_val[:, iexp] < 1.0, iexp] = ivar_stack[sn_val[:, iexp] < 1.0, iexp] - weights[:, iexp] = smooth_weights(sn_val[:, iexp]**2, mask_stack[:, iexp], sn_smooth_npix) + weights.append(smooth_weights(sn_val[iexp]**2, sn_gpms[iexp], sn_smooth_npix)) if verbose: msgs.info('Using {:s} weights for coadding, S/N '.format(weight_method) + '= {:4.2f}, weight = {:4.2f} for {:}th exposure'.format( - rms_sn[iexp], np.mean(weights[:, iexp]), iexp)) + rms_sn[iexp], np.mean(weights[iexp]), iexp)) - if fluxes.ndim == 3: - rms_sn = np.reshape(rms_sn, (norder, nexp), order='F') - weights = np.reshape(weights, (nspec, norder, nexp), order='F') # Finish - return rms_sn, weights + return np.array(rms_sn), weights # TODO: This was commented out and would need to be refactored if brought back @@ -894,42 +941,48 @@ def robust_median_ratio(flux, ivar, flux_ref, ivar_ref, mask=None, mask_ref=None Parameters ---------- - flux: `numpy.ndarray`_ - spectrum that will be rescaled. shape=(nspec,) - ivar: `numpy.ndarray`_ - inverse variance for the spectrum that will be rescaled. - Same shape as flux - flux_ref: `numpy.ndarray`_ - reference spectrum. Same shape as flux - mask: `numpy.ndarray`_, optional - boolean mask for the spectrum that will be rescaled. True=Good. - If not input, computed from inverse variance - ivar_ref: `numpy.ndarray`_, optional - inverse variance of reference spectrum. - mask_ref: `numpy.ndarray`_, optional - Boolean mask for reference spectrum. True=Good. If not input, computed from inverse variance. - ref_percentile: float, optional, default=70.0 - Percentile fraction used for selecting the minimum SNR cut from the reference spectrum. Pixels above this - percentile cut are deemed the "good" pixels and are used to compute the ratio. This must be a number - between 0 and 100. - min_good: float, optional, default = 0.05 - Minimum fraction of good pixels determined as a fraction of the total pixels for estimating the median ratio - maxiters: int, optional, default = 5 - Maximum number of iterations for astropy.stats.SigmaClip - sigrej: float, optional, default = 3.0 - Rejection threshold for astropy.stats.SigmaClip - max_factor: float, optional, default = 10.0, - Maximum allowed value of the returned ratio - snr_do_not_rescale: float, optional default = 1.0 - If the S/N ratio of the set of pixels (defined by upper ref_percentile in the reference spectrum) in the - input spectrum have a median value below snr_do_not_rescale, median rescaling will not be attempted - and the code returns ratio = 1.0. We also use this parameter to define the set of pixels (determined from - the reference spectrum) to compare for the rescaling. + flux : `numpy.ndarray`_ + spectrum that will be rescaled. shape=(nspec,) + ivar : `numpy.ndarray`_ + inverse variance for the spectrum that will be rescaled. Same shape as + flux + flux_ref : `numpy.ndarray`_ + reference spectrum. Same shape as flux + mask : `numpy.ndarray`_, optional + boolean mask for the spectrum that will be rescaled. True=Good. If not + input, computed from inverse variance + ivar_ref : `numpy.ndarray`_, optional + inverse variance of reference spectrum. + mask_ref : `numpy.ndarray`_, optional + Boolean mask for reference spectrum. True=Good. If not input, computed + from inverse variance. + ref_percentile : float, optional, default=70.0 + Percentile fraction used for selecting the minimum SNR cut from the + reference spectrum. Pixels above this percentile cut are deemed the + "good" pixels and are used to compute the ratio. This must be a number + between 0 and 100. + min_good : float, optional, default = 0.05 + Minimum fraction of good pixels determined as a fraction of the total + pixels for estimating the median ratio + maxiters : int, optional, default = 5 + Maximum number of iterations for astropy.stats.SigmaClip + sigrej : float, optional, default = 3.0 + Rejection threshold for astropy.stats.SigmaClip + max_factor : float, optional, default = 10.0, + Maximum allowed value of the returned ratio + snr_do_not_rescale : float, optional, default = 1.0 + If the S/N ratio of the set of pixels (defined by upper ref_percentile + in the reference spectrum) in the input spectrum have a median value + below snr_do_not_rescale, median rescaling will not be attempted and the + code returns ratio = 1.0. We also use this parameter to define the set + of pixels (determined from the reference spectrum) to compare for the + rescaling. Returns ------- - ratio: float - the number that must be multiplied into flux in order to get it to match up with flux_ref + ratio : float + the number that must be multiplied into flux in order to get it to match + up with flux_ref """ ## Mask for reference spectrum and your spectrum @@ -950,13 +1003,13 @@ def robust_median_ratio(flux, ivar, flux_ref, ivar_ref, mask=None, mask_ref=None # Take the best part of the higher SNR reference spectrum sigclip = stats.SigmaClip(sigma=sigrej, maxiters=maxiters, cenfunc='median', stdfunc=utils.nan_mad_std) - flux_ref_ma = np.ma.MaskedArray(flux_ref, np.invert(calc_mask)) + flux_ref_ma = np.ma.MaskedArray(flux_ref, np.logical_not(calc_mask)) flux_ref_clipped, lower, upper = sigclip(flux_ref_ma, masked=True, return_bounds=True) - mask_ref_clipped = np.invert(flux_ref_clipped.mask) # mask_stack = True are good values + mask_ref_clipped = np.logical_not(flux_ref_clipped.mask) # mask_stack = True are good values - flux_ma = np.ma.MaskedArray(flux_ref, np.invert(calc_mask)) + flux_ma = np.ma.MaskedArray(flux_ref, np.logical_not(calc_mask)) flux_clipped, lower, upper = sigclip(flux_ma, masked=True, return_bounds=True) - mask_clipped = np.invert(flux_clipped.mask) # mask_stack = True are good values + mask_clipped = np.logical_not(flux_clipped.mask) # mask_stack = True are good values new_mask = mask_ref_clipped & mask_clipped @@ -981,121 +1034,6 @@ def robust_median_ratio(flux, ivar, flux_ref, ivar_ref, mask=None, mask_ref=None return ratio -def order_median_scale(waves, fluxes, ivars, masks, min_good=0.05, maxiters=5, - max_factor=10., sigrej=3, debug=False, show=False): - ''' - Function to scaling different orders by the median S/N - - - Args: - waves (`numpy.ndarray`_): wavelength array of your spectra with the shape of (nspec, norder) - fluxes (`numpy.ndarray`_): flux array of your spectra with the shape of (nspec, norder) - ivars (`numpy.ndarray`_): ivar array of your spectra with the shape of (nspec, norder) - masks (`numpy.ndarray`_): mask for your spectra with the shape of (nspec, norder) - min_good (float, optional): minimum fraction of the total number of good pixels needed for estimate the median ratio - maxiters (int or float, optional): maximum iterations for rejecting outliers - max_factor (float, optional): maximum scale factor - sigrej (float, optional): sigma used for rejecting outliers - debug (bool, optional): if True show intermediate QA - show (bool, optional): if True show the final QA - - Returns: - tuple: (1) fluxes_new (`numpy.ndarray`_): re-scaled fluxes with the shape - of (nspec, norder). (2) ivars_new (`numpy.ndarray`_): re-scaled ivars - with the shape of (nspec, norder) (3) order_ratios (`numpy.ndarray`_): an - array of scale factor with the length of norder - ''' - - norder = np.shape(waves)[1] - order_ratios = np.ones(norder) - - ## re-scale bluer orders to match the reddest order. - # scaling spectrum order by order. We use the reddest order as the reference since slit loss in redder is smaller - for ii in range(norder - 1): - iord = norder - ii - 1 - wave_blue, flux_blue, ivar_blue, mask_blue = waves[:, iord-1], fluxes[:, iord-1],\ - ivars[:, iord-1], masks[:, iord-1] - - wave_red_tmp, flux_red_tmp = waves[:, iord], fluxes[:, iord]*order_ratios[iord] - ivar_red_tmp, mask_red_tmp = ivars[:, iord]*1.0/order_ratios[iord]**2, masks[:, iord] - wave_mask = wave_red_tmp>1.0 - wave_red, flux_red, ivar_red, mask_red = wave_red_tmp[wave_mask], flux_red_tmp[wave_mask], \ - ivar_red_tmp[wave_mask], mask_red_tmp[wave_mask], - - # interpolate iord-1 (bluer) to iord-1 (redder) - flux_blue_inter, ivar_blue_inter, mask_blue_inter = interp_spec(wave_red, wave_blue, flux_blue, ivar_blue, mask_blue) - - npix_overlap = np.sum(mask_blue_inter & mask_red) - percentile_iord = np.fmax(100.0 * (npix_overlap / np.sum(mask_red)-0.05), 10) - - mask_both = mask_blue_inter & mask_red - snr_median_red = np.median(flux_red[mask_both]*np.sqrt(ivar_red[mask_both])) - snr_median_blue = np.median(flux_blue_inter[mask_both]*np.sqrt(ivar_blue_inter[mask_both])) - - ## TODO: we set the SNR to be minimum of 300 to turn off the scaling but we need the QA plot - ## need to think more about whether we need to scale different orders, it seems make the spectra - ## much bluer than what it should be. - if (snr_median_blue>300.0) & (snr_median_red>300.0): - order_ratio_iord = robust_median_ratio(flux_blue_inter, ivar_blue_inter, flux_red, ivar_red, mask=mask_blue_inter, - mask_ref=mask_red, ref_percentile=percentile_iord, min_good=min_good, - maxiters=maxiters, max_factor=max_factor, sigrej=sigrej) - order_ratios[iord - 1] = np.fmax(np.fmin(order_ratio_iord, max_factor), 1.0/max_factor) - msgs.info('Scaled {}th order to {}th order by {:}'.format(iord-1, iord, order_ratios[iord-1])) - else: - if ii>0: - order_ratios[iord - 1] = order_ratios[iord] - msgs.warn('Scaled {}th order to {}th order by {:} using the redder order scaling ' - 'factor'.format(iord-1, iord, order_ratios[iord-1])) - else: - msgs.warn('The SNR in the overlapped region is too low or there is not enough overlapped pixels.'+ msgs.newline() + - 'Median scale between order {:} and order {:} was not attempted'.format(iord-1, iord)) - - if debug: - plt.figure(figsize=(12, 8)) - plt.plot(wave_red[mask_red], flux_red[mask_red], 'k-', label='reference spectrum') - plt.plot(wave_blue[mask_blue], flux_blue[mask_blue],color='dodgerblue', lw=3, label='raw spectrum') - plt.plot(wave_blue[mask_blue], flux_blue[mask_blue]*order_ratios[iord-1], color='r', - alpha=0.5, label='re-scaled spectrum') - ymin, ymax = get_ylim(flux_blue, ivar_blue, mask_blue) - plt.ylim([ymin, ymax]) - plt.xlim([np.min(wave_blue[mask_blue]), np.max(wave_red[mask_red])]) - plt.legend() - plt.xlabel('wavelength') - plt.ylabel('Flux') - plt.show() - - # Update flux and ivar - fluxes_new = np.zeros_like(fluxes) - ivars_new = np.zeros_like(ivars) - for ii in range(norder): - fluxes_new[:, ii] *= order_ratios[ii] - ivars_new[:, ii] *= 1.0/order_ratios[ii]**2 - - if show: - plt.figure(figsize=(12, 8)) - ymin = [] - ymax = [] - for ii in range(norder): - wave_stack_iord = waves[:, ii] - flux_stack_iord = fluxes_new[:, ii] - ivar_stack_iord = ivars_new[:, ii] - mask_stack_iord = masks[:, ii] - med_width = (2.0 * np.ceil(0.1 / 10.0 * np.size(wave_stack_iord[mask_stack_iord])) + 1).astype(int) - flux_med, ivar_med = median_filt_spec(flux_stack_iord, ivar_stack_iord, mask_stack_iord, med_width) - plt.plot(wave_stack_iord[mask_stack_iord], flux_med[mask_stack_iord], alpha=0.7) - #plt.plot(wave_stack_iord[mask_stack_iord], flux_stack_iord[mask_stack_iord], alpha=0.5) - # plt.plot(wave_stack_iord[mask_stack_iord],1.0/np.sqrt(ivar_stack_iord[mask_stack_iord])) - ymin_ii, ymax_ii = get_ylim(flux_stack_iord, ivar_stack_iord, mask_stack_iord) - ymax.append(ymax_ii) - ymin.append(ymin_ii) - plt.xlim([np.min(waves[masks]), np.max(waves[masks])]) - plt.ylim([-0.15*np.median(ymax), 1.5*np.median(ymax)]) - plt.xlabel('Wavelength ($\\rm\\AA$)') - plt.ylabel('Flux') - plt.show() - - return fluxes_new, ivars_new, order_ratios - def scale_spec(wave, flux, ivar, sn, wave_ref, flux_ref, ivar_ref, mask=None, mask_ref=None, scale_method='auto', min_good=0.05, ref_percentile=70.0, maxiters=5, sigrej=3, max_median_factor=10.0, @@ -1160,12 +1098,14 @@ def scale_spec(wave, flux, ivar, sn, wave_ref, flux_ref, ivar_ref, mask=None, ma Returns ------- - Multiple items: tuple - (1) flux_scale: ndarray (nspec,) scaled spectrum; (2) - ivar_scale: ndarray (nspec,) inverse variance for scaled - spectrum; (3) scale: `numpy.ndarray`_ (nspec,) scale factor applied to - the spectrum and inverse variance; (4) scale_method: str, method - that was used to scale the spectra. + flux_scale : `numpy.ndarray`_, shape is (nspec,) + Scaled spectrum + ivar_scale : `numpy.ndarray`_, shape is (nspec,) + Inverse variance of scaled spectrum + scale : `numpy.ndarray`_, shape is (nspec,) + Scale factor applied to the spectrum and inverse variance + method_used : :obj:`str` + Method that was used to scale the spectra. """ if mask is None: @@ -1175,7 +1115,7 @@ def scale_spec(wave, flux, ivar, sn, wave_ref, flux_ref, ivar_ref, mask=None, ma # Interpolate the reference spectrum onto the wavelengths of the spectrum that will be rescaled - flux_ref_int, ivar_ref_int, mask_ref_int = interp_spec(wave, wave_ref, flux_ref, ivar_ref, mask_ref) + flux_ref_int, ivar_ref_int, mask_ref_int, _ = interp_spec(wave, wave_ref, flux_ref, ivar_ref, mask_ref) # estimates the SNR of each spectrum and the stacked mean SNR #rms_sn, weights = sn_weights(wave, flux, ivar, mask, sn_smooth_npix) @@ -1235,71 +1175,78 @@ def scale_spec(wave, flux, ivar, sn, wave_ref, flux_ref, ivar_ref, mask=None, ma return flux_scale, ivar_scale, scale, method_used - -def compute_stack(wave_grid, waves, fluxes, ivars, masks, weights, min_weight=1e-8): - ''' - Compute a stacked spectrum from a set of exposures on the specified wave_grid with proper treatment of - weights and masking. This code uses np.histogram to combine the data using NGP and does not perform any - interpolations and thus does not correlate errors. It uses wave_grid to determine the set of wavelength bins that - the data are averaged on. The final spectrum will be on an ouptut wavelength grid which is not the same as wave_grid. - The ouput wavelength grid is the weighted average of the individual wavelengths used for each exposure that fell into - a given wavelength bin in the input wave_grid. This 1d coadding routine thus maintains the independence of the - errors for each pixel in the combined spectrum and computes the weighted averaged wavelengths of each pixel - in an analogous way to the 2d extraction procedure which also never interpolates to avoid correlating erorrs. +def compute_stack(wave_grid, waves, fluxes, ivars, gpms, weights, min_weight=1e-8): + """ + Compute a stacked spectrum from a set of exposures on the specified + wave_grid with proper treatment of weights and masking. This code uses + np.histogram to combine the data using NGP and does not perform any + interpolations and thus does not correlate errors. It uses wave_grid to + determine the set of wavelength bins that the data are averaged on. The + final spectrum will be on an ouptut wavelength grid which is not the same as + wave_grid. The ouput wavelength grid is the weighted average of the + individual wavelengths used for each exposure that fell into a given + wavelength bin in the input wave_grid. This 1d coadding routine thus + maintains the independence of the errors for each pixel in the combined + spectrum and computes the weighted averaged wavelengths of each pixel in an + analogous way to the 2d extraction procedure which also never interpolates + to avoid correlating erorrs. Parameters ---------- - wave_grid: `numpy.ndarray`_ - new wavelength grid desired. This will typically be a reguarly spaced grid created by the get_wave_grid routine. - The reason for the ngrid+1 is that this is the general way to specify a set of bins if you desire ngrid - bin centers, i.e. the output stacked spectra have ngrid elements. The spacing of this grid can be regular in - lambda (better for multislit) or log lambda (better for echelle). This new wavelength grid should be designed - with the sampling of the data in mind. For example, the code will work fine if you choose the sampling to be - too fine, but then the number of exposures contributing to any given wavelength bin will be one or zero in the - limiting case of very small wavelength bins. For larger wavelength bins, the number of exposures contributing - to a given bin will be larger. shape=(ngrid +1,) - waves: `numpy.ndarray`_ - wavelength arrays for spectra to be stacked. Note that the wavelength grids can in general be different for - each exposure and irregularly spaced. - shape=(nspec, nexp) - fluxes: `numpy.ndarray`_ - fluxes for each exposure on the waves grid - shape=(nspec, nexp) - ivars: `numpy.ndarray`_ - Inverse variances for each exposure on the waves grid - shape=(nspec, nexp) - masks: `numpy.ndarray`_ - Boolean masks for each exposure on the waves grid. True=Good. - shape=(nspec, nexp) - weights: `numpy.ndarray`_ - Weights to be used for combining your spectra. These are computed using sn_weights - shape=(nspec, nexp) - min_weight: float, optional - Minimum allowed weight for any individual spectrum + wave_grid : `numpy.ndarray`_ + new wavelength grid desired. This will typically be a reguarly spaced + grid created by the get_wave_grid routine. The reason for the ngrid+1 + is that this is the general way to specify a set of bins if you desire + ngrid bin centers, i.e. the output stacked spectra have ngrid elements. + The spacing of this grid can be regular in lambda (better for multislit) + or log lambda (better for echelle). This new wavelength grid should be + designed with the sampling of the data in mind. For example, the code + will work fine if you choose the sampling to be too fine, but then the + number of exposures contributing to any given wavelength bin will be one + or zero in the limiting case of very small wavelength bins. For larger + wavelength bins, the number of exposures contributing to a given bin + will be larger. shape=(ngrid +1,) + waves : list + List of length nexp `numpy.ndarray`_ float wavelength arrays for spectra + to be stacked. Note that the wavelength grids can in general be + different for each exposure, irregularly spaced, and can have different + sizes. + fluxes : list + List of length nexp `numpy.ndarray`_ float flux arrays for spectra to be + stacked. These are aligned to the wavelength grids in waves. + ivars : list + List of length nexp `numpy.ndarray`_ float inverse variance arrays for + spectra to be stacked. These are aligned to the wavelength grids in + waves. + gpms : list + List of length nexp `numpy.ndarray`_ boolean good pixel mask arrays for + spectra to be stacked. These are aligned to the wavelength grids in + waves. True=Good. + weights : list + List of length nexp `numpy.ndarray`_ float weights to be used for + combining the spectra. These are aligned to the wavelength grids in + waves and were computed using sn_weights. + min_weight : float, optional + Minimum allowed weight for any individual spectrum Returns ------- - wave_stack: `numpy.ndarray`_ - Wavelength grid for stacked - spectrum. As discussed above, this is the weighted average - of the wavelengths of each spectrum that contriuted to a - bin in the input wave_grid wavelength grid. It thus has - ngrid elements, whereas wave_grid has ngrid+1 elements to - specify the ngrid total number of bins. Note that - wave_stack is NOT simply the wave_grid bin centers, since - it computes the weighted average. shape=(ngrid,) - flux_stack: `numpy.ndarray`_ + wave_stack : `numpy.ndarray`_ + Wavelength grid for stacked spectrum. As discussed above, this is the + weighted average of the wavelengths of each spectrum that contriuted to + a bin in the input wave_grid wavelength grid. It thus has ngrid + elements, whereas wave_grid has ngrid+1 elements to specify the ngrid + total number of bins. Note that wave_stack is NOT simply the wave_grid + bin centers, since it computes the weighted average. shape=(ngrid,) + flux_stack : `numpy.ndarray`_, shape=(ngrid,) Final stacked spectrum on wave_stack wavelength grid - shape=(ngrid,) - ivar_stack: `numpy.ndarray`_ + ivar_stack : `numpy.ndarray`_, shape=(ngrid,) Inverse variance spectrum on wave_stack wavelength grid. Errors are propagated according to weighting and masking. - shape=(ngrid,) - mask_stack: `numpy.ndarray`_ + gpm_stack : `numpy.ndarray`_, shape=(ngrid,) Boolean Mask for stacked spectrum on wave_stack wavelength grid. True=Good. - shape=(ngrid,) - nused: `numpy.ndarray`_ + nused : `numpy.ndarray`_, shape=(ngrid,) Number of exposures which contributed to each pixel in the wave_stack. Note that this is in general different from nexp because of masking, but also becuse of @@ -1307,16 +1254,24 @@ def compute_stack(wave_grid, waves, fluxes, ivars, masks, weights, min_weight=1e sometimes more spectral pixels in the irregularly gridded input wavelength array waves will land in one bin versus another depending on the sampling. - shape=(ngrid,) - ''' + """ #mask bad values and extreme values (usually caused by extreme low sensitivity at the edge of detectors) - ubermask = masks & (weights > 0.0) & (waves > 1.0) & (ivars > 0.0) & (utils.inverse(ivars)<1e10) - waves_flat = waves[ubermask].flatten() - fluxes_flat = fluxes[ubermask].flatten() - ivars_flat = ivars[ubermask].flatten() - vars_flat = utils.inverse(ivars_flat) - weights_flat = weights[ubermask].flatten() + #TODO cutting on the value of ivar is dicey for data in different units. This should be removed. + uber_gpms = [gpm & (weight > 0.0) & (wave > 1.0) & (ivar > 0.0) & (utils.inverse(ivar)<1e10) + for gpm, weight, wave, ivar in zip(gpms, weights, waves, ivars)] + waves_flat, fluxes_flat, ivars_flat, weights_flat = [], [], [], [] + for wave, flux, ivar, weight, ugpm in zip(waves, fluxes, ivars, weights, uber_gpms): + waves_flat += wave[ugpm].tolist() + fluxes_flat += flux[ugpm].tolist() + ivars_flat += ivar[ugpm].tolist() + weights_flat += weight[ugpm].tolist() + + waves_flat = np.array(waves_flat) + fluxes_flat = np.array(fluxes_flat) + ivars_flat = np.array(ivars_flat) + weights_flat = np.array(weights_flat) + vars_flat = utils.inverse(np.array(ivars_flat)) # Counts how many pixels in each wavelength bin nused, wave_edges = np.histogram(waves_flat,bins=wave_grid,density=False) @@ -1340,9 +1295,8 @@ def compute_stack(wave_grid, waves, fluxes, ivars, masks, weights, min_weight=1e ivar_stack = utils.inverse(var_stack) # New mask for the stack - mask_stack = (weights_total > min_weight) & (nused > 0.0) - - return wave_stack, flux_stack, ivar_stack, mask_stack, nused + gpm_stack = (weights_total > min_weight) & (nused > 0.0) + return wave_stack, flux_stack, ivar_stack, gpm_stack, nused def get_ylim(flux, ivar, mask): """ @@ -1493,11 +1447,11 @@ def coadd_iexp_qa(wave, flux, rejivar, mask, wave_stack, flux_stack, ivar_stack, ymin, ymax = get_ylim(flux_stack, ivar_stack, mask_stack) # Plot spectrum - rejmask = mask & np.invert(outmask) + rejmask = mask & np.logical_not(outmask) wave_mask = wave > 1.0 wave_stack_mask = wave_stack > 1.0 spec_plot.plot(wave[rejmask], flux[rejmask],'s',zorder=10,mfc='None', mec='r', label='rejected pixels') - spec_plot.plot(wave[np.invert(mask)], flux[np.invert(mask)],'v', zorder=10, mfc='None', mec='orange', + spec_plot.plot(wave[np.logical_not(mask)], flux[np.logical_not(mask)],'v', zorder=10, mfc='None', mec='orange', label='originally masked') if norder is None: @@ -1545,103 +1499,84 @@ def coadd_iexp_qa(wave, flux, rejivar, mask, wave_stack, flux_stack, ivar_stack, msgs.info("Wrote QA: {:s}".format(qafile)) plt.show() - -def weights_qa(waves, weights, masks, title=''): - ''' +def weights_qa(waves, weights, gpms, title='', colors=None): + """ Routine to make a QA plot for the weights used to compute a stacked spectrum. Parameters ---------- - wave: `numpy.ndarray`_ - wavelength array for spectra that went into a stack; - shape=(nspec, nexp) - weights: `numpy.ndarray`_ - (S/N)^2 weights for the exposures that went into a stack. This would have been computed by sn_weights - shape=(nspec, nexp) - mask: `numpy.ndarray`_ - Boolean array indicating pixels which were masked - in each individual exposure which go into the stack. - shape=(nspec, nexp) - title: str, optional - Title for the plot. - ''' - - if waves.ndim == 1: - nstack = 1 - nspec = waves.shape[0] - waves_stack = waves.reshape((nspec, nstack)) - weights_stack = weights.reshape((nspec, nstack)) - masks_stack = masks.reshape((nspec, nstack)) - elif waves.ndim == 2: - nspec, nstack = waves.shape - waves_stack = waves - weights_stack = weights - masks_stack = masks - elif weights.ndim == 3: - nspec, norder, nexp = waves.shape - waves_stack = np.reshape(waves, (nspec, norder * nexp), order='F') - weights_stack = np.reshape(weights, (nspec, norder * nexp), order='F') - masks_stack = np.reshape(masks, (nspec, norder * nexp), order='F') - nstack = norder*nexp - else: - msgs.error('Unrecognized dimensionality for waves') + wave : list + List of `numpy.ndarray`_ float 1d wavelength arrays for spectra that + went into a stack. + weights : list + List of `numpy.ndarray`_ float 1d (S/N)^2 weight arrays for the + exposures that went into a stack. This would have been computed by + sn_weights + gpm : list + List of `numpy.ndarray`_ boolean 1d good-pixel mask arrays for the + exposures that went into a stack. Good=True. + title : str, optional + Title for the plot. + """ + if colors is None: + colors = utils.distinct_colors(len(waves)) fig = plt.figure(figsize=(12, 8)) - for iexp in range(nstack): - wave_mask = waves_stack[:, iexp] > 1.0 - plt.plot(waves_stack[wave_mask,iexp], weights_stack[wave_mask,iexp]*masks_stack[wave_mask,iexp]) - - plt.xlim(waves_stack[(waves_stack > 1.0)].min(), waves_stack[(waves_stack > 1.0)].max()) + wave_min, wave_max = [], [] + for ii, (wave, weight, gpm) in enumerate(zip(waves, weights, gpms)): + wave_gpm = wave > 1.0 + plt.plot(wave[wave_gpm], weight[wave_gpm]*gpm[wave_gpm], color=colors[ii]) + wave_min.append(wave[wave_gpm].min()) + wave_max.append(wave[wave_gpm].max()) + + plt.xlim(np.min(wave_min), np.max(wave_max)) plt.xlabel('Wavelength (Angstrom)') plt.ylabel('Weights') plt.title(title, fontsize=16, color='red') plt.show() -def coadd_qa(wave, flux, ivar, nused, mask=None, tell=None, +def coadd_qa(wave, flux, ivar, nused, gpm=None, tell=None, title=None, qafile=None): - ''' - Routine to make QA plot of the final stacked spectrum. It works for both longslit/mulitslit, coadded individual - order spectrum of the Echelle data and the final coadd of the Echelle data. + """ + Routine to make QA plot of the final stacked spectrum. It works for both + longslit/mulitslit, coadded individual order spectrum of the Echelle data + and the final coadd of the Echelle data. Parameters ---------- - wave: `numpy.ndarray`_ - one-d wavelength array of your spectrum; - shape=(nspec,) - flux: `numpy.ndarray`_ - one-d flux array of your spectrum; - shape=(nspec,) - ivar: `numpy.ndarray`_ - one-d ivar array of your spectrum; - shape=(nspec,) - nused: `numpy.ndarray`_ - how many exposures used in the stack for each pixel, the same size with flux - shape=(nspec,) - mask: `numpy.ndarray`_, optional - boolean mask array for your spectrum; - shape=(nspec,) - tell: `numpy.ndarray`_, optional - one-d telluric array for your spectrum; shape=(nspec,) - title: str, optional - plot title - qafile: str, optional - QA file name - ''' + wave : `numpy.ndarray`_, shape=(nspec,) + one-d wavelength array of your spectrum + flux : `numpy.ndarray`_, shape=(nspec,) + one-d flux array of your spectrum + ivar : `numpy.ndarray`_, shape=(nspec,) + one-d ivar array of your spectrum + nused : `numpy.ndarray`_, shape=(nspec,) + how many exposures used in the stack for each pixel, the same size with + flux. + gpm : `numpy.ndarray`_, optional, shape=(nspec,) + boolean good pixel mask array for your spectrum. Good=True. + tell : `numpy.ndarray`_, optional, shape=(nspec,) + one-d telluric array for your spectrum + title : str, optional + plot title + qafile : str, optional + QA file name + """ #TODO: This routine should take a parset - if mask is None: - mask = ivar > 0.0 + if gpm is None: + gpm = ivar > 0.0 - wave_mask = wave > 1.0 - wave_min = wave[wave_mask].min() - wave_max = wave[wave_mask].max() + wave_gpm = wave > 1.0 + wave_min = wave[wave_gpm].min() + wave_max = wave[wave_gpm].max() fig = plt.figure(figsize=(12, 8)) # plot how may exposures you used at each pixel # [left, bottom, width, height] num_plot = fig.add_axes([0.10, 0.70, 0.80, 0.23]) spec_plot = fig.add_axes([0.10, 0.10, 0.80, 0.60]) - num_plot.plot(wave[wave_mask],nused[wave_mask],drawstyle='steps-mid',color='k',lw=2) + num_plot.plot(wave[wave_gpm],nused[wave_gpm],drawstyle='steps-mid',color='k',lw=2) num_plot.set_xlim([wave_min, wave_max]) num_plot.set_ylim([0.0, np.fmax(1.1*nused.max(), nused.max()+1.0)]) num_plot.set_ylabel('$\\rm N_{EXP}$') @@ -1649,22 +1584,22 @@ def coadd_qa(wave, flux, ivar, nused, mask=None, tell=None, num_plot.yaxis.set_minor_locator(NullLocator()) # Plot spectrum - spec_plot.plot(wave[wave_mask], flux[wave_mask], color='black', drawstyle='steps-mid',zorder=1,alpha=0.8, label='Single exposure') - spec_plot.plot(wave[wave_mask], np.sqrt(utils.inverse(ivar[wave_mask])),zorder=2, color='red', alpha=0.7, + spec_plot.plot(wave[wave_gpm], flux[wave_gpm], color='black', drawstyle='steps-mid',zorder=1,alpha=0.8, label='Single exposure') + spec_plot.plot(wave[wave_gpm], np.sqrt(utils.inverse(ivar[wave_gpm])),zorder=2, color='red', alpha=0.7, drawstyle='steps-mid', linestyle=':') # Get limits - ymin, ymax = get_ylim(flux, ivar, mask) + ymin, ymax = get_ylim(flux, ivar, gpm) # Plot transmission - if (np.max(wave[mask])>9000.0) and (tell is None): + if (np.max(wave[gpm])>9000.0) and (tell is None): skytrans_file = data.get_skisim_filepath('atm_transmission_secz1.5_1.6mm.dat') skycat = np.genfromtxt(skytrans_file,dtype='float') scale = 0.8*ymax spec_plot.plot(skycat[:,0]*1e4,skycat[:,1]*scale,'m-',alpha=0.5,zorder=11) elif tell is not None: scale = 0.8*ymax - spec_plot.plot(wave[wave_mask], tell[wave_mask]*scale, drawstyle='steps-mid', color='m',alpha=0.5,zorder=11) + spec_plot.plot(wave[wave_gpm], tell[wave_gpm]*scale, drawstyle='steps-mid', color='m',alpha=0.5,zorder=11) spec_plot.set_ylim([ymin, ymax]) spec_plot.set_xlim([wave_min, wave_max]) @@ -1682,7 +1617,6 @@ def coadd_qa(wave, flux, ivar, nused, mask=None, tell=None, msgs.info("Wrote QA: {:s}".format(qafile)) plt.show() - def update_errors(fluxes, ivars, masks, fluxes_stack, ivars_stack, masks_stack, sn_clip=30.0, title='', debug=False): ''' @@ -1799,59 +1733,75 @@ def update_errors(fluxes, ivars, masks, fluxes_stack, ivars_stack, masks_stack, return rejivars, sigma_corrs, outchi, maskchi -def spec_reject_comb(wave_grid, waves, fluxes, ivars, masks, weights, sn_clip=30.0, lower=3.0, upper=3.0, +def spec_reject_comb(wave_grid, wave_grid_mid, waves_list, fluxes_list, ivars_list, gpms_list, weights_list, sn_clip=30.0, lower=3.0, upper=3.0, maxrej=None, maxiter_reject=5, title='', debug=False, verbose=False): """ - Routine for executing the iterative combine and rejection of a set of spectra to compute a final stacked spectrum. + Routine for executing the iterative combine and rejection of a set of + spectra to compute a final stacked spectrum. Parameters ---------- - wave_grid: `numpy.ndarray`_ - new wavelength grid desired. This will typically be a reguarly spaced grid created by the get_wave_grid routine. - The reason for the ngrid+1 is that this is the general way to specify a set of bins if you desire ngrid - bin centers, i.e. the output stacked spectra have ngrid elements. The spacing of this grid can be regular in - lambda (better for multislit) or log lambda (better for echelle). This new wavelength grid should be designed - with the sampling of the data in mind. For example, the code will work fine if you choose the sampling to be - too fine, but then the number of exposures contributing to any given wavelength bin will be one or zero in the - limiting case of very small wavelength bins. For larger wavelength bins, the number of exposures contributing - to a given bin will be larger. - shape=(ngrid +1,) - waves: `numpy.ndarray`_ - wavelength arrays for spectra to be stacked. Note that the wavelength grids can in general be different for - each exposure and irregularly spaced. - shape=(nspec, nexp) - fluxes: `numpy.ndarray`_ - fluxes for each exposure on the waves grid - ivars: `numpy.ndarray`_ - Inverse variances for each exposure on the waves grid - masks: `numpy.ndarray`_ - Boolean masks for each exposure on the waves grid. True=Good. - weights: `numpy.ndarray`_ - Weights to be used for combining your spectra. These are computed using sn_weights - sn_clip: float, optional, default=30.0 - Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection - in high S/N ratio spectrum which neverthless differ at a level greater than the implied S/N due to - systematics. - lower: float, optional, default=3.0 - lower rejection threshold for djs_reject - upper: float, optional, default=3.0 - upper rejection threshold for djs_reject - maxrej: int, optional, default=None - maximum number of pixels to reject in each iteration for djs_reject. - maxiter_reject: int, optional, default=5 - maximum number of iterations for stacking and rejection. The code stops iterating either when - the output mask does not change betweeen successive iterations or when maxiter_reject is reached. - title: str, optional - Title for QA plot - debug: bool, optional, default=False - Show QA plots useful for debugging. - verbose: bool, optional, default=False - Level + wave_grid : `numpy.ndarray`_ + new wavelength grid desired. This will typically be a reguarly spaced + grid created by the get_wave_grid routine. The reason for the ngrid+1 + is that this is the general way to specify a set of bins if you desire + ngrid bin centers, i.e. the output stacked spectra have ngrid elements. + The spacing of this grid can be regular in lambda (better for multislit) + or log lambda (better for echelle). This new wavelength grid should be + designed with the sampling of the data in mind. For example, the code + will work fine if you choose the sampling to be too fine, but then the + number of exposures contributing to any given wavelength bin will be one + or zero in the limiting case of very small wavelength bins. For larger + wavelength bins, the number of exposures contributing to a given bin + will be larger. shape=(ngrid +1,) + wave_grid_mid : `numpy.ndarray`_ + Wavelength grid (in Angstrom) evaluated at the bin centers, + uniformly-spaced either in lambda or log10-lambda/velocity. See + core.wavecal.wvutils.py for more. shape=(ngrid,) + waves : list + List of length nexp float `numpy.ndarray`_ wavelength arrays for spectra + to be stacked. Note that the wavelength grids can in general be + different for each exposure, irregularly spaced, and have different + sizes. + fluxes : list + List of length nexp float `numpy.ndarray`_ fluxes for each exposure + aligned with the wavelength arrays in waves + ivars : list + List of length nexp float `numpy.ndarray`_ inverse variances for each + exposure aligned with the wavelength arrays in waves + gpms : list + List of length nexp boolean `numpy.ndarray`_ good pixel masks for each + exposure aligned with the wavelength arrays in waves. True=Good. + weights : list + List of length nexp float `numpy.ndarray`_ weights for each exposure + aligned with the wavelength arrays in waves. These are computed using + sn_weights + sn_clip : float, optional, default=30.0 + Errors are capped during rejection so that the S/N is never greater than + sn_clip. This prevents overly aggressive rejection in high S/N ratio + spectrum which neverthless differ at a level greater than the implied + S/N due to systematics. + lower : float, optional, default=3.0 + lower rejection threshold for djs_reject + upper : float, optional, default=3.0 + upper rejection threshold for djs_reject + maxrej : int, optional, default=None + maximum number of pixels to reject in each iteration for djs_reject. + maxiter_reject : int, optional, default=5 + maximum number of iterations for stacking and rejection. The code stops + iterating either when the output mask does not change betweeen + successive iterations or when maxiter_reject is reached. + title : str, optional + Title for QA plot + debug : bool, optional, default=False + Show QA plots useful for debugging. + verbose : bool, optional, default=False + Level of verbosity. Returns ------- - wave_stack: `numpy.ndarray`_ + wave_stack : `numpy.ndarray`_ Wavelength grid for stacked spectrum. As discussed above, this is the weighted average of the wavelengths of each spectrum that contriuted to a @@ -1861,25 +1811,20 @@ def spec_reject_comb(wave_grid, waves, fluxes, ivars, masks, weights, sn_clip=30 wave_stack is NOT simply the wave_grid bin centers, since it computes the weighted average. shape=(ngrid,) - flux_stack: `numpy.ndarray`_ + flux_stack : `numpy.ndarray`_ Final stacked spectrum on wave_stack wavelength grid shape=(ngrid,) - ivar_stack: `numpy.ndarray`_ + ivar_stack : `numpy.ndarray`_ Inverse variance spectrum on wave_stack wavelength grid. Erors are propagated according to weighting and masking. shape=(ngrid,) - mask_stack: `numpy.ndarray`_ + gpm_stack : `numpy.ndarray`_ Boolean mask for stacked spectrum on wave_stack wavelength grid. True=Good. shape=(ngrid,) - outmask: `numpy.ndarray`_ - Output bool mask with shape=(nspec, nexp) - indicating which pixels are rejected in each exposure of - the original input spectra after performing all of the - iterations of combine/rejection - nused: `numpy.ndarray`_ + nused : `numpy.ndarray`_ Number of exposures which contributed to each pixel in the wave_stack. Note that this is in general different from nexp because of masking, @@ -1888,32 +1833,52 @@ def spec_reject_comb(wave_grid, waves, fluxes, ivars, masks, weights, sn_clip=30 irregularly gridded input wavelength array waves will land in one bin versus another depending on the sampling. shape=(ngrid,) - + out_gpms : list + List of length nexp output `numpy.ndarray`_ good pixel masks for each + exposure aligned with the wavelength arrays in waves indicating which + pixels are rejected in each exposure of the original input spectra after + performing all of the iterations of combine/rejection """ - thismask = np.copy(masks) + # Conver the input lists to arrays. This is currently needed since the rejection routine pydl.djs_reject requires + # arrays as input. + waves, nspec_list = utils.explist_to_array(waves_list, pad_value=0.0) + fluxes, _ = utils.explist_to_array(fluxes_list, pad_value=0.0) + ivars, _ = utils.explist_to_array(ivars_list, pad_value=0.0) + weights, _ = utils.explist_to_array(weights_list, pad_value=0.0) + gpms, _ = utils.explist_to_array(gpms_list, pad_value=False) + this_gpms = np.copy(gpms) iter = 0 qdone = False while (not qdone) and (iter < maxiter_reject): - wave_stack, flux_stack, ivar_stack, mask_stack, nused = compute_stack( - wave_grid, waves, fluxes, ivars, thismask, weights) - flux_stack_nat, ivar_stack_nat, mask_stack_nat = interp_spec( - waves, wave_stack, flux_stack, ivar_stack, mask_stack) - rejivars, sigma_corrs, outchi, maskchi = update_errors(fluxes, ivars, thismask, - flux_stack_nat, ivar_stack_nat, mask_stack_nat, - sn_clip=sn_clip) - thismask, qdone = pydl.djs_reject(fluxes, flux_stack_nat, outmask=thismask,inmask=masks, invvar=rejivars, + # Compute the stack + #from IPython import embed + #embed() + wave_stack, flux_stack, ivar_stack, gpm_stack, nused = compute_stack( + wave_grid, waves_list, fluxes_list, ivars_list, utils.array_to_explist(this_gpms, nspec_list=nspec_list), weights_list) + # Interpolate the individual spectra onto the wavelength grid of the stack. Use wave_grid_mid for this + # since it has no masked values + flux_stack_nat, ivar_stack_nat, gpm_stack_nat, _ = interp_spec( + waves, wave_grid_mid, flux_stack, ivar_stack, gpm_stack) + ## TESTING + #nused_stack_nat, _, _ = interp_spec( + # waves, wave_grid_mid, nused, ivar_stack, mask_stack) + #embed() + rejivars, sigma_corrs, outchi, chigpm = update_errors(fluxes, ivars, this_gpms, + flux_stack_nat, ivar_stack_nat, gpm_stack_nat, sn_clip=sn_clip) + this_gpms, qdone = pydl.djs_reject(fluxes, flux_stack_nat, outmask=this_gpms,inmask=gpms, invvar=rejivars, lower=lower,upper=upper, maxrej=maxrej, sticky=False) iter += 1 if (iter == maxiter_reject) & (maxiter_reject != 0): msgs.warn('Maximum number of iterations maxiter={:}'.format(maxiter_reject) + ' reached in spec_reject_comb') - outmask = np.copy(thismask) + out_gpms = np.copy(this_gpms) + out_gpms_list = utils.array_to_explist(out_gpms, nspec_list=nspec_list) # print out a summary of how many pixels were rejected nexp = waves.shape[1] - nrej = np.sum(np.invert(outmask) & masks, axis=0) - norig = np.sum((waves > 1.0) & np.invert(masks), axis=0) + nrej = np.sum(np.logical_not(out_gpms) & gpms, axis=0) + norig = np.sum((waves > 1.0) & np.logical_not(gpms), axis=0) if verbose: for iexp in range(nexp): @@ -1921,124 +1886,137 @@ def spec_reject_comb(wave_grid, waves, fluxes, ivars, masks, weights, sn_clip=30 msgs.info("Rejected {:d} pixels in exposure {:d}/{:d}".format(nrej[iexp], iexp, nexp)) # Compute the final stack using this outmask - wave_stack, flux_stack, ivar_stack, mask_stack, nused = compute_stack(wave_grid, waves, fluxes, ivars, outmask, weights) + wave_stack, flux_stack, ivar_stack, gpm_stack, nused = compute_stack( + wave_grid, waves_list, fluxes_list, ivars_list, out_gpms_list, weights_list) # Used only for plotting below if debug: # TODO Add a line here to optionally show the distribution of all pixels about the stack as we do for X-shooter. - #flux_stack_nat, ivar_stack_nat, mask_stack_nat = interp_spec(waves, wave_stack, flux_stack, ivar_stack, mask_stack) for iexp in range(nexp): # plot the residual distribution for each exposure title_renorm = title + ': Error distriution about stack for exposure {:d}/{:d}'.format(iexp,nexp) - renormalize_errors_qa(outchi[:, iexp], maskchi[:, iexp], sigma_corrs[iexp], title=title_renorm) + renormalize_errors_qa(outchi[:, iexp], chigpm[:, iexp], sigma_corrs[iexp], title=title_renorm) # plot the rejections for each exposures title_coadd_iexp = title + ': nrej={:d} pixels rejected,'.format(nrej[iexp]) + \ ' norig={:d} originally masked,'.format(norig[iexp]) + \ ' for exposure {:d}/{:d}'.format(iexp,nexp) - coadd_iexp_qa(waves[:, iexp], fluxes[:, iexp], rejivars[:, iexp], masks[:, iexp], wave_stack, flux_stack, - ivar_stack, mask_stack, outmask[:, iexp], qafile=None, title=title_coadd_iexp) + # JFH: QA should use wave_grid_mid + coadd_iexp_qa(waves[:, iexp], fluxes[:, iexp], rejivars[:, iexp], gpms[:, iexp], wave_grid_mid, flux_stack, + ivar_stack, gpm_stack, out_gpms[:, iexp], qafile=None, title=title_coadd_iexp) # weights qa title_weights = title + ': Weights Used -- nrej={:d} total pixels rejected,'.format(np.sum(nrej)) + \ ' norig={:d} originally masked'.format(np.sum(norig)) - weights_qa(waves, weights, outmask, title=title_weights) + weights_qa(waves_list, weights_list, out_gpms_list, title=title_weights) - return wave_stack, flux_stack, ivar_stack, mask_stack, outmask, nused -def scale_spec_stack(wave_grid, waves, fluxes, ivars, masks, sn, weights, + return wave_stack, flux_stack, ivar_stack, gpm_stack, nused, out_gpms_list + +def scale_spec_stack(wave_grid, wave_grid_mid, waves, fluxes, ivars, gpms, sns, weights, ref_percentile=70.0, maxiter_scale=5, sigrej_scale=3.0, scale_method='auto', hand_scale=None, sn_min_polyscale=2.0, sn_min_medscale=0.5, debug=False, show=False): """ - THIS NEEDS A PROPER DESCRIPTION + Scales a set of spectra to a common flux scale. This is done by first + computing a stack of the spectra and then scaling each spectrum to match the + composite with the :func:`scale_spec` algorithm which performs increasingly + sophisticated scaling depending on the S/N ratio. Parameters ---------- - wave_grid: `numpy.ndarray`_ - New wavelength grid desired. This will typically be a reguarly spaced grid created by the get_wave_grid routine. - The reason for the ngrid+1 is that this is the general way to specify a set of bins if you desire ngrid - bin centers, i.e. the output stacked spectra have ngrid elements. The spacing of this grid can be regular in - lambda (better for multislit) or log lambda (better for echelle). This new wavelength grid should be designed - with the sampling of the data in mind. For example, the code will work fine if you choose the sampling to be - too fine, but then the number of exposures contributing to any given wavelength bin will be one or zero in the - limiting case of very small wavelength bins. For larger wavelength bins, the number of exposures contributing - to a given bin will be larger. - shape=(ngrid +1,) - waves: `numpy.ndarray`_ - wavelength arrays for spectra to be stacked. Note that the wavelength grids can in general be different for - each exposure and irregularly spaced. - shape=(nspec, nexp) - fluxes: `numpy.ndarray`_ - fluxes for each exposure on the waves grid - shape=(nspec, nexp) - ivars: `numpy.ndarray`_ - Inverse variances for each exposure on the waves grid - shape=(nspec, nexp) - masks: `numpy.ndarray`_ - Bool masks for each exposure on the waves grid. True=Good. - shape=(nspec, nexp) - sn: `numpy.ndarray`_ - sn of each spectrum in the stack used to determine which scaling method should be used. This can - be computed using sn_weights. shape=(nexp,) - sigrej_scale: float, optional, default=3.0 - Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. - ref_percentile: float, optional, default=70.0 - percentile fraction cut used for selecting minimum SNR cut for robust_median_ratio - maxiter_scale: int, optional, default=5 + wave_grid : `numpy.ndarray`_ + New wavelength grid desired. This will typically be a reguarly spaced + grid created by the get_wave_grid routine. The reason for the ngrid+1 + is that this is the general way to specify a set of bins if you desire + ngrid bin centers, i.e. the output stacked spectra have ngrid elements. + The spacing of this grid can be regular in lambda (better for multislit) + or log lambda (better for echelle). This new wavelength grid should be + designed with the sampling of the data in mind. For example, the code + will work fine if you choose the sampling to be too fine, but then the + number of exposures contributing to any given wavelength bin will be one + or zero in the limiting case of very small wavelength bins. For larger + wavelength bins, the number of exposures contributing to a given bin + will be larger. shape=(ngrid +1,) + wave_grid_mid : `numpy.ndarray`_ + Wavelength grid (in Angstrom) evaluated at the bin centers, + uniformly-spaced either in lambda or log10-lambda/velocity. See + core.wavecal.wvutils.py for more. shape=(ngrid,) + waves : list + List of length nexp float `numpy.ndarray`_ of wavelengths of the spectra + to be stacked. Note that wavelength arrays can in general be different, + irregularly spaced, and have different sizes. + fluxes : list + List of length nexp float `numpy.ndarray`_ of fluxes for each exposure + aligned with the wavelength grids in waves. + ivars : list + List of length nexp float `numpy.ndarray`_ of inverse variances for each + exposure aligned with the wavelength grids in waves. + gpms : list + List of length nexp bool `numpy.ndarray`_ of good pixel masks for each + exposure aligned with the wavelength grids in waves. True=Good. + sns : `numpy.ndarray`_ + sn of each spectrum in the list of exposures used to determine which + scaling method should be used. This can be computed using sn_weights. + shape=(nexp,) + sigrej_scale : float, optional, default=3.0 + Rejection threshold used for rejecting pixels when rescaling spectra + with scale_spec. + ref_percentile : float, optional, default=70.0 + percentile fraction cut used for selecting minimum SNR cut for + robust_median_ratio + maxiter_scale : int, optional, default=5 Maximum number of iterations performed for rescaling spectra. - scale_method: str, optional, default='auto' + scale_method : str, optional, default='auto' Options are auto, poly, median, none, or hand. Hand is not well tested. User can optionally specify the rescaling method. Default is to let the code determine this automitically which works well. - hand_scale: `numpy.ndarray`_, optional + hand_scale : `numpy.ndarray`_, optional array of hand scale factors, not well tested - sn_min_polyscale: float, optional, default=2.0 + sn_min_polyscale : float, optional, default=2.0 maximum SNR for perforing median scaling - sn_min_medscale: float, optional default=0.5 + sn_min_medscale : float, optional, default=0.5 minimum SNR for perforing median scaling - debug: bool, optional, default=False + debug : bool, optional, default=False show interactive QA plot Returns ------- - fluxes_scales: `numpy.ndarray`_ - Scale factors applied to the fluxes - shape=(nspec, nexp) - ivars_scales: `numpy.ndarray`_ - Scale factors applied to the ivars; - shape=(nspec, nexp) - scales: `numpy.ndarray`_ - shape=(nspec, nexp); Scale factors applied to - each individual spectrum before the combine computed by - scale_spec - scale_method_used: list + fluxes_scales : list + List of length nexp `numpy.ndarray`_ rescaled fluxes + ivars_scales : list + List of length nexp `numpy.ndarray`_ rescaled ivars. shape=(nspec, + nexp) + scales : list + List of length nexp `numpy.ndarray`_ scale factors applied to each + individual spectra and their inverse variances. + scale_method_used : list List of methods used for rescaling spectra. """ # Compute an initial stack as the reference, this has its own wave grid based on the weighted averages - wave_stack, flux_stack, ivar_stack, mask_stack, nused = compute_stack(wave_grid, waves, fluxes, ivars, masks, weights) + wave_stack, flux_stack, ivar_stack, gpm_stack, nused = compute_stack(wave_grid, waves, fluxes, ivars, gpms, weights) # Rescale spectra to line up with our preliminary stack so that we can sensibly reject outliers - nexp = np.shape(fluxes)[1] - fluxes_scale = np.zeros_like(fluxes) - ivars_scale = np.zeros_like(ivars) - scales = np.zeros_like(fluxes) - scale_method_used = [] - for iexp in range(nexp): + fluxes_scale, ivars_scale, scales, scale_method_used = [], [], [], [] + for iexp, (wave, flux, ivar, gpm, sn) in enumerate(zip(waves, fluxes, ivars, gpms, sns)): hand_scale_iexp = None if hand_scale is None else hand_scale[iexp] - fluxes_scale[:, iexp], ivars_scale[:, iexp], scales[:, iexp], scale_method_iexp = scale_spec( - waves[:, iexp], fluxes[:, iexp], ivars[:, iexp], sn[iexp], wave_stack, flux_stack, ivar_stack, - mask=masks[:, iexp], mask_ref=mask_stack, ref_percentile=ref_percentile, maxiters=maxiter_scale, + # JFH Changed to used wave_grid_mid in interpolation to avoid interpolating onto zeros + fluxes_scale_iexp, ivars_scale_iexp, scales_iexp, scale_method_iexp = scale_spec( + wave, flux, ivar, sn, wave_grid_mid, flux_stack, ivar_stack, + mask=gpm, mask_ref=gpm_stack, ref_percentile=ref_percentile, maxiters=maxiter_scale, sigrej=sigrej_scale, scale_method=scale_method, hand_scale=hand_scale_iexp, sn_min_polyscale=sn_min_polyscale, sn_min_medscale=sn_min_medscale, debug=debug, show=show) + fluxes_scale.append(fluxes_scale_iexp) + ivars_scale.append(ivars_scale_iexp) + scales.append(scales_iexp) scale_method_used.append(scale_method_iexp) return fluxes_scale, ivars_scale, scales, scale_method_used -def combspec(waves, fluxes, ivars, masks, sn_smooth_npix, +def combspec(waves, fluxes, ivars, gpms, sn_smooth_npix, wave_method='linear', dwave=None, dv=None, dloglam=None, spec_samp_fact=1.0, wave_grid_min=None, wave_grid_max=None, ref_percentile=70.0, maxiter_scale=5, wave_grid_input=None, @@ -2046,8 +2024,8 @@ def combspec(waves, fluxes, ivars, masks, sn_smooth_npix, sn_min_polyscale=2.0, sn_min_medscale=0.5, const_weights=False, maxiter_reject=5, sn_clip=30.0, lower=3.0, upper=3.0, maxrej=None, qafile=None, title='', debug=False, - debug_scale=False, show_scale=False, show=False, - verbose=True): + debug_scale=False, debug_order_stack=False, + debug_global_stack=False, show_scale=False, show=False, verbose=True): ''' Main driver routine for coadding longslit/multi-slit spectra. @@ -2055,87 +2033,107 @@ def combspec(waves, fluxes, ivars, masks, sn_smooth_npix, Parameters ---------- - waves: `numpy.ndarray`_ - Wavelength arrays for spectra to be stacked. - shape=(nspec, nexp) - fluxes: `numpy.ndarray`_ - Flux arrays for spectra to be stacked. - shape=(nspec, nexp) - ivars: `numpy.ndarray`_ - ivar arrays for spectra to be stacked. - shape=(nspec, nexp) - sn_smooth_npix: int - Number of pixels to median filter by when computing S/N used to decide how to scale and weight spectra - wave_method: str, optional - method for generating new wavelength grid with get_wave_grid. Deafult is 'linear' which creates a uniformly - space grid in lambda. See docuementation on get_wave_grid for description of the options. - dwave: float, optional - dispersion in units of A in case you want to specify it for get_wave_grid, otherwise the code computes the - median spacing from the data. - dv: float, optional - Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), - otherwise a median value is computed from the data. - spec_samp_fact: float, optional - Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or coarser (spec_samp_fact > 1.0) by this - sampling factor. This basically multiples the 'native' spectral pixels by spec_samp_fact, i.e. units + waves : list + List of length nexp containing `numpy.ndarray`_ float wavelength arrays + for spectra to be stacked. These wavelength arrays can in general be + different, irregularly spaced, and have different sizes. + fluxes : list + List of length nexp containing `numpy.ndarray`_ float flux arrays for + spectra to be stacked. These should be aligned with the wavelength + arrays in waves. + ivars : list + List of length nexp containing `numpy.ndarray`_ float ivar arrays for + spectra to be stacked. These should be aligned with the wavelength + arrays in waves. + gpms : list + List of length nexp containing `numpy.ndarray`_ boolean good pixel mask + arrays for spectra to be stacked. These should be aligned with the + wavelength arrays in waves. + sn_smooth_npix : int + Number of pixels to median filter by when computing S/N used to decide + how to scale and weight spectra + wave_method : str, optional + method for generating new wavelength grid with get_wave_grid. Deafult is + 'linear' which creates a uniformly space grid in lambda. See + docuementation on get_wave_grid for description of the options. + dwave : float, optional + dispersion in units of A in case you want to specify it for + get_wave_grid, otherwise the code computes the median spacing from the + data. + dv : float, optional + Dispersion in units of km/s in case you want to specify it in the + get_wave_grid (for the 'velocity' option), otherwise a median value is + computed from the data. + spec_samp_fact : float, optional + Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or + coarser (spec_samp_fact > 1.0) by this sampling factor. This basically + multiples the 'native' spectral pixels by spec_samp_fact, i.e. units spec_samp_fact are pixels. - wave_grid_min: float, optional - In case you want to specify the minimum wavelength in your wavelength grid, default=None computes from data. - wave_grid_max: float, optional - In case you want to specify the maximum wavelength in your wavelength grid, default=None computes from data. - ref_percentile: - percentile fraction cut used for selecting minimum SNR cut for robust_median_ratio - maxiter_scale: int, optional, default=5 + wave_grid_min : float, optional + In case you want to specify the minimum wavelength in your wavelength + grid, default=None computes from data. + wave_grid_max : float, optional + In case you want to specify the maximum wavelength in your wavelength + grid, default=None computes from data. + ref_percentile : float, optional + percentile fraction cut used for selecting minimum SNR cut for + robust_median_ratio + maxiter_scale : int, optional, default=5 Maximum number of iterations performed for rescaling spectra. - wave_grid_input : `numpy.ndarray`_ - User input wavelength grid to be used with the 'user_input' wave_method. Shape is (nspec_input,) - maxiter_reject: int, optional, default=5 - maximum number of iterations for stacking and rejection. The code stops iterating either when - the output mask does not change betweeen successive iterations or when maxiter_reject is reached. - sigrej_scale: float, optional, default=3.0 - Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. - scale_method: str, optional + wave_grid_input : `numpy.ndarray`_, optional + User input wavelength grid to be used with the 'user_input' wave_method. + Shape is (nspec_input,) + maxiter_reject : int, optional, default=5 + maximum number of iterations for stacking and rejection. The code stops + iterating either when the output mask does not change betweeen + successive iterations or when maxiter_reject is reached. + sigrej_scale : float, optional, default=3.0 + Rejection threshold used for rejecting pixels when rescaling spectra + with scale_spec. + scale_method : str, optional Options are auto, poly, median, none, or hand. Hand is not well tested. User can optionally specify the rescaling method. Default is 'auto' will let the code determine this automitically which works well. - hand_scale: `numpy.ndarray`_, optional - Array of hand scale factors, not well tested - sn_min_polyscale: float, optional, default = 2.0 - maximum SNR for perforing median scaling - sn_min_medscale: float, optional, default = 0.5 - minimum SNR for perforing median scaling - const_weights: bool, optional - If True, apply constant weight + hand_scale : `numpy.ndarray`_, optional + Array of hand scale factors, not well tested + sn_min_polyscale : float, optional, default = 2.0 + maximum SNR for perforing median scaling + sn_min_medscale : float, optional, default = 0.5 + minimum SNR for perforing median scaling + const_weights : bool, optional + If True, apply constant weight sn_clip: float, optional, default=30.0 - Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection - in high S/N ratio spectrum which neverthless differ at a level greater than the implied S/N due to - systematics. - lower: float, optional, default=3.0 - lower rejection threshold for djs_reject - upper: float: optional, default=3.0 - upper rejection threshold for djs_reject - maxrej: int, optional - maximum number of pixels to reject in each iteration for djs_reject. - qafile: str, default=None - Root name for QA, if None, it will be determined from the outfile - title: str, optional + Errors are capped during rejection so that the S/N is never greater than + sn_clip. This prevents overly aggressive rejection in high S/N ratio + spectrum which neverthless differ at a level greater than the implied + S/N due to systematics. + lower : float, optional, default=3.0 + lower rejection threshold for djs_reject + upper : float, optional, default=3.0 + upper rejection threshold for djs_reject + maxrej : int, optional + maximum number of pixels to reject in each iteration for djs_reject. + qafile : str, default=None + Root name for QA, if None, it will be determined from the outfile + title : str, optional Title for QA plots debug: bool, default=False - Show all QA plots useful for debugging. Note there are lots of QA plots, so only set this to True if you want to inspect them all. - debug_scale: bool, default=False - show interactive QA plots for the rescaling of the spectra - show: bool, default=False + Show all QA plots useful for debugging. Note there are lots of QA plots, + so only set this to True if you want to inspect them all. + debug_scale : bool, default=False + show interactive QA plots for the rescaling of the spectra + show : bool, default=False If True, show key QA plots or not show_scale: bool, default=False If True, show interactive QA plots for the rescaling of the spectra Returns ------- - wave_grid_mid: `numpy.ndarray`_ + wave_grid_mid : `numpy.ndarray`_ Wavelength grid (in Angstrom) evaluated at the bin centers, - uniformly-spaced either in lambda or log10-lambda/velocity. See core.wavecal.wvutils.py for more. - shape=(ngrid,) - wave_stack: `numpy.ndarray`_ + uniformly-spaced either in lambda or log10-lambda/velocity. See + core.wavecal.wvutils.py for more. shape=(ngrid,) + wave_stack : `numpy.ndarray`_ Wavelength grid for stacked spectrum. As discussed above, this is the weighted average of the wavelengths of each spectrum that contriuted to a @@ -2145,50 +2143,50 @@ def combspec(waves, fluxes, ivars, masks, sn_smooth_npix, wave_stack is NOT simply the wave_grid bin centers, since it computes the weighted average. shape=(ngrid,) - flux_stack: `numpy.ndarray`_ + flux_stack : `numpy.ndarray`_ Final stacked spectrum on wave_stack wavelength grid shape=(ngrid,) - ivar_stack: `numpy.ndarray`_ + ivar_stack : `numpy.ndarray`_ Inverse variance spectrum on wave_stack wavelength grid. Erors are propagated according to weighting and masking. shape=(ngrid,) - mask_stack: `numpy.ndarray`_ + mask_stack : `numpy.ndarray`_ Boolean mask for stacked spectrum on wave_stack wavelength grid. True=Good. shape=(ngrid,) ''' - + #from IPython import embed + #embed() # We cast to float64 because of a bug in np.histogram - waves = np.float64(waves) - fluxes = np.float64(fluxes) - ivars = np.float64(ivars) + _waves = [np.float64(wave) for wave in waves] + _fluxes = [np.float64(flux) for flux in fluxes] + _ivars = [np.float64(ivar) for ivar in ivars] # Generate a giant wave_grid wave_grid, wave_grid_mid, _ = wvutils.get_wave_grid( - waves=waves, masks = masks, wave_method=wave_method, - wave_grid_min=wave_grid_min, wave_grid_max=wave_grid_max, - wave_grid_input=wave_grid_input, + waves=_waves, gpms=gpms, wave_method=wave_method, + wave_grid_min=wave_grid_min, wave_grid_max=wave_grid_max, + wave_grid_input=wave_grid_input, dwave=dwave, dv=dv, dloglam=dloglam, spec_samp_fact=spec_samp_fact) # Evaluate the sn_weights. This is done once at the beginning - rms_sn, weights = sn_weights(waves, fluxes, ivars, masks, sn_smooth_npix, const_weights=const_weights, verbose=verbose) - + rms_sn, weights = sn_weights(_fluxes, _ivars, gpms, sn_smooth_npix=sn_smooth_npix, const_weights=const_weights, verbose=verbose) fluxes_scale, ivars_scale, scales, scale_method_used = scale_spec_stack( - wave_grid, waves, fluxes, ivars, masks, rms_sn, weights, ref_percentile=ref_percentile, maxiter_scale=maxiter_scale, + wave_grid, wave_grid_mid, _waves, _fluxes, _ivars, gpms, rms_sn, weights, ref_percentile=ref_percentile, maxiter_scale=maxiter_scale, sigrej_scale=sigrej_scale, scale_method=scale_method, hand_scale=hand_scale, sn_min_polyscale=sn_min_polyscale, sn_min_medscale=sn_min_medscale, debug=debug_scale, show=show_scale) - # Rejecting and coadding - wave_stack, flux_stack, ivar_stack, mask_stack, outmask, nused = spec_reject_comb( - wave_grid, waves, fluxes_scale, ivars_scale, masks, weights, sn_clip=sn_clip, lower=lower, upper=upper, + wave_stack, flux_stack, ivar_stack, gpm_stack, nused, outmask = spec_reject_comb( + wave_grid, wave_grid_mid, _waves, fluxes_scale, ivars_scale, gpms, weights, sn_clip=sn_clip, lower=lower, upper=upper, maxrej=maxrej, maxiter_reject=maxiter_reject, debug=debug, title=title) if show: - coadd_qa(wave_stack, flux_stack, ivar_stack, nused, mask=mask_stack, title='Stacked spectrum', qafile=qafile) + # JFH Use wave_grid_mid for QA plots + coadd_qa(wave_grid_mid, flux_stack, ivar_stack, nused, gpm=gpm_stack, title='Stacked spectrum', qafile=qafile) - return wave_grid_mid, wave_stack, flux_stack, ivar_stack, mask_stack + return wave_grid_mid, wave_stack, flux_stack, ivar_stack, gpm_stack def multi_combspec(waves, fluxes, ivars, masks, sn_smooth_npix=None, wave_method='linear', dwave=None, dv=None, dloglam=None, spec_samp_fact=1.0, wave_grid_min=None, @@ -2201,113 +2199,132 @@ def multi_combspec(waves, fluxes, ivars, masks, sn_smooth_npix=None, Routine for coadding longslit/multi-slit spectra. Calls combspec which is the main stacking algorithm. - Args: - waves (`numpy.ndarray`_): - Wavelength array with shape (nspec, nexp) containing the spectra to be coadded. - fluxes (`numpy.ndarray`_): - Flux array with shape (nspec, nexp) containing the spectra to be coadded. - ivars (`numpy.ndarray`_): - Ivar array with shape (nspec, nexp) containing the spectra to be coadded. - masks (`numpy.ndarray`_): - Maks array with shape (nspec, nexp) containing the spectra to be coadded. - sn_smooth_npix (int): optional - Number of pixels to median filter by when computing S/N used to decide how to scale and weight spectra. If - set to None, the code will determine the effective number of good pixels per spectrum - in the stack that is being co-added and use 10% of this neff. - wave_method (str, optional): - method for generating new wavelength grid with get_wave_grid. Deafult is 'linear' which creates a uniformly - space grid in lambda. See docuementation on get_wave_grid for description of the options. - dwave (float, optional): - dispersion in units of A in case you want to specify it for get_wave_grid, otherwise the code computes the - median spacing from the data. - dv (float, optional): - Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), - otherwise a median value is computed from the data. - spec_samp_fact (float, optional): - Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or coarser (spec_samp_fact > 1.0) by this - sampling factor. This basically multiples the 'native' spectral pixels by spec_samp_fact, i.e. units - spec_samp_fact are pixels. - wave_grid_min (float, optional): - In case you want to specify the minimum wavelength in your wavelength grid, default=None computes from data. - wave_grid_max (float, optional): - In case you want to specify the maximum wavelength in your wavelength grid, default=None computes from data. - wave_grid_input (`numpy.ndarray`_): - User input wavelength grid to be used with the 'user_input' wave_method. Shape is (nspec_input,) - maxiter_reject (int): optional - maximum number of iterations for stacking and rejection. The code stops iterating either when - the output mask does not change betweeen successive iterations or when maxiter_reject is reached. Default=5. - ref_percentile (float, optional): - percentile fraction cut used for selecting minimum SNR cut for robust_median_ratio. Should be a number between - 0 and 100, default = 70.0 - maxiter_scale (int, optional): - Maximum number of iterations performed for rescaling spectra. Default=5. - sigrej_scale (float, optional): - Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. Default=3.0 - scale_method (str, optional): - Options are auto, poly, median, none, or hand. Hand is not well tested. - User can optionally specify the rescaling method. Default='auto' will let the - code determine this automitically which works well. - hand_scale (`numpy.ndarray`_, optional): - Array of hand scale factors, not well tested - sn_min_polyscale (float, optional): - maximum SNR for perforing median scaling - sn_min_medscale (float, optional): - minimum SNR for perforing median scaling - const_weights (`numpy.ndarray`_, optional): - Constant weight factors specified - maxiter_reject (int, optional): - maximum number of iterations for stacking and rejection. The code stops iterating either when - the output mask does not change betweeen successive iterations or when maxiter_reject is reached. - sn_clip (float, optional): - Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection - in high S/N ratio spectrum which neverthless differ at a level greater than the implied S/N due to - systematics. - lower (float, optional): - lower rejection threshold for djs_reject - upper (float, optional): - upper rejection threshold for djs_reject - maxrej (int, optional): - maximum number of pixels to reject in each iteration for djs_reject. - nmaskedge (int, optional): - Number of edge pixels to mask. This should be removed/fixed. - qafile (str, optional): optional, default=None - Root name for QA, if None, it will be determined from the outfile - outfile (str, optional): optional, default=None, - Root name for QA, if None, it will come from the target name from the fits header. - debug (bool, optional): optinoal, default=False, - Show all QA plots useful for debugging. Note there are lots of QA plots, so only set this to True if you want to inspect them all. - debug_scale (bool, optional): optional, default=False - show interactive QA plots for the rescaling of the spectra - show (bool, optional): optional, default=False, - Show key QA plots or not - - Returns: - :obj:`tuple`: - - - wave_grid_mid: `numpy.ndarray`_, (ngrid,): Wavelength grid (in Angstrom) - evaluated at the bin centers, uniformly-spaced either in lambda or - log10-lambda/velocity. See core.wavecal.wvutils.py for more. - - wave_stack: `numpy.ndarray`_, (ngrid,): Wavelength grid for stacked - spectrum. As discussed above, this is the weighted average of the - wavelengths of each spectrum that contriuted to a bin in the input - wave_grid wavelength grid. It thus has ngrid elements, whereas - wave_grid has ngrid+1 elements to specify the ngrid total number - of bins. Note that wave_stack is NOT simply the wave_grid bin - centers, since it computes the weighted average. - - flux_stack: `numpy.ndarray`_, (ngrid,): Final stacked spectrum on - wave_stack wavelength grid - - ivar_stack: `numpy.ndarray`_, (ngrid,): Inverse variance spectrum on - wave_stack wavelength grid. Erors are propagated according to - weighting and masking. - - mask_stack: `numpy.ndarray`_, bool, (ngrid,): Mask for stacked spectrum on - wave_stack wavelength grid. True=Good. + Parameters + ---------- + waves : list + List of `numpy.ndarray`_ wavelength arrays with shape (nspec_i,) with + the wavelength arrays of the spectra to be coadded. + fluxes : list + List of `numpy.ndarray`_ wavelength arrays with shape (nspec_i,) with + the flux arrays of the spectra to be coadded. + ivars : list + List of `numpy.ndarray`_ wavelength arrays with shape (nspec_i,) with + the ivar arrays of the spectra to be coadded. + masks : list + List of `numpy.ndarray`_ wavelength arrays with shape (nspec_i,) with + the mask arrays of the spectra to be coadded. + sn_smooth_npix : int, optional + Number of pixels to median filter by when computing S/N used to decide + how to scale and weight spectra. If set to None, the code will determine + the effective number of good pixels per spectrum in the stack that is + being co-added and use 10% of this neff. + wave_method : str, optional + method for generating new wavelength grid with get_wave_grid. Deafult is + 'linear' which creates a uniformly space grid in lambda. See + docuementation on get_wave_grid for description of the options. + dwave : float, optional + dispersion in units of A in case you want to specify it for + get_wave_grid, otherwise the code computes the median spacing from the + data. + dv : float, optional + Dispersion in units of km/s in case you want to specify it in the + get_wave_grid (for the 'velocity' option), otherwise a median value is + computed from the data. + spec_samp_fact : float, optional + Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or + coarser (spec_samp_fact > 1.0) by this sampling factor. This basically + multiples the 'native' spectral pixels by spec_samp_fact, i.e. units + spec_samp_fact are pixels. + wave_grid_min : float, optional + In case you want to specify the minimum wavelength in your wavelength + grid, default=None computes from data. + wave_grid_max : float, optional + In case you want to specify the maximum wavelength in your wavelength + grid, default=None computes from data. + wave_grid_input : `numpy.ndarray`_ + User input wavelength grid to be used with the 'user_input' wave_method. + Shape is (nspec_input,) + maxiter_reject : int, optional + maximum number of iterations for stacking and rejection. The code stops + iterating either when the output mask does not change betweeen + successive iterations or when maxiter_reject is reached. Default=5. + ref_percentile : float, optional + percentile fraction cut used for selecting minimum SNR cut for + robust_median_ratio. Should be a number between 0 and 100, default = + 70.0 + maxiter_scale : int, optional + Maximum number of iterations performed for rescaling spectra. Default=5. + sigrej_scale : float, optional + Rejection threshold used for rejecting pixels when rescaling spectra + with scale_spec. Default=3.0 + scale_method : str, optional + Options are auto, poly, median, none, or hand. Hand is not well tested. + User can optionally specify the rescaling method. Default='auto' will + let the code determine this automitically which works well. + hand_scale : `numpy.ndarray`_, optional + Array of hand scale factors, not well tested + sn_min_polyscale : float, optional + maximum SNR for perforing median scaling + sn_min_medscale : float, optional + minimum SNR for perforing median scaling + const_weights : `numpy.ndarray`_, optional + Constant weight factors specified + maxiter_reject : int, optional + maximum number of iterations for stacking and rejection. The code stops + iterating either when the output mask does not change betweeen + successive iterations or when maxiter_reject is reached. + sn_clip : float, optional + Errors are capped during rejection so that the S/N is never greater than + sn_clip. This prevents overly aggressive rejection in high S/N ratio + spectrum which neverthless differ at a level greater than the implied + S/N due to systematics. + lower : float, optional + lower rejection threshold for djs_reject + upper : float, optional + upper rejection threshold for djs_reject + maxrej : int, optional + maximum number of pixels to reject in each iteration for djs_reject. + nmaskedge : int, optional + Number of edge pixels to mask. This should be removed/fixed. + qafile : str, optional, default=None + Root name for QA, if None, it will be determined from the outfile + outfile : str, optional, default=None, + Root name for QA, if None, it will come from the target name from the + fits header. + debug : bool, optional, default=False, + Show all QA plots useful for debugging. Note there are lots of QA plots, + so only set this to True if you want to inspect them all. + debug_scale : bool, optional, default=False + show interactive QA plots for the rescaling of the spectra + show : bool, optional, default=False + Show key QA plots or not + Returns + ------- + wave_grid_mid : `numpy.ndarray`_, (ngrid,) + Wavelength grid (in Angstrom) evaluated at the bin centers, + uniformly-spaced either in lambda or log10-lambda/velocity. See + core.wavecal.wvutils.py for more. + wave_stack : `numpy.ndarray`_, (ngrid,) + Wavelength grid for stacked spectrum. As discussed above, this is the + weighted average of the wavelengths of each spectrum that contriuted to + a bin in the input wave_grid wavelength grid. It thus has ngrid + elements, whereas wave_grid has ngrid+1 elements to specify the ngrid + total number of bins. Note that wave_stack is NOT simply the wave_grid + bin centers, since it computes the weighted average. + flux_stack : `numpy.ndarray`_, (ngrid,) + Final stacked spectrum on wave_stack wavelength grid + ivar_stack : `numpy.ndarray`_, (ngrid,) + Inverse variance spectrum on wave_stack wavelength grid. Erors are + propagated according to weighting and masking. + mask_stack : `numpy.ndarray`_, bool, (ngrid,) + Mask for stacked spectrum on wave_stack wavelength grid. True=Good. """ # Decide how much to smooth the spectra by if this number was not passed in if sn_smooth_npix is None: - nspec, nexp = waves.shape + nexp = len(waves) # This is the effective good number of spectral pixels in the stack - nspec_eff = np.sum(waves > 1.0)/nexp + nspec_eff = np.sum([np.sum(wave > 1.0) for wave in waves]) / nexp sn_smooth_npix = int(np.round(0.1*nspec_eff)) msgs.info('Using a sn_smooth_npix={:d} to decide how to scale and weight your spectra'.format(sn_smooth_npix)) @@ -2320,15 +2337,12 @@ def multi_combspec(waves, fluxes, ivars, masks, sn_smooth_npix=None, maxrej=maxrej, qafile=qafile, title='multi_combspec', debug=debug, debug_scale=debug_scale, show_scale=show_scale, show=show) - # Write to disk? - #if outfile is not None: - # save.save_coadd1d_to_fits(outfile, wave_stack, flux_stack, ivar_stack, mask_stack, header=header, - # ex_value=ex_value, overwrite=True) - return wave_grid_mid, wave_stack, flux_stack, ivar_stack, mask_stack -def ech_combspec(waves, fluxes, ivars, masks, weights_sens, nbest=None, +# TODO: Describe the "b_tuple" +def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_setup, weights_sens_arr_setup, + setup_ids = None, nbests=None, wave_method='log10', dwave=None, dv=None, dloglam=None, spec_samp_fact=1.0, wave_grid_min=None, wave_grid_max=None, ref_percentile=70.0, maxiter_scale=5, niter_order_scale=3, @@ -2336,123 +2350,198 @@ def ech_combspec(waves, fluxes, ivars, masks, weights_sens, nbest=None, hand_scale=None, sn_min_polyscale=2.0, sn_min_medscale=0.5, sn_smooth_npix=None, const_weights=False, maxiter_reject=5, sn_clip=30.0, lower=3.0, upper=3.0, - maxrej=None, qafile=None, debug_scale=False, debug=False, + maxrej=None, qafile=None, debug_scale=False, debug_order_stack=False, + debug_global_stack=False, debug=False, show_order_stacks=False, show_order_scale=False, show_exp=False, show=False, verbose=False): """ - Driver routine for coadding Echelle spectra. Calls combspec which is the main stacking algorithm. It will deliver - three fits files: spec1d_order_XX.fits (stacked individual orders, one order per extension), spec1d_merge_XX.fits - (straight combine of stacked individual orders), spec1d_stack_XX.fits (a giant stack of all exposures and all orders). - In most cases, you should use spec1d_stack_XX.fits for your scientific analyses since it reject most outliers. - - ..todo.. -- Clean up the doc formatting + Driver routine for coadding Echelle spectra. Calls combspec which is the + main stacking algorithm. It will deliver three fits files: + spec1d_order_XX.fits (stacked individual orders, one order per extension), + spec1d_merge_XX.fits (straight combine of stacked individual orders), + spec1d_stack_XX.fits (a giant stack of all exposures and all orders). In + most cases, you should use spec1d_stack_XX.fits for your scientific analyses + since it reject most outliers. Parameters ---------- - waves: `numpy.ndarray`_ - Wavelength arrays for spectra to be stacked. - shape=(nspec, norder, nexp) - fluxes: `numpy.ndarray`_ - Flux arrays for spectra to be stacked. - shape=(nspec, norder, nexp) - ivars: `numpy.ndarray`_ - ivar arrays for spectra to be stacked. + waves_arr_setup : list of `numpy.ndarray`_ + List of wavelength arrays for spectra to be stacked. The length of the + list is nsetups. Each element of the list corresponds to a distinct + setup and each numpy array has shape=(nspec, norder, nexp) + fluxes_arr_setup : list of `numpy.ndarray`_ + List of flux arrays for spectra to be stacked. The length of the list is + nsetups. Each element of the list corresponds to a dinstinct setup and + each numpy array has shape=(nspec, norder, nexp) + ivars_arr_setup : list of `numpy.ndarray`_ + List of ivar arrays for spectra to be stacked. The length of the list is + nsetups. Each element of the list corresponds to a dinstinct setup and + each numpy array has shape=(nspec, norder, nexp) + gpms_arr_setup : list of `numpy.ndarray`_ + List of good pixel mask arrays for spectra to be stacked. The length of + the list is nsetups. Each element of the list corresponds to a + dinstinct setup and each numpy array has shape=(nspec, norder, nexp) + weights_sens_arr_setup : list of `numpy.ndarray`_ + List of sensitivity function weights required for relative weighting of + the orders. The length of the list is nsetups. Each element of the + list corresponds to a dinstinct setup and each numpy array has shape=(nspec, norder, nexp) - masks: `numpy.ndarray`_ - Mask array with shape (nspec, norders, nexp) containing the spectra to be coadded. - weights_sens: `numpy.ndarray`_ - Sensitivity function weights required for relatively weighting of the - orders. Must have the same shape as waves, etc. - nbest: int, optional - Number of orders to use for estimating the per exposure weights. - Default is nbest=None, which will just use one fourth of the orders. - wave_method: str, optional - method for generating new wavelength grid with get_wave_grid. Deafult is 'log10' which creates a uniformly - space grid in log10(lambda), which is typically the best for echelle spectrographs - dwave: float, optional - dispersion in units of A in case you want to specify it for get_wave_grid, otherwise the code computes the - median spacing from the data. - dv: float, optional - Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), - otherwise a median value is computed from the data. - dloglam: float, optional + setup_ids : list, optional, default=None + List of strings indicating the name of each setup. If None uppercase + letters A, B, C, etc. will be used + nbests : int or list or `numpy.ndarray`_, optional + Integer or list of integers indicating the number of orders to use for + estimating the per exposure weights per echelle setup. Default is + nbests=None, which will just use one fourth of the orders for a given + setup. + wave_method : str, optional + method for generating new wavelength grid with get_wave_grid. Deafult is + 'log10' which creates a uniformly space grid in log10(lambda), which is + typically the best for echelle spectrographs + dwave : float, optional + dispersion in units of A in case you want to specify it for + get_wave_grid, otherwise the code computes the median spacing from the + data. + dv : float, optional + Dispersion in units of km/s in case you want to specify it in the + get_wave_grid (for the 'velocity' option), otherwise a median value is + computed from the data. + dloglam : float, optional Dispersion in dimensionless units - spec_samp_fact: float, optional - Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or coarser (spec_samp_fact > 1.0) by this - sampling factor. This basically multiples the 'native' spectral pixels by spec_samp_fact, i.e. units - spec_samp_fact are pixels. - wave_grid_min: float, optional - In case you want to specify the minimum wavelength in your wavelength grid, default=None computes from data. - wave_grid_max: float, optional - In case you want to specify the maximum wavelength in your wavelength grid, default=None computes from data. - wave_grid_input : `numpy.ndarray`_ - User input wavelength grid to be used with the 'user_input' wave_method. Shape is (nspec_input,) - ref_percentile: - percentile fraction cut used for selecting minimum SNR cut for robust_median_ratio - maxiter_scale: int, optional, default=5 + spec_samp_fact : float, optional + Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or + coarser (spec_samp_fact > 1.0) by this sampling factor. This basically + multiples the 'native' spectral pixels by spec_samp_fact, i.e. units + spec_samp_fact are pixels. + wave_grid_min : float, optional + In case you want to specify the minimum wavelength in your wavelength + grid, default=None computes from data. + wave_grid_max : float, optional + In case you want to specify the maximum wavelength in your wavelength + grid, default=None computes from data. + wave_grid_input : `numpy.ndarray`_, optional + User input wavelength grid to be used with the 'user_input' wave_method. + Shape is (nspec_input,) + ref_percentile : float, optional + percentile fraction cut used for selecting minimum SNR cut for + robust_median_ratio + maxiter_scale : int, optional, default=5 Maximum number of iterations performed for rescaling spectra. - sigrej_scale: float, optional, default=3.0 - Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. - hand_scale: `numpy.ndarray`_, optional - Array of hand scale factors, not well tested - sn_min_polyscale: float, optional, default = 2.0 - maximum SNR for perforing median scaling - sn_min_medscale: float, optional, default = 0.5 - minimum SNR for perforing median scaling - const_weights: bool, optional - If True, apply constant weight - maxiter_reject: int, optional, default=5 - maximum number of iterations for stacking and rejection. The code stops iterating either when - the output mask does not change betweeen successive iterations or when maxiter_reject is reached. - sn_clip: float, optional, default=30.0 - Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection - in high S/N ratio spectrum which neverthless differ at a level greater than the implied S/N due to - lower: float, optional, default=3.0 - lower rejection threshold for djs_reject - upper: float: optional, default=3.0 - upper rejection threshold for djs_reject - maxrej: int, optional - maximum number of pixels to reject in each iteration for djs_reject. + sigrej_scale : float, optional, default=3.0 + Rejection threshold used for rejecting pixels when rescaling spectra + with scale_spec. + hand_scale : `numpy.ndarray`_, optional + Array of hand scale factors, not well tested + sn_min_polyscale : float, optional, default = 2.0 + maximum SNR for perforing median scaling + sn_min_medscale : float, optional, default = 0.5 + minimum SNR for perforing median scaling + const_weights : bool, optional + If True, apply constant weight + maxiter_reject : int, optional, default=5 + maximum number of iterations for stacking and rejection. The code stops + iterating either when the output mask does not change betweeen + successive iterations or when maxiter_reject is reached. + sn_clip : float, optional, default=30.0 + Errors are capped during rejection so that the S/N is never greater than + sn_clip. This prevents overly aggressive rejection in high S/N ratio + spectrum which neverthless differ at a level greater than the implied + S/N due to + lower : float, optional, default=3.0 + lower rejection threshold for djs_reject + upper : float, optional, default=3.0 + upper rejection threshold for djs_reject + maxrej : int, optional + maximum number of pixels to reject in each iteration for djs_reject. + debug_order_stack : bool, default=False + show interactive QA plots for the order stacking + debug_global_stack : bool, default=False + show interactive QA plots for the global stacking of all the + setups/orders/exposures + debug : bool, optional, default=False + show all QA plots useful for debugging. Note there are lots of QA plots, + so only set this to True if you want to inspect them all. + debug_scale : bool, optional, default=False + show interactive QA plots for the rescaling of the spectra + show_order_scale : bool, optional, default=False + show interactive QA plots for the order scaling + show : bool, optional, default=False, + show coadded spectra QA plot or not + show_exp : bool, optional, default=False + show individual exposure spectra QA plot or not Returns ------- - wave_grid_mid: `numpy.ndarray`_ + wave_grid_mid : `numpy.ndarray`_ Wavelength grid (in Angstrom) evaluated at the bin centers, - uniformly-spaced either in lambda or log10-lambda/velocity. See core.wavecal.wvutils.py for more. - shape=(ngrid,) - a_tuple: tuple - (wave_giant_stack: ndarray, (ngrid,): Wavelength grid for - stacked spectrum. As discussed above, this is the weighted - average of the wavelengths of each spectrum that - contriuted to a bin in the input wave_grid wavelength - grid. It thus has ngrid elements, whereas wave_grid has - ngrid+1 elements to specify the ngrid total number of - bins. Note that wave_giant_stack is NOT simply the - wave_grid bin centers, since it computes the weighted - average; - flux_giant_stack: ndarray, (ngrid,): Final stacked - spectrum on wave_stack wavelength grid; - ivar_giant_stack: ndarray, (ngrid,): Inverse variance - spectrum on wave_stack wavelength grid. Erors are - propagated according to weighting and masking.; - mask_giant_stack: ndarray, bool, (ngrid,): Mask for - stacked spectrum on wave_stack wavelength grid. True=Good. - ) - another_tuple: tuple - (waves_stack_orders, - fluxes_stack_orders, - ivars_stack_orders, - masks_stack_orders, - None) + uniformly-spaced either in lambda or log10-lambda/velocity. See + core.wavecal.wvutils.py for more. shape=(ngrid,) + a_tuple : tuple + Contains: + + - ``wave_giant_stack``: ndarray, (ngrid,): Wavelength grid for stacked + spectrum. As discussed above, this is the weighted average of the + wavelengths of each spectrum that contriuted to a bin in the input + wave_grid wavelength grid. It thus has ngrid elements, whereas + wave_grid has ngrid+1 elements to specify the ngrid total number + of bins. Note that wave_giant_stack is NOT simply the wave_grid + bin centers, since it computes the weighted average; + + - ``flux_giant_stack``: ndarray, (ngrid,): Final stacked spectrum on + wave_stack wavelength grid; + + - ``ivar_giant_stack``: ndarray, (ngrid,): Inverse variance spectrum on + wave_stack wavelength grid. Erors are propagated according to + weighting and masking.; + + - ``mask_giant_stack``: ndarray, bool, (ngrid,): Mask for stacked + spectrum on wave_stack wavelength grid. True=Good. + + b_tuple : tuple + Contains: + + - ``waves_stack_orders`` + + - ``fluxes_stack_orders`` + + - ``ivars_stack_orders`` + + - ``masks_stack_orders`` + """ -# TODO: Please leave this commented docstring entry here for now. -# merge_stack: bool, default=False, -# Compute an experimental combine of the high S/N combined orders in addition to the default algorithm, -# which is to compute one giant stack using all order overlaps - # output filenams for fits and QA plots - #outfile_order = outfile.replace('.fits', '_order.fits') if outfile is not None else None + # Notes on object/list formats + + # waves_arr_setup -- is a list of length nsetups, one for each setup. Each element is a numpy + # array with shape = (nspec, norder, nexp) which is the data model for echelle spectra + # for an individual setup. The utiltities utils.arr_setup_to_setup_list and + # utils.setup_list_to_arr convert between arr_setup and setup_list + + # + # waves_setup_list -- is a list of length nsetups, one for each setup. Each element of it is a list of length + # norder*nexp elements, each of which contains the shape = (nspec1,) wavelength arrays + # for the order/exposure in setup1. The list is arranged such that the nexp1 spectra + # for iorder=0 appear first, then com nexp1 spectra for iorder=1, i.e. the outer or + # fastest varying dimension in python array ordering is the exposure number. The utility + # functions utils.echarr_to_echlist and utils.echlist_to_echarr convert between + # the multi-d numpy arrays in the waves_arr_setup and the lists of numpy arrays in + # waves_setup_list + # + # waves_concat -- is a list of length = \Sum_i norder_i*nexp_i where the index i runs over the setups. The + # elements of the list contains a numpy array of wavelengths for the + # setup, order, exposure in question. The utility routines utils.setup_list_to_concat and + # utils.concat_to_setup_list convert between waves_setup_lists and waves_concat + + + if debug: + show=True + show_exp=True + show_order_scale=True + debug_order_stack=True + debug_global_stack=True + debug_scale=True + if qafile is not None: qafile_stack = qafile.replace('.pdf', '_stack.pdf') @@ -2462,194 +2551,273 @@ def ech_combspec(waves, fluxes, ivars, masks, weights_sens, nbest=None, qafile_chi = None # data shape - nspec, norder, nexp = waves.shape - if nbest is None: - # Decide how many orders to use for estimating the per exposure weights. If nbest was not passed in - # default to using one fourth of the orders - nbest = int(np.ceil(norder/4)) + nsetups=len(waves_arr_setup) + if setup_ids is None: + setup_ids = list(string.ascii_uppercase[:nsetups]) + setup_colors = utils.distinct_colors(nsetups) + + norders = [] + nexps = [] + nspecs = [] + for wave in waves_arr_setup: + nspecs.append(wave.shape[0]) + norders.append(wave.shape[1]) + nexps.append(wave.shape[2]) + + if nbests is None: + _nbests = [int(np.ceil(norder/4)) for norder in norders] + else: + _nbests = nbests if isinstance(nbests, (list, np.ndarray)) else [nbests]*nsetups + + # nspec, norder, nexp = shape # Decide how much to smooth the spectra by if this number was not passed in + nspec_good = [] + ngood = [] if sn_smooth_npix is None: - # This is the effective good number of spectral pixels in the stack - nspec_eff = np.sum(waves > 1.0)/(norder*nexp) + # Loop over setups + for wave, norder, nexp in zip(waves_arr_setup, norders, nexps): + # This is the effective good number of spectral pixels in the stack + nspec_good.append(np.sum(wave > 1.0)) + ngood.append(norder*nexp) + nspec_eff = np.sum(nspec_good)/np.sum(ngood) sn_smooth_npix = int(np.round(0.1 * nspec_eff)) msgs.info('Using a sn_smooth_pix={:d} to decide how to scale and weight your spectra'.format(sn_smooth_npix)) - # create some arrays - scales = np.zeros_like(waves) + # Create the setup lists + waves_setup_list = [utils.echarr_to_echlist(wave)[0] for wave in waves_arr_setup] + fluxes_setup_list = [utils.echarr_to_echlist(flux)[0] for flux in fluxes_arr_setup] + ivars_setup_list = [utils.echarr_to_echlist(ivar)[0] for ivar in ivars_arr_setup] + gpms_setup_list= [utils.echarr_to_echlist(gpm)[0] for gpm in gpms_arr_setup] + #shapes_setup_list = [wave.shape for wave in waves_arr_setup] + + # Generate some concatenated lists for wavelength grid determination + waves_concat = utils.setup_list_to_concat(waves_setup_list) + gpms_concat = utils.setup_list_to_concat(gpms_setup_list) # Generate a giant wave_grid - wave_grid, wave_grid_mid, _ = wvutils.get_wave_grid(waves, masks=masks, wave_method=wave_method, + wave_grid, wave_grid_mid, _ = wvutils.get_wave_grid(waves_concat, gpms=gpms_concat, wave_method=wave_method, wave_grid_min=wave_grid_min, wave_grid_max=wave_grid_max, dwave=dwave, dv=dv, dloglam=dloglam, spec_samp_fact=spec_samp_fact) # Evaluate the sn_weights. This is done once at the beginning - rms_sn, weights_sn = sn_weights(waves, fluxes, ivars, masks, sn_smooth_npix, const_weights=const_weights, verbose=verbose) - # Isolate the nbest best orders, and then use the average S/N of these to determine the per exposure relative weights. - mean_sn_ord = np.mean(rms_sn, axis=1) - best_orders = np.argsort(mean_sn_ord)[::-1][0:nbest] - rms_sn_per_exp = np.mean(rms_sn[best_orders, :], axis=0) - weights_exp = np.tile(rms_sn_per_exp**2, (nspec, norder, 1)) - weights = weights_exp*weights_sens - # - # Old code below for ivar weights if the sensfile was not passed in - #msgs.error('Using ivar weights is deprecated.') - #msgs.warn('No sensfunc is available for weighting, using smoothed ivar weights which is not optimal!') - #_, weights_ivar = sn_weights(waves, fluxes, ivars, masks, sn_smooth_npix, const_weights=const_weights, - # ivar_weights=True, verbose=True) - #weights = weights_exp*weights_ivar + weights = [] + rms_sn_setup_list = [] + colors = [] + for isetup in range(nsetups): + rms_sn_vec, _ = sn_weights(fluxes_setup_list[isetup], ivars_setup_list[isetup], gpms_setup_list[isetup], + sn_smooth_npix=sn_smooth_npix, const_weights=const_weights, verbose=verbose) + rms_sn = rms_sn_vec.reshape(norders[isetup], nexps[isetup]) + mean_sn_ord = np.mean(rms_sn, axis=1) + best_orders = np.argsort(mean_sn_ord)[::-1][0:_nbests[isetup]] + rms_sn_per_exp = np.mean(rms_sn[best_orders, :], axis=0) + weights_exp = np.tile(np.square(rms_sn_per_exp), (nspecs[isetup], norders[isetup], 1)) + weights_isetup = weights_exp * weights_sens_arr_setup[isetup] + weights.append(weights_isetup) + rms_sn_setup_list.append(rms_sn) + colors.append([setup_colors[isetup]]*norders[isetup]*nexps[isetup]) + + + # Create the waves_setup_list + weights_setup_list = [utils.echarr_to_echlist(weight)[0] for weight in weights] if debug: - weights_qa(waves, weights, masks, title='ech_combspec') - - fluxes_scl_interord = np.zeros_like(fluxes) - ivars_scl_interord = np.zeros_like(ivars) - scales_interord = np.zeros_like(fluxes) - # First perform inter-order scaling once - for iord in range(norder): - # TODO Add checking here such that orders with low S/N ratio are instead scaled using scale factors from - # higher S/N ratio. The point is it makes no sense to take 0.0/0.0. In the low S/N regime, i.e. DLAs, - # GP troughs, we should be rescaling using scale factors from orders with signal. This also applies - # to the echelle combine below. - fluxes_scl_interord[:, iord], ivars_scl_interord[:,iord], scales_interord[:,iord], scale_method_used = \ - scale_spec_stack(wave_grid, waves[:, iord, :], fluxes[:, iord, :], ivars[:, iord, :], masks[:, iord, :], - rms_sn[iord, :], weights[:, iord, :], ref_percentile=ref_percentile, - maxiter_scale=maxiter_scale, sigrej_scale=sigrej_scale, scale_method=scale_method, - hand_scale=hand_scale, - sn_min_polyscale=sn_min_polyscale, sn_min_medscale=sn_min_medscale, debug=debug_scale) - - # Arrays to store rescaled spectra. Need Fortran like order reshaping to create a (nspec, norder*nexp) stack of spectra. - # The order of the reshaping in the second dimension is such that blocks norder long for each exposure are stacked - # sequentially, i.e. for order number [:, 0:norder] would be the 1st exposure, [:,norder:2*norder] would be the - # 2nd exposure, etc. - shape_2d = (nspec, norder * nexp) - waves_2d = np.reshape(waves, shape_2d, order='F') - fluxes_2d = np.reshape(fluxes_scl_interord, shape_2d, order='F') - ivars_2d = np.reshape(ivars_scl_interord, shape_2d, order='F') - masks_2d = np.reshape(masks, shape_2d, order='F') - scales_2d = np.reshape(scales_interord, shape_2d, order='F') - weights_2d = np.reshape(weights, shape_2d, order='F') - rms_sn_2d = np.reshape(rms_sn, (norder*nexp), order='F') - # Iteratively scale and stack the spectra, this takes or the order re-scaling we were doing previously - fluxes_pre_scale = fluxes_2d.copy() - ivars_pre_scale = ivars_2d.copy() + weights_qa(utils.setup_list_to_concat(waves_setup_list),utils.setup_list_to_concat(weights_setup_list), + utils.setup_list_to_concat(gpms_setup_list), colors=utils.setup_list_to_concat(colors), + title='ech_combspec') + + ####################### + # Inter-order rescaling + ####################### + #debug_scale=True + fluxes_scl_interord_setup_list, ivars_scl_interord_setup_list, scales_interord_setup_list = [], [], [] + for isetup in range(nsetups): + fluxes_scl_interord_isetup, ivars_scl_interord_isetup, scales_interord_isetup = [], [], [] + for iord in range(norders[isetup]): + ind_start = iord*nexps[isetup] + ind_end = (iord+1)*nexps[isetup] + fluxes_scl_interord_iord, ivars_scl_interord_iord, scales_interord_iord, scale_method_used = \ + scale_spec_stack(wave_grid, wave_grid_mid, waves_setup_list[isetup][ind_start:ind_end], + fluxes_setup_list[isetup][ind_start:ind_end],ivars_setup_list[isetup][ind_start:ind_end], + gpms_setup_list[isetup][ind_start:ind_end], rms_sn_setup_list[isetup][iord, :], + weights_setup_list[isetup][ind_start:ind_end], ref_percentile=ref_percentile, + maxiter_scale=maxiter_scale, sigrej_scale=sigrej_scale, scale_method=scale_method, + hand_scale=hand_scale, + sn_min_polyscale=sn_min_polyscale, sn_min_medscale=sn_min_medscale, debug=debug_scale) + fluxes_scl_interord_isetup += fluxes_scl_interord_iord + ivars_scl_interord_isetup += ivars_scl_interord_iord + scales_interord_isetup += scales_interord_iord + + fluxes_scl_interord_setup_list.append(fluxes_scl_interord_isetup) + ivars_scl_interord_setup_list.append(ivars_scl_interord_isetup) + scales_interord_setup_list.append(scales_interord_isetup) + + # TODO Add checking above in inter-order scaling such that orders with low S/N ratio are instead scaled using + # scale factors from higher S/N ratio. The point is it makes no sense to take 0.0/0.0. In the low S/N regime, + # i.e. DLAs, GP troughs, we should be rescaling using scale factors from orders with signal. This also applies + # to the echelle combine below. + + + ####################### + # Global Rescaling Computation -- Scale each setup/order/exp to match a preliminary global stack + ####################### + #show_order_scale=True + fluxes_concat = utils.setup_list_to_concat(fluxes_scl_interord_setup_list) + ivars_concat = utils.setup_list_to_concat(ivars_scl_interord_setup_list) + scales_concat = utils.setup_list_to_concat(scales_interord_setup_list) + weights_concat = utils.setup_list_to_concat(weights_setup_list) + rms_sn_concat = [] + for rms_sn in rms_sn_setup_list: + rms_sn_concat += rms_sn.flatten().tolist() + rms_sn_concat = np.array(rms_sn_concat) + fluxes_pre_scale_concat = copy.deepcopy(fluxes_concat) + ivars_pre_scale_concat = copy.deepcopy(ivars_concat) # For the first iteration use the scale_method input as an argument (default=None, which will allow # soly_poly_ratio scaling which is very slow). For all the other iterations simply use median rescaling since # we are then applying tiny corrections and median scaling is much faster scale_method_iter = [scale_method] + ['median']*(niter_order_scale - 1) - for iter in range(niter_order_scale): - fluxes_scale_2d, ivars_scale_2d, scales_iter, scale_method_used = scale_spec_stack( - wave_grid, waves_2d, fluxes_pre_scale, ivars_pre_scale, masks_2d, rms_sn_2d, weights_2d, ref_percentile=ref_percentile, - maxiter_scale=maxiter_scale, sigrej_scale=sigrej_scale, scale_method=scale_method_iter[iter], hand_scale=hand_scale, + # Iteratively scale and stack the entire set of spectra arcoss all setups, orders, and exposures + for iteration in range(niter_order_scale): + # JFH This scale_spec_stack routine takes a list of [(nspec1,), (nspec2,), ...] arrays, so a loop needs to be + # added here over the outer setup dimension of the lists + fluxes_scale_concat, ivars_scale_concat, scales_iter_concat, scale_method_used = scale_spec_stack( + wave_grid, wave_grid_mid, waves_concat, fluxes_pre_scale_concat, ivars_pre_scale_concat, gpms_concat, rms_sn_concat, + weights_concat, ref_percentile=ref_percentile, + maxiter_scale=maxiter_scale, sigrej_scale=sigrej_scale, scale_method=scale_method_iter[iteration], hand_scale=hand_scale, sn_min_polyscale=sn_min_polyscale, sn_min_medscale=sn_min_medscale, - show=(show_order_scale & (iter == (niter_order_scale-1)))) - scales_2d *= scales_iter - fluxes_pre_scale = fluxes_scale_2d.copy() - ivars_pre_scale = ivars_scale_2d.copy() - - # Reshape the outputs to be (nspec, norder, nexp) - fluxes_scale = np.reshape(fluxes_scale_2d, (nspec, norder, nexp), order='F') - ivars_scale = np.reshape(ivars_scale_2d, (nspec, norder, nexp), order='F') - scales = np.reshape(scales_2d, (nspec, norder, nexp), order='F') - - # Arrays to store stacked individual order spectra. - waves_stack_orders = np.zeros((np.size(wave_grid) - 1, norder)) - fluxes_stack_orders = np.zeros_like(waves_stack_orders) - ivars_stack_orders = np.zeros_like(waves_stack_orders) - masks_stack_orders = np.zeros_like(waves_stack_orders, dtype=bool) - outmasks_orders = np.zeros_like(masks) - # Now perform stacks order by order - for iord in range(norder): - # Rejecting and coadding - waves_stack_orders[:, iord], fluxes_stack_orders[:, iord], ivars_stack_orders[:, iord], \ - masks_stack_orders[:, iord], outmasks_orders[:,iord,:], nused_iord = spec_reject_comb( - wave_grid, waves[:, iord, :], fluxes_scale[:, iord, :], ivars_scale[:, iord, :], masks[:, iord, :], weights[:, iord, :], - sn_clip=sn_clip, lower=lower, upper=upper, maxrej=maxrej, maxiter_reject=maxiter_reject, debug=debug, - title='order_stacks') - if show_order_stacks: - # TODO This will probably crash since sensfile is not guarnetted to have telluric. - #if sensfile is not None: - # tell_iord = get_tell_from_file(sensfile, waves_stack_orders[:, iord], masks_stack_orders[:, iord], iord=iord) - #else: - # tell_iord = None - tell_iord=None - coadd_qa(waves_stack_orders[:, iord], fluxes_stack_orders[:, iord], ivars_stack_orders[:, iord], nused_iord, - mask=masks_stack_orders[:, iord], tell=tell_iord, - title='Coadded spectrum of order {:d}/{:d}'.format(iord, norder)) - - # Now compute the giant stack - wave_giant_stack, flux_giant_stack, ivar_giant_stack, mask_giant_stack, outmask_giant_stack, nused_giant_stack = \ - spec_reject_comb(wave_grid, waves_2d, fluxes_2d, ivars_2d, masks_2d, weights_2d, sn_clip=sn_clip, - lower=lower, upper=upper, maxrej=maxrej, maxiter_reject=maxiter_reject, debug=debug) - - # Reshape everything now exposure-wise - waves_2d_exps = waves_2d.reshape((nspec * norder, nexp), order='F') - fluxes_2d_exps = fluxes_2d.reshape(np.shape(waves_2d_exps), order='F') - ivars_2d_exps = ivars_2d.reshape(np.shape(waves_2d_exps), order='F') - masks_2d_exps = masks_2d.reshape(np.shape(waves_2d_exps), order='F') - outmasks_2d_exps = outmask_giant_stack.reshape(np.shape(waves_2d_exps), order='F') - # rejection statistics, exposure by exposure - nrej = np.sum(np.invert(outmasks_2d_exps) & masks_2d_exps, axis=0) # rejected pixels - norig = np.sum((waves_2d_exps > 1.0) & np.invert(masks_2d_exps), axis=0) # originally masked pixels + show=(show_order_scale & (iteration == (niter_order_scale-1)))) + scales_concat = [scales_orig*scales_new for scales_orig, scales_new in zip(scales_concat, scales_iter_concat)] + fluxes_pre_scale_concat = copy.deepcopy(fluxes_scale_concat) + ivars_pre_scale_concat = copy.deepcopy(ivars_scale_concat) + + + ########################### + # Giant stack computation -- Perform the final stack with rejection for the globally rescaled spectra + ########################### + #debug=True + wave_final_stack, flux_final_stack, ivar_final_stack, gpm_final_stack, nused_final_stack, out_gpms_concat = \ + spec_reject_comb(wave_grid, wave_grid_mid, waves_concat, fluxes_scale_concat, ivars_scale_concat, gpms_concat, + weights_concat, sn_clip=sn_clip, lower=lower, upper=upper, maxrej=maxrej, + maxiter_reject=maxiter_reject, debug=debug_global_stack) + + # Generate setup_lists and arr_setup for some downstream computations + fluxes_scale_setup_list = utils.concat_to_setup_list(fluxes_scale_concat, norders, nexps) + ivars_scale_setup_list = utils.concat_to_setup_list(ivars_scale_concat, norders, nexps) + out_gpms_setup_list = utils.concat_to_setup_list(out_gpms_concat, norders, nexps) + fluxes_scale_arr_setup = utils.setup_list_to_arr_setup(fluxes_scale_setup_list, norders, nexps) + ivars_scale_arr_setup = utils.setup_list_to_arr_setup(ivars_scale_setup_list, norders, nexps) + out_gpms_arr_setup = utils.setup_list_to_arr_setup(out_gpms_setup_list, norders, nexps) + + + ############################ + # Order Stack computation -- These are returned. CURRENTLY NOT USED FOR ANYTHING + ############################ + #show_order_stacks=True + waves_order_stack_setup, fluxes_order_stack_setup, ivars_order_stack_setup, gpms_order_stack_setup = [], [], [], [] + out_gpms_order_stack_setup_list = [] + for isetup in range(nsetups): + waves_order_stack, fluxes_order_stack, ivars_order_stack, gpms_order_stack, out_gpms_order_stack = [], [], [], [], [] + for iord in range(norders[isetup]): + ind_start = iord*nexps[isetup] + ind_end = (iord+1)*nexps[isetup] + wave_order_stack_iord, flux_order_stack_iord, ivar_order_stack_iord, gpm_order_stack_iord, \ + nused_order_stack_iord, outgpms_order_stack_iord = spec_reject_comb( + wave_grid, wave_grid_mid, waves_setup_list[isetup][ind_start:ind_end], + fluxes_scale_setup_list[isetup][ind_start:ind_end], ivars_scale_setup_list[isetup][ind_start:ind_end], + gpms_setup_list[isetup][ind_start:ind_end], weights_setup_list[isetup][ind_start:ind_end], + sn_clip=sn_clip, lower=lower, upper=upper, maxrej=maxrej, maxiter_reject=maxiter_reject, debug=debug_order_stack, + title='order_stacks') + waves_order_stack.append(wave_order_stack_iord) + fluxes_order_stack.append(flux_order_stack_iord) + ivars_order_stack.append(ivar_order_stack_iord) + gpms_order_stack.append(gpm_order_stack_iord) + out_gpms_order_stack.append(outgpms_order_stack_iord) + if show_order_stacks: + coadd_qa(wave_order_stack_iord, flux_order_stack_iord, ivar_order_stack_iord, nused_order_stack_iord, + gpm=gpm_order_stack_iord, + title='Coadded spectrum of order {:d}/{:d} for setup={:s}'.format( + iord, norders[isetup], setup_ids[isetup])) + waves_order_stack_setup.append(waves_order_stack) + fluxes_order_stack_setup.append(fluxes_order_stack) + ivars_order_stack_setup.append(ivars_order_stack) + gpms_order_stack_setup.append(gpms_order_stack) + out_gpms_order_stack_setup_list.append(out_gpms_order_stack) + + ############################ + # QA Generation + ############################ if debug or show: - # Interpolate stack onto native 2d wavelength grids reshaped exposure-wise - flux_stack_2d_exps, ivar_stack_2d_exps, mask_stack_2d_exps = interp_spec( - waves_2d_exps, wave_giant_stack, flux_giant_stack, ivar_giant_stack, mask_giant_stack) - if show_exp: - # Show QA plots for each exposure - rejivars_2d_exps, sigma_corrs_2d_exps, outchi_2d_exps, maskchi_2d_exps = update_errors( - fluxes_2d_exps, ivars_2d_exps, outmasks_2d_exps, flux_stack_2d_exps, ivar_stack_2d_exps, - mask_stack_2d_exps, sn_clip=sn_clip) - # QA for individual exposures - for iexp in range(nexp): - # plot the residual distribution - msgs.info('QA plots for exposure {:} with new_sigma = {:}'.format(iexp, sigma_corrs_2d_exps[iexp])) - # plot the residual distribution for each exposure - title_renorm = 'ech_combspec: Error distribution about stack for exposure {:d}/{:d}'.format(iexp, nexp) - renormalize_errors_qa(outchi_2d_exps[:, iexp], maskchi_2d_exps[:, iexp], sigma_corrs_2d_exps[iexp], - title=title_renorm) - title_coadd_iexp = 'ech_combspec: nrej={:d} pixels rejected,'.format(nrej[iexp]) + \ - ' norig={:d} originally masked,'.format(norig[iexp]) + \ - ' for exposure {:d}/{:d}'.format(iexp, nexp) - coadd_iexp_qa(waves_2d_exps[:,iexp], fluxes_2d_exps[:,iexp], rejivars_2d_exps[:,iexp], masks_2d_exps[:,iexp], - wave_giant_stack, flux_giant_stack, ivar_giant_stack, mask_giant_stack, outmasks_2d_exps[:, iexp], - norder=norder, qafile=None, title=title_coadd_iexp) - # Global QA - rejivars_1d, sigma_corrs_1d, outchi_1d, maskchi_1d = update_errors( - fluxes_2d_exps.flatten(), ivars_2d_exps.flatten(), outmasks_2d_exps.flatten(), - flux_stack_2d_exps.flatten(), ivar_stack_2d_exps.flatten(), mask_stack_2d_exps.flatten(), sn_clip=sn_clip) - renormalize_errors_qa(outchi_1d, maskchi_1d, sigma_corrs_1d[0], qafile=qafile_chi, title='Global Chi distribution') + fluxes_exps, ivars_exps, out_gpms_exps = np.array([],dtype=float), np.array([],dtype=float), np.array([],dtype=bool) + flux_stack_exps, ivar_stack_exps, gpm_stack_exps = np.array([],dtype=float), np.array([],dtype=float), np.array([], dtype=bool) + nrej_setup, norig_setup = [], [] + for isetup in range(nsetups): + # Reshape everything now exposure-wise + new_shape = (nspecs[isetup] * norders[isetup], nexps[isetup]) + waves_2d_exps = waves_arr_setup[isetup].reshape(new_shape, order='F') + fluxes_2d_exps = fluxes_scale_arr_setup[isetup].reshape(new_shape, order='F') + ivars_2d_exps = ivars_scale_arr_setup[isetup].reshape(new_shape, order='F') + gpms_2d_exps = gpms_arr_setup[isetup].reshape(new_shape, order='F') + out_gpms_2d_exps = out_gpms_arr_setup[isetup].reshape(new_shape, order='F') + + nrej = np.sum(np.logical_not(out_gpms_2d_exps) & gpms_2d_exps, axis=0) # rejected pixels per exposure + norig = np.sum((waves_2d_exps > 1.0) & np.logical_not(gpms_2d_exps), axis=0) # originally masked pixels per exposure + nrej_setup.append(np.sum(nrej)) + norig_setup.append(np.sum(norig)) + # Interpolate stack onto native 2d wavelength grids reshaped exposure-wise + # JFH changed to wave_grid_mid + flux_stack_2d_exps, ivar_stack_2d_exps, gpm_stack_2d_exps, _ = interp_spec( + waves_2d_exps, wave_grid_mid, flux_final_stack, ivar_final_stack, gpm_final_stack) + if show_exp: + # Show QA plots for each exposure + rejivars_2d_exps, sigma_corrs_2d_exps, outchi_2d_exps, gpm_chi_2d_exps = update_errors( + fluxes_2d_exps, ivars_2d_exps, out_gpms_2d_exps, flux_stack_2d_exps, ivar_stack_2d_exps, + gpm_stack_2d_exps, sn_clip=sn_clip) + # QA for individual exposures + for iexp in range(nexps[isetup]): + # plot the residual distribution + msgs.info('QA plots for exposure {:} with new_sigma = {:}'.format(iexp, sigma_corrs_2d_exps[iexp])) + # plot the residual distribution for each exposure + title_renorm = 'ech_combspec: Error distribution about stack for exposure {:d}/{:d} for setup={:s}'.format(iexp, nexps[isetup], setup_ids[isetup]) + renormalize_errors_qa(outchi_2d_exps[:, iexp], gpm_chi_2d_exps[:, iexp], sigma_corrs_2d_exps[iexp], + title=title_renorm) + title_coadd_iexp = 'ech_combspec: nrej={:d} pixels rejected,'.format(nrej[iexp]) + \ + ' norig={:d} originally masked,'.format(norig[iexp]) + \ + ' for exposure {:d}/{:d}'.format(iexp, nexps[isetup]) + # JFH changed QA to use wave_grid_mid + coadd_iexp_qa(waves_2d_exps[:,iexp], fluxes_2d_exps[:,iexp], rejivars_2d_exps[:,iexp], gpms_2d_exps[:,iexp], + wave_grid_mid, flux_final_stack, ivar_final_stack, gpm_final_stack, out_gpms_2d_exps[:, iexp], + norder=norders[isetup], qafile=None, title=title_coadd_iexp) + # Global QA for this setup + rejivars_1d_isetup, sigma_corrs_1d_isetup, outchi_1d_isetup, gpm_chi_1d_isetup = update_errors( + fluxes_2d_exps.flatten(), ivars_2d_exps.flatten(), out_gpms_2d_exps.flatten(), + flux_stack_2d_exps.flatten(), ivar_stack_2d_exps.flatten(), gpm_stack_2d_exps.flatten(), sn_clip=sn_clip) + renormalize_errors_qa(outchi_1d_isetup, gpm_chi_1d_isetup, sigma_corrs_1d_isetup[0], + qafile=qafile_chi, title='Global Chi distribution for setup={:s}'.format(setup_ids[isetup])) + fluxes_exps = np.append(fluxes_exps, fluxes_2d_exps.flatten()) + ivars_exps = np.append(ivars_exps, ivars_2d_exps.flatten()) + out_gpms_exps = np.append(out_gpms_exps, out_gpms_2d_exps.flatten()) + flux_stack_exps = np.append(flux_stack_exps, flux_stack_2d_exps.flatten()) + ivar_stack_exps = np.append(ivar_stack_exps, ivar_stack_2d_exps.flatten()) + gpm_stack_exps = np.append(gpm_stack_exps, gpm_stack_2d_exps.flatten()) + + # TODO Print out rejection statistics per setup? + + # Show a global chi distribution now for all setups, but only if there are multiple setups otherwise it is + # identical to the isetup=0 plot. + if nsetups > 1: + rejivars_1d, sigma_corrs_1d, outchi_1d, gpm_chi_1d = update_errors( + fluxes_exps, ivars_exps, out_gpms_exps, flux_stack_exps, ivar_stack_exps, gpm_stack_exps, sn_clip=sn_clip) + renormalize_errors_qa(outchi_1d, gpm_chi_1d, sigma_corrs_1d[0], qafile=qafile_chi, + title='Global Chi distribution for all nsetups={:d} setups'.format(nsetups)) # show the final coadded spectrum - coadd_qa(wave_giant_stack, flux_giant_stack, ivar_giant_stack, nused_giant_stack, mask=mask_giant_stack, + coadd_qa(wave_grid_mid, flux_final_stack, ivar_final_stack, nused_final_stack, gpm=gpm_final_stack, title='Final stacked spectrum', qafile=qafile_stack) -# TODO: Please leave this commented code in for now. -# ## Stack with an altnernative method: combine the stacked individual order spectra directly. This is deprecated -# merge_stack = False -# if merge_stack: -# order_weights = sensfunc_weights(sensfile, waves_stack_orders, debug=debug) -# wave_merge, flux_merge, ivar_merge, mask_merge, nused_merge = compute_stack( -# wave_grid, waves_stack_orders, fluxes_stack_orders, ivars_stack_orders, masks_stack_orders, order_weights) -# if show_order_stacks: -# qafile_merge = 'spec1d_merge_{:}'.format(qafile) -# coadd_qa(wave_merge, flux_merge, ivar_merge, nused_merge, mask=mask_merge, tell = None, -# title='Straight combined spectrum of the stacked individual orders', qafile=qafile_merge) -# #if outfile is not None: -# # outfile_merge = outfile.replace('.fits', '_merge.fits') -# # save.save_coadd1d_to_fits(outfile_merge, wave_merge, flux_merge, ivar_merge, mask_merge, header=header, -# # ex_value=ex_value, overwrite=True) - - # Save stacked individual order spectra - #save.save_coadd1d_to_fits(outfile_order, waves_stack_orders, fluxes_stack_orders, ivars_stack_orders, masks_stack_orders, - # header=header, ex_value = ex_value, overwrite=True) - #save.save_coadd1d_to_fits(outfile, wave_giant_stack, flux_giant_stack, ivar_giant_stack, mask_giant_stack, - # header=header, ex_value=ex_value, overwrite=True) - - - return wave_grid_mid, (wave_giant_stack, flux_giant_stack, ivar_giant_stack, mask_giant_stack), \ - (waves_stack_orders, fluxes_stack_orders, ivars_stack_orders, masks_stack_orders,) + + return wave_grid_mid, (wave_final_stack, flux_final_stack, ivar_final_stack, gpm_final_stack), \ + (waves_order_stack_setup, fluxes_order_stack_setup, ivars_order_stack_setup, gpms_order_stack_setup,) # #################################################################### @@ -2703,24 +2871,27 @@ def get_wave_bins(thismask_stack, waveimg_stack, wave_grid): Parameters ---------- - thismask_stack: list - List of boolean arrays containing the masks indicating which pixels are on - the slit in question. `True` values are on the slit; - `False` values are off the slit. Length of the list is nimgs. Shapes of the individual elements in the list - are (nspec, nspat), but each image can have a different shape. - waveimg_stack: list - List of the wavelength images, each of which is a float `numpy.ndarray`_. Length of the list is nimgs. - Shapes of the individual elements in the list are (nspec, nspat), but each image can have a different shape. - wave_grid: array shape (ngrid) + thismask_stack : list + List of boolean arrays containing the masks indicating which pixels are + on the slit in question. `True` values are on the slit; `False` values + are off the slit. Length of the list is nimgs. Shapes of the + individual elements in the list are (nspec, nspat), but each image can + have a different shape. + + waveimg_stack : list + List of the wavelength images, each of which is a float + `numpy.ndarray`_. Length of the list is nimgs. Shapes of the individual + elements in the list are (nspec, nspat), but each image can have a + different shape. + wave_grid : `numpy.ndarray`_, shape=(ngrid,) The wavelength grid created for the 2d coadd Returns ------- - wave_bins : `numpy.ndarray`_ - shape (ind_upper-ind_lower + 1, ) - Wavelength bins that are relevant given the illuminated pixels (thismask_stack) and - wavelength coverage (waveimg_stack) of the image stack - + wave_bins : `numpy.ndarray`_, shape = (ind_upper-ind_lower + 1, ) + Wavelength bins that are relevant given the illuminated pixels + (thismask_stack) and wavelength coverage (waveimg_stack) of the image + stack """ # Determine the wavelength grid that we will use for the current slit/order @@ -2812,7 +2983,7 @@ def get_spat_bins(thismask_stack, trace_stack, spat_samp_fact=1.0): def compute_coadd2d(ref_trace_stack, sciimg_stack, sciivar_stack, skymodel_stack, inmask_stack, thismask_stack, waveimg_stack, wave_grid, spat_samp_fact=1.0, maskdef_dict=None, - weights='uniform', interp_dspat=True): + weights=None, interp_dspat=True): """ Construct a 2d co-add of a stack of PypeIt spec2d reduction outputs. @@ -2859,17 +3030,14 @@ def compute_coadd2d(ref_trace_stack, sciimg_stack, sciivar_stack, skymodel_stack the slit in question. `True` values are on the slit; `False` values are off the slit. Length of the list is nimgs. Shapes of the individual elements in the list are (nspec, nspat), but each image can have a different shape. - weights (list or str, optional): + weights (list, optional): The weights used when combining the rectified images (see - :func:`weighted_combine`). If weights is set to 'uniform' then a - uniform weighting is used. Weights are broadast to the correct size - of the image stacks (see :func:`broadcast_weights`), as necessary. - If a list is passed it must have a length of nimgs. The individual elements - of the list can either be floats, indiciating the weight to be used for each image, or + :func:`~pypeit.core.combine.weighted_combine`). If weights is None a + uniform weighting is used. If weights is not None then it must be a list of length nimgs. + The individual elements of the list can either be floats, indiciating the weight to be used for each image, or arrays with shape = (nspec,) or shape = (nspec, nspat), indicating pixel weights for the individual images. Weights are broadast to the correct size - of the image stacks (see :func:`broadcast_weights`), as necessary. - (TODO: JFH I think the str option should be changed here, but am leaving it for now.) + of the image stacks (see :func:`~pypeit.core.combine.broadcast_weights`), as necessary. spat_samp_fact (float, optional): Spatial sampling for 2d coadd spatial bins in pixels. A value > 1.0 (i.e. bigger pixels) will downsample the images spatially, whereas < @@ -2878,7 +3046,7 @@ def compute_coadd2d(ref_trace_stack, sciimg_stack, sciivar_stack, skymodel_stack Wavelength grid in log10(wave) onto which the image stacks will be rectified. The code will automatically choose the subset of this grid encompassing the wavelength coverage of - the image stacks provided (see :func:`waveimg_stack`). + the image stacks provided (see ``waveimg_stack``). Either `loglam_grid` or `wave_grid` must be provided. wave_grid (`numpy.ndarray`_, optional): Same as `loglam_grid` but in angstroms instead of @@ -2927,6 +3095,7 @@ def compute_coadd2d(ref_trace_stack, sciimg_stack, sciivar_stack, skymodel_stack - maskdef_slitcen: int - maskdef_objpos: int - maskdef_designtab: int + """ #nimgs, nspec, nspat = sciimg_stack.shape @@ -2934,9 +3103,9 @@ def compute_coadd2d(ref_trace_stack, sciimg_stack, sciivar_stack, skymodel_stack # Maybe the doc string above is inaccurate? nimgs =len(sciimg_stack) - if isinstance(weights,str) and weights == 'uniform': + if weights is None: msgs.info('No weights were provided. Using uniform weights.') - weights = np.ones(nimgs)/float(nimgs) + weights = (np.ones(nimgs)/float(nimgs)).tolist() shape_list = [sciimg.shape for sciimg in sciimg_stack] weights_stack = combine.broadcast_lists_of_weights(weights, shape_list) @@ -3047,64 +3216,57 @@ def rebin2d(spec_bins, spat_bins, waveimg_stack, spatimg_stack, Parameters ---------- - spec_bins: `numpy.ndarray`_ + spec_bins : `numpy.ndarray`_, float, shape = (nspec_rebin) Spectral bins to rebin to. - float, shape = (nspec_rebin) - spat_bins: `numpy.ndarray`_ + + spat_bins : `numpy.ndarray`_, float ndarray, shape = (nspat_rebin) Spatial bins to rebin to. - float ndarray, shape = (nspat_rebin) - waveimg_stack: `numpy.ndarray`_ + waveimg_stack : `numpy.ndarray`_, float , shape = (nimgs, nspec, nspat) Stack of nimgs wavelength images with shape = (nspec, nspat) each - float , shape = (nimgs, nspec, nspat) - spatimg_stack: `numpy.ndarray`_ + spatimg_stack : `numpy.ndarray`_, float, shape = (nimgs, nspec, nspat) Stack of nimgs spatial position images with shape = (nspec, nspat) each - float, shape = (nimgs, nspec, nspat) - thismask_stack: `numpy.ndarray`_ - Stack of nimgs images with shape = (nspec, nspat) indicating the locatons on the pixels on an image that - are on the slit in question. - bool, shape = (nimgs, nspec, nspat) - inmask_stack: `numpy.ndarray`_ - Stack of nimgs images with shape = (nspec, nspat) indicating which pixels on an image are masked. - True = Good, False = Bad - bool ndarray, shape = (nimgs, nspec, nspat) - - sci_list: list - Nested list of images, i.e. list of lists of images, where sci_list[i][j] is a shape = (nspec, nspat) where - the shape can be different for each image. The ith index is the image type, i.e. sciimg, skysub, tilts, waveimg, - the jth index is the exposure or image number, i.e. nimgs. These images are to be rebinned onto - the commong grid. - var_list: list - Nested list of variance images, i.e. list of lists of images. The format is the same as for sci_list, but - note that sci_list and var_list can have different lengths. Since this routine performs a NGP rebinning, - it effectively comptues the average of a science image landing on a pixel. This means that the science - is weighted by the 1/norm_rebin_stack, and hence variances must be weighted by that factor squared, - which his why they must be input here as a separate list. - + thismask_stack : `numpy.ndarray`_, bool, shape = (nimgs, nspec, nspat) + Stack of nimgs images with shape = (nspec, nspat) indicating the + locatons on the pixels on an image that are on the slit in question. + inmask_stack : `numpy.ndarray`_, bool ndarray, shape = (nimgs, nspec, nspat) + Stack of nimgs images with shape = (nspec, nspat) indicating which + pixels on an image are masked. True = Good, False = Bad + sci_list : list + Nested list of images, i.e. list of lists of images, where + sci_list[i][j] is a shape = (nspec, nspat) where the shape can be + different for each image. The ith index is the image type, i.e. sciimg, + skysub, tilts, waveimg, the jth index is the exposure or image number, + i.e. nimgs. These images are to be rebinned onto the commong grid. + var_list : list + Nested list of variance images, i.e. list of lists of images. The format + is the same as for sci_list, but note that sci_list and var_list can + have different lengths. Since this routine performs a NGP rebinning, it + effectively comptues the average of a science image landing on a pixel. + This means that the science is weighted by the 1/norm_rebin_stack, and + hence variances must be weighted by that factor squared, which his why + they must be input here as a separate list. Returns ------- - sci_list_out: list. The list of ndarray rebinned images - with new shape (nimgs, nspec_rebin, nspat_rebin) - var_list_out: list. The list of ndarray rebinned variance - images with correct error propagation with shape (nimgs, - nspec_rebin, nspat_rebin) - norm_rebin_stack: int ndarray, shape (nimgs, nspec_rebin, - nspat_rebin). An image stack indicating the integer - occupation number of a given pixel. In other words, this - number would be zero for empty bins, one for bins that - were populated by a single pixel, etc. This image takes - the input inmask_stack into account. The output mask for - each image can be formed via outmask_rebin_satck = - (norm_rebin_stack > 0) - nsmp_rebin_stack: int ndarray, shape (nimgs, nspec_rebin, - nspat_rebin). An image stack indicating the integer - occupation number of a given pixel taking only the - thismask_stack into account, but taking the inmask_stack - into account. This image is mainly constructed for - bookeeping purposes, as it represents the number of times - each pixel in the rebin image was populated taking only - the "geometry" of the rebinning into account (i.e. the - thismask_stack), but not the masking (inmask_stack). + sci_list_out: list + The list of ndarray rebinned images with new shape (nimgs, nspec_rebin, + nspat_rebin) + var_list_out : list + The list of ndarray rebinned variance images with correct error + propagation with shape (nimgs, nspec_rebin, nspat_rebin). + norm_rebin_stack : int ndarray, shape (nimgs, nspec_rebin, nspat_rebin) + An image stack indicating the integer occupation number of a given + pixel. In other words, this number would be zero for empty bins, one for + bins that were populated by a single pixel, etc. This image takes the + input inmask_stack into account. The output mask for each image can be + formed via outmask_rebin_stack = (norm_rebin_stack > 0). + nsmp_rebin_stack : int ndarray, shape (nimgs, nspec_rebin, nspat_rebin) + An image stack indicating the integer occupation number of a given pixel + taking only the thismask_stack into account, but taking the inmask_stack + into account. This image is mainly constructed for bookeeping purposes, + as it represents the number of times each pixel in the rebin image was + populated taking only the "geometry" of the rebinning into account (i.e. + the thismask_stack), but not the masking (inmask_stack). """ # allocate the output mages @@ -3126,7 +3288,7 @@ def rebin2d(spec_bins, spat_bins, waveimg_stack, spatimg_stack, spec_rebin_this = waveimg[thismask] spat_rebin_this = spatimg[thismask] - # This fist image is purely for bookeeping purposes to determine the number of times each pixel + # This first image is purely for bookkeeping purposes to determine the number of times each pixel # could have been sampled nsmp_rebin_stack[img, :, :], spec_edges, spat_edges = np.histogram2d(spec_rebin_this, spat_rebin_this, bins=[spec_bins, spat_bins], density=False) @@ -3153,3 +3315,5 @@ def rebin2d(spec_bins, spat_bins, waveimg_stack, spatimg_stack, var_list_out[indx][img, :, :] = (norm_img > 0.0)*weigh_var/(norm_img + (norm_img == 0.0))**2 return sci_list_out, var_list_out, norm_rebin_stack.astype(int), nsmp_rebin_stack.astype(int) + + diff --git a/pypeit/core/collate.py b/pypeit/core/collate.py old mode 100755 new mode 100644 index 0c4bb2a949..caa1695634 --- a/pypeit/core/collate.py +++ b/pypeit/core/collate.py @@ -4,6 +4,8 @@ # -*- coding: utf-8 -*- """ This module contains code for collating multiple 1d spectra source object. + +.. include:: ../include/links.rst """ #TODO -- Consider moving this into the top-level, i.e. out of core @@ -32,7 +34,7 @@ class SourceObject: Args: spec1d_obj (:obj:`pypeit.specobj.SpecObj`): The initial spectra of the group as a SpecObj. - spec1d_header (:obj:`astropy.io.fits.Header`): + spec1d_header (`astropy.io.fits.Header`_): The header for the first spec1d file in the group. spec1d_file (str): Filename of the first spec1d file in the group. spectrograph (:obj:`pypeit.spectrographs.spectrograph.Spectrograph`): @@ -47,7 +49,7 @@ class SourceObject: The list of spectra in the group as SpecObj objects. spec1d_file_list (list of str): The pathnames of the spec1d files in the group. - spec1d_header_list: (list of :obj:`astropy.io.fits.Header`): + spec1d_header_list: (list of `astropy.io.fits.Header`_): The headers of the spec1d files in the group """ def __init__(self, spec1d_obj, spec1d_header, spec1d_file, spectrograph, match_type): @@ -66,23 +68,25 @@ def __init__(self, spec1d_obj, spec1d_header, spec1d_file, spectrograph, match_t self.coord = spec1d_obj['SPAT_PIXPOS'] @classmethod - def build_source_objects(cls, spec1d_files, match_type): + def build_source_objects(cls, specobjs_list, spec1d_files, match_type): """Build a list of SourceObjects from a list of spec1d files. There will be one SourceObject per SpecObj in the resulting list (i.e. no combining or collating is done by this method). Args: - spec1d_files (list of str): List of spec1d filenames + specobjs_list (list of :obj:`pypeit.specobjs.SpecObjs`): List of SpecObjs objects to build from. + + spec1d_files (list of str): List of spec1d filenames corresponding to each SpecObjs object. + match_type (str): What type of matching the SourceObjects will be configured for. Must be either 'ra/dec' or 'pixel' Returns: list of :obj:`SourceObject`: A list of uncollated SourceObjects with one SpecObj per SourceObject. """ result = [] - for spec1d_file in spec1d_files: - sobjs = specobjs.SpecObjs.from_fitsfile(spec1d_file) + for i, sobjs in enumerate(specobjs_list): spectrograph = load_spectrograph(sobjs.header['PYP_SPEC']) for sobj in sobjs: - result.append(SourceObject(sobj, sobjs.header, spec1d_file, spectrograph, match_type)) + result.append(SourceObject(sobj, sobjs.header, spec1d_files[i], spectrograph, match_type)) return result @@ -92,7 +96,7 @@ def _config_key_match(self, header): ones for this SourceObject. Args: - header (:obj:`astropy.io.fits.Header`): + header (`astropy.io.fits.Header`_): Header from a spec1d file. Returns: @@ -129,16 +133,13 @@ def match(self, spec_obj, spec1d_header, tolerance, unit = u.arcsec): Args: spec_obj (:obj:`pypeit.specobj.SpecObj`): The SpecObj to compare with this SourceObject. - - spec1d_header (:obj:`astropy.io.fits.Header`): + spec1d_header (`astropy.io.fits.Header`_): The header from the spec1d that dontains the SpecObj. - tolerance (float): Maximum distance that two spectra can be from each other to be considered to be from the same source. Measured in floating point pixels or as an angular distance (see ``unit1`` argument). - - unit (:obj:`astropy.units.Unit`): + unit (`astropy.units.Unit`_): Units of ``tolerance`` argument if match_type is 'ra/dec'. Defaults to arcseconds. Igored if match_type is 'pixel'. @@ -189,7 +190,7 @@ def collate_spectra_by_source(source_list, tolerance, unit=u.arcsec): Maximum distance that two spectra can be from each other to be considered to be from the same source. Measured in floating point pixels or as an angular distance (see ``unit`` argument). - unit (:obj:`astropy.units.Unit`): + unit (`astropy.units.Unit`_): Units of ``tolerance`` argument if match_type is 'ra/dec'. Defaults to arcseconds. Ignored if match_type is 'pixel'. diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 9c80e67fbb..2e0115c10a 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -1,7 +1,6 @@ """ Module containing routines used by 3D datacubes. -.. include common links, assuming primary doc root is up one directory .. include:: ../include/links.rst """ @@ -13,7 +12,7 @@ from astropy.coordinates import AltAz, SkyCoord from astropy.io import fits import scipy.optimize as opt -from scipy.interpolate import interp1d, RegularGridInterpolator, RBFInterpolator +from scipy.interpolate import interp1d import numpy as np from pypeit import msgs @@ -23,6 +22,12 @@ from pypeit.core.procimg import grow_mask from pypeit.spectrographs.util import load_spectrograph +# Use a fast histogram for speed! +try: + from fast_histogram import histogramdd +except ImportError: + histogramdd = None + from IPython import embed @@ -54,25 +59,31 @@ class DataCube(datamodel.DataContainer): If the cube has been flux calibrated, this will be set to "True" Attributes: - head0 (`astropy.io.fits.Header`): + head0 (`astropy.io.fits.Header`_): Primary header filename (str): Filename to use when loading from file spect_meta (:obj:`dict`): Parsed meta from the header - spectrograph (:class:`pypeit.spectrographs.spectrograph.Spectrograph`): + spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): Build from PYP_SPEC """ version = '1.1.0' - datamodel = {'flux': dict(otype=np.ndarray, atype=np.floating, descr='Flux datacube in units of counts/s/Ang/arcsec^2' - 'or 10^-17 erg/s/cm^2/Ang/arcsec^2'), - 'sig': dict(otype=np.ndarray, atype=np.floating, descr='Error datacube (matches units of flux)'), - 'bpm': dict(otype=np.ndarray, atype=np.uint8, descr='Bad pixel mask of the datacube (0=good, 1=bad)'), - 'blaze_wave': dict(otype=np.ndarray, atype=np.floating, descr='Wavelength array of the spectral blaze function'), - 'blaze_spec': dict(otype=np.ndarray, atype=np.floating, descr='The spectral blaze function'), - 'sensfunc': dict(otype=np.ndarray, atype=np.floating, descr='Sensitivity function 10^-17 erg/(counts/cm^2)'), + datamodel = {'flux': dict(otype=np.ndarray, atype=np.floating, + descr='Flux datacube in units of counts/s/Ang/arcsec^2 or ' + '10^-17 erg/s/cm^2/Ang/arcsec^2'), + 'sig': dict(otype=np.ndarray, atype=np.floating, + descr='Error datacube (matches units of flux)'), + 'bpm': dict(otype=np.ndarray, atype=np.uint8, + descr='Bad pixel mask of the datacube (0=good, 1=bad)'), + 'blaze_wave': dict(otype=np.ndarray, atype=np.floating, + descr='Wavelength array of the spectral blaze function'), + 'blaze_spec': dict(otype=np.ndarray, atype=np.floating, + descr='The spectral blaze function'), + 'sensfunc': dict(otype=np.ndarray, atype=np.floating, + descr='Sensitivity function 10^-17 erg/(counts/cm^2)'), 'PYP_SPEC': dict(otype=str, descr='PypeIt: Spectrograph name'), 'fluxed': dict(otype=bool, descr='Boolean indicating if the datacube is fluxed.')} @@ -82,7 +93,8 @@ class DataCube(datamodel.DataContainer): 'spect_meta' ] - def __init__(self, flux, sig, bpm, PYP_SPEC, blaze_wave, blaze_spec, sensfunc=None, fluxed=None): + def __init__(self, flux, sig, bpm, PYP_SPEC, blaze_wave, blaze_spec, sensfunc=None, + fluxed=None): args, _, _, values = inspect.getargvalues(inspect.currentframe()) _d = dict([(k, values[k]) for k in args[1:]]) @@ -121,15 +133,19 @@ def _bundle(self): def to_file(self, ofile, primary_hdr=None, hdr=None, **kwargs): """ - Over-load :func:`pypeit.datamodel.DataContainer.to_file` + Over-load :func:`~pypeit.datamodel.DataContainer.to_file` to deal with the header Args: - ofile (:obj:`str`): Filename + ofile (:obj:`str`): + Filename primary_hdr (`astropy.io.fits.Header`_, optional): + Base primary header. Updated with new subheader keywords. If + None, initialized using :func:`~pypeit.io.initialize_header`. wcs (`astropy.io.fits.Header`_, optional): The World Coordinate System, represented by a fits header - **kwargs: Passed to super.to_file() + kwargs (dict): + Keywords passed directly to parent ``to_file`` function. """ if primary_hdr is None: @@ -149,7 +165,7 @@ def to_file(self, ofile, primary_hdr=None, hdr=None, **kwargs): @classmethod def from_file(cls, ifile): """ - Over-load :func:`pypeit.datamodel.DataContainer.from_file` + Over-load :func:`~pypeit.datamodel.DataContainer.from_file` to deal with the header Args: @@ -175,12 +191,16 @@ def wcs(self): return wcs.WCS(self.head0) -def dar_fitfunc(radec, coord_ra, coord_dec, datfit, wave, obstime, location, pressure, temperature, rel_humidity): - """ Generates a fitting function to calculate the offset due to differential atmospheric refraction +def dar_fitfunc(radec, coord_ra, coord_dec, datfit, wave, obstime, location, pressure, + temperature, rel_humidity): + """ + Generates a fitting function to calculate the offset due to differential + atmospheric refraction Args: radec (tuple): - A tuple containing two floats representing the shift in ra and dec due to DAR. + A tuple containing two floats representing the shift in ra and dec + due to DAR. coord_ra (float): RA in degrees coord_dec (float): @@ -199,8 +219,7 @@ def dar_fitfunc(radec, coord_ra, coord_dec, datfit, wave, obstime, location, pre Outside relative humidity at `location`. This should be between 0 to 1. Returns: - chisq (float): - chi-squared difference between datfit and model + float: chi-squared difference between datfit and model """ (diff_ra, diff_dec) = radec # Generate the coordinate with atmospheric conditions @@ -212,8 +231,8 @@ def dar_fitfunc(radec, coord_ra, coord_dec, datfit, wave, obstime, location, pre return np.sum((np.array([coord_altaz.alt.value, coord_altaz.az.value])-datfit)**2) -def dar_correction(wave_arr, coord, obstime, location, pressure, temperature, rel_humidity, - wave_ref=None, numgrid=10): +def correct_dar(wave_arr, coord, obstime, location, pressure, temperature, rel_humidity, + wave_ref=None, numgrid=10): """ Apply a differental atmospheric refraction correction to the input ra/dec. @@ -249,9 +268,9 @@ def dar_correction(wave_arr, coord, obstime, location, pressure, temperature, re Returns ------- ra_diff : `numpy.ndarray`_ - Relative RA shift at each wavelength given by `wave_arr` + Relative RA shift at each wavelength given by ``wave_arr`` dec_diff : `numpy.ndarray`_ - Relative DEC shift at each wavelength given by `wave_arr` + Relative DEC shift at each wavelength given by ``wave_arr`` """ msgs.info("Performing differential atmospheric refraction correction") if wave_ref is None: @@ -286,11 +305,12 @@ def dar_correction(wave_arr, coord, obstime, location, pressure, temperature, re return ra_diff, dec_diff -def calc_grating_corr(wave_eval, wave_curr, spl_curr, wave_ref, spl_ref, order=2): - """ Using spline representations of the blaze profile, calculate the grating correction - that should be applied to the current spectrum (suffix 'curr') relative to the reference - spectrum (suffix 'ref'). The grating correction is then evaluated at the wavelength - array given by 'wave_eval'. +def correct_grating_shift(wave_eval, wave_curr, spl_curr, wave_ref, spl_ref, order=2): + """ + Using spline representations of the blaze profile, calculate the grating + correction that should be applied to the current spectrum (suffix ``curr``) + relative to the reference spectrum (suffix ``ref``). The grating correction + is then evaluated at the wavelength array given by ``wave_eval``. Args: wave_eval (`numpy.ndarray`_): @@ -307,7 +327,7 @@ def calc_grating_corr(wave_eval, wave_curr, spl_curr, wave_ref, spl_ref, order=2 Polynomial order used to fit the grating correction. Returns: - grat_corr (`numpy.ndarray`_): The grating correction to apply + `numpy.ndarray`_: The grating correction to apply """ msgs.info("Calculating the grating correction") # Calculate the grating correction @@ -324,15 +344,19 @@ def calc_grating_corr(wave_eval, wave_curr, spl_curr, wave_ref, spl_ref, order=2 def gaussian2D_cube(tup, intflux, xo, yo, dxdz, dydz, sigma_x, sigma_y, theta, offset): - """ Fit a 2D Gaussian function to a datacube. This function assumes that each wavelength - slice of the datacube is well-fit by a 2D Gaussian. The centre of the Gaussian is allowed - to vary linearly as a function of wavelength. + """ + Fit a 2D Gaussian function to a datacube. This function assumes that each + wavelength slice of the datacube is well-fit by a 2D Gaussian. The centre of + the Gaussian is allowed to vary linearly as a function of wavelength. + + .. note:: - NOTE : the integrated flux does not vary with wavelength. + The integrated flux does not vary with wavelength. Args: tup (:obj:`tuple`): - A three element tuple containing the x, y, and z locations of each pixel in the cube + A three element tuple containing the x, y, and z locations of each + pixel in the cube intflux (float): The Integrated flux of the Gaussian xo (float): @@ -353,7 +377,7 @@ def gaussian2D_cube(tup, intflux, xo, yo, dxdz, dydz, sigma_x, sigma_y, theta, o Constant offset Returns: - gtwod (`numpy.ndarray`_): The 2D Gaussian evaluated at the coordinate (x, y, z) + `numpy.ndarray`_: The 2D Gaussian evaluated at the coordinate (x, y, z) """ # Extract the (x, y, z) coordinates of each pixel from the tuple (x, y, z) = tup @@ -371,11 +395,13 @@ def gaussian2D_cube(tup, intflux, xo, yo, dxdz, dydz, sigma_x, sigma_y, theta, o def gaussian2D(tup, intflux, xo, yo, sigma_x, sigma_y, theta, offset): - """ Fit a 2D Gaussian function to an image. + """ + Fit a 2D Gaussian function to an image. Args: tup (:obj:`tuple`): - A two element tuple containing the x and y coordinates of each pixel in the image + A two element tuple containing the x and y coordinates of each pixel + in the image intflux (float): The Integrated flux of the 2D Gaussian xo (float): @@ -392,7 +418,7 @@ def gaussian2D(tup, intflux, xo, yo, sigma_x, sigma_y, theta, offset): Constant offset Returns: - gtwod (`numpy.ndarray`_): The 2D Gaussian evaluated at the coordinate (x, y) + `numpy.ndarray`_: The 2D Gaussian evaluated at the coordinate (x, y) """ # Extract the (x, y, z) coordinates of each pixel from the tuple (x, y) = tup @@ -434,7 +460,6 @@ def fitGaussian2D(image, norm=False): of the model. pcov : `numpy.ndarray`_ Corresponding covariance matrix - """ # Normalise if requested wlscl = np.max(image) if norm else 1 @@ -454,46 +479,30 @@ def fitGaussian2D(image, norm=False): return popt, pcov -def rebinND(img, shape): - """ - Rebin a 2D image to a smaller shape. For example, if img.shape=(100,100), - then shape=(10,10) would take the mean of the first 10x10 pixels into a - single output pixel, then the mean of the next 10x10 pixels will be output - into the next pixel - - Args: - img (`numpy.ndarray`_): - A 2D input image - shape (:obj:`tuple`): - The desired shape to be returned. The elements of img.shape - should be an integer multiple of the elements of shape. - - Returns: - img_out (`numpy.ndarray`_): The input image rebinned to shape - """ - # Convert input 2D image into a 4D array to make the rebinning easier - sh = shape[0], img.shape[0]//shape[0], shape[1], img.shape[1]//shape[1] - # Rebin to the 4D array and then average over the second and last elements. - img_out = img.reshape(sh).mean(-1).mean(1) - return img_out - - def extract_standard_spec(stdcube, subpixel=20, method='boxcar'): - """ Extract a spectrum of a standard star from a datacube + """ + Extract a spectrum of a standard star from a datacube - Args: - std_cube (`astropy.io.fits.HDUList`_): - An HDU list of fits files - subpixel (int): - Number of pixels to subpixelate spectrum when creating mask - method (str): - Method used to extract standard star spectrum. Currently, only 'boxcar' is supported + Parameters + ---------- + std_cube : `astropy.io.fits.HDUList`_ + An HDU list of fits files + subpixel : int + Number of pixels to subpixelate spectrum when creating mask + method : str + Method used to extract standard star spectrum. Currently, only 'boxcar' + is supported - Returns: - wave (`numpy.ndarray`_): Wavelength of the star. - Nlam_star (`numpy.ndarray`_): counts/second/Angstrom - Nlam_ivar_star (`numpy.ndarray`_): inverse variance of Nlam_star - gpm_star (`numpy.ndarray`_): good pixel mask for Nlam_star + Returns + ------- + wave : `numpy.ndarray`_ + Wavelength of the star. + Nlam_star : `numpy.ndarray`_ + counts/second/Angstrom + Nlam_ivar_star : `numpy.ndarray`_ + inverse variance of Nlam_star + gpm_star : `numpy.ndarray`_ + good pixel mask for Nlam_star """ # Extract some information from the HDU list flxcube = stdcube['FLUX'].data.T.copy() @@ -522,7 +531,7 @@ def extract_standard_spec(stdcube, subpixel=20, method='boxcar'): nsig = 4 # 4 sigma should be far enough... Note: percentage enclosed for 2D Gaussian = 1-np.exp(-0.5 * nsig**2) ww = np.where((np.sqrt((xx - popt[1]) ** 2 + (yy - popt[2]) ** 2) < nsig * wid)) mask[ww] = 1 - mask = rebinND(mask, (flxcube.shape[0], flxcube.shape[1])).reshape(flxcube.shape[0], flxcube.shape[1], 1) + mask = utils.rebinND(mask, (flxcube.shape[0], flxcube.shape[1])).reshape(flxcube.shape[0], flxcube.shape[1], 1) # Generate a sky mask newshape = (flxcube.shape[0] * subpixel, flxcube.shape[1] * subpixel) @@ -530,7 +539,7 @@ def extract_standard_spec(stdcube, subpixel=20, method='boxcar'): nsig = 8 # 8 sigma should be far enough ww = np.where((np.sqrt((xx - popt[1]) ** 2 + (yy - popt[2]) ** 2) < nsig * wid)) smask[ww] = 1 - smask = rebinND(smask, (flxcube.shape[0], flxcube.shape[1])).reshape(flxcube.shape[0], flxcube.shape[1], 1) + smask = utils.rebinND(smask, (flxcube.shape[0], flxcube.shape[1])).reshape(flxcube.shape[0], flxcube.shape[1], 1) smask -= mask # Subtract the residual sky @@ -626,7 +635,7 @@ def extract_standard_spec(stdcube, subpixel=20, method='boxcar'): # Prepare for optimal msgs.info("Starting optimal extraction") - thismask = np.ones(box_sciimg.shape, dtype=np.bool) + thismask = np.ones(box_sciimg.shape, dtype=bool) nspec, nspat = thismask.shape[0], thismask.shape[1] slit_left = np.zeros(nspec) slit_right = np.ones(nspec)*(nspat-1) @@ -652,10 +661,11 @@ def extract_standard_spec(stdcube, subpixel=20, method='boxcar'): def make_good_skymask(slitimg, tilts): - """ Mask the spectral edges of each slit (i.e. the pixels near the ends of - the detector in the spectral direction). Some extreme values of the tilts are - only sampled with a small fraction of the pixels of the slit width. This leads - to a bad extrapolation/determination of the sky model. + """ + Mask the spectral edges of each slit (i.e. the pixels near the ends of the + detector in the spectral direction). Some extreme values of the tilts are + only sampled with a small fraction of the pixels of the slit width. This + leads to a bad extrapolation/determination of the sky model. Args: slitimg (`numpy.ndarray`_): @@ -664,7 +674,7 @@ def make_good_skymask(slitimg, tilts): Spectral tilts. Returns: - gpm (`numpy.ndarray`_): A mask of the good sky pixels (True = good) + `numpy.ndarray`_: A mask of the good sky pixels (True = good) """ msgs.info("Masking edge pixels where the sky model is poor") # Initialise the GPM @@ -673,7 +683,7 @@ def make_good_skymask(slitimg, tilts): unq = np.unique(slitimg[slitimg>0]) for uu in range(unq.size): # Find the x,y pixels in this slit - ww = np.where(slitimg==unq[uu]) + ww = np.where((slitimg == unq[uu]) & (tilts != 0.0)) # Mask the bottom pixels first wb = np.where(ww[0] == 0)[0] wt = np.where(ww[0] == np.max(ww[0]))[0] @@ -681,27 +691,97 @@ def make_good_skymask(slitimg, tilts): maxtlt = np.max(tilts[0, ww[1][wb]]) mintlt = np.min(tilts[-1, ww[1][wt]]) # Mask all values below this maximum - gpm[ww] = (tilts[ww]>=maxtlt) & (tilts[ww]<=mintlt) # The signs are correct here. + gpm[ww] = (tilts[ww] >= maxtlt) & (tilts[ww] <= mintlt) # The signs are correct here. return gpm +def get_whitelight_pixels(all_wave, min_wl, max_wl): + """ + Determine which pixels are included within the specified wavelength range + + Args: + all_wave (`numpy.ndarray`_): + The wavelength of each individual pixel + min_wl (float): + Minimum wavelength to consider + max_wl (float): + Maximum wavelength to consider + + Returns: + :obj:`tuple`: A `numpy.ndarray`_ object with the indices of all_wave + that contain pixels within the requested wavelength range, and a float + with the wavelength range (i.e. maximum wavelength - minimum wavelength) + """ + wavediff = np.max(all_wave) - np.min(all_wave) + if min_wl < max_wl: + ww = np.where((all_wave > min_wl) & (all_wave < max_wl)) + wavediff = max_wl - min_wl + else: + msgs.warn("Datacubes do not completely overlap in wavelength. Offsets may be unreliable...") + ww = (np.arange(all_wave.size),) + return ww, wavediff + + +def get_whitelight_range(wavemin, wavemax, wl_range): + """ + Get the wavelength range to use for the white light images + + Parameters + ---------- + wavemin : float + Automatically determined minimum wavelength to use for making the white + light image. + wavemax : float + Automatically determined maximum wavelength to use for making the white + light image. + wl_range : list + Two element list containing the user-specified values to manually + override the automated values determined by PypeIt. + + Returns + ------- + wlrng : list + A two element list containing the minimum and maximum wavelength to use + for the white light images + """ + wlrng = [wavemin, wavemax] + if wl_range[0] is not None: + if wl_range[0] < wavemin: + msgs.warn("The user-specified minimum wavelength ({0:.2f}) to use for the white light".format(wl_range[0]) + + msgs.newline() + "images is lower than the recommended value ({0:.2f}),".format(wavemin) + + msgs.newline() + "which ensures that all spaxels cover the same wavelength range.") + wlrng[0] = wl_range[0] + if wl_range[1] is not None: + if wl_range[1] > wavemax: + msgs.warn("The user-specified maximum wavelength ({0:.2f}) to use for the white light".format(wl_range[1]) + + msgs.newline() + "images is greater than the recommended value ({0:.2f}),".format(wavemax) + + msgs.newline() + "which ensures that all spaxels cover the same wavelength range.") + wlrng[1] = wl_range[1] + msgs.info("The white light images will cover the wavelength range: {0:.2f}A - {1:.2f}A".format(wlrng[0], wlrng[1])) + return wlrng + + def make_whitelight_fromcube(cube, wave=None, wavemin=None, wavemax=None): - """ Generate a white light image using an input cube. + """ + Generate a white light image using an input cube. Args: cube (`numpy.ndarray`_): 3D datacube (the final element contains the wavelength dimension) wave (`numpy.ndarray`_, optional): - 1D wavelength array. Only required if wavemin or wavemax are not None. - wavemin (float, None): - Minimum wavelength (same units as wave) to be included in the whitelight image. - You must provide wave as well if you want to reduce the wavelength range. - wavemax (float, None): - Maximum wavelength (same units as wave) to be included in the whitelight image. - You must provide wave as well if you want to reduce the wavelength range. + 1D wavelength array. Only required if wavemin or wavemax are not + None. + wavemin (float, optional): + Minimum wavelength (same units as wave) to be included in the + whitelight image. You must provide wave as well if you want to + reduce the wavelength range. + wavemax (float, optional): + Maximum wavelength (same units as wave) to be included in the + whitelight image. You must provide wave as well if you want to + reduce the wavelength range. Returns: - wl_img (`numpy.ndarray`_): Whitelight image of the input cube. + `numpy.ndarray`_: Whitelight image of the input cube. """ # Make a wavelength cut, if requested cutcube = cube.copy() @@ -727,83 +807,62 @@ def make_whitelight_fromcube(cube, wave=None, wavemin=None, wavemax=None): return wl_img -def make_whitelight_fromref(all_ra, all_dec, all_wave, all_sci, all_wghts, all_idx, dspat, ref_filename): - """ Generate a whitelight image using the individual pixels of every - input frame, based on a reference image. Note the, the reference - image must have a well-defined WCS. +def load_imageWCS(filename, ext=0): + """ + Load an image and return the image and the associated WCS. Args: - all_ra (`numpy.ndarray`_): - 1D flattened array containing the RA values of each pixel from all spec2d files - all_dec (`numpy.ndarray`_): - 1D flattened array containing the DEC values of each pixel from all spec2d files - all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength values of each pixel from all spec2d files - all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all spec2d files - all_wghts (`numpy.ndarray`_): - 1D flattened array containing the weights attributed to each pixel from all spec2d files - all_idx (`numpy.ndarray`_): - 1D flattened array containing an integer identifier indicating which spec2d file - each pixel originates from. For example, a 0 would indicate that a pixel originates - from the first spec2d frame listed in the input file. a 1 would indicate that this - pixel originates from the second spec2d file, and so forth. - dspat (float): - The size of each spaxel on the sky (in degrees) - ref_filename (str): - A fits filename of a reference image to be used when generating white light + filename (str): + A fits filename of an image to be used when generating white light images. Note, the fits file must have a valid 3D WCS. + ext (bool, optional): + The extension that contains the image and WCS Returns: - tuple : two `numpy.ndarray`_ and one WCS will be returned. The first is a 2D reference image - loaded from ref_filename. The second element is a 3D array of shape [N, M, numfiles], - where N and M are the spatial dimensions of the combined white light images. The third is - the WCS of the white light image. + :obj:`tuple`: An `numpy.ndarray`_ with the 2D image data and a + `astropy.wcs.WCS`_ with the image WCS. """ - refhdu = fits.open(ref_filename) - reference_image = refhdu[0].data.T[:, :, 0] - refwcs = wcs.WCS(refhdu[0].header) - numra, numdec = reference_image.shape - # Generate coordinate system (i.e. update wavelength range to include all values) - coord_min = refwcs.wcs.crval - coord_dlt = refwcs.wcs.cdelt - coord_min[2] = np.min(all_wave) - coord_dlt[2] = np.max(all_wave) - np.min(all_wave) # For white light, we want to bin all wavelength pixels - wlwcs = generate_WCS(coord_min, coord_dlt) - - # Generate white light images - whitelight_imgs, _, _ = make_whitelight_frompixels(all_ra, all_dec, all_wave, all_sci, all_wghts, all_idx, dspat, - whitelightWCS=wlwcs, numra=numra, numdec=numdec) + imghdu = fits.open(filename) + image = imghdu[ext].data.T + imgwcs = wcs.WCS(imghdu[ext].header) # Return required info - return reference_image, whitelight_imgs, wlwcs + return image, imgwcs def make_whitelight_frompixels(all_ra, all_dec, all_wave, all_sci, all_wghts, all_idx, dspat, all_ivar=None, whitelightWCS=None, numra=None, numdec=None, trim=1): - """ Generate a whitelight image using the individual pixels of every input frame + """ + Generate a whitelight image using the individual pixels of every input frame Args: all_ra (`numpy.ndarray`_): - 1D flattened array containing the RA values of each pixel from all spec2d files + 1D flattened array containing the RA values of each pixel from all + spec2d files all_dec (`numpy.ndarray`_): - 1D flattened array containing the DEC values of each pixel from all spec2d files + 1D flattened array containing the DEC values of each pixel from all + spec2d files all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength values of each pixel from all spec2d files + 1D flattened array containing the wavelength values of each pixel + from all spec2d files all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all spec2d files + 1D flattened array containing the counts of each pixel from all + spec2d files all_wghts (`numpy.ndarray`_): - 1D flattened array containing the weights attributed to each pixel from all spec2d files + 1D flattened array containing the weights attributed to each pixel + from all spec2d files all_idx (`numpy.ndarray`_): - 1D flattened array containing an integer identifier indicating which spec2d file - each pixel originates from. For example, a 0 would indicate that a pixel originates - from the first spec2d frame listed in the input file. a 1 would indicate that this - pixel originates from the second spec2d file, and so forth. + 1D flattened array containing an integer identifier indicating which + spec2d file each pixel originates from. For example, a 0 would + indicate that a pixel originates from the first spec2d frame listed + in the input file. a 1 would indicate that this pixel originates + from the second spec2d file, and so forth. dspat (float): The size of each spaxel on the sky (in degrees) all_ivar (`numpy.ndarray`_, optional): - 1D flattened array containing of the inverse variance of each pixel from all spec2d files. - If provided, inverse variance images will be calculated and returned for each white light image. - whitelightWCS (`astropy.wcs.wcs.WCS`_, optional): + 1D flattened array containing of the inverse variance of each pixel + from all spec2d files. If provided, inverse variance images will be + calculated and returned for each white light image. + whitelightWCS (`astropy.wcs.WCS`_, optional): The WCS of a reference white light image. If supplied, you must also supply numra and numdec. numra (int, optional): @@ -814,10 +873,11 @@ def make_whitelight_frompixels(all_ra, all_dec, all_wave, all_sci, all_wghts, al Number of pixels to grow around a masked region Returns: - tuple : two 3D arrays will be returned, each of shape [N, M, numfiles], - where N and M are the spatial dimensions of the combined white light images. - The first array is a white light image, and the second array is the corresponding - inverse variance image. If all_ivar is None, this will be an empty array. + tuple: two 3D arrays will be returned, each of shape [N, M, numfiles], + where N and M are the spatial dimensions of the combined white light + images. The first array is a white light image, and the second array is + the corresponding inverse variance image. If all_ivar is None, this will + be an empty array. """ # Determine number of files numfiles = np.unique(all_idx).size @@ -872,7 +932,106 @@ def make_whitelight_frompixels(all_ra, all_dec, all_wave, all_sci, all_wghts, al return whitelight_Imgs, whitelight_ivar, whitelightWCS -def generate_WCS(crval, cdelt, equinox=2000.0, name="Instrument Unknown"): +def create_wcs(cubepar, all_ra, all_dec, all_wave, dspat, dwv, collapse=False, equinox=2000.0, + specname="PYP_SPEC"): + """ + Create a WCS and the expected edges of the voxels, based on user-specified + parameters or the extremities of the data. + + Parameters + ---------- + cubepar : :class:`~pypeit.par.pypeitpar.CubePar` + An instance of the CubePar parameter set, contained parameters of the + datacube reduction + all_ra : `numpy.ndarray`_ + 1D flattened array containing the RA values of each pixel from all + spec2d files + all_dec : `numpy.ndarray`_ + 1D flattened array containing the DEC values of each pixel from all + spec2d files + all_wave : `numpy.ndarray`_ + 1D flattened array containing the wavelength values of each pixel from + all spec2d files + dspat : float + Spatial size of each square voxel (in arcsec). The default is to use the + values in cubepar. + dwv : float + Linear wavelength step of each voxel (in Angstroms) + collapse : bool, optional + If True, the spectral dimension will be collapsed to a single channel + (primarily for white light images) + equinox : float, optional + Equinox of the WCS + specname : str, optional + Name of the spectrograph + + Returns + ------- + cubewcs : `astropy.wcs.WCS`_ + astropy WCS to be used for the combined cube + voxedges : tuple + A three element tuple containing the bin edges in the x, y (spatial) and + z (wavelength) dimensions + reference_image : `numpy.ndarray`_ + The reference image to be used for the cross-correlation. Can be None. + """ + # Grab cos(dec) for convenience + cosdec = np.cos(np.mean(all_dec) * np.pi / 180.0) + + # Setup the cube ranges + reference_image = None # The default behaviour is that the reference image is not used + ra_min = cubepar['ra_min'] if cubepar['ra_min'] is not None else np.min(all_ra) + ra_max = cubepar['ra_max'] if cubepar['ra_max'] is not None else np.max(all_ra) + dec_min = cubepar['dec_min'] if cubepar['dec_min'] is not None else np.min(all_dec) + dec_max = cubepar['dec_max'] if cubepar['dec_max'] is not None else np.max(all_dec) + wav_min = cubepar['wave_min'] if cubepar['wave_min'] is not None else np.min(all_wave) + wav_max = cubepar['wave_max'] if cubepar['wave_max'] is not None else np.max(all_wave) + dwave = cubepar['wave_delta'] if cubepar['wave_delta'] is not None else dwv + + # Number of voxels in each dimension + numra = int((ra_max-ra_min) * cosdec / dspat) + numdec = int((dec_max-dec_min)/dspat) + numwav = int(np.round((wav_max-wav_min)/dwave)) + + # If a white light WCS is being generated, make sure there's only 1 wavelength bin + if collapse: + wav_min = np.min(all_wave) + wav_max = np.max(all_wave) + dwave = wav_max - wav_min + numwav = 1 + + # Generate a master WCS to register all frames + coord_min = [ra_min, dec_min, wav_min] + coord_dlt = [dspat, dspat, dwave] + + # If a reference image is being used and a white light image is requested (collapse=True) update the celestial parts + if cubepar["reference_image"] is not None: + # Load the requested reference image + reference_image, imgwcs = load_imageWCS(cubepar["reference_image"]) + # Update the celestial WCS + coord_min[:2] = imgwcs.wcs.crval + coord_dlt[:2] = imgwcs.wcs.cdelt + numra, numdec = reference_image.shape + + cubewcs = generate_WCS(coord_min, coord_dlt, equinox=equinox, name=specname) + msgs.info(msgs.newline() + "-" * 40 + + msgs.newline() + "Parameters of the WCS:" + + msgs.newline() + "RA min = {0:f}".format(coord_min[0]) + + msgs.newline() + "DEC min = {0:f}".format(coord_min[1]) + + msgs.newline() + "WAVE min, max = {0:f}, {1:f}".format(wav_min, wav_max) + + msgs.newline() + "Spaxel size = {0:f} arcsec".format(3600.0*dspat) + + msgs.newline() + "Wavelength step = {0:f} A".format(dwave) + + msgs.newline() + "-" * 40) + + # Generate the output binning + xbins = np.arange(1+numra)-0.5 + ybins = np.arange(1+numdec)-0.5 + spec_bins = np.arange(1+numwav)-0.5 + voxedges = (xbins, ybins, spec_bins) + return cubewcs, voxedges, reference_image + + +def generate_WCS(crval, cdelt, equinox=2000.0, name="PYP_SPEC"): """ Generate a WCS that will cover all input spec2D files @@ -883,11 +1042,11 @@ def generate_WCS(crval, cdelt, equinox=2000.0, name="Instrument Unknown"): cdelt (list): 3 element list containing the delta values of the [RA, DEC, WAVELENGTH] - equinox (float): + equinox (float, optional): Equinox of the WCS Returns: - `astropy.wcs.wcs.WCS`_ : astropy WCS to be used for the combined cube + `astropy.wcs.WCS`_ : astropy WCS to be used for the combined cube """ # Create a new WCS object. msgs.info("Generating WCS") @@ -910,40 +1069,49 @@ def generate_WCS(crval, cdelt, equinox=2000.0, name="Instrument Unknown"): def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, whitelight_img, dspat, dwv, sn_smooth_npix=None, relative_weights=False): - """ Calculate wavelength dependent optimal weights. The weighting - is currently based on a relative (S/N)^2 at each wavelength + r""" + Calculate wavelength dependent optimal weights. The weighting is currently + based on a relative :math:`(S/N)^2` at each wavelength Args: all_ra (`numpy.ndarray`_): - 1D flattened array containing the RA values of each pixel from all spec2d files + 1D flattened array containing the RA values of each pixel from all + spec2d files all_dec (`numpy.ndarray`_): - 1D flattened array containing the DEC values of each pixel from all spec2d files + 1D flattened array containing the DEC values of each pixel from all + spec2d files all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength values of each pixel from all spec2d files + 1D flattened array containing the wavelength values of each pixel + from all spec2d files all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all spec2d files + 1D flattened array containing the counts of each pixel from all + spec2d files all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel from all spec2d files + 1D flattened array containing the inverse variance of each pixel + from all spec2d files all_idx (`numpy.ndarray`_): - 1D flattened array containing an integer identifier indicating which spec2d file - each pixel originates from. For example, a 0 would indicate that a pixel originates - from the first spec2d frame listed in the input file. a 1 would indicate that this - pixel originates from the second spec2d file, and so forth. + 1D flattened array containing an integer identifier indicating which + spec2d file each pixel originates from. For example, a 0 would + indicate that a pixel originates from the first spec2d frame listed + in the input file. a 1 would indicate that this pixel originates + from the second spec2d file, and so forth. whitelight_img (`numpy.ndarray`_): - A 2D array containing a whitelight image, that was created with the input all_* arrays. + A 2D array containing a whitelight image, that was created with the + input ``all_`` arrays. dspat (float): The size of each spaxel on the sky (in degrees) dwv (float): The size of each wavelength pixel (in Angstroms) sn_smooth_npix (float, optional): - Number of pixels used for determining smoothly varying S/N ratio weights. - This is currently not required, since a relative weighting scheme with a - polynomial fit is used to calculate the S/N weights. + Number of pixels used for determining smoothly varying S/N ratio + weights. This is currently not required, since a relative weighting + scheme with a polynomial fit is used to calculate the S/N weights. relative_weights (bool, optional): Calculate weights by fitting to the ratio of spectra? + Returns: - `numpy.ndarray`_ : a 1D array the same size as all_sci, containing relative wavelength - dependent weights of each input pixel. + `numpy.ndarray`_ : a 1D array the same size as all_sci, containing + relative wavelength dependent weights of each input pixel. """ msgs.info("Calculating the optimal weights of each pixel") # Determine number of files @@ -989,67 +1157,196 @@ def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, white # Compute the smoothing scale to use if sn_smooth_npix is None: sn_smooth_npix = int(np.round(0.1 * wave_spec.size)) - rms_sn, weights = coadd.sn_weights(wave_spec, flux_stack, ivar_stack, mask_stack, sn_smooth_npix, - relative_weights=relative_weights) + rms_sn, weights = coadd.sn_weights(utils.array_to_explist(flux_stack), utils.array_to_explist(ivar_stack), utils.array_to_explist(mask_stack), + sn_smooth_npix=sn_smooth_npix, relative_weights=relative_weights) # Because we pass back a weights array, we need to interpolate to assign each detector pixel a weight all_wghts = np.ones(all_idx.size) for ff in range(numfiles): ww = (all_idx == ff) - all_wghts[ww] = interp1d(wave_spec, weights[:, ff], kind='cubic', + all_wghts[ww] = interp1d(wave_spec, weights[ff], kind='cubic', bounds_error=False, fill_value="extrapolate")(all_wave[ww]) msgs.info("Optimal weighting complete") return all_wghts -def generate_cube_subpixel(outfile, output_wcs, all_sci, all_ivar, all_wghts, all_wave, tilts, slits, slitid_img_gpm, - astrom_trans, bins, spec_subpixel=10, spat_subpixel=10, overwrite=False, blaze_wave=None, - blaze_spec=None, fluxcal=False, sensfunc=None, specname="PYP_SPEC", debug=False): +def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, all_spatpos, all_specpos, + all_spatid, tilts, slits, astrom_trans, bins, all_idx=None, + spec_subpixel=10, spat_subpixel=10, combine=False): """ - Save a datacube using the subpixel algorithm. This algorithm splits - each detector pixel into multiple subpixels, and then assigns each - subpixel to a voxel. For example, if spec_subpixel = spat_subpixel = 10, - then each detector pixel is divided into 10^2=100 subpixels. Alternatively, - when spec_subpixel = spat_subpixel = 1, this corresponds to the nearest - grid point (NGP) algorithm. + Generate a white light image from the input pixels - Important Note: If spec_subpixel > 1 or spat_subpixel > 1, the errors will - be correlated, and the covariance is not being tracked, so the errors will - not be (quite) right. There is a tradeoff one has to make between sampling - and better looking cubes, versus no sampling and better behaved errors. + Args: + image_wcs (`astropy.wcs.WCS`_): + World coordinate system to use for the white light images. + all_ra (`numpy.ndarray`_): + 1D flattened array containing the right ascension of each pixel + (units = degrees) + all_dec (`numpy.ndarray`_): + 1D flattened array containing the declination of each pixel (units = + degrees) + all_wave (`numpy.ndarray`_): + 1D flattened array containing the wavelength of each pixel (units = + Angstroms) + all_sci (`numpy.ndarray`_): + 1D flattened array containing the counts of each pixel from all + spec2d files + all_ivar (`numpy.ndarray`_): + 1D flattened array containing the inverse variance of each pixel + from all spec2d files + all_wghts (`numpy.ndarray`_): + 1D flattened array containing the weights of each pixel to be used + in the combination + all_spatpos (`numpy.ndarray`_): + 1D flattened array containing the detector pixel location in the + spatial direction + all_specpos (`numpy.ndarray`_): + 1D flattened array containing the detector pixel location in the + spectral direction + all_spatid (`numpy.ndarray`_): + 1D flattened array containing the spatid of each pixel + tilts (`numpy.ndarray`_, list): + 2D wavelength tilts frame, or a list of tilt frames (see all_idx) + slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): + Information stored about the slits, or a list of SlitTraceSet (see + all_idx) + astrom_trans (:class:`~pypeit.alignframe.AlignmentSplines`, list): + A Class containing the transformation between detector pixel + coordinates and WCS pixel coordinates, or a list of Alignment + Splines (see all_idx) + bins (tuple): + A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial + and z wavelength coordinates + all_idx (`numpy.ndarray`_, optional): + If tilts, slits, and astrom_trans are lists, this should contain a + 1D flattened array, of the same length as all_sci, containing the + index the tilts, slits, and astrom_trans lists that corresponds to + each pixel. Note that, in this case all of these lists need to be + the same length. + spec_subpixel (:obj:`int`, optional): + What is the subpixellation factor in the spectral direction. Higher + values give more reliable results, but note that the time required + goes as (``spec_subpixel * spat_subpixel``). The default value is 5, + which divides each detector pixel into 5 subpixels in the spectral + direction. + spat_subpixel (:obj:`int`, optional): + What is the subpixellation factor in the spatial direction. Higher + values give more reliable results, but note that the time required + goes as (``spec_subpixel * spat_subpixel``). The default value is 5, + which divides each detector pixel into 5 subpixels in the spatial + direction. + combine (:obj:`bool`, optional): + If True, all of the input frames will be combined into a single + output. Otherwise, individual images will be generated. + + Returns: + `numpy.ndarray`_: The white light images for all frames + """ + # Perform some checks on the input -- note, more complete checks are performed in subpixellate() + _all_idx = np.zeros(all_sci.size) if all_idx is None else all_idx + if combine: + numfr = 1 + else: + numfr = np.unique(_all_idx).size + if len(tilts) != numfr or len(slits) != numfr or len(astrom_trans) != numfr: + msgs.error("The following arguments must be the same length as the expected number of frames to be combined:" + + msgs.newline() + "tilts, slits, astrom_trans") + # Prepare the array of white light images to be stored + numra = bins[0].size-1 + numdec = bins[1].size-1 + all_wl_imgs = np.zeros((numra, numdec, numfr)) + + # Loop through all frames and generate white light images + for fr in range(numfr): + msgs.info(f"Creating image {fr+1}/{numfr}") + if combine: + # Subpixellate + img, _, _ = subpixellate(image_wcs, all_ra, all_dec, all_wave, + all_sci, all_ivar, all_wghts, all_spatpos, + all_specpos, all_spatid, tilts, slits, astrom_trans, bins, + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, all_idx=_all_idx) + else: + ww = np.where(_all_idx == fr) + # Subpixellate + img, _, _ = subpixellate(image_wcs, all_ra[ww], all_dec[ww], all_wave[ww], + all_sci[ww], all_ivar[ww], all_wghts[ww], all_spatpos[ww], + all_specpos[ww], all_spatid[ww], tilts[fr], slits[fr], astrom_trans[fr], bins, + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) + all_wl_imgs[:, :, fr] = img[:, :, 0] + # Return the constructed white light images + return all_wl_imgs + + +def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, + all_spatpos, all_specpos, all_spatid, tilts, slits, astrom_trans, bins, + all_idx=None, spec_subpixel=10, spat_subpixel=10, overwrite=False, blaze_wave=None, + blaze_spec=None, fluxcal=False, sensfunc=None, whitelight_range=None, + specname="PYP_SPEC", debug=False): + r""" + Save a datacube using the subpixel algorithm. Refer to the subpixellate() + docstring for further details about this algorithm Args: outfile (str): Filename to be used to save the datacube - output_wcs (`astropy.wcs.wcs.WCS`_): + output_wcs (`astropy.wcs.WCS`_): Output world coordinate system. + all_ra (`numpy.ndarray`_): + 1D flattened array containing the right ascension of each pixel + (units = degrees) + all_dec (`numpy.ndarray`_): + 1D flattened array containing the declination of each pixel (units = + degrees) + all_wave (`numpy.ndarray`_): + 1D flattened array containing the wavelength of each pixel (units = + Angstroms) all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all spec2d files + 1D flattened array containing the counts of each pixel from all + spec2d files all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel from all spec2d files + 1D flattened array containing the inverse variance of each pixel + from all spec2d files all_wghts (`numpy.ndarray`_): - 1D flattened array containing the weights of each pixel to be used in the combination - all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength of each pixel (units = Angstroms) - tilts (`numpy.ndarray`_): - 2D wavelength tilts frame - slits (:class:`pypeit.slittrace.SlitTraceSet`): - Information stored about the slits - slitid_img_gpm (`numpy.ndarray`_): - An image indicating which pixels belong to a slit (0 = not on a slit or a masked pixel). - Any positive value indicates the spatial ID of the pixel. - astrom_trans (:class:`pypeit.alignframe.AlignmentSplines`): - A Class containing the transformation between detector pixel coordinates and WCS pixel coordinates + 1D flattened array containing the weights of each pixel to be used + in the combination + all_spatpos (`numpy.ndarray`_): + 1D flattened array containing the detector pixel location in the + spatial direction + all_specpos (`numpy.ndarray`_): + 1D flattened array containing the detector pixel location in the + spectral direction + all_spatid (`numpy.ndarray`_): + 1D flattened array containing the spatid of each pixel + tilts (`numpy.ndarray`_, list): + 2D wavelength tilts frame, or a list of tilt frames (see all_idx) + slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): + Information stored about the slits, or a list of SlitTraceSet (see + all_idx) + astrom_trans (:class:`~pypeit.alignframe.AlignmentSplines`, list): + A Class containing the transformation between detector pixel + coordinates and WCS pixel coordinates, or a list of Alignment + Splines (see all_idx) bins (tuple): - A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial and z wavelength coordinates + A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial + and z wavelength coordinates + all_idx (`numpy.ndarray`_, optional): + If tilts, slits, and astrom_trans are lists, this should contain a + 1D flattened array, of the same length as all_sci, containing the + index the tilts, slits, and astrom_trans lists that corresponds to + each pixel. Note that, in this case all of these lists need to be + the same length. spec_subpixel (int, optional): - What is the subpixellation factor in the spectral direction. Higher values give more reliable results, - but note that the time required goes as (spec_subpixel * spat_subpixel). The default value is 5, - which divides each detector pixel into 5 subpixels in the spectral direction. + What is the subpixellation factor in the spectral direction. Higher + values give more reliable results, but note that the time required + goes as (``spec_subpixel * spat_subpixel``). The default value is 5, + which divides each detector pixel into 5 subpixels in the spectral + direction. spat_subpixel (int, optional): - What is the subpixellation factor in the spatial direction. Higher values give more reliable results, - but note that the time required goes as (spec_subpixel * spat_subpixel). The default value is 5, - which divides each detector pixel into 5 subpixels in the spatial direction. + What is the subpixellation factor in the spatial direction. Higher + values give more reliable results, but note that the time required + goes as (``spec_subpixel * spat_subpixel``). The default value is 5, + which divides each detector pixel into 5 subpixels in the spatial + direction. overwrite (bool, optional): If True, the output cube will be overwritten. blaze_wave (`numpy.ndarray`_, optional): @@ -1057,66 +1354,27 @@ def generate_cube_subpixel(outfile, output_wcs, all_sci, all_ivar, all_wghts, al blaze_spec (`numpy.ndarray`_, optional): Spectral blaze function fluxcal (bool, optional): - Are the data flux calibrated? If True, the units are: erg/s/cm^2/Angstrom/arcsec^2 - multiplied by the PYPEIT_FLUX_SCALE. Otherwise, the units are: counts/s/Angstrom/arcsec^2") + Are the data flux calibrated? If True, the units are: :math:`{\rm + erg/s/cm}^2{\rm /Angstrom/arcsec}^2` multiplied by the + PYPEIT_FLUX_SCALE. Otherwise, the units are: :math:`{\rm + counts/s/Angstrom/arcsec}^2`. sensfunc (`numpy.ndarray`_, None, optional): Sensitivity function that has been applied to the datacube + whitelight_range (None, list, optional): + A two element list that specifies the minimum and maximum + wavelengths (in Angstroms) to use when constructing the white light + image (format is: [min_wave, max_wave]). If None, the cube will be + collapsed over the full wavelength range. If a list is provided an + either element of the list is None, then the minimum/maximum + wavelength range of that element will be set by the minimum/maximum + wavelength of all_wave. specname (str, optional): Name of the spectrograph debug (bool, optional): - If True, a residuals cube will be output. If the datacube generation is correct, the - distribution of pixels in the residual cube with no flux should have mean=0 and std=1. + If True, a residuals cube will be output. If the datacube generation + is correct, the distribution of pixels in the residual cube with no + flux should have mean=0 and std=1. """ - # Prepare the output arrays - outshape = (bins[0].size-1, bins[1].size-1, bins[2].size-1) - datacube, varcube, normcube = np.zeros(outshape), np.zeros(outshape), np.zeros(outshape) - if debug: residcube = np.zeros(outshape) - # Divide each pixel into subpixels - spec_offs = np.arange(0.5/spec_subpixel, 1, 1/spec_subpixel) - 0.5 # -0.5 is to offset from the centre of each pixel. - spat_offs = np.arange(0.5/spat_subpixel, 1, 1/spat_subpixel) - 0.5 # -0.5 is to offset from the centre of each pixel. - area = 1 / (spec_subpixel * spat_subpixel) - all_wght_subpix = all_wghts * area - # Loop through all slits - all_sltid = slitid_img_gpm[(slitid_img_gpm > 0)] - all_var = utils.inverse(all_ivar) - wave0, wave_delta = output_wcs.wcs.crval[2], output_wcs.wcs.cd[2, 2] - for sl, spatid in enumerate(slits.spat_id): - msgs.info(f"Resampling slit {sl+1}/{slits.nslits} into the datacube") - this_sl = np.where(all_sltid == spatid) - wpix = np.where(slitid_img_gpm == spatid) - slitID = np.ones(wpix[0].size) * sl - output_wcs.wcs.crpix[0] - # Generate a spline between spectral pixel position and wavelength - yspl = tilts[wpix]*(slits.nspec - 1) - wspl = all_wave[this_sl] - asrt = np.argsort(yspl) - wave_spl = interp1d(yspl[asrt], wspl[asrt], kind='linear', bounds_error=False, fill_value='extrapolate') - for xx in range(spat_subpixel): - for yy in range(spec_subpixel): - # Calculate the transformation from detector pixels to voxels - spatpos = astrom_trans.transform(sl, wpix[1] + spat_offs[xx], wpix[0] + spec_offs[yy]) - # TODO :: The tilts in the following line is evaluated at the pixel location, not the subpixel locations - # A simple fix is implemented for the spectral direction, but this is not so straightforward for the spatial direction - # Probably, the correction in the spatial direction is so tiny, that this doesn't matter... - specpos = (wave_spl(tilts[wpix]*(slits.nspec - 1) + spec_offs[yy]) - wave0) / wave_delta - # Now assemble this position of the datacube - pix_coord = np.column_stack((slitID, spatpos, specpos)) - tmp_dc, _ = np.histogramdd(pix_coord, bins=bins, weights=all_sci[this_sl] * all_wght_subpix[this_sl]) - tmp_vr, _ = np.histogramdd(pix_coord, bins=bins, weights=all_var[this_sl] * all_wght_subpix[this_sl]**2) - tmp_nm, _ = np.histogramdd(pix_coord, bins=bins, weights=all_wght_subpix[this_sl]) - if debug: - tmp_rsd, _ = np.histogramdd(pix_coord, bins=bins, weights=all_sci[this_sl] * np.sqrt(all_ivar[this_sl])) - residcube += tmp_rsd - datacube += tmp_dc - varcube += tmp_vr - normcube += tmp_nm - # Normalise the datacube and variance cube - nc_inverse = utils.inverse(normcube) - datacube *= nc_inverse - varcube *= nc_inverse**2 - bpmcube = (normcube == 0).astype(np.uint8) - if debug: - residcube *= nc_inverse - # Prepare the header, and add the unit of flux to the header hdr = output_wcs.to_header() if fluxcal: @@ -1124,89 +1382,236 @@ def generate_cube_subpixel(outfile, output_wcs, all_sci, all_ivar, all_wghts, al else: hdr['FLUXUNIT'] = (1, "Flux units -- counts/s/Angstrom/arcsec^2") - # Write out the datacube - msgs.info("Saving datacube as: {0:s}".format(outfile)) - final_cube = DataCube(datacube.T, np.sqrt(varcube.T), bpmcube.T, specname, blaze_wave, blaze_spec, sensfunc=sensfunc, fluxed=fluxcal) - final_cube.to_file(outfile, hdr=hdr, overwrite=overwrite) - - # Save a residuals cube, if requested + # Subpixellate + subpix = subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, all_spatpos, all_specpos, + all_spatid, tilts, slits, astrom_trans, bins, all_idx=all_idx, + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, debug=debug) + # Extract the variables that we need if debug: + datacube, varcube, bpmcube, residcube = subpix + # Save a residuals cube outfile_resid = outfile.replace(".fits", "_resid.fits") msgs.info("Saving residuals datacube as: {0:s}".format(outfile_resid)) hdu = fits.PrimaryHDU(residcube.T, header=hdr) hdu.writeto(outfile_resid, overwrite=overwrite) + else: + datacube, varcube, bpmcube = subpix + + # Check if the user requested a white light image + if whitelight_range is not None: + # Grab the WCS of the white light image + whitelight_wcs = output_wcs.celestial + # Determine the wavelength range of the whitelight image + if whitelight_range[0] is None: + whitelight_range[0] = np.min(all_wave) + if whitelight_range[1] is None: + whitelight_range[1] = np.max(all_wave) + msgs.info("White light image covers the wavelength range {0:.2f} A - {1:.2f} A".format( + whitelight_range[0], whitelight_range[1])) + # Get the output filename for the white light image + out_whitelight = get_output_whitelight_filename(outfile) + nspec = datacube.shape[2] + # Get wavelength of each pixel, and note that the WCS gives this in m, so convert to Angstroms (x 1E10) + wave = 1.0E10 * output_wcs.spectral.wcs_pix2world(np.arange(nspec), 0)[0] + whitelight_img = make_whitelight_fromcube(datacube, wave=wave, wavemin=whitelight_range[0], wavemax=whitelight_range[1]) + msgs.info("Saving white light image as: {0:s}".format(out_whitelight)) + img_hdu = fits.PrimaryHDU(whitelight_img.T, header=whitelight_wcs.to_header()) + img_hdu.writeto(out_whitelight, overwrite=overwrite) + # Write out the datacube + msgs.info("Saving datacube as: {0:s}".format(outfile)) + final_cube = DataCube(datacube.T, np.sqrt(varcube.T), bpmcube.T, specname, blaze_wave, blaze_spec, + sensfunc=sensfunc, fluxed=fluxcal) + final_cube.to_file(outfile, hdr=hdr, overwrite=overwrite) -def generate_cube_ngp(outfile, hdr, all_sci, all_ivar, all_wghts, vox_coord, bins, - overwrite=False, blaze_wave=None, blaze_spec=None, fluxcal=False, - sensfunc=None, specname="PYP_SPEC", debug=False): - """ - Save a datacube using the Nearest Grid Point (NGP) algorithm. + +def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, all_spatpos, all_specpos, + all_spatid, tilts, slits, astrom_trans, bins, all_idx=None, + spec_subpixel=10, spat_subpixel=10, debug=False): + r""" + Subpixellate the input data into a datacube. This algorithm splits each + detector pixel into multiple subpixels, and then assigns each subpixel to a + voxel. For example, if ``spec_subpixel = spat_subpixel = 10``, then each + detector pixel is divided into :math:`10^2=100` subpixels. Alternatively, + when spec_subpixel = spat_subpixel = 1, this corresponds to the nearest grid + point (NGP) algorithm. + + Important Note: If spec_subpixel > 1 or spat_subpixel > 1, the errors will + be correlated, and the covariance is not being tracked, so the errors will + not be (quite) right. There is a tradeoff one has to make between sampling + and better looking cubes, versus no sampling and better behaved errors. Args: - outfile (`str`): - Filename to be used to save the datacube - hdr (`astropy.io.fits.header_`): - Header of the output datacube (must contain WCS) + output_wcs (`astropy.wcs.WCS`_): + Output world coordinate system. + all_ra (`numpy.ndarray`_): + 1D flattened array containing the right ascension of each pixel + (units = degrees) + all_dec (`numpy.ndarray`_): + 1D flattened array containing the declination of each pixel (units = + degrees) + all_wave (`numpy.ndarray`_): + 1D flattened array containing the wavelength of each pixel (units = + Angstroms) all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all spec2d files + 1D flattened array containing the counts of each pixel from all + spec2d files all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel from all spec2d files + 1D flattened array containing the inverse variance of each pixel + from all spec2d files all_wghts (`numpy.ndarray`_): - 1D flattened array containing the weights of each pixel to be used in the combination - vox_coord (`numpy.ndarray`_): - The voxel coordinates of each pixel in the spec2d frames. vox_coord is returned by the - function `astropy.wcs.WCS.wcs_world2pix_` once a WCS is setup and every spec2d detector - pixel has an RA, DEC, and WAVELENGTH. + 1D flattened array containing the weights of each pixel to be used + in the combination + all_spatpos (`numpy.ndarray`_): + 1D flattened array containing the detector pixel location in the + spatial direction + all_specpos (`numpy.ndarray`_): + 1D flattened array containing the detector pixel location in the + spectral direction + all_spatid (`numpy.ndarray`_): + 1D flattened array containing the spatid of each pixel + tilts (`numpy.ndarray`_, list): + 2D wavelength tilts frame, or a list of tilt frames (see all_idx) + slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): + Information stored about the slits, or a list of SlitTraceSet (see + all_idx) + astrom_trans (:class:`~pypeit.alignframe.AlignmentSplines`, list): + A Class containing the transformation between detector pixel + coordinates and WCS pixel coordinates, or a list of Alignment + Splines (see all_idx) bins (tuple): - A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial and z wavelength coordinates - overwrite (`bool`): - If True, the output cube will be overwritten. - blaze_wave (`numpy.ndarray`_): - Wavelength array of the spectral blaze function - blaze_spec (`numpy.ndarray`_): - Spectral blaze function - fluxcal (bool): - Are the data flux calibrated? - sensfunc (`numpy.ndarray`_, None): - Sensitivity function that has been applied to the datacube - specname (str): - Name of the spectrograph + A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial + and z wavelength coordinates + all_idx (`numpy.ndarray`_, optional): + If tilts, slits, and astrom_trans are lists, this should contain a + 1D flattened array, of the same length as all_sci, containing the + index the tilts, slits, and astrom_trans lists that corresponds to + each pixel. Note that, in this case all of these lists need to be + the same length. + spec_subpixel (:obj:`int`, optional): + What is the subpixellation factor in the spectral direction. Higher + values give more reliable results, but note that the time required + goes as (``spec_subpixel * spat_subpixel``). The default value is 5, + which divides each detector pixel into 5 subpixels in the spectral + direction. + spat_subpixel (:obj:`int`, optional): + What is the subpixellation factor in the spatial direction. Higher + values give more reliable results, but note that the time required + goes as (``spec_subpixel * spat_subpixel``). The default value is 5, + which divides each detector pixel into 5 subpixels in the spatial + direction. debug (bool): - Debug the code by writing out a residuals cube? + If True, a residuals cube will be output. If the datacube generation + is correct, the distribution of pixels in the residual cube with no + flux should have mean=0 and std=1. + + Returns: + :obj:`tuple`: Three or four `numpy.ndarray`_ objects containing (1) the + datacube generated from the subpixellated inputs, (2) the corresponding + variance cube, (3) the corresponding bad pixel mask cube, and (4) the + residual cube. The latter is only returned if debug is True. """ - # Add the unit of flux to the header - if fluxcal: - hdr['FLUXUNIT'] = (flux_calib.PYPEIT_FLUX_SCALE, "Flux units -- erg/s/cm^2/Angstrom/arcsec^2") + # Check for combinations of lists or not + if type(tilts) is list and type(slits) is list and type(astrom_trans) is list: + # Several frames are being combined. Check the lists have the same length + numframes = len(tilts) + if len(slits) != numframes or len(astrom_trans) != numframes: + msgs.error("The following lists must have the same length:" + msgs.newline() + + "tilts, slits, astrom_trans") + # Check all_idx has been set + if all_idx is None: + if numframes != 1: + msgs.error("Missing required argument for combining frames: all_idx") + else: + all_idx = np.zeros(all_sci.size) + else: + tmp = np.unique(all_idx).size + if tmp != numframes: + msgs.warn("Indices in argument 'all_idx' does not match the number of frames expected.") + # Store in the following variables + _tilts, _slits, _astrom_trans = tilts, slits, astrom_trans + elif type(tilts) is not list and type(slits) is not list and \ + type(astrom_trans) is not list: + # Just a single frame - store as lists for this code + _tilts, _slits, _astrom_trans = [tilts], [slits], [astrom_trans], + all_idx = np.zeros(all_sci.size) + numframes = 1 else: - hdr['FLUXUNIT'] = (1, "Flux units -- counts/s/Angstrom/arcsec^2") - - # Use NGP to generate the cube - this ensures errors between neighbouring voxels are not correlated - datacube, edges = np.histogramdd(vox_coord, bins=bins, weights=all_sci * all_wghts) - normcube, edges = np.histogramdd(vox_coord, bins=bins, weights=all_wghts) + msgs.error("The following input arguments should all be of type 'list', or all not be type 'list':" + + msgs.newline() + "tilts, slits, astrom_trans") + # Prepare the output arrays + outshape = (bins[0].size-1, bins[1].size-1, bins[2].size-1) + binrng = [[bins[0][0], bins[0][-1]], [bins[1][0], bins[1][-1]], [bins[2][0], bins[2][-1]]] + datacube, varcube, normcube = np.zeros(outshape), np.zeros(outshape), np.zeros(outshape) + if debug: + residcube = np.zeros(outshape) + # Divide each pixel into subpixels + spec_offs = np.arange(0.5/spec_subpixel, 1, 1/spec_subpixel) - 0.5 # -0.5 is to offset from the centre of each pixel. + spat_offs = np.arange(0.5/spat_subpixel, 1, 1/spat_subpixel) - 0.5 # -0.5 is to offset from the centre of each pixel. + spat_x, spec_y = np.meshgrid(spat_offs, spec_offs) + num_subpixels = spec_subpixel * spat_subpixel + area = 1 / num_subpixels + all_wght_subpix = all_wghts * area + all_var = utils.inverse(all_ivar) + # Loop through all exposures + for fr in range(numframes): + # Extract tilts and slits for convenience + this_tilts = _tilts[fr] + this_slits = _slits[fr] + # Loop through all slits + for sl, spatid in enumerate(this_slits.spat_id): + if numframes == 1: + msgs.info(f"Resampling slit {sl+1}/{this_slits.nslits}") + else: + msgs.info(f"Resampling slit {sl+1}/{this_slits.nslits} of frame {fr+1}/{numframes}") + this_sl = np.where((all_spatid == spatid) & (all_idx == fr)) + wpix = (all_specpos[this_sl], all_spatpos[this_sl]) + # Generate a spline between spectral pixel position and wavelength + yspl = this_tilts[wpix]*(this_slits.nspec - 1) + tiltpos = np.add.outer(yspl, spec_y).flatten() + wspl = all_wave[this_sl] + asrt = np.argsort(yspl) + wave_spl = interp1d(yspl[asrt], wspl[asrt], kind='linear', bounds_error=False, fill_value='extrapolate') + # Calculate spatial and spectral positions of the subpixels + spat_xx = np.add.outer(wpix[1], spat_x.flatten()).flatten() + spec_yy = np.add.outer(wpix[0], spec_y.flatten()).flatten() + # Transform this to spatial location + spatpos_subpix = _astrom_trans[fr].transform(sl, spat_xx, spec_yy) + spatpos = _astrom_trans[fr].transform(sl, all_spatpos[this_sl], all_specpos[this_sl]) + ra_coeff = np.polyfit(spatpos, all_ra[this_sl], 1) + dec_coeff = np.polyfit(spatpos, all_dec[this_sl], 1) + this_ra = np.polyval(ra_coeff, spatpos_subpix)#ra_spl(spatpos_subpix) + this_dec = np.polyval(dec_coeff, spatpos_subpix)#dec_spl(spatpos_subpix) + # ssrt = np.argsort(spatpos) + # ra_spl = interp1d(spatpos[ssrt], all_ra[this_sl][ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') + # dec_spl = interp1d(spatpos[ssrt], all_dec[this_sl][ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') + # this_ra = ra_spl(spatpos_subpix) + # this_dec = dec_spl(spatpos_subpix) + this_wave = wave_spl(tiltpos) + # Convert world coordinates to voxel coordinates, then histogram + vox_coord = output_wcs.wcs_world2pix(np.vstack((this_ra, this_dec, this_wave * 1.0E-10)).T, 0) + if histogramdd is not None: + # use the "fast histogram" algorithm, that assumes regular bin spacing + datacube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_sci[this_sl] * all_wght_subpix[this_sl], num_subpixels)) + varcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_var[this_sl] * all_wght_subpix[this_sl]**2, num_subpixels)) + normcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_wght_subpix[this_sl], num_subpixels)) + if debug: + residcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_sci[this_sl] * np.sqrt(all_ivar[this_sl]), num_subpixels)) + else: + datacube += np.histogramdd(vox_coord, bins=outshape, weights=np.repeat(all_sci[this_sl] * all_wght_subpix[this_sl], num_subpixels))[0] + varcube += np.histogramdd(vox_coord, bins=outshape, weights=np.repeat(all_var[this_sl] * all_wght_subpix[this_sl]**2, num_subpixels))[0] + normcube += np.histogramdd(vox_coord, bins=outshape, weights=np.repeat(all_wght_subpix[this_sl], num_subpixels))[0] + if debug: + residcube += np.histogramdd(vox_coord, bins=outshape, weights=np.repeat(all_sci[this_sl] * np.sqrt(all_ivar[this_sl]), num_subpixels))[0] + # Normalise the datacube and variance cube nc_inverse = utils.inverse(normcube) datacube *= nc_inverse - # Create the variance cube, including weights - msgs.info("Generating variance cube") - all_var = utils.inverse(all_ivar) - var_cube, edges = np.histogramdd(vox_coord, bins=bins, weights=all_var * all_wghts ** 2) - var_cube *= nc_inverse**2 + varcube *= nc_inverse**2 bpmcube = (normcube == 0).astype(np.uint8) - - # Save the datacube if debug: - datacube_resid, edges = np.histogramdd(vox_coord, bins=bins, weights=all_sci * np.sqrt(all_ivar)) - normcube, edges = np.histogramdd(vox_coord, bins=bins) - nc_inverse = utils.inverse(normcube) - outfile_resid = "datacube_resid.fits" - msgs.info("Saving datacube as: {0:s}".format(outfile_resid)) - hdu = fits.PrimaryHDU((datacube_resid*nc_inverse).T, header=hdr) - hdu.writeto(outfile_resid, overwrite=overwrite) - - msgs.info("Saving datacube as: {0:s}".format(outfile)) - final_cube = DataCube(datacube.T, np.sqrt(var_cube.T), bpmcube.T, specname, blaze_wave, blaze_spec, - sensfunc=sensfunc, fluxed=fluxcal) - final_cube.to_file(outfile, hdr=hdr, overwrite=overwrite) + residcube *= nc_inverse + return datacube, varcube, bpmcube, residcube + return datacube, varcube, bpmcube def get_output_filename(fil, par_outfile, combine, idx=1): @@ -1224,7 +1629,7 @@ def get_output_filename(fil, par_outfile, combine, idx=1): Index of filename to be saved. Required if combine=False. Returns: - outfile (str): The output filename to use. + str: The output filename to use. """ if combine: if par_outfile == "": @@ -1241,6 +1646,22 @@ def get_output_filename(fil, par_outfile, combine, idx=1): return outfile +def get_output_whitelight_filename(outfile): + """ + Given the output filename of a datacube, create an appropriate whitelight + fits file name + + Args: + outfile (str): + The output filename used for the datacube. + + Returns: + str: The output filename to use for the whitelight image. + """ + out_wl_filename = os.path.splitext(outfile)[0] + "_whitelight.fits" + return out_wl_filename + + def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): """ Main routine to coadd spec2D files into a 3D datacube @@ -1275,7 +1696,7 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): # Grab the parset, if not provided if parset is None: - # TODO: Use config_specific_par instead? + # TODO :: Use config_specific_par instead? parset = spec.default_pypeit_par() cubepar = parset['reduce']['cube'] flatpar = parset['calibrations']['flatfield'] @@ -1283,12 +1704,36 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): # prep numfiles = len(files) - combine = cubepar['combine'] method = cubepar['method'].lower() + combine = cubepar['combine'] + align = cubepar['align'] + # If there is only one frame being "combined" AND there's no reference image, then don't compute the translation. + if numfiles == 1 and cubepar["reference_image"] is None: + if not align: + msgs.warn("Parameter 'align' should be False when there is only one frame and no reference image") + msgs.info("Setting 'align' to False") + align = False + if opts['ra_offset'] is not None: + if not align: + msgs.warn("When 'ra_offset' and 'dec_offset' are set, 'align' must be True.") + msgs.info("Setting 'align' to True") + align = True + # TODO :: The default behaviour (combine=False, align=False) produces a datacube that uses the instrument WCS + # It should be possible (and perhaps desirable) to do a spatial alignment (i.e. align=True), apply this to the + # RA,Dec values of each pixel, and then use the instrument WCS to save the output (or, just adjust the crval). + # At the moment, if the user wishes to spatially align the frames, a different WCS is generated. + if histogramdd is None: + msgs.warn("Generating a datacube is faster if you install fast-histogram:"+msgs.newline()+ + "https://pypi.org/project/fast-histogram/") + if method != 'ngp': + msgs.warn("Forcing NGP algorithm, because fast-histogram is not installed") + method = 'ngp' # Determine what method is requested + spec_subpixel, spat_subpixel = 1, 1 if method == "subpixel": msgs.info("Adopting the subpixel algorithm to generate the datacube.") + spec_subpixel, spat_subpixel = cubepar['spec_subpixel'], cubepar['spat_subpixel'] elif method == "ngp": msgs.info("Adopting the nearest grid point (NGP) algorithm to generate the datacube.") else: @@ -1301,7 +1746,7 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): # Check if the output file exists if combine: outfile = get_output_filename("", cubepar['output_filename'], combine) - out_whitelight = os.path.splitext(outfile)[0] + "_whitelight.fits" + out_whitelight = get_output_whitelight_filename(outfile) if os.path.exists(outfile) and not overwrite: msgs.error("Output filename already exists:"+msgs.newline()+outfile) if os.path.exists(out_whitelight) and cubepar['save_whitelight'] and not overwrite: @@ -1310,7 +1755,7 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): # Finally, if there's just one file, check if the output filename is given if numfiles == 1 and cubepar['output_filename'] != "": outfile = get_output_filename("", cubepar['output_filename'], True, -1) - out_whitelight = os.path.splitext(outfile)[0] + "_whitelight.fits" + out_whitelight = get_output_whitelight_filename(outfile) if os.path.exists(outfile) and not overwrite: msgs.error("Output filename already exists:" + msgs.newline() + outfile) if os.path.exists(out_whitelight) and cubepar['save_whitelight'] and not overwrite: @@ -1318,7 +1763,7 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): else: for ff in range(numfiles): outfile = get_output_filename(files[ff], cubepar['output_filename'], combine, ff+1) - out_whitelight = os.path.splitext(outfile)[0] + "_whitelight.fits" + out_whitelight = get_output_whitelight_filename(outfile) if os.path.exists(outfile) and not overwrite: msgs.error("Output filename already exists:" + msgs.newline() + outfile) if os.path.exists(out_whitelight) and cubepar['save_whitelight'] and not overwrite: @@ -1352,7 +1797,7 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): blaze_spline = interp1d(blaze_wave, blaze_spec, kind='linear', bounds_error=False, fill_value="extrapolate") # Perform a grating correction - grat_corr = calc_grating_corr(wave.value, blaze_wave_curr, blaze_spline_curr, blaze_wave, blaze_spline) + grat_corr = correct_grating_shift(wave.value, blaze_wave_curr, blaze_spline_curr, blaze_wave, blaze_spline) # Apply the grating correction to the standard star spectrum Nlam_star /= grat_corr Nlam_ivar_star *= grat_corr**2 @@ -1362,6 +1807,8 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): star_mag=senspar['star_mag'], ra=star_ra, dec=star_dec) # Calculate the sensitivity curve + # TODO :: This needs to be addressed... unify flux calibration into the main PypeIt routines. + msgs.warn("Datacubes are currently flux-calibrated using the UVIS algorithm... this will be deprecated soon") zeropoint_data, zeropoint_data_gpm, zeropoint_fit, zeropoint_fit_gpm =\ flux_calib.fit_zeropoint(wave.value, Nlam_star, Nlam_ivar_star, gpm_star, std_dict, mask_hydrogen_lines=senspar['mask_hydrogen_lines'], @@ -1380,8 +1827,11 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): msgs.error("Reference image does not exist:" + msgs.newline() + cubepar['reference_image']) # Initialise arrays for storage + ifu_ra, ifu_dec = np.array([]), np.array([]) # The RA and Dec at the centre of the IFU, as stored in the header all_ra, all_dec, all_wave = np.array([]), np.array([]), np.array([]) all_sci, all_ivar, all_idx, all_wghts = np.array([]), np.array([]), np.array([]), np.array([]) + all_spatpos, all_specpos, all_spatid = np.array([], dtype=int), np.array([], dtype=int), np.array([], dtype=int) + all_tilts, all_slits, all_align = [], [], [] all_wcs = [] dspat = None if cubepar['spatial_delta'] is None else cubepar['spatial_delta']/3600.0 # binning size on the sky (/3600 to convert to degrees) dwv = cubepar['wave_delta'] # binning size in wavelength direction (in Angstroms) @@ -1390,27 +1840,37 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): weights = np.ones(numfiles) # Weights to use when combining cubes flat_splines = dict() # A dictionary containing the splines of the flatfield # Load the default scaleimg frame for the scale correction + scalecorr_default = "none" relScaleImgDef = np.array([1]) if cubepar['scale_corr'] is not None: - msgs.info("Loading default scale image for relative spectral illumination correction:" + - msgs.newline() + cubepar['scale_corr']) - try: - spec2DObj = spec2dobj.Spec2DObj.from_file(cubepar['scale_corr'], detname) - relScaleImgDef = spec2DObj.scaleimg - except: - msgs.warn("Could not load scaleimg from spec2d file:" + msgs.newline() + - cubepar['scale_corr'] + msgs.newline() + - "scale correction will not be performed unless you have specified the correct" + msgs.newline() + - "scale_corr file in the spec2d block") - cubepar['scale_corr'] = None + if cubepar['scale_corr'] == "image": + msgs.info("The default relative spectral illumination correction will use the science image") + scalecorr_default = "image" + else: + msgs.info("Loading default scale image for relative spectral illumination correction:" + + msgs.newline() + cubepar['scale_corr']) + try: + spec2DObj = spec2dobj.Spec2DObj.from_file(cubepar['scale_corr'], detname) + relScaleImgDef = spec2DObj.scaleimg + scalecorr_default = cubepar['scale_corr'] + except: + msgs.warn("Could not load scaleimg from spec2d file:" + msgs.newline() + + cubepar['scale_corr'] + msgs.newline() + + "scale correction will not be performed unless you have specified the correct" + msgs.newline() + + "scale_corr file in the spec2d block") + cubepar['scale_corr'] = None + scalecorr_default = "none" + # Load the default sky frame to be used for sky subtraction skysub_default = "image" - skysubImgDef = None # This is the default behaviour (i.e. to use the "image" for the sky subtraction) - + skyImgDef, skySclDef = None, None # This is the default behaviour (i.e. to use the "image" for the sky subtraction) if cubepar['skysub_frame'] in [None, 'none', '', 'None']: skysub_default = "none" - skysubImgDef = np.array([0.0]) # Do not perform sky subtraction + skyImgDef = np.array([0.0]) # Do not perform sky subtraction + skySclDef = np.array([0.0]) # Do not perform sky subtraction elif cubepar['skysub_frame'].lower() == "image": + msgs.info("The sky model in the spec2d science frames will be used for sky subtraction" +msgs.newline() + + "(unless specific skysub frames have been specified)") skysub_default = "image" else: msgs.info("Loading default image for sky subtraction:" + @@ -1421,17 +1881,21 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): except: msgs.error("Could not load skysub image from spec2d file:" + msgs.newline() + cubepar['skysub_frame']) skysub_default = cubepar['skysub_frame'] - skysubImgDef = spec2DObj.skymodel/skysub_exptime # Sky counts/second + skyImgDef = spec2DObj.skymodel/skysub_exptime # Sky counts/second + skySclDef = spec2DObj.scaleimg # Load all spec2d files and prepare the data for making a datacube for ff, fil in enumerate(files): # Load it up + msgs.info("Loading PypeIt spec2d frame:" + msgs.newline() + fil) spec2DObj = spec2dobj.Spec2DObj.from_file(fil, detname) detector = spec2DObj.detector - flexure = None #spec2DObj.sci_spat_flexure + spat_flexure = None #spec2DObj.sci_spat_flexure # Load the header - hdr = spec2DObj.head0 #fits.open(fil)[0].header + hdr = spec2DObj.head0 + ifu_ra = np.append(ifu_ra, spec.compound_meta([hdr], 'ra')) + ifu_dec = np.append(ifu_dec, spec.compound_meta([hdr], 'dec')) # Get the exposure time exptime = hdr['EXPTIME'] @@ -1439,41 +1903,45 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): # Setup for PypeIt imports msgs.reset(verbosity=2) - # Try to load the relative scale image, if something other than the default has been provided - relScaleImg = relScaleImgDef.copy() - if opts['scale_corr'][ff] is not None: - msgs.info("Loading relative scale image:" + msgs.newline() + opts['scale_corr'][ff]) - spec2DObj_scl = spec2dobj.Spec2DObj.from_file(opts['scale_corr'][ff], detname) - try: - relScaleImg = spec2DObj_scl.scaleimg - except: - msgs.warn("Could not load scaleimg from spec2d file:" + msgs.newline() + opts['scale_corr'][ff]) - if cubepar['scale_corr'] is not None: - msgs.info("Using the default scaleimg from spec2d file:" + msgs.newline() + cubepar['scale_corr']) - else: - msgs.warn("Scale correction will not be performed") - relScaleImg = relScaleImgDef.copy() + # TODO :: Consider loading all calibrations into a single variable. + + # Initialise the slit edges + msgs.info("Constructing slit image") + slits = spec2DObj.slits + slitid_img_init = slits.slit_img(pad=0, initial=True, flexure=spat_flexure) + slits_left, slits_right, _ = slits.select_edges(initial=True, flexure=spat_flexure) + + # The order of operations below proceeds as follows: + # (1) Get science image + # (2) Subtract sky (note, if a joint fit has been performed, the relative scale correction is applied in the reduction!) + # (3) Apply relative scale correction to both science and ivar # Set the default behaviour if a global skysub frame has been specified this_skysub = skysub_default if skysub_default == "image": - skysubImg = spec2DObj.skymodel + skyImg = spec2DObj.skymodel + skyScl = spec2DObj.scaleimg else: - skysubImg = skysubImgDef.copy() * exptime + skyImg = skyImgDef.copy() * exptime + skyScl = skySclDef.copy() # See if there's any changes from the default behaviour if opts['skysub_frame'][ff] is not None: if opts['skysub_frame'][ff].lower() == 'default': if skysub_default == "image": - skysubImg = spec2DObj.skymodel + skyImg = spec2DObj.skymodel + skyScl = spec2DObj.scaleimg this_skysub = "image" # Use the current spec2d for sky subtraction else: - skysubImg = skysubImgDef.copy() * exptime + skyImg = skyImgDef.copy() * exptime + skyScl = skySclDef.copy() * exptime this_skysub = skysub_default # Use the global value for sky subtraction elif opts['skysub_frame'][ff].lower() == 'image': - skysubImg = spec2DObj.skymodel + skyImg = spec2DObj.skymodel + skyScl = spec2DObj.scaleimg this_skysub = "image" # Use the current spec2d for sky subtraction elif opts['skysub_frame'][ff].lower() == 'none': - skysubImg = np.array([0.0]) + skyImg = np.array([0.0]) + skyScl = np.array([1.0]) this_skysub = "none" # Don't do sky subtraction else: # Load a user specified frame for sky subtraction @@ -1483,26 +1951,68 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): skysub_exptime = fits.open(opts['skysub_frame'][ff])[0].header['EXPTIME'] except: msgs.error("Could not load skysub image from spec2d file:" + msgs.newline() + opts['skysub_frame'][ff]) - skysubImg = spec2DObj_sky.skymodel * exptime / skysub_exptime # Sky counts + # TODO :: Consider allowing the actual frame (instead of the skymodel) to be used as the skysub image - make sure the BPM is carried over. + # :: Allow sky data fitting (i.e. scale the flux of a skysub frame to the science frame data) + skyImg = spec2DObj_sky.skymodel * exptime / skysub_exptime # Sky counts + skyScl = spec2DObj_sky.scaleimg this_skysub = opts['skysub_frame'][ff] # User specified spec2d for sky subtraction if this_skysub == "none": msgs.info("Sky subtraction will not be performed.") else: msgs.info("Using the following frame for sky subtraction:"+msgs.newline()+this_skysub) - # Extract the information - relscl = 1.0 - if cubepar['scale_corr'] is not None or opts['scale_corr'][ff] is not None: - relscl = spec2DObj.scaleimg/relScaleImg - sciimg = (spec2DObj.sciimg-skysubImg)*relscl # Subtract sky - ivar = spec2DObj.ivarraw / relscl**2 + # Load the relative scale image, if something other than the default has been provided + this_scalecorr = scalecorr_default + relScaleImg = relScaleImgDef.copy() + if opts['scale_corr'][ff] is not None: + if opts['scale_corr'][ff].lower() == 'default': + if scalecorr_default == "image": + relScaleImg = spec2DObj.scaleimg + this_scalecorr = "image" # Use the current spec2d for the relative spectral illumination scaling + else: + this_scalecorr = scalecorr_default # Use the default value for the scale correction + elif opts['scale_corr'][ff].lower() == 'image': + relScaleImg = spec2DObj.scaleimg + this_scalecorr = "image" # Use the current spec2d for the relative spectral illumination scaling + elif opts['scale_corr'][ff].lower() == 'none': + relScaleImg = np.array([1]) + this_scalecorr = "none" # Don't do relative spectral illumination scaling + else: + # Load a user specified frame for sky subtraction + msgs.info("Loading the following frame for the relative spectral illumination correction:" + + msgs.newline() + opts['scale_corr'][ff]) + try: + spec2DObj_scl = spec2dobj.Spec2DObj.from_file(opts['scale_corr'][ff], detname) + except: + msgs.error("Could not load skysub image from spec2d file:" + msgs.newline() + opts['skysub_frame'][ff]) + relScaleImg = spec2DObj_scl.scaleimg + this_scalecorr = opts['scale_corr'][ff] + if this_scalecorr == "none": + msgs.info("Relative spectral illumination correction will not be performed.") + else: + msgs.info("Using the following frame for the relative spectral illumination correction:" + + msgs.newline()+this_scalecorr) + + # Prepare the relative scaling factors + relSclSky = skyScl/spec2DObj.scaleimg # This factor ensures the sky has the same relative scaling as the science frame + relScale = spec2DObj.scaleimg/relScaleImg # This factor is applied to the sky subtracted science frame + + # Extract the relevant information from the spec2d file + sciImg = (spec2DObj.sciimg - skyImg*relSclSky)*relScale # Subtract sky and apply relative illumination + ivar = spec2DObj.ivarraw / relScale**2 waveimg = spec2DObj.waveimg bpmmask = spec2DObj.bpmmask - # Grab the slit edges - slits = spec2DObj.slits + # TODO :: Really need to write some detailed information in the docs about all of the various corrections that can optionally be applied + + # TODO :: Include a flexure correction from the sky frame? Note, you cannot use the waveimg from a sky frame, + # since the heliocentric correction may have been applied to the sky frame. Need to recalculate waveimg using + # the slitshifts from a skyimage, and then apply the vel_corr from the science image. - wave0 = waveimg[waveimg != 0.0].min() + wnonzero = (waveimg != 0.0) + if not np.any(wnonzero): + msgs.error("The wavelength image contains only zeros - You need to check the data reduction.") + wave0 = waveimg[wnonzero].min() # Calculate the delta wave in every pixel on the slit waveimp = np.roll(waveimg, 1, axis=0) waveimn = np.roll(waveimg, -1, axis=0) @@ -1520,14 +2030,9 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): msgs.info("Using wavelength solution: wave0={0:.3f}, dispersion={1:.3f} Angstrom/pixel".format(wave0, dwv)) - msgs.info("Constructing slit image") - slitid_img_init = slits.slit_img(pad=0, initial=True, flexure=flexure) - # Obtain the minimum and maximum wavelength of all slits if mnmx_wv is None: - mnmx_wv = np.zeros((1, slits.nslits, 2)) - else: - mnmx_wv = np.append(mnmx_wv, np.zeros((1, slits.nslits, 2)), axis=0) + mnmx_wv = np.zeros((len(files), slits.nslits, 2)) for slit_idx, slit_spat in enumerate(slits.spat_id): onslit_init = (slitid_img_init == slit_spat) mnmx_wv[ff, slit_idx, 0] = np.min(waveimg[onslit_init]) @@ -1537,8 +2042,7 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): sky_is_good = make_good_skymask(slitid_img_init, spec2DObj.tilts) # Construct a good pixel mask - # TODO: This should use the mask function to figure out which elements - # are masked. + # TODO: This should use the mask function to figure out which elements are masked. onslit_gpm = (slitid_img_init > 0) & (bpmmask.mask == 0) & sky_is_good # Grab the WCS of this frame @@ -1572,7 +2076,7 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): msgs.info("Using slit edges for astrometric transform") # If nothing better was provided, use the slit edges if alignments is None: - left, right, _ = slits.select_edges(initial=True, flexure=flexure) + left, right, _ = slits.select_edges(initial=True, flexure=spat_flexure) locations = [0.0, 1.0] traces = np.append(left[:,None,:], right[:,None,:], axis=1) else: @@ -1582,8 +2086,7 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): msgs.info("Generating RA/DEC image") alignSplines = alignframe.AlignmentSplines(traces, locations, spec2DObj.tilts) raimg, decimg, minmax = slits.get_radec_image(frame_wcs, alignSplines, spec2DObj.tilts, - initial=True, flexure=flexure) - + initial=True, flexure=spat_flexure) # Perform the DAR correction if wave_ref is None: wave_ref = 0.5*(np.min(waveimg[onslit_gpm]) + np.max(waveimg[onslit_gpm])) @@ -1603,14 +2106,14 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): " Pressure = {0:f} bar".format(pressure) + msgs.newline() + " Temperature = {0:f} deg C".format(temperature) + msgs.newline() + " Humidity = {0:f}".format(rel_humidity)) - ra_corr, dec_corr = dar_correction(waveimg[onslit_gpm], coord, obstime, location, - pressure*units.bar, temperature*units.deg_C, rel_humidity, wave_ref=wave_ref) + ra_corr, dec_corr = correct_dar(waveimg[onslit_gpm], coord, obstime, location, + pressure * units.bar, temperature * units.deg_C, rel_humidity, wave_ref=wave_ref) raimg[onslit_gpm] += ra_corr*np.cos(np.mean(decimg[onslit_gpm]) * np.pi / 180.0) decimg[onslit_gpm] += dec_corr # Get copies of arrays to be saved wave_ext = waveimg[onslit_gpm].copy() - flux_ext = sciimg[onslit_gpm].copy() + flux_ext = sciImg[onslit_gpm].copy() ivar_ext = ivar[onslit_gpm].copy() dwav_ext = dwaveimg[onslit_gpm].copy() @@ -1620,17 +2123,26 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): if key not in spec2DObj.calibs: msgs.error('Processed flat calibration file not recorded by spec2d file!') flatfile = os.path.join(spec2DObj.calibs['DIR'], spec2DObj.calibs[key]) - # TODO: Check that the file exists? if cubepar['grating_corr'] and flatfile not in flat_splines.keys(): msgs.info("Calculating relative sensitivity for grating correction") + # Check if the Flat file exists + if not os.path.exists(flatfile): + msgs.error("Grating correction requested, but the following file does not exist:" + + msgs.newline() + flatfile) + # Load the Flat file flatimages = flatfield.FlatImages.from_file(flatfile) - flatframe = flatimages.illumflat_raw/flatimages.fit2illumflat(slits, frametype='illum', initial=True, - spat_flexure=flexure) - # Calculate the relative scale - scale_model = flatfield.illum_profile_spectral(flatframe, waveimg, slits, - slit_illum_ref_idx=flatpar['slit_illum_ref_idx'], model=None, - skymask=None, trim=flatpar['slit_trim'], flexure=flexure, - smooth_npix=flatpar['slit_illum_smooth_npix']) + total_illum = flatimages.fit2illumflat(slits, finecorr=False, frametype='illum', initial=True, spat_flexure=spat_flexure) * \ + flatimages.fit2illumflat(slits, finecorr=True, frametype='illum', initial=True, spat_flexure=spat_flexure) + flatframe = flatimages.pixelflat_raw / total_illum + if flatimages.pixelflat_spec_illum is None: + # Calculate the relative scale + scale_model = flatfield.illum_profile_spectral(flatframe, waveimg, slits, + slit_illum_ref_idx=flatpar['slit_illum_ref_idx'], model=None, + skymask=None, trim=flatpar['slit_trim'], flexure=spat_flexure, + smooth_npix=flatpar['slit_illum_smooth_npix']) + else: + msgs.info("Using relative spectral illumination from FlatImages") + scale_model = flatimages.pixelflat_spec_illum # Apply the relative scale and generate a 1D "spectrum" onslit = waveimg != 0 wavebins = np.linspace(np.min(waveimg[onslit]), np.max(waveimg[onslit]), slits.nspec) @@ -1662,8 +2174,8 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): # Grating correction grat_corr = 1.0 if cubepar['grating_corr']: - grat_corr = calc_grating_corr(wave_ext[wvsrt], flat_splines[flatfile+"_wave"], flat_splines[flatfile], - blaze_wave, blaze_spline) + grat_corr = correct_grating_shift(wave_ext[wvsrt], flat_splines[flatfile + "_wave"], flat_splines[flatfile], + blaze_wave, blaze_spline) # Sensitivity function sens_func = 1.0 if fluxcal: @@ -1688,27 +2200,20 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): numpix = raimg[onslit_gpm].size # Calculate the weights relative to the zeroth cube - weights[ff] = exptime #np.median(flux_sav[resrt]*np.sqrt(ivar_sav[resrt]))**2 + weights[ff] = 1.0#exptime #np.median(flux_sav[resrt]*np.sqrt(ivar_sav[resrt]))**2 + + # Get the slit image and then unset pixels in the slit image that are bad + this_specpos, this_spatpos = np.where(onslit_gpm) + this_spatid = slitid_img_init[onslit_gpm] - # If individual frames are to be output, there's no need to store information, just make the cubes now - if not combine: + # If individual frames are to be output without aligning them, + # there's no need to store information, just make the cubes now + if not combine and not align: # Get the output filename if numfiles == 1 and cubepar['output_filename'] != "": outfile = get_output_filename("", cubepar['output_filename'], True, -1) else: outfile = get_output_filename(fil, cubepar['output_filename'], combine, ff+1) - # Generate individual whitelight images of each spec2d file - # TODO :: May be better to generate this after making the cube? - if cubepar['save_whitelight']: - # if method == 'resample': - # msgs.warn("Whitelight images are not implemented with the 'resample' algorithm.") - # msgs.info("Generating a whitelight image with the NGP algorithm.") - out_whitelight = os.path.splitext(outfile)[0] + "_whitelight.fits" - whitelight_img, _, wlwcs = make_whitelight_frompixels(raimg[onslit_gpm], decimg[onslit_gpm], wave_ext, - flux_sav[resrt], np.ones(numpix), np.zeros(numpix), dspat) - msgs.info("Saving white light image as: {0:s}".format(out_whitelight)) - img_hdu = fits.PrimaryHDU(whitelight_img.T, header=wlwcs.to_header()) - img_hdu.writeto(out_whitelight, overwrite=overwrite) # Get the coordinate bounds slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) numwav = int((np.max(waveimg) - wave0) / dwv) @@ -1716,35 +2221,26 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): # Generate the output WCS for the datacube crval_wv = cubepar['wave_min'] if cubepar['wave_min'] is not None else 1.0E10 * frame_wcs.wcs.crval[2] cd_wv = cubepar['wave_delta'] if cubepar['wave_delta'] is not None else 1.0E10 * frame_wcs.wcs.cd[2, 2] - cd_spat = cubepar['spatial_delta'] if cubepar['spatial_delta'] is not None else px_deg * 3600.0 - output_wcs = spec.get_wcs(spec2DObj.head0, slits, detector.platescale, crval_wv, cd_wv, spatial_scale=cd_spat) + output_wcs = spec.get_wcs(spec2DObj.head0, slits, detector.platescale, crval_wv, cd_wv) + # Set the wavelength range of the white light image. + wl_wvrng = None + if cubepar['save_whitelight']: + wl_wvrng = get_whitelight_range(np.max(mnmx_wv[ff, :, 0]), + np.min(mnmx_wv[ff, :, 1]), + cubepar['whitelight_range']) # Make the datacube if method in ['subpixel', 'ngp']: - if method == 'ngp': - spec_subpixel, spat_subpixel = 1, 1 - else: - spec_subpixel, spat_subpixel = cubepar['spec_subpixel'], cubepar['spat_subpixel'] - # Get the slit image and then unset pixels in the slit image that are bad - slitid_img_gpm = slitid_img_init.copy() - slitid_img_gpm[(bpmmask.mask != 0) | (~sky_is_good)] = 0 - generate_cube_subpixel(outfile, output_wcs, flux_sav[resrt], ivar_sav[resrt], np.ones(numpix), - wave_ext, spec2DObj.tilts, slits, slitid_img_gpm, alignSplines, bins, - overwrite=overwrite, blaze_wave=blaze_wave, blaze_spec=blaze_spec, - fluxcal=fluxcal, specname=specname, + # Generate the datacube + generate_cube_subpixel(outfile, output_wcs, raimg[onslit_gpm], decimg[onslit_gpm], wave_ext, + flux_sav[resrt], ivar_sav[resrt], np.ones(numpix), + this_spatpos, this_specpos, this_spatid, + spec2DObj.tilts, slits, alignSplines, bins, + all_idx=None, overwrite=overwrite, blaze_wave=blaze_wave, blaze_spec=blaze_spec, + fluxcal=fluxcal, specname=specname, whitelight_range=wl_wvrng, spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) - # elif method == 'resample': - # fluximg, ivarimg = np.zeros_like(raimg), np.zeros_like(raimg) - # fluximg[onslit_gpm] = flux_sav[resrt] - # ivarimg[onslit_gpm] = ivar_sav[resrt] - # # Now generate the cube - # generate_cube_resample(outfile, frame_wcs, slits, fluximg, ivarimg, raimg, decimg, waveimg, slitid_img_init, onslit_gpm, - # overwrite=overwrite, output_wcs=output_wcs, blaze_wave=blaze_wave, blaze_spec=blaze_spec, - # fluxcal=fluxcal, specname=specname) - # else: - # msgs.error(f"The following method is not yet implemented: {method}") continue - # Store the information + # Store the information if we are combining multiple frames all_ra = np.append(all_ra, raimg[onslit_gpm].copy()) all_dec = np.append(all_dec, decimg[onslit_gpm].copy()) all_wave = np.append(all_wave, wave_ext.copy()) @@ -1752,138 +2248,132 @@ def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): all_ivar = np.append(all_ivar, ivar_sav[resrt].copy()) all_idx = np.append(all_idx, ff*np.ones(numpix)) all_wghts = np.append(all_wghts, weights[ff]*np.ones(numpix)/weights[0]) - - # No need to continue if we are not combining frames - if not combine: + all_spatpos = np.append(all_spatpos, this_spatpos) + all_specpos = np.append(all_specpos, this_specpos) + all_spatid = np.append(all_spatid, this_spatid) + all_tilts.append(spec2DObj.tilts) + all_slits.append(slits) + all_align.append(alignSplines) + + # No need to continue if we are not combining nor aligning frames + if not combine and not align: return # Grab cos(dec) for convenience cosdec = np.cos(np.mean(all_dec) * np.pi / 180.0) # Register spatial offsets between all frames - # Check if a reference whitelight image should be used to register the offsets - numiter=2 - for dd in range(numiter): - msgs.info(f"Iterating on spatial translation - ITERATION #{dd+1}/{numiter}") - if cubepar["reference_image"] is None: - # Find the wavelength range where all frames overlap - min_wl, max_wl = np.max(mnmx_wv[:,:,0]), np.min(mnmx_wv[:,:,1]) # This is the max blue wavelength and the min red wavelength - # Generate white light images - if min_wl < max_wl: - ww = np.where((all_wave > min_wl) & (all_wave < max_wl)) - else: - msgs.warn("Datacubes do not completely overlap in wavelength. Offsets may be unreliable...") - ww = np.where((all_wave > 0) & (all_wave < 99999999)) - whitelight_imgs, _, _ = make_whitelight_frompixels(all_ra[ww], all_dec[ww], all_wave[ww], all_sci[ww], all_wghts[ww], all_idx[ww], dspat) - # ref_idx will be the index of the cube with the highest S/N - ref_idx = np.argmax(weights) - reference_image = whitelight_imgs[:, :, ref_idx].copy() - msgs.info("Calculating spatial translation of each cube relative to cube #{0:d})".format(ref_idx+1)) - else: - ref_idx = -1 # Don't use an index - # Load reference information - reference_image, whitelight_imgs, wlwcs = \ - make_whitelight_fromref(all_ra, all_dec, all_wave, all_sci, all_wghts, all_idx, dspat, - cubepar['reference_image']) - msgs.info("Calculating the spatial translation of each cube relative to user-defined 'reference_image'") - # Calculate the image offsets - check the reference is a zero shift - for ff in range(numfiles): - # Calculate the shift - ra_shift, dec_shift = calculate_image_phase(reference_image.copy(), whitelight_imgs[:, :, ff], maskval=0.0) - # Convert pixel shift to degress shift - ra_shift *= dspat/cosdec - dec_shift *= dspat - msgs.info("Spatial shift of cube #{0:d}: RA, DEC (arcsec) = {1:+0.3f}, {2:+0.3f}".format(ff+1, ra_shift*3600.0, dec_shift*3600.0)) - # Apply the shift - all_ra[all_idx == ff] += ra_shift - all_dec[all_idx == ff] += dec_shift - # Generate a white light image of *all* data - msgs.info("Generating global white light image") - if cubepar["reference_image"] is None: - # Find the wavelength range where all frames overlap - min_wl, max_wl = np.max(mnmx_wv[:, :, 0]), np.min(mnmx_wv[:, :, 1]) # This is the max blue wavelength and the min red wavelength - if min_wl < max_wl: - ww = np.where((all_wave > min_wl) & (all_wave < max_wl)) - msgs.info("Whitelight image covers the wavelength range {0:.2f} A - {1:.2f} A".format(min_wl, max_wl)) + if align: + if opts['ra_offset'] is not None: + # First, translate all coordinates to the coordinates of the first frame + # Note :: Don't need cosdec here, this just overrides the IFU coordinate centre of each frame + ref_shift_ra = ifu_ra[0] - ifu_ra + ref_shift_dec = ifu_dec[0] - ifu_dec + for ff in range(numfiles): + # Apply the shift + all_ra[all_idx == ff] += ref_shift_ra[ff] + opts['ra_offset'][ff]/3600.0 + all_dec[all_idx == ff] += ref_shift_dec[ff] + opts['dec_offset'][ff]/3600.0 + msgs.info("Spatial shift of cube #{0:d}: RA, DEC (arcsec) = {1:+0.3f} E, {2:+0.3f} N".format(ff + 1, opts['ra_offset'][ff], opts['dec_offset'][ff])) else: - msgs.warn("Datacubes do not completely overlap in wavelength. Offsets may be unreliable...") - ww = np.where((all_wave > 0) & (all_wave < 99999999)) - whitelight_img, _, wlwcs = make_whitelight_frompixels(all_ra[ww], all_dec[ww], all_wave[ww], all_sci[ww], - all_wghts[ww], np.zeros(ww[0].size), dspat) - else: - _, whitelight_img, wlwcs = \ - make_whitelight_fromref(all_ra, all_dec, all_wave, all_sci, all_wghts, np.zeros(all_ra.size), - dspat, cubepar['reference_image']) + # Find the wavelength range where all frames overlap + min_wl, max_wl = get_whitelight_range(np.max(mnmx_wv[:, :, 0]), # The max blue wavelength + np.min(mnmx_wv[:, :, 1]), # The min red wavelength + cubepar['whitelight_range']) # The user-specified values (if any) + # Get the good whitelight pixels + ww, wavediff = get_whitelight_pixels(all_wave, min_wl, max_wl) + # Iterate over white light image generation and spatial shifting + numiter = 2 + for dd in range(numiter): + msgs.info(f"Iterating on spatial translation - ITERATION #{dd+1}/{numiter}") + # Setup the WCS to use for all white light images + ref_idx = None # Don't use an index - This is the default behaviour when a reference image is supplied + image_wcs, voxedge, reference_image = create_wcs(cubepar, all_ra[ww], all_dec[ww], all_wave[ww], + dspat, wavediff, collapse=True) + if voxedge[2].size != 2: + msgs.error("Spectral range for WCS is incorrect for white light image") + + wl_imgs = generate_image_subpixel(image_wcs, all_ra[ww], all_dec[ww], all_wave[ww], + all_sci[ww], all_ivar[ww], all_wghts[ww], + all_spatpos[ww], all_specpos[ww], all_spatid[ww], + all_tilts, all_slits, all_align, voxedge, all_idx=all_idx[ww], + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) + if reference_image is None: + # ref_idx will be the index of the cube with the highest S/N + ref_idx = np.argmax(weights) + reference_image = wl_imgs[:, :, ref_idx].copy() + msgs.info("Calculating spatial translation of each cube relative to cube #{0:d})".format(ref_idx+1)) + else: + msgs.info("Calculating the spatial translation of each cube relative to user-defined 'reference_image'") + + # Calculate the image offsets relative to the reference image + for ff in range(numfiles): + # Calculate the shift + ra_shift, dec_shift = calculate_image_phase(reference_image.copy(), wl_imgs[:, :, ff], maskval=0.0) + # Convert pixel shift to degrees shift + ra_shift *= dspat/cosdec + dec_shift *= dspat + msgs.info("Spatial shift of cube #{0:d}: RA, DEC (arcsec) = {1:+0.3f} E, {2:+0.3f} N".format(ff+1, ra_shift*3600.0, dec_shift*3600.0)) + # Apply the shift + all_ra[all_idx == ff] += ra_shift + all_dec[all_idx == ff] += dec_shift # Calculate the relative spectral weights of all pixels - all_wghts = compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, - whitelight_img[:, :, 0], dspat, dwv, - relative_weights=cubepar['relative_weights']) - - # Check if a whitelight image should be saved - if cubepar['save_whitelight']: - # Check if the white light image still needs to be generated - if so, generate it now - if whitelight_img is None: - msgs.info("Generating global white light image") - if cubepar["reference_image"] is None: - whitelight_img, _, wlwcs = make_whitelight_frompixels(all_ra, all_dec, all_wave, all_sci, all_wghts, - np.zeros(all_ra.size), dspat) - else: - _, whitelight_img, wlwcs = \ - make_whitelight_fromref(all_ra, all_dec, all_wave, all_sci, all_wghts, - np.zeros(all_ra.size), - dspat, cubepar['reference_image']) - # Prepare and save the fits file - msgs.info("Saving white light image as: {0:s}".format(out_whitelight)) - img_hdu = fits.PrimaryHDU(whitelight_img.T, header=wlwcs.to_header()) - img_hdu.writeto(out_whitelight, overwrite=overwrite) - - # Setup the cube ranges - ra_min = cubepar['ra_min'] if cubepar['ra_min'] is not None else np.min(all_ra) - ra_max = cubepar['ra_max'] if cubepar['ra_max'] is not None else np.max(all_ra) - dec_min = cubepar['dec_min'] if cubepar['dec_min'] is not None else np.min(all_dec) - dec_max = cubepar['dec_max'] if cubepar['dec_max'] is not None else np.max(all_dec) - wav_min = cubepar['wave_min'] if cubepar['wave_min'] is not None else np.min(all_wave) - wav_max = cubepar['wave_max'] if cubepar['wave_max'] is not None else np.max(all_wave) - if cubepar['wave_delta'] is not None: dwv = cubepar['wave_delta'] - - # Generate a WCS to register all frames - coord_min = [ra_min, dec_min, wav_min] - coord_dlt = [dspat, dspat, dwv] - coord_wcs = generate_WCS(coord_min, coord_dlt, name=specname) - msgs.info(msgs.newline() + "-"*40 + - msgs.newline() + "Parameters of the WCS:" + - msgs.newline() + "RA min, max = {0:f}, {1:f}".format(ra_min, ra_max) + - msgs.newline() + "DEC min, max = {0:f}, {1:f}".format(dec_min, dec_max) + - msgs.newline() + "WAVE min, max = {0:f}, {1:f}".format(wav_min, wav_max) + - msgs.newline() + "Spaxel size = {0:f} arcsec".format(3600.0*dspat) + - msgs.newline() + "Wavelength step = {0:f} A".format(dwv) + - msgs.newline() + "-" * 40) - - # Generate the output binning - numra = int((ra_max-ra_min) * cosdec / dspat) - numdec = int((dec_max-dec_min)/dspat) - numwav = int((wav_max-wav_min)/dwv) - xbins = np.arange(1+numra)-0.5 - ybins = np.arange(1+numdec)-0.5 - spec_bins = np.arange(1+numwav)-0.5 - bins = (xbins, ybins, spec_bins) - - # Make the cube - msgs.info("Generating pixel coordinates") - pix_coord = coord_wcs.wcs_world2pix(all_ra, all_dec, all_wave * 1.0E-10, 0) - hdr = coord_wcs.to_header() + if numfiles == 1: + # No need to calculate weights if there's just one frame + all_wghts = np.ones_like(all_sci) + else: + # Find the wavelength range where all frames overlap + min_wl, max_wl = get_whitelight_range(np.max(mnmx_wv[:, :, 0]), # The max blue wavelength + np.min(mnmx_wv[:, :, 1]), # The min red wavelength + cubepar['whitelight_range']) # The user-specified values (if any) + # Get the good white light pixels + ww, wavediff = get_whitelight_pixels(all_wave, min_wl, max_wl) + # Get a suitable WCS + image_wcs, voxedge, reference_image = create_wcs(cubepar, all_ra, all_dec, all_wave, dspat, wavediff, collapse=True) + # Generate the white light image (note: hard-coding subpixel=1 in both directions, and combining into a single image) + wl_full = generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, + all_sci, all_ivar, all_wghts, + all_spatpos, all_specpos, all_spatid, + all_tilts, all_slits, all_align, voxedge, all_idx=all_idx, + spec_subpixel=1, spat_subpixel=1, combine=True) + # Compute the weights + all_wghts = compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, wl_full[:, :, 0], + dspat, dwv, relative_weights=cubepar['relative_weights']) + + # Generate the WCS, and the voxel edges + cube_wcs, vox_edges, _ = create_wcs(cubepar, all_ra, all_dec, all_wave, dspat, dwv) sensfunc = None if flux_spline is not None: - wcs_wav = coord_wcs.wcs_pix2world(np.vstack((np.zeros(numwav), np.zeros(numwav), np.arange(numwav))).T, 0) - senswave = wcs_wav[:, 2] * 1.0E10 + # Get wavelength of each pixel, and note that the WCS gives this in m, so convert to Angstroms (x 1E10) + numwav = vox_edges[2].size-1 + senswave = cube_wcs.spectral.wcs_pix2world(np.arange(numwav), 0)[0] * 1.0E10 sensfunc = flux_spline(senswave) - # Generate a datacube using nearest grid point (NGP) - msgs.info("Generating data cube") - generate_cube_ngp(outfile, hdr, all_sci, all_ivar, all_wghts, pix_coord, bins, overwrite=overwrite, - blaze_wave=blaze_wave, blaze_spec=blaze_spec, sensfunc=sensfunc, fluxcal=fluxcal, - specname=specname) + # Generate a datacube + outfile = get_output_filename("", cubepar['output_filename'], True, -1) + if method in ['subpixel', 'ngp']: + # Generate the datacube + wl_wvrng = None + if cubepar['save_whitelight']: + wl_wvrng = get_whitelight_range(np.max(mnmx_wv[:, :, 0]), + np.min(mnmx_wv[:, :, 1]), + cubepar['whitelight_range']) + if combine: + generate_cube_subpixel(outfile, cube_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, + np.ones(all_wghts.size), # all_wghts, + all_spatpos, all_specpos, all_spatid, all_tilts, all_slits, all_align, vox_edges, + all_idx=all_idx, overwrite=overwrite, blaze_wave=blaze_wave, blaze_spec=blaze_spec, + fluxcal=fluxcal, sensfunc=sensfunc, specname=specname, whitelight_range=wl_wvrng, + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) + else: + for ff in range(numfiles): + outfile = get_output_filename("", cubepar['output_filename'], False, ff) + ww = np.where(all_idx == ff) + generate_cube_subpixel(outfile, cube_wcs, all_ra[ww], all_dec[ww], all_wave[ww], all_sci[ww], all_ivar[ww], np.ones(all_wghts[ww].size), + all_spatpos[ww], all_specpos[ww], all_spatid[ww], all_tilts[ff], all_slits[ff], all_align[ff], vox_edges, + all_idx=all_idx[ww], overwrite=overwrite, blaze_wave=blaze_wave, blaze_spec=blaze_spec, + fluxcal=fluxcal, sensfunc=sensfunc, specname=specname, whitelight_range=wl_wvrng, + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) diff --git a/pypeit/core/extract.py b/pypeit/core/extract.py index d7f3022e68..0e26639340 100644 --- a/pypeit/core/extract.py +++ b/pypeit/core/extract.py @@ -24,13 +24,14 @@ def extract_optimal(sciimg, ivar, mask, waveimg, skyimg, thismask, oprof, - spec, min_frac_use=0.05, base_var=None, count_scale=None, noise_floor=None): + spec, min_frac_use=0.05, fwhmimg=None, base_var=None, count_scale=None, noise_floor=None): r""" - Perform optimal extraction `(Horne 1986) `_ - for a single :class:`~pypeit.specobjs.SpecObj`. + Perform optimal extraction `(Horne 1986) + `__ for a + single :class:`~pypeit.specobj.SpecObj`. - The :class:`~pypeit.specobjs.SpecObj` object is changed in place with optimal attributes + The :class:`~pypeit.specobj.SpecObj` object is changed in place with optimal attributes being filled with the extraction parameters, and additional sky and noise estimates being added. The following are the attributes that are filled here: @@ -40,6 +41,7 @@ def extract_optimal(sciimg, ivar, mask, waveimg, skyimg, thismask, oprof, - spec.OPT_COUNTS_SIG --> Optimally extracted noise from IVAR - spec.OPT_COUNTS_NIVAR --> Optimally extracted noise variance (sky + read noise) only - spec.OPT_MASK --> Mask for optimally extracted flux + - spec.OPT_FWHM --> Spectral FWHM (in A) for optimally extracted flux - spec.OPT_COUNTS_SKY --> Optimally extracted sky - spec.OPT_COUNTS_SIG_DET --> Square root of optimally extracted read noise squared - spec.OPT_FRAC_USE --> Fraction of pixels in the object profile subimage used for this extraction @@ -82,6 +84,9 @@ def extract_optimal(sciimg, ivar, mask, waveimg, skyimg, thismask, oprof, For each spectral pixel, if the majority of the object profile has been masked, i.e., the sum of the normalized object profile across the spatial direction is less than `min_frac_use`, the optimal extraction will also be masked. The default value is 0.05. + fwhmimg : `numpy.ndarray`_, None, optional: + Floating-point image containing the modeled spectral FWHM (in pixels) at every pixel location. + Must have the same shape as ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. base_var : `numpy.ndarray`_, optional Floating-point "base-level" variance image set by the detector properties and the image processing steps. See :func:`~pypeit.core.procimg.base_variance`. @@ -137,6 +142,8 @@ def extract_optimal(sciimg, ivar, mask, waveimg, skyimg, thismask, oprof, img_sub = imgminsky[:,mincol:maxcol] sky_sub = skyimg[:,mincol:maxcol] oprof_sub = oprof[:,mincol:maxcol] + if fwhmimg is not None: + fwhmimg_sub = fwhmimg[:,mincol:maxcol] # enforce normalization and positivity of object profiles norm = np.nansum(oprof_sub,axis = 1) norm_oprof = np.outer(norm, np.ones(nsub)) @@ -179,7 +186,9 @@ def extract_optimal(sciimg, ivar, mask, waveimg, skyimg, thismask, oprof, wave_opt = np.nansum(mask_sub*ivar_sub*wave_sub*oprof_sub**2, axis=1)/(mivar_num + (mivar_num == 0.0)) mask_opt = (tot_weight > 0.0) & (frac_use > min_frac_use) & (mivar_num > 0.0) & (ivar_denom > 0.0) & \ np.isfinite(wave_opt) & (wave_opt > 0.0) - + fwhm_opt = None + if fwhmimg is not None: + fwhm_opt = np.nansum(mask_sub*ivar_sub*fwhmimg_sub*oprof_sub, axis=1) * utils.inverse(tot_weight) # Interpolate wavelengths over masked pixels badwvs = (mivar_num <= 0) | np.invert(np.isfinite(wave_opt)) | (wave_opt <= 0.0) if badwvs.any(): @@ -208,6 +217,9 @@ def extract_optimal(sciimg, ivar, mask, waveimg, skyimg, thismask, oprof, chi2_denom = np.fmax(np.nansum(ivar_sub*mask_sub > 0.0, axis=1) - 1.0, 1.0) chi2 = chi2_num/chi2_denom + # Calculate the Angstroms/pixel and Spectral FWHM + if fwhm_opt is not None: + fwhm_opt *= np.gradient(wave_opt) # Convert pixel FWHM to Angstroms # Fill in the optimally extraction tags spec.OPT_WAVE = wave_opt # Optimally extracted wavelengths spec.OPT_COUNTS = flux_opt # Optimally extracted flux @@ -215,6 +227,7 @@ def extract_optimal(sciimg, ivar, mask, waveimg, skyimg, thismask, oprof, spec.OPT_COUNTS_SIG = np.sqrt(utils.inverse(spec.OPT_COUNTS_IVAR)) spec.OPT_COUNTS_NIVAR = None if nivar_opt is None else nivar_opt*np.logical_not(badwvs) # Optimally extracted noise variance (sky + read noise) only spec.OPT_MASK = mask_opt*np.logical_not(badwvs) # Mask for optimally extracted flux + spec.OPT_FWHM = fwhm_opt # Spectral FWHM (in Angstroms) for the optimally extracted spectrum spec.OPT_COUNTS_SKY = sky_opt # Optimally extracted sky spec.OPT_COUNTS_SIG_DET = base_opt # Square root of optimally extracted read noise squared spec.OPT_FRAC_USE = frac_use # Fraction of pixels in the object profile subimage used for this extraction @@ -293,14 +306,14 @@ def extract_asym_boxcar(sciimg, left_trace, righ_trace, gpm=None, ivar=None): return flux_out, gpm_box, box_npix, ivar_out -def extract_boxcar(sciimg, ivar, mask, waveimg, skyimg, spec, base_var=None, +def extract_boxcar(sciimg, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, base_var=None, count_scale=None, noise_floor=None): r""" - Perform boxcar extraction for a single :class:`~pypeit.specobjs.SpecObj`. + Perform boxcar extraction for a single :class:`~pypeit.specobj.SpecObj`. The size of the boxcar must be available as an attribute of the :class:`~pypeit.specobj.SpecObj` object. - The :class:`~pypeit.specobjs.SpecObj` object is changed in place with boxcar attributes + The :class:`~pypeit.specobj.SpecObj` object is changed in place with boxcar attributes being filled with the extraction parameters, and additional sky and noise estimates being added. The following are the attributes that are filled here: @@ -310,6 +323,7 @@ def extract_boxcar(sciimg, ivar, mask, waveimg, skyimg, spec, base_var=None, - spec.BOX_COUNTS_SIG --> Box car extracted error - spec.BOX_COUNTS_NIVAR --> Box car extracted noise variance - spec.BOX_MASK --> Box car extracted mask + - spec.BOX_FWHM --> Box car extracted spectral FWHM - spec.BOX_COUNTS_SKY --> Box car extracted sky - spec.BOX_COUNTS_SIG_DET --> Box car extracted read noise - spec.BOX_NPIX --> Number of pixels used in boxcar sum @@ -338,6 +352,9 @@ def extract_boxcar(sciimg, ivar, mask, waveimg, skyimg, spec, base_var=None, Container that holds object, trace, and extraction information for the object in question. **This object is altered in place!** Note that this routine operates one object at a time. + fwhmimg : `numpy.ndarray`_, None, optional: + Floating-point image containing the modeled spectral FWHM (in pixels) at every pixel location. + Must have the same shape as ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. base_var : `numpy.ndarray`_, optional Floating-point "base-level" variance image set by the detector properties and the image processing steps. See :func:`~pypeit.core.procimg.base_variance`. @@ -384,6 +401,9 @@ def extract_boxcar(sciimg, ivar, mask, waveimg, skyimg, spec, base_var=None, row=spec.trace_spec)[0] wave_box = moment1d(waveimg*mask, spec.TRACE_SPAT, 2*box_radius, row=spec.trace_spec)[0] / (box_denom + (box_denom == 0.0)) + fwhm_box = None + if fwhmimg is not None: + fwhm_box = moment1d(fwhmimg*mask, spec.TRACE_SPAT, 2*box_radius, row=spec.trace_spec)[0] varimg = 1.0/(ivar + (ivar == 0.0)) var_box = moment1d(varimg*mask, spec.TRACE_SPAT, 2*box_radius, row=spec.trace_spec)[0] nvar_box = None if var_no is None \ @@ -412,6 +432,10 @@ def extract_boxcar(sciimg, ivar, mask, waveimg, skyimg, spec, base_var=None, ivar_box = 1.0/(var_box + (var_box == 0.0)) nivar_box = None if nvar_box is None else 1.0/(nvar_box + (nvar_box == 0.0)) + # Calculate the Angstroms/pixel and the final spectral FWHM value + if fwhm_box is not None: + ang_per_pix = np.gradient(wave_box) + fwhm_box *= ang_per_pix / (pixtot - pixmsk) # Need to divide by total number of unmasked pixels # Fill em up! spec.BOX_WAVE = wave_box spec.BOX_COUNTS = flux_box*mask_box @@ -419,6 +443,7 @@ def extract_boxcar(sciimg, ivar, mask, waveimg, skyimg, spec, base_var=None, spec.BOX_COUNTS_SIG = np.sqrt(utils.inverse( spec.BOX_COUNTS_IVAR)) spec.BOX_COUNTS_NIVAR = None if nivar_box is None else nivar_box*mask_box*np.logical_not(bad_box) spec.BOX_MASK = mask_box*np.logical_not(bad_box) + spec.BOX_FWHM = fwhm_box # Spectral FWHM (in Angstroms) for the boxcar extracted spectrum spec.BOX_COUNTS_SKY = sky_box spec.BOX_COUNTS_SIG_DET = base_box # TODO - Confirm this should be float, not int diff --git a/pypeit/core/findobj_skymask.py b/pypeit/core/findobj_skymask.py index 00b9d3ca78..ff158ba251 100644 --- a/pypeit/core/findobj_skymask.py +++ b/pypeit/core/findobj_skymask.py @@ -150,7 +150,7 @@ def create_skymask(sobjs, thismask, slit_left, slit_righ, box_rad_pix=None, trim def ech_findobj_ineach_order( image, ivar, slitmask, slit_left, slit_righ, slit_spats, - order_vec, orders_gpm, spec_min_max, plate_scale_ord, + order_vec, spec_min_max, plate_scale_ord, det='DET01', inmask=None, std_trace=None, ncoeff=5, hand_extract_dict=None, box_radius=2.0, fwhm=3.0, @@ -204,9 +204,6 @@ def ech_findobj_ineach_order( found. This is saved to the output :class:`~pypeit.specobj.SpecObj` objects. If the orders are not known, this can be ``np.arange(norders)`` (but this is *not* recommended). - order_gpm (`numpy.ndarray`_): - Boolean array indicating which orders are good (True), - i.e. have good calibrations (wavelengths, etc.). Shape is spec_min_max (`numpy.ndarray`_): 2D array defining the minimum and maximum pixel in the spectral direction with useable data for each order. Shape must be (2, @@ -296,9 +293,6 @@ def ech_findobj_ineach_order( # Loop over orders and find objects sobjs = specobjs.SpecObjs() for iord, iorder in enumerate(order_vec): - if not orders_gpm[iord]: - continue - # qa_title = 'Finding objects on order # {:d}'.format(iorder) msgs.info(qa_title) thisslit_gpm = slitmask == slit_spats[iord] @@ -334,8 +328,9 @@ def ech_findobj_ineach_order( def ech_fof_sobjs(sobjs:specobjs.SpecObjs, slit_left:np.ndarray, - slit_righ:np.ndarray, - plate_scale_ord:np.ndarray, + slit_righ:np.ndarray, + order_vec:np.ndarray, + plate_scale_ord:np.ndarray, fof_link:float=1.5): """ Links together objects previously found using a @@ -355,6 +350,9 @@ def ech_fof_sobjs(sobjs:specobjs.SpecObjs, Right boundary of orders to be extracted (given as floating point pixels). Shape is (nspec, norders), where norders is the total number of traced echelle orders. + order_vec (`numpy.ndarray`_): + Vector identifying the Echelle orders for each pair of order edges + found. plate_scale_ord (`numpy.ndarray`_): An array with shape (norders,) providing the plate scale of each order in arcsec/pix, @@ -399,12 +397,11 @@ def ech_fof_sobjs(sobjs:specobjs.SpecObjs, nobj_init = len(uni_obj_id_init) for iobj in range(nobj_init): for iord in range(norders): - on_order = (obj_id_init == uni_obj_id_init[iobj]) & ( - sobjs.ECH_ORDERINDX == iord) + on_order = (obj_id_init == uni_obj_id_init[iobj]) & (sobjs.ECH_ORDER == order_vec[iord]) if (np.sum(on_order) > 1): - msgs.warn('Found multiple objects in a FOF group on order iord={:d}'.format(iord) + msgs.newline() + + msgs.warn('Found multiple objects in a FOF group on order iord={:d}'.format(order_vec[iord]) + msgs.newline() + 'Spawning new objects to maintain a single object per order.') - off_order = (obj_id_init == uni_obj_id_init[iobj]) & (sobjs.ECH_ORDERINDX != iord) + off_order = (obj_id_init == uni_obj_id_init[iobj]) & (sobjs.ECH_ORDER != order_vec[iord]) ind = np.where(on_order)[0] if np.any(off_order): # Keep the closest object to the location of the rest of the group (on other orders) @@ -430,11 +427,10 @@ def ech_fof_sobjs(sobjs:specobjs.SpecObjs, def ech_fill_in_orders(sobjs:specobjs.SpecObjs, slit_left:np.ndarray, - slit_righ:np.ndarray, + slit_righ:np.ndarray, + slit_spat_id: np.ndarray, order_vec:np.ndarray, - order_gpm:np.ndarray, obj_id:np.ndarray, - slit_spat_id:np.ndarray, std_trace:specobjs.SpecObjs=None, show:bool=False): """ @@ -465,20 +461,17 @@ def ech_fill_in_orders(sobjs:specobjs.SpecObjs, Right boundary of orders to be extracted (given as floating point pixels). Shape is (nspec, norders), where norders is the total number of traced echelle orders. + slit_spat_id (`numpy.ndarray`_): + slit_spat values (spatial position 1/2 way up the detector) + for the orders order_vec (`numpy.ndarray`_): Vector identifying the Echelle orders for each pair of order edges found. This is saved to the output :class:`~pypeit.specobj.SpecObj` objects. If the orders are not known, this can be ``np.arange(norders)`` (but this is *not* recommended). - order_gpm (`numpy.ndarray`_): - Boolean array indicating which orders are good (True), - i.e. have good calibrations (wavelengths, etc.). Shape is obj_id (`numpy.ndarray`_): Object IDs of the objects linked together. - slit_spat_id (`numpy.ndarray`_): - slit_spat values (spatial position 1/2 way up the detector) - for the orders - std_trace (:class:`~pypeit.specobjs.SpecObjs`, optional): + std_trace (:class:`~pypeit.specobjs.SpecObjs`, optional): Standard star objects (including the traces) Defaults to None. show (bool, optional): @@ -496,12 +489,11 @@ def ech_fill_in_orders(sobjs:specobjs.SpecObjs, fracpos = sobjs.SPAT_FRACPOS # Prep - ngd_orders = np.sum(order_gpm) - gd_orders = order_vec[order_gpm] + norders = order_vec.size slit_width = slit_righ - slit_left # Check standard star - if std_trace is not None and std_trace.shape[1] != ngd_orders: + if std_trace is not None and std_trace.shape[1] != norders: msgs.error('Standard star trace does not match the number of orders in the echelle data.') # For traces @@ -526,7 +518,7 @@ def ech_fill_in_orders(sobjs:specobjs.SpecObjs, # Loop over the orders and assign each specobj a fractional position and a obj_id number for iobj in range(nobj): #for iord in range(norders): - for iord in order_vec[order_gpm]: + for iord in order_vec: on_order = (obj_id == uni_obj_id[iobj]) & (sobjs_align.ECH_ORDER == iord) sobjs_align[on_order].ECH_FRACPOS = uni_frac[iobj] sobjs_align[on_order].ECH_OBJID = uni_obj_id[iobj] @@ -543,9 +535,12 @@ def ech_fill_in_orders(sobjs:specobjs.SpecObjs, indx_obj_id = sobjs_align.ECH_OBJID == uni_obj_id[iobj] nthisobj_id = np.sum(indx_obj_id) # Perform the fit if this objects shows up on more than three orders - if (nthisobj_id > 3) and (nthisobj_id 3) and (nthisobj_id= 0]).astype(int) # Unique sorts + #if gdslit_spat.size != norders: + #msgs.error('Number of slitidsin slitmask and the number of left/right slits must be the same.') + + if slit_righ.shape[1] != norders: + msgs.error('Number of left and right slits must be the same.') + if order_vec.size != norders: + msgs.error('Number of orders in order_vec and left/right slits must be the same.') + if spec_min_max.shape[1] != norders: + msgs.error('Number of orders in spec_min_max and left/right slits must be the same.') + if specobj_dict is None: specobj_dict = {'SLITID': 999, 'ECH_ORDERINDX': 999, 'DET': det, 'OBJTYPE': 'unknown', 'PYPELINE': 'Echelle'} + # Loop over the orders and find the objects within them (if any) - order_gpm = np.invert(slits_bpm) sobjs_in_orders = ech_findobj_ineach_order( - image, ivar, slitmask, slit_left, + image, ivar, slitmask, slit_left, slit_righ, slit_spat_id, - order_vec, order_gpm, + order_vec, spec_min_max, plate_scale, det=det, inmask=inmask, @@ -1256,31 +1267,24 @@ def ech_objfind(image, ivar, slitmask, slit_left, slit_righ, order_vec, slits_bp objfindQA_filename=objfindQA_filename, hand_extract_dict=manual_extract_dict) + # No sources and no manual? if len(sobjs_in_orders) == 0: return sobjs_in_orders - # Additional work for slits with sources (found or input manually) + # Perform some additional work for slits with sources (found or input manually) # Friend of friend algorithm to group objects - obj_id = ech_fof_sobjs( - sobjs_in_orders, slit_left, - slit_righ, plate_scale, - fof_link=fof_link) + obj_id = ech_fof_sobjs(sobjs_in_orders, slit_left, slit_righ, order_vec, plate_scale, fof_link=fof_link) # Fill in Orders sobjs_filled = ech_fill_in_orders( - sobjs_in_orders, - slit_left, slit_righ, - order_vec, order_gpm, - obj_id, #obj_id[tmp], - slit_spat_id, - std_trace=std_trace) + sobjs_in_orders, slit_left, slit_righ, slit_spat_id, order_vec, obj_id, std_trace=std_trace) # Cut on SNR and number of objects sobjs_pre_final = ech_cutobj_on_snr( sobjs_filled, image, ivar, slitmask, - order_vec[order_gpm], + order_vec, plate_scale, inmask=inmask, nperorder=nperorder, @@ -1289,12 +1293,15 @@ def ech_objfind(image, ivar, slitmask, slit_left, slit_righ, order_vec, slits_bp nabove_min_snr=nabove_min_snr, box_radius=box_radius) + if len(sobjs_pre_final) == 0: + return sobjs_pre_final + # PCA sobjs_ech = ech_pca_traces( sobjs_pre_final, image, slitmask, inmask, - order_vec[order_gpm], - spec_min_max[:, order_gpm], + order_vec, + spec_min_max, coeff_npoly=coeff_npoly, ncoeff=ncoeff, npca=npca, pca_explained_var=pca_explained_var, diff --git a/pypeit/core/fitting.py b/pypeit/core/fitting.py index cf56e6e481..7f4346ebe4 100644 --- a/pypeit/core/fitting.py +++ b/pypeit/core/fitting.py @@ -271,6 +271,8 @@ def evaluate_fit(fitc, func, x, x2=None, minx=None, `numpy.ndarray`_: Evaluated fit at the x (and x2) locations """ + if func is None: + return None # For two-d fits x = x, y = x2, y = z if ('2d' in func) and (x2 is not None): # Is this a 2d fit? @@ -285,6 +287,9 @@ def evaluate_fit(fitc, func, x, x2=None, minx=None, else np.polynomial.chebyshev.chebval2d(xv, x2v, fitc)) else: msgs.error("Function {0:s} has not yet been implemented for 2d fits".format(func)) + # TODO: Why is this return here? The code will never reach this point + # because of the if/elif/else above. What should the behavior be, raise + # an exception or return None? return None elif func == "polynomial": return np.polynomial.polynomial.polyval(x, fitc) @@ -471,7 +476,7 @@ def robust_optimize(ydata, fitfunc, arg_dict, maxiter=10, inmask=None, invvar=No one to fit a more general model using the optimizer of the users choice. If you are fitting simple functions like Chebyshev or Legednre polynomials using a linear least-squares algorithm, you - should use :func:robust_polyfit_djs` instead of this function. + should use :func:`robust_fit` instead of this function. Args: ydata (`numpy.ndarray`_): @@ -580,7 +585,7 @@ def robust_optimize(ydata, fitfunc, arg_dict, maxiter=10, inmask=None, invvar=No inmask = np.ones(ydata.size, dtype=bool) nin_good = np.sum(inmask) - iter = 0 + iIter = 0 qdone = False thismask = np.copy(inmask) @@ -588,7 +593,7 @@ def robust_optimize(ydata, fitfunc, arg_dict, maxiter=10, inmask=None, invvar=No # results in signficant speedup for e.g. differential_evolution optimization. Thus # init_from_last is None on the first iteration and then is updated in the iteration loop. init_from_last = None - while (not qdone) and (iter < maxiter): + while (not qdone) and (iIter < maxiter): ret_tuple = fitfunc(ydata, thismask, arg_dict, init_from_last=init_from_last, **kwargs_optimizer) if (len(ret_tuple) == 2): result, ymodel = ret_tuple @@ -610,9 +615,9 @@ def robust_optimize(ydata, fitfunc, arg_dict, maxiter=10, inmask=None, invvar=No msgs.info( 'Iteration #{:d}: nrej={:d} new rejections, nrej_tot={:d} total rejections out of ntot={:d} ' 'total pixels'.format(iter, nrej, nrej_tot, nin_good)) - iter += 1 + iIter += 1 - if (iter == maxiter) & (maxiter != 0): + if (iIter == maxiter) & (maxiter != 0): msgs.warn('Maximum number of iterations maxiter={:}'.format(maxiter) + ' reached in robust_optimize') outmask = np.copy(thismask) if np.sum(outmask) == 0: @@ -833,7 +838,8 @@ def polyfit2d_general(x, y, z, deg, w=None, function='polynomial', def twoD_Gaussian(tup, amplitude, xo, yo, sigma_x, sigma_y, theta, offset): - """ A 2D Gaussian to be used to fit the cross-correlation + """ + A 2D Gaussian to be used to fit the cross-correlation Args: tup (tuple): @@ -842,7 +848,7 @@ def twoD_Gaussian(tup, amplitude, xo, yo, sigma_x, sigma_y, theta, offset): The amplitude of the 2D Gaussian xo (float): The centre of the Gaussian in the x direction - yo (float: + yo (float): The centre of the Gaussian in the y direction sigma_x (float): The dispersion of the Gaussian in the x direction @@ -1015,7 +1021,7 @@ def iterfit(xdata, ydata, invvar=None, inmask=None, upper=5, lower=5, x2=None, elif error == 0: # ToDO JFH by setting inmask to be tempin which is maskwork, we are basically implicitly enforcing sticky rejection # here. See djs_reject.py. I'm leaving this as is for consistency with the IDL version, but this may require - # further consideration. I think requiring stick to be set is the more transparent behavior. + # further consideration. I think requiring sticky to be set is the more transparent behavior. maskwork, qdone = pydl.djs_reject(ywork, yfit, invvar=invwork, inmask=inmask_rej, outmask=maskwork, upper=upper, lower=lower, **kwargs_reject) else: @@ -1134,7 +1140,7 @@ def bspline_profile(xdata, ydata, invvar, profile_basis, ingpm=None, upper=5, lo yfit = np.zeros(ydata.shape) reduced_chi = 0. - # TODO: Instanting these place-holder arrays can be expensive. Can we avoid doing this? + # TODO: Instantiating these place-holder arrays can be expensive. Can we avoid doing this? outmask = True if invvar.size == 1 else np.ones(invvar.shape, dtype=bool) if ingpm is None: diff --git a/pypeit/core/flat.py b/pypeit/core/flat.py index 908dd810ee..cb3ba6b1a2 100644 --- a/pypeit/core/flat.py +++ b/pypeit/core/flat.py @@ -151,7 +151,7 @@ def construct_illum_profile(norm_spec, spat_coo, slitwidth, spat_gpm=None, spat_ good-pixel mask should be for a single slit. The iterations involve constructing the illumination profile - using :func:`illum_profile` and then rejecting deviant residuals. + using :func:`illum_filter` and then rejecting deviant residuals. Each rejection iteration recomputes the standard deviation and pixels to reject from the full input set (i.e., rejected pixels are not kept between iterations). Rejection iterations are only @@ -283,6 +283,77 @@ def construct_illum_profile(norm_spec, spat_coo, slitwidth, spat_gpm=None, spat_ return _spat_gpm, spat_srt, spat_coo_data, spat_flat_data_raw, \ illum_filter(spat_flat_data_raw, med_width) + +def illum_profile_spectral_poly(rawimg, waveimg, slitmask, slitmask_trim, model, slit_illum_ref_idx=0, + gpmask=None, thismask=None, nbins=20): + """ + Use a polynomial fit to control points along the spectral direction to determine the relative + spectral illumination of all slits. Currently, this routine is only used for image slicer IFUs. + + Parameters + ---------- + rawimg : `numpy.ndarray`_ + Image data that will be used to estimate the spectral relative sensitivity + waveimg : `numpy.ndarray`_ + Wavelength image + slitmask : `numpy.ndarray`_ + A 2D int mask, the same shape as rawimg, indicating which pixels are on a slit. A zero value + indicates not on a slit, while any pixels on a slit should have the value of the slit spatial ID + number. + slitmask_trim : + Same as slitmask, but the slit edges are trimmed. + model : `numpy.ndarray`_ + A model of the rawimg data. + slit_illum_ref_idx : int + Index of slit that is used as the reference. + gpmask : `numpy.ndarray`_, optional + Boolean good pixel mask (True = Good) + thismask : `numpy.ndarray`_, optional + A boolean mask (True = good) that indicates all pixels where the scaleImg should be constructed. + If None, the slitmask that is generated by this routine will be used. + nbins : int + Number of bins in the spectral direction to sample the relative spectral sensitivity + + Returns + ------- + scale_model: `numpy.ndarray`_ + An image containing the appropriate scaling + """ + msgs.info(f"Performing relative spectral sensitivity correction (reference slit = {slit_illum_ref_idx})") + # Generate the mask + _thismask = thismask if (thismask is not None) else slitmask + gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool) + # Extract the list of spatial IDs from the slitmask + slitmask_spatid = np.unique(slitmask) + slitmask_spatid = np.sort(slitmask_spatid[slitmask_spatid != 0]) + # Initialise the scale image that will be returned + scaleImg = np.ones_like(rawimg) + # Divide the slit into several bins and calculate the median of each bin + for sl, spatid in enumerate(slitmask_spatid): + # Prepare the masks, edges, and fitting variables + this_slit = (slitmask == spatid) + this_slit_trim = (slitmask_trim == spatid) + this_slit_mask = gpm & this_slit_trim + this_wave = waveimg[this_slit_mask] + wavedg = np.linspace(np.min(this_wave), np.max(this_wave), nbins + 1) + wavcen = 0.5 * (wavedg[1:] + wavedg[:-1]) + scale_all = rawimg[this_slit_mask] * utils.inverse(model[this_slit_mask]) + scale_bin = np.zeros(nbins) + scale_err = np.zeros(nbins) + for bb in range(nbins): + cond = (this_wave >= wavedg[bb]) & (this_wave <= wavedg[bb + 1]) + scale_bin[bb] = np.median(scale_all[cond]) + scale_err[bb] = 1.4826 * np.median(np.abs(scale_all[cond] - scale_bin[bb])) + wgd = np.where(scale_err > 0) + coeff = np.polyfit(wavcen[wgd], scale_bin[wgd], w=1/scale_err[wgd], deg=2) + scaleImg[this_slit] *= np.polyval(coeff, waveimg[this_slit]) + if sl == slit_illum_ref_idx: + scaleImg[_thismask] /= np.polyval(coeff, waveimg[_thismask]) + minv, maxv = np.min(scaleImg[_thismask]), np.max(scaleImg[_thismask]) + msgs.info("Minimum/Maximum scales = {0:.5f}, {1:.5f}".format(minv, maxv)) + return scaleImg + + # TODO: See pypeit/deprecated/flat.py for the previous version. We need # to continue to vet this algorithm to make sure there are no # unforeseen corner cases that cause errors. diff --git a/pypeit/core/flexure.py b/pypeit/core/flexure.py index 1e1cba161f..744279ddbf 100644 --- a/pypeit/core/flexure.py +++ b/pypeit/core/flexure.py @@ -121,9 +121,9 @@ def spec_flex_shift(obj_skyspec, arx_skyspec, arx_fwhm_pix, spec_fwhm_pix=None, """ Calculate shift between object sky spectrum and archive sky spectrum Args: - obj_skyspec (:class:`linetools.spectra.xspectrum1d.XSpectrum1d`): + obj_skyspec (`linetools.spectra.xspectrum1d.XSpectrum1d`_): Spectrum of the sky related to our object - arx_skyspec (:class:`linetools.spectra.xspectrum1d.XSpectrum1d`): + arx_skyspec (`linetools.spectra.xspectrum1d.XSpectrum1d`_): Archived sky spectrum arx_fwhm_pix (:obj:`float`): Spectral FWHM (in pixels) of the archived sky spectrum. @@ -320,9 +320,9 @@ def get_fwhm_gauss_smooth(arx_skyspec, obj_skyspec, arx_fwhm_pix, spec_fwhm_pix= """ Args: - arx_skyspec (:class:`linetools.spectra.xspectrum1d.XSpectrum1d`): + arx_skyspec (`linetools.spectra.xspectrum1d.XSpectrum1d`_): Archived sky spectrum. - obj_skyspec (:class:`linetools.spectra.xspectrum1d.XSpectrum1d`): + obj_skyspec (`linetools.spectra.xspectrum1d.XSpectrum1d`_): Sky spectrum associated with the science target. arx_fwhm_pix (:obj:`float`): Spectral FWHM (in pixels) of the archived sky spectrum. @@ -408,7 +408,7 @@ def spec_flex_shift_global(slit_specs, islit, sky_spectrum, arx_fwhm_pix, empty_ this list are sky spectra, extracted from the center of each slit. islit (:obj:`int`): Index of the slit where the sky spectrum related to our object is. - sky_spectrum (:class:`linetools.spectra.xspectrum1d.XSpectrum1d`): + sky_spectrum (`linetools.spectra.xspectrum1d.XSpectrum1d`_): Archived sky spectrum. arx_fwhm_pix (:obj:`float`): Spectral FWHM (in pixels) of the archived sky spectrum. @@ -477,11 +477,11 @@ def spec_flex_shift_local(slits, slitord, specobjs, islit, sky_spectrum, arx_fwh Slit trace set. slitord (`numpy.ndarray`_): Array of slit/order numbers. - specobjs (:class:`~pypeit.specobjs.Specobjs`, optional): + specobjs (:class:`~pypeit.specobjs.SpecObjs`, optional): Spectral extractions. islit (:obj:`int`): Index of the slit where the sky spectrum related to our object is. - sky_spectrum (:class:`linetools.spectra.xspectrum1d.XSpectrum1d`): + sky_spectrum (`linetools.spectra.xspectrum1d.XSpectrum1d`_): Archived sky spectrum. arx_fwhm_pix (:obj:`float`): Spectral FWHM (in pixels) of the archived sky spectrum. @@ -563,7 +563,7 @@ def spec_flex_shift_local(slits, slitord, specobjs, islit, sky_spectrum, arx_fwh f'object(s) in slit {slits.spat_id[islit]}') # get the median shift among all objects in this slit idx_med_shift = np.where(flex_dict['shift'] == np.percentile(flex_dict['shift'], 50, - interpolation='nearest'))[0][0] + method='nearest'))[0][0] msgs.info(f"Median value of the measured flexure shifts in this slit, equal to " f"{flex_dict['shift'][idx_med_shift]:.3f} pixels, will be used") @@ -610,7 +610,7 @@ def spec_flexure_slit(slits, slitord, slit_bpm, sky_file, method="boxcar", speco extracted. This method uses a spectrum (stored in slitspecs) that is extracted from the center of each slit. - specobjs (:class:`~pypeit.specobjs.Specobjs`, optional): + specobjs (:class:`~pypeit.specobjs.SpecObjs`, optional): Spectral extractions slit_specs (:obj:`list`, optional): A list of linetools.xspectrum1d, one for each slit. The spectra stored in @@ -664,10 +664,10 @@ def spec_flexure_slit(slits, slitord, slit_bpm, sky_file, method="boxcar", speco # get spectral FWHM (in pixels) if available spec_fwhm_pix = None if wv_calib is not None: - iwv = np.where(wv_calib.spat_ids == slits.spat_id[islit])[0][0] # Allow for wavelength failures - if wv_calib.wv_fits is not None and wv_calib.wv_fits[iwv].fwhm is not None: - spec_fwhm_pix = wv_calib.wv_fits[iwv].fwhm + if wv_calib.fwhm_map is not None: + # Evaluate the spectral FWHM at the centre of the slit (in both the spectral and spatial directions) + spec_fwhm_pix = wv_calib.fwhm_map[islit].eval(slits.nspec/2, 0.5) if slit_cen: # global flexure @@ -684,7 +684,7 @@ def spec_flexure_slit(slits, slitord, slit_bpm, sky_file, method="boxcar", speco if len(return_later_slits) > 0: msgs.warn(f'Flexure shift calculation failed for {len(return_later_slits)} slits') # take the median value to deal with the cases when there are more than one shift per slit (e.g., local flexure) - saved_shifts = np.array([np.percentile(flex['shift'], 50, interpolation='nearest') + saved_shifts = np.array([np.percentile(flex['shift'], 50, method='nearest') if len(flex['shift']) > 0 else None for flex in flex_list]) if np.all(saved_shifts == None): # If all the elements in saved_shifts are None means that there are no saved shifts available @@ -695,7 +695,7 @@ def spec_flexure_slit(slits, slitord, slit_bpm, sky_file, method="boxcar", speco flex_list.append(empty_flex_dict.copy()) else: # get the median shift value among all slit - med_shift = np.percentile(saved_shifts[saved_shifts!= None], 50, interpolation='nearest') + med_shift = np.percentile(saved_shifts[saved_shifts!= None], 50, method='nearest') # in which slit the median is? islit_med_shift = np.where(saved_shifts == med_shift)[0][0] msgs.info(f"Median value of all the measured flexure shifts, equal to " @@ -815,8 +815,8 @@ def get_archive_spectrum(sky_file): Sky file Returns: - (:obj:`XSpectrum1D`): Sky spectrum - (float): FWHM of the sky lines in pixels. + tuple: The sky spectrum (`linetools.spectra.xspectrum1d.XSpectrum1D`_) + and the FWHM (float) of the sky lines in pixels. """ # Load Archive. Save the fwhm to avoid the performance hit from calling it on the archive sky spectrum # multiple times @@ -858,7 +858,7 @@ def get_sky_spectrum(sciimg, ivar, waveimg, thismask, global_sky, box_radius, sl extracted. For example, DET01. Returns: - (:obj:`XSpectrum1D`): Sky spectrum + (`linetools.spectra.xspectrum1d.XSpectrum1D`_): Sky spectrum """ spec = specobj.SpecObj(PYPELINE=pypeline, SLITID=-1, DET=str(det)) spec.trace_spec = np.arange(slits.nspec) @@ -917,9 +917,9 @@ def spec_flexure_qa(slitords, bpm, basename, flex_list, Used to generate the output file name flex_list (list): list of :obj:`dict` objects containing the flexure information - specobjs (:class:`~pypeit.specobjs.Specobjs`, optional): + specobjs (:class:`~pypeit.specobjs.SpecObjs`, optional): Spectrally extracted objects - out_dir (str, optonal): + out_dir (str, optional): Path to the output directory for the QA plots. If None, the current is used. """ @@ -1064,29 +1064,30 @@ def calculate_image_phase(imref, imshift, gpm_ref=None, gpm_shift=None, maskval= skimage is not installed, a standard (unmasked) cross-correlation is used. - Args: - im_ref (`numpy.ndarray`_): - Reference image - imshift (`numpy.ndarray`_): - Image that we want to measure the shift of (relative to im_ref) - gpm_ref (`numpy.ndarray`_): - Mask of good pixels (True = good) in the reference image - gpm_shift (`numpy.ndarray`_): - Mask of good pixels (True = good) in the shifted image - maskval (float, optional): - If gpm_ref and gpm_shift are both None, a single value can be specified - and this value will be masked in both images. - - Returns: - ra_diff (float): - Relative shift (in pixels) of image relative to im_ref (x direction). - In order to align image with im_ref, ra_diff should be added to the - x-coordinates of image - dec_diff (float): - Relative shift (in pixels) of image relative to im_ref (y direction). - In order to align image with im_ref, dec_diff should be added to the - y-coordinates of image - + Parameters + ---------- + im_ref : `numpy.ndarray`_ + Reference image + imshift : `numpy.ndarray`_ + Image that we want to measure the shift of (relative to im_ref) + gpm_ref : `numpy.ndarray`_ + Mask of good pixels (True = good) in the reference image + gpm_shift : `numpy.ndarray`_ + Mask of good pixels (True = good) in the shifted image + maskval : float, optional + If gpm_ref and gpm_shift are both None, a single value can be specified + and this value will be masked in both images. + + Returns + ------- + ra_diff : float + Relative shift (in pixels) of image relative to im_ref (x direction). + In order to align image with im_ref, ra_diff should be added to the + x-coordinates of image + dec_diff : float + Relative shift (in pixels) of image relative to im_ref (y direction). + In order to align image with im_ref, dec_diff should be added to the + y-coordinates of image """ # Do some checks first try: @@ -1327,7 +1328,7 @@ class MultiSlitFlexure(DataContainer): internals = ['flex_par', # Parameters (FlexurePar) 'spectrograph', # spectrograph - 'specobjs', # Specobjs object + 'specobjs', # SpecObjs object 'sobj_idx', # (ndet, nslits); Index to specobjs (tuple of arrays) 'sky_table', # Sky line table # 2D models diff --git a/pypeit/core/flux_calib.py b/pypeit/core/flux_calib.py index 339e0e7dd9..bee3a66872 100644 --- a/pypeit/core/flux_calib.py +++ b/pypeit/core/flux_calib.py @@ -27,6 +27,7 @@ from pypeit.wavemodel import conv2res from pypeit.core.wavecal import wvutils from pypeit.core import fitting +from pypeit.core import wave from pypeit import data @@ -120,7 +121,7 @@ def find_standard_file(ra, dec, toler=20.*units.arcmin, check=False): Object right-ascension in decimal deg dec : float Object declination in decimal deg - toler : :class:`astropy.units.quantity.Quantity`, optional + toler : `astropy.units.Quantity`_, optional Tolerance on matching archived standards to input. Expected to be in arcmin. check : bool, optional @@ -199,6 +200,8 @@ def find_standard_file(ra, dec, toler=20.*units.arcmin, check=False): std_dict['wave'] = std_spec['col1'] * units.AA std_dict['flux'] = std_spec['col2'] / PYPEIT_FLUX_SCALE * \ units.erg / units.s / units.cm ** 2 / units.AA + # Xshooter standard files use air wavelengths, convert them to vacuum + std_dict['wave'] = wave.airtovac(std_dict['wave']) elif sset == 'calspec': std_dict['std_source'] = sset std_spec = io.fits_open(fil)[1].data @@ -439,11 +442,14 @@ def load_extinction_data(longitude, latitude, extinctfilepar, Parameters ---------- - longitude, latitude: Geocentric coordinates in degrees (floats). - extinctfilepar : (str) + longitude : float + Geocentric longitude in degrees. + latitude : float + Geocentric latitude in degrees. + extinctfilepar : str The sensfunc['extinct_file'] parameter, used to determine which extinction file to load. - toler : Angle, optional + toler : `astropy.coordinates.Angle`_, optional Tolerance for matching detector to site (5 deg) Returns @@ -496,17 +502,18 @@ def load_extinction_data(longitude, latitude, extinctfilepar, def extinction_correction(wave, airmass, extinct): """ - Derive extinction correction + Derive extinction correction. + Based on algorithm in LowRedux (long_extinct) Parameters ---------- - wave: `numpy.ndarray`_ + wave : `numpy.ndarray`_ Wavelengths for interpolation. Should be sorted. Assumes angstroms. airmass : float Airmass - extinct : Table + extinct : `astropy.table.Table`_ Table of extinction values Returns @@ -615,7 +622,7 @@ def sensfunc(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, Args: wave (`numpy.ndarray`_): Wavelength of the star. Shape (nspec,) or (nspec, norders) - counts (ndarray): + counts (`numpy.ndarray`_): Flux (in counts) of the star. Shape (nspec,) or (nspec, norders) counts_ivar (`numpy.ndarray`_): Inverse variance of the star counts. Shape (nspec,) or (nspec, norders) @@ -669,7 +676,7 @@ def sensfunc(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, """ - wave_arr, counts_arr, ivar_arr, mask_arr, nspec, norders = utils.spec_atleast_2d(wave, counts, counts_ivar, counts_mask) + wave_arr, counts_arr, ivar_arr, mask_arr, log10_blaze_func, nspec, norders = utils.spec_atleast_2d(wave, counts, counts_ivar, counts_mask) zeropoint_data = np.zeros_like(wave_arr) zeropoint_data_gpm = np.zeros_like(wave_arr, dtype=bool) zeropoint_fit = np.zeros_like(wave_arr) @@ -1198,6 +1205,42 @@ def mask_stellar_helium(wave_star, mask_width=5.0, mask_star=None): # These are physical limits on the allowed values of the zeropoint in magnitudes +def eval_zeropoint(theta, func, wave, wave_min, wave_max, log10_blaze_func_per_ang=None): + """ + Evaluate the zeropoint model. + + Parameters + ---------- + theta : `numpy.ndarray`_ + Parameter vector for the zeropoint model + func : callable + Function for the zeropoint model from the set of available functions in + :func:`~pypeit.core.fitting.evaluate_fit`. + wave : `numpy.ndarray`_, shape = (nspec,) + Wavelength vector for zeropoint. + wave_min : float + Minimum wavelength for the zeropoint fit to be passed as an argument to + :func:`~pypeit.core.fitting.evaluate_fit` + wave_max : float + Maximum wavelength for the zeropoint fit to be passed as an argument to + :func:`~pypeit.core.fitting.evaluate_fit` + log10_blaze_func_per_ang : `numpy.ndarray`_, optional, shape = (nspec,) + Log10 blaze function per angstrom. This option is used if the zeropoint + model is relative to the non-parametric blaze function determined from + flats. The blaze function is defined on the wavelength grid wave. + + Returns + ------- + zeropoint : `numpy.ndarray`_, shape = (nspec,) + Zeropoint evaluated on the wavelength grid wave. + """ + poly_model = fitting.evaluate_fit(theta, func, wave, minx=wave_min, maxx=wave_max) + zeropoint = poly_model - 5.0 * np.log10(wave) + ZP_UNIT_CONST + if log10_blaze_func_per_ang is not None: + zeropoint += 2.5*log10_blaze_func_per_ang + + return zeropoint + def Nlam_to_Flam(wave, zeropoint, zp_min=5.0, zp_max=30.0): r""" @@ -1242,7 +1285,7 @@ def Flam_to_Nlam(wave, zeropoint, zp_min=5.0, zp_max=30.0): Returns ------- factor: `numpy.ndarray`_ - Factor that when multiplied into F_lam converts to N_lam + Factor that when multiplied into F_lam converts to N_lam, i.e. 1/S_lam """ gpm = (wave > 1.0) & (zeropoint > zp_min) & (zeropoint < zp_max) @@ -1264,7 +1307,7 @@ def compute_zeropoint(wave, N_lam, N_lam_gpm, flam_std_star, tellmodel=None): N_lam_gpm: `numpy.ndarray`_ N_lam mask, good pixel mask, boolean, shape (nspec,) flam_std_star: `numpy.ndarray`_ - True standard star spectrum units set of PYPEIT_FLUX_SCALE erg/s/cm^2/sm/Angstrom + True standard star spectrum in units of PYPEIT_FLUX_SCALE erg/s/cm^2/Angstrom tellmodel: `numpy.ndarray`_ Telluric absorption model, optional, shape (nspec,) @@ -1331,21 +1374,21 @@ def zeropoint_qa_plot(wave, zeropoint_data, zeropoint_data_gpm, zeropoint_fit, z Parameters ---------- - wave: `numpy.ndarray`_ + wave : `numpy.ndarray`_ Wavelength array - zeropoint_data: `numpy.ndarray`_ + zeropoint_data : `numpy.ndarray`_ Zeropoint data array - zeropoint_data_gpm: bool `numpy.ndarray`_ + zeropoint_data_gpm : boolean `numpy.ndarray`_ Good pixel mask array for zeropoint_data - zeropoint_fit: `numpy.ndarray`_ + zeropoint_fit : `numpy.ndarray`_ Zeropoint fitting array - zeropoint_fit_gpm: bool zeropoint_fit + zeropoint_fit_gpm : boolean `numpy.ndarray`_ Good pixel mask array for zeropoint_fit - title: str, optional + title : str, optional Title for the QA plot - axis: None or matplotlib.pyplot axis, optional - axis used for ploting. - show: bool, optional + axis : `matplotlib.axes.Axes`_, optional + axis used for ploting. If None, a new plot is created + show : bool, optional Whether to show the QA plot """ @@ -1389,26 +1432,26 @@ def standard_zeropoint(wave, Nlam, Nlam_ivar, Nlam_gpm, flam_true, mask_recomb=N inverse variance of counts/s/Angstrom Nlam_gpm : `numpy.ndarray`_ mask for bad pixels. True is good. - flam_true : Quantity array - standard star true flux (erg/s/cm^2/A) + flam_true : `astropy.units.Quantity`_ + array with true standard star flux (erg/s/cm^2/A) mask_recomb: `numpy.ndarray`_ mask for hydrogen (and/or helium II) recombination lines. True is good. mask_tell: `numpy.ndarray`_ mask for telluric regions. True is good. - maxiter : integer + maxiter : int maximum number of iterations for polynomial fit - upper : integer + upper : int number of sigma for rejection in polynomial - lower : integer + lower : int number of sigma for rejection in polynomial - polyorder : integer + polyorder : int order of polynomial fit balm_mask_wid: float Mask parameter for Balmer absorption. A region equal to balm_mask_wid in units of angstrom is masked. - nresln: integer/float + nresln: int, float number of resolution elements between breakpoints - resolution: integer/float. + resolution: int, float The spectral resolution. This paramters should be removed in the future. The resolution should be estimated from spectra directly. debug : bool diff --git a/pypeit/core/gui/identify.py b/pypeit/core/gui/identify.py index 545f8d8d79..8902f1b093 100644 --- a/pypeit/core/gui/identify.py +++ b/pypeit/core/gui/identify.py @@ -1,3 +1,6 @@ +""" +.. include:: ../include/links.rst +""" from datetime import datetime import os import copy @@ -18,7 +21,7 @@ from pypeit.par import pypeitpar from pypeit.core.wavecal import wv_fitting, waveio, wvutils -from pypeit import data, msgs +from pypeit import msgs from astropy.io import ascii as ascii_io from astropy.table import Table @@ -199,12 +202,12 @@ def initialise(cls, arccen, lamps, slits, slit=0, par=None, wv_calib_all=None, Parameters ---------- - arccen : ndarray + arccen : `numpy.ndarray`_ Arc spectrum lamps : :obj:`list` List of arc lamps to be used for wavelength calibration. E.g., ['ArI','NeI','KrI','XeI'] - slits : :class:`SlitTraceSet` + slits : :class:`~pypeit.slittrace.SlitTraceSet` Data container with slit trace information slit : int, optional The slit to be used for wavelength calibration @@ -272,11 +275,7 @@ def initialise(cls, arccen, lamps, slits, slit=0, par=None, wv_calib_all=None, detns = tdetns[icut] # Load line lists - if 'ThAr' in lamps: - line_lists_all = waveio.load_line_lists(lamps) - line_lists = line_lists_all[np.where(line_lists_all['ion'] != 'UNKNWN')] - else: - line_lists = waveio.load_line_lists(lamps) + line_lists, _, _ = waveio.load_line_lists(lamps, include_unknown=False) # Trim the wavelength scale if requested if wavelim is not None: @@ -408,7 +407,7 @@ def linelist_select(self, event): Note, only the LMB works. Args: - event (Event): A matplotlib event instance + event (`matplotlib.backend_bases.Event`_): A matplotlib event instance """ if event.button == 1: self.update_line_id() @@ -594,7 +593,7 @@ def draw_callback(self, event): """Draw the lines and annotate with their IDs Args: - event (Event): A matplotlib event instance + event (`matplotlib.backend_bases.Event`_): A matplotlib event instance """ # Get the background self.background = self.canvas.copy_from_bbox(self.axes['main'].bbox) @@ -624,10 +623,11 @@ def get_ann_ypos(self, scale=1.02): """Calculate the y locations of the annotated IDs Args: - scale (float): Scale the location relative to the maximum value of the spectrum + scale (float): + Scale the location relative to the maximum value of the spectrum Returns: - ypos (ndarray): y locations of the annotations + `numpy.ndarray`_: y locations of the annotations """ ypos = np.zeros(self._detns.size) for xx in range(self._detns.size): @@ -644,10 +644,11 @@ def get_ind_under_point(self, event): """Get the index of the line closest to the cursor Args: - event (Event): Matplotlib event instance containing information about the event + event (`matplotlib.backend_bases.Event`_): + Matplotlib event instance containing information about the event Returns: - ind (int): Index of the spectrum where the event occurred + int: Index of the spectrum where the event occurred """ ind = np.argmin(np.abs(self.plotx - event.xdata)) return ind @@ -656,10 +657,11 @@ def get_axisID(self, event): """Get the ID of the axis where an event has occurred Args: - event (Event): Matplotlib event instance containing information about the event + event (`matplotlib.backend_bases.Event`_): + Matplotlib event instance containing information about the event Returns: - axisID (int, None): Axis where the event has occurred + int: Axis where the event has occurred """ if event.inaxes == self.axes['main']: return 0 @@ -680,7 +682,7 @@ def get_results(self): manually identified all lines. Returns: - wvcalib (dict): Dict of wavelength calibration solutions + dict: Dict of wavelength calibration solutions """ wvcalib = {} # Check that a result exists: @@ -763,22 +765,7 @@ def store_solution(self, final_fit, binspec, rmstol=0.15, # Instead of a generic name, save the wvarxiv with a unique identifier date_str = datetime.now().strftime("%Y%m%dT%H%M") wvarxiv_name = f"wvarxiv_{self.specname}_{date_str}.fits" - wvutils.write_template(wavelengths, self.specdata, binspec, - './', wvarxiv_name) - - # Also copy the file to the cache for direct use - data.write_file_to_cache(wvarxiv_name, - wvarxiv_name, - "arc_lines/reid_arxiv") - - msgs.info(f"Your arxiv solution has been written to ./{wvarxiv_name}\n") - msgs.info(f"Your arxiv solution has also been cached.{msgs.newline()}" - f"To utilize this wavelength solution, insert the{msgs.newline()}" - f"following block in your PypeIt Reduction File:{msgs.newline()}" - f" [calibrations]{msgs.newline()}" - f" [[wavelengths]]{msgs.newline()}" - f" reid_arxiv = {wvarxiv_name}{msgs.newline()}" - f" method = full_template\n") + wvutils.write_template(wavelengths, self.specdata, binspec, './', wvarxiv_name, cache=True) # Write the WVCalib file outfname = "wvcalib.fits" @@ -815,7 +802,8 @@ def button_press_callback(self, event): """What to do when the mouse button is pressed Args: - event (Event): Matplotlib event instance containing information about the event + event (`matplotlib.backend_bases.Event`_): + Matplotlib event instance containing information about the event """ if event.inaxes is None: return @@ -849,10 +837,8 @@ def button_release_callback(self, event): """What to do when the mouse button is released Args: - event (Event): Matplotlib event instance containing information about the event - - Returns: - None + event (`matplotlib.backend_bases.Event`_): + Matplotlib event instance containing information about the event """ self._msedown = False if event.inaxes is None: @@ -900,10 +886,8 @@ def key_press_callback(self, event): """What to do when a key is pressed Args: - event (Event): Matplotlib event instance containing information about the event - - Returns: - None + event (`matplotlib.backend_bases.Event`_): + Matplotlib event instance containing information about the event """ # Check that the event is in an axis... if not event.inaxes: @@ -1084,20 +1068,20 @@ def delete_line_id(self): self._lineflg[rmid] = 0 def fitsol_value(self, xfit=None, idx=None): - """Calculate the wavelength at a pixel + """ + Calculate the wavelength at a pixel Parameters ---------- - - xfit : ndarray, float + xfit : `numpy.ndarray`_, float Pixel values that the user wishes to evaluate the wavelength - idx : ndarray, int + idx : `numpy.ndarray`_, int Index of the arc line detections that the user wishes to evaluate the wavelength Returns ------- - - disp : The wavelength (Angstroms) of the requested pixels + disp : `numpy.ndarray`_, float + The wavelength (Angstroms) of the requested pixels """ if xfit is None: xfit = self._detns @@ -1114,11 +1098,15 @@ def fitsol_deriv(self, xfit=None, idx=None): """Calculate the dispersion as a function of wavelength Args: - xfit (ndarray, float): Pixel values that the user wishes to evaluate the wavelength - idx (int): Index of the arc line detections that the user wishes to evaluate the wavelength + xfit (`numpy.ndarray`_, float): + Pixel values that the user wishes to evaluate the wavelength + idx (int): + Index of the arc line detections that the user wishes to + evaluate the wavelength Returns: - disp (ndarray, float, None): The dispersion (Angstroms/pixel) as a function of wavelength + ndarray, float: The dispersion (Angstroms/pixel) as a function of + wavelength """ if xfit is None: xfit = self._detns diff --git a/pypeit/core/gui/object_find.py b/pypeit/core/gui/object_find.py index b06d0370aa..54526c5a3c 100644 --- a/pypeit/core/gui/object_find.py +++ b/pypeit/core/gui/object_find.py @@ -5,6 +5,8 @@ Implement color scaling with RMB click+drag +.. include:: ../include/links.rst + """ import os @@ -89,16 +91,25 @@ def from_dict(self, obj_dict, det): self.add_object(det, pos_spat, pos_spec, obj_dict['traces'][ii], spec_trace, obj_dict['fwhm'][ii], addrm=0) def add_object(self, det, pos_spat, pos_spec, trc_spat, trc_spec, fwhm, addrm=1): - """Add an object trace + """ + Add an object trace Args: - det (int): Detector to add a slit on - pos_spat (float): Spatial pixel position - pos_spec (float): Spectral pixel position - trc_spat (ndarray): Spatial trace of object - trc_spec (ndarray): Spectral trace of object - fwhm (float): FWHM of the object - addrm (int): Flag to say if an object was been added (1), removed (-1), or was an auto found slit (0) + det (int): + Detector to add a slit on + pos_spat (float): + patial pixel position + pos_spec (float): + Spectral pixel position + trc_spat (`numpy.ndarray`_): + Spatial trace of object + trc_spec (`numpy.ndarray`_): + Spectral trace of object + fwhm (float): + FWHM of the object + addrm (int): + Flag to say if an object was been added (1), removed (-1), or + was an auto found slit (0) """ self._det.append(det) self._add_rm.append(addrm) @@ -402,7 +413,7 @@ def draw_callback(self, event): """Draw callback (i.e. everytime the canvas is being drawn/updated) Args: - event (Event): A matplotlib event instance + event (`matplotlib.backend_bases.Event`_): A matplotlib event instance """ # Get the background self.background = self.canvas.copy_from_bbox(self.axes['main'].bbox) @@ -412,7 +423,7 @@ def get_ind_under_point(self, event): """Get the index of the object trace closest to the cursor Args: - event (Event): Matplotlib event instance containing information about the event + event (`matplotlib.backend_bases.Event`_): Matplotlib event instance containing information about the event """ mindist = self._spatpos.shape[0]**2 self._obj_idx = -1 @@ -430,7 +441,7 @@ def get_axisID(self, event): """Get the ID of the axis where an event has occurred Args: - event (Event): Matplotlib event instance containing information about the event + event (`matplotlib.backend_bases.Event`_): Matplotlib event instance containing information about the event Returns: int, None: Axis where the event has occurred @@ -456,7 +467,7 @@ def button_press_callback(self, event): """What to do when the mouse button is pressed Args: - event (Event): Matplotlib event instance containing information about the event + event (`matplotlib.backend_bases.Event`_): Matplotlib event instance containing information about the event """ if event.inaxes is None: return @@ -475,7 +486,7 @@ def button_release_callback(self, event): """What to do when the mouse button is released Args: - event (Event): Matplotlib event instance containing information about the event + event (`matplotlib.backend_bases.Event`_): Matplotlib event instance containing information about the event """ if event.inaxes is None: return @@ -527,7 +538,7 @@ def key_press_callback(self, event): """What to do when a key is pressed Args: - event (Event): Matplotlib event instance containing information about the event + event (`matplotlib.backend_bases.Event`_): Matplotlib event instance containing information about the event """ # Check that the event is in an axis... if not event.inaxes: @@ -859,11 +870,11 @@ def initialise(det, frame, left, right, obj_trace, trace_models, sobjs, slit_ids Args: det (int): 1-indexed detector number - frame (numpy.ndarray): + frame (`numpy.ndarray`_): Sky subtracted science image - left (numpy.ndarray): + left (`numpy.ndarray`_): Slit left edges - right (numpy.ndarray): + right (`numpy.ndarray`_): Slit right edges obj_trace (dict): Result of diff --git a/pypeit/core/gui/skysub_regions.py b/pypeit/core/gui/skysub_regions.py index 57e3a1bae8..26417eaa94 100644 --- a/pypeit/core/gui/skysub_regions.py +++ b/pypeit/core/gui/skysub_regions.py @@ -1,5 +1,8 @@ """ This script allows the user to manually select the sky background regions + +.. include:: ../include/links.rst + """ import os @@ -97,7 +100,7 @@ def __init__(self, canvas, image, frame, outname, det, slits, axes, pypeline, sp self._nslits = slits.nslits self._maxslitlength = np.max(self.slits.get_slitlengths(initial=initial)) self._resolution = int(10.0 * self._maxslitlength) if resolution is None else int(resolution) - self._allreg = np.zeros(int(self._resolution), dtype=np.bool) + self._allreg = np.zeros(int(self._resolution), dtype=bool) self._specx = np.arange(int(self._resolution)) self._start = [0, 0] self._end = [0, 0] @@ -154,7 +157,7 @@ def initialize(cls, det, frame, slits, pypeline, spectrograph, outname="skyregio ---------- det : int Detector index - frame : ndarray + frame : `numpy.ndarray`_ Sky subtracted science image slits : :class:`~pypeit.slittrace.SlitTraceSet` Object with the image coordinates of the slit edges @@ -284,7 +287,7 @@ def button_regb(self, event): would select the first 10%, the inner 30%, and the final 20% of all slits Args: - event : Event + event : `matplotlib.backend_bases.Event`_ A matplotlib event instance """ self.update_infobox(message='Enter regions in the terminal (see terminal for help)', yesno=False) @@ -315,7 +318,7 @@ def button_cont(self, event): """What to do when the 'exit and save' button is clicked Args: - event : Event + event : `matplotlib.backend_bases.Event`_ A matplotlib event instance """ self._respreq = [True, "exit_update"] @@ -326,7 +329,7 @@ def button_exit(self, event): """What to do when the 'exit and do not save changes' button is clicked Args: - event : Event + event : `matplotlib.backend_bases.Event`_ A matplotlib event instance """ self._respreq = [True, "exit_restore"] @@ -373,7 +376,7 @@ def draw_callback(self, event): """Draw callback (i.e. everytime the canvas is being drawn/updated) Args: - event : Event + event : `matplotlib.backend_bases.Event`_ A matplotlib event instance """ # Get the background @@ -384,7 +387,7 @@ def get_current_slit(self, event): """Get the index of the slit closest to the cursor Args: - event : Event + event : `matplotlib.backend_bases.Event`_ Matplotlib event instance containing information about the event """ # Find the current slit @@ -401,7 +404,7 @@ def get_axisID(self, event): """Get the ID of the axis where an event has occurred Args: - event : Event + event : `matplotlib.backend_bases.Event`_ Matplotlib event instance containing information about the event Returns: @@ -427,7 +430,7 @@ def button_press_callback(self, event): """What to do when the mouse button is pressed Args: - event : Event + event : `matplotlib.backend_bases.Event`_ Matplotlib event instance containing information about the event """ if event.inaxes is None: @@ -449,7 +452,7 @@ def button_release_callback(self, event): """What to do when the mouse button is released Args: - event : Event + event : `matplotlib.backend_bases.Event`_ Matplotlib event instance containing information about the event """ if event.inaxes is None: @@ -494,7 +497,7 @@ def key_press_callback(self, event): """What to do when a key is pressed Args: - event : Event + event : `matplotlib.backend_bases.Event`_ Matplotlib event instance containing information about the event """ # Check that the event is in an axis... @@ -579,7 +582,8 @@ def get_result(self): outfil = self._outname if os.path.exists(self._outname) and not self._overwrite: outfil = 'temp.fits' - msgs.warn("File exists:\n{0:s}\nSaving regions to 'temp.fits'") + msgs.warn(f"A SkyRegions file already exists and you have not forced an overwrite:\n{self._outname}") + msgs.info(f"Saving regions to: {outfil}") self._overwrite = True msskyreg = buildimage.SkyRegions(image=inmask.astype(float), PYP_SPEC=self.spectrograph) msskyreg.to_file(file_path=outfil) @@ -676,6 +680,6 @@ def add_region_all(self): def reset_regions(self): """ Reset the sky regions for all slits simultaneously """ - self._skyreg = [np.zeros(self._resolution, dtype=np.bool) for all in range(self._nslits)] + self._skyreg = [np.zeros(self._resolution, dtype=bool) for all in range(self._nslits)] self._allreg[:] = False diff --git a/pypeit/core/meta.py b/pypeit/core/meta.py index ab4ae14b83..a4eacf5f9b 100644 --- a/pypeit/core/meta.py +++ b/pypeit/core/meta.py @@ -1,6 +1,6 @@ """ -Provides methods common to :class:`pypeit.metadata.PypeItMetaData` and -:class:`pypeit.spectographs.spectrograph.Spectrograph` that define the +Provides methods common to :class:`~pypeit.metadata.PypeItMetaData` and +:class:`~pypeit.spectrographs.spectrograph.Spectrograph` that define the common metadata used for all spectrographs. .. include common links, assuming primary doc root is up one directory @@ -164,6 +164,7 @@ def define_additional_meta(nlamps=20): 'slitlength': dict(dtype=float, comment='Slit length, used only for long slits'), 'temperature': dict(dtype=float, comment='Temperature (units.K) at observation time'), 'utc': dict(dtype=str, comment='UTC of observation'), + 'mirror': dict(dtype=str, comment='Position of an instrument mirror (e.g. IN or OUT)'), 'xd': dict(dtype=float, comment='Cross disperser (e.g. red or blue for HIRES)'), 'xdangle':dict(dtype=float, comment='Cross disperser angle'), } diff --git a/pypeit/core/parse.py b/pypeit/core/parse.py index 52cfdeae5a..acea048ae2 100644 --- a/pypeit/core/parse.py +++ b/pypeit/core/parse.py @@ -115,7 +115,7 @@ def binning2string(binspectral, binspatial): return '{0},{1}'.format(binspectral, binspatial) -def parse_binning(binning): +def parse_binning(binning:str): """ Parse input binning into binspectral, binspatial @@ -123,7 +123,7 @@ def parse_binning(binning): parsed directly from the Header. The developer needs to react accordingly.. Args: - binning (str, ndarray or tuple): + binning (str, `numpy.ndarray`_, tuple): Returns: tuple: binspectral, binspatial as integers @@ -149,8 +149,7 @@ def parse_binning(binning): # Return return binspectral, binspatial -# TODO: Allow this to differentiate between detectors and mosaics. Input syntax -# likely to become, e.g., DET01:175,DET01:205. + def parse_slitspatnum(slitspatnum): """ Parse the ``slitspatnum`` into a list of detectors and SPAT_IDs. @@ -165,16 +164,10 @@ def parse_slitspatnum(slitspatnum): array is ``(nslits,)``, where ``nslits`` is the number of ``slitspatnum`` entries parsed (1 if only a single string is provided). """ - dets = [] - spat_ids = [] - if isinstance(slitspatnum,list): - slitspatnum = ",".join(slitspatnum) - for item in slitspatnum.split(','): - spt = item.split(':') - dets.append(spt[0]) - spat_ids.append(int(spt[1])) - # Return - return np.array(dets).astype(str), np.array(spat_ids).astype(int) + _slitspatnum = slitspatnum if isinstance(slitspatnum,list) else [slitspatnum] + _slitspatnum = np.concatenate([item.split(',') for item in _slitspatnum]) + _slitspatnum = np.array([item.split(':') for item in _slitspatnum]) + return _slitspatnum[:,0], _slitspatnum[:,1].astype(int) def sec2slice(subarray, one_indexed=False, include_end=False, require_dim=None, binning=None): diff --git a/pypeit/core/pca.py b/pypeit/core/pca.py index 4f94241a1d..9ee6dd4520 100644 --- a/pypeit/core/pca.py +++ b/pypeit/core/pca.py @@ -33,8 +33,7 @@ def pca_decomposition(vectors, npca=None, pca_explained_var=99.0, mean=None): This is a fully generalized convenience function for a specific use of `sklearn.decomposition.PCA`_. When used - within PypeIt, the vectors to decompose (see, e.g., - :class:`pypeit.edgetrace.EdgeTracePCA`) typically have the + within PypeIt, the vectors to decompose typically have the length of the spectral axis. This means that, within PypeIt, arrays are typically transposed when passed to this function. @@ -133,12 +132,12 @@ def fit_pca_coefficients(coeff, order, ivar=None, weights=None, function='legend The coefficients of each PCA component are fit by a low-order polynomial, where the abscissa is set by the `coo` argument (see - :func:`pypeit.fitting.robust_fit`). + :func:`~pypeit.core.fitting.robust_fit`). .. note:: This is a general function, not really specific to the PCA; and is really just a wrapper for - :func:`pypeit.fitting.robust_fit`. + :func:`~pypeit.core.fitting.robust_fit`. Args: coeff (`numpy.ndarray`_): @@ -157,7 +156,7 @@ def fit_pca_coefficients(coeff, order, ivar=None, weights=None, function='legend ivar (`numpy.ndarray`_, optional): Inverse variance in the PCA coefficients to use during the fit; see the `invvar` parameter of - :func:`pypeit.fitting.robust_fit`. If None, fit is + :func:`~pypeit.core.fitting.robust_fit`. If None, fit is not error weighted. If a vector with shape :math:`(N_{\rm vec},)`, the same error will be assumed for all PCA components (i.e., `ivar` will be expanded to match the @@ -166,7 +165,7 @@ def fit_pca_coefficients(coeff, order, ivar=None, weights=None, function='legend weights (`numpy.ndarray`_, optional): Weights to apply to the PCA coefficients during the fit; see the `weights` parameter of - :func:`pypeit.fitting.robust_fit`. If None, the + :func:`~pypeit.core.fitting.robust_fit`. If None, the weights are uniform. If a vector with shape :math:`(N_{\rm vec},)`, the same weights will be assumed for all PCA components (i.e., `weights` will be expanded @@ -177,14 +176,14 @@ def fit_pca_coefficients(coeff, order, ivar=None, weights=None, function='legend lower (:obj:`float`, optional): Number of standard deviations used for rejecting data **below** the mean residual. If None, no rejection is - performed. See :func:`fitting.robust_fit`. + performed. See :func:`~pypeit.core.fitting.robust_fit`. upper (:obj:`float`, optional): Number of standard deviations used for rejecting data **above** the mean residual. If None, no rejection is - performed. See :func:`fitting.robust_fit`. + performed. See :func:`~pypeit.core.fitting.robust_fit`. maxrej (:obj:`int`, optional): Maximum number of points to reject during fit iterations. - See :func:`fitting.robust_fit`. + See :func:`~pypeit.core.fitting.robust_fit`. maxiter (:obj:`int`, optional): Maximum number of rejection iterations allows. To force no rejection iterations, set to 0. @@ -197,7 +196,7 @@ def fit_pca_coefficients(coeff, order, ivar=None, weights=None, function='legend Minimum and maximum values used to rescale the independent axis data. If None, the minimum and maximum values of `coo` are used. See - :func:`fitting.robust_fit`. + :func:`~pypeit.core.fitting.robust_fit`. debug (:obj:`bool`, optional): Show plots useful for debugging. diff --git a/pypeit/core/pixels.py b/pypeit/core/pixels.py index c1d423021e..65b51db151 100644 --- a/pypeit/core/pixels.py +++ b/pypeit/core/pixels.py @@ -26,7 +26,7 @@ def phys_to_pix(array, pixlocn, axis): Returns ------- - pixarr : ndarray + pixarr : `numpy.ndarray`_ The pixel locations of the input array (as seen on a computer screen) """ if len(array.shape) > 2: diff --git a/pypeit/core/procimg.py b/pypeit/core/procimg.py index ef2c2cce29..6c356c1506 100644 --- a/pypeit/core/procimg.py +++ b/pypeit/core/procimg.py @@ -217,7 +217,7 @@ def lacosmic(sciframe, saturation=None, nonlinear=1., bpm=None, varframe=None, m msgs.info("Convolving image with Laplacian kernel") deriv = convolve(boxcar_replicate(_sciframe, 2), laplkernel, normalize_kernel=False, boundary='extend') - s = utils.rebin_evlist(np.clip(deriv, 0, None), _sciframe.shape) * _inv_err / 2.0 + s = utils.rebinND(np.clip(deriv, 0, None), _sciframe.shape) * _inv_err / 2.0 # Remove the large structures sp = s - scipy.ndimage.median_filter(s, size=5, mode='mirror') @@ -1084,7 +1084,7 @@ def trim_frame(frame, mask): `numpy.ndarray`_: Trimmed image Raises: - PypitError: + :class:`~pypeit.pypmsgs.PypeItError`: Error raised if the trimmed image includes masked values because the shape of the valid region is odd. """ diff --git a/pypeit/core/pydl.py b/pypeit/core/pydl.py index 98da6833f3..b7916dfbd4 100644 --- a/pypeit/core/pydl.py +++ b/pypeit/core/pydl.py @@ -325,7 +325,7 @@ class TraceSet(object): When initialized with x,y positions, this contains the fitted y values. pypeitFits : list - Holds a list of :class:`pypeit.fitting.PypeItFit` fits + Holds a list of :class:`~pypeit.core.fitting.PypeItFit` fits """ # ToDO Remove the kwargs and put in all the djs_reject parameters here def __init__(self, *args, **kwargs): @@ -897,16 +897,16 @@ def djs_reject(data, model, outmask=None, inmask=None, # print(newmask) if grow > 0: - rejects = newmask == 0 - if rejects.any(): - irejects = rejects.nonzero()[0] - for k in range(1, grow): - newmask[(irejects - k) > 0] = 0 - newmask[(irejects + k) < (data.shape[0]-1)] = 0 + bpm = np.logical_not(newmask) + if bpm.any(): + irejects = np.where(bpm)[0] + for k in range(1, grow+1): + newmask[np.clip(irejects - k, 0,None)] = False + newmask[np.clip(irejects + k, None, data.shape[0]-1)] = False if inmask is not None: - newmask = newmask & inmask + newmask &= inmask if sticky: - newmask = newmask & outmask + newmask &= outmask # # Set qdone if the input outmask is identical to the output outmask. # @@ -915,7 +915,7 @@ def djs_reject(data, model, outmask=None, inmask=None, # to python True and False booleans outmask = newmask - return (outmask, qdone) + return outmask, qdone diff --git a/pypeit/core/qa.py b/pypeit/core/qa.py index 7984766500..1929f9c5c2 100644 --- a/pypeit/core/qa.py +++ b/pypeit/core/qa.py @@ -1,4 +1,7 @@ """ Module for QA in PypeIt + +.. include:: ../include/links.rst + """ import os import datetime @@ -49,6 +52,8 @@ def set_qa_filename(root, method, det=None, slit=None, prefix=None, out_dir=None elif method == 'arc_fit_qa': # outfile = 'QA/PNGs/Arc_1dfit_{:s}_S{:04d}.png'.format(root, slit) outfile = 'PNGs/Arc_1dfit_{:s}_S{:04d}.png'.format(root, slit) + elif method == 'arc_fwhm_qa': + outfile = 'PNGs/Arc_FWHMfit_{:s}_S{:04d}.png'.format(root, slit) elif method == 'plot_orderfits_Arc': # This is root for multiple PNGs outfile = 'QA/PNGs/Arc_lines_{:s}_S{:04d}_'.format(root, slit) elif method == 'arc_fit2d_global_qa': @@ -186,11 +191,12 @@ def html_header(title): return head def html_end(f, body, links=None): - """ Fill in the HTML file with a proper ending + """ + Fill in the HTML file with a proper ending Parameters ---------- - f : file + f : `io.TextIOWrapper`_ body : str links : str, optional @@ -220,8 +226,10 @@ def html_init(f, title): Initialize the HTML file Args: - f (fileobj): file object to write to - title (str): title + f (`io.TextIOWrapper`_): + file object to write to + title (str): + title Returns: str: Initial HTML text incluing the header and links @@ -471,11 +479,14 @@ def gen_exp_html(): def close_qa(pypeit_file, qa_path): - """Tie off QA under a crash + """ + Tie off QA under a crash Args: - pypeit_file (_type_): _description_ - qa_path (_type_): _description_ + pypeit_file (str): + PypeIt file name + qa_path (str): + Path to QA directory """ if pypeit_file is None: return diff --git a/pypeit/core/skysub.py b/pypeit/core/skysub.py index 5650915170..0634505bd3 100644 --- a/pypeit/core/skysub.py +++ b/pypeit/core/skysub.py @@ -7,12 +7,12 @@ import scipy.ndimage import scipy.special +from scipy.interpolate import RegularGridInterpolator import matplotlib.pyplot as plt from IPython import embed -from pypeit.images import imagebitmask from pypeit.core import basis, pixels, extract from pypeit.core import fitting from pypeit.core import procimg @@ -136,7 +136,7 @@ def global_skysub(image, ivar, tilts, thismask, slit_left, slit_righ, inmask=Non msgs.error("Type of inmask should be bool and is of type: {:}".format(inmask.dtype)) # Sky pixels for fitting - gpm = thismask & (ivar > 0.0) & inmask & np.logical_not(edgmask) + gpm = thismask & (ivar > 0.0) & inmask & np.logical_not(edgmask) & np.isfinite(image) & np.isfinite(ivar) bad_pixel_frac = np.sum(thismask & np.logical_not(gpm))/np.sum(thismask) if bad_pixel_frac > max_mask_frac: msgs.warn('This slit/order has {:5.3f}% of the pixels masked, which exceeds the threshold of {:f}%. '.format(100.0*bad_pixel_frac, 100.0*max_mask_frac) @@ -395,6 +395,8 @@ def optimal_bkpts(bkpts_optimal, bsp_min, piximg, sampmask, samp_frac=0.80, Parameters ---------- + bkpts_optimal: bool + If True, then the breakpoints are optimally spaced. If False, then the breakpoints are spaced uniformly. bsp_min: float Desired B-spline breakpoint spacing in pixels piximg: `numpy.ndarray`_ @@ -537,7 +539,7 @@ def optimal_bkpts(bkpts_optimal, bsp_min, piximg, sampmask, samp_frac=0.80, def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, slit_left, - slit_righ, sobjs, ingpm=None, spat_pix=None, adderr=0.01, bsp=0.6, + slit_righ, sobjs, ingpm=None, fwhmimg=None, spat_pix=None, adderr=0.01, bsp=0.6, trim_edg=(3,3), std=False, prof_nsigma=None, niter=4, extract_good_frac=0.005, sigrej=3.5, bkpts_optimal=True, debug_bkpts=False, force_gauss=False, sn_gauss=4.0, model_full_slit=False, @@ -572,13 +574,16 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, slit_righ : `numpy.ndarray`_ Right slit boundary in floating point pixels. shape (nspec, 1) or (nspec) - sobjs : :class:`~pypeit.specobjs.SpecoObjs` object + sobjs : :class:`~pypeit.specobjs.SpecObjs` object Object containing the information about the objects found on the slit/order from objfind or ech_objfind ingpm : `numpy.ndarray`_, optional Input mask with any non-zero item flagged as False using :class:`pypeit.images.imagebitmask.ImageBitMask` shape=(nspec, nspat) + fwhmimg : `numpy.ndarray`_, None, optional: + Floating-point image containing the modeled spectral FWHM (in pixels) at every pixel location. + Must have the same shape as ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. spat_pix: `numpy.ndarray`_, optional Image containing the spatial location of pixels. If not input, it will be computed from ``spat_img = @@ -793,6 +798,9 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, min_spat_img = min_spat1[:, None] max_spat_img = max_spat1[:, None] localmask = (spat_img > min_spat_img) & (spat_img < max_spat_img) & thismask + if np.sum(localmask) == 0: + msgs.error('There are no pixels on the localmask for group={}. ' + 'Something is very wrong with either your slit edges or your object traces'.format(group)) npoly = skysub_npoly(localmask) # Some bookeeping to define the sub-image and make sure it does not land off the mask @@ -825,7 +833,7 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, # TODO -- Use extract_specobj_boxcar to avoid code duplication extract.extract_boxcar(sciimg, modelivar, outmask, waveimg, skyimage, - sobjs[iobj], base_var=base_var, count_scale=count_scale, + sobjs[iobj], fwhmimg=fwhmimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) flux = sobjs[iobj].BOX_COUNTS fluxivar = sobjs[iobj].BOX_COUNTS_IVAR * sobjs[iobj].BOX_MASK @@ -837,12 +845,12 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, objmask = ((spat_img >= (trace - 2.0 * sobjs[iobj].BOX_RADIUS)) & (spat_img <= (trace + 2.0 * sobjs[iobj].BOX_RADIUS))) # Boxcar extract.extract_boxcar(sciimg, modelivar, (outmask & objmask), waveimg, - skyimage, sobjs[iobj], base_var=base_var, + skyimage, sobjs[iobj], fwhmimg=fwhmimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) # Optimal extract.extract_optimal(sciimg, modelivar, (outmask & objmask), waveimg, skyimage, thismask, last_profile, sobjs[iobj], - base_var=base_var, count_scale=count_scale, + fwhmimg=fwhmimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) # If the extraction is bad do not update if sobjs[iobj].OPT_MASK is not None: @@ -969,11 +977,11 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, objmask = ((spat_img >= (trace - 2.0 * sobjs[iobj].BOX_RADIUS)) & (spat_img <= (trace + 2.0 * sobjs[iobj].BOX_RADIUS))) extract.extract_optimal(sciimg, modelivar * thismask, (outmask_extract & objmask), waveimg, skyimage, thismask, this_profile, sobjs[iobj], - base_var=base_var, count_scale=count_scale, + fwhmimg=fwhmimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) # Boxcar extract.extract_boxcar(sciimg, modelivar*thismask, (outmask_extract & objmask), - waveimg, skyimage, sobjs[iobj], base_var=base_var, + waveimg, skyimage, sobjs[iobj], fwhmimg=fwhmimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) sobjs[iobj].min_spat = min_spat sobjs[iobj].max_spat = max_spat @@ -1023,7 +1031,7 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, global_sky, left, right, - slitmask, sobjs, order_vec, spat_pix=None, + slitmask, sobjs, spat_pix=None, fit_fwhm=False, min_snr=2.0, bsp=0.6, trim_edg=(3,3), std=False, prof_nsigma=None, niter=4, sigrej=3.5, bkpts_optimal=True, @@ -1032,9 +1040,24 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, show_resids=False, show_fwhm=False, adderr=0.01, base_var=None, count_scale=None): """ - Perform local sky subtraction, profile fitting, and optimal extraction slit by slit - - IMPROVE THIS DOCSTRING + Perform local sky subtraction, profile fitting, and optimal extraction slit + by slit. Objects are sky/subtracted extracted in order of the highest + average (across all orders) S/N ratio object first, and then for a given + object the highest S/N ratio orders are extracted first. The profile fitting + FWHM are stored and progressively fit as the objects are extracted to + properly ensure that low S/N orders use Gaussian extracted FWHMs from + higher-S/N orders (i.e. in the regime where the data is too noisy for a + non-parametric object profile fit). The FWHM of higher S/N ratio objects are + used for lower S/N ratio objects (note this assumes point sources with FWHM + set by the seeing). + + Note on masking: This routine requires that all masking be performed in the + upstream calling routine (:class:`~pypeit.extraction.Extract`) and thus the left and + right slit edge arrays must only contain these slits. Similarly, the sobjs + object must only include the unmasked (good) objects that are to be + extracted. The number of sobjs objects must equal to an integer multiple of + the number of good slits/orders. The routine will fault if any of these + criteria are not met. Parameters ---------- @@ -1055,19 +1078,17 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, Global sky model produced by global_skysub left : `numpy.ndarray`_ Spatial-pixel coordinates for the left edges of each - order. + order. Shape = (nspec, norders) right : `numpy.ndarray`_ Spatial-pixel coordinates for the right edges of each - order. + order. Shape = (nspec, norders) slitmask : `numpy.ndarray`_ Image identifying the 0-indexed order associated with each pixel. Pixels with -1 are not associatead with any order. - sobjs : :class:`~pypeit.specobjs.SpecoObjs` object + sobjs : :class:`~pypeit.specobjs.SpecObjs` object Object containing the information about the objects found on the slit/order from objfind or ech_objfind - order_vec: `numpy.ndarray`_ - Vector of order numbers spat_pix: `numpy.ndarray`_, optional Image containing the spatial location of pixels. If not input, it will be computed from ``spat_img = @@ -1075,7 +1096,7 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, should generally not be used unless one is extracting 2d coadds for which a rectified image contains sub-pixel spatial information. - shape (nspec, nspat) + shape=(nspec, nspat) fit_fwhm: bool, optional if True, perform a fit to the FWHM of the object profiles to use for non-detected sources @@ -1152,14 +1173,16 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, (does not have the right count levels). In principle this could be improved if the user could pass in a model of what the sky is for near-IR difference imaging + residual subtraction - debug_bkpts: + debug_bkpts : bool, optional + debug show_profile : bool, default=False Show QA for the object profile fitting to the screen. Note that this will show interactive matplotlib plots which will block the execution of the code until the window is closed. show_resids : bool, optional Show the model fits and residuals. - show_fwhm: + show_fwhm : bool, optional + show fwhm adderr : float, default = 0.01 Error floor. The quantity adderr**2*sciframe**2 is added to the variance to ensure that the S/N is never > 1/adderr, effectively setting a floor @@ -1193,7 +1216,7 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, Model inverse variance where ``thismask`` is true. outmask : `numpy.ndarray`_ Model mask where ``thismask`` is true. - sobjs : :class:`~pypeit.specobjs.SpecoObjs` object + sobjs : :class:`~pypeit.specobjs.SpecObjs` object Same object as passed in """ @@ -1210,35 +1233,52 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, ivarmodel = np.copy(sciivar) sobjs = sobjs.copy() - norders = order_vec.size - slit_vec = np.arange(norders) + # Identify the unique SLITIDs and orders in the sobjs object + slitids = np.unique(sobjs.SLITID) # This will also sort the slitids + norders = (np.unique(sobjs.ECH_ORDER)).size # Find the spat IDs - gdslit_spat = np.unique(slitmask[slitmask >= 0]).astype(int) # Unique sorts - #if gdslit_spat.size != norders: - # msgs.error("You have not dealt with masked orders properly") - - #if (np.sum(sobjs.sign > 0) % norders) == 0: - # nobjs = int((np.sum(sobjs.sign > 0)/norders)) - #else: - # msgs.error('Number of specobjs in sobjs is not an integer multiple of the number or ordres!') - - # Set bad obj to -nan + if norders != len(slitids): + msgs.error('The number of orders in the sobjs object does not match the number of good slits in the ' + 'slitmask image! There is a problem with the object/slitmask masking. This routine ' + 'requires that all masking is performed in the calling routine.') + + # Check that the slit edges are masked consistent with the number of orders and the number of unique spatids + nleft = left.shape[1] + nrigh = right.shape[1] + if nleft != nrigh or norders != nleft or norders != nrigh: + msgs.error('The number of left and right edges must be the same as the number of orders. ' + 'There is likely a problem with your masking') + + # Now assign the order_sn, and generate an order_vec aligned with the slitids uni_objid = np.unique(sobjs[sobjs.sign > 0].ECH_OBJID) nobjs = len(uni_objid) - order_snr = np.zeros((norders, nobjs)) - order_snr_gpm = np.ones_like(order_snr) - for iord in range(norders): + order_snr = np.full((norders, nobjs), np.nan) + order_vec = np.zeros(norders, dtype=int) + for islit, slitid in enumerate(slitids): for iobj in range(nobjs): - ind = (sobjs.ECH_ORDERINDX == iord) & (sobjs.ECH_OBJID == uni_objid[iobj]) - # Allow for missed/bad order + ind = (sobjs.SLITID == slitid) & (sobjs.ECH_OBJID == uni_objid[iobj]) + # Check for a missed order and fault if they exist if np.sum(ind) == 0: - order_snr_gpm[iord,iobj] = False - else: - order_snr[iord,iobj] = sobjs[ind].ech_snr + msgs.error('There is a missing order for object {0:d} on slit {1:d}!'.format(iobj, slitid)) + if iobj == 0: + order_vec[islit] = sobjs[ind].ECH_ORDER + order_snr[islit,iobj] = sobjs[ind].ech_snr + + # Enforce that the number of objects in the sobjs object is an integer multiple of the number of good orders + if (np.sum(sobjs.sign > 0) % norders) == 0: + nobjs = int((np.sum(sobjs.sign > 0)/norders)) + else: + msgs.error('Number of specobjs in sobjs is not an integer multiple of the number or orders!') + # Enforce that every object in sobj has an specobj on every good order + if np.any(np.isnan(order_snr)): + msgs.error('There are missing orders for one or more objects in sobjs. There is a problem with how you have ' + 'masked objects in sobjs or slits in slitmask in the calling routine') + + # Compute the average SNR and find the brightest object - snr_bar = np.sum(order_snr,axis=0) / np.sum(order_snr_gpm,axis=0) + snr_bar = np.mean(order_snr,axis=0) srt_obj = snr_bar.argsort()[::-1] ibright = srt_obj[0] # index of the brightest object @@ -1250,8 +1290,7 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, # Print out a status message str_out = '' for iord in srt_order_snr: - if order_snr_gpm[iord,ibright]: - str_out += '{:<8d}{:<8d}{:>10.2f}'.format(slit_vec[iord], order_vec[iord], order_snr[iord,ibright]) + msgs.newline() + str_out += '{:<8d}{:<8d}{:>10.2f}'.format(slitids[iord], order_vec[iord], order_snr[iord,ibright]) + msgs.newline() dash = '-'*27 dash_big = '-'*40 msgs.info(msgs.newline() + 'Reducing orders in order of S/N of brightest object:' + msgs.newline() + dash + @@ -1259,9 +1298,6 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, msgs.newline() + str_out) # Loop over orders in order of S/N ratio (from highest to lowest) for the brightest object for iord in srt_order_snr: - # Is this a bad slit? - if not np.any(order_snr_gpm[iord,:]): - continue order = order_vec[iord] msgs.info("Local sky subtraction and extraction for slit/order: {:d}/{:d}".format(iord,order)) other_orders = (fwhm_here > 0) & np.invert(fwhm_was_fit) @@ -1292,13 +1328,13 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, fwhm_this_ord = np.median(fwhm_here[other_orders]) fwhm_all = np.full(norders,fwhm_this_ord) fwhm_str = 'median ' - indx = (sobjs.ECH_OBJID == uni_objid[iobj]) & (sobjs.ECH_ORDERINDX == iord) + indx = (sobjs.ECH_OBJID == uni_objid[iobj]) & (sobjs.SLITID == slitids[iord]) for spec in sobjs[indx]: spec.FWHM = fwhm_this_ord str_out = '' for slit_now, order_now, snr_now, fwhm_now in zip( - slit_vec[other_orders], order_vec[other_orders], + slitids[other_orders], order_vec[other_orders], order_snr[other_orders,ibright], fwhm_here[other_orders]): str_out += '{:<8d}{:<8d}{:>10.2f}{:>10.2f}'.format(slit_now, order_now, snr_now, fwhm_now) + msgs.newline() @@ -1325,18 +1361,18 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, plt.show() else: # If this is not the brightest object then assign it the FWHM of the brightest object - indx = np.where((sobjs.ECH_OBJID == uni_objid[iobj]) & (sobjs.ECH_ORDERINDX == iord))[0][0] - indx_bri = np.where((sobjs.ECH_OBJID == uni_objid[ibright]) & (sobjs.ECH_ORDERINDX == iord))[0][0] + indx = np.where((sobjs.ECH_OBJID == uni_objid[iobj]) & (sobjs.SLITID == slitids[iord]))[0][0] + indx_bri = np.where((sobjs.ECH_OBJID == uni_objid[ibright]) & (sobjs.SLITID == slitids[iord]))[0][0] spec = sobjs[indx] spec.FWHM = sobjs[indx_bri].FWHM - thisobj = (sobjs.ECH_ORDERINDX == iord) # indices of objects for this slit - thismask = slitmask == gdslit_spat[iord] # pixels for this slit + thisobj = (sobjs.SLITID == slitids[iord]) # indices of objects for this slit + thismask = slitmask == slitids[iord] # pixels for this slit # True = Good, False = Bad for inmask inmask = fullmask.flagged(invert=True) & thismask # Local sky subtraction and extraction skymodel[thismask], objmodel[thismask], ivarmodel[thismask], extractmask[thismask] \ - = local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, + = local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, left[:,iord], right[:,iord], sobjs[thisobj], spat_pix=spat_pix, ingpm=inmask, std=std, bsp=bsp, trim_edg=trim_edg, prof_nsigma=prof_nsigma, niter=niter, @@ -1346,9 +1382,8 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, model_noise=model_noise, debug_bkpts=debug_bkpts, show_resids=show_resids, show_profile=show_profile, adderr=adderr, base_var=base_var, count_scale=count_scale) - # update the FWHM fitting vector for the brighest object - indx = (sobjs.ECH_OBJID == uni_objid[ibright]) & (sobjs.ECH_ORDERINDX == iord) + indx = (sobjs.ECH_OBJID == uni_objid[ibright]) & (sobjs.SLITID == slitids[iord]) fwhm_here[iord] = np.median(sobjs[indx].FWHMFIT) # Did the FWHM get updated by the profile fitting routine in local_skysub_extract? If so, include this value # for future fits @@ -1365,6 +1400,69 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, return skymodel, objmodel, ivarmodel, outmask, sobjs +def convolve_skymodel(input_img, fwhm_map, thismask, subpixel=5, nsample=10): + """Convolve an input image with a Gaussian function that ensures all pixels have + the same FWHM (i.e. the returned image has a spectral resolution corresponding to + the maximum FWHM specified in the input fwhm_map). To speed up the computation, + the input image is uniformly convolved by a grid of FWHM values, and the final + pixel-by-pixel convolved map is an interpolation of the grid point values. + + Args: + input_img (`numpy.ndarray`_): + The science frame, shape = nspec, nspat + fwhm_map (`numpy.ndarray`_): + An array (same shape as input_img), that specifies the FWHM at every pixel + in the image. + thismask (`numpy.ndarray`_): + A boolean mask (True = good), same shape as input_img, of the detector pixels + that fall in a slit and have a measured FWHM. + subpixel (int, optional): + Divide each pixel into this many subpixels to improve the accuracy of the + convolution (a higher number gives a more accurate result, at the expense + of computational time). + nsample (int, optional): + Number of grid points that will be used to evaluate the convolved image. + + Returns: + `numpy.ndarray`_: The convolved input_img, same shape as input_img. + """ + fwhm_to_sig = 2 * np.sqrt(2 * np.log(2)) + # Make a subpixellated input science image + _input_img = np.repeat(input_img, subpixel, axis=0) + _input_msk = np.repeat(thismask, subpixel, axis=0) + # Calculate the excess sigma (in subpixel coordinates) + sig_exc = subpixel * np.sqrt(np.max(fwhm_map[thismask])**2 - fwhm_map[thismask]**2)/fwhm_to_sig + nspec, nspat = _input_img.shape + # We need to loop over a range of kernel widths, and then interpolate. This assumes that the FWHM + # locally around every pixel is roughly constant to within +/-5 sigma of the profile width + kernwids = np.linspace(0.0, np.max(sig_exc), nsample) # Need to include the zero index + # Setup the output array. Note that the first image is the input (i.e. no convolution) + conv_allkern = np.zeros(input_img.shape + (nsample,)) + conv_allkern[:, :, 0] = input_img + for kk in range(nsample): + if kk == 0: + # The first element is the original image + continue + msgs.info(f"Image spectral convolution - Evaluating grid point {kk}/{nsample - 1}") + # Generate a kernel and normalise + kernsize = 2 * int(5 * kernwids[kk] + 0.5) + 1 # Use a Gaussian kernel, covering +/-5sigma + midp = (kernsize - 1) // 2 + xkern = np.arange(kernsize, dtype=int) - midp + kern = np.exp(-0.5 * (xkern / kernwids[kk]) ** 2) + kern = kern / np.sum(kern) + conv_allkern[:, :, kk] = utils.rebinND(utils.convolve_fft(_input_img, kern, _input_msk), input_img.shape) + + # Collect all of the images + msgs.info(f"Collating all convolution steps") + conv_interp = RegularGridInterpolator((np.arange(conv_allkern.shape[0]), np.arange(nspat), kernwids), conv_allkern) + msgs.info(f"Applying the convolution solution") + eval_spec, eval_spat = np.where(thismask) + sciimg_conv = np.copy(input_img) + sciimg_conv[thismask] = conv_interp((eval_spec, eval_spat, sig_exc)) + # Return the convolved image + return sciimg_conv + + def read_userregions(skyreg, nslits, maxslitlength): """ Parse the sky regions defined by the user. The text should be a comma @@ -1446,7 +1544,7 @@ def generate_mask(pypeline, skyreg, slits, slits_left, slits_right, spat_flexure skyreg : list A list of size nslits. Each element contains a numpy array (dtype=bool) where a True value indicates a value that is part of the sky region. - slits : :class:`SlitTraceSet` + slits : :class:`~pypeit.slittrace.SlitTraceSet` Data container with slit trace information slits_left : `numpy.ndarray`_ A 2D array containing the pixel coordinates of the left slit edges diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 7afdf247e5..378e57ce81 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -25,7 +25,7 @@ from pypeit import utils from pypeit import msgs from pypeit import onespec -from pypeit import datamodel + from pypeit.spectrographs.util import load_spectrograph from pypeit import datamodel @@ -33,6 +33,7 @@ ############################## # Telluric model functions # ############################## +ZP_UNIT_CONST = flux_calib.zp_unit_const() # TODO These codes should probably be in a separate qso_pca module. Also pickle functionality needs to be removed. @@ -200,7 +201,8 @@ def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): dwave, dloglam, resln_guess, pix_per_sigma = wvutils.get_sampling(wave_grid) tell_pad_pix = int(np.ceil(10.0 * pix_per_sigma)) - return dict(wave_grid=wave_grid, + + return dict(wave_grid=wave_grid, dloglam=dloglam, resln_guess=resln_guess, pix_per_sigma=pix_per_sigma, @@ -260,7 +262,7 @@ def conv_telluric(tell_model, dloglam, res): general different from the size of the telluric grid (read in by read_telluric_grid above) because it is trimmed to relevant wavelenghts using ind_lower, ind_upper. See eval_telluric below. dloglam (float): - Wavelength spacing of the telluric grid expressed as a a dlog10(lambda), i.e. stored in the + Wavelength spacing of the telluric grid expressed as a dlog10(lambda), i.e. stored in the tell_dict as tell_dict['dloglam'] res (float): Desired resolution expressed as lambda/dlambda. Note that here dlambda is linear, whereas dloglam is @@ -272,13 +274,20 @@ def conv_telluric(tell_model, dloglam, res): """ + pix_per_sigma = 1.0/res/(dloglam*np.log(10.0))/(2.0 * np.sqrt(2.0 * np.log(2))) # number of dloglam pixels per 1 sigma dispersion sig2pix = 1.0/pix_per_sigma # number of sigma per 1 pix - #conv_model = scipy.ndimage.gaussian_filter1d(tell_model, pix) + if sig2pix > 2.0: + msgs.warn('The telluric model grid is not sampled finely enough to properly convolve to the desired resolution. ' + 'Skipping resolution convolution for now. Create a higher resolution telluric model grid') + return tell_model + # x = loglam/sigma on the wavelength grid from -4 to 4, symmetric, centered about zero. x = np.hstack([-1*np.flip(np.arange(sig2pix,4,sig2pix)),np.arange(0,4,sig2pix)]) # g = Gaussian evaluated at x, sig2pix multiplied in to properly normalize the convolution - g = (1.0/(np.sqrt(2*np.pi)))*np.exp(-0.5*(x)**2)*sig2pix + #g = (1.0/(np.sqrt(2*np.pi)))*np.exp(-0.5*np.square(x))*sig2pix + g=np.exp(-0.5*np.square(x)) + g /= g.sum() conv_model = scipy.signal.convolve(tell_model,g,mode='same') return conv_model @@ -529,7 +538,7 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): init_obj = np.array([[np.clip(param + ballsize*(bounds_obj[i][1] - bounds_obj[i][0]) * rng.standard_normal(1)[0], bounds_obj[i][0], bounds_obj[i][1]) for i, param in enumerate(arg_dict['obj_dict']['init_obj_opt_theta'])] for jsamp in range(nsamples)]) - tell_lhs = utils.lhs(7, samples=nsamples) + tell_lhs = utils.lhs(7, samples=nsamples, seed_or_rng=rng) init_tell = np.array([[bounds[-idim][0] + tell_lhs[isamp, idim] * (bounds[-idim][1] - bounds[-idim][0]) for idim in range(7)] for isamp in range(nsamples)]) init = np.hstack((init_obj, init_tell)) @@ -569,7 +578,7 @@ def unpack_orders(sobjs, ret_flam=False): arrays necessary for telluric fitting. Args: - sobjs (obj): + sobjs (:class:`~pypeit.specobjs.SpecObjs`): SpecObjs object ret_flam (bool): If true return the FLAM, otherwise return COUNTS @@ -757,6 +766,7 @@ def init_sensfunc_model(obj_params, iord, wave, counts_per_ang, ivar, gpm, tellm telluric model fit. """ + # Model parameter guess for starting the optimizations flam_true = scipy.interpolate.interp1d(obj_params['std_dict']['wave'].value, obj_params['std_dict']['flux'].value, kind='linear', @@ -771,15 +781,21 @@ def init_sensfunc_model(obj_params, iord, wave, counts_per_ang, ivar, gpm, tellm = flux_calib.compute_zeropoint(wave, N_lam, (gpm & flam_true_gpm), flam_true, tellmodel=tellmodel) + zeropoint_poly = zeropoint_data + 5.0*np.log10(wave) - ZP_UNIT_CONST + if obj_params['log10_blaze_func_per_ang'] is not None: + zeropoint_poly -= 2.5*obj_params['log10_blaze_func_per_ang'] # Perform an initial fit to the sensitivity function to set the starting # point for optimization - pypeitFit = fitting.robust_fit(wave, zeropoint_data, obj_params['polyorder_vec'][iord], - function=obj_params['func'], minx=wave.min(), maxx=wave.max(), + wave_min, wave_max = wave.min(), wave.max() + pypeitFit = fitting.robust_fit(wave, zeropoint_poly, obj_params['polyorder_vec'][iord], + function=obj_params['func'], minx=wave_min, maxx=wave_max, in_gpm=zeropoint_data_gpm, lower=obj_params['sigrej'], upper=obj_params['sigrej'], use_mad=True) - zeropoint_fit = pypeitFit.eval(wave) + zeropoint_fit = flux_calib.eval_zeropoint(pypeitFit.fitc, obj_params['func'], wave, wave_min, wave_max, + log10_blaze_func_per_ang=obj_params['log10_blaze_func_per_ang']) zeropoint_fit_gpm = pypeitFit.bool_gpm + # Polynomial coefficient bounds bounds_obj = [(np.fmin(np.abs(this_coeff)*obj_params['delta_coeff_bounds'][0], obj_params['minmax_coeff_bounds'][0]), @@ -788,15 +804,19 @@ def init_sensfunc_model(obj_params, iord, wave, counts_per_ang, ivar, gpm, tellm # Create the obj_dict obj_dict = dict(wave=wave, wave_min=wave.min(), wave_max=wave.max(), + log10_blaze_func_per_ang=obj_params['log10_blaze_func_per_ang'], exptime=obj_params['exptime'], flam_true=flam_true, flam_true_gpm=flam_true_gpm, func=obj_params['func'], polyorder=obj_params['polyorder_vec'][iord], bounds_obj=bounds_obj, init_obj_opt_theta = pypeitFit.fitc) + if obj_params['debug']: title = 'Zeropoint Initialization Guess for order/det={:d}'.format(iord + 1) # +1 to account 0-index starting flux_calib.zeropoint_qa_plot(wave, zeropoint_data, zeropoint_data_gpm, zeropoint_fit, - zeropoint_fit_gpm, title=title, show=True) + zeropoint_fit_gpm, title=title, show=True) + + return obj_dict, bounds_obj @@ -829,10 +849,11 @@ def eval_sensfunc_model(theta, obj_dict): gpm : `numpy.ndarray`_, bool, shape is the same as obj_dict['wave_star'] Good pixel mask indicating where the model is valid """ - zeropoint = fitting.evaluate_fit(theta, obj_dict['func'], obj_dict['wave'], - minx=obj_dict['wave_min'], maxx=obj_dict['wave_max']) + zeropoint = flux_calib.eval_zeropoint(theta, obj_dict['func'], obj_dict['wave'], + obj_dict['wave_min'], obj_dict['wave_max'], + log10_blaze_func_per_ang=obj_dict['log10_blaze_func_per_ang']) counts_per_angstrom_model = obj_dict['exptime'] \ - * flux_calib.Flam_to_Nlam(obj_dict['wave'],zeropoint) \ + * flux_calib.Flam_to_Nlam(obj_dict['wave'], zeropoint) \ * obj_dict['flam_true'] * obj_dict['flam_true_gpm'] return counts_per_angstrom_model, obj_dict['flam_true_gpm'] @@ -1181,7 +1202,7 @@ def eval_poly_model(theta, obj_dict): def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, - telgridfile, ech_orders=None, polyorder=8, mask_hydrogen_lines=True, + telgridfile, log10_blaze_function=None, ech_orders=None, polyorder=8, mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., resln_guess=None, resln_frac_bounds=(0.5, 1.5), delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), sn_clip=30.0, ballsize=5e-4, only_orders=None, maxiter=3, lower=3.0, @@ -1218,9 +1239,15 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict : :obj:`dict` Dictionary containing the information for the true flux of the standard star. + log10_blaze_function : `numpy.ndarray`_ , optional + The log10 blaze function determined from a flat field image. If this is + passed in the sensitivity function model will be a (parametric) + polynomial fit multiplied into the (non-parametric) + log10_blaze_function. Shape must match ``wave``, i.e. (nspec,) or + (nspec, norddet). telgridfile : :obj:`str` - File containing grid of HITRAN atmosphere models. This file is given by - :func:`~pypeit.spectrographs.spectrograph.Spectrograph.telluric_grid_file`. + File containing grid of HITRAN atmosphere models; see + :class:`~pypeit.par.pypeitpar.TelluricPar`. ech_orders : `numpy.ndarray`_, shape is (norders,), optional If passed, provides the true order numbers for the spectra provided. polyorder : :obj:`int`, optional, default = 8 @@ -1303,7 +1330,6 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, TelObj : :class:`Telluric` Best-fitting telluric model """ - # Turn on disp for the differential_evolution if debug mode is turned on. if debug: disp = True @@ -1339,7 +1365,7 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, # Since we are fitting a sensitivity function, first compute counts per second per angstrom. TelObj = Telluric(wave, counts, counts_ivar, mask_tot, telgridfile, obj_params, - init_sensfunc_model, eval_sensfunc_model, ech_orders=ech_orders, + init_sensfunc_model, eval_sensfunc_model, log10_blaze_function=log10_blaze_function, ech_orders=ech_orders, resln_guess=resln_guess, resln_frac_bounds=resln_frac_bounds, sn_clip=sn_clip, maxiter=maxiter, lower=lower, upper=upper, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, @@ -1814,6 +1840,7 @@ class Telluric(datamodel.DataContainer): .. include:: ../include/class_datamodel_telluric.rst .. todo:: + - List the elements of ``obj_params``. Args: @@ -1829,8 +1856,8 @@ class Telluric(datamodel.DataContainer): Good pixel gpm for the object in question. Same shape as ``wave``. telgridfile (:obj:`str`): - File containing grid of HITRAN atmosphere models. This file is - given by :func:`~pypeit.spectrographs.spectrograph.Spectrograph.telluric_grid_file`. + File containing grid of HITRAN atmosphere models; see + :class:`~pypeit.par.pypeitpar.TelluricPar`. obj_params (:obj:`dict`): Dictionary of parameters for initializing the object model. init_obj_model (callable): @@ -1852,6 +1879,12 @@ class Telluric(datamodel.DataContainer): Where ``obj_dict`` is one of the return values from the ``init_obj_model`` above. See, e.g., :func:`eval_star_model` for a detailed explanation of these paramaters and return values. + log10_blaze_function (`numpy.ndarray`_, optional): + The log10 blaze function determined from a flat field image. If + this is passed in the sensitivity function model will be a + (parametric) polynomial fit multiplied into the (non-parametric) + log10_blaze_function. Shape = (nspec,) or (nspec, norddet), i.e. + the same as ``wave``. ech_orders (`numpy.ndarray`_, optional): If passed the echelle orders will be included in the output data. Must be a numpy array of integers with the shape (norders,) @@ -1903,7 +1936,7 @@ class Telluric(datamodel.DataContainer): option, you will see that the f(x) loss function gets progressively better during the iterations. sticky (:obj:`bool`, optional): - Sticky parameter for the :func:`~pypeit.utils.djs_reject` + Sticky parameter for the :func:`~pypeit.core.pydl.djs_reject` algorithm for iterative model fit rejection. If True then points rejected from a previous iteration are kept rejected, in other words the bad pixel mask is the OR of all previous iterations and @@ -1928,7 +1961,7 @@ class Telluric(datamodel.DataContainer): into the formal errors. In this way, a rejection threshold of i.e. 3-sigma, will always correspond to roughly the same percentile. This renormalization is performed with - :func:`~pypeit.coadd1d.renormalize_errors`, and guarantees that + :func:`~pypeit.core.coadd.renormalize_errors`, and guarantees that rejection is not too agressive in cases where the empirical errors determined from the chi-distribution differ significantly from the formal noise, which is used to determine ``chi``. @@ -2070,6 +2103,7 @@ class Telluric(datamodel.DataContainer): 'flux_in_arr', 'ivar_in_arr', 'mask_in_arr', + 'log10_blaze_func_in_arr', 'nspec_in', 'norders', @@ -2085,6 +2119,7 @@ class Telluric(datamodel.DataContainer): 'flux_arr', 'ivar_arr', 'mask_arr', + 'log10_blaze_func_arr', 'wave_mask_arr', 'ind_lower', @@ -2169,7 +2204,7 @@ def empty_model_table(norders, nspec, n_obj_par=0): description='Maximum wavelength included in the fit')]) def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_model, - eval_obj_model, ech_orders=None, sn_clip=30.0, airmass_guess=1.5, + eval_obj_model, log10_blaze_function=None, ech_orders=None, sn_clip=30.0, airmass_guess=1.5, resln_guess=None, resln_frac_bounds=(0.5, 1.5), pix_shift_bounds=(-5.0, 5.0), pix_stretch_bounds=(0.9,1.1), maxiter=2, sticky=True, lower=3.0, upper=3.0, seed=777, ballsize = 5e-4, tol=1e-3, diff_evol_maxiter=1000, popsize=30, @@ -2215,11 +2250,17 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode self.disp = disp or debug self.sensfunc = sensfunc self.debug = debug + self.log10_blaze_func_in_arr = None # 2) Reshape all spectra to be (nspec, norders) - self.wave_in_arr, self.flux_in_arr, self.ivar_in_arr, self.mask_in_arr, self.nspec_in, \ - self.norders = utils.spec_atleast_2d(wave, flux, ivar, gpm) - + if log10_blaze_function is not None: + self.wave_in_arr, self.flux_in_arr, self.ivar_in_arr, self.mask_in_arr, self.log10_blaze_func_in_arr, \ + self.nspec_in, self.norders = utils.spec_atleast_2d( + wave, flux, ivar, gpm, log10_blaze_function=log10_blaze_function) + else: + self.wave_in_arr, self.flux_in_arr, self.ivar_in_arr, self.mask_in_arr, _, \ + self.nspec_in, self.norders = utils.spec_atleast_2d( + wave, flux, ivar, gpm) # 3) Read the telluric grid and initalize associated parameters wv_gpm = self.wave_in_arr > 1.0 self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), @@ -2235,13 +2276,20 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode # 4) Interpolate the input values onto the fixed telluric wavelength # grid, clip S/N and process inmask - self.flux_arr, self.ivar_arr, self.mask_arr \ + if log10_blaze_function is not None: + self.flux_arr, self.ivar_arr, self.mask_arr, self.log10_blaze_func_arr \ + = coadd.interp_spec(self.wave_grid, self.wave_in_arr, self.flux_in_arr, + self.ivar_in_arr, self.mask_in_arr, log10_blaze_function=self.log10_blaze_func_in_arr, + sensfunc=self.sensfunc) + else: + self.flux_arr, self.ivar_arr, self.mask_arr, _ \ = coadd.interp_spec(self.wave_grid, self.wave_in_arr, self.flux_in_arr, - self.ivar_in_arr, self.mask_in_arr, sensfunc=self.sensfunc) + self.ivar_in_arr, self.mask_in_arr, + sensfunc=self.sensfunc) # This is a hack to get an interpolate mask indicating where wavelengths # are good on each order - _, _, self.wave_mask_arr = coadd.interp_spec(self.wave_grid, self.wave_in_arr, + _, _, self.wave_mask_arr, _ = coadd.interp_spec(self.wave_grid, self.wave_in_arr, np.ones_like(self.flux_in_arr), np.ones_like(self.ivar_in_arr), (self.wave_in_arr > 1.0).astype(float)) @@ -2267,6 +2315,12 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode tellmodel = eval_telluric(self.tell_guess, self.tell_dict, ind_lower=self.ind_lower[iord], ind_upper=self.ind_upper[iord]) + # TODO This is a pretty ugly way to pass in the blaze function. Particularly since now all the other models + # (star, qso, poly) are going to have this parameter set in their obj_params dictionary. + # Is there something more elegant that can be done with e.g. functools.partial? + obj_params['log10_blaze_func_per_ang'] = \ + self.log10_blaze_func_arr[self.ind_lower[iord]:self.ind_upper[iord] + 1, iord] \ + if log10_blaze_function is not None else None obj_dict, bounds_obj \ = init_obj_model(obj_params, iord, self.wave_grid[self.ind_lower[iord]:self.ind_upper[iord]+1], @@ -2371,6 +2425,7 @@ def show_fit_qa(self, iord): plt.title('QA plot for order/det: {:d}/{:d}'.format(iord + 1, self.norders)) # +1 to account 0-index starting plt.show() + def init_output(self): """ Method to initialize the outputs diff --git a/pypeit/core/trace.py b/pypeit/core/trace.py index e0bf0a67d6..fd7d03cbac 100644 --- a/pypeit/core/trace.py +++ b/pypeit/core/trace.py @@ -603,8 +603,7 @@ def follow_centroid(flux, start_row, start_cen, ivar=None, bpm=None, fwgt=None, Object used to flag the feature traces. If None, assessments use a boolean array to flag traces. If not None, errors will be raised if the object cannot - interpret the correct flag names defined. In addition to - flags used by :func:`_recenter_trace_row`, this function + interpret the correct flag names defined. This function uses the DISCONTINUOUS flag. Returns: @@ -911,10 +910,10 @@ def fit_trace(flux, trace_cen, order, ivar=None, bpm=None, trace_bpm=None, weigh maxdev (:obj:`float`, optional): If provided, reject points with `abs(data-model) > maxdev` during the fitting. If None, no points are - rejected. See :func:`pypeit.utils.robust_polyfit_djs`. + rejected. See :func:`~pypeit.core.fitting.robust_fit`. maxiter (:obj:`int`, optional): Maximum number of rejection iterations allowed during the - fitting. See :func:`pypeit.utils.robust_polyfit_djs`. + fitting. See :func:`~pypeit.core.fitting.robust_fit`. niter (:obj:`int`, optional): The number of iterations for this method; i.e., the number of times the two-step fitting algorithm described @@ -929,10 +928,10 @@ def fit_trace(flux, trace_cen, order, ivar=None, bpm=None, trace_bpm=None, weigh if `debug` is true for the plotting. Default is just a running number. xmin (:obj:`float`, optional): - Lower reference for robust_polyfit polynomial fitting. + Lower reference for robust_fit polynomial fitting. Default is to use zero xmax (:obj:`float`, optional): - Upper reference for robust_polyfit polynomial fitting. + Upper reference for robust_fit polynomial fitting. Default is to use the image size in nspec direction flavor (:obj:`str`, optional): Defines the type of fit performed. Only used by QA @@ -1300,11 +1299,11 @@ def peak_trace(flux, ivar=None, bpm=None, trace_map=None, extract_width=None, sm length of the detector. The tuple gives the minimum and maximum in the fraction of the full spectral length (nspec). If None, the full image is collapsed. - peak_thresh (:obj:`float, optional): + peak_thresh (:obj:`float`, optional): The threshold for detecting peaks in the image. See the ``input_thresh`` parameter for :func:`~pypeit.core.arc.detect_lines`. - peak_clip (:obj:`float, optional): + peak_clip (:obj:`float`, optional): Sigma-clipping threshold used to clip peaks with small values; no large values are clipped. If None, no clipping is performed. Generally, if the peak detection algorithm diff --git a/pypeit/core/tracewave.py b/pypeit/core/tracewave.py index c9da4f75be..6715c5d3ef 100644 --- a/pypeit/core/tracewave.py +++ b/pypeit/core/tracewave.py @@ -59,6 +59,9 @@ def tilts_find_lines(arc_spec, slit_cen, tracethresh=10.0, sig_neigh=5.0, nfwhm_ either because the detection wasn't significant enough or the line was too close to a more-significant, neighboring line. """ + # Setup some convenience variables + npix_neigh = nfwhm_neigh * fwhm + # Find peaks tampl_tot, tampl_cont_tot, tcent_tot, twid_tot, _, wgood, arc_cont_sub, nsig_tot \ = arc.detect_lines(arc_spec, sigdetect=np.min([sig_neigh, tracethresh]), fwhm=fwhm, @@ -67,18 +70,11 @@ def tilts_find_lines(arc_spec, slit_cen, tracethresh=10.0, sig_neigh=5.0, nfwhm_ niter_cont=niter_cont, nonlinear_counts=nonlinear_counts, bpm=bpm, debug=debug_peaks) - # good = np.zeros(tampl_tot.size, dtype=bool) - # good[wgood] = True - # arc.find_lines_qa(arc_cont_sub, tcent_tot, tampl_cont_tot, good, bpm=bpm, - # nonlinear=nonlinear_counts) - # Good lines arcdet = tcent_tot[wgood] arc_ampl = tampl_cont_tot[wgood] nsig = nsig_tot[wgood] - npix_neigh = nfwhm_neigh * fwhm - # Determine the best lines to use to trace the tilts aduse = np.zeros(arcdet.size, dtype=bool) # Which lines should be used to trace the tilts aduse[nsig >= tracethresh] = True @@ -530,60 +526,72 @@ def trace_tilts(arcimg, lines_spec, lines_spat, thismask, slit_cen, inmask=None, max_badpix_frac=0.30, tcrude_nave=5, npca=2, coeff_npoly_pca=2, sigrej_pca=2.0, debug_pca=False, show_tracefits=False): """ - Use a PCA model to determine the best object (or slit edge) traces for echelle spectrographs. + Use a PCA model to determine the best object (or slit edge) traces for + echelle spectrographs. Parameters ---------- - arcimg: ndarray, float (nspec, nspat) + arcimg : `numpy.ndarray`_, float (nspec, nspat) Image of arc or sky that will be used for tracing tilts. - lines_spec: ndarray, float (nlines,) - Array containing arc line centroids along the center of the slit for each arc line that will be traced. This is - in pixels in image coordinates. - lines_spat: ndarray, float (nlines,) - Array contianing the spatial position of the center of the slit along which the arc was extracted. This is is in - pixels in image coordinates. - thismask: ndarray, boolean (nspec, nsapt) - Boolean mask image specifying the pixels which lie on the slit/order to search for objects on. - The convention is: True = on the slit/order, False = off the slit/order. This must be the same size as the arcimg. - inmask: float ndarray, default = None, optional + lines_spec : `numpy.ndarray`_, float (nlines,) + Array containing arc line centroids along the center of the slit for + each arc line that will be traced. This is in pixels in image + coordinates. + lines_spat : `numpy.ndarray`_, float (nlines,) + Array contianing the spatial position of the center of the slit along + which the arc was extracted. This is is in pixels in image coordinates. + thismask : `numpy.ndarray`_, boolean (nspec, nsapt) + Boolean mask image specifying the pixels which lie on the slit/order to + search for objects on. The convention is: True = on the slit/order, + False = off the slit/order. This must be the same size as the arcimg. + inmask : float ndarray, default = None, optional Input mask image. - gauss: bool, default = False, optional - If true the code will trace the arc lines usign Gaussian weighted centroiding (trace_gweight) instead of the default, - which is flux weighted centroiding (trace_fweight) - fwhm: float, optional - Expected FWHM of the arc lines. - spat_order: int, default = None, optional - Order of the legendre polynomial that will be fit to the tilts. + gauss : bool, default = False, optional + If true the code will trace the arc lines usign Gaussian weighted + centroiding (trace_gweight) instead of the default, which is flux + weighted centroiding (trace_fweight) + fwhm : float, optional + Expected FWHM of the arc lines. + spat_order : int, default = None, optional + Order of the legendre polynomial that will be fit to the tilts. maxdev_tracefit: float, default = 1.0, optional - Maximum absolute deviation for the arc tilt fits during iterative trace fitting expressed in units of the fwhm. - sigrej_trace: float, default = 3.0, optional - From each line we compute a median absolute deviation of the trace from the polynomial fit. We then - analyze the distribution of maximxum absolute deviations (MADs) for all the lines, and reject sigrej_trace outliers - from that distribution. + Maximum absolute deviation for the arc tilt fits during iterative trace + fitting expressed in units of the fwhm. + sigrej_trace : float, default = 3.0, optional + From each line we compute a median absolute deviation of the trace from + the polynomial fit. We then analyze the distribution of maximxum + absolute deviations (MADs) for all the lines, and reject sigrej_trace + outliers from that distribution. max_badpix_frac: float, default = 0.30, optional - Maximum fraction of total pixels that can be masked by the trace_gweight algorithm - (because the residuals are too large) to still be usable for tilt fitting. - tcrude_nave: int, default = 5, optional - Trace crude is used to determine the initial arc line tilts, which are then iteratively fit. Trace crude - can optionally boxcar smooth the image (along the spatial direction of the image, i.e. roughly along the arc line tilts) - to improve the tracing. + Maximum fraction of total pixels that can be masked by the trace_gweight + algorithm (because the residuals are too large) to still be usable for + tilt fitting. + tcrude_nave : int, default = 5, optional + Trace crude is used to determine the initial arc line tilts, which are + then iteratively fit. Trace crude can optionally boxcar smooth the image + (along the spatial direction of the image, i.e. roughly along the arc + line tilts) to improve the tracing. npca: int, default = 1, optional - Tilts are initially traced and then a PCA is performed. The PCA is used to determine better crutches for a second - round of improved tilt tracing. This parameter is the order of that PCA and determined how much the tilts behavior - is being compressed. npca = 0 would be just using the mean tilt. This PCA is only an intermediate step to - improve the crutches and is an attempt to make the tilt tracing that goes into the final fit more robust. + Tilts are initially traced and then a PCA is performed. The PCA is used + to determine better crutches for a second round of improved tilt + tracing. This parameter is the order of that PCA and determined how much + the tilts behavior is being compressed. npca = 0 would be just using the + mean tilt. This PCA is only an intermediate step to improve the crutches + and is an attempt to make the tilt tracing that goes into the final fit + more robust. coeff_npoly_pca: int, default = 1, optional - Order of polynomial fits used for PCA coefficients fitting for the PCA described above. + Order of polynomial fits used for PCA coefficients fitting for the PCA + described above. sigrej_pca: float, default = 2.0, optional - Significance threhsold for rejection of outliers from fits to PCA coefficients for the PCA described above. + Significance threhsold for rejection of outliers from fits to PCA + coefficients for the PCA described above. show_tracefits: bool, default = False, optional - If true the fits will be shown to each arc line trace by iter_fitting.py + If true the fits will be shown to each arc line trace by iter_fitting.py Returns ------- trace_tilts_dict : dict - See trace_tilts_work for a complete description - + See :func:`trace_tilts_work` for a complete description """ #show_tracefits = True #debug_pca = True @@ -650,7 +658,8 @@ def fit_tilts(trc_tilt_dict, thismask, slit_cen, spat_order=3, spec_order=4, max ---------- trc_tilt_dict: dict Diciontary containing tilt info - slitord_id (int): Slit ID, spatial; only used for QA + slitord_id : int + Slit ID, spatial; only used for QA all_tilts: order: yorder: @@ -685,7 +694,7 @@ def fit_tilts(trc_tilt_dict, thismask, slit_cen, spat_order=3, spec_order=4, max # tilts_spat = trc_tilt_dict['tilts_dspat'][:,use_tilt] # spatial offset from the central trace tilts_spec = trc_tilt_dict['tilts_spec'] # line spectral pixel position from legendre fit evaluated at slit center tilts_mask = trc_tilt_dict['tilts_mask'] # Reflects if trace is on the slit - tilts_mad = trc_tilt_dict['tilts_mad'] # quantitfies aggregate error of this tilt + tilts_mad = trc_tilt_dict['tilts_mad'] # quantifies aggregate error of this tilt use_mask = np.outer(np.ones(nspat, dtype=bool), use_tilt) # tot_mask = tilts_mask & (tilts_err < 900) & use_mask @@ -697,21 +706,20 @@ def fit_tilts(trc_tilt_dict, thismask, slit_cen, spat_order=3, spec_order=4, max # TODO: Make adderr a parameter? Where does this come from? adderr = 0.03 - tilts_sigma = ((tilts_mad < 100.0) & (tilts_mad > 0.0)) \ - * np.sqrt(np.abs(tilts_mad) ** 2 + adderr ** 2) + tilts_sigma = ((tilts_mad < 100.0) & (tilts_mad > 0.0)) * np.sqrt(np.abs(tilts_mad) ** 2 + adderr ** 2) tilts_ivar = utils.inverse((tilts_sigma.flatten() / xnspecmin1) ** 2) pypeitFit = fitting.robust_fit(tilts_spec.flatten() / xnspecmin1, - (tilts.flatten() - tilts_spec.flatten()) / xnspecmin1, - fitxy, x2=tilts_dspat.flatten() / xnspatmin1, - in_gpm=tot_mask.flatten(), invvar=tilts_ivar, - function=func2d, maxiter=maxiter, lower=sigrej, - upper=sigrej, maxdev=maxdev_pix / xnspecmin1, - minx=-0.0, maxx=1.0, minx2=-1.0, maxx2=1.0, - use_mad=False, sticky=False) + (tilts.flatten() - tilts_spec.flatten()) / xnspecmin1, + fitxy, x2=tilts_dspat.flatten() / xnspatmin1, + in_gpm=tot_mask.flatten(), invvar=tilts_ivar, + function=func2d, maxiter=maxiter, lower=sigrej, + upper=sigrej, maxdev=maxdev_pix / xnspecmin1, + minx=-0.0, maxx=1.0, minx2=-1.0, maxx2=1.0, + use_mad=False, sticky=False) fitmask = pypeitFit.bool_gpm.reshape(tilts_dspat.shape) # Compute a rejection mask that we will use later. These are - # locations that were fit but were rejectedK + # locations that were fit but were rejected rej_mask = tot_mask & np.invert(fitmask) # Compute and store the 2d tilts fit delta_tilt_1 = xnspecmin1 * pypeitFit.eval(tilts_spec[tilts_mask] / xnspecmin1, @@ -849,11 +857,11 @@ def fit2tilts(shape, coeff2, func2d, spat_shift=None): Parameters ---------- - shape: tuple of ints, + shape : tuple of ints, shape of image - coeff2: ndarray, float + coeff2 : `numpy.ndarray`_, float result of griddata tilt fit - func2d: str + func2d : str the 2d function used to fit the tilts spat_shift : float, optional Spatial shift to be added to image pixels before evaluation @@ -862,7 +870,7 @@ def fit2tilts(shape, coeff2, func2d, spat_shift=None): Returns ------- - tilts: ndarray, float + tilts : `numpy.ndarray`_, float Image indicating how spectral pixel locations move across the image. This output is used in the pipeline. @@ -1035,15 +1043,18 @@ def arc_tilts_spec_qa(tilts_spec_fit, tilts, tilts_model, tot_mask, rej_mask, rm plt.rcdefaults() -def arc_tilts_spat_qa(tilts_dspat, tilts, tilts_model, tilts_spec_fit, tot_mask, rej_mask, spat_order, spec_order, rms, - fwhm, - setup='A', slitord_id=0, outfile=None, show_QA=False, out_dir=None): +def arc_tilts_spat_qa(tilts_dspat, tilts, tilts_model, tilts_spec_fit, tot_mask, rej_mask, + spat_order, spec_order, rms, fwhm, setup='A', slitord_id=0, outfile=None, + show_QA=False, out_dir=None): + """ + NEEDS A DOC STRING! + """ plt.rcdefaults() plt.rcParams['font.family'] = 'sans-serif' - # Outfil + # Output file method = inspect.stack()[0][3] - if (outfile is None): + if outfile is None: outfile = qa.set_qa_filename(setup, method, slit=slitord_id, out_dir=out_dir) nspat, nuse = tilts_dspat.shape @@ -1082,7 +1093,7 @@ def arc_tilts_spat_qa(tilts_dspat, tilts, tilts_model, tilts_spec_fit, tot_mask, ax.legend(handles=legend_elements) ax.set_title('Tilts vs Fit (spat_order, spec_order)=({:d},{:d}) for slit={:d}: RMS = {:5.3f}, ' 'RMS/FWHM={:5.3f}'.format(spat_order, spec_order, slitord_id, rms, rms / fwhm), fontsize=15) - cb = fig.colorbar(dummie_cax, ticks=lines_spec) + cb = fig.colorbar(dummie_cax, ax=ax, ticks=lines_spec) cb.set_label('Spectral Pixel') # Finish diff --git a/pypeit/core/transform.py b/pypeit/core/transform.py index b7771f0771..d1b916b697 100644 --- a/pypeit/core/transform.py +++ b/pypeit/core/transform.py @@ -158,7 +158,7 @@ def coordinate_transform_2d(coo, matrix, inverse=False): second column (``coo[:,1]``). matrix (`numpy.ndarray`_): The :math:3\times 3` affine-transformation matrix. See - :func:`~pypeit.core.mosaic.affine_transform_matrix`. + :func:`affine_transform_matrix`. inverse (:obj:`bool`, optional): By default, the function performs the *active* transformation; i.e., applying the transformation to the coordinates, moving them within diff --git a/pypeit/core/wave.py b/pypeit/core/wave.py index a8122bba21..bd623d5711 100644 --- a/pypeit/core/wave.py +++ b/pypeit/core/wave.py @@ -29,7 +29,7 @@ def geomotion_calculate(radec, time, longitude, latitude, elevation, refframe): Args: radec (`astropy.coordinates.SkyCoord`_): RA, DEC of source - time (:obj:`astropy.time.Time`): + time (`astropy.time.Time`_): Time of observation longitude (float): Telescope longitude in deg @@ -56,7 +56,7 @@ def geomotion_correct(radec, time, longitude, latitude, elevation, refframe): Args: radec (`astropy.coordinates.SkyCoord`_): RA, DEC of source - time (:obj:`astropy.time.Time`): + time (`astropy.time.Time`_): Time of observation gd_slitord (`numpy.ndarray`_): Array of good slit/order IDs diff --git a/pypeit/core/wavecal/autoid.py b/pypeit/core/wavecal/autoid.py index 41d48dfd48..76b0edd3cf 100644 --- a/pypeit/core/wavecal/autoid.py +++ b/pypeit/core/wavecal/autoid.py @@ -24,6 +24,7 @@ from pypeit.core.wavecal import wv_fitting from pypeit.core.wavecal import wvutils from pypeit.core import arc +from pypeit.core import fitting from pypeit.core import pca from pypeit import utils @@ -32,25 +33,30 @@ from matplotlib import pyplot as plt from matplotlib import gridspec +from mpl_toolkits.axes_grid1 import make_axes_locatable +from matplotlib import colorbar +import matplotlib.colors from matplotlib.backends.backend_pdf import PdfPages from matplotlib.patches import Patch -def arc_fit_qa(waveFit, outfile=None, ids_only=False, title=None, +def arc_fit_qa(waveFit, + outfile=None, ids_only=False, title=None, log=True): """ QA for Arc spectrum Args: waveFit (:class:`pypeit.core.wavecal.wv_fitting.WaveFit`): - outfile (:obj:`str`, optional): Name of output file or 'show' to show on screen + Wavelength solution object + outfile (:obj:`str`, optional): + Name of output file or 'show' to show on screen ids_only (bool, optional): + ?? title (:obj:`str`, optional): + Add a title to the spectrum plot log (:obj:`bool`, optional): If True, use log scaling for the spectrum - - Returns: - """ plt.rcdefaults() plt.rcParams['font.family']= 'serif' @@ -135,8 +141,7 @@ def arc_fit_qa(waveFit, outfile=None, ids_only=False, title=None, # Title if title is not None: - ax_spec.text(0.04, 0.93, title, transform=ax_spec.transAxes, - size='x-large', ha='left')#, bbox={'facecolor':'white'}) + fig.suptitle(title, fontsize='x-large', va='top') if ids_only: plt.tight_layout(pad=0.2, h_pad=0.0, w_pad=0.0) if outfile is None: @@ -171,8 +176,8 @@ def arc_fit_qa(waveFit, outfile=None, ids_only=False, title=None, # Stats wave_soln_fit = waveFit.pypeitfit.eval(waveFit.pixel_fit/waveFit.xnorm)#, 'legendre',minx=fit['fmin'], maxx=fit['fmax']) - ax_fit.text(0.1*len(arc_spec), 0.90*ymin+(ymax-ymin),r'$\Delta\lambda$={:.3f}$\AA$ (per pix)'.format(waveFit.cen_disp), size='small') - ax_fit.text(0.1*len(arc_spec), 0.80*ymin+(ymax-ymin),'RMS={:.3f} (pixels)'.format(waveFit.rms), size='small') + ax_fit.text(0.1, 0.9, r'$\Delta\lambda$={:.3f}$\AA$ (per pix)'.format(waveFit.cen_disp), size='small', transform=ax_fit.transAxes) + ax_fit.text(0.1, 0.8, 'RMS={:.3f} (pixels)'.format(waveFit.rms), size='small', transform=ax_fit.transAxes) # Arc Residuals ax_res = plt.subplot(gs[1,1]) res = waveFit.wave_fit-wave_soln_fit @@ -197,6 +202,96 @@ def arc_fit_qa(waveFit, outfile=None, ids_only=False, title=None, return +def arc_fwhm_qa(fwhmFit, spat_id, slit_txt="slit", outfile=None, show_QA=False): + """ + QA for spectral FWHM fitting + + Args: + fwhmFit (:class:`pypeit.core.fitting.PypeItFit`): + 2D fit (spatial+spectral) to the measured spectral FWHM (usually based on the arc lines). + spat_id (int): + The spatial ID of the slit. It is the spatial midpoint of the slit, + halfway along the spectral direction. + slit_txt (:obj:`str`, optional): + String indicating if the QA should use "slit" (MultiSlit, IFU) or "order" (Echelle) + outfile (:obj:`str`, optional): + Name of output file or 'show' to show on screen + show_QA (bool, optional): + If True, the generated QA will be shown on the screen (default is False) + """ + spec_order, spat_order = (fwhmFit.fitc.shape[0]-1, fwhmFit.fitc.shape[1]-1) + plt.rcdefaults() + plt.rcParams['font.family']= 'serif' + # Calculate the model spectral FWHM at the measured positions, and the RMS of the fit + model = fwhmFit.eval(fwhmFit.xval, fwhmFit.x2) + gpm = (fwhmFit.gpm == 0) + dev = (model-fwhmFit.yval)[gpm] + med = np.median(dev) + rms = 1.4826 * np.median(np.abs(dev-med)) + # Calculate the typical fractional error + dev = (model/fwhmFit.yval)[gpm] - 1 + med = np.median(dev) + rmsfwhm = 1.4826 * np.median(np.abs(dev-med)) + # Determine the unique spatial positions where the spectral FWHM was measured + unq = np.unique(fwhmFit.x2) + colors = plt.cm.Spectral(unq) + spec_vec = np.linspace(0, fwhmFit.xval.max(), 10) + # Begin + plt.close('all') + # Show the fit + fig, ax = plt.subplots(figsize=(6, 9)) + ax.cla() + # Plot this for all spatial locations considered + # ax.scatter(fwhmFit.x2, fwhmFit.yval-model, s=200, c=fwhmFit.xval, cmap='Spectral') + # Plot the model fits with the same colors + for uu in range(unq.size): + # The mask to use for this spatial location + this_fitmask = (fwhmFit.gpm == 1) & (fwhmFit.x2 == unq[uu]) + this_rejmask = (fwhmFit.gpm == 0) & (fwhmFit.x2 == unq[uu]) + # Plot the data + ax.scatter(fwhmFit.xval[this_rejmask], fwhmFit.yval[this_rejmask], s=50, facecolors='none', edgecolors=colors[uu]) + ax.scatter(fwhmFit.xval[this_fitmask], fwhmFit.yval[this_fitmask], s=50, facecolors=colors[uu], edgecolors='none') + this_model = fwhmFit.eval(spec_vec, unq[uu]*np.ones(spec_vec.size)) + ax.plot(spec_vec, this_model, color=colors[uu]) + # Finalise the plot details + mdiff = np.max(model)-np.min(model) + ymin = np.min(model)-0.5*mdiff + ymax = np.max(model)+0.5*mdiff + ax.set_ylim((ymin, ymax)) + ax.set_xlabel('Spectral coordinate (pixels)', fontsize=12) + ax.set_ylabel('Spectral FWHM (pixels)', fontsize=12) + titletxt = f'Spectral FWHM residual map for {slit_txt} {spat_id}\n' \ + f'spat_order, spec_order = {spat_order}, {spec_order}\n' \ + f'rms={rms:.2f}, rms/FWHM={rmsfwhm:.2f}\n' \ + f'filled (unfilled) symbols = included (excluded) in fit' + ax.set_title(titletxt, fontsize=12) + + if unq.size >= 2: + # Make a colorbar to illustrate the spectral FWHM along the slit in the spatial direction + cmap = matplotlib.colors.ListedColormap(colors) + divider = make_axes_locatable(ax) + cax = divider.append_axes("right", size="5%", pad=0.05) + cbar = colorbar.Colorbar(cax, + orientation='vertical', + cmap=cmap, + norm=plt.Normalize(unq[0]-0.5*(unq[1]-unq[0]), unq[-1]+0.5*(unq[-1]-unq[-2]))) + cbar_labels = [f"{uu:.3f}" for uu in unq] + cbar.set_ticks(unq) + cbar.ax.set_yticklabels(cbar_labels, fontsize=10) + cbar.solids.set_edgecolor('black') + cbar.set_label(label='Fraction along the slit in the spatial direction', weight='bold', fontsize=12) + + plt.tight_layout(pad=0.2, h_pad=0.0, w_pad=0.0) + if outfile is not None: + plt.savefig(outfile, dpi=400) + + if show_QA: + plt.show() + + plt.close() + plt.rcdefaults() + + def match_qa(arc_spec, tcent, line_list, IDs, scores, outfile = None, title=None, path=None): """ Parameters @@ -280,12 +375,11 @@ def match_qa(arc_spec, tcent, line_list, IDs, scores, outfile = None, title=None return - - - -def reidentify(spec, spec_arxiv_in, wave_soln_arxiv_in, line_list, nreid_min, det_arxiv=None, detections=None, cc_thresh=0.8,cc_local_thresh = 0.8, - match_toler=2.0, nlocal_cc=11, nonlinear_counts=1e10,sigdetect=5.0,fwhm=4.0, - debug_xcorr=False, debug_reid=False, debug_peaks = False): +def reidentify(spec, spec_arxiv_in, wave_soln_arxiv_in, line_list, + nreid_min, det_arxiv=None, + detections=None, cc_thresh=0.8, cc_local_thresh=0.8, + match_toler=2.0, nlocal_cc=11, nonlinear_counts=1e10, + sigdetect=5.0, fwhm=4.0, debug_xcorr=False, debug_reid=False, debug_peaks = False): """ Determine a wavelength solution for a set of spectra based on archival wavelength solutions Parameters @@ -349,6 +443,12 @@ def reidentify(spec, spec_arxiv_in, wave_soln_arxiv_in, line_list, nreid_min, de debug_reid: bool, default = False Show plots useful for debugging the line reidentification + sigdetect: float, default = 5.0 + Threshold for detecting arcliens + + fwhm: float, default = 4.0 + Full width at half maximum for the arc lines + Returns ------- (detections, spec_cont_sub, patt_dict) @@ -364,6 +464,7 @@ def reidentify(spec, spec_arxiv_in, wave_soln_arxiv_in, line_list, nreid_min, de ---------------- November 2018 by J.F. Hennawi. Built from an initial version of cross_match code written by Ryan Cooke. """ + # TODO -- Break up this morass into multiple methods # Determine the seed for scipy.optimize.differential_evolution optimizer. Just take the sum of all the elements # and round that to an integer @@ -437,6 +538,7 @@ def reidentify(spec, spec_arxiv_in, wave_soln_arxiv_in, line_list, nreid_min, de wvc_arxiv = np.zeros(narxiv, dtype=float) disp_arxiv = np.zeros(narxiv, dtype=float) + # Determine the central wavelength and dispersion of wavelength arxiv for iarxiv in range(narxiv): wvc_arxiv[iarxiv] = wave_soln_arxiv[nspec//2, iarxiv] @@ -549,9 +651,10 @@ def reidentify(spec, spec_arxiv_in, wave_soln_arxiv_in, line_list, nreid_min, de patt_dict_slit['sigdetect'] = sigdetect return detections, spec_cont_sub, patt_dict_slit - # Finalize the best guess of each line - patt_dict_slit = patterns.solve_xcorr(detections, wvdata, det_indx, line_indx, line_cc,nreid_min=nreid_min,cc_local_thresh=cc_local_thresh) + patt_dict_slit = patterns.solve_xcorr( + detections, wvdata, det_indx, line_indx, line_cc, + nreid_min=nreid_min,cc_local_thresh=cc_local_thresh) patt_dict_slit['sign'] = sign # This is not used anywhere patt_dict_slit['bwv'] = np.median(wcen[wcen != 0.0]) patt_dict_slit['bdisp'] = np.median(disp[disp != 0.0]) @@ -590,6 +693,230 @@ def reidentify(spec, spec_arxiv_in, wave_soln_arxiv_in, line_list, nreid_min, de return detections, spec_cont_sub, patt_dict_slit +def match_to_arxiv(lamps:list, spec:np.ndarray, wv_guess:np.ndarray, + spec_arxiv:np.ndarray, wave_arxiv:np.ndarray, nreid_min:int, + match_toler=2.0, nonlinear_counts=1e10, sigdetect=5.0, fwhm=4.0, + debug_peaks:bool=False, use_unknowns:bool=False): + """ + Algorithm to match an input arc spectrum to an archival arc spectrum using a + set wavelength guess for the input. This is an alternative to + shifting/stretching to match to the archival arc spectrum as we (hopefully) + have a good guess of the wavelength solution for the input spectrum. + + Used only for missing orders of echelle spectrographs (so far) + + Args: + lamps (list): + List of lamps used in the arc + spec (`numpy.ndarray`_): + Spectrum to match + wv_guess (`numpy.ndarray`_): + Wavelength solution guess for the input arc spectrum + spec_arxiv (`numpy.ndarray`_): + Archival spectrum to match to + wave_arxiv (`numpy.ndarray`_): + Wavelegnth solution for the archival spectrum + nreid_min (int): + Minimum number of times that a given candidate reidentified line + must be properly matched with a line in the arxiv to be considered a + good reidentification. If there is a lot of duplication in the arxiv + of the spectra in question (i.e. multislit) set this to a number + like 2-4. For echelle this depends on the number of solutions in the + arxiv. For fixed format echelle (ESI, X-SHOOTER, NIRES) set this 1. + For an echelle with a tiltable grating, it will depend on the number + of solutions in the arxiv. + match_toler (float, optional): + Matching tolerance in pixels for a line reidentification. A good + line match must match within this tolerance to the the shifted and + stretched archive spectrum, and the archive wavelength solution at + this match must be within match_toler dispersion elements from the + line in line list. Defaults to 2.0. + nonlinear_counts (float, optional): + For arc line detection: Arc lines above this saturation threshold + are not used in wavelength solution fits because they cannot be + accurately centroided. Defaults to 1e10. + sigdetect (float, optional): + Threshold for detecting arcliens. Defaults to 5.0. + fwhm (float, optional): + Full width at half maximum for the arc lines. Defaults to 4.0. + debug_peaks (bool, optional): + Defaults to False. + use_unknowns (bool, optional): + If True, use the unknowns in the solution (not recommended). + Defaults to False. + + Returns: + tuple: tcent (np.ndarray; centroid of lines), spec_cont_sub (np.ndarray; + subtracted continuum), patt_dict_slit (dict; dictionary on the lines), + tot_line_list (astropy.table.Table; line list) + """ + # Load line list + tot_line_list, _, _ = waveio.load_line_lists(lamps, include_unknown=use_unknowns) + + + # Generate the wavelengths from the line list and sort + wvdata = np.array(tot_line_list['wave'].data) # Removes mask if any + wvdata.sort() + + # Search for lines in the input arc + tcent, ecent, cut_tcent, icut, spec_cont_sub = wvutils.arc_lines_from_spec( + spec, sigdetect=sigdetect, + nonlinear_counts=nonlinear_counts, + fwhm=fwhm, debug = debug_peaks) + # If there are no lines in the input arc, return + if tcent.size == 0: + return None + + # Search for lines in the arxiv arc + tcent_arxiv, ecent_arxiv, cut_tcent_arxiv, icut_arxiv, spec_cont_sub_now = wvutils.arc_lines_from_spec( + spec_arxiv, sigdetect=sigdetect, + nonlinear_counts=nonlinear_counts, fwhm = fwhm, debug = debug_peaks) + # If there are no lines in the arxiv arc, return + if tcent_arxiv.size == 0: + return None + + # Interpolate the input wavelengths + fwv_guess = scipy.interpolate.interp1d(np.arange(len(wv_guess)), wv_guess, + kind='cubic', bounds_error=False, + fill_value='extrapolate') + # Interpolate the arxiv both ways + fpix_arxiv = scipy.interpolate.interp1d(wave_arxiv, np.arange(len(wave_arxiv)), + kind='cubic', bounds_error=False, + fill_value='extrapolate') + fwv_arxiv = scipy.interpolate.interp1d(np.arange(len(wave_arxiv)), wave_arxiv, + kind='cubic', bounds_error=False, + fill_value='extrapolate') + # Find the wavelengths of the input arc lines and then the pixels + wv_cent = fwv_guess(tcent) + pix_arxiv = fpix_arxiv(wv_cent) + + # Other bits + wvc_arxiv = wave_arxiv[wave_arxiv.size//2] + igood = wave_arxiv > 1.0 + disp_arxiv = np.median(wave_arxiv[igood] - np.roll(wave_arxiv[igood], 1)) + + line_indx = np.array([], dtype=int) + det_indx = np.array([], dtype=int) + line_cc = np.array([], dtype=float) + #line_iarxiv = np.array([], dtype=int) + + # Match with tolerance + for ss, ipix_arxiv in enumerate(pix_arxiv): + pdiff = np.abs(ipix_arxiv - tcent_arxiv) + bstpx = np.argmin(pdiff) + # If a match is found within 2 pixels, consider this a successful match + if pdiff[bstpx] < match_toler: + # Using the arxiv arc wavelength solution, search for the nearest line in the line list + bstwv = np.abs(wvdata - fwv_arxiv(tcent_arxiv[bstpx])) + # This is a good wavelength match if it is within match_toler disperion elements + if bstwv[np.argmin(bstwv)] < match_toler*disp_arxiv: + line_indx = np.append(line_indx, np.argmin(bstwv)) # index in the line list array wvdata of this match + det_indx = np.append(det_indx, ss) # index of this line in the detected line array detections + #line_iarxiv = np.append(line_iarxiv,iarxiv) + line_cc = np.append(line_cc,1.) # Fakery + + # Initialise the patterns dictionary, sigdetect not used anywhere + if (len(np.unique(line_indx)) < 3): + patt_dict_slit = patterns.empty_patt_dict(pix_arxiv.size) + patt_dict_slit['sigdetect'] = sigdetect + else: + # Finalize the best guess of each line + patt_dict_slit = patterns.solve_xcorr( + tcent, wvdata, det_indx, line_indx, line_cc, + nreid_min=nreid_min,cc_local_thresh=-1) + patt_dict_slit['bwv'] = wvc_arxiv + patt_dict_slit['bdisp'] = disp_arxiv + patt_dict_slit['sigdetect'] = sigdetect + + return tcent, spec_cont_sub, patt_dict_slit, tot_line_list + + +def map_fwhm(image, gpm, slits_left, slits_right, slitmask, npixel=None, nsample=None, sigdetect=10., specord=1, + spatord=0, fwhm=5., box_rad=3.0, slit_bpm=None): + """ + Map the spectral FWHM at all spectral and spatial locations of all slits, using an input image (usually an arc) + + Args: + image (`numpy.ndarray`_): + Arc image (nspec, nspat) + gpm (`numpy.ndarray`_): + Good pixel mask corresponding to the input arc image (nspec, nspat) + slits_left (`numpy.ndarray`_): + Left slit edges + slits_right (`numpy.ndarray`_): + Right slit edges + slitmask (`numpy.ndarray`_): + 2D array indicating which pixels are on the slit + npixel (int, optional): + Number of spatial detector pixels between each estimate of the FWHM + Only nsample or npixel should be specified. Precedence is given to nsample. + nsample (int, optional): + Number of positions along the spatial direction of the slit to estimate the FWHM. + Only nsample or npixel should be specified. Precedence is given to nsample. + sigdetect (:obj:`float`, optional): + Sigma threshold above fluctuations for arc-line detection. + Used by :func:`~pypeit.core.arc.detect_lines`. + specord (tuple, optional): + The spectral polynomial order to use in the 2D polynomial fit to the + FWHM of the arc lines. See also, spatord. + spatord (tuple, optional): + The spatial polynomial order to use in the 2D polynomial fit to the + FWHM of the arc lines. See also, specord. + fwhm (:obj:`float`, optional): + Number of pixels per FWHM resolution element. + Used by :func:`~pypeit.core.arc.detect_lines`. + box_rad (:obj:`float`, optional): + Half-width of the boxcar (floating-point pixels) in the spatial + direction used to extract the arc. + slit_bpm (`numpy.ndarray`_, bool, optional): + Bad pixel mask for the slits. True = bad. Shape must be (nslits,). Arc + spectra are filled with np.nan for masked slits. + + Returns: + `numpy.ndarray`_: Numpy array of PypeItFit objects that provide the + spectral FWHM (in pixels) given a spectral pixel and the spatial + coordinate (expressed as a fraction along the slit in the spatial + direction) + """ + nslits = slits_left.shape[1] + scale = 2 * np.sqrt(2 * np.log(2)) + _npixel = 10 if npixel is None else npixel # Sample every 10 pixels unless the argument is set (Note: this is only used if nsample is not set) + _ord = (specord, spatord) # The 2D polynomial orders to fit to the resolution map. + _slit_bpm = np.zeros(nslits, dtype=bool) if slit_bpm is None else slit_bpm + + # TODO deal with slits not being defined beyond the slitmask in spectral direction + slit_lengths = np.mean(slits_right-slits_left, axis=0) + resmap = [None for sl in range(nslits)] # Setup the resmap + for sl in range(nslits): + if _slit_bpm[sl]: + msgs.warn(f"Skipping FWHM map computation for masked slit {sl+1}/{nslits}") + # Assign it an empty PypeItFit object so that we can still write to file + resmap[sl] = fitting.PypeItFit() + continue + msgs.info(f"Calculating spectral resolution of slit {sl + 1}/{nslits}") + # Fraction along the slit in the spatial direction to sample the arc line width + nmeas = int(0.5+slit_lengths[sl]/_npixel) if nsample is None else nsample + slitsamp = np.linspace(0.01, 0.99, nmeas) + this_samp, this_cent, this_fwhm = np.array([]), np.array([]), np.array([]) + for ss in range(nmeas): + spat_vec = np.atleast_2d((1-slitsamp[ss]) * slits_left[:, sl] + slitsamp[ss] * slits_right[:, sl]).T + arc_spec, arc_spec_bpm, bpm_mask = arc.get_censpec(spat_vec, slitmask, image, gpm=gpm, box_rad=box_rad, + slit_bpm=np.array([_slit_bpm[sl]]), verbose=False) + if bpm_mask[0]: + msgs.warn('Failed to extract the arc at fractional location {0:.2f} along slit {1:d}'.format(slitsamp[ss], sl+1)) + continue + # Detect lines and store the spectral FWHM + _, _, cent, wdth, _, best, _, nsig = arc.detect_lines(arc_spec.squeeze(), sigdetect=sigdetect, fwhm=fwhm, bpm=arc_spec_bpm.squeeze()) + this_cent = np.append(this_cent, cent[best]) + this_fwhm = np.append(this_fwhm, scale*wdth[best]) # Scale convert sig to spectral FWHM + this_samp = np.append(this_samp, slitsamp[ss]*np.ones(wdth[best].size)) + # Perform a 2D robust fit on the measures for this slit + resmap[sl] = fitting.robust_fit(this_cent, this_fwhm, _ord, x2=this_samp, lower=3, upper=3, function='polynomial2d') + + # Return an array containing the PypeIt fits + return np.array(resmap) + + def measure_fwhm(spec, sigdetect=10., fwhm=5.): """ Measure the arc lines FWHM, i.e, approximate spectral resolution @@ -639,7 +966,7 @@ def set_fwhm(par, measured_fwhm=None): and the measured_fwhm Args: - par (:class:`~pypeit.par.pypeitpar.WaveSolutionPar`): + par (:class:`~pypeit.par.pypeitpar.WavelengthSolutionPar`): Key parameters that drive the behavior of the wavelength-solution algorithms. measured_fwhm (:obj:`float`): @@ -663,7 +990,8 @@ def set_fwhm(par, measured_fwhm=None): return fwhm -def full_template(spec, lamps, par, ok_mask, det, binspectral, nsnippet=2, +# TODO: Docstring is missing many arguments +def full_template(spec, lamps, par, ok_mask, det, binspectral, nsnippet=2, slit_ids=None, measured_fwhms=None, debug_xcorr=False, debug_reid=False, x_percentile=50., template_dict=None, debug=False, nonlinear_counts=1e10): @@ -676,44 +1004,45 @@ def full_template(spec, lamps, par, ok_mask, det, binspectral, nsnippet=2, 3. Loop on snippets of the input spectrum to ID lines using reidentify() 4. Fit with fitting.iterative_fitting() - Args: - spec: ndarray (nspec, nslit) - Spectra to be calibrated - lamps : :obj:`list` - List of arc lamps to be used for wavelength calibration. - E.g., ['ArI','NeI','KrI','XeI'] - par: WavelengthSolutionPar ParSet - Calibration parameters - ok_mask: ndarray, bool - Mask of indices of good slits - det: int - Detector index - binspectral: int - Binning of the input arc in the spectral dimension - nsnippet: int, optional - Number of snippets to chop the input spectrum into when ID'ing lines - This deals with differences due to non-linearity between the template - and input spectrum. - measured_fwhms: ndarray, optional - Array of FWHM (in pixels) measured from the arc lines. Shape (nslit,) - x_percentile: float, optional - Passed to reidentify to reduce the dynamic range of arc line amplitudes - template_dict (dict, optional): Dict containing tempmlate items, largely for development + Parameters + ---------- + spec : `numpy.ndarray`_ + Spectra to be calibrated. Shape is (nspec, nslit). + lamps : :obj:`list` + List of arc lamps to be used for wavelength calibration. + E.g., ['ArI','NeI','KrI','XeI'] + par : :class:`~pypeit.par.pypeitpar.WavelengthSolutionPar` + Calibration parameters + ok_mask : `numpy.ndarray`_ + Mask of indices of good slits + det : int + Detector index + binspectral : int + Binning of the input arc in the spectral dimension + nsnippet : int, optional + Number of snippets to chop the input spectrum into when IDing lines. + This deals with differences due to non-linearity between the template + and input spectrum. + slit_ids: ndarray, optional + Array of slit/order IDs. Shape (nslit,) + measured_fwhms : `numpy.ndarray`_, optional + Array of FWHM (in pixels) measured from the arc lines. Shape (nslit,) + x_percentile : float, optional + Passed to reidentify to reduce the dynamic range of arc line amplitudes + template_dict : dict, optional + Dict containing tempmlate items, largely for development - Returns: - wvcalib: dict - Dict of wavelength calibration solutions + Returns + ------- + wvcalib : dict + Dict of wavelength calibration solutions """ #debug = True #debug_xcorr = True #debug_reid = True # Load line lists - if 'ThAr' in lamps: - line_lists_all = waveio.load_line_lists(lamps) - line_lists = line_lists_all[np.where(line_lists_all['ion'] != 'UNKNWN')] - else: - line_lists = waveio.load_line_lists(lamps) + line_lists, _, _ = waveio.load_line_lists(lamps, include_unknown=False) # Load template if template_dict is None: @@ -748,7 +1077,8 @@ def full_template(spec, lamps, par, ok_mask, det, binspectral, nsnippet=2, if slit not in ok_mask: wvcalib[str(slit)] = None continue - msgs.info("Processing slit {}".format(slit)) + slit_txt = f'slit/order {slit_ids[slit]} ({slit+1}/{nslits})' if slit_ids is not None else f'slit {slit+1}/{nslits}' + msgs.info("Processing " + slit_txt) msgs.info("Using sigdetect = {}".format(sigdetect)) # Grab the observed arc spectrum obs_spec_i = spec[:,slit] @@ -866,8 +1196,11 @@ def full_template(spec, lamps, par, ok_mask, det, binspectral, nsnippet=2, # Finish return wvcalib -def echelle_wvcalib(spec, orders, spec_arxiv, wave_arxiv, lamps, par, ok_mask=None, use_unknowns=True, debug_all=False, - debug_peaks=False, debug_xcorr=False, debug_reid=False, debug_fits=False, nonlinear_counts=1e10): +def echelle_wvcalib(spec, orders, spec_arxiv, wave_arxiv, lamps, par, + ok_mask=None, use_unknowns=True, debug_all=False, + debug_peaks=False, debug_xcorr=False, debug_reid=False, + debug_fits=False, nonlinear_counts=1e10, + redo_slits:list=None): r""" Algorithm to wavelength calibrate echelle data based on a predicted or archived wavelength solution @@ -887,7 +1220,7 @@ def echelle_wvcalib(spec, orders, spec_arxiv, wave_arxiv, lamps, par, ok_mask=No lamps : :obj:`list` List of arc lamps to be used for wavelength calibration. E.g., ['ArI','NeI','KrI','XeI'] - par : :class:`~pypeit.par.pypeitpar.WaveSolutionPar` + par : :class:`~pypeit.par.pypeitpar.WavelengthSolutionPar` Key parameters that drive the behavior of the wavelength-solution algorithms. ok_mask : `numpy.ndarray`, optional @@ -917,6 +1250,9 @@ def echelle_wvcalib(spec, orders, spec_arxiv, wave_arxiv, lamps, par, ok_mask=No For arc line detection: Arc lines above this saturation threshold are not used in wavelength solution fits because they cannot be accurately centroided + redo_slits: list, optional + If provided, only perform the wavelength calibration for the + given slit(s). Returns ------- @@ -949,15 +1285,7 @@ def echelle_wvcalib(spec, orders, spec_arxiv, wave_arxiv, lamps, par, ok_mask=No 'spec array.') # Load the line lists - if 'ThAr' in lamps: - line_lists_all = waveio.load_line_lists(lamps) - line_lists = line_lists_all[np.where(line_lists_all['ion'] != 'UNKNWN')] - unknwns = line_lists_all[np.where(line_lists_all['ion'] == 'UNKNWN')] - else: - line_lists = waveio.load_line_lists(lamps) - unknwns = waveio.load_unknown_list(lamps) - - tot_line_list = astropy.table.vstack([line_lists, unknwns]) if use_unknowns else line_lists + tot_line_list, _, _ = waveio.load_line_lists(lamps, include_unknown=use_unknowns) # Array to hold continuum subtracted arcs spec_cont_sub = np.zeros_like(spec) @@ -969,11 +1297,13 @@ def echelle_wvcalib(spec, orders, spec_arxiv, wave_arxiv, lamps, par, ok_mask=No bad_orders = np.array([], dtype=int) # Reidentify each slit, and perform a fit for iord in range(norders): + if redo_slits is not None and orders[iord] not in redo_slits: + continue # ToDO should we still be populating wave_calib with an empty dict here? if iord not in ok_mask: wv_calib[str(iord)] = None continue - msgs.info('Reidentifying and fitting Order = {0:d}, which is {1:d}/{2:d}'.format(orders[iord], iord, norders - 1)) + msgs.info('Reidentifying and fitting Order = {0:d}, which is {1:d}/{2:d}'.format(orders[iord], iord+1, norders)) sigdetect = wvutils.parse_param(par, 'sigdetect', iord) cc_thresh = wvutils.parse_param(par, 'cc_thresh', iord) rms_threshold = wvutils.parse_param(par, 'rms_threshold', iord) @@ -987,18 +1317,24 @@ def echelle_wvcalib(spec, orders, spec_arxiv, wave_arxiv, lamps, par, ok_mask=No debug_peaks=(debug_peaks or debug_all), debug_xcorr=(debug_xcorr or debug_all), debug_reid=(debug_reid or debug_all)) + # Perform the fit + #if debug_fits or debug_all: + # embed(header='1115 of autoid') # Check if an acceptable reidentification solution was found if not all_patt_dict[str(iord)]['acceptable']: wv_calib[str(iord)] = None bad_orders = np.append(bad_orders, iord) continue - # Perform the fit n_final = wvutils.parse_param(par, 'n_final', iord) - final_fit = wv_fitting.fit_slit(spec_cont_sub[:, iord], all_patt_dict[str(iord)], detections[str(iord)], - tot_line_list, match_toler=par['match_toler'], func=par['func'], - n_first=par['n_first'], - sigrej_first=par['sigrej_first'], n_final=n_final, sigrej_final=par['sigrej_final']) + final_fit = wv_fitting.fit_slit( + spec_cont_sub[:, iord], all_patt_dict[str(iord)], + detections[str(iord)], tot_line_list, + match_toler=par['match_toler'], + func=par['func'], n_first=par['n_first'], + sigrej_first=par['sigrej_first'], + n_final=n_final, + sigrej_final=par['sigrej_final']) # Did the fit succeed? if final_fit is None: @@ -1009,7 +1345,7 @@ def echelle_wvcalib(spec, orders, spec_arxiv, wave_arxiv, lamps, par, ok_mask=No # Is the RMS below the threshold? if final_fit['rms'] > rms_threshold: msgs.warn('---------------------------------------------------' + msgs.newline() + - 'Reidentify report for slit {0:d}/{1:d}:'.format(iord, norders - 1) + msgs.newline() + + 'Reidentify report for slit {0:d}/{1:d}:'.format(iord+1, norders) + msgs.newline() + ' Poor RMS ({0:.3f})! Need to add additional spectra to arxiv to improve fits'.format( final_fit['rms']) + msgs.newline() + '---------------------------------------------------') @@ -1019,15 +1355,20 @@ def echelle_wvcalib(spec, orders, spec_arxiv, wave_arxiv, lamps, par, ok_mask=No # Add the patt_dict and wv_calib to the output dicts wv_calib[str(iord)] = copy.deepcopy(final_fit) if (debug_fits or debug_all): - arc_fit_qa(wv_calib[str(iord)], title='Silt: {}'.format(str(iord))) + arc_fit_qa(wv_calib[str(iord)], title='Silt: {}'.format(str(iord)), log=par['qa_log']) # Print the final report of all lines - report_final(norders, all_patt_dict, detections, wv_calib, ok_mask, bad_orders) + report_final(norders, all_patt_dict, detections, + wv_calib, ok_mask, bad_orders, + redo_slits=redo_slits, orders=orders) return all_patt_dict, wv_calib -def report_final(nslits, all_patt_dict, detections, wv_calib, ok_mask, bad_slits): +def report_final(nslits, all_patt_dict, detections, + wv_calib, ok_mask, bad_slits, + redo_slits:list=None, + orders:np.ndarray=None): """ Print out the final report for wavelength calibration @@ -1044,12 +1385,21 @@ def report_final(nslits, all_patt_dict, detections, wv_calib, ok_mask, bad_slits Mask of indices of good slits bad_slits (ndarray, bool): List of slits that are bad + redo_slits (list, optional): + Report on only these slits + orders (ndarray, optional): + Array of echelle orders to be printed out during the report. """ for slit in range(nslits): # Prepare a message for bad wavelength solutions badmsg = '---------------------------------------------------' + msgs.newline() + \ - 'Final report for slit {0:d}/{1:d}:'.format(slit, nslits) + msgs.newline() + \ - ' Wavelength calibration not performed!' + 'Final report for slit {0:d}/{1:d}:'.format(slit+1, nslits) + msgs.newline() + if orders is not None: + badmsg += f' Order: {orders[slit]}' + msgs.newline() + badmsg += ' Wavelength calibration not performed!' + # Redo? + if redo_slits is not None and orders[slit] not in redo_slits: + continue if slit not in ok_mask or slit in bad_slits or all_patt_dict[str(slit)] is None: msgs.warn(badmsg) continue @@ -1064,9 +1414,8 @@ def report_final(nslits, all_patt_dict, detections, wv_calib, ok_mask, bad_slits # Report cen_wave = wv_calib[st]['cen_wave'] cen_disp = wv_calib[st]['cen_disp'] - msgs.info(msgs.newline() + - '---------------------------------------------------' + msgs.newline() + - 'Final report for slit {0:d}/{1:d}:'.format(slit, nslits - 1) + msgs.newline() + + sreport = str('---------------------------------------------------' + msgs.newline() + + 'Final report for slit {0:d}/{1:d}:'.format(slit+1, nslits) + msgs.newline() + ' Pixels {:s} with wavelength'.format(signtxt) + msgs.newline() + ' Number of lines detected = {:d}'.format(detections[st].size) + msgs.newline() + ' Number of lines that were fit = {:d}'.format( @@ -1075,6 +1424,9 @@ def report_final(nslits, all_patt_dict, detections, wv_calib, ok_mask, bad_slits ' Central dispersion = {:g}A/pix'.format(cen_disp) + msgs.newline() + ' Central wave/disp = {:g}'.format(cen_wave / cen_disp) + msgs.newline() + ' Final RMS of fit = {:g}'.format(wv_calib[st]['rms'])) + if orders is not None: + sreport + msgs.newline()+ f' Echelle order = {orders[slit]}' + msgs.info(msgs.newline() + sreport) class ArchiveReid: @@ -1090,7 +1442,7 @@ class ArchiveReid: lamps : :obj:`list` List of arc lamps to be used for wavelength calibration. E.g., ['ArI','NeI','KrI','XeI'] - par : :class:`~pypeit.par.pypeitpar.WaveSolutionPar` + par : :class:`~pypeit.par.pypeitpar.WavelengthSolutionPar` Key parameters that drive the behavior of the wavelength-solution algorithms. ech_fixed_format: bool @@ -1207,16 +1559,8 @@ def __init__(self, spec, lamps, par, ech_fixed_format=False, ok_mask=None, meas self.sigrej_final= self.par['sigrej_final'] # Load the line lists - if 'ThAr' in self.lamps: - line_lists_all = waveio.load_line_lists(self.lamps) - self.line_lists = line_lists_all[np.where(line_lists_all['ion'] != 'UNKNWN')] - self.unknwns = line_lists_all[np.where(line_lists_all['ion'] == 'UNKNWN')] - else: - self.line_lists = waveio.load_line_lists(self.lamps) - self.unknwns = waveio.load_unknown_list(self.lamps) - - self.tot_line_list = astropy.table.vstack([self.line_lists, self.unknwns]) if self.use_unknowns \ - else self.line_lists + self.tot_line_list, self.line_lists, self.unknwns = waveio.load_line_lists( + lamps, include_unknown=self.use_unknowns) # Read in the wv_calib_arxiv and pull out some relevant quantities # ToDO deal with different binnings! @@ -1248,9 +1592,9 @@ def __init__(self, spec, lamps, par, ech_fixed_format=False, ok_mask=None, meas self.wave_soln_arxiv[:, iarxiv] = self.wv_calib_arxiv[str(iarxiv)]['wave_soln'] # arxiv orders (echelle only) if ech_fixed_format: - arxiv_orders = [] + self.arxiv_orders = [] for iarxiv in range(narxiv): - arxiv_orders.append(self.wv_calib_arxiv[str(iarxiv)]['order']) + self.arxiv_orders.append(self.wv_calib_arxiv[str(iarxiv)]['order']) # orders, _ = self.spectrograph.slit2order(slit_spat_pos) ind_arxiv = np.arange(narxiv, dtype=int) @@ -1265,10 +1609,10 @@ def __init__(self, spec, lamps, par, ech_fixed_format=False, ok_mask=None, meas if slit not in self.ok_mask: self.wv_calib[str(slit)] = None continue - msgs.info('Reidentifying and fitting slit # {0:d}/{1:d}'.format(slit,self.nslits-1)) + msgs.info('Reidentifying and fitting slit # {0:d}/{1:d}'.format(slit+1,self.nslits)) # If this is a fixed format echelle, arxiv has exactly the same orders as the data and so # we only pass in the relevant arxiv spectrum to make this much faster - ind_sp = arxiv_orders.index(orders[slit]) if ech_fixed_format else ind_arxiv + ind_sp = self.arxiv_orders.index(orders[slit]) if ech_fixed_format else ind_arxiv if ech_fixed_format: msgs.info(f'Order: {orders[slit]}') sigdetect = wvutils.parse_param(self.par, 'sigdetect', slit) @@ -1279,7 +1623,8 @@ def __init__(self, spec, lamps, par, ech_fixed_format=False, ok_mask=None, meas # get FWHM for this slit fwhm = set_fwhm(self.par, measured_fwhm=measured_fwhms[slit]) self.detections[str(slit)], self.spec_cont_sub[:,slit], self.all_patt_dict[str(slit)] = \ - reidentify(self.spec[:,slit], self.spec_arxiv[:,ind_sp], self.wave_soln_arxiv[:,ind_sp], + reidentify(self.spec[:,slit], self.spec_arxiv[:,ind_sp], + self.wave_soln_arxiv[:,ind_sp], self.tot_line_list, self.nreid_min, cc_thresh=cc_thresh, match_toler=self.match_toler, cc_local_thresh=self.cc_local_thresh, nlocal_cc=self.nlocal_cc, nonlinear_counts=self.nonlinear_counts, sigdetect=sigdetect, fwhm=fwhm, debug_peaks=self.debug_peaks, debug_xcorr=self.debug_xcorr, @@ -1305,8 +1650,9 @@ def __init__(self, spec, lamps, par, ech_fixed_format=False, ok_mask=None, meas continue # Is the RMS below the threshold? if final_fit['rms'] > rms_threshold: + order_str = '' if orders is None else ', order={}'.format(orders[slit]) msgs.warn('---------------------------------------------------' + msgs.newline() + - 'Reidentify report for slit {0:d}/{1:d}:'.format(slit, self.nslits-1) + msgs.newline() + + 'Reidentify report for slit {0:d}/{1:d}'.format(slit, self.nslits-1) + order_str + msgs.newline() + ' Poor RMS ({0:.3f})! Need to add additional spectra to arxiv to improve fits'.format( final_fit['rms']) + msgs.newline() + '---------------------------------------------------') @@ -1325,6 +1671,25 @@ def __init__(self, spec, lamps, par, ech_fixed_format=False, ok_mask=None, meas def get_results(self): return copy.deepcopy(self.all_patt_dict), copy.deepcopy(self.wv_calib) + def get_arxiv(self, orders): + """ Grab the arxiv spectrum and wavelength solution for the provided orders + + Args: + orders (list, `numpy.ndarray`_): Orders to retrieve + + Returns: + tuple: wavelengths arrays, spec arrays aligned with orders + """ + # Collate + wave_soln_arxiv = [] + arcspec_arxiv = [] + for order in orders: + ind_sp = self.arxiv_orders.index(order) + wave_soln_arxiv.append(self.wave_soln_arxiv[:,ind_sp]) + arcspec_arxiv.append(self.spec_arxiv[:,ind_sp]) + + # Return + return np.stack(wave_soln_arxiv,axis=-1), np.stack(arcspec_arxiv,axis=-1) @@ -1345,7 +1710,8 @@ class HolyGrail: ok_mask : ndarray, optional Array of good slits islinelist : bool, optional - Is lines a linelist (True), or a list of ions (False) + Is lamps a linelist (True), or a list of ions (False) + The former is not recommended except by expert users/developers outroot : str, optional Name of output file debug : bool, optional @@ -1417,25 +1783,21 @@ def __init__(self, spec, lamps, par=None, ok_mask=None, islinelist=False, self._debug = debug self._verbose = verbose - # Load the linelist to be used for pattern matching + # Line list provided? (not recommended) if self._islinelist: self._line_lists = self._lamps self._unknwns = self._lamps[:0].copy() - else: - if 'ThAr' in self._lamps: - line_lists_all = waveio.load_line_lists(self._lamps) - self._line_lists = line_lists_all[np.where(line_lists_all['ion'] != 'UNKNWN')] - self._unknwns = line_lists_all[np.where(line_lists_all['ion'] == 'UNKNWN')] + if self._use_unknowns: + self._tot_list = astropy.table.vstack([self._line_lists, self._unknwns]) else: - restrict = spectrograph if self._par['use_instr_flag'] else None - self._line_lists = waveio.load_line_lists( - self._lamps, restrict_on_instr=restrict) - self._unknwns = waveio.load_unknown_list(self._lamps) - - if self._use_unknowns: - self._tot_list = astropy.table.vstack([self._line_lists, self._unknwns]) + self._tot_list = self._line_lists else: - self._tot_list = self._line_lists + # Load the linelist to be used for pattern matching + restrict = spectrograph if self._par['use_instr_flag'] else None + self._tot_list, self._line_lists, self._unknwns = waveio.load_line_lists( + self._lamps, include_unknown=self._use_unknowns, + restrict_on_instr=restrict) + # Generate the final linelist and sort self._wvdata = np.array(self._tot_list['wave'].data) # Removes mask if any @@ -1536,10 +1898,12 @@ def run_brute(self, min_nlines=10): self._det_weak = {} self._det_stro = {} for slit in range(self._nslit): - msgs.info("Working on slit: {}".format(slit)) if slit not in self._ok_mask: self._all_final_fit[str(slit)] = None + msgs.info('Ignoring masked slit {}'.format(slit+1)) continue + else: + msgs.info("Working on slit: {}".format(slit+1)) # TODO Pass in all the possible params for detect_lines to arc_lines_from_spec, and update the parset # Detect lines, and decide which tcent to use sigdetect = wvutils.parse_param(self._par, 'sigdetect', slit) @@ -1551,7 +1915,7 @@ def run_brute(self, min_nlines=10): # Were there enough lines? This mainly deals with junk slits if self._all_tcent.size < min_nlines: - msgs.warn("Not enough lines to identify in slit {0:d}!".format(slit)) + msgs.warn("Not enough lines to identify in slit {0:d}!".format(slit+1)) self._det_weak[str(slit)] = [None,None] self._det_stro[str(slit)] = [None,None] # Remove from ok mask @@ -1570,9 +1934,10 @@ def run_brute(self, min_nlines=10): # Print preliminary report good_fit[slit] = self.report_prelim(slit, best_patt_dict, best_final_fit) - # Now that all slits have been inspected, cross match to generate a + # Now that all slits have been inspected, cross match (if there are bad fit) to generate a # list of all lines in every slit, and refit all spectra - if self._nslit > 1: + # in self.cross_match() good fits are cross correlate with each other, so we need to have at least 2 good fits + if np.where(good_fit[self._ok_mask])[0].size > 1 and np.any(np.logical_not(good_fit[self._ok_mask])): msgs.info('Checking wavelength solution by cross-correlating with all slits') msgs.info('Cross-correlation iteration #1') @@ -1834,7 +2199,7 @@ def cross_match(self, good_fit, detections): # to be classified as a bad slit here. Is this the behavior we want?? Maybe we should be more # conservative and call a bad any slit which results in an outlier here? good_slits = np.sort(sort_idx[np.unique(slit_ids[gdmsk, :].flatten())]) - bad_slits = np.setdiff1d(np.arange(self._nslit), good_slits, assume_unique=True) + bad_slits = np.setdiff1d(np.arange(self._nslit)[self._ok_mask], good_slits, assume_unique=True) nbad = bad_slits.size if nbad > 0: msgs.info('Working on {:d}'.format(nbad) + ' bad slits: {:}'.format(bad_slits + 1)) @@ -2196,22 +2561,26 @@ def get_use_tcent_old(self, corr, cut=True, arr_err=None, weak=False): """ Grab the lines to use - Args: - corr: int - Set if pixels correlate with wavelength (corr==1) or - anticorrelate (corr=-1) - arr_err: - A list [tcent, ecent] indicating which detection list - should be used. Note that if arr_err is set then the - weak keyword is ignored. - weak: bool, optional - If True, return the weak lines - cut: bool, optional - Cut on the lines according to significance - - Returns: - tuple: arr, err - + Parameters + ---------- + corr : int + Set if pixels correlate with wavelength (corr==1) or + anticorrelate (corr=-1) + cut: bool, optional + Cut on the lines according to significance + arr_err : list, optional + A list [tcent, ecent] indicating which detection list + should be used. Note that if arr_err is set then the + weak keyword is ignored. + weak: bool, optional + If True, return the weak lines + + Returns + ------- + arr : `numpy.ndarray` + ??? + err : `numpy.ndarray` + ??? """ # Decide which array to use if arr_err is None: @@ -2239,17 +2608,22 @@ def get_use_tcent(self, corr, tcent_ecent): """ Grab the lines to use - Args: - corr: int - Set if pixels correlate with wavelength (corr==1) or - anticorrelate (corr=-1) - tcent_ecent: - A list [tcent, ecent] indicating which detection list - should be used. Note that if arr_err is set then the weak - keyword is ignored. - - Returns: - tuple: arr, err + Parameters + ---------- + corr : int + Set if pixels correlate with wavelength (corr==1) or + anticorrelate (corr=-1) + tcent_ecent : list + A list [tcent, ecent] indicating which detection list + should be used. Note that if arr_err is set then the weak + keyword is ignored. + + Returns + ------- + arr : `numpy.ndarray` + ??? + err : `numpy.ndarray` + ??? """ # Return the appropriate tcent tcent, ecent = tcent_ecent[0], tcent_ecent[1] @@ -2259,6 +2633,7 @@ def get_use_tcent(self, corr, tcent_ecent): return (self._npix - 1.0) - tcent[::-1], ecent[::-1] + # TODO: Docstring missing return statement def results_brute(self, tcent_ecent, poly=3, pix_tol=0.5, detsrch=5, lstsrch=5, wavedata=None): """ Need some docs here. I think this routine generates the @@ -2266,21 +2641,32 @@ def results_brute(self, tcent_ecent, poly=3, pix_tol=0.5, detsrch=5, lstsrch=5, Parameters ---------- - tcent_ecent: list of ndarrays, [tcent, ecent] - poly, optional: + tcent_ecent : list + List of `numpy.ndarray`_ objects, [tcent, ecent] + poly : int, optional algorithms to use for pattern matching. Only triangles (3) and quadrangles (4) are supported - pix_tol, optional: + pix_tol : float, optional tolerance that is used to determine if a pattern match is successful (in units of pixels) - detsrch, optional: + detsrch: int, optional Number of lines to search over for the detected lines - lstsrch, optional: + lstsrch : int, optional Number of lines to search over for the detected lines - wavedata, optional: - arrerr, optional: + wavedata : `numpy.ndarray`), optional + Line list; see ``linelist`` argument in, e.g., + :func:`~pypeit.core.wavecal.patterns.triangles`. """ + + # TODO: These imports should go at the top. E.g. + # from pypeit.core.wavecal.patterns import triangles, quadrangles + # You would then assign the relevant function to generate_patterns here. E.g. + # if poly == 3: + # generate_patterns = triangles + # elif poly == 4: + # generate_patterns = quadrangles + # Import the pattern matching algorithms if poly == 3: from pypeit.core.wavecal.patterns import triangles as generate_patterns @@ -2348,22 +2734,31 @@ def solve_slit(self, slit, psols, msols, tcent_ecent, nstore=1, nselw=3, nseld=3 and log10(disp). Then it attempts to fit each value determined (default of 1) to try to figure out if it is a reasonable fit. - Args: - slit: - psols: - msols: - tcent_ecent: list, [tcent, ecent] - nstore: - Number of pattern matches to store and fit - nselw: - All solutions around the best central wavelength - solution within +- nselw are selected to be fit - nseld: - All solutions around the best log10(dispersion) solution - within +- nseld are selected to be fit - - Returns: - tuple: patt_dict, final_dict + Parameters + ---------- + slit : int + Slit ID number + psols : tuple + ?? + msols : tuple + ?? + tcent_ecent: list + List with [tcent, ecent] + nstore : int, optional + Number of pattern matches to store and fit + nselw : int, optional + All solutions around the best central wavelength + solution within +- nselw are selected to be fit + nseld : int, optional + All solutions around the best log10(dispersion) solution + within +- nseld are selected to be fit + + Returns + ------- + patt_dict : dict + ?? + final_dict : dict + ?? """ # Extract the solutions @@ -2502,19 +2897,30 @@ def solve_patterns(self, slit, bestlist, tcent_ecent): Args: slit (int): The ID of the slit - bestlist (list or ndarray): - A 5 element list, each containing a numpy.ndarray, with the following values required for each index: - 0: central wavelength of the pattern - 1: central dispersion of pattern - 2: sign of the pattern (note, sign = 1 [-1] if pixels correlate [anticorrelate] with wavelength - 3: index of the full list of patterns that were created from the detected arc lines - 4: index of the full list of patterns that were created from the line list. + bestlist (list, `numpy.ndarray`_): + A 5 element list, each containing a numpy.ndarray, with the + following values required for each index: + + #. central wavelength of the pattern + + #. central dispersion of pattern + + #. sign of the pattern (note, sign = 1 [-1] if pixels + correlate [anticorrelate] with wavelength + + #. index of the full list of patterns that were created from + the detected arc lines + + #. index of the full list of patterns that were created from + the line list. + tcent_ecent (list): - A list [tcent, ecent] indicating which detection list should be used. Note that if arr_err is set then the weak keyword is ignored. + A list [tcent, ecent] indicating which detection list should be + used. Note that if arr_err is set then the weak keyword is + ignored. Returns: - patt_dict (dict): - Dictionary containing information about the best patterns. + dict: Dictionary containing information about the best patterns. """ @@ -2604,14 +3010,14 @@ def report_prelim(self, slit, best_patt_dict, best_final_fit): # Report on the best preliminary result if best_final_fit is None: msgs.warn('---------------------------------------------------' + msgs.newline() + - 'Preliminary report for slit {0:d}/{1:d}:'.format(slit, self._nslit-1) + msgs.newline() + + 'Preliminary report for slit {0:d}/{1:d}:'.format(slit+1, self._nslit) + msgs.newline() + ' No matches! Attempting to cross match.' + msgs.newline() + '---------------------------------------------------') self._all_patt_dict[str(slit)] = None self._all_final_fit[str(slit)] = None elif best_final_fit['rms'] > wvutils.parse_param(self._par, 'rms_threshold', slit): msgs.warn('---------------------------------------------------' + msgs.newline() + - 'Preliminary report for slit {0:d}/{1:d}:'.format(slit, self._nslit-1) + msgs.newline() + + 'Preliminary report for slit {0:d}/{1:d}:'.format(slit+1, self._nslit) + msgs.newline() + ' Poor RMS ({0:.3f})! Attempting to cross match.'.format(best_final_fit['rms']) + msgs.newline() + '---------------------------------------------------') self._all_patt_dict[str(slit)] = None @@ -2624,7 +3030,7 @@ def report_prelim(self, slit, best_patt_dict, best_final_fit): signtxt = 'anitcorrelate' # Report msgs.info('---------------------------------------------------' + msgs.newline() + - 'Preliminary report for slit {0:d}/{1:d}:'.format(slit, self._nslit-1) + msgs.newline() + + 'Preliminary report for slit {0:d}/{1:d}:'.format(slit+1, self._nslit) + msgs.newline() + ' Pixels {:s} with wavelength'.format(signtxt) + msgs.newline() + ' Number of weak lines = {:d}'.format(self._det_weak[str(slit)][0].size) + msgs.newline() + ' Number of strong lines = {:d}'.format(self._det_stro[str(slit)][0].size) + msgs.newline() + @@ -2644,13 +3050,12 @@ def report_final(self): for slit in range(self._nslit): # Prepare a message for bad wavelength solutions badmsg = '---------------------------------------------------' + msgs.newline() +\ - 'Final report for slit {0:d}/{1:d}:'.format(slit, self._nslit-1) + msgs.newline() +\ - ' Wavelength calibration not performed!' + 'Final report for slit {0:d}/{1:d}:'.format(slit+1, self._nslit) + msgs.newline() if slit not in self._ok_mask: - msgs.warn(badmsg) + msgs.warn(badmsg + 'Masked slit ignored') continue if self._all_patt_dict[str(slit)] is None: - msgs.warn(badmsg) + msgs.warn(badmsg + ' Wavelength calibration not performed!') continue st = str(slit) if self._all_patt_dict[st]['sign'] == +1: @@ -2663,7 +3068,7 @@ def report_final(self): centdisp = abs(centwave-tempwave) msgs.info(msgs.newline() + '---------------------------------------------------' + msgs.newline() + - 'Final report for slit {0:d}/{1:d}:'.format(slit, self._nslit-1) + msgs.newline() + + 'Final report for slit {0:d}/{1:d}:'.format(slit+1, self._nslit) + msgs.newline() + ' Pixels {:s} with wavelength'.format(signtxt) + msgs.newline() + ' Number of weak lines = {:d}'.format(self._det_weak[str(slit)][0].size) + msgs.newline() + ' Number of strong lines = {:d}'.format(self._det_stro[str(slit)][0].size) + msgs.newline() + diff --git a/pypeit/core/wavecal/defs.py b/pypeit/core/wavecal/defs.py index c7557345be..2038f90490 100644 --- a/pypeit/core/wavecal/defs.py +++ b/pypeit/core/wavecal/defs.py @@ -66,6 +66,7 @@ def __init__(self): #('OH_FIRE_Echelle', 'Comment'), #('Ar_IR_GNIRS', 'Comment'), ('Ar', 'This is for Ar_IR_GNIRS which should specify the real ion'), + ('NeII', 'Comment'), ]) super(LinesBitMask, self).__init__(list(mask.keys()), descr=list(mask.values())) @@ -115,6 +116,8 @@ def lines(): line_dict['OH_MOSFIRE_H'] = 2 ** 28 line_dict['OH_MOSFIRE_K'] = 2 ** 29 line_dict['ThAr_XSHOOTER_UVB'] = 2**30 + line_dict['NeII'] = 2 ** 31 + line_dict['FeAr'] = 2 ** 32 # return line_dict diff --git a/pypeit/core/wavecal/echelle.py b/pypeit/core/wavecal/echelle.py index 41349db2ba..cfeab522fd 100644 --- a/pypeit/core/wavecal/echelle.py +++ b/pypeit/core/wavecal/echelle.py @@ -17,13 +17,14 @@ from pypeit import data -def predict_ech_order_coverage(angle_fits_params, xd_angle_coeffs, xdisp, xdangle, norders, pad=0): +def predict_ech_order_coverage(angle_fits_params, xd_angle_coeffs, + xdisp, xdangle, norders, pad=0): """ Predict the coverage of orders in the echelle spectrum using the disperser dependent fits of the reddest order as a function of xdangle. Args: - angle_fits_params (astropy.table.Table): + angle_fits_params (`astropy.table.Table`_): Table holding the arxiv parameters xd_angle_coeffs Table holding the arxiv data @@ -56,14 +57,14 @@ def predict_ech_wave_soln(angle_fits_params, ech_angle_coeffs, ech_angle, order_ wavelength solution coefficients vs echelle angle at the given echelle angle. Args: - angle_fits_params (astropy.table.Table): + angle_fits_params (`astropy.table.Table`_): Table holding the parameters governing the echelle angle fits - ech_angle_coeffs (numpy.ndarray): + ech_angle_coeffs (`numpy.ndarray`_): Array holding the polynomial coefficients for the fits of the wavelength solution polynomial coefficients vs echelle angle. ech_angle (float): Echelle angle - order_vec (numpy.ndarray): + order_vec (`numpy.ndarray`_): Array of order numbers for the deisred predicted spectrum. Shape = (norders,) nspec (int): Number of spectral pixels in the echelle spectrum @@ -165,7 +166,8 @@ def predict_ech_arcspec(angle_fits_file, composite_arc_file, echangle, xdangle, return order_vec_guess, wave_soln_guess, arcspec_guess -def identify_ech_orders(arcspec, echangle, xdangle, dispname, angle_fits_file, +def identify_ech_orders(arcspec, echangle, xdangle, dispname, + angle_fits_file, composite_arc_file, debug=False, pad=3): """ Identify the orders in the echelle spectrum via cross correlation with the best guess predicted arc based @@ -205,7 +207,8 @@ def identify_ech_orders(arcspec, echangle, xdangle, dispname, angle_fits_file, # Predict the echelle order coverage and wavelength solution order_vec_guess, wave_soln_guess_pad, arcspec_guess_pad = predict_ech_arcspec( - angle_fits_file, composite_arc_file, echangle, xdangle, dispname, nspec, norders, pad=pad) + angle_fits_file, composite_arc_file, echangle, xdangle, dispname, + nspec, norders, pad=pad) norders_guess = order_vec_guess.size # Since we padded the guess we need to pad the data to the same size @@ -219,16 +222,20 @@ def identify_ech_orders(arcspec, echangle, xdangle, dispname, angle_fits_file, percent_ceil=50.0, sigdetect=5.0, sig_ceil=10.0, fwhm=4.0, debug=debug) # Finish - x_ordr_shift = shift_cc / nspec ordr_shift = int(np.round(shift_cc / nspec)) spec_shift = int(np.round(shift_cc - ordr_shift * nspec)) - msgs.info('Shift in order number between prediction and reddest order: {:.3f}'.format(ordr_shift + pad)) + msgs.info('Shift in order number between prediction and reddest order: {:.3f}'.format( + ordr_shift + pad)) msgs.info('Shift in spectral pixels between prediction and data: {:.3f}'.format(spec_shift)) - order_vec = order_vec_guess[-1] - ordr_shift + np.arange(norders)[::-1] + # Assign + order_vec = order_vec_guess[0] + ordr_shift - np.arange(norders) ind = np.isin(order_vec_guess, order_vec, assume_unique=True) + #if debug: + # embed(header='identify_ech_orders 232 of echelle.py') + # Return return order_vec, wave_soln_guess_pad[:, ind], arcspec_guess_pad[:, ind] diff --git a/pypeit/core/wavecal/patterns.py b/pypeit/core/wavecal/patterns.py index 54afe12892..fd03c4858b 100644 --- a/pypeit/core/wavecal/patterns.py +++ b/pypeit/core/wavecal/patterns.py @@ -1,4 +1,6 @@ """ Module for finding patterns in arc line spectra + +.. include:: ../include/links.rst """ import numpy as np import scipy.ndimage @@ -620,19 +622,28 @@ def empty_patt_dict(nlines): return patt_dict -def solve_xcorr(detlines, linelist, dindex, lindex, line_cc, nreid_min = 4, cc_local_thresh = 0.8): +def solve_xcorr(detlines, linelist, dindex, lindex, line_cc, + nreid_min:int=4, cc_local_thresh:float=0.8): """ Given a starting solution, find the best match for all detlines Parameters ---------- - detlines : ndarray + detlines : `numpy.ndarray`_ list of detected lines in pixels (sorted, increasing) - linelist : ndarray + linelist : `numpy.ndarray`_ list of lines that should be detected (sorted, increasing) - dindex : ndarray + dindex : `numpy.ndarray`_ Index array of all detlines (pixels) used in each triangle - lindex : ndarray + lindex : `numpy.ndarray`_ Index array of the assigned line (wavelengths)to each index in dindex + line_cc : `numpy.ndarray`_ + local cross correlation coefficient computed for each line + cc_local_thresh : float, default = 0.8, optional + Threshold to satisy for local cross-correlation + nreid_min: int, default = 4, optional + Minimum number of matches + to receive a score of 'Perfect' or 'Very Good' + Passed to score_xcorr() Returns ------- @@ -709,9 +720,11 @@ def score_xcorr(counts, cc_avg, nreid_min = 4, cc_local_thresh = -1.0): counts. The more different wavelengths that are attributed to the same detected line (i.e. not ideal) the longer the counts list will be. - nmin_match: int, default = 4, optional - Minimum number of slits/solutions that have to have been matched + nreid_min: int, default = 4, optional + Minimum number of matches to receive a score of 'Perfect' or 'Very Good' + cc_local_thresh: float, default = -1.0, optional + What does this do?? Returns ------- diff --git a/pypeit/core/wavecal/spectrographs/templ_keck_lris.py b/pypeit/core/wavecal/spectrographs/templ_keck_lris.py deleted file mode 100644 index 87a69530ba..0000000000 --- a/pypeit/core/wavecal/spectrographs/templ_keck_lris.py +++ /dev/null @@ -1,36 +0,0 @@ -""" Generate the wavelength templates for Keck/LRIS""" -import os - -from pypeit.core.wavecal import templates - - -# Keck/DEIMOS - -def keck_lris_red_mark4_R400(overwrite=False): - binspec = 1 - outroot = 'keck_lris_red_mark4_R400.fits' - # PypeIt fits - wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'Mark4', 'R400') - - basefiles = ['MasterWaveCalib_A_1_01_long.fits', - 'MasterWaveCalib_A_1_01_sunil.fits'] - wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] - # Snippets - ifiles = [0,1] - slits = [2048, 2045] - wv_cuts = [9320.] - assert len(wv_cuts) == len(slits)-1 - # det_dict - det_cut = None - # - templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, - ifiles=ifiles, det_cut=det_cut, chk=True, - normalize=True, lowredux=False, - subtract_conti=True, overwrite=overwrite, - shift_wave=True) - - - -# Run em -if __name__ == '__main__': - keck_lris_red_mark4_R400()#overwrite=True) diff --git a/pypeit/core/wavecal/spectrographs/templ_keck_lris_blue.py b/pypeit/core/wavecal/spectrographs/templ_keck_lris_blue.py new file mode 100644 index 0000000000..dae1288d2c --- /dev/null +++ b/pypeit/core/wavecal/spectrographs/templ_keck_lris_blue.py @@ -0,0 +1,80 @@ +""" Generate the wavelength templates for Keck/LRIS Blue""" +import os + +from pypeit.core.wavecal import templates + + +def keck_lris_blue_B300_5000(overwrite=False): + binspec = 1 + outroot = 'keck_lris_blue_B300_5000_d680_ArCdHgKrNeXeZnFeAr.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_blue', 'B300_5000') + + basefiles = ['WaveCalib_A_0_DET01_S1676.fits', 'WaveCalib_A_0_DET01_S0947.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1] + slits = [1676, 947] + wv_cuts = [6830.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_blue_B400_3400(overwrite=False): + binspec = 1 + outroot = 'keck_lris_blue_B400_3400_d560_ArCdHgNeZnFeAr.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_blue', 'B400_3400') + + basefiles = ['WaveCalib_A_0_DET02_S1642.fits', 'WaveCalib_A_0_DET01_S1753.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1] + slits = [1642, 1753] + wv_cuts = [4970.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_blue_B1200_3400(overwrite=False): + binspec = 1 + outroot = 'keck_lris_blue_B1200_3400_d560_ArCdHgNeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_blue', 'B1200_3400') + + basefiles = ['WaveCalib_A_0_DET01_S1252.fits', 'WaveCalib_A_0_DET02_S0985.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1] + slits = [1252, 985] + wv_cuts = [3800.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=False, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + +# Run em +if __name__ == '__main__': + # keck_lris_blue_B300_5000(overwrite=False) + # keck_lris_blue_B400_3400(overwrite=False) + keck_lris_blue_B1200_3400(overwrite=False) + diff --git a/pypeit/core/wavecal/spectrographs/templ_keck_lris_red.py b/pypeit/core/wavecal/spectrographs/templ_keck_lris_red.py new file mode 100644 index 0000000000..574bb7142d --- /dev/null +++ b/pypeit/core/wavecal/spectrographs/templ_keck_lris_red.py @@ -0,0 +1,248 @@ +""" Generate the wavelength templates for Keck/LRIS RED""" +import os + +from pypeit.core.wavecal import templates + + +def keck_lris_red_mark4_R400(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_mark4_R400.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'Mark4', 'R400') + + basefiles = ['MasterWaveCalib_A_1_01_long.fits', + 'MasterWaveCalib_A_1_01_sunil.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1] + slits = [2048, 2045] + wv_cuts = [9320.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_R150_7500(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_R150_7500_ArCdHgNeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red', 'R150_7500') + + basefiles = ['WaveCalib_A_0_DET02_S0302.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0] + slits = [302] + wv_cuts = [] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_R300_5000(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_R300_5000_ArCdHgKrNeXeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red', 'R300_5000') + + basefiles = ['WaveCalib_A_0_DET02_S0309.fits', 'WaveCalib_A_0_DET02_S1045.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1] + slits = [309, 1045] + wv_cuts = [5680.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + +def keck_lris_red_R400_8500(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_R400_8500_ArCdHgKrNeXeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red', 'R400_8500') + + basefiles = ['WaveCalib_A_0_DET01_S1549.fits', 'WaveCalib_A_0_DET02_S0876.fits', 'WaveCalib_A_0_DET01_S0694.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1,2] + slits = [1549, 876, 694] + wv_cuts = [5510., 6800.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_R600_5000(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_R600_5000_ArCdHgKrNeXeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red', 'R600_5000') + + basefiles = ['WaveCalib_A_0_DET01_1783.fits','WaveCalib_A_0_DET01_S1781.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1] + slits = [1783,1781] + wv_cuts = [6800.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_R600_7500(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_R600_7500_ArCdHgKrNeXeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red', 'R600_7500') + + basefiles = ['WaveCalib_A_0_DET02_S0302.fits', 'WaveCalib_A_0_DET02_S1077.fits', 'WaveCalib_A_0_DET01_S0757.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1,2] + slits = [302, 1077, 757] + wv_cuts = [7416., 8560.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_R600_10000(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_R600_10000_ArCdHgKrNeXeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red', 'R600_10000') + + basefiles = ['WaveCalib_A_0_DET01_S0305.fits', 'WaveCalib_A_0_DET01_S0577.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1] + slits = [305, 577] + wv_cuts = [8170.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_R831_8200(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_R831_8200_ArCdHgKrNeXeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red', 'R831_8200') + + basefiles = ['WaveCalib_A_0_DET02_S0328.fits', 'WaveCalib_A_0_DET02_S1090.fits', 'WaveCalib_A_0_DET01_S0862.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1,2] + slits = [328, 1090, 862] + wv_cuts = [7780., 9080.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + +def keck_lris_red_R900_5500(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_R900_5500_ArCdHgNeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red', 'R900_5500') + + basefiles = ['WaveCalib_A_0_DET02_S1723.fits', 'WaveCalib_A_0_DET02_S0285.fits', 'WaveCalib_A_0_DET01_S1240.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1,2] + slits = [1723,285,1240] + wv_cuts = [5700., 6800.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_R1200_7500(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_R1200_7500_ArCdHgKrNeXeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red', 'R1200_7500') + + basefiles = ['WaveCalib_A_0_DET02_S0046.fits', 'WaveCalib_A_0_DET01_S0290.fits', 'WaveCalib_A_0_DET01_S0899.fits', + 'WaveCalib_A_0_DET01_S1134.fits', 'WaveCalib_A_0_DET02_S0125.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1,2,3,4] + slits = [46,290,899,1134,125] + wv_cuts = [5700., 6800., 7800., 8700.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + +# Run em +if __name__ == '__main__': + # keck_lris_red_mark4_R400()#overwrite=True) + # keck_lris_red_R150_7500(overwrite=False) + # keck_lris_red_R300_5000(overwrite=False) + # keck_lris_red_R400_8500(overwrite=False) + # keck_lris_red_R600_5000(overwrite=False) + # keck_lris_red_R600_7500(overwrite=False) + # keck_lris_red_R600_10000(overwrite=False) + # keck_lris_red_R831_8200(overwrite=False) + # keck_lris_red_R900_5500(overwrite=False) + keck_lris_red_R1200_7500(overwrite=False) + diff --git a/pypeit/core/wavecal/spectrographs/templ_keck_lris_red_orig.py b/pypeit/core/wavecal/spectrographs/templ_keck_lris_red_orig.py new file mode 100644 index 0000000000..64aaf23328 --- /dev/null +++ b/pypeit/core/wavecal/spectrographs/templ_keck_lris_red_orig.py @@ -0,0 +1,229 @@ +""" Generate the wavelength templates for Keck/LRIS RED ORIG""" +import os + +from pypeit.core.wavecal import templates + + + +def keck_lris_red_orig_R150_7500(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_orig_R150_7500_ArHgNe.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red_orig', 'R150_7500_orig') + + basefiles = ['WaveCalib_A_0_DET01_S0523.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0] + slits = [523] + wv_cuts = [] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_orig_R300_5000(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_orig_R300_5000_ArCdHgNeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red_orig', 'R300_5000_orig') + + basefiles = ['WaveCalib_A_0_DET01_S1456.fits', 'WaveCalib_A_0_DET01_S0783.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1] + slits = [1456, 783] + wv_cuts = [5377.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + +def keck_lris_red_orig_R400_8500(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_orig_R400_8500_ArCdHgNeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red_orig', 'R400_8500_orig') + + basefiles = ['WaveCalib_A_0_DET01_S0180.fits', 'WaveCalib_A_0_DET01_S0131.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1] + slits = [180,131] + wv_cuts = [7330.] + assert len(wv_cuts) == len(slits)-1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_orig_R600_5000(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_orig_R600_5000_ArCdHgNeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red_orig', 'R600_5000_orig') + + basefiles = ['WaveCalib_A_0_DET01_S0038.fits', 'WaveCalib_A_0_DET01_S0258.fits', 'WaveCalib_A_0_DET01_S0596.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0, 1, 2] + slits = [38, 258, 596] + wv_cuts = [6450., 6825.] + assert len(wv_cuts) == len(slits) - 1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_orig_R600_7500(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_orig_R600_7500_ArCdHgNeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red_orig', 'R600_7500_orig') + + basefiles = ['WaveCalib_A_0_DET01_S1179.fits', 'WaveCalib_A_0_DET01_S0476.fits', + 'WaveCalib_A_0_DET01_S0741.fits', 'WaveCalib_A_0_DET01_S1872.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0, 1, 2,3] + slits = [1179, 476, 741, 1872] + wv_cuts = [5250., 7585., 9480.] + assert len(wv_cuts) == len(slits) - 1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_orig_R600_10000(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_orig_R600_10000_ArCdHgNeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red_orig', 'R600_10000_orig') + + basefiles = ['WaveCalib_A_0_DET01_S1567.fits', 'WaveCalib_A_0_DET01_S0543.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0, 1] + slits = [1567, 543] + wv_cuts = [9030.] + assert len(wv_cuts) == len(slits) - 1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_orig_R831_8200(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_orig_R831_8200_ArCdHgNeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red_orig', 'R831_8200_orig') + + basefiles = ['WaveCalib_A_0_DET01_S0216.fits', 'WaveCalib_A_0_DET01_S1025.fits', 'WaveCalib_A_0_DET01_S1783.fits', + 'WaveCalib_A_0_DET01_S0757.fits', 'WaveCalib_A_0_DET01_S1332.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0, 1, 2, 3, 4] + slits = [216, 1025, 1783, 757, 1332] + wv_cuts = [6060., 6290., 7100., 8330.] + assert len(wv_cuts) == len(slits) - 1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_orig_R900_5500(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_orig_R900_5500_ArCdHgNeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red_orig', 'R900_5500_orig') + + basefiles = ['WaveCalib_A_0_DET01_S1966.fits', 'WaveCalib_A_0_DET01_S1864.fits', 'WaveCalib_A_0_DET01_S0590.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0, 1, 2] + slits = [1966, 1864, 590] + wv_cuts = [6195., 6630.] + assert len(wv_cuts) == len(slits) - 1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +def keck_lris_red_orig_R1200_7500(overwrite=False): + binspec = 1 + outroot = 'keck_lris_red_orig_R1200_7500_ArCdHgNeZn.fits' + # PypeIt fits + wpath = os.path.join(templates.template_path, 'Keck_LRIS', 'keck_lris_red_orig', 'R1200_7500_orig') + + basefiles = ['WaveCalib_A_0_DET01_S0997.fits','WaveCalib_A_0_DET01_S0967.fits', 'WaveCalib_A_0_DET01_S0502.fits', + 'WaveCalib_A_0_DET01_S1161.fits', 'WaveCalib_A_0_DET01_S0473.fits'] + wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + # Snippets + ifiles = [0,1,2,3,4] + slits = [997, 967, 502, 1161, 473] + wv_cuts = [5200., 6348., 7600., 8648.] + assert len(wv_cuts) == len(slits) - 1 + # det_dict + det_cut = None + # + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + ifiles=ifiles, det_cut=det_cut, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + +# Run em +if __name__ == '__main__': + # keck_lris_red_orig_R300_5000(overwrite=False) + keck_lris_red_orig_R150_7500(overwrite=False) + # keck_lris_red_orig_R400_8500(overwrite=False) + # keck_lris_red_orig_R600_5000(overwrite=False) + # keck_lris_red_orig_R600_7500(overwrite=False) + # keck_lris_red_orig_R600_10000(overwrite=False) + # keck_lris_red_orig_R831_8200(overwrite=False) + # keck_lris_red_orig_R900_5500(overwrite=False) + # keck_lris_red_orig_R1200_7500(overwrite=False) + + diff --git a/pypeit/core/wavecal/spectrographs/templ_mdm_modspec_echelle.py b/pypeit/core/wavecal/spectrographs/templ_mdm_modspec_echelle.py new file mode 100644 index 0000000000..f50edf02a4 --- /dev/null +++ b/pypeit/core/wavecal/spectrographs/templ_mdm_modspec_echelle.py @@ -0,0 +1,49 @@ +""" Generate the wavelength templates for MDM/Modspec """ + +import os + +from pypeit.core.wavecal import templates +import numpy as np + +def mdm_modspec_echelle(overwrite=True): + """mdm_modspec_echelle Template from the 1200/5100 grism [INSERT UNITS HERE] + + Args: + overwrite: bool, optional + Overwrite the existing file? [Default: False] + """ + + binspec = 1 + outroot = 'mdm_modspec_echelle_Ar.fits' + + # PypeIt fits + ##IF could find dev files/folder system, would set up properly and somewhat like this: + ## wpath = os.path.join(templates.template_path, 'mdm_modspec_echelle') + ##basefiles = ['MasterWaveCalib_ArI_5100.fits','MasterWaveCalib_NeI_5100.fits','MasterWaveCalib_XeI_5100.fits'] + ##wfiles = [os.path.join(wpath, basefile) for basefile in basefiles] + + ##but since I cannot find them, for now: + #filePath = r'C:\Users\thela\Desktop\coolege\Fall2022\upTo\Summer\Space_ze_Final_Frontier\python_scripts\testingSite\folderForReductionTest\testHolyGrail5\__template' + #basefiles = ['MasterWaveCalib_ArI_5100.fits','MasterWaveCalib_NeI_5100.fits','MasterWaveCalib_XeI_5100.fits'] + #wfiles = [os.path.join(filePath, basefile) for basefile in basefiles] ## these are the MasterWaveCalib files + + filePath = r'C:\Users\thela\Desktop\coolege\Fall2022\upTo\Summer\Space_ze_Final_Frontier\python_scripts\testingSite\folderForReductionTest\testHolyGrail6\ArI\mdm_modspec_echelle_A' + wfiles = os.path.join(filePath, 'wvcalib.fits') + + # Snippets + slits = [150] ## spatial id + wv_cuts = [] + assert len(wv_cuts) == len(slits)-1 + + + templates.build_template(wfiles, slits, wv_cuts, binspec, outroot, + det_cut=None, chk=True, + normalize=True, lowredux=False, + subtract_conti=True, overwrite=overwrite, + shift_wave=True) + + + +if __name__ == '__main__': + mdm_modspec_echelle(overwrite=True) + diff --git a/pypeit/core/wavecal/templates.py b/pypeit/core/wavecal/templates.py index 889f7cc4c2..bc856b73c8 100644 --- a/pypeit/core/wavecal/templates.py +++ b/pypeit/core/wavecal/templates.py @@ -242,7 +242,7 @@ def pypeit_arcspec(in_file, slit, binspec, binning=None): slit index Returns: - tuple: np.ndarray, np.ndarray, PypeItFit: wave, flux, pypeitFitting + tuple: `numpy.ndarray`_, `numpy.ndarray`_, PypeItFit: wave, flux, pypeitFitting """ if '.json' in in_file: @@ -351,12 +351,12 @@ def poly_val(coeff, x, nrm): IDL style function for polynomial Args: - coeff (np.ndarray): Polynomial coefficients - x (np.ndarray): x array - nrm (np.ndarray): Normalization terms + coeff (`numpy.ndarray`_): Polynomial coefficients + x (`numpy.ndarray`_): x array + nrm (`numpy.ndarray`_): Normalization terms Returns: - np.ndarray: Same shape as x + `numpy.ndarray`_: Same shape as x """ # @@ -446,9 +446,11 @@ def xidl_arcspec(xidl_file, slit): return wv_vac.value, spec -def xidl_hires(xidl_file, specbin=1): +def xidl_esihires(xidl_file, specbin=1, order_vec=None, + log10=True): """ Read an XIDL format solution for Keck/HIRES + or Keck/ESI Note: They used air @@ -457,10 +459,12 @@ def xidl_hires(xidl_file, specbin=1): Keck/HIRES save file Returns: + tuple: np.ndarray, np.ndarray, np.ndarray of orders, wavelength, flux """ xidl_dict = readsav(xidl_file) - order_vec = xidl_dict['guess_ordr'] + if order_vec is None: + order_vec = xidl_dict['guess_ordr'] norders = order_vec.size nspec = xidl_dict['sv_aspec'].shape[1] @@ -485,6 +489,8 @@ def xidl_hires(xidl_file, specbin=1): else: order_mask[kk]=False continue + if not log10: + log10_wv_air = np.log10(log10_wv_air) wv_vac = airtovac(10**log10_wv_air * units.AA).value ispec = xidl_dict['sv_aspec'][kk,:] diff --git a/pypeit/core/wavecal/waveio.py b/pypeit/core/wavecal/waveio.py index 2e524543cb..e765ebe90a 100644 --- a/pypeit/core/wavecal/waveio.py +++ b/pypeit/core/wavecal/waveio.py @@ -1,4 +1,7 @@ """ Module for I/O in arclines + +.. include:: ../include/links.rst + """ import pathlib @@ -51,17 +54,23 @@ def load_template(arxiv_file, det, wvrng=None): """ Load a full template file from disk - Args: - arxiv_file: str - det: int - wvrng (list, optional): - min, max wavelength range for the arxiv - + Parameters + ---------- + arxiv_file : str + File with archive spectrum + det : int + Detector number + wvrng : list, optional + min, max wavelength range for the arxiv - Returns: - wave: ndarray - flux: ndarray - binning: int, Of the template arc spectrum + Returns + ------- + wave : ndarray + Wavelength vector + flux : ndarray + Flux vector + binning : int + binning of the template arc spectrum """ # Path already included? @@ -148,7 +157,7 @@ def load_line_list(line_file, use_ion=False): Returns ------- - line_list : Table + line_list : `astropy.table.Table`_ """ line_file = data.get_linelist_filepath(f'{line_file}_lines.dat') if use_ion else \ @@ -156,7 +165,7 @@ def load_line_list(line_file, use_ion=False): return astropy.table.Table.read(line_file, format='ascii.fixed_width', comment='#') -def load_line_lists(lamps, unknown=False, all=False, restrict_on_instr=None): +def load_line_lists(lamps, all=False, include_unknown:bool=False, restrict_on_instr=None): """ Loads a series of line list files @@ -165,14 +174,18 @@ def load_line_lists(lamps, unknown=False, all=False, restrict_on_instr=None): lamps : list List of arc lamps to be used for wavelength calibration. E.g., ['ArI','NeI','KrI','XeI'] - unknown : bool, optional - Load the unknown list restrict_on_instr : str, optional Restrict according to the input spectrograph + all : bool, optional + Load all line lists, independent of the input lamps (not recommended) + include_unknown : bool, optional + If True, the tot_line_list includes the unknown lines Returns ------- - line_list : Table + tot_line_list : astropy Table of line lists (including unknown lines, if requested) + line_list : astropy Table of line lists + unkn_lines : astropy Table of unknown lines """ # All? @@ -194,23 +207,28 @@ def load_line_lists(lamps, unknown=False, all=False, restrict_on_instr=None): # Stack if len(lists) == 0: return None - line_lists = astropy.table.vstack(lists, join_type='exact') + line_lists_all = astropy.table.vstack(lists, join_type='exact') # Restrict on the spectrograph? if restrict_on_instr is not None: instr_dict = defs.instruments() - gdI = (line_lists['Instr'] & instr_dict[restrict_on_instr]) > 0 - line_lists = line_lists[gdI] + gdI = (line_lists_all['Instr'] & instr_dict[restrict_on_instr]) > 0 + line_lists_all = line_lists_all[gdI] - # Unknown - if unknown: + # Load Unknowns + if 'ThAr' in lamps: + line_lists = line_lists_all[line_lists_all['ion'] != 'UNKNWN'] + unkn_lines = line_lists_all[line_lists_all['ion'] == 'UNKNWN'] + else: + line_lists = line_lists_all unkn_lines = load_unknown_list(lamps) - unkn_lines.remove_column('line_flag') # may wish to have this info - # Stack - line_lists = astropy.table.vstack([line_lists, unkn_lines]) + #unkn_lines.remove_column('line_flag') # may wish to have this info + + # Stack? + tot_line_list = astropy.table.vstack([line_lists, unkn_lines]) if include_unknown else line_lists_all # Return - return line_lists + return tot_line_list, line_lists, unkn_lines def load_tree(polygon=4, numsearch=20): @@ -279,7 +297,7 @@ def load_unknown_list(lines, unknwn_file=None, all=False): Returns ------- - unknwn_lines : Table + unknwn_lines : `astropy.table.Table`_ """ line_dict = defs.lines() @@ -294,6 +312,10 @@ def load_unknown_list(lines, unknwn_file=None, all=False): # Otherwise msk = np.zeros(len(line_list), dtype=bool) for line in lines: + # Skip if the lines is not even in the line list + if line not in line_dict.keys(): + continue + # Else consider masking line_flag = line_dict[line] match = line_list['line_flag'] % (2*line_flag) >= line_flag msk[match] = True diff --git a/pypeit/core/wavecal/wv_fitting.py b/pypeit/core/wavecal/wv_fitting.py index b2d8f88f0d..3d25b06900 100644 --- a/pypeit/core/wavecal/wv_fitting.py +++ b/pypeit/core/wavecal/wv_fitting.py @@ -178,45 +178,45 @@ def fit_slit(spec, patt_dict, tcent, line_lists, vel_tol = 1.0, outroot=None, sl Parameters ---------- spec : ndarray - arc spectrum + arc spectrum patt_dict : dict - dictionary of patterns + dictionary of patterns tcent: ndarray - List of the detections in this slit to be fit using the patt_dict - line_lists: astropy Table - Table containing the line list - Optional Parameters - ------------------- + List of the detections in this slit to be fit using the patt_dict + line_lists: `astropy.table.Table`_ + Table containing the line list vel_tol: float, default = 1.0 - Tolerance in km/s for matching lines in the IDs to lines in the NIST database. The default is 1.0 km/s + Tolerance in km/s for matching lines in the IDs to lines in the NIST + database. The default is 1.0 km/s outroot: str - Path for QA file. + Path for QA file. slittxt : str - Label used for QA + Label used for QA thar: bool, default = False - True if this is a ThAr fit + True if this is a ThAr fit match_toler: float, default = 3.0 - Matching tolerance when searching for new lines. This is the difference in pixels between the wavlength assigned to - an arc line by an iteration of the wavelength solution to the wavelength in the line list. + Matching tolerance when searching for new lines. This is the difference + in pixels between the wavlength assigned to an arc line by an iteration + of the wavelength solution to the wavelength in the line list. func: str, default = 'legendre' - Name of function used for the wavelength solution + Name of function used for the wavelength solution n_first: int, default = 2 - Order of first guess to the wavelength solution. + Order of first guess to the wavelength solution. sigrej_first: float, default = 2.0 - Number of sigma for rejection for the first guess to the wavelength solution. + Number of sigma for rejection for the first guess to the wavelength solution. n_final: int, default = 4 - Order of the final wavelength solution fit + Order of the final wavelength solution fit sigrej_final: float, default = 3.0 - Number of sigma for rejection for the final fit to the wavelength solution. + Number of sigma for rejection for the final fit to the wavelength solution. verbose : bool - If True, print out more information. + If True, print out more information. plot_fil: - Filename for plotting some QA? + Filename for plotting some QA? Returns ------- final_fit : dict - A dictionary containing all of the information about the fit + A dictionary containing all of the information about the fit """ # Check that patt_dict and tcent refer to each other @@ -264,47 +264,47 @@ def iterative_fitting(spec, tcent, ifit, IDs, llist, disp, Parameters ---------- spec : ndarray, shape = (nspec,) - arcline spectrum + arcline spectrum tcent : ndarray - Centroids in pixels of lines identified in spec + Centroids in pixels of lines identified in spec ifit : ndarray - Indices of the lines that will be fit + Indices of the lines that will be fit IDs: ndarray - wavelength IDs of the lines that will be fit (I think?) + wavelength IDs of the lines that will be fit (I think?) llist: dict - Linelist dictionary + Linelist dictionary disp: float - dispersion - - Optional Parameters - ------------------- + dispersion match_toler: float, default = 3.0 - Matching tolerance when searching for new lines. This is the difference in pixels between the wavlength assigned to - an arc line by an iteration of the wavelength solution to the wavelength in the line list. + Matching tolerance when searching for new lines. This is the difference + in pixels between the wavlength assigned to an arc line by an iteration + of the wavelength solution to the wavelength in the line list. func: str, default = 'legendre' - Name of function used for the wavelength solution + Name of function used for the wavelength solution n_first: int, default = 2 - Order of first guess to the wavelength solution. + Order of first guess to the wavelength solution. sigrej_first: float, default = 2.0 - Number of sigma for rejection for the first guess to the wavelength solution. + Number of sigma for rejection for the first guess to the wavelength solution. n_final: int, default = 4 - Order of the final wavelength solution fit + Order of the final wavelength solution fit sigrej_final: float, default = 3.0 - Number of sigma for rejection for the final fit to the wavelength solution. + Number of sigma for rejection for the final fit to the wavelength + solution. input_only: bool - If True, the routine will only perform a robust polyfit to the input IDs. - If False, the routine will fit the input IDs, and then include additional - lines in the linelist that are a satisfactory fit. + If True, the routine will only perform a robust polyfit to the input + IDs. If False, the routine will fit the input IDs, and then include + additional lines in the linelist that are a satisfactory fit. weights: ndarray - Weights to be used? + Weights to be used? verbose : bool - If True, print out more information. + If True, print out more information. plot_fil: - Filename for plotting some QA? + Filename for plotting some QA? Returns ------- final_fit: :class:`pypeit.core.wavecal.wv_fitting.WaveFit` + Fit result """ #TODO JFH add error checking here to ensure that IDs and ifit have the same size! diff --git a/pypeit/core/wavecal/wvutils.py b/pypeit/core/wavecal/wvutils.py index bffa05637c..63c15dad94 100644 --- a/pypeit/core/wavecal/wvutils.py +++ b/pypeit/core/wavecal/wvutils.py @@ -17,7 +17,7 @@ from astropy import constants from pypeit import msgs -from pypeit import utils +from pypeit import utils, data from pypeit.core import arc from pypeit.pypmsgs import PypeItError @@ -99,14 +99,13 @@ def get_sampling(waves, pix_per_R=3.0): Computes the median wavelength sampling of wavelength vector(s) Args: - waves (float `numpy.ndarray`_): shape = (nspec,) or (nspec, nimgs) - Array of wavelengths. Can be one or two dimensional where - the nimgs dimension can represent the orders, exposures, or - slits - pix_per_R (float): default=3.0 + waves (list, `numpy.ndarray`_): + List of `numpy.ndarray`_ wavelength arrays or a single 1d + 'numpy.ndarray'_ + pix_per_R (float): Number of pixels per resolution element used for the resolution guess. The default of 3.0 assumes roughly Nyquist - smampling + sampling. Returns: tuple: Returns dlam, dloglam, resln_guess, pix_per_sigma. @@ -114,55 +113,67 @@ def get_sampling(waves, pix_per_R=3.0): vector(s) """ - - if waves.ndim == 1: - norders = 1 - nspec = waves.shape[0] - waves_stack = waves.reshape((nspec, norders)) - elif waves.ndim == 2: - waves_stack = waves - elif waves.ndim == 3: - nspec, norder, nexp = waves.shape - waves_stack = np.reshape(waves, (nspec, norder * nexp), order='F') + if isinstance(waves, np.ndarray): + if waves.ndim == 1: + waves_out = [waves] + elif waves.ndim == 2: + waves_out = utils.array_to_explist(waves) + else: + msgs.error('Array inputs can only be 1D or 2D') + elif isinstance(waves, list): + ndim = np.array([wave.ndim for wave in waves], dtype=int) + if np.any(ndim > 1): + msgs.error('Input list can only contain 1D arrays') + waves_out = waves else: - msgs.error('The shape of your wavelength array does not make sense.') - - wave_mask = waves_stack > 1.0 - waves_ma = np.ma.array(waves_stack, mask=np.invert(wave_mask)) - loglam = np.ma.log10(waves_ma) - wave_diff = np.diff(waves_ma, axis=0) - loglam_diff = np.diff(loglam, axis=0) - dwave = np.ma.median(wave_diff) - dloglam = np.ma.median(loglam_diff) - #dloglam_ord = np.ma.median(loglam_roll, axis=0) - #dloglam = np.median(dloglam_ord) + msgs.error('Input must be a list or numpy.ndarray') + + wave_diff_flat = [] + dloglam_flat = [] + for wave in waves_out: + wave_good = wave[wave > 1.0] + wave_diff_flat += np.diff(wave_good).tolist() + dloglam_flat += np.diff(np.log10(wave_good)).tolist() + + + dwave = np.median(wave_diff_flat) + dloglam = np.median(dloglam_flat) resln_guess = 1.0 / (pix_per_R* dloglam * np.log(10.0)) pix_per_sigma = 1.0 / resln_guess / (dloglam * np.log(10.0)) / (2.0 * np.sqrt(2.0 * np.log(2))) return dwave, dloglam, resln_guess, pix_per_sigma # TODO: the other methods iref should be deprecated or removed -def get_wave_grid(waves=None, masks=None, wave_method='linear', iref=0, wave_grid_min=None, +def get_wave_grid(waves=None, gpms=None, wave_method='linear', iref=0, wave_grid_min=None, wave_grid_max=None, dwave=None, dv=None, dloglam=None, wave_grid_input=None, spec_samp_fact=1.0): """ Create a new wavelength grid for spectra to be rebinned and coadded. Args: - waves (`numpy.ndarray`_, optional): - Set of N original wavelength arrays. Shape is (nspec, nexp). - Required unless wave_method='user_input' in which case it need not be passed in. - masks (`numpy.ndarray`_, optional): - Good-pixel mask for wavelengths. Shape must match waves. + waves (list): + List of the `numpy.ndarray`_ N original 1d wavelength arrays. + Shapes of the input arrays are arbitrary. Required unless + wave_method='user_input' in which case it need not be passed in. + gpms (list): + Good-pixel mask for wavelengths. Same format as waves and shapes of + the individual arrays must match. wave_method (:obj:`str`, optional): Desired method for creating new wavelength grid: - * 'iref' -- Use the first wavelength array (default) - * 'velocity' -- Grid is uniform in velocity - * 'log10' -- Grid is uniform in log10(wave). This is the same as velocity. - * 'linear' -- Constant pixel grid - * 'concatenate' -- Meld the input wavelength arrays - * 'user_input' -- Use a user input wavelength grid. wave_grid_input must be set for this option. + - 'iref' -- Use the first wavelength array (default) + + - 'velocity' -- Grid is uniform in velocity + + - 'log10' -- Grid is uniform in log10(wave). This is the same + as velocity. + + - 'linear' -- Constant pixel grid + + - 'concatenate' -- Meld the input wavelength arrays + + - 'user_input' -- Use a user input wavelength grid. + wave_grid_input must be set for this option. iref (:obj:`int`, optional): Index in waves array for reference spectrum @@ -172,12 +183,13 @@ def get_wave_grid(waves=None, masks=None, wave_method='linear', iref=0, wave_gri max wavelength value for the final grid dwave (:obj:`float`, optional): Pixel size in same units as input wavelength array (e.g. Angstroms). - If not input, the median pixel size is calculated and used. + Used with the 'linear' method. If not input, the median pixel size + is calculated and used. dv (:obj:`float`, optional): - Pixel size in km/s for velocity method. If not input, the median + Pixel size in km/s for 'velocity' method. If not input, the median km/s per pixel is calculated and used dloglam (:obj:`float`, optional): - Pixel size in log10(wave) for the log10 method. + Pixel size in log10(wave) for the log10 or velocity method. spec_samp_fact (:obj:`float`, optional): Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or coarser (spec_samp_fact > 1.0) by this sampling factor. This @@ -189,30 +201,37 @@ def get_wave_grid(waves=None, masks=None, wave_method='linear', iref=0, wave_gri Returns: :obj:`tuple`: Returns two `numpy.ndarray`_ objects and a float: - - ``wave_grid``: (ndarray, (ngrid +1,)) New wavelength grid, not masked. - This is a set of bin edges (rightmost edge for the last bin and leftmost edges for the rest), - while wave_grid_mid is a set of bin centers, hence wave_grid has 1 more value than wave_grid_mid. - - ``wave_grid_mid``: ndarray, (ngrid,) New wavelength grid evaluated at the centers of - the wavelength bins, that is this grid is simply offset from - ``wave_grid`` by ``dsamp/2.0``, in either linear space or log10 - depending on whether linear or (log10 or velocity) was requested. - Last bin center is removed since it falls outside wave_grid. - For iref or concatenate, the linear wavelength sampling will be - calculated. + + - ``wave_grid``: (ndarray, (ngrid +1,)) New wavelength grid, not + masked. This is a set of bin edges (rightmost edge for the last + bin and leftmost edges for the rest), while wave_grid_mid is a set + of bin centers, hence wave_grid has 1 more value than + wave_grid_mid. + + - ``wave_grid_mid``: ndarray, (ngrid,) New wavelength grid evaluated + at the centers of the wavelength bins, that is this grid is simply + offset from ``wave_grid`` by ``dsamp/2.0``, in either linear space + or log10 depending on whether linear or (log10 or velocity) was + requested. Last bin center is removed since it falls outside + wave_grid. For iref or concatenate, the linear wavelength + sampling will be calculated. + - ``dsamp``: The pixel sampling for wavelength grid created. + """ + c_kms = constants.c.to('km/s').value if wave_method == 'user_input': wave_grid = wave_grid_input else: - if masks is None: - masks = waves > 1.0 + if gpms is None: + gpms = [wave > 1.0 for wave in waves] if wave_grid_min is None: - wave_grid_min = waves[masks].min() + wave_grid_min = np.min([wave[gpm].min() for wave, gpm in zip(waves, gpms)]) if wave_grid_max is None: - wave_grid_max = waves[masks].max() + wave_grid_max = np.max([wave[gpm].max() for wave, gpm in zip(waves, gpms)]) dwave_data, dloglam_data, resln_guess, pix_per_sigma = get_sampling(waves) @@ -240,15 +259,15 @@ def get_wave_grid(waves=None, masks=None, wave_method='linear', iref=0, wave_gri elif wave_method == 'concatenate': # Concatenate # Setup - loglam = np.log10(waves) # This deals with padding (0's) just fine, i.e. they get masked.. - nexp = waves.shape[1] - newloglam = loglam[:, iref] # Deals with mask + loglam = [np.log10(wave) for wave in waves] # This deals with padding (0's) just fine, i.e. they get masked.. + nexp = len(waves) + newloglam = loglam[iref] # Deals with mask # Loop for j in range(nexp): if j == iref: continue # - iloglam = loglam[:, j] + iloglam = loglam[j] dloglam_0 = (newloglam[1]-newloglam[0]) dloglam_n = (newloglam[-1] - newloglam[-2]) # Assumes sorted if (newloglam[0] - iloglam[0]) > dloglam_0: @@ -262,8 +281,8 @@ def get_wave_grid(waves=None, masks=None, wave_method='linear', iref=0, wave_gri wave_grid = np.power(10.0,newloglam) elif wave_method == 'iref': # Use the iref index wavelength array - wave_tmp = waves[:, iref] - wave_grid = wave_tmp[ wave_tmp > 1.0] + wave_tmp = waves[iref] + wave_grid = wave_tmp[wave_tmp > 1.0] else: msgs.error("Bad method for wavelength grid: {:s}".format(wave_method)) @@ -378,16 +397,16 @@ def zerolag_shift_stretch(theta, y1, y2): Parameters ---------- - theta (float `numpy.ndarray`_): + theta : float `numpy.ndarray`_ Function parameters to optmize over. theta[0] = shift, theta[1] = stretch - y1 (float `numpy.ndarray`_): shape = (nspec,) + y1 : float `numpy.ndarray`_, shape = (nspec,) First spectrum which acts as the refrence - y2 (float `numpy.ndarray`_): shape = (nspec,) + y2 : float `numpy.ndarray`_, shape = (nspec,) Second spectrum which will be transformed by a shift and stretch to match y1 Returns ------- - corr_norm: float + corr_norm : float Negative of the zero lag cross-correlation coefficient (since we are miniziming with scipy.optimize). scipy.optimize will thus determine the shift,stretch that maximize the cross-correlation. @@ -406,10 +425,10 @@ def zerolag_shift_stretch(theta, y1, y2): return -corr_norm -def get_xcorr_arc(inspec1, sigdetect=5.0, sig_ceil=10.0, percent_ceil=50.0, use_raw_arc=False, fwhm = 4.0): - - """ Utility routine to create an synthetic arc spectrum for cross-correlation using the location of the peaks in - the input spectrum. +def get_xcorr_arc(inspec1, sigdetect=5.0, sig_ceil=10.0, percent_ceil=50.0, use_raw_arc=False, fwhm = 4.0, debug=False): + """ + Utility routine to create a synthetic arc spectrum for cross-correlation + using the location of the peaks in the input spectrum. Args: inspec1 (`numpy.ndarray`_): @@ -427,6 +446,8 @@ def get_xcorr_arc(inspec1, sigdetect=5.0, sig_ceil=10.0, percent_ceil=50.0, use_ If True, use amplitudes from the raw arc, i.e. do not continuum subtract. Default = False fwhm (float, optional): Fwhm of arc lines. Used for peak finding and to assign a fwhm in the xcorr_arc. + debug (bool, optional): + Show plots for line detection debugging. Default = False Returns: `numpy.ndarray`_: Synthetic arc spectrum to be used for @@ -436,7 +457,8 @@ def get_xcorr_arc(inspec1, sigdetect=5.0, sig_ceil=10.0, percent_ceil=50.0, use_ # Run line detection to get the locations and amplitudes of the lines - tampl1, tampl1_cont, tcent1, twid1, centerr1, w1, arc1, nsig1 = arc.detect_lines(inspec1, sigdetect=sigdetect, fwhm=fwhm) + tampl1, tampl1_cont, tcent1, twid1, centerr1, w1, arc1, nsig1 = arc.detect_lines(inspec1, sigdetect=sigdetect, + fwhm=fwhm, debug=debug) ampl = tampl1 if use_raw_arc else tampl1_cont @@ -448,6 +470,9 @@ def get_xcorr_arc(inspec1, sigdetect=5.0, sig_ceil=10.0, percent_ceil=50.0, use_ ceil_upper = np.inf ampl_clip = np.clip(ampl, None, ceil_upper) + if ampl_clip.size == 0: + msgs.warn('No lines were detected in the arc spectrum. Cannot create a synthetic arc spectrum for cross-correlation.') + return None # Make a fake arc by plopping down Gaussians at the location of every centroided line we found xcorr_arc = np.zeros_like(inspec1) @@ -460,6 +485,7 @@ def get_xcorr_arc(inspec1, sigdetect=5.0, sig_ceil=10.0, percent_ceil=50.0, use_ continue xcorr_arc += ampl_clip[ind]*np.exp(-0.5*((spec_vec - tcent1[ind])/sigma)**2) + return xcorr_arc @@ -551,12 +577,18 @@ def xcorr_shift_stretch(inspec1, inspec2, cc_thresh=-1.0, percent_ceil=50.0, use shift_mnmx=(-0.2,0.2), stretch_mnmx=(0.95,1.05), sigdetect = 5.0, sig_ceil=10.0, fwhm = 4.0, debug=False, toler=1e-5, seed = None): - """ Determine the shift and stretch of inspec2 relative to inspec1. This routine computes an initial - guess for the shift via maximimizing the cross-correlation. It then performs a two parameter search for the shift and stretch - by optimizing the zero lag cross-correlation between the inspec1 and the transformed inspec2 (shifted and stretched via - wvutils.shift_and_stretch()) in a narrow window about the initial estimated shift. The convention for the shift is that - positive shift means inspec2 is shifted to the right (higher pixel values) relative to inspec1. The convention for the stretch is - that it is float near unity that increases the size of the inspec2 relative to the original size (which is the size of inspec1) + """ + Determine the shift and stretch of inspec2 relative to inspec1. This + routine computes an initial guess for the shift via maximimizing the + cross-correlation. It then performs a two parameter search for the shift and + stretch by optimizing the zero lag cross-correlation between the inspec1 and + the transformed inspec2 (shifted and stretched via + wvutils.shift_and_stretch()) in a narrow window about the initial estimated + shift. The convention for the shift is that positive shift means inspec2 is + shifted to the right (higher pixel values) relative to inspec1. The + convention for the stretch is that it is float near unity that increases the + size of the inspec2 relative to the original size (which is the size of + inspec1) Parameters ---------- @@ -604,10 +636,10 @@ def xcorr_shift_stretch(inspec1, inspec2, cc_thresh=-1.0, percent_ceil=50.0, use seed: int or np.random.RandomState, optional, default = None Seed for scipy.optimize.differential_evolution optimizer. If not specified, the calculation will not be repeatable - toler (float): + toler : float Tolerance for differential evolution optimizaiton. debug = False - Show plots to the screen useful for debugging. + Show plots to the screen useful for debugging. Returns ------- @@ -616,7 +648,9 @@ def xcorr_shift_stretch(inspec1, inspec2, cc_thresh=-1.0, percent_ceil=50.0, use - success = 1, shift and stretch performed via sucessful optimization + - success = 0, shift and stretch optimization failed + - success = -1, initial x-correlation is below cc_thresh (see above), so shift/stretch optimization was not attempted @@ -634,12 +668,12 @@ def xcorr_shift_stretch(inspec1, inspec2, cc_thresh=-1.0, percent_ceil=50.0, use which unity indicating a perfect match between the two spectra. If cc_thresh is set, and the initial cross-correlation is < cc_thresh, this will be just the initial cross-correlation - shift_init: + shift_init: float The initial shift determined by maximizing the cross-correlation coefficient without allowing for a stretch. If cc_thresh is set, and the initial cross-correlation is < cc_thresh, this will be just the shift from the initial cross-correlation - cross_corr_init: + cross_corr_init: float The maximum of the initial cross-correlation coefficient determined without allowing for a stretch. If cc_thresh is set, and the initial cross-correlation is < cc_thresh, this will be @@ -655,9 +689,13 @@ def xcorr_shift_stretch(inspec1, inspec2, cc_thresh=-1.0, percent_ceil=50.0, use y2 = get_xcorr_arc(inspec2, percent_ceil=percent_ceil, use_raw_arc=use_raw_arc, sigdetect=sigdetect, sig_ceil=sig_ceil, fwhm=fwhm) + if y1 is None or y2 is None: + msgs.warn('No lines detected punting on shift/stretch') + return 0, None, None, None, None, None # Do the cross-correlation first and determine the initial shift shift_cc, corr_cc = xcorr_shift(y1, y2, percent_ceil = None, do_xcorr_arc=False, sigdetect = sigdetect, fwhm=fwhm, debug = debug) + # TODO JFH Is this a good idea? Stretch fitting seems to recover better values #if corr_cc < -np.inf: # < cc_thresh: # return -1, shift_cc, 1.0, corr_cc, shift_cc, corr_cc @@ -771,7 +809,7 @@ def wavegrid(wave_min, wave_max, dwave, spec_samp_fact=1.0, log10=False): def write_template(nwwv, nwspec, binspec, outpath, outroot, det_cut=None, - order=None, overwrite=True): + order=None, overwrite=True, cache=False): """ Write the template spectrum into a binary FITS table @@ -783,7 +821,9 @@ def write_template(nwwv, nwspec, binspec, outpath, outroot, det_cut=None, binspec (int): Binning of the template outpath (str): + Directory to store the wavelength template file outroot (str): + Filename to use for the template det_cut (bool, optional): Cuts in wavelength for detector snippets Used primarily for DEIMOS @@ -791,6 +831,8 @@ def write_template(nwwv, nwspec, binspec, outpath, outroot, det_cut=None, Echelle order numbers overwrite (bool, optional): If True, overwrite any existing file + cache (bool, optional): + Store the wavelength solution in the pypeit cache? """ tbl = Table() tbl['wave'] = nwwv @@ -809,4 +851,17 @@ def write_template(nwwv, nwspec, binspec, outpath, outroot, det_cut=None, # Write outfile = os.path.join(outpath, outroot) tbl.write(outfile, overwrite=overwrite) - print("Wrote: {}".format(outfile)) + msgs.info(f"Your arxiv solution has been written to {outfile}\n") + if cache: + # Also copy the file to the cache for direct use + data.write_file_to_cache(outroot, outroot, "arc_lines/reid_arxiv") + + msgs.info(f"Your arxiv solution has also been cached.{msgs.newline()}" + f"To utilize this wavelength solution, insert the{msgs.newline()}" + f"following block in your PypeIt Reduction File:{msgs.newline()}" + f" [calibrations]{msgs.newline()}" + f" [[wavelengths]]{msgs.newline()}" + f" reid_arxiv = {outroot}{msgs.newline()}" + f" method = full_template\n") + print("") # Empty line for clarity + msgs.info("Please consider sharing your solution with the PypeIt Developers.") diff --git a/pypeit/data/arc_lines/NIST/NeII_vacuum.ascii b/pypeit/data/arc_lines/NIST/NeII_vacuum.ascii new file mode 100644 index 0000000000..0d508e81ea --- /dev/null +++ b/pypeit/data/arc_lines/NIST/NeII_vacuum.ascii @@ -0,0 +1,978 @@ +# 27-Jun-2023 (grabbed from NIST website by DP) +# Vacuum, 3000-10,000 Ang +#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Observed | Ritz | Rel. | Aki | Acc. | Ei Ek | Lower level | Upper level |Type| TP | Line | +# Wavelength | Wavelength | Int. | (s^-1) | | (cm-1) (cm-1) |-----------------------------------|-----------------------------------| | Ref. | Ref. | +# Vac (Å) | Vac (Å) | | | | | Conf. | Term | J | Conf. | Term | J | | | | +#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 3000.331 | 3000.330 | 22 | | | 305364.004 - 338693.67 | 2s2.2p4.(1D).3d | 2G | 9/2 | 2s2.2p4.(1D<2>).5f | 2[4]* | 9/2 | | | L1406 | + 3000.422 | 3000.425 | 22 | | | 305365.116 - 338693.73 | 2s2.2p4.(1D).3d | 2G | 7/2 | 2s2.2p4.(1D<2>).5f | 2[4]* | 7/2 | | | L1406 | + 3002.54352 | 3002.54360 | 300 | 8.7e+07 | D | 219648.4248 - 252953.5198 | 2s2.2p4.(3P).3s | 4P | 3/2 | 2s2.2p4.(3P).3p | 4S* | 3/2 | | T867LS | L5951 | + 3007.905 | 3007.8992 | 22 | | | 280473.5550 - 313719.3496 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<0>).5f | 2[3]* | 5/2 | | | L1406 | + 3008.709 | 3008.708 | 160 | | | 305364.004 - 338600.86 | 2s2.2p4.(1D).3d | 2G | 9/2 | 2s2.2p4.(1D<2>).5f | 2[5]* | 11/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3008.810 | 3008.810 | 140 | | | 305365.116 - 338600.85 | 2s2.2p4.(1D).3d | 2G | 7/2 | 2s2.2p4.(1D<2>).5f | 2[5]* | 9/2 | | | L1406 | + 3011.328 | 3011.3145 | 22 | | | 280262.3163 - 313470.4054 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P<1>).5f | 2[3]* | 5/2 | | | L1406 | + 3014.853 | 3014.8494 | 40 | | | 280262.3163 - 313431.469 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P<1>).5f | 2[4]* | 9/2 | | | L1406 | + | 3016.58756 | | 1.3e+06 | D | 219648.4248 - 252798.4654 | 2s2.2p4.(3P).3s | 4P | 3/2 | 2s2.2p4.(3P).3p | 2S* | 1/2 | | T380 | | + 3018.1882 | 3018.18818 | 240bl | 3.5e+07 | D | 246192.4130 - 279324.8733 | 2s2.2p4.(3P).3p | 4P* | 5/2 | 2s2.2p4.(3P).3d | 4D | 3/2 | | T380 | L5951 | + | | | | | | | | | | | | | | | + 3027.89652 | 3027.89644 | 200 | 1.4e+08 | D | 246192.4130 - 279218.6416 | 2s2.2p4.(3P).3p | 4P* | 5/2 | 2s2.2p4.(3P).3d | 4D | 5/2 | | T380 | L5951 | + 3029.5818 | 3029.5819 | 180 | 8.5e+07 | D | 246415.0144 - 279422.869 | 2s2.2p4.(3P).3p | 4P* | 3/2 | 2s2.2p4.(3P).3d | 4D | 1/2 | | T380 | L5951 | + 3029.74518 | 3029.74533 | 240 | 4.7e+07 | D | 219947.4453 - 252953.5198 | 2s2.2p4.(3P).3s | 4P | 1/2 | 2s2.2p4.(3P).3p | 4S* | 3/2 | | T867LS | L5951 | + 3030.595 | 3030.5923 | 120 | | | 280473.5550 - 313470.4054 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<1>).5f | 2[3]* | 5/2 | | | L1406 | + 3031.66988 | 3031.66973 | 200 | | | 249695.5051 - 282680.6284 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).4s | 4P | 1/2 | | | L5951 | + | | | | | | | | | | | | | | | + 3031.812 | 3031.813 | 22 | | | 305566.923 - 338550.49 | 2s2.2p4.(1D).3d | 2P | 3/2 | 2s2.2p4.(1D<2>).5f | 2[1]* | 3/2 | | | L1406 | + 3035.223 | 3035.2255 | 120w,bl | | | 280473.5550 - 313420.0366 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<1>).5f | 2[2]* | 5/2 | | | L1406 | + 3035.34416 | 3035.34425 | 240 | 3.1e+08 | D | 246192.4130 - 279137.6053 | 2s2.2p4.(3P).3p | 4P* | 5/2 | 2s2.2p4.(3P).3d | 4D | 7/2 | | T380 | L5951 | + 3036.80524 | 3036.80532 | 200 | | | 249445.9632 - 282375.3049 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).4s | 4P | 3/2 | | | L5951 | + 3037.510 | 3037.5090 | 160 | | | 280797.2387 - 313718.9516 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<0>).5f | 2[3]* | 7/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3038.60320 | 3038.60312 | 200 | 2.1e+08 | D | 246415.0144 - 279324.8733 | 2s2.2p4.(3P).3p | 4P* | 3/2 | 2s2.2p4.(3P).3d | 4D | 3/2 | | T380 | L5951 | + 3040.47005 | 3040.47021 | 200 | | | 249108.6138 - 281998.2635 | 2s2.2p4.(3P).3p | 4D* | 7/2 | 2s2.2p4.(3P).4s | 4P | 5/2 | | | L5951 | + 3044.9734 | 3044.97336 | 200 | | | 249839.6186 - 282680.6284 | 2s2.2p4.(3P).3p | 4D* | 1/2 | 2s2.2p4.(3P).4s | 4P | 1/2 | | | L5951 | + 3046.442 | 3046.4410 | 200 | 2.5e+08 | D | 246597.6805 - 279422.869 | 2s2.2p4.(3P).3p | 4P* | 1/2 | 2s2.2p4.(3P).3d | 4D | 1/2 | | T380 | L1406 | + 3048.44343 | 3048.44337 | 240 | 1.8e+08 | D | 246415.0144 - 279218.6416 | 2s2.2p4.(3P).3p | 4P* | 3/2 | 2s2.2p4.(3P).3d | 4D | 5/2 | | T380 | L5951 | + | | | | | | | | | | | | | | | + 3051.3596 | 3051.3599 | 180 | | | 280947.0768 - 313719.3496 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<0>).5f | 2[3]* | 5/2 | | | L5951 | + 3051.650 | 3051.6498 | 140 | | | 280700.7426 - 313469.9028 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<1>).5f | 2[3]* | 7/2 | | | L1406 | + 3055.2331 | 3055.2331 | 200 | | | 280700.7426 - 313431.469 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<1>).5f | 2[4]* | 9/2 | | | L5951 | + 3055.313 | 3055.3171 | 180 | | | 280989.5229 - 313719.3496 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<0>).5f | 2[3]* | 5/2 | | | L1406 | + 3055.56303 | 3055.56302 | 200 | 9.4e+07 | D | 246597.6805 - 279324.8733 | 2s2.2p4.(3P).3p | 4P* | 1/2 | 2s2.2p4.(3P).3d | 4D | 3/2 | | T380 | L5951 | + | | | | | | | | | | | | | | | + 3058.7560 | 3058.7570 | 180 | | | 281025.9326 - 313718.9516 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<0>).5f | 2[3]* | 7/2 | | | L5951 | + 3059.9944 | 3059.99427 | 200 | | | 249695.5051 - 282375.3049 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).4s | 4P | 3/2 | | | L5951 | + 3060.614 | 3060.6155 | 160 | | | 280797.2387 - 313470.4054 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<1>).5f | 2[3]* | 5/2 | | | L1406 | + 3062.744 | 3062.7375 | 100 | | | 280768.508 - 313419.0375 | 2s2.2p4.(3P).3d | 4P | 1/2 | 2s2.2p4.(3P<1>).5f | 2[2]* | 3/2 | | | L1406 | + 3063.3815 | 3063.3816 | 200 | | | 280172.9854 - 312816.6500 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P<2>).5f | 2[5]* | 11/2 | | | L5951 | + | | | | | | | | | | | | | | | + 3064.1919 | 3064.1927 | 200 | | | 280797.2387 - 313432.262 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<1>).5f | 2[4]* | 7/2 | | | L5951 | + 3065.447 | 3065.4349 | 22 | | | 280797.2387 - 313419.0375 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<1>).5f | 2[2]* | 3/2 | | | L1406 | + 3067.574 | 3067.5781 | 22 | | | 280172.9854 - 312771.9922 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P<2>).5f | 2[3]* | 7/2 | | | L1406 | + 3068.3409 | 3068.3405 | 160 | | | 280172.9854 - 312763.8929 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P<2>).5f | 2[4]* | 9/2 | | | L5951 | + 3071.7839 | 3071.7840 | 200 | | | 280262.3163 - 312816.689 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P<2>).5f | 2[5]* | 9/2 | | | L5951 | + | | | | | | | | | | | | | | | + 3071.9795 | 3071.97952 | 180 | | | 249445.9632 - 281998.2635 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).4s | 4P | 5/2 | | | L5951 | + 3072.4235 | 3072.4236 | 200 | | | 281171.3566 - 313718.9516 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<0>).5f | 2[3]* | 7/2 | | | L5951 | + 3073.195 | 3073.194 | 40 | | | 306011.056 - 338550.49 | 2s2.2p4.(1D).3d | 2S | 1/2 | 2s2.2p4.(1D<2>).5f | 2[1]* | 3/2 | | | L1406 | + 3073.547 | 3073.5405 | 180bl* | | | 280268.9453 - 312804.7127 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P<2>).5f | 2[2]* | 5/2 | | | L5951 | + 3073.547 | 3073.54820 | 180bl* | | | 249839.6186 - 282375.3049 | 2s2.2p4.(3P).3p | 4D* | 1/2 | 2s2.2p4.(3P).4s | 4P | 3/2 | | | L5951 | + | | | | | | | | | | | | | | | + 3074.714 | 3074.7160 | 140 | | | 280947.0768 - 313470.4054 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<1>).5f | 2[3]* | 5/2 | | | L1406 | + 3075.970 | 3075.9658 | 22 | | | 280262.3163 - 312772.4309 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P<2>).5f | 2[3]* | 5/2 | | | L1406 | + 3076.632 | 3076.6234 | 200bl* | | | 280262.3163 - 312765.4821 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P<2>).5f | 2[4]* | 7/2 | | | L5951 | + 3076.632 | 3076.6346 | 200bl* | | | 280268.9453 - 312771.9922 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P<2>).5f | 2[3]* | 7/2 | | | L5951 | + 3077.2506 | 3077.2510 | 180 | | | 280268.9453 - 312765.4821 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P<2>).5f | 2[4]* | 7/2 | | | L5951 | + | | | | | | | | | | | | | | | + 3078.7395 | 3078.7341 | 160 | | | 280989.5229 - 313470.4054 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<1>).5f | 2[3]* | 5/2 | | | L5951 | + 3079.4815 | 3079.4852 | 160 | | | 280947.0768 - 313420.0366 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<1>).5f | 2[2]* | 5/2 | | | L5951 | + 3082.2371 | 3082.2368 | 180 | | | 281025.9326 - 313469.9028 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<1>).5f | 2[3]* | 7/2 | | | L5951 | + 3083.508 | 3083.5158 | 160 | | | 280989.5229 - 313420.0366 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<1>).5f | 2[2]* | 5/2 | | | L1406 | + 3083.601 | 3083.6108 | 140 | | | 280989.5229 - 313419.0375 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<1>).5f | 2[2]* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3084.915 | 3084.916 | 60 | | | 306262.755 - 338678.55 | 2s2.2p4.(1D).3d | 2D | 3/2 | 2s2.2p4.(1D<2>).5f | 2[3]* | 5/2 | | | L1406 | + 3085.815 | 3085.8169 | 160 | | | 281025.9326 - 313432.262 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<1>).5f | 2[4]* | 7/2 | | | L5951 | + 3086.976 | 3086.9815 | 140 | | | 281025.9326 - 313420.0366 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<1>).5f | 2[2]* | 5/2 | | | L1406 | + 3087.078 | 3087.0767 | 22 | | | 281025.9326 - 313419.0375 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<1>).5f | 2[2]* | 3/2 | | | L1406 | + 3089.06075 | 3089.06073 | 240 | | | 251522.0967 - 283894.3965 | 2s2.2p4.(3P).3p | 2D* | 3/2 | 2s2.2p4.(3P).4s | 2P | 1/2 | | | L5951 | + | | | | | | | | | | | | | | | + 3089.173 | 3089.1848 | 160 | | | 280473.5550 - 312844.5547 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<2>).5f | 2[1]* | 3/2 | | | L1406 | + 3092.9917 | 3092.9916 | 200 | | | 280473.5550 - 312804.7127 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<2>).5f | 2[2]* | 5/2 | | | L5951 | + 3093.7999 | 3093.7999 | 240 | 1.3e+08 | D | 274364.5596 - 306687.271 | 2s2.2p4.(1D).3p | 2F* | 5/2 | 2s2.2p4.(1D).3d | 2F | 5/2 | | T8043 | L5951 | + 3094.90409 | 3094.90392 | 200 | | | 251011.1511 - 283322.3319 | 2s2.2p4.(3P).3p | 2D* | 5/2 | 2s2.2p4.(3P).4s | 2P | 3/2 | | | L5951 | + 3096.0018 | 3096.0020 | 200 | | | 276677.1142 - 308976.8347 | 2s2.2p4.(1S).3s | 2S | 1/2 | 2s2.2p4.(3P).5p | 2P* | 1/2 | | | L5951 | + | | | | | | | | | | | | | | | + 3096.082 | 3096.0664 | 180* | | | 281171.3566 - 313470.4054 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<1>).5f | 2[3]* | 5/2 | | | L5951 | + 3096.082 | 3096.0830 | 180* | | | 280473.5550 - 312772.4309 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<2>).5f | 2[3]* | 5/2 | | | L5951 | + 3098.03076 | 3098.03058 | 200 | 1.3e+08 | D | 274409.0830 - 306687.654 | 2s2.2p4.(1D).3p | 2F* | 7/2 | 2s2.2p4.(1D).3d | 2F | 7/2 | | T8043 | L5951 | + 3098.434 | 3098.43937 | 80 | 1.5e+06 | D | 249445.9632 - 281720.2755 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 2P | 3/2 | | T380 | L1406 | + 3099.726 | 3099.7270 | 120 | | | 281171.3566 - 313432.262 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<1>).5f | 2[4]* | 7/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3100.905 | 3100.9021 | 22 | | | 281171.3566 - 313420.0366 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<1>).5f | 2[2]* | 5/2 | | | L1406 | + 3113.724 | 3113.7180 | 40 | | | 280700.7426 - 312816.689 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<2>).5f | 2[5]* | 9/2 | | | L1406 | + 3116.576 | 3116.5739 | 160 | | | 281332.521 - 313419.0375 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P<1>).5f | 2[2]* | 3/2 | | | L1406 | + 3117.599 | 3117.5912 | 180bl | | | 280768.508 - 312844.5547 | 2s2.2p4.(3P).3d | 4P | 1/2 | 2s2.2p4.(3P<2>).5f | 2[1]* | 3/2 | | | L1406 | + 3117.666 | 3117.6835 | 180bl | | | 280768.508 - 312843.6043 | 2s2.2p4.(3P).3d | 4P | 1/2 | 2s2.2p4.(3P<2>).5f | 2[1]* | 1/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3118.057 | 3118.0575 | 100 | | | 280700.7426 - 312771.9922 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<2>).5f | 2[3]* | 7/2 | | | L1406 | + 3118.697 | 3118.6905 | 60 | | | 280700.7426 - 312765.4821 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<2>).5f | 2[4]* | 7/2 | | | L1406 | + 3118.88489 | 3118.88476 | 200 | 4.2e+06 | D | 249108.6138 - 281171.3566 | 2s2.2p4.(3P).3p | 4D* | 7/2 | 2s2.2p4.(3P).3d | 4P | 5/2 | | T380 | L5951 | + 3119.0642 | 3119.0644 | 240 | | | 276677.1142 - 308738.0108 | 2s2.2p4.(1S).3s | 2S | 1/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L5951 | + 3119.84 | 3119.848 | 30 | | | 305408.498 - 337461.34 | 2s2.2p4.(1S).3p | 2P* | 1/2 | 2s2.2p4.(1D).5d | 2S | 1/2 | | | L11529 | + | | | | | | | | | | | | | | | + 3120.377 | 3120.3861 | 40 | | | 280797.2387 - 312844.5547 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<2>).5f | 2[1]* | 3/2 | | | L1406 | + 3121.540 | 3121.5352 | 40 | | | 280768.508 - 312804.0268 | 2s2.2p4.(3P).3d | 4P | 1/2 | 2s2.2p4.(3P<2>).5f | 2[2]* | 3/2 | | | L1406 | + 3124.273 | 3124.2703 | 22 | | | 280797.2387 - 312804.7127 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<2>).5f | 2[2]* | 5/2 | | | L1406 | + 3124.382 | 3124.369 | 140bl | | | 306687.271 - 338693.73 | 2s2.2p4.(1D).3d | 2F | 5/2 | 2s2.2p4.(1D<2>).5f | 2[4]* | 7/2 | | | L1406 | + 3124.406 | 3124.413 | 140bl | | | 306687.654 - 338693.67 | 2s2.2p4.(1D).3d | 2F | 7/2 | 2s2.2p4.(1D<2>).5f | 2[4]* | 9/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3125.0904 | 3125.0904 | 180 | | | 281720.2755 - 313719.3496 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<0>).5f | 2[3]* | 5/2 | | | L5951 | + 3125.854 | 3125.852 | 80 | | | 306687.271 - 338678.55 | 2s2.2p4.(1D).3d | 2F | 5/2 | 2s2.2p4.(1D<2>).5f | 2[3]* | 5/2 | | | L1406 | + 3133.098 | 3133.09525 | 180 | 2.2e+06 | D | 249108.6138 - 281025.9326 | 2s2.2p4.(3P).3p | 4D* | 7/2 | 2s2.2p4.(3P).3d | 2F | 5/2 | | T380 | L1406 | + 3134.974 | 3134.9736 | 80 | 2.6e+07 | D | 274364.5596 - 306262.755 | 2s2.2p4.(1D).3p | 2F* | 5/2 | 2s2.2p4.(1D).3d | 2D | 3/2 | | T8043 | L1406 | + 3135.040 | 3135.0441 | 140 | | | 280947.0768 - 312844.5547 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<2>).5f | 2[1]* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + | 3136.69823 | | 2.0e+06 | D | 249839.6186 - 281720.2755 | 2s2.2p4.(3P).3p | 4D* | 1/2 | 2s2.2p4.(3P).3d | 2P | 3/2 | | T380 | | + 3136.726 | 3136.72447 | 160 | 6.5e+05 | D | 219130.7609 - 251011.1511 | 2s2.2p4.(3P).3s | 4P | 5/2 | 2s2.2p4.(3P).3p | 2D* | 5/2 | | T380 | L1406 | + 3137.385 | 3137.38562 | 100 | 4.9e+05 | D | 219648.4248 - 251522.0967 | 2s2.2p4.(3P).3s | 4P | 3/2 | 2s2.2p4.(3P).3p | 2D* | 3/2 | | T380 | L1406 | + 3138.967 | 3138.9649 | 160 | | | 280947.0768 - 312804.7127 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<2>).5f | 2[2]* | 5/2 | | | L1406 | + 3141.268 | 3141.26343 | 100 | 2.4e+07 | D | 274409.0830 - 306243.4077 | 2s2.2p4.(1D).3p | 2F* | 7/2 | 2s2.2p4.(1D).3d | 2D | 5/2 | | T8043 | L1406 | + | | | | | | | | | | | | | | | + 3142.24205 | 3142.24200 | 240 | | | 276276.8361 - 308101.2468 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(1D).4s | 2D | 5/2 | | | L5951 | + 3143.215 | 3143.2205 | 22 | | | 280989.5229 - 312804.0268 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<2>).5f | 2[2]* | 3/2 | | | L1406 | + 3144.6310 | 3144.63083 | 200 | | | 251522.0967 - 283322.3319 | 2s2.2p4.(3P).3p | 2D* | 3/2 | 2s2.2p4.(3P).4s | 2P | 3/2 | | | L5951 | + 3146.336 | 3146.3452 | 22 | | | 280989.5229 - 312772.4309 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<2>).5f | 2[3]* | 5/2 | | | L1406 | + 3149.5924 | 3149.5934 | 200bl | | | 281720.2755 - 313470.4054 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<1>).5f | 2[3]* | 5/2 | | | L5951 | + | | | | | | | | | | | | | | | + 3152.0489 | 3152.04917 | 180 | 4.8e+06 | D | 249445.9632 - 281171.3566 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 4P | 5/2 | | T380 | L5951 | + 3152.509 | 3152.5167 | 80 | | | 281998.2635 - 313718.9516 | 2s2.2p4.(3P).4s | 4P | 5/2 | 2s2.2p4.(3P<0>).5f | 2[3]* | 7/2 | | | L1406 | + 3154.591 | 3154.5979 | 140 | | | 281720.2755 - 313420.0366 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<1>).5f | 2[2]* | 5/2 | | | L1406 | + 3155.708 | 3155.70651 | 140 | 1.8e+06 | D | 249108.6138 - 280797.2387 | 2s2.2p4.(3P).3p | 4D* | 7/2 | 2s2.2p4.(3P).3d | 4F | 5/2 | | T380 | L1406 | + 3157.247 | 3157.2435 | 22 | | | 281171.3566 - 312844.5547 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<2>).5f | 2[1]* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3160.852 | 3160.85437 | 60 | 1.9e+06 | D | 249695.5051 - 281332.521 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).3d | 2P | 1/2 | | T380 | L1406 | + 3164.494 | 3164.4933 | 60 | | | 281171.3566 - 312771.9922 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<2>).5f | 2[3]* | 7/2 | | | L1406 | + 3165.144 | 3165.1454 | 40 | | | 281171.3566 - 312765.4821 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<2>).5f | 2[4]* | 7/2 | | | L1406 | + 3165.3454 | 3165.34541 | 200 | 1.6e+07 | D | 249108.6138 - 280700.7426 | 2s2.2p4.(3P).3p | 4D* | 7/2 | 2s2.2p4.(3P).3d | 4F | 7/2 | | T380 | L5951 | + 3165.57775 | 3165.57800 | 240 | | | 276511.800 - 308101.6076 | 2s2.2p4.(1D).3p | 2P* | 1/2 | 2s2.2p4.(1D).4s | 2D | 3/2 | | | L5951 | + | | | | | | | | | | | | | | | + 3166.5641 | 3166.56418 | 200 | 1.2e+07 | D | 249445.9632 - 281025.9326 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 2F | 5/2 | | T380 | L5951 | + 3167.092 | 3167.09752 | 140 | 4.2e+05 | D | 219947.4453 - 251522.0967 | 2s2.2p4.(3P).3s | 4P | 1/2 | 2s2.2p4.(3P).3p | 2D* | 3/2 | | T380 | L1406 | + 3170.225 | 3170.21924 | 12 | 1.7e+07 | D | 249445.9632 - 280989.5229 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 4P | 3/2 | | T380 | L1406 | + 3173.392 | 3173.3909 | 160 | | | 281332.521 - 312844.5547 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P<2>).5f | 2[1]* | 3/2 | | | L1406 | + 3174.4908 | 3174.49095 | 180 | 4.5e+06 | D | 249445.9632 - 280947.0768 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 4F | 3/2 | | T380 | L5951 | + | | | | | | | | | | | | | | | + 3174.589 | 3174.59611 | 160 | | | 276677.1142 - 308177.1843 | 2s2.2p4.(1S).3s | 2S | 1/2 | 2s2.2p4.(3P).5p | 2D* | 3/2 | | | L1406 | + 3177.0387 | 3177.03875 | 180 | 6.0e+06 | D | 249695.5051 - 281171.3566 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).3d | 4P | 5/2 | | T380 | L5951 | + 3177.471 | 3177.4640 | 180* | | | 281998.2635 - 313469.9028 | 2s2.2p4.(3P).4s | 4P | 5/2 | 2s2.2p4.(3P<1>).5f | 2[3]* | 7/2 | | | L1406 | + 3177.471 | 3177.4775 | 180* | | | 281332.521 - 312804.0268 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P<2>).5f | 2[2]* | 3/2 | | | L1406 | + 3178.659 | 3178.66437 | 100 | | | 276677.1142 - 308136.8685 | 2s2.2p4.(1S).3s | 2S | 1/2 | 2s2.2p4.(3P).5p | 4S* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3188.4980 | 3188.49832 | 180 | 1.4e+06 | D | 219648.4248 - 251011.1511 | 2s2.2p4.(3P).3s | 4P | 3/2 | 2s2.2p4.(3P).3p | 2D* | 5/2 | | T380 | L5951 | + 3189.6630 | 3189.66289 | 200 | 3.9e+07 | D | 249445.9632 - 280797.2387 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 4F | 5/2 | | T380 | L5951 | + 3191.7855 | 3191.78537 | 180 | 1.5e+07 | D | 249695.5051 - 281025.9326 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).3d | 2F | 5/2 | | T380 | L5951 | + 3191.843 | 3191.8562 | 140 | | | 276677.1142 - 308006.8460 | 2s2.2p4.(1S).3s | 2S | 1/2 | 2s2.2p4.(3P).5p | 2S* | 1/2 | | | L1406 | + 3195.49885 | 3195.49892 | 200 | 5.2e+07 | D | 249695.5051 - 280989.5229 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).3d | 4P | 3/2 | | T380 | L5951 | + | | | | | | | | | | | | | | | + 3199.51074 | 3199.51066 | 300 | 1.7e+08 | D | 249445.9632 - 280700.7426 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 4F | 7/2 | | T380 | L5951 | + 3199.841 | 3199.83907 | 120 | 2.3e+07 | D | 249695.5051 - 280947.0768 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).3d | 4F | 3/2 | | T8043 | L1406 | + | 3209.20848 | | 9.1e+05 | D | 249108.6138 - 280268.9453 | 2s2.2p4.(3P).3p | 4D* | 7/2 | 2s2.2p4.(3P).3d | 2D | 5/2 | | T380 | | + 3209.892 | 3209.89134 | 200 | 1.6e+07 | D | 249108.6138 - 280262.3163 | 2s2.2p4.(3P).3p | 4D* | 7/2 | 2s2.2p4.(3P).3d | 2F | 7/2 | | T380 | L1406 | + 3210.28267 | 3210.28274 | 240 | 6.0e+07 | D | 249839.6186 - 280989.5229 | 2s2.2p4.(3P).3p | 4D* | 1/2 | 2s2.2p4.(3P).3d | 4P | 3/2 | | T380 | L5951 | + | | | | | | | | | | | | | | | + 3212.917 | 3212.9258 | 22 | | | 281720.2755 - 312844.5547 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<2>).5f | 2[1]* | 3/2 | | | L1406 | + 3214.66311 | 3214.66316 | 240 | 1.7e+08 | D | 249839.6186 - 280947.0768 | 2s2.2p4.(3P).3p | 4D* | 1/2 | 2s2.2p4.(3P).3d | 4F | 3/2 | | T8043 | L5951 | + 3215.25472 | 3215.25486 | 300 | 2.2e+08 | D | 249695.5051 - 280797.2387 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).3d | 4F | 5/2 | | T380 | L5951 | + 3215.680 | 3215.6852 | 160 | | | 276677.1142 - 307774.6858 | 2s2.2p4.(1S).3s | 2S | 1/2 | 2s2.2p4.(3P).5p | 4D* | 3/2 | | | L1406 | + 3215.8566 | 3215.85482 | 160 | | | 252798.4654 - 283894.3965 | 2s2.2p4.(3P).3p | 2S* | 1/2 | 2s2.2p4.(3P).4s | 2P | 1/2 | | | L5951 | + | | | | | | | | | | | | | | | + | 3218.22774 | | 1.3e+07 | D | 249695.5051 - 280768.508 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).3d | 4P | 1/2 | | T380 | | + 3219.12198 | 3219.12193 | 300 | 3.6e+08 | D | 249108.6138 - 280172.9854 | 2s2.2p4.(3P).3p | 4D* | 7/2 | 2s2.2p4.(3P).3d | 4F | 9/2 | | T380 | L5951 | + | 3222.93785 | | 2.0e+06 | D | 249445.9632 - 280473.5550 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 2D | 3/2 | | T380 | | + 3225.74858 | 3225.7486 | 240 | 3.5e+08 | D | 274364.5596 - 305365.116 | 2s2.2p4.(1D).3p | 2F* | 5/2 | 2s2.2p4.(1D).3d | 2G | 7/2 | | T8043 | L5951 | + 3230.389 | 3230.3881 | 180 | 1.3e+07 | D | 274409.0830 - 305365.116 | 2s2.2p4.(1D).3p | 2F* | 7/2 | 2s2.2p4.(1D).3d | 2G | 7/2 | | T8043 | L1406 | + | | | | | | | | | | | | | | | + 3230.50408 | 3230.50413 | 240 | 3.6e+08 | D | 274409.0830 - 305364.004 | 2s2.2p4.(1D).3p | 2F* | 7/2 | 2s2.2p4.(1D).3d | 2G | 9/2 | | T8043 | L5951 | + 3231.00232 | 3231.00239 | 400 | 1.8e+08 | D | 246394.1202 - 277344.2675 | 2s2.2p4.(1D).3s | 2D | 5/2 | 2s2.2p4.(1D).3p | 2D* | 5/2 | | T867LS | L5951 | + 3231.3534 | 3231.35328 | 240 | 1.4e+07 | D | 246397.4810 - 277344.2675 | 2s2.2p4.(1D).3s | 2D | 3/2 | 2s2.2p4.(1D).3p | 2D* | 5/2 | | T867LS | L5951 | + 3232.955 | 3232.95488 | 240 | 2.7e+07 | D | 246394.1202 - 277325.5757 | 2s2.2p4.(1D).3s | 2D | 5/2 | 2s2.2p4.(1D).3p | 2D* | 3/2 | | T867LS | L1406 | + | 3233.22311 | | 3.3e+06 | D | 249839.6186 - 280768.508 | 2s2.2p4.(3P).3p | 4D* | 1/2 | 2s2.2p4.(3P).3d | 4P | 1/2 | | T380 | | + | | | | | | | | | | | | | | | + 3233.30618 | 3233.30619 | 300 | 1.6e+08 | D | 246397.4810 - 277325.5757 | 2s2.2p4.(1D).3s | 2D | 3/2 | 2s2.2p4.(1D).3p | 2D* | 3/2 | | T867LS | L5951 | + 3244.3321 | 3244.33242 | 200 | 2.3e+07 | D | 249445.9632 - 280268.9453 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 2D | 5/2 | | T380 | L5951 | + 3245.03026 | 3245.03031 | 200 | 1.5e+08 | D | 249445.9632 - 280262.3163 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 2F | 7/2 | | T380 | L5951 | + 3249.0684 | 3249.06875 | 180 | 2.4e+07 | D | 249695.5051 - 280473.5550 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).3d | 2D | 3/2 | | T380 | L5951 | + 3249.2820 | 3249.28179 | 200 | | | 277325.5757 - 308101.6076 | 2s2.2p4.(1D).3p | 2D* | 3/2 | 2s2.2p4.(1D).4s | 2D | 3/2 | | | L5951 | + | | | | | | | | | | | | | | | + 3249.3197 | 3249.31988 | 190 | | | 277325.5757 - 308101.2468 | 2s2.2p4.(1D).3p | 2D* | 3/2 | 2s2.2p4.(1D).4s | 2D | 5/2 | | | L12157c153 | + 3251.29468 | 3251.29458 | 200 | | | 277344.2675 - 308101.2468 | 2s2.2p4.(1D).3p | 2D* | 5/2 | 2s2.2p4.(1D).4s | 2D | 5/2 | | | L5951 | + 3256.3612 | 3256.36116 | 120 | 3.8e+06 | D | 251011.1511 - 281720.2755 | 2s2.2p4.(3P).3p | 2D* | 5/2 | 2s2.2p4.(3P).3d | 2P | 3/2 | | T380 | L5951 | + 3264.3537 | 3264.35358 | 140 | 3.9e+07 | D | 249839.6186 - 280473.5550 | 2s2.2p4.(3P).3p | 4D* | 1/2 | 2s2.2p4.(3P).3d | 2D | 3/2 | | T380 | L5951 | + 3270.8131 | 3270.81281 | 180 | 5.1e+07 | D | 249695.5051 - 280268.9453 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).3d | 2D | 5/2 | | T380 | L5951 | + | | | | | | | | | | | | | | | + 3271.743 | 3271.74340 | 180 | 5.7e+06 | D | 219130.7609 - 249695.5051 | 2s2.2p4.(3P).3s | 4P | 5/2 | 2s2.2p4.(3P).3p | 4D* | 3/2 | | T867LS | L1406 | + 3276.1249 | 3276.12493 | 180 | | | 252798.4654 - 283322.3319 | 2s2.2p4.(3P).3p | 2S* | 1/2 | 2s2.2p4.(3P).4s | 2P | 3/2 | | | L5951 | + 3298.67488 | 3298.67500 | 300 | 4.3e+07 | D | 219130.7609 - 249445.9632 | 2s2.2p4.(3P).3s | 4P | 5/2 | 2s2.2p4.(3P).3p | 4D* | 5/2 | | T867LS | L5951 | + 3310.69251 | 3310.69271 | 300 | 3.1e+07 | D | 224087.0092 - 254292.1683 | 2s2.2p4.(3P).3s | 2P | 3/2 | 2s2.2p4.(3P).3p | 2P* | 1/2 | | T867LS | L5951 | + 3311.452 | 3311.45797 | 60 | 6.9e+06 | D | 251522.0967 - 281720.2755 | 2s2.2p4.(3P).3p | 2D* | 3/2 | 2s2.2p4.(3P).3d | 2P | 3/2 | | T380 | L1406 | + | | | | | | | | | | | | | | | + 3312.22421 | 3312.22411 | 180 | 2.6e+07 | D | 219648.4248 - 249839.6186 | 2s2.2p4.(3P).3s | 4P | 3/2 | 2s2.2p4.(3P).3p | 4D* | 1/2 | | T867LS | L5951 | + 3315.6244 | 3315.62728 | 40 | 4.4e+06 | D | 251011.1511 - 281171.3566 | 2s2.2p4.(3P).3p | 2D* | 5/2 | 2s2.2p4.(3P).3d | 4P | 5/2 | | T380 | L5951 | + 3320.677 | 3320.67944 | 600 | 1.6e+08 | D | 246397.4810 - 276511.800 | 2s2.2p4.(1D).3s | 2D | 3/2 | 2s2.2p4.(1D).3p | 2P* | 1/2 | | T867LS | L1406 | + 3321.1527 | 3321.15270 | 160 | 2.1e+07 | D | 249108.6138 - 279218.6416 | 2s2.2p4.(3P).3p | 4D* | 7/2 | 2s2.2p4.(3P).3d | 4D | 5/2 | | T380 | L5951 | + 3324.69130 | 3324.69140 | 2000 | 1.6e+08 | D | 224087.0092 - 254164.9888 | 2s2.2p4.(3P).3s | 2P | 3/2 | 2s2.2p4.(3P).3p | 2P* | 3/2 | | T867LS | L5951 | + | | | | | | | | | | | | | | | + 3328.11056 | 3328.11039 | 300 | 9.1e+07 | D | 219648.4248 - 249695.5051 | 2s2.2p4.(3P).3s | 4P | 3/2 | 2s2.2p4.(3P).3p | 4D* | 3/2 | | T867LS | L5951 | + 3330.11516 | 3330.11517 | 200 | 8.8e+07 | D | 249108.6138 - 279137.6053 | 2s2.2p4.(3P).3p | 4D* | 7/2 | 2s2.2p4.(3P).3d | 4D | 7/2 | | T380 | L5951 | + 3331.6915 | 3331.69175 | 120 | 3.9e+06 | D | 251011.1511 - 281025.9326 | 2s2.2p4.(3P).3p | 2D* | 5/2 | 2s2.2p4.(3P).3d | 2F | 5/2 | | T380 | L5951 | + | 3335.73820 | | 3.0e+06 | D | 251011.1511 - 280989.5229 | 2s2.2p4.(3P).3p | 2D* | 5/2 | 2s2.2p4.(3P).3d | 4P | 3/2 | | T380 | | + 3335.79592 | 3335.79594 | 400 | 1.8e+08 | D | 219130.7609 - 249108.6138 | 2s2.2p4.(3P).3s | 4P | 5/2 | 2s2.2p4.(3P).3p | 4D* | 7/2 | | T867LS | L5951 | + | | | | | | | | | | | | | | | + 3337.05164 | 3337.05174 | 180 | 1.1e+08 | D | 276276.8361 - 306243.4077 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(1D).3d | 2D | 5/2 | | T8043 | L5951 | + 3345.35714 | 3345.35729 | 300 | 1.5e+08 | D | 219947.4453 - 249839.6186 | 2s2.2p4.(3P).3s | 4P | 1/2 | 2s2.2p4.(3P).3p | 4D* | 1/2 | | T867LS | L5951 | + 3346.41616 | 3346.41605 | 600 | 1.4e+08 | D | 246394.1202 - 276276.8361 | 2s2.2p4.(1D).3s | 2D | 5/2 | 2s2.2p4.(1D).3p | 2P* | 3/2 | | T867LS | L5951 | + 3346.79229 | 3346.79245 | 300 | 2.2e+07 | D | 246397.4810 - 276276.8361 | 2s2.2p4.(1D).3s | 2D | 3/2 | 2s2.2p4.(1D).3p | 2P* | 3/2 | | T867LS | L5951 | + | 3346.84229 | | 2.2e+07 | D | 249445.9632 - 279324.8733 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 4D | 3/2 | | T380 | | + | | | | | | | | | | | | | | | + 3354.5314 | 3354.5313 | 140 | 1.2e+07 | D | 251522.0967 - 281332.521 | 2s2.2p4.(3P).3p | 2D* | 3/2 | 2s2.2p4.(3P).3d | 2P | 1/2 | | T380 | L5951 | + 3355.98183 | 3355.98192 | 400 | 1.3e+08 | D | 219648.4248 - 249445.9632 | 2s2.2p4.(3P).3s | 4P | 3/2 | 2s2.2p4.(3P).3p | 4D* | 5/2 | | T867LS | L5951 | + 3357.2724 | 3357.27207 | 180 | 2.0e+07 | D | 251011.1511 - 280797.2387 | 2s2.2p4.(3P).3p | 2D* | 5/2 | 2s2.2p4.(3P).3d | 4F | 5/2 | | T380 | L5951 | + 3358.78398 | 3358.78414 | 240 | 5.0e+07 | D | 249445.9632 - 279218.6416 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 4D | 5/2 | | T380 | L5951 | + 3361.237 | 3361.2366 | 180 | 8.6e+07 | D | 276511.800 - 306262.755 | 2s2.2p4.(1D).3p | 2P* | 1/2 | 2s2.2p4.(1D).3d | 2D | 3/2 | | T8043 | L1406 | + | | | | | | | | | | | | | | | + 3361.563 | 3361.56377 | 400 | 8.2e+07 | D | 219947.4453 - 249695.5051 | 2s2.2p4.(3P).3s | 4P | 1/2 | 2s2.2p4.(3P).3p | 4D* | 3/2 | | T867LS | L1406 | + 3363.12840 | 3363.12842 | 240 | | | 276276.8361 - 306011.056 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(1D).3d | 2S | 1/2 | | | L5951 | + 3363.67292 | 3363.67280 | 200 | | | 254164.9888 - 283894.3965 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).4s | 2P | 1/2 | | | L5951 | + 3363.9040 | 3363.9041 | 180 | 3.5e+07 | D | 249695.5051 - 279422.869 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).3d | 4D | 1/2 | | T380 | L5951 | + 3367.952 | 3367.95114 | 60 | 3.5e+06 | D | 249445.9632 - 279137.6053 | 2s2.2p4.(3P).3p | 4D* | 5/2 | 2s2.2p4.(3P).3d | 4D | 7/2 | | T380 | L1406 | + | | | | | | | | | | | | | | | + 3368.18376 | 3368.18376 | 240 | 1.0e+08 | D | 251011.1511 - 280700.7426 | 2s2.2p4.(3P).3p | 2D* | 5/2 | 2s2.2p4.(3P).3d | 4F | 7/2 | | T380 | L5951 | + 3372.768 | 3372.76547 | 200 | 2.2e+07 | D | 251522.0967 - 281171.3566 | 2s2.2p4.(3P).3p | 2D* | 3/2 | 2s2.2p4.(3P).3d | 4P | 5/2 | | T380 | L1406 | + 3375.0298 | 3375.02978 | 160 | 3.0e+07 | D | 249695.5051 - 279324.8733 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).3d | 4D | 3/2 | | T380 | L5951 | + 3378.1242 | 3378.12408 | 240 | | | 254292.1683 - 283894.3965 | 2s2.2p4.(3P).3p | 2P* | 1/2 | 2s2.2p4.(3P).4s | 2P | 1/2 | | | L5951 | + 3379.18946 | 3379.18930 | 1000 | 1.7e+08 | D | 224699.2716 - 254292.1683 | 2s2.2p4.(3P).3s | 2P | 1/2 | 2s2.2p4.(3P).3p | 2P* | 1/2 | | T867LS | L5951 | + | | | | | | | | | | | | | | | + 3380.2913 | 3380.2912 | 160 | 3.0e+07 | D | 249839.6186 - 279422.869 | 2s2.2p4.(3P).3p | 4D* | 1/2 | 2s2.2p4.(3P).3d | 4D | 1/2 | | T380 | L5951 | + 3387.1739 | 3387.17399 | 120 | 5.5e+06 | D | 249695.5051 - 279218.6416 | 2s2.2p4.(3P).3p | 4D* | 3/2 | 2s2.2p4.(3P).3d | 4D | 5/2 | | T380 | L5951 | + 3389.38969 | 3389.38978 | 300 | 2.2e+08 | D | 251522.0967 - 281025.9326 | 2s2.2p4.(3P).3p | 2D* | 3/2 | 2s2.2p4.(3P).3d | 2F | 5/2 | | T380 | L5951 | + 3389.9160 | 3389.91600 | 240 | | | 276511.800 - 306011.056 | 2s2.2p4.(1D).3p | 2P* | 1/2 | 2s2.2p4.(1D).3d | 2S | 1/2 | | | L5951 | + 3391.5254 | 3391.52573 | 140 | 7.7e+06 | D | 249839.6186 - 279324.8733 | 2s2.2p4.(3P).3p | 4D* | 1/2 | 2s2.2p4.(3P).3d | 4D | 3/2 | | T380 | L5951 | + | | | | | | | | | | | | | | | + 3393.581 | 3393.57769 | 160 | 1.4e+07 | D | 251522.0967 - 280989.5229 | 2s2.2p4.(3P).3p | 2D* | 3/2 | 2s2.2p4.(3P).3d | 4P | 3/2 | | T380 | L1406 | + 3393.77449 | 3393.77451 | 600 | 4.4e+07 | D | 224699.2716 - 254164.9888 | 2s2.2p4.(3P).3s | 2P | 1/2 | 2s2.2p4.(3P).3p | 2P* | 3/2 | | T867LS | L5951 | + 3394.157 | 3394.15617 | 120 | 2.2e+06 | D | 251011.1511 - 280473.5550 | 2s2.2p4.(3P).3p | 2D* | 5/2 | 2s2.2p4.(3P).3d | 2D | 3/2 | | T380 | L1406 | + | 3398.47299 | | 4.9e+06 | D | 251522.0967 - 280947.0768 | 2s2.2p4.(3P).3p | 2D* | 3/2 | 2s2.2p4.(3P).3d | 4F | 3/2 | | T380 | | + 3398.841 | 3398.84204 | 100 | | | 252953.5198 - 282375.3049 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).4s | 4P | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3405.16 | 3405.155 | 8 | | | 294086.4608 - 323453.7 | 2s2.2p4.(3P).4p | 2P* | 3/2 | 2s2.2p4.(3P).8d | 2D | 5/2 | | | L11529 | + 3405.79780 | 3405.79789 | 200 | 1.9e+08 | D | 277325.5757 - 306687.271 | 2s2.2p4.(1D).3p | 2D* | 3/2 | 2s2.2p4.(1D).3d | 2F | 5/2 | | T8043 | L5951 | + 3407.92260 | 3407.92294 | 240 | 2.3e+08 | D | 277344.2675 - 306687.654 | 2s2.2p4.(1D).3p | 2D* | 5/2 | 2s2.2p4.(1D).3d | 2F | 7/2 | | T8043 | L5951 | + 3412.3390 | 3412.3389 | 160 | 6.1e+07 | D | 276276.8361 - 305582.249 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(1D).3d | 2P | 1/2 | | T8043 | L5951 | + 3414.12438 | 3414.12439 | 200 | 1.8e+08 | D | 276276.8361 - 305566.923 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(1D).3d | 2P | 3/2 | | T8043 | L5951 | + | | | | | | | | | | | | | | | + 3415.8681 | 3415.86729 | 140 | 1.8e+06 | D | 251522.0967 - 280797.2387 | 2s2.2p4.(3P).3p | 2D* | 3/2 | 2s2.2p4.(3P).3d | 4F | 5/2 | | T380 | L5951 | + 3417.89262 | 3417.89266 | 240 | 6.4e+07 | D | 251011.1511 - 280268.9453 | 2s2.2p4.(3P).3p | 2D* | 5/2 | 2s2.2p4.(3P).3d | 2D | 5/2 | | T380 | L5951 | + 3418.66727 | 3418.66723 | 240 | 1.6e+08 | D | 251011.1511 - 280262.3163 | 2s2.2p4.(3P).3p | 2D* | 5/2 | 2s2.2p4.(3P).3d | 2F | 7/2 | | T380 | L5951 | + 3429.6681 | 3429.66777 | 240 | | | 254164.9888 - 283322.3319 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).4s | 2P | 3/2 | | | L9088 | + 3439.921 | 3439.9193 | 180 | 1.4e+08 | D | 276511.800 - 305582.249 | 2s2.2p4.(1D).3p | 2P* | 1/2 | 2s2.2p4.(1D).3d | 2P | 1/2 | | T8043 | L1406 | + | | | | | | | | | | | | | | | + 3441.7335 | 3441.7338 | 140 | 3.5e+07 | D | 276511.800 - 305566.923 | 2s2.2p4.(1D).3p | 2P* | 1/2 | 2s2.2p4.(1D).3d | 2P | 3/2 | | T8043 | L5951 | + 3442.9627 | 3442.96376 | 100 | | | 252953.5198 - 281998.2635 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).4s | 4P | 5/2 | | | L5951 | + 3444.6934 | 3444.69295 | 160 | | | 254292.1683 - 283322.3319 | 2s2.2p4.(3P).3p | 2P* | 1/2 | 2s2.2p4.(3P).4s | 2P | 3/2 | | | L5951 | + 3454.0578 | 3454.05744 | 160 | 4.6e+07 | D | 251522.0967 - 280473.5550 | 2s2.2p4.(3P).3p | 2D* | 3/2 | 2s2.2p4.(3P).3d | 2D | 3/2 | | T380 | L5951 | + 3455.76178 | 3455.76184 | 180 | 1.6e+08 | D | 277325.5757 - 306262.755 | 2s2.2p4.(1D).3p | 2D* | 3/2 | 2s2.2p4.(1D).3d | 2D | 3/2 | | T8043 | L5951 | + | | | | | | | | | | | | | | | + 3457.59837 | 3457.59825 | 200 | 9.6e+07 | D | 252798.4654 - 281720.2755 | 2s2.2p4.(3P).3p | 2S* | 1/2 | 2s2.2p4.(3P).3d | 2P | 3/2 | | T380 | L5951 | + 3457.9982 | 3457.9955 | 140 | | | 277344.2675 - 306262.755 | 2s2.2p4.(1D).3p | 2D* | 5/2 | 2s2.2p4.(1D).3d | 2D | 3/2 | | | L5951 | + 3458.074 | 3458.07390 | 140 | 9.9e+06 | D | 277325.5757 - 306243.4077 | 2s2.2p4.(1D).3p | 2D* | 3/2 | 2s2.2p4.(1D).3d | 2D | 5/2 | | T8043 | L1406 | + 3460.31062 | 3460.31056 | 200 | 1.6e+08 | D | 277344.2675 - 306243.4077 | 2s2.2p4.(1D).3p | 2D* | 5/2 | 2s2.2p4.(1D).3d | 2D | 5/2 | | T8043 | L5951 | + 3464.20 | 3464.22196 | | 2.8e+06 | D | 224087.0092 - 252953.5198 | 2s2.2p4.(3P).3s | 2P | 3/2 | 2s2.2p4.(3P).3p | 4S* | 3/2 | | T380 | L9088c153 | + | | | | | | | | | | | | | | | + 3476.235 | 3476.23490 | 60 | 1.2e+06 | D | 252953.5198 - 281720.2755 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 2P | 3/2 | | T380 | L1406 | + 3478.6422 | 3478.64218 | 160 | 4.3e+07 | D | 251522.0967 - 280268.9453 | 2s2.2p4.(3P).3p | 2D* | 3/2 | 2s2.2p4.(3P).3d | 2D | 5/2 | | T380 | L5951 | + 3480.5147 | 3480.5146 | 300 | 1.6e+08 | D | 276677.1142 - 305408.498 | 2s2.2p4.(1S).3s | 2S | 1/2 | 2s2.2p4.(1S).3p | 2P* | 1/2 | | T380 | L9088 | + 3481.71448 | 3481.71442 | 400 | 1.6e+08 | D | 276677.1142 - 305398.5968 | 2s2.2p4.(1S).3s | 2S | 1/2 | 2s2.2p4.(1S).3p | 2P* | 3/2 | | T380 | L5951 | + 3482.93040 | 3482.93027 | 400 | 1.4e+08 | D | 224087.0092 - 252798.4654 | 2s2.2p4.(3P).3s | 2P | 3/2 | 2s2.2p4.(3P).3p | 2S* | 1/2 | | T867LS | L5951 | + | | | | | | | | | | | | | | | + 3501.881 | 3501.88560 | 40 | | | 279218.6416 - 307774.6858 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P).5p | 4D* | 3/2 | | | L1406 | + 3504.5843 | 3504.58419 | 180 | 2.0e+08 | D | 252798.4654 - 281332.521 | 2s2.2p4.(3P).3p | 2S* | 1/2 | 2s2.2p4.(3P).3d | 2P | 1/2 | | T380 | L5951 | + 3508.3734 | 3508.3682 | 180 | | | 280473.5550 - 308976.8347 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P).5p | 2P* | 1/2 | | | L5951 | + 3510.68 | 3510.681 | 6 | | | 308976.8347 - 337461.34 | 2s2.2p4.(3P).5p | 2P* | 1/2 | 2s2.2p4.(1D).5d | 2S | 1/2 | | | L11529 | + 3512.584 | 3512.5846 | 180 | | | 280268.9453 - 308738.0108 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3523.729 | 3523.7322 | 40 | 2.3e+06 | D | 252953.5198 - 281332.521 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 2P | 1/2 | | T380 | L1406 | + 3526.52 | 3526.4791 | 24? | | | 290583.1503 - 318940.046 | 2s2.2p4.(3P).4p | 4P* | 3/2 | 2s2.2p4.(3P).7s | 4P | 3/2 | | | L11529c153 | + 3536.11 | 3536.098 | 4 | | | 290583.1503 - 318862.91 | 2s2.2p4.(3P).4p | 4P* | 3/2 | 2s2.2p4.(3P).6d | 2F | 5/2 | | | L11529 | + 3538.013 | 3538.0126 | 120 | | | 280473.5550 - 308738.0108 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L1406 | + 3538.9868 | 3538.9870 | 140 | 7.6e+07 | D | 277325.5757 - 305582.249 | 2s2.2p4.(1D).3p | 2D* | 3/2 | 2s2.2p4.(1D).3d | 2P | 1/2 | | T8043 | L5951 | + | | | | | | | | | | | | | | | + 3539.27 | 3539.29077 | | 9.0e+05 | D | 224699.2716 - 252953.5198 | 2s2.2p4.(3P).3s | 2P | 1/2 | 2s2.2p4.(3P).3p | 4S* | 3/2 | | T380 | L9088c153 | + 3540.907 | 3540.9076 | 60 | 3.6e+06 | D | 277325.5757 - 305566.923 | 2s2.2p4.(1D).3p | 2D* | 3/2 | 2s2.2p4.(1D).3d | 2P | 3/2 | | T8043 | L1406 | + 3543.2528 | 3543.2527 | 160 | 6.0e+07 | D | 277344.2675 - 305566.923 | 2s2.2p4.(1D).3p | 2D* | 5/2 | 2s2.2p4.(1D).3d | 2P | 3/2 | | T8043 | L5951 | + 3543.85756 | 3543.85776 | 240 | 1.2e+08 | D | 252953.5198 - 281171.3566 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 4P | 5/2 | | T380 | L5951 | + 3544.8033 | 3544.80253 | 140 | | | 254164.9888 - 282375.3049 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).4s | 4P | 3/2 | | | L5951 | + | | | | | | | | | | | | | | | + | 3545.1576 | | 3.7e+05 | D | 251011.1511 - 279218.6416 | 2s2.2p4.(3P).3p | 2D* | 5/2 | 2s2.2p4.(3P).3d | 4D | 5/2 | | T380 | | + 3545.828 | 3545.8248 | 40 | | | 279218.6416 - 307420.8248 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P).5p | 2D* | 5/2 | | | L1406 | + 3547.224 | 3547.2242 | 100 | 6.3e+06 | D | 252798.4654 - 280989.5229 | 2s2.2p4.(3P).3p | 2S* | 1/2 | 2s2.2p4.(3P).3d | 4P | 3/2 | | T380 | L1406 | + 3552.572 | 3552.5731 | 40 | 3.7e+06 | D | 252798.4654 - 280947.0768 | 2s2.2p4.(3P).3p | 2S* | 1/2 | 2s2.2p4.(3P).3d | 4F | 3/2 | | T380 | L1406 | + | 3555.3717 | | 1.3e+06 | D | 251011.1511 - 279137.6053 | 2s2.2p4.(3P).3p | 2D* | 5/2 | 2s2.2p4.(3P).3d | 4D | 7/2 | | T380 | | + | | | | | | | | | | | | | | | + 3558.329 | 3558.3270 | 100 | | | 279137.6053 - 307240.7001 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P).5p | 4D* | 7/2 | | | L1406 | + 3558.821 | 3558.8210 | 240 | 1.9e+07 | D | 224699.2716 - 252798.4654 | 2s2.2p4.(3P).3s | 2P | 1/2 | 2s2.2p4.(3P).3p | 2S* | 1/2 | | T867LS | L1406 | + 3559.234 | 3559.2317 | 12 | | | 279324.8733 - 307420.8248 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P).5p | 2D* | 5/2 | | | L1406 | + 3560.857 | 3560.85581 | 60 | | | 254292.1683 - 282375.3049 | 2s2.2p4.(3P).3p | 2P* | 1/2 | 2s2.2p4.(3P).4s | 4P | 3/2 | | | L1406 | + 3562.2161 | 3562.21607 | 200 | 2.1e+07 | D | 252953.5198 - 281025.9326 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 2F | 5/2 | | T380 | L5951 | + | | | | | | | | | | | | | | | + 3566.844 | 3566.8422 | 180 | 6.2e+07 | D | 252953.5198 - 280989.5229 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 4P | 3/2 | | T380 | L1406 | + 3568.627 | 3568.6172 | 12 | | | 279218.6416 - 307240.7001 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P).5p | 4D* | 7/2 | | | L1406 | + 3569.52116 | 3569.52107 | 500 | 1.4e+08 | D | 246394.1202 - 274409.0830 | 2s2.2p4.(1D).3s | 2D | 5/2 | 2s2.2p4.(1D).3p | 2F* | 7/2 | | T867LS | L5951 | + 3572.25079 | 3572.25057 | 200 | 6.3e+07 | D | 252953.5198 - 280947.0768 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 4F | 3/2 | | T380 | L5951 | + 3573.041 | 3573.0477 | 100 | | | 280989.5229 - 308976.8347 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P).5p | 2P* | 1/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3573.396 | 3573.3958 | 80 | | | 279324.8733 - 307309.4584 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P).5p | 4P* | 1/2 | | | L1406 | + 3575.2030 | 3575.20304 | 200 | 1.0e+07 | D | 246394.1202 - 274364.5596 | 2s2.2p4.(1D).3s | 2D | 5/2 | 2s2.2p4.(1D).3p | 2F* | 5/2 | | T867LS | L5951 | + | 3575.2538 | | 4.6e+06 | D | 252798.4654 - 280768.508 | 2s2.2p4.(3P).3p | 2S* | 1/2 | 2s2.2p4.(3P).3d | 4P | 1/2 | | T380 | | + 3575.63278 | 3575.63267 | 400 | 1.3e+08 | D | 246397.4810 - 274364.5596 | 2s2.2p4.(1D).3s | 2D | 3/2 | 2s2.2p4.(1D).3p | 2F* | 5/2 | | T867LS | L5951 | + 3578.9986 | 3578.9992 | 140 | | | 280797.2387 - 308738.0108 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L5951 | + | | | | | | | | | | | | | | | + 3583.173 | 3583.1713 | 60 | | | 280268.9453 - 308177.1843 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P).5p | 2D* | 3/2 | | | L1406 | + 3585.957 | 3585.9530 | 80 | | | 279422.869 - 307309.4584 | 2s2.2p4.(3P).3d | 4D | 1/2 | 2s2.2p4.(3P).5p | 4P* | 1/2 | | | L1406 | + 3586.267 | 3586.2627 | 40 | | | 290371.9268 - 318256.108 | 2s2.2p4.(3P).4p | 4P* | 5/2 | 2s2.2p4.(3P).7s | 4P | 5/2 | | | L1406 | + 3590.518 | 3590.5139 | 100 | | | 279218.6416 - 307069.8080 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P).5p | 4P* | 3/2 | | | L1406 | + 3591.474 | 3591.4743 | 80 | 3.6e+06 | D | 252953.5198 - 280797.2387 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 4F | 5/2 | | T380 | L1406 | + | | | | | | | | | | | | | | | + 3592.821 | 3592.82194 | 22 | | | 254164.9888 - 281998.2635 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).4s | 4P | 5/2 | | | L1406 | + 3595.1838 | 3595.1840 | 160 | 1.3e+08 | D | 252953.5198 - 280768.508 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 4P | 1/2 | | T380 | L5951 | + 3602.087 | 3602.0868 | 120 | | | 279137.6053 - 306899.2903 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P).5p | 4P* | 5/2 | | | L1406 | + 3603.803 | 3603.8000 | 80 | | | 280989.5229 - 308738.0108 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L1406 | + 3604.262 | 3604.2615 | 100 | | | 279324.8733 - 307069.8080 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P).5p | 4P* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3608.537 | 3608.5349 | 120 | | | 281025.9326 - 308738.0108 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L1406 | + 3609.634 | 3609.6354 | 60 | | | 280473.5550 - 308177.1843 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P).5p | 2D* | 3/2 | | | L1406 | + 3612.626 | 3612.6321 | 100 | | | 279218.6416 - 306899.2903 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P).5p | 4P* | 5/2 | | | L1406 | + 3613.3573 | 3613.3578 | 140 | 2.6e+07 | D | 252798.4654 - 280473.5550 | 2s2.2p4.(3P).3p | 2S* | 1/2 | 2s2.2p4.(3P).3d | 2D | 3/2 | | T380 | L9088 | + 3613.638 | 3613.6361 | 12 | | | 290583.1503 - 318256.108 | 2s2.2p4.(3P).4p | 4P* | 3/2 | 2s2.2p4.(3P).7s | 4P | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3617.376 | 3617.3805 | 22 | | | 281332.521 - 308976.8347 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P).5p | 2P* | 1/2 | | | L1406 | + 3619.795 | 3619.7949 | 80 | | | 280268.9453 - 307894.8198 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P).5p | 4D* | 5/2 | | | L1406 | + 3627.5702 | 3627.5712 | 140 | | | 281171.3566 - 308738.0108 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L5951 | + 3629.0672 | 3629.06767 | 180 | 6.0e+07 | D | 254164.9888 - 281720.2755 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).3d | 2P | 3/2 | | T380 | L5951 | + 3631.964 | 3631.9668 | 80 | | | 280473.5550 - 308006.8460 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P).5p | 2S* | 1/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3633.7159 | 3633.7163 | 140 | 1.3e+07 | D | 252953.5198 - 280473.5550 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 2D | 3/2 | | T380 | L5951 | + 3635.605 | 3635.6047 | 140 | | | 280268.9453 - 307774.6858 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P).5p | 4D* | 3/2 | | | L1406 | + 3640.58 | 3640.579 | 13 | | | 294086.4608 - 321554.62 | 2s2.2p4.(3P).4p | 2P* | 3/2 | 2s2.2p4.(3P).8s | 2P | 3/2 | | | L11529 | + 3644.96751 | 3644.96742 | 300 | 3.2e+07 | D | 224087.0092 - 251522.0967 | 2s2.2p4.(3P).3s | 2P | 3/2 | 2s2.2p4.(3P).3p | 2D* | 3/2 | | T867LS | L5951 | + 3645.8953 | 3645.89504 | 180 | 9.9e+07 | D | 254292.1683 - 281720.2755 | 2s2.2p4.(3P).3p | 2P* | 1/2 | 2s2.2p4.(3P).3d | 2P | 3/2 | | T380 | L5951 | + | | | | | | | | | | | | | | | + 3648.898 | 3648.9040 | 100 | | | 281332.521 - 308738.0108 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L1406 | + 3652.302 | 3652.3082 | 80 | | | 280797.2387 - 308177.1843 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P).5p | 2D* | 3/2 | | | L1406 | + 3653.855 | 3653.8542 | 80 | | | 280768.508 - 308136.8685 | 2s2.2p4.(3P).3d | 4P | 1/2 | 2s2.2p4.(3P).5p | 4S* | 3/2 | | | L1406 | + 3657.692 | 3657.6940 | 80 | | | 280797.2387 - 308136.8685 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P).5p | 4S* | 3/2 | | | L1406 | + 3660.9347 | 3660.9351 | 120 | 6.7e+06 | D | 252953.5198 - 280268.9453 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 2D | 5/2 | | T380 | L5951 | + | | | | | | | | | | | | | | | + 3662.855 | 3662.8519 | 100 | | | 280473.5550 - 307774.6858 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P).5p | 4D* | 3/2 | | | L1406 | + 3665.11766 | 3665.11768 | 400 | 7.0e+07 | D | 219130.7609 - 246415.0144 | 2s2.2p4.(3P).3s | 4P | 5/2 | 2s2.2p4.(3P).3p | 4P* | 3/2 | | T867LS | L5951 | + 3668.8415 | 3668.8417 | 140 | | | 281720.2755 - 308976.8347 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P).5p | 2P* | 1/2 | | | L5951 | + 3677.270 | 3677.2713 | 140 | | | 280700.7426 - 307894.8198 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P).5p | 4D* | 5/2 | | | L1406 | + 3677.852 | 3677.8509 | 100 | | | 280947.0768 - 308136.8685 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P).5p | 4S* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3677.98 | 3677.987 | 15 | | | 291801.858 - 318990.64 | 2s2.2p4.(3P).4p | 4D* | 3/2 | 2s2.2p4.(3P).7s | 4P | 1/2 | | | L11529 | + 3678.138 | 3678.1391 | 100 | | | 280989.5229 - 308177.1843 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P).5p | 2D* | 3/2 | | | L1406 | + 3680.8637 | 3680.8643 | 100 | 3.2e+07 | D | 254164.9888 - 281332.521 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).3d | 2P | 1/2 | | T380 | L5951 | + 3682.084 | 3682.0873 | 140 | | | 280262.3163 - 307420.8248 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P).5p | 2D* | 5/2 | | | L5951 | + 3682.995 | 3682.9863 | 100 | | | 280268.9453 - 307420.8248 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P).5p | 2D* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3683.071 | 3683.0715 | 120 | | | 281025.9326 - 308177.1843 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P).5p | 2D* | 3/2 | | | L5951 | + 3683.596 | 3683.6014 | 100 | | | 280989.5229 - 308136.8685 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P).5p | 4S* | 3/2 | | | L1406 | + 3684.82 | 3684.8444 | 9 | | | 291801.858 - 318940.046 | 2s2.2p4.(3P).4p | 4D* | 3/2 | 2s2.2p4.(3P).7s | 4P | 3/2 | | | L11529c153 | + 3688.548 | 3688.5484 | 100 | | | 281025.9326 - 308136.8685 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P).5p | 4S* | 3/2 | | | L1406 | + 3690.369 | 3690.3663 | 60 | | | 280797.2387 - 307894.8198 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P).5p | 4D* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3693.323 | 3693.3238 | 80 | | | 280947.0768 - 308022.9587 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P).5p | 4D* | 1/2 | | | L1406 | + 3694.440 | 3694.4382 | 140 | | | 280172.9854 - 307240.7001 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P).5p | 4D* | 7/2 | | | L1406 | + 3695.26597 | 3695.26589 | 400 | 1.0e+08 | D | 219130.7609 - 246192.4130 | 2s2.2p4.(3P).3s | 4P | 5/2 | 2s2.2p4.(3P).3p | 4P* | 5/2 | | T867LS | L5951 | + 3695.511 | 3695.5230 | 22 | | | 280947.0768 - 308006.8460 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P).5p | 2S* | 1/2 | | | L1406 | + 3696.293 | 3696.2976 | 80 | | | 291202.0096 - 318256.108 | 2s2.2p4.(3P).4p | 4D* | 7/2 | 2s2.2p4.(3P).7s | 4P | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3696.58 | 3696.588 | 11 | | | 291495.0176 - 318546.99 | 2s2.2p4.(3P).4p | 4D* | 5/2 | 2s2.2p4.(3P).6d | 4F | 7/2 | | | L11529c153 | + 3698.1756 | 3698.1766 | 100 | 2.8e+07 | D | 254292.1683 - 281332.521 | 2s2.2p4.(3P).3p | 2P* | 1/2 | 2s2.2p4.(3P).3d | 2P | 1/2 | | T380 | L5951 | + 3699.120 | 3699.1228 | 80 | | | 280989.5229 - 308022.9587 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P).5p | 4D* | 1/2 | | | L1406 | + 3700.59 | 3700.617 | 10 | | | 291968.116 - 318990.64 | 2s2.2p4.(3P).4p | 4D* | 1/2 | 2s2.2p4.(3P).7s | 4P | 1/2 | | | L11529 | + 3701.273 | 3701.2725 | 120 | | | 281720.2755 - 308738.0108 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3702.8303 | 3702.83041 | 180 | 2.7e+07 | D | 254164.9888 - 281171.3566 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).3d | 4P | 5/2 | | T380 | L5951 | + 3706.799 | 3706.8000 | 120 | | | 280797.2387 - 307774.6858 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P).5p | 4D* | 3/2 | | | L1406 | + 3708.439 | 3708.4406 | 120 | | | 281171.3566 - 308136.8685 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P).5p | 4S* | 3/2 | | | L1406 | + 3709.051 | 3709.0532 | 100 | | | 276276.8361 - 303237.8943 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(3P).4d | 2P | 3/2 | | | L1406 | + 3710.67760 | 3710.67762 | 300 | 1.1e+08 | D | 219648.4248 - 246597.6805 | 2s2.2p4.(3P).3s | 4P | 3/2 | 2s2.2p4.(3P).3p | 4P* | 1/2 | | T867LS | L5951 | + | | | | | | | | | | | | | | | + 3714.13892 | 3714.13880 | 500 | 1.3e+08 | D | 224087.0092 - 251011.1511 | 2s2.2p4.(3P).3s | 2P | 3/2 | 2s2.2p4.(3P).3p | 2D* | 5/2 | | T867LS | L5951 | + 3716.515 | 3716.5161 | 100 | | | 292033.1263 - 318940.046 | 2s2.2p4.(3P).4p | 2D* | 5/2 | 2s2.2p4.(3P).7s | 4P | 3/2 | | | L1406 | + 3718.6782 | 3718.6785 | 31 | | | 291495.0176 - 318386.291 | 2s2.2p4.(3P).4p | 4D* | 5/2 | 2s2.2p4.(3P).7s | 2P | 3/2 | | | L11529 | + 3721.777 | 3721.7768 | 12 | | | 281025.9326 - 307894.8198 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P).5p | 4D* | 5/2 | | | L1406 | + 3722.878 | 3722.87738 | 100 | 2.0e+07 | D | 254164.9888 - 281025.9326 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).3d | 2F | 5/2 | | T380 | L1406 | + | | | | | | | | | | | | | | | + 3727.932 | 3727.9305 | 60 | 1.2e+07 | D | 254164.9888 - 280989.5229 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).3d | 4P | 3/2 | | T380 | L1406 | + 3728.16805 | 3728.16807 | 500 | 9.8e+07 | D | 224699.2716 - 251522.0967 | 2s2.2p4.(3P).3s | 2P | 1/2 | 2s2.2p4.(3P).3p | 2D* | 3/2 | | T867LS | L5951 | + 3730.26 | 3730.267 | 26 | | | 292451.7592 - 319259.49 | 2s2.2p4.(3P).4p | 2D* | 3/2 | 2s2.2p4.(3P).7s | 2P | 1/2 | | | L11529 | + 3732.037 | 3732.0446 | 100 | | | 276276.8361 - 303071.8002 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(3P).4d | 4P | 5/2 | | | L1406 | + 3733.407 | 3733.4102 | 60 | | | 280989.5229 - 307774.6858 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P).5p | 4D* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3733.836 | 3733.8388 | 12w | 3.0e+06 | D | 254164.9888 - 280947.0768 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).3d | 4F | 3/2 | | T380 | L1406 | + 3736.00082 | 3736.00079 | 180 | 1.9e+07 | D | 219648.4248 - 246415.0144 | 2s2.2p4.(3P).3s | 4P | 3/2 | 2s2.2p4.(3P).3p | 4P* | 3/2 | | T867LS | L5951 | + 3739.746 | 3739.7511 | 12 | | | 281998.2635 - 308738.0108 | 2s2.2p4.(3P).4s | 4P | 5/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L1406 | + 3741.6622 | 3741.6616 | 120 | | | 276511.800 - 303237.8943 | 2s2.2p4.(1D).3p | 2P* | 1/2 | 2s2.2p4.(3P).4d | 2P | 3/2 | | | L5951 | + 3742.028 | 3742.0300 | 12 | | | 281171.3566 - 307894.8198 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P).5p | 4D* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3745.6890 | 3745.6895 | 100 | 2.6e+07 | D | 254292.1683 - 280989.5229 | 2s2.2p4.(3P).3p | 2P* | 1/2 | 2s2.2p4.(3P).3d | 4P | 3/2 | | T380 | L5951 | + 3747.531 | 3747.5352 | 80 | | | 276276.8361 - 302961.0417 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(3P).4d | 4F | 3/2 | | | L1406 | + 3748.920 | 3748.9234 | 80 | | | 281332.521 - 308006.8460 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P).5p | 2S* | 1/2 | | | L1406 | + 3751.655 | 3751.6542 | 12 | | | 254292.1683 - 280947.0768 | 2s2.2p4.(3P).3p | 2P* | 1/2 | 2s2.2p4.(3P).3d | 4F | 3/2 | | | L1406 | + 3752.31218 | 3752.31210 | 160 | 1.8e+07 | D | 219947.4453 - 246597.6805 | 2s2.2p4.(3P).3s | 4P | 1/2 | 2s2.2p4.(3P).3p | 4P* | 1/2 | | T867LS | L5951 | + | | | | | | | | | | | | | | | + 3754.8461 | 3754.84611 | 140 | 4.5e+07 | D | 254164.9888 - 280797.2387 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).3d | 4F | 5/2 | | T380 | L5951 | + 3755.946 | 3755.9527 | 12 | 9.5e+05 | D | 252798.4654 - 279422.869 | 2s2.2p4.(3P).3p | 2S* | 1/2 | 2s2.2p4.(3P).3d | 4D | 1/2 | | T380 | L1406 | + 3757.458 | 3757.4631 | 22 | | | 276276.8361 - 302890.5369 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(3P).4d | 2F | 5/2 | | | L1406 | + 3761.58 | 3761.600 | 9 | | | 291801.858 - 318386.291 | 2s2.2p4.(3P).4p | 4D* | 3/2 | 2s2.2p4.(3P).7s | 2P | 3/2 | | | L11529 | + 3764.713 | 3764.7153 | 80 | | | 276276.8361 - 302839.2693 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(3P).4d | 4F | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3765.780 | 3765.7794 | 60 | | | 276276.8361 - 302831.7640 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(3P).4d | 2P | 1/2 | | | L1406 | + 3767.329 | 3767.33139 | 200 | 2.9e+07 | D | 219648.4248 - 246192.4130 | 2s2.2p4.(3P).3s | 4P | 3/2 | 2s2.2p4.(3P).3p | 4P* | 5/2 | | T867LS | L1406 | + 3767.881 | 3767.8799 | 12 | | | 276276.8361 - 302816.9602 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(3P).4d | 4P | 3/2 | | | L1406 | + 3771.61 | 3771.612 | 8 | | | 292033.1263 - 318546.99 | 2s2.2p4.(3P).4p | 2D* | 5/2 | 2s2.2p4.(3P).6d | 4F | 7/2 | | | L11529 | + 3775.24 | 3775.2536 | 3 | | | 292451.7592 - 318940.046 | 2s2.2p4.(3P).4p | 2D* | 3/2 | 2s2.2p4.(3P).7s | 4P | 3/2 | | | L11529c153 | + | | | | | | | | | | | | | | | + | 3777.9546 | | 7.4e+05 | D | 252953.5198 - 279422.869 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 4D | 1/2 | | T380 | | + 3778.20882 | 3778.20871 | 200 | 4.2e+07 | D | 219947.4453 - 246415.0144 | 2s2.2p4.(3P).3s | 4P | 1/2 | 2s2.2p4.(3P).3p | 4P* | 3/2 | | T867LS | L5951 | + 3786.27 | 3786.280 | 5 | | | 292451.7592 - 318862.91 | 2s2.2p4.(3P).4p | 2D* | 3/2 | 2s2.2p4.(3P).6d | 2F | 5/2 | | | L11529 | + 3791.999 | 3791.9935 | 120 | 1.7e+06 | D | 252953.5198 - 279324.8733 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 4D | 3/2 | | T380 | L1406 | + 3793.236 | 3793.2373 | 80 | | | 282375.3049 - 308738.0108 | 2s2.2p4.(3P).4s | 4P | 3/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3794.63 | 3794.611 | 16 | | | 292033.1263 - 318386.291 | 2s2.2p4.(3P).4p | 2D* | 5/2 | 2s2.2p4.(3P).7s | 2P | 3/2 | | | L11529 | + 3796.62 | 3796.602 | 5 | | | 292651.298 - 318990.64 | 2s2.2p4.(3P).4p | 4S* | 3/2 | 2s2.2p4.(3P).7s | 4P | 1/2 | | | L11529 | + 3799.401 | 3799.3973 | 100* | | | 276511.800 - 302831.7640 | 2s2.2p4.(1D).3p | 2P* | 1/2 | 2s2.2p4.(3P).4d | 2P | 1/2 | | | L5951 | + 3799.401 | 3799.4014 | 100* | | | 280989.5229 - 307309.4584 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P).5p | 4P* | 1/2 | | | L5951 | + 3801.0434 | 3801.04333 | 160 | 3.7e+07 | D | 254164.9888 - 280473.5550 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).3d | 2D | 3/2 | | T380 | L5951 | + | | | | | | | | | | | | | | | + 3801.536 | 3801.5355 | 60 | | | 276511.800 - 302816.9602 | 2s2.2p4.(1D).3p | 2P* | 1/2 | 2s2.2p4.(3P).4d | 4P | 3/2 | | | L1406 | + 3801.895 | 3801.8935 | 12 | | | 281720.2755 - 308022.9587 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P).5p | 4D* | 1/2 | | | L1406 | + 3802.092 | 3802.0934 | 12 | | | 280768.508 - 307069.8080 | 2s2.2p4.(3P).3d | 4P | 1/2 | 2s2.2p4.(3P).5p | 4P* | 3/2 | | | L1406 | + 3803.91 | 3803.9088 | 5 | | | 292651.298 - 318940.046 | 2s2.2p4.(3P).4p | 4S* | 3/2 | 2s2.2p4.(3P).7s | 4P | 3/2 | | | L11529c153 | + 3807.332 | 3807.33053 | 120 | 1.3e+06 | D | 252953.5198 - 279218.6416 | 2s2.2p4.(3P).3p | 4S* | 3/2 | 2s2.2p4.(3P).3d | 4D | 5/2 | | T380 | L1406 | + | | | | | | | | | | | | | | | + 3819.5073 | 3819.50739 | 200 | 6.1e+07 | D | 254292.1683 - 280473.5550 | 2s2.2p4.(3P).3p | 2P* | 1/2 | 2s2.2p4.(3P).3d | 2D | 3/2 | | T380 | L5951 | + 3824.242 | 3824.2413 | 100 | | | 276276.8361 - 302425.8139 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(3P).4d | 2D | 3/2 | | | L1406 | + 3830.83690 | 3830.83691 | 240 | 8.4e+07 | D | 254164.9888 - 280268.9453 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).3d | 2D | 5/2 | | T380 | L5951 | + 3841.589 | 3841.5902 | 140 | | | 276276.8361 - 302307.7230 | 2s2.2p4.(1D).3p | 2P* | 3/2 | 2s2.2p4.(3P).4d | 2D | 5/2 | | | L1406 | + 3858.906 | 3858.9159 | 60 | | | 276511.800 - 302425.8139 | 2s2.2p4.(1D).3p | 2P* | 1/2 | 2s2.2p4.(3P).4d | 2D | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3870.03 | 3870.028 | 8 | | | 292546.687 - 318386.291 | 2s2.2p4.(3P).4p | 2S* | 1/2 | 2s2.2p4.(3P).7s | 2P | 3/2 | | | L11529 | + 3897.951 | 3897.9512 | 100 | | | 283322.3319 - 308976.8347 | 2s2.2p4.(3P).4s | 2P | 3/2 | 2s2.2p4.(3P).5p | 2P* | 1/2 | | | L1406 | + | 3904.95406 | | 8.4e+04 | D | 224087.0092 - 249695.5051 | 2s2.2p4.(3P).3s | 2P | 3/2 | 2s2.2p4.(3P).3p | 4D* | 3/2 | | T380 | | + 3931.411 | 3931.4081 | 12 | | | 276511.800 - 301947.9789 | 2s2.2p4.(1D).3p | 2P* | 1/2 | 2s2.2p4.(3P).4d | 4D | 3/2 | | | L1406 | + 3934.583 | 3934.5791 | 180 | | | 283322.3319 - 308738.0108 | 2s2.2p4.(3P).4s | 2P | 3/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 3943.374 | 3943.38031 | 160 | 1.0e+06 | D | 224087.0092 - 249445.9632 | 2s2.2p4.(3P).3s | 2P | 3/2 | 2s2.2p4.(3P).3p | 4D* | 5/2 | | T380 | L1406 | + 3959.158 | 3959.1604 | 22 | | | 254164.9888 - 279422.869 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).3d | 4D | 1/2 | | | L1406 | + 3972.52 | 3972.506 | 10 | | | 294086.4608 - 319259.49 | 2s2.2p4.(3P).4p | 2P* | 3/2 | 2s2.2p4.(3P).7s | 2P | 1/2 | | | L11529 | + 3974.579 | 3974.5810 | 12 | | | 254164.9888 - 279324.8733 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(3P).3d | 4D | 3/2 | | | L1406 | + 3986.852 | 3986.8532 | 160 | | | 283894.3965 - 308976.8347 | 2s2.2p4.(3P).4s | 2P | 1/2 | 2s2.2p4.(3P).5p | 2P* | 1/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4000.602 | 4000.60273 | 100 | 3.7e+05 | D | 224699.2716 - 249695.5051 | 2s2.2p4.(3P).3s | 2P | 1/2 | 2s2.2p4.(3P).3p | 4D* | 3/2 | | T380 | L1406 | + 4011.81 | 4011.827 | 10 | | | 294333.192 - 319259.49 | 2s2.2p4.(3P).4p | 2P* | 1/2 | 2s2.2p4.(3P).7s | 2P | 1/2 | | | L11529 | + 4023.56 | 4023.5644 | 5 | | | 294086.4608 - 318940.046 | 2s2.2p4.(3P).4p | 2P* | 3/2 | 2s2.2p4.(3P).7s | 4P | 3/2 | | | L11529c153 | + 4025.180 | 4025.1792 | 80 | | | 283894.3965 - 308738.0108 | 2s2.2p4.(3P).4s | 2P | 1/2 | 2s2.2p4.(3P).5p | 2P* | 3/2 | | | L1406 | + 4055.58 | 4055.570 | 6 | | | 294333.192 - 318990.64 | 2s2.2p4.(3P).4p | 2P* | 1/2 | 2s2.2p4.(3P).7s | 4P | 1/2 | | | L11529 | + | | | | | | | | | | | | | | | + 4064.1204 | 4064.1204 | 160 | | | 279218.6416 - 303824.2119 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 7/2 | | | L5951 | + 4081.662 | 4081.6664 | 140 | | | 279324.8733 - 303824.6705 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 5/2 | | | L9088 | + 4087.926 | 4087.9223 | 80 | | | 279137.6053 - 303599.9098 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 7/2 | | | L9088 | + 4099.889 | 4099.8886 | 12 | | | 279137.6053 - 303528.5122 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P<1>).4f | 2[4]* | 7/2 | | | L1406 | + 4100.0213 | 4100.0206 | 160 | | | 279137.6053 - 303527.7267 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P<1>).4f | 2[4]* | 9/2 | | | L9088 | + | | | | | | | | | | | | | | | + 4101.510 | 4101.5094 | 120 | | | 279218.6416 - 303599.9098 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 7/2 | | | L1406 | + 4103.07 | 4103.0845 | 12w | | | 279137.6053 - 303509.514 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 5/2 | | | L1406 | + 4113.554 | 4113.5554 | 100 | | | 279218.6416 - 303528.5122 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P<1>).4f | 2[4]* | 7/2 | | | L1406 | + 4115.29 | 4115.255 | 16 | | | 294086.4608 - 318386.291 | 2s2.2p4.(3P).4p | 2P* | 3/2 | 2s2.2p4.(3P).7s | 2P | 3/2 | | | L11529 | + 4119.361 | 4119.3585 | 80 | | | 279324.8733 - 303600.498 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4134.857 | 4134.8557 | 180 | | | 279324.8733 - 303509.514 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 5/2 | | | L1406 | + 4135.034 | 4135.0363 | 60 | | | 279324.8733 - 303508.458 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 3/2 | | | L1406 | + 4151.8597 | 4151.8603 | 180 | | | 279422.869 - 303508.458 | 2s2.2p4.(3P).3d | 4D | 1/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 3/2 | | | L5951 | + 4187.83 | 4187.8417 | 40w | | | 252798.4654 - 276677.1142 | 2s2.2p4.(3P).3p | 2S* | 1/2 | 2s2.2p4.(1S).3s | 2S | 1/2 | | | L1406 | + 4206.781 | 4206.7777 | 80 | | | 279218.6416 - 302989.805 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4207.686 | 4207.6864 | 100 | | | 279137.6053 - 302903.635 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | | | L1406 | + | 4218.2560 | | 7.4e+06 | D | 279137.6053 - 302844.085 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | | T1115CA,LS | | + 4218.3581 | 4218.3585 | 180 | | | 279137.6053 - 302843.509 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 7/2 | | | L9088 | + 4220.555 | 4220.5542 | 160 | 3.3e+07 | D | 279137.6053 - 302831.176 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P<2>).4f | 2[4]* | 7/2 | | T1115CA,LS | L1406 | + 4220.9337 | 4220.9338 | 300 | | | 279137.6053 - 302829.0453 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P<2>).4f | 2[4]* | 9/2 | | | L9088 | + | | | | | | | | | | | | | | | + 4222.0823 | 4222.0827 | 180 | | | 279218.6416 - 302903.635 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | | | L9088 | + 4222.275 | 4222.2748 | 120 | 1.4e+07 | D | 279218.6416 - 302902.557 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 3/2 | | T1115CA,LS | L1406 | + 4225.663 | 4225.6619 | 100 | | | 279324.8733 - 302989.805 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | | | L1406 | + 4225.831 | 4225.8308 | 120 | 1.9e+07 | D | 279324.8733 - 302988.859 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 1/2 | | T1115CA,LS | L1406 | + | 4228.1105 | | 7.3e+05 | D | 280172.9854 - 303824.2119 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 7/2 | | T1115CA,LS | | + | | | | | | | | | | | | | | | + 4232.7248 | 4232.7248 | 160bl | 2.2e+07 | D | 279218.6416 - 302844.085 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | | T1115CA,LS | L5951 | + 4232.8278 | 4232.8280 | 180bl | | | 279218.6416 - 302843.509 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 7/2 | | | L9088 | + 4235.0388 | 4235.0388 | 200 | 5.5e+06 | D | 279218.6416 - 302831.176 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P<2>).4f | 2[4]* | 7/2 | | T1115CA,LS | L9088 | + 4241.1128 | 4241.1048 | 160 | 1.5e+07 | D | 279324.8733 - 302903.635 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | | T1115CA,LS | L5951 | + 4241.2988 | 4241.2987 | 180 | | | 279324.8733 - 302902.557 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 3/2 | | | L5951 | + | | | | | | | | | | | | | | | + 4243.232 | 4243.2330 | 60 | | | 279422.869 - 302989.805 | 2s2.2p4.(3P).3d | 4D | 1/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | | | L1406 | + 4243.404 | 4243.4033 | 120 | 1.9e+07 | D | 279422.869 - 302988.859 | 2s2.2p4.(3P).3d | 4D | 1/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 1/2 | | T1115CA,LS | L1406 | + | 4244.0581 | | 4.8e+06 | D | 280262.3163 - 303824.6705 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 5/2 | | T1115CA,LS | | + | 4244.1407 | | 3.3e+07 | D | 280262.3163 - 303824.2119 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 7/2 | | T1115CA,LS | | + 4251.8435 | 4251.8432 | 240 | 9.0e+06 | D | 279324.8733 - 302844.085 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | | T1115CA,LS | L9088 | + | | | | | | | | | | | | | | | + 4258.379 | 4258.378 | 80 | | | 305364.004 - 328847.124 | 2s2.2p4.(1D).3d | 2G | 9/2 | 2s2.2p4.(1D<2>).4f | 2[4]* | 9/2 | | | L1406 | + 4258.593 | 4258.597 | 80 | | | 305365.116 - 328847.027 | 2s2.2p4.(1D).3d | 2G | 7/2 | 2s2.2p4.(1D<2>).4f | 2[4]* | 7/2 | | | L1406 | + 4259.0009 | 4259.0004 | 180 | 9.6e+06 | D | 279422.869 - 302902.557 | 2s2.2p4.(3P).3d | 4D | 1/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 3/2 | | T1115CA,LS | L9088 | + 4263.642 | 4263.656 | 40 | | | 305364.004 - 328818.054 | 2s2.2p4.(1D).3d | 2G | 9/2 | 2s2.2p4.(1D<2>).4f | 2[3]* | 7/2 | | | L1406 | + 4268.583 | 4268.5928 | 60 | | | 280172.9854 - 303599.9098 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 7/2 | | | L1406 | + | | | | | | | | | | | | | | | + | 4281.6418 | | 2.8e+06 | D | 280172.9854 - 303528.5122 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P<1>).4f | 2[4]* | 7/2 | | T1115CA,LS | | + | 4281.7858 | | 2.0e+07 | D | 280172.9854 - 303527.7267 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P<1>).4f | 2[4]* | 9/2 | | T1115CA,LS | | + 4291.581 | 4291.581 | 180 | | | 305364.004 - 328665.44 | 2s2.2p4.(1D).3d | 2G | 9/2 | 2s2.2p4.(1D<2>).4f | 2[5]* | 11/2 | | | L1406 | + 4291.809 | 4291.810 | 160 | | | 305365.116 - 328665.31 | 2s2.2p4.(1D).3d | 2G | 7/2 | 2s2.2p4.(1D<2>).4f | 2[5]* | 9/2 | | | L1406 | + 4295.48 | 4295.4929 | 19 | | | 290371.9268 - 313652.1424 | 2s2.2p4.(3P).4p | 4P* | 5/2 | 2s2.2p4.(3P).6s | 4P | 3/2 | | | L11529c153 | + | | | | | | | | | | | | | | | + 4299.303 | 4299.3062 | 12 | | | 280268.9453 - 303528.5122 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P<1>).4f | 2[4]* | 7/2 | | | L1406 | + 4323.587 | 4323.587 | 160 | | | 305566.923 - 328695.864 | 2s2.2p4.(1D).3d | 2P | 3/2 | 2s2.2p4.(1D<2>).4f | 2[2]* | 5/2 | | | L1406 | + 4323.958 | 4323.9610 | 140 | | | 280473.5550 - 303600.498 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 5/2 | | | L9088 | + 4325.84 | 4325.8293 | 30 | | | 290583.1503 - 313700.1049 | 2s2.2p4.(3P).4p | 4P* | 3/2 | 2s2.2p4.(3P).6s | 4P | 1/2 | | | L11529c153 | + 4326.448 | 4326.451 | 120 | | | 305582.249 - 328695.882 | 2s2.2p4.(1D).3d | 2P | 1/2 | 2s2.2p4.(1D<2>).4f | 2[2]* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4341.0410 | 4341.0392 | 160 | | | 280473.5550 - 303509.514 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 5/2 | | | L9088 | + 4341.240 | 4341.2382 | 40 | | | 280473.5550 - 303508.458 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 3/2 | | | L1406 | + 4342.735 | 4342.7332 | 160 | | | 280797.2387 - 303824.2119 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 7/2 | | | L1406 | + 4343.419 | 4343.4275 | 22 | | | 282375.3049 - 305398.5968 | 2s2.2p4.(3P).4s | 4P | 3/2 | 2s2.2p4.(1S).3p | 2P* | 3/2 | | | L1406 | + 4347.461 | 4347.459 | 160 | | | 305566.923 - 328568.864 | 2s2.2p4.(1D).3d | 2P | 3/2 | 2s2.2p4.(1D<2>).4f | 2[1]* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4350.358 | 4350.357 | 40w | | | 305582.249 - 328568.87 | 2s2.2p4.(1D).3d | 2P | 1/2 | 2s2.2p4.(1D<2>).4f | 2[1]* | 1/2 | | | L1406 | + | 4366.8589 | | 1.2e+06 | D | 280700.7426 - 303600.498 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 5/2 | | T1115CA,LS | | + 4366.972 | 4366.9710 | 160 | | | 280700.7426 - 303599.9098 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 7/2 | | | L1406 | + 4371.090 | 4371.0891 | 240 | 3.5e+06 | D | 280947.0768 - 303824.6705 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 5/2 | | T1115CA,LS | L1406 | + 4379.2131 | 4379.2141 | 180 | | | 280989.5229 - 303824.6705 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 5/2 | | | L9088 | + | | | | | | | | | | | | | | | + 4380.631 | 4380.6295 | 140bl | 2.0e+07 | D | 280700.7426 - 303528.5122 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<1>).4f | 2[4]* | 7/2 | | T1115CA,LS | L1406 | + 4380.7802 | 4380.78020 | 300 | 2.2e+08 | D | 280700.7426 - 303527.7267 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<1>).4f | 2[4]* | 9/2 | | T1115CA,LS | L5951 | + 4385.3397 | 4385.3380 | 180 | | | 280797.2387 - 303600.498 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 5/2 | | | L5951 | + 4385.455 | 4385.4511 | 80 | | | 280797.2387 - 303599.9098 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 7/2 | | | L1406 | + | 4386.2077 | | 1.8e+07 | D | 281025.9326 - 303824.6705 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 5/2 | | T1115CA,LS | | + | | | | | | | | | | | | | | | + 4386.2962 | 4386.2959 | 200 | 2.0e+08 | D | 281025.9326 - 303824.2119 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 7/2 | | T1115CA,LS | L5951 | + 4393.2240 | 4393.2240 | 400 | | | 280172.9854 - 302935.312 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P<2>).4f | 2[5]* | 11/2 | | | L9088 | + | 4393.2279 | | 2.4e+07 | D | 280172.9854 - 302935.2919 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P<2>).4f | 2[5]* | 9/2 | | T1115CA,LS | | + 4397.548 | 4397.5470 | 120 | | | 280768.508 - 303508.458 | 2s2.2p4.(3P).3d | 4P | 1/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 3/2 | | | L1406 | + 4399.2256 | 4399.2256 | 300 | | | 280797.2387 - 303528.5122 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<1>).4f | 2[4]* | 7/2 | | | L5951 | + | | | | | | | | | | | | | | | + 4410.53711 | 4410.53710 | 300 | | | 280262.3163 - 302935.2919 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P<2>).4f | 2[5]* | 9/2 | | | L5951 | + 4411.017 | 4411.0141 | 40w | | | 280172.9854 - 302843.509 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 7/2 | | | L1406 | + 4413.44 | 4413.4151 | 80w | | | 280172.9854 - 302831.176 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P<2>).4f | 2[4]* | 7/2 | | | L1406 | + 4413.830 | 4413.8301 | 180 | | | 280172.9854 - 302829.0453 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P<2>).4f | 2[4]* | 9/2 | | | L1406 | + 4414.352 | 4414.3443 | 160* | 2.0e+08 | D | 280947.0768 - 303600.498 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 5/2 | | T1115CA,LS | L9088 | + | | | | | | | | | | | | | | | + 4414.352 | 4414.3652 | 160* | | | 281171.3566 - 303824.6705 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 5/2 | | | L9088 | + 4414.4544 | 4414.4545 | 200 | | | 281171.3566 - 303824.2119 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 7/2 | | | L5951 | + 4417.9956 | 4417.9974 | 160 | | | 280268.9453 - 302903.635 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | | | L9088 | + 4422.6304 | 4422.6310 | 200 | | | 280989.5229 - 303600.498 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 5/2 | | | L5951 | + 4428.351 | 4428.3511 | 60 | | | 280262.3163 - 302844.085 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4429.6509 | 4429.6514 | 160bl | | | 280268.9453 - 302844.085 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | | | L9088 | + 4429.763 | 4429.748 | 200bl* | | | 306243.4077 - 328818.054 | 2s2.2p4.(1D).3d | 2D | 5/2 | 2s2.2p4.(1D<2>).4f | 2[3]* | 7/2 | | | L5951 | + 4429.763 | 4429.7641 | 200bl* | 3.3e+07 | D | 281025.9326 - 303600.498 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 5/2 | | T1115CA,LS | L5951 | + 4429.763 | 4429.7645 | 200bl* | | | 280268.9453 - 302843.509 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 7/2 | | | L5951 | + 4429.8794 | 4429.8796 | 200bl | | | 281025.9326 - 303599.9098 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 7/2 | | | L9088 | + | | | | | | | | | | | | | | | + 4430.884 | 4430.8840 | 180 | | | 280262.3163 - 302831.176 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P<2>).4f | 2[4]* | 7/2 | | | L1406 | + 4432.1459 | 4432.1453 | 300bl | | | 280947.0768 - 303509.514 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 5/2 | | | L5951 | + 4432.1858 | 4432.1859 | 300bl | | | 280268.9453 - 302831.176 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P<2>).4f | 2[4]* | 7/2 | | | L5951 | + 4432.349 | 4432.3527 | 12 | 2.1e+07 | D | 280947.0768 - 303508.458 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 3/2 | | T1115CA,LS | L1406 | + 4433.054 | 4433.054 | 160* | | | 306011.056 - 328568.87 | 2s2.2p4.(1D).3d | 2S | 1/2 | 2s2.2p4.(1D<2>).4f | 2[1]* | 1/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4433.054 | 4433.055 | 160* | | | 306011.056 - 328568.864 | 2s2.2p4.(1D).3d | 2S | 1/2 | 2s2.2p4.(1D<2>).4f | 2[1]* | 3/2 | | | L1406 | + 4433.547 | 4433.549 | 160 | | | 306262.755 - 328818.048 | 2s2.2p4.(1D).3d | 2D | 3/2 | 2s2.2p4.(1D<2>).4f | 2[3]* | 5/2 | | | L1406 | + 4438.225 | 4438.2234 | 100 | | | 290371.9268 - 312903.4650 | 2s2.2p4.(3P).4p | 4P* | 5/2 | 2s2.2p4.(3P).6s | 4P | 5/2 | | | L1406 | + 4440.4988 | 4440.4991 | 180 | | | 280989.5229 - 303509.514 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 5/2 | | | L5951 | + 4440.7073 | 4440.7073 | 180 | | | 280989.5229 - 303508.458 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 3/2 | | | L5951 | + | | | | | | | | | | | | | | | + 4441.2379 | 4441.2369 | 180 | | | 280473.5550 - 302989.805 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | | | L5951 | + 4442.051 | 4442.0506 | 80 | | | 254164.9888 - 276677.1142 | 2s2.2p4.(3P).3p | 2P* | 3/2 | 2s2.2p4.(1S).3s | 2S | 1/2 | | | L1406 | + | 4442.3375 | | 3.4e+05 | D | 224087.0092 - 246597.6805 | 2s2.2p4.(3P).3s | 2P | 3/2 | 2s2.2p4.(3P).3p | 4P* | 1/2 | | T380 | | + 4443.9337 | 4443.9350 | 180 | 3.5e+06 | D | 281025.9326 - 303528.5122 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<1>).4f | 2[4]* | 7/2 | | T1115CA,LS | L5951 | + 4447.6880 | 4447.6900 | 180 | | | 281025.9326 - 303509.514 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 5/2 | | | L9088 | + | | | | | | | | | | | | | | | + | 4447.8989 | | 5.2e+06 | D | 281025.9326 - 303508.458 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 3/2 | | T1115CA,LS | | + 4453.857 | 4453.856 | 140 | | | 306243.4077 - 328695.864 | 2s2.2p4.(1D).3d | 2D | 5/2 | 2s2.2p4.(1D<2>).4f | 2[2]* | 5/2 | | | L1406 | + 4457.696 | 4457.693 | 80 | | | 306262.755 - 328695.882 | 2s2.2p4.(1D).3d | 2D | 3/2 | 2s2.2p4.(1D<2>).4f | 2[2]* | 3/2 | | | L1406 | + 4458.2983 | 4458.2989 | 240 | | | 280473.5550 - 302903.635 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | | | L9088 | + 4458.493 | 4458.4854 | 100* | | | 281171.3566 - 303600.498 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4458.493 | 4458.5131 | 100* | | | 280473.5550 - 302902.557 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 3/2 | | | L1406 | + 4458.6018 | 4458.6023 | 160 | | | 281171.3566 - 303599.9098 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 7/2 | | | L9088 | + 4467.290 | 4467.2880 | 40bl | | | 254292.1683 - 276677.1142 | 2s2.2p4.(3P).3p | 2P* | 1/2 | 2s2.2p4.(1S).3s | 2S | 1/2 | | | L1406 | + 4468.989 | 4468.9909 | 40 | | | 290803.7989 - 313180.2149 | 2s2.2p4.(3P).4p | 4P* | 1/2 | 2s2.2p4.(3P).6s | 2P | 3/2 | | | L1406 | + 4470.1673 | 4470.1668 | 180 | | | 280473.5550 - 302844.085 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | | | L5951 | + | | | | | | | | | | | | | | | + 4472.841 | 4472.8409 | 160 | | | 281171.3566 - 303528.5122 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<1>).4f | 2[4]* | 7/2 | | | L1406 | + 4476.646 | 4476.6450 | 60 | | | 281171.3566 - 303509.514 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 5/2 | | | L1406 | + 4478.7 | 4478.6804 | | | | 224087.0092 - 246415.0144 | 2s2.2p4.(3P).3s | 2P | 3/2 | 2s2.2p4.(3P).3p | 4P* | 3/2 | | | L9088c153 | + 4479.190 | 4479.192 | 22 | | | 306243.4077 - 328568.864 | 2s2.2p4.(1D).3d | 2D | 5/2 | 2s2.2p4.(1D<2>).4f | 2[1]* | 3/2 | | | L1406 | + 4497.504 | 4497.5052 | 60 | 2.1e+06 | D | 280700.7426 - 302935.2919 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<2>).4f | 2[5]* | 9/2 | | T1115CA,LS | L1406 | + | | | | | | | | | | | | | | | + 4500.1867 | 4500.1874 | 180 | | | 280768.508 - 302989.805 | 2s2.2p4.(3P).3d | 4P | 1/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | | | L9088 | + 4500.3787 | 4500.3790 | 180 | | | 280768.508 - 302988.859 | 2s2.2p4.(3P).3d | 4P | 1/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 1/2 | | | L5951 | + 4509.3931 | 4509.3923 | 180 | | | 281332.521 - 303508.458 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 3/2 | | | L9088 | + 4512.685 | 4512.685 | 160 | | | 306687.271 - 328847.027 | 2s2.2p4.(1D).3d | 2F | 5/2 | 2s2.2p4.(1D<2>).4f | 2[4]* | 7/2 | | | L1406 | + 4512.743 | 4512.743 | 160 | | | 306687.654 - 328847.124 | 2s2.2p4.(1D).3d | 2F | 7/2 | 2s2.2p4.(1D<2>).4f | 2[4]* | 9/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4516.150 | 4516.1475 | 140 | | | 280700.7426 - 302843.509 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 7/2 | | | L1406 | + 4517.930 | 4517.9262 | 60 | | | 280768.508 - 302902.557 | 2s2.2p4.(3P).3d | 4P | 1/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 3/2 | | | L1406 | + 4518.596 | 4518.594 | 80 | | | 306687.271 - 328818.048 | 2s2.2p4.(1D).3d | 2F | 5/2 | 2s2.2p4.(1D<2>).4f | 2[3]* | 5/2 | | | L1406 | + | 4518.6643 | m | | | 280700.7426 - 302831.176 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<2>).4f | 2[4]* | 7/2 | | | L1406 | + 4518.671 | 4518.671 | 120 | | | 306687.654 - 328818.054 | 2s2.2p4.(1D).3d | 2F | 7/2 | 2s2.2p4.(1D<2>).4f | 2[3]* | 7/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4519.1004 | 4519.0994 | 160 | | | 280700.7426 - 302829.0453 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P<2>).4f | 2[4]* | 9/2 | | | L5951 | + 4523.582 | 4523.5776 | 60 | | | 280797.2387 - 302903.635 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | | | L1406 | + 4523.7 | 4523.7807 | | | | 224087.0092 - 246192.4130 | 2s2.2p4.(3P).3s | 2P | 3/2 | 2s2.2p4.(3P).3p | 4P* | 5/2 | | | L9088c153 | + 4523.792 | 4523.7982 | 12 | | | 280797.2387 - 302902.557 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 3/2 | | | L1406 | + 4523.9874 | 4523.9872 | 200 | | | 281720.2755 - 303824.6705 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 5/2 | | | L5951 | + | | | | | | | | | | | | | | | + 4527.727 | 4527.7211 | 80w | | | 283322.3319 - 305408.498 | 2s2.2p4.(3P).4s | 2P | 3/2 | 2s2.2p4.(1S).3p | 2P* | 1/2 | | | L1406 | + 4529.753 | 4529.7518 | 160 | | | 283322.3319 - 305398.5968 | 2s2.2p4.(3P).4s | 2P | 3/2 | 2s2.2p4.(1S).3p | 2P* | 3/2 | | | L1406 | + 4535.79 | 4535.7961 | 22w | | | 280797.2387 - 302844.085 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | | | L1406 | + 4535.915 | 4535.9146 | 80w | | | 280797.2387 - 302843.509 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 7/2 | | | L1406 | + 4536.646 | 4536.6435 | 140 | | | 280947.0768 - 302989.805 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4536.844 | 4536.8382 | 40w | | | 280947.0768 - 302988.859 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 1/2 | | | L1406 | + 4538.467 | 4538.4535 | 22 | | | 280797.2387 - 302831.176 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P<2>).4f | 2[4]* | 7/2 | | | L1406 | + 4543.66 | 4543.676 | 12bl | | | 306687.271 - 328695.882 | 2s2.2p4.(1D).3d | 2F | 5/2 | 2s2.2p4.(1D<2>).4f | 2[2]* | 3/2 | | | L1406 | + 4543.75 | 4543.759 | 12bl | | | 306687.654 - 328695.864 | 2s2.2p4.(1D).3d | 2F | 7/2 | 2s2.2p4.(1D<2>).4f | 2[2]* | 5/2 | | | L1406 | + 4545.39 | 4545.3963 | 12w | | | 280989.5229 - 302989.805 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4545.59 | 4545.5917 | 12w | | | 280989.5229 - 302988.859 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 1/2 | | | L1406 | + 4552.95 | 4552.9312 | 12w | | | 281025.9326 - 302989.805 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | | | L1406 | + 4554.4495 | 4554.4479 | 140 | | | 280947.0768 - 302903.635 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | | | L9088 | + 4554.675 | 4554.6715 | 100 | | | 280947.0768 - 302902.557 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 3/2 | | | L1406 | + 4563.273 | 4563.2695 | 12 | | | 280989.5229 - 302903.635 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4563.48 | 4563.4940 | 40w | | | 280989.5229 - 302902.557 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 3/2 | | | L1406 | + 4566.4 | 4566.5418 | | | | 224699.2716 - 246597.6805 | 2s2.2p4.(3P).3s | 2P | 1/2 | 2s2.2p4.(3P).3p | 4P* | 1/2 | | | L9088c153 | + 4566.819 | 4566.8339 | 22 | | | 280947.0768 - 302844.085 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | | | L9088 | + 4570.338 | 4570.3374 | 200 | | | 281720.2755 - 303600.498 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 5/2 | | | L1406 | + 4570.885 | 4570.8639 | 22 | | | 281025.9326 - 302903.635 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4575.702 | 4575.7037 | 40 | | | 280989.5229 - 302844.085 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | | | L1406 | + 4576.593 | 4576.5995 | 40 | | | 291801.858 - 313652.1424 | 2s2.2p4.(3P).4p | 4D* | 3/2 | 2s2.2p4.(3P).6s | 4P | 3/2 | | | L1406 | + 4581.7016 | 4581.7024 | 160 | | | 281998.2635 - 303824.2119 | 2s2.2p4.(3P).4s | 4P | 5/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 7/2 | | | L9088 | + 4583.28 | 4583.2773 | 40bl | | | 281171.3566 - 302989.805 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | | | L1406 | + 4583.327 | 4583.3395 | 60bl | | | 281025.9326 - 302844.085 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4589.4186 | 4589.4215 | 160 | | | 281720.2755 - 303509.514 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 5/2 | | | L9088 | + 4589.644 | 4589.6440 | 22 | | | 281720.2755 - 303508.458 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 3/2 | | | L1406 | + 4601.441 | 4601.4503 | 40bl | | | 281171.3566 - 302903.635 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | | | L9088 | + 4605.759 | 4605.761 | 40 | | | 301801.0594 - 323513.00 | 2s2.2p4.(3P).4d | 4D | 7/2 | 2s2.2p4.(3P<2>).8f | 2[4]* | 9/2 | | | L1406 | + 4607.988 | 4607.9859 | 100 | | | 291202.0096 - 312903.4650 | 2s2.2p4.(3P).4p | 4D* | 7/2 | 2s2.2p4.(3P).6s | 4P | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4611.442 | 4611.4406 | 80 | | | 291495.0176 - 313180.2149 | 2s2.2p4.(3P).4p | 4D* | 5/2 | 2s2.2p4.(3P).6s | 2P | 3/2 | | | L1406 | + 4614.094 | 4614.0937 | 12 | | | 281171.3566 - 302844.085 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | | | L1406 | + 4614.220 | 4614.2164 | 22w | | | 281171.3566 - 302843.509 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 7/2 | | | L1406 | + 4616.856 | 4616.8437 | 22w | | | 281171.3566 - 302831.176 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P<2>).4f | 2[4]* | 7/2 | | | L1406 | + 4617.3841 | 4617.3842 | 180 | | | 281332.521 - 302989.805 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | | | L5951 | + | | | | | | | | | | | | | | | + 4617.59 | 4617.5859 | 12w | | | 281332.521 - 302988.859 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 1/2 | | | L1406 | + 4625.546 | 4625.5574 | 120 | | | 292033.1263 - 313652.1424 | 2s2.2p4.(3P).4p | 2D* | 5/2 | 2s2.2p4.(3P).6s | 4P | 3/2 | | | L9088 | + 4629.2771 | 4629.2768 | 160 | | | 281998.2635 - 303599.9098 | 2s2.2p4.(3P).4s | 4P | 5/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 7/2 | | | L9088 | + 4636.0602 | 4636.0609 | 140 | | | 281332.521 - 302902.557 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 3/2 | | | L5951 | + 4636.680 | 4636.6805 | 100 | | | 292451.7592 - 314018.9129 | 2s2.2p4.(3P).4p | 2D* | 3/2 | 2s2.2p4.(3P).6s | 2P | 1/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4648.111 | 4648.1142 | 100 | | | 283894.3965 - 305408.498 | 2s2.2p4.(3P).4s | 2P | 1/2 | 2s2.2p4.(1S).3p | 2P* | 1/2 | | | L1406 | + 4648.730 | 4648.7302 | 40 | | | 281998.2635 - 303509.514 | 2s2.2p4.(3P).4s | 4P | 5/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 5/2 | | | L1406 | + 4650.246 | 4650.2543 | 22 | | | 283894.3965 - 305398.5968 | 2s2.2p4.(3P).4s | 2P | 1/2 | 2s2.2p4.(1S).3p | 2P* | 3/2 | | | L1406 | + 4651.302 | 4651.3048 | 40 | | | 291495.0176 - 312994.3611 | 2s2.2p4.(3P).4p | 4D* | 5/2 | 2s2.2p4.(3P).5d | 4F | 7/2 | | | L1406 | + 4662.142 | 4662.1425 | 60 | | | 282375.3049 - 303824.6705 | 2s2.2p4.(3P).4s | 4P | 3/2 | 2s2.2p4.(3P<0>).4f | 2[3]* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4670.679 | 4670.678 | 60 | | | 302750.7840 - 324160.95 | 2s2.2p4.(3P).4d | 4F | 7/2 | 2s2.2p4.(3P<1>).8f | 2[4]* | 9/2 | | | L1406 | + 4679.623 | 4679.624 | 60 | | | 303071.8002 - 324441.04 | 2s2.2p4.(3P).4d | 4P | 5/2 | 2s2.2p4.(3P<0>).8f | 2[3]* | 7/2 | | | L1406 | + 4687.667 | 4687.665 | 60 | | | 302192.9098 - 323525.49 | 2s2.2p4.(3P).4d | 4F | 9/2 | 2s2.2p4.(3P<2>).8f | 2[5]* | 11/2 | | | L1406 | + 4689.983 | 4689.985 | 40 | | | 302839.2693 - 324161.30 | 2s2.2p4.(3P).4d | 4F | 5/2 | 2s2.2p4.(3P<1>).8f | 2[4]* | 7/2 | | | L1406 | + 4699.189 | 4699.192 | 22 | | | 302890.5369 - 324170.79 | 2s2.2p4.(3P).4d | 2F | 5/2 | 2s2.2p4.(3P<1>).8f | 2[3]* | 7/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4699.745 | 4699.746 | 60 | | | 302247.7634 - 323525.51 | 2s2.2p4.(3P).4d | 2F | 7/2 | 2s2.2p4.(3P<2>).8f | 2[5]* | 9/2 | | | L1406 | + 4701.558 | 4701.5615 | 12 | | | 281720.2755 - 302989.805 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | | | L1406 | + 4701.798 | 4701.7706 | 12 | | | 281720.2755 - 302988.859 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<2>).4f | 2[1]* | 1/2 | | | L1406 | + | 4711.3823 | m | | | 282375.3049 - 303600.498 | 2s2.2p4.(3P).4s | 4P | 3/2 | 2s2.2p4.(3P<1>).4f | 2[3]* | 5/2 | | | L1406 | + 4714.287 | 4714.2907 | 22 | | | 291968.116 - 313180.2149 | 2s2.2p4.(3P).4p | 4D* | 1/2 | 2s2.2p4.(3P).6s | 2P | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4714.777 | 4714.773 | 22 | | | 302961.0417 - 324170.97 | 2s2.2p4.(3P).4d | 4F | 3/2 | 2s2.2p4.(3P<1>).8f | 2[3]* | 5/2 | | | L1406 | + 4715.732 | 4715.7327 | 120 | | | 291202.0096 - 312407.6220 | 2s2.2p4.(3P).4p | 4D* | 7/2 | 2s2.2p4.(3P).5d | 4F | 9/2 | | | L1406 | + 4716.254 | 4716.250 | 40 | | | 303237.8943 - 324441.18 | 2s2.2p4.(3P).4d | 2P | 3/2 | 2s2.2p4.(3P<0>).8f | 2[3]* | 5/2 | | | L1406 | + 4720.683 | 4720.6865 | 22 | | | 281720.2755 - 302903.635 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | | | L1406 | + 4720.928 | 4720.9268 | 12 | | | 281720.2755 - 302902.557 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4731.662 | 4731.6651 | 22 | | | 282375.3049 - 303509.514 | 2s2.2p4.(3P).4s | 4P | 3/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 5/2 | | | L1406 | + 4731.90 | 4731.9016 | 22w | | | 282375.3049 - 303508.458 | 2s2.2p4.(3P).4s | 4P | 3/2 | 2s2.2p4.(3P<1>).4f | 2[2]* | 3/2 | | | L1406 | + 4733.991 | 4733.9946 | 80bl | | | 281720.2755 - 302844.085 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | | | L1406 | + 4738.110 | 4738.1115 | 22 | | | 292546.687 - 313652.1424 | 2s2.2p4.(3P).4p | 2S* | 1/2 | 2s2.2p4.(3P).6s | 4P | 3/2 | | | L1406 | + 4770.709 | 4770.7113 | 100 | | | 292033.1263 - 312994.3611 | 2s2.2p4.(3P).4p | 2D* | 5/2 | 2s2.2p4.(3P).5d | 4F | 7/2 | | | L1406 | + | | | | | | | | | | | | | | | + 4774.262 | 4774.2634 | 120 | | | 291495.0176 - 312440.6573 | 2s2.2p4.(3P).4p | 4D* | 5/2 | 2s2.2p4.(3P).5d | 2F | 7/2 | | | L1406 | + 4783.458 | 4783.4596 | 60 | | | 281998.2635 - 302903.635 | 2s2.2p4.(3P).4s | 4P | 5/2 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | | | L1406 | + 4797.259 | 4797.2570 | 60 | | | 281998.2635 - 302843.509 | 2s2.2p4.(3P).4s | 4P | 5/2 | 2s2.2p4.(3P<2>).4f | 2[3]* | 7/2 | | | L1406 | + 4900.152 | 4900.1518 | 22 | | | 292033.1263 - 312440.6573 | 2s2.2p4.(3P).4p | 2D* | 5/2 | 2s2.2p4.(3P).5d | 2F | 7/2 | | | L1406 | + 4937.748 | 4937.7432 | 40 | | | 292651.298 - 312903.4650 | 2s2.2p4.(3P).4p | 4S* | 3/2 | 2s2.2p4.(3P).6s | 4P | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 5079.830 | 5079.8241 | 22 | | | 294333.192 - 314018.9129 | 2s2.2p4.(3P).4p | 2P* | 1/2 | 2s2.2p4.(3P).6s | 2P | 1/2 | | | L1406 | + 5100.967 | 5100.955 | 100 | | | 301801.0594 - 321405.23 | 2s2.2p4.(3P).4d | 4D | 7/2 | 2s2.2p4.(3P<2>).7f | 2[4]* | 9/2 | | | L9088 | + | 5115.027 | m | | | 301855.6707 - 321405.91 | 2s2.2p4.(3P).4d | 4D | 5/2 | 2s2.2p4.(3P<2>).7f | 2[4]* | 7/2 | | | L1406 | + 5138.581 | 5138.582 | 40 | | | 301947.9789 - 321408.60 | 2s2.2p4.(3P).4d | 4D | 3/2 | 2s2.2p4.(3P<2>).7f | 2[3]* | 5/2 | | | L1406 | + 5179.950 | 5179.953 | 140 | | | 302750.7840 - 322055.98 | 2s2.2p4.(3P).4d | 4F | 7/2 | 2s2.2p4.(3P<1>).7f | 2[4]* | 9/2 | | | L1406 | + | | | | | | | | | | | | | | | + 5190.609 | 5190.612 | 120 | | | 303071.8002 - 322337.35 | 2s2.2p4.(3P).4d | 4P | 5/2 | 2s2.2p4.(3P<0>).7f | 2[3]* | 7/2 | | | L1406 | + 5199.012 | 5199.005 | 80 | | | 302816.9602 - 322051.41 | 2s2.2p4.(3P).4d | 4P | 3/2 | 2s2.2p4.(3P<1>).7f | 2[2]* | 5/2 | | | L1406 | + 5199.929 | 5199.924 | 160 | | | 302192.9098 - 321423.96 | 2s2.2p4.(3P).4d | 4F | 9/2 | 2s2.2p4.(3P<2>).7f | 2[5]* | 11/2 | | | L1406 | + 5203.689 | 5203.693 | 140 | | | 302839.2693 - 322056.39 | 2s2.2p4.(3P).4d | 4F | 5/2 | 2s2.2p4.(3P<1>).7f | 2[4]* | 7/2 | | | L1406 | + 5205.03 | 5204.993 | 22bl | | | 302192.9098 - 321405.23 | 2s2.2p4.(3P).4d | 4F | 9/2 | 2s2.2p4.(3P<2>).7f | 2[4]* | 9/2 | | | L1406 | + | | | | | | | | | | | | | | | + 5213.814 | 5213.812 | 80 | | | 302890.5369 - 322070.36 | 2s2.2p4.(3P).4d | 2F | 5/2 | 2s2.2p4.(3P<1>).7f | 2[3]* | 7/2 | | | L1406 | + 5214.791 | 5214.790 | 140 | | | 302247.7634 - 321423.99 | 2s2.2p4.(3P).4d | 2F | 7/2 | 2s2.2p4.(3P<2>).7f | 2[5]* | 9/2 | | | L1406 | + 5219.721 | 5219.712 | 22 | | | 302247.7634 - 321405.91 | 2s2.2p4.(3P).4d | 2F | 7/2 | 2s2.2p4.(3P<2>).7f | 2[4]* | 7/2 | | | L1406 | + 5232.979 | 5232.983 | 80 | | | 302961.0417 - 322070.60 | 2s2.2p4.(3P).4d | 4F | 3/2 | 2s2.2p4.(3P<1>).7f | 2[3]* | 5/2 | | | L1406 | + | 5235.41 | m | | | 302307.7230 - 321408.42 | 2s2.2p4.(3P).4d | 2D | 5/2 | 2s2.2p4.(3P<2>).7f | 2[3]* | 7/2 | | | L1406 | + | | | | | | | | | | | | | | | + 5236.094 | 5236.099 | 60 | | | 302307.7230 - 321405.91 | 2s2.2p4.(3P).4d | 2D | 5/2 | 2s2.2p4.(3P<2>).7f | 2[4]* | 7/2 | | | L1406 | + 5237.325 | 5237.3148 | 80 | | | 294086.4608 - 313180.2149 | 2s2.2p4.(3P).4p | 2P* | 3/2 | 2s2.2p4.(3P).6s | 2P | 3/2 | | | L1406 | + 5247.294 | 5247.289 | 40 | | | 303279.8915 - 322337.35 | 2s2.2p4.(3P).5s | 4P | 5/2 | 2s2.2p4.(3P<0>).7f | 2[3]* | 7/2 | | | L1406 | + 5264.588 | 5264.586 | 80 | | | 302425.8139 - 321420.66 | 2s2.2p4.(3P).4d | 2D | 3/2 | 2s2.2p4.(3P<2>).7f | 2[2]* | 5/2 | | | L1406 | + 5378.130 | 5378.138 | 80 | | | 302829.0453 - 321422.84 | 2s2.2p4.(3P<2>).4f | 2[4]* | 9/2 | 2s2.2p4.(3P<2>).7g | 2[5] | 11/2 | | | L1406 | + | | | | | | | | | | | | | | | + 5378.750 | 5378.758 | 80 | | | 302831.176 - 321422.83 | 2s2.2p4.(3P<2>).4f | 2[4]* | 7/2 | 2s2.2p4.(3P<2>).7g | 2[5] | 9/2 | | | L1406 | + 5382.217 | 5382.218 | 80 | | | 302843.509 - 321423.21 | 2s2.2p4.(3P<2>).4f | 2[3]* | 7/2 | 2s2.2p4.(3P<2>).7g | 2[4] | 9/2 | | | L1406 | + 5382.382 | 5382.382 | 60 | | | 302844.085 - 321423.22 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | 2s2.2p4.(3P<2>).7g | 2[4] | 7/2 | | | L1406 | + 5388.396 | 5388.395 | 60 | | | 303508.458 - 322066.86 | 2s2.2p4.(3P<1>).4f | 2[2]* | 3/2 | 2s2.2p4.(3P<1>).7g | 2[3] | 5/2 | | | L1406 | + 5388.707 | 5388.707 | 80 | | | 303509.514 - 322066.84 | 2s2.2p4.(3P<1>).4f | 2[2]* | 5/2 | 2s2.2p4.(3P<1>).7g | 2[3] | 7/2 | | | L1406 | + | | | | | | | | | | | | | | | + 5393.500 | 5393.501 | 100 | | | 303527.7267 - 322068.56 | 2s2.2p4.(3P<1>).4f | 2[4]* | 9/2 | 2s2.2p4.(3P<1>).7g | 2[5] | 11/2 | | | L1406 | + 5393.747 | 5393.747 | 100 | | | 303528.5122 - 322068.50 | 2s2.2p4.(3P<1>).4f | 2[4]* | 7/2 | 2s2.2p4.(3P<1>).7g | 2[5] | 9/2 | | | L1406 | + 5397.97 | 5397.97 | 22w | | | 302902.557 - 321428.05 | 2s2.2p4.(3P<2>).4f | 2[2]* | 3/2 | 2s2.2p4.(3P<2>).7g | 2[3] | 5/2 | | | L1406 | + 5398.282 | 5398.281 | 60 | | | 302903.635 - 321428.05 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | 2s2.2p4.(3P<2>).7g | 2[3] | 7/2 | | | L1406 | + 5398.466 | 5398.464 | 120 | | | 303824.2119 - 322348.00 | 2s2.2p4.(3P<0>).4f | 2[3]* | 7/2 | 2s2.2p4.(3P<0>).7g | 2[4] | 9/2 | | | L1406 | + | | | | | | | | | | | | | | | + 5398.597 | 5398.597 | 100 | | | 303824.6705 - 322348.00 | 2s2.2p4.(3P<0>).4f | 2[3]* | 5/2 | 2s2.2p4.(3P<0>).7g | 2[4] | 7/2 | | | L1406 | + 5406.655 | 5406.65 | 140* | | | 302935.312 - 321431.04 | 2s2.2p4.(3P<2>).4f | 2[5]* | 11/2 | 2s2.2p4.(3P<2>).7g | 2[6] | 13/2 | | | L1406 | + 5406.655 | 5406.65 | 140* | | | 302935.2919 - 321431.02 | 2s2.2p4.(3P<2>).4f | 2[5]* | 9/2 | 2s2.2p4.(3P<2>).7g | 2[6] | 11/2 | | | L1406 | + 5412.587 | 5412.588 | 60 | | | 303599.9098 - 322075.36 | 2s2.2p4.(3P<1>).4f | 2[3]* | 7/2 | 2s2.2p4.(3P<1>).7g | 2[4] | 9/2 | | | L1406 | + 5412.767 | 5412.766 | 100 | | | 303600.498 - 322075.34 | 2s2.2p4.(3P<1>).4f | 2[3]* | 5/2 | 2s2.2p4.(3P<1>).7g | 2[4] | 7/2 | | | L1406 | + | | | | | | | | | | | | | | | + 5421.60 | 5421.60 | 12w,bl | | | 302989.805 - 321434.54 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | 2s2.2p4.(3P<2>).7g | 2[2] | 5/2 | | | L1406 | + 5663.771 | 5663.7721 | 40 | | | 276677.1142 - 294333.192 | 2s2.2p4.(1S).3s | 2S | 1/2 | 2s2.2p4.(3P).4p | 2P* | 1/2 | | | L1406 | + 5744.030 | 5744.0410 | 80 | | | 276677.1142 - 294086.4608 | 2s2.2p4.(1S).3s | 2S | 1/2 | 2s2.2p4.(3P).4p | 2P* | 3/2 | | | L1406 | + 6112.57 | 6112.5021 | 100 | | | 301801.0594 - 318160.9720 | 2s2.2p4.(3P).4d | 4D | 7/2 | 2s2.2p4.(3P<2>).6f | 2[3]* | 7/2 | | | L1406 | + 6114.41 | 6114.3409 | 160 | | | 301801.0594 - 318156.0520 | 2s2.2p4.(3P).4d | 4D | 7/2 | 2s2.2p4.(3P<2>).6f | 2[4]* | 9/2 | | | L1406 | + | | | | | | | | | | | | | | | + 6134.47 | 6134.4153 | 100 | | | 301855.6707 - 318157.1429 | 2s2.2p4.(3P).4d | 4D | 5/2 | 2s2.2p4.(3P<2>).6f | 2[4]* | 7/2 | | | L1406 | + 6167.80 | 6167.7726 | 80 | | | 301947.9789 - 318161.2874 | 2s2.2p4.(3P).4d | 4D | 3/2 | 2s2.2p4.(3P<2>).6f | 2[3]* | 5/2 | | | L1406 | + 6226.20 | 6226.1347 | 160 | | | 302750.7840 - 318812.1133 | 2s2.2p4.(3P).4d | 4F | 7/2 | 2s2.2p4.(3P<1>).6f | 2[4]* | 9/2 | | | L1406 | + 6240.758 | 6240.7514 | 120 | | | 303071.8002 - 319095.5118 | 2s2.2p4.(3P).4d | 4P | 5/2 | 2s2.2p4.(3P<0>).6f | 2[3]* | 7/2 | | | L1406 | + 6252.654 | 6252.6535 | 120 | | | 302192.9098 - 318186.1197 | 2s2.2p4.(3P).4d | 4F | 9/2 | 2s2.2p4.(3P<2>).6f | 2[5]* | 11/2 | | | L1406 | + | | | | | | | | | | | | | | | + 6254.635 | 6254.6281 | 80 | | | 302816.9602 - 318805.121 | 2s2.2p4.(3P).4d | 4P | 3/2 | 2s2.2p4.(3P<1>).6f | 2[2]* | 5/2 | | | L1406 | + 6260.42 | 6260.3819 | 180bl | | | 302839.2693 - 318812.7356 | 2s2.2p4.(3P).4d | 4F | 5/2 | 2s2.2p4.(3P<1>).6f | 2[4]* | 7/2 | | | L1406 | + 6271.865 | 6271.8663 | 80 | | | 302890.5369 - 318834.7543 | 2s2.2p4.(3P).4d | 2F | 5/2 | 2s2.2p4.(3P<1>).6f | 2[3]* | 7/2 | | | L1406 | + 6274.155 | 6274.1565 | 140 | | | 302247.7634 - 318186.1608 | 2s2.2p4.(3P).4d | 2F | 7/2 | 2s2.2p4.(3P<2>).6f | 2[5]* | 9/2 | | | L1406 | + 6299.582 | 6299.5752 | 80 | | | 302961.0417 - 318835.128 | 2s2.2p4.(3P).4d | 4F | 3/2 | 2s2.2p4.(3P<1>).6f | 2[3]* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + | 6305.9994 | m | | | 303237.8943 - 319095.8089 | 2s2.2p4.(3P).4d | 2P | 3/2 | 2s2.2p4.(3P<0>).6f | 2[3]* | 5/2 | | | L1406 | + 6307.857 | 6307.8553 | 100 | | | 302307.7230 - 318160.9720 | 2s2.2p4.(3P).4d | 2D | 5/2 | 2s2.2p4.(3P<2>).6f | 2[3]* | 7/2 | | | L1406 | + 6309.40 | 6309.3792 | 180bl | | | 302307.7230 - 318157.1429 | 2s2.2p4.(3P).4d | 2D | 5/2 | 2s2.2p4.(3P<2>).6f | 2[4]* | 7/2 | | | L1406 | + 6322.864 | 6322.8630 | 60 | | | 303279.8915 - 319095.5118 | 2s2.2p4.(3P).5s | 4P | 5/2 | 2s2.2p4.(3P<0>).6f | 2[3]* | 7/2 | | | L1406 | + 6347.408 | 6347.3983 | 60 | | | 302425.8139 - 318180.3002 | 2s2.2p4.(3P).4d | 2D | 3/2 | 2s2.2p4.(3P<2>).6f | 2[2]* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 6512.676 | 6512.6865 | 60 | | | 302829.0453 - 318183.6919 | 2s2.2p4.(3P<2>).4f | 2[4]* | 9/2 | 2s2.2p4.(3P<2>).6g | 2[5] | 11/2 | | | L1406 | + 6513.588 | 6513.5956 | 140 | | | 302831.176 - 318183.6797 | 2s2.2p4.(3P<2>).4f | 2[4]* | 7/2 | 2s2.2p4.(3P<2>).6g | 2[5] | 9/2 | | | L1406 | + 6518.590 | 6518.599 | 120 | | | 302843.509 - 318184.2286 | 2s2.2p4.(3P<2>).4f | 2[3]* | 7/2 | 2s2.2p4.(3P<2>).6g | 2[4] | 9/2 | | | L1406 | + 6518.842 | 6518.8483 | 100 | | | 302844.085 - 318184.2181 | 2s2.2p4.(3P<2>).4f | 2[3]* | 5/2 | 2s2.2p4.(3P<2>).6g | 2[4] | 7/2 | | | L1406 | + 6527.387 | 6527.399 | 22 | | | 303508.458 - 318828.4968 | 2s2.2p4.(3P<1>).4f | 2[2]* | 3/2 | 2s2.2p4.(3P<1>).6g | 2[3] | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 6527.83 | 6527.851 | 120w | | | 303509.514 - 318828.4917 | 2s2.2p4.(3P<1>).4f | 2[2]* | 5/2 | 2s2.2p4.(3P<1>).6g | 2[3] | 7/2 | | | L1406 | + 6540.491 | 6540.491 | 80 | | | 302902.557 - 318191.9290 | 2s2.2p4.(3P<2>).4f | 2[2]* | 3/2 | 2s2.2p4.(3P<2>).6g | 2[3] | 5/2 | | | L1406 | + 6540.950 | 6540.946 | 120 | | | 302903.635 - 318191.9428 | 2s2.2p4.(3P<2>).4f | 2[2]* | 5/2 | 2s2.2p4.(3P<2>).6g | 2[3] | 7/2 | | | L1406 | + 6541.299 | 6541.294 | 140 | | | 303824.2119 - 319111.7080 | 2s2.2p4.(3P<0>).4f | 2[3]* | 7/2 | 2s2.2p4.(3P<0>).6g | 2[4] | 9/2 | | | L1406 | + 6541.499 | 6541.497 | 60 | | | 303824.6705 - 319111.6901 | 2s2.2p4.(3P<0>).4f | 2[3]* | 5/2 | 2s2.2p4.(3P<0>).6g | 2[4] | 7/2 | | | L1406 | + | | | | | | | | | | | | | | | + 6552.463 | 6552.4529 | 140* | | | 302935.312 - 318196.7724 | 2s2.2p4.(3P<2>).4f | 2[5]* | 11/2 | 2s2.2p4.(3P<2>).6g | 2[6] | 13/2 | | | L1406 | + 6552.463 | 6552.467 | 140* | | | 302935.2919 - 318196.72 | 2s2.2p4.(3P<2>).4f | 2[5]* | 9/2 | 2s2.2p4.(3P<2>).6g | 2[6] | 11/2 | | | L1406 | + 6560.836 | 6560.832 | 100 | | | 303599.9098 - 318841.8788 | 2s2.2p4.(3P<1>).4f | 2[3]* | 7/2 | 2s2.2p4.(3P<1>).6g | 2[4] | 9/2 | | | L1406 | + 6561.097 | 6561.099 | 100 | | | 303600.498 - 318841.8468 | 2s2.2p4.(3P<1>).4f | 2[3]* | 5/2 | 2s2.2p4.(3P<1>).6g | 2[4] | 7/2 | | | L1406 | + 6573.12 | 6573.106 | 22w | | | 302988.859 - 318202.3663 | 2s2.2p4.(3P<2>).4f | 2[1]* | 1/2 | 2s2.2p4.(3P<2>).6g | 2[2] | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 6573.52 | 6573.504 | 60w | | | 302989.805 - 318202.3922 | 2s2.2p4.(3P<2>).4f | 2[1]* | 3/2 | 2s2.2p4.(3P<2>).6g | 2[2] | 5/2 | | | L1406 | + 7069.673 | 7069.6757 | 80 | | | 303518.1843 - 317663.1046 | 2s2.2p4.(3P).5s | 4P | 3/2 | 2s2.2p4.(1D).4p | 2P* | 3/2 | | | L1406 | + 7135.326 | 7135.3212 | 40 | | | 294086.4608 - 308101.2468 | 2s2.2p4.(3P).4p | 2P* | 3/2 | 2s2.2p4.(1D).4s | 2D | 5/2 | | | L1406 | + 7215.198 | 7215.1962 | 300 | | | 280473.5550 - 294333.192 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P).4p | 2P* | 1/2 | | | L5951 | + 7237.1917 | 7237.1911 | 300 | | | 280268.9453 - 294086.4608 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P).4p | 2P* | 3/2 | | | L9088 | + | | | | | | | | | | | | | | | + 7305.228 | 7305.2200 | 80 | | | 303974.2630 - 317663.1046 | 2s2.2p4.(3P).5s | 2P | 3/2 | 2s2.2p4.(1D).4p | 2P* | 3/2 | | | L1406 | + 7345.968 | 7345.9702 | 300 | | | 280473.5550 - 294086.4608 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P).4p | 2P* | 3/2 | | | L1406 | + 7351.678 | 7351.6783 | 40 | | | 290371.9268 - 303974.2630 | 2s2.2p4.(3P).4p | 4P* | 5/2 | 2s2.2p4.(3P).5s | 2P | 3/2 | | | L1406 | + 7469.30 | 7469.3175 | 60c | | | 304440.2089 - 317828.3130 | 2s2.2p4.(3P).5s | 2P | 1/2 | 2s2.2p4.(1D).4p | 2P* | 1/2 | | | L1406 | + 7493.471 | 7493.4852 | 60 | | | 290583.1503 - 303928.0756 | 2s2.2p4.(3P).4p | 4P* | 3/2 | 2s2.2p4.(3P).5s | 4P | 1/2 | | | L1406 | + | | | | | | | | | | | | | | | + 7494.165 | 7494.1906 | 200 | | | 280989.5229 - 294333.192 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P).4p | 2P* | 1/2 | | | L1406 | + 7524.893 | 7524.8949 | 300 | | | 280797.2387 - 294086.4608 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P).4p | 2P* | 3/2 | | | L5951 | + 7592.728 | 7592.7469 | 60 | | | 290803.7989 - 303974.2630 | 2s2.2p4.(3P).4p | 4P* | 1/2 | 2s2.2p4.(3P).5s | 2P | 3/2 | | | L1406 | + 7606.727 | 7606.7276 | 60 | | | 290371.9268 - 303518.1843 | 2s2.2p4.(3P).4p | 4P* | 5/2 | 2s2.2p4.(3P).5s | 4P | 3/2 | | | L1406 | + 7635.368 | 7635.3725 | 140 | | | 280989.5229 - 294086.4608 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P).4p | 2P* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 7656.653 | 7656.6582 | 180 | | | 281025.9326 - 294086.4608 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P).4p | 2P* | 3/2 | | | L1406 | + 7691.905 | 7691.9107 | 120 | | | 281332.521 - 294333.192 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P).4p | 2P* | 1/2 | | | L1406 | + 7730.930 | 7730.9422 | 22 | | | 290583.1503 - 303518.1843 | 2s2.2p4.(3P).4p | 4P* | 3/2 | 2s2.2p4.(3P).5s | 4P | 3/2 | | | L1406 | + 7742.870 | 7742.8721 | 240 | | | 281171.3566 - 294086.4608 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P).4p | 2P* | 3/2 | | | L5951 | + 7747.150 | 7747.1547 | 160 | | | 290371.9268 - 303279.8915 | 2s2.2p4.(3P).4p | 4P* | 5/2 | 2s2.2p4.(3P).5s | 4P | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 7840.713 | 7840.7144 | 160 | | | 281332.521 - 294086.4608 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P).4p | 2P* | 3/2 | | | L1406 | + 7865.109 | 7865.1069 | 140 | | | 290803.7989 - 303518.1843 | 2s2.2p4.(3P).4p | 4P* | 1/2 | 2s2.2p4.(3P).5s | 4P | 3/2 | | | L1406 | + 7874.090 | 7874.0942 | 60 | | | 290371.9268 - 303071.8002 | 2s2.2p4.(3P).4p | 4P* | 5/2 | 2s2.2p4.(3P).4d | 4P | 5/2 | | | L1406 | + 7876.039 | 7876.0367 | 160 | | | 290583.1503 - 303279.8915 | 2s2.2p4.(3P).4p | 4P* | 3/2 | 2s2.2p4.(3P).5s | 4P | 5/2 | | | L1406 | + 7909.359 | 7909.3633 | 12 | | | 279324.8733 - 291968.116 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P).4p | 4D* | 1/2 | | | L1406 | + | | | | | | | | | | | | | | | + 7928.381 | 7928.3804 | 240 | | | 281720.2755 - 294333.192 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P).4p | 2P* | 1/2 | | | L5951 | + 7947.104 | 7947.0937 | 60 | | | 279218.6416 - 291801.858 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P).4p | 4D* | 3/2 | | | L1406 | + 7988.082 | 7988.1072 | 22 | | | 290371.9268 - 302890.5369 | 2s2.2p4.(3P).4p | 4P* | 5/2 | 2s2.2p4.(3P).4d | 2F | 5/2 | | | L1406 | + 8020.961 | 8020.9555 | 22 | | | 290371.9268 - 302839.2693 | 2s2.2p4.(3P).4p | 4P* | 5/2 | 2s2.2p4.(3P).4d | 4F | 5/2 | | | L1406 | + 8076.000 | 8076.010 | 22 | | | 280268.9453 - 292651.298 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P).4p | 4S* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 8078.214 | 8078.2041 | 22 | | | 279422.869 - 291801.858 | 2s2.2p4.(3P).3d | 4D | 1/2 | 2s2.2p4.(3P).4p | 4D* | 3/2 | | | L1406 | + 8078.303 | 8078.2901 | 40 | | | 290371.9268 - 302750.7840 | 2s2.2p4.(3P).4p | 4P* | 5/2 | 2s2.2p4.(3P).4d | 4F | 7/2 | | | L1406 | + 8086.568 | 8086.5681 | 200 | | | 281720.2755 - 294086.4608 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P).4p | 2P* | 3/2 | | | L1406 | + 8092.27 | 8092.3091 | 160bl | | | 279137.6053 - 291495.0176 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P).4p | 4D* | 5/2 | | | L1406 | + 8145.732 | 8145.7264 | 100 | | | 279218.6416 - 291495.0176 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P).4p | 4D* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 8152.226 | 8152.224 | 60 | | | 316882.1781 - 329148.77 | 2s2.2p4.(1D).4p | 2F* | 5/2 | 2s2.2p4.(1D).5s | 2D | 3/2 | | | L1406 | + 8155.690 | 8155.6822 | 80 | | | 305566.923 - 317828.3130 | 2s2.2p4.(1D).3d | 2P | 3/2 | 2s2.2p4.(1D).4p | 2P* | 1/2 | | | L1406 | + 8162.295 | 8162.289 | 100 | | | 316897.1647 - 329148.63 | 2s2.2p4.(1D).4p | 2F* | 7/2 | 2s2.2p4.(1D).5s | 2D | 5/2 | | | L1406 | + 8165.900 | 8165.889 | 120 | | | 305582.249 - 317828.3130 | 2s2.2p4.(1D).3d | 2P | 1/2 | 2s2.2p4.(1D).4p | 2P* | 1/2 | | | L1406 | + 8187.015 | 8187.004 | 80 | | | 305566.923 - 317781.404 | 2s2.2p4.(1D).3d | 2P | 3/2 | 2s2.2p4.(1D).4p | 2D* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 8197.08 | 8197.045 | 360bl | | | 305582.249 - 317781.767 | 2s2.2p4.(1D).3d | 2P | 1/2 | 2s2.2p4.(1D).4p | 2D* | 3/2 | | | L1406 | + 8208.299 | 8208.2843 | 140 | | | 280268.9453 - 292451.7592 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P).4p | 2D* | 3/2 | | | L1406 | + 8215.293 | 8215.3034 | 22 | | | 291801.858 - 303974.2630 | 2s2.2p4.(3P).4p | 4D* | 3/2 | 2s2.2p4.(3P).5s | 2P | 3/2 | | | L1406 | + 8216.834 | 8216.8294 | 60 | | | 279324.8733 - 291495.0176 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P).4p | 4D* | 5/2 | | | L1406 | + 8246.604 | 8246.5946 | 140 | | | 291801.858 - 303928.0756 | 2s2.2p4.(3P).4p | 4D* | 3/2 | 2s2.2p4.(3P).5s | 4P | 1/2 | | | L1406 | + | | | | | | | | | | | | | | | + 8267.079 | 8267.0717 | 200 | | | 305566.923 - 317663.1046 | 2s2.2p4.(1D).3d | 2P | 3/2 | 2s2.2p4.(1D).4p | 2P* | 3/2 | | | L1406 | + 8279.603 | 8279.5974 | 120 | | | 291202.0096 - 303279.8915 | 2s2.2p4.(3P).4p | 4D* | 7/2 | 2s2.2p4.(3P).5s | 4P | 5/2 | | | L1406 | + 8282.874 | 8282.8549 | 60 | | | 280473.5550 - 292546.687 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P).4p | 2S* | 1/2 | | | L1406 | + 8288.852 | 8288.8469 | 180 | | | 279137.6053 - 291202.0096 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P).4p | 4D* | 7/2 | | | L1406 | + 8317.280 | 8317.2763 | 200 | | | 291495.0176 - 303518.1843 | 2s2.2p4.(3P).4p | 4D* | 5/2 | 2s2.2p4.(3P).5s | 4P | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 8324.206 | 8324.2036 | 100 | | | 290803.7989 - 302816.9602 | 2s2.2p4.(3P).4p | 4P* | 1/2 | 2s2.2p4.(3P).4d | 4P | 3/2 | | | L1406 | + 8341.357 | 8341.3621 | 180 | | | 292451.7592 - 304440.2089 | 2s2.2p4.(3P).4p | 2D* | 3/2 | 2s2.2p4.(3P).5s | 2P | 1/2 | | | L1406 | + 8344.899 | 8344.8994 | 60 | | | 279218.6416 - 291202.0096 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P).4p | 4D* | 7/2 | | | L1406 | + 8348.494 | 8348.4968 | 120 | | | 280473.5550 - 292451.7592 | 2s2.2p4.(3P).3d | 2D | 3/2 | 2s2.2p4.(3P).4p | 2D* | 3/2 | | | L1406 | + 8361.226 | 8361.2323 | 120 | | | 291968.116 - 303928.0756 | 2s2.2p4.(3P).4p | 4D* | 1/2 | 2s2.2p4.(3P).5s | 4P | 1/2 | | | L1406 | + | | | | | | | | | | | | | | | + 8362.683 | 8362.6814 | 100 | | | 282375.3049 - 294333.192 | 2s2.2p4.(3P).4s | 4P | 3/2 | 2s2.2p4.(3P).4p | 2P* | 1/2 | | | L1406 | + 8374.407 | 8374.4121 | 200 | | | 292033.1263 - 303974.2630 | 2s2.2p4.(3P).4p | 2D* | 5/2 | 2s2.2p4.(3P).5s | 2P | 3/2 | | | L1406 | + 8408.706 | 8408.7102 | 80 | | | 290583.1503 - 302475.5806 | 2s2.2p4.(3P).4p | 4P* | 3/2 | 2s2.2p4.(3P).4d | 4P | 1/2 | | | L1406 | + 8415.524 | 8415.532 | 120 | | | 280768.508 - 292651.298 | 2s2.2p4.(3P).3d | 4P | 1/2 | 2s2.2p4.(3P).4p | 4S* | 3/2 | | | L1406 | + 8424.739 | 8424.7485 | 100 | | | 291202.0096 - 303071.8002 | 2s2.2p4.(3P).4p | 4D* | 7/2 | 2s2.2p4.(3P).4d | 4P | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 8435.954 | 8435.929 | 12 | | | 280797.2387 - 292651.298 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P).4p | 4S* | 3/2 | | | L1406 | + 8462.18 | 8462.2007 | 100bl | | | 306011.056 - 317828.3130 | 2s2.2p4.(1D).3d | 2S | 1/2 | 2s2.2p4.(1D).4p | 2P* | 1/2 | | | L1406 | + 8495.586 | 8495.5921 | 140 | | | 280262.3163 - 292033.1263 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P).4p | 2D* | 5/2 | | | L1406 | + 8500.375 | 8500.3792 | 80 | | | 280268.9453 - 292033.1263 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P).4p | 2D* | 5/2 | | | L1406 | + 8535.097 | 8535.0986 | 40 | | | 291801.858 - 303518.1843 | 2s2.2p4.(3P).4p | 4D* | 3/2 | 2s2.2p4.(3P).5s | 4P | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 8538.860 | 8538.8668 | 160 | | | 282375.3049 - 294086.4608 | 2s2.2p4.(3P).4s | 4P | 3/2 | 2s2.2p4.(3P).4p | 2P* | 3/2 | | | L1406 | + 8543.922 | 8543.926 | 100 | | | 280947.0768 - 292651.298 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P).4p | 4S* | 3/2 | | | L1406 | + 8555.399 | 8555.3977 | 60 | | | 291202.0096 - 302890.5369 | 2s2.2p4.(3P).4p | 4D* | 7/2 | 2s2.2p4.(3P).4d | 2F | 5/2 | | | L1406 | + 8575.010 | 8575.024 | 80 | | | 280989.5229 - 292651.298 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P).4p | 4S* | 3/2 | | | L1406 | + 8581.805 | 8581.8025 | 22 | | | 282680.6284 - 294333.192 | 2s2.2p4.(3P).4s | 4P | 1/2 | 2s2.2p4.(3P).4p | 2P* | 1/2 | | | L1406 | + | | | | | | | | | | | | | | | + 8582.176 | 8582.1818 | 80 | | | 306011.056 - 317663.1046 | 2s2.2p4.(1D).3d | 2S | 1/2 | 2s2.2p4.(1D).4p | 2P* | 3/2 | | | L1406 | + 8601.870 | 8601.880 | 100 | | | 281025.9326 - 292651.298 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P).4p | 4S* | 3/2 | | | L1406 | + 8667.008 | 8667.016 | 180 | | | 306243.4077 - 317781.404 | 2s2.2p4.(1D).3d | 2D | 5/2 | 2s2.2p4.(1D).4p | 2D* | 5/2 | | | L1406 | + 8670.648 | 8670.6500 | 200 | | | 305364.004 - 316897.1647 | 2s2.2p4.(1D).3d | 2G | 9/2 | 2s2.2p4.(1D).4p | 2F* | 7/2 | | | L5951 | + 8681.25 | 8681.300 | 100bl | | | 306262.755 - 317781.767 | 2s2.2p4.(1D).3d | 2D | 3/2 | 2s2.2p4.(1D).4p | 2D* | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 8682.751 | 8682.7699 | 180 | | | 305365.116 - 316882.1781 | 2s2.2p4.(1D).3d | 2G | 7/2 | 2s2.2p4.(1D).4p | 2F* | 5/2 | | | L1406 | + 8707.949 | 8707.9615 | 60 | | | 290371.9268 - 301855.6707 | 2s2.2p4.(3P).4p | 4P* | 5/2 | 2s2.2p4.(3P).4d | 4D | 5/2 | | | L1406 | + 8710.844 | 8710.846 | 60 | | | 281171.3566 - 292651.298 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P).4p | 4S* | 3/2 | | | L5951 | + 8749.572 | 8749.5704 | 100 | | | 290371.9268 - 301801.0594 | 2s2.2p4.(3P).4p | 4P* | 5/2 | 2s2.2p4.(3P).4d | 4D | 7/2 | | | L5951 | + 8752.100 | 8752.1020 | 100 | | | 281025.9326 - 292451.7592 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P).4p | 2D* | 3/2 | | | L5951 | + | | | | | | | | | | | | | | | + 8797.34 | 8797.22 | 80w* | | | 317781.404 - 329148.63 | 2s2.2p4.(1D).4p | 2D* | 5/2 | 2s2.2p4.(1D).5s | 2D | 5/2 | | | L1406 | + 8797.34 | 8797.39 | 80w* | | | 317781.767 - 329148.77 | 2s2.2p4.(1D).4p | 2D* | 3/2 | 2s2.2p4.(1D).5s | 2D | 3/2 | | | L1406 | + 8799.076 | 8799.0768 | 40 | | | 290583.1503 - 301947.9789 | 2s2.2p4.(3P).4p | 4P* | 3/2 | 2s2.2p4.(3P).4d | 4D | 3/2 | | | L1406 | + 8799.323 | 8799.3245 | 40 | | | 279218.6416 - 290583.1503 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P).4p | 4P* | 3/2 | | | L1406 | + 8824.269 | 8824.2688 | 140 | | | 280700.7426 - 292033.1263 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P).4p | 2D* | 5/2 | | | L5951 | + | | | | | | | | | | | | | | | + 8864.930 | 8864.9318 | 60 | | | 281171.3566 - 292451.7592 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P).4p | 2D* | 3/2 | | | L1406 | + 8871.122 | 8871.1305 | 40 | | | 290583.1503 - 301855.6707 | 2s2.2p4.(3P).4p | 4P* | 3/2 | 2s2.2p4.(3P).4d | 4D | 5/2 | | | L1406 | + 8882.346 | 8882.3538 | 60 | | | 279324.8733 - 290583.1503 | 2s2.2p4.(3P).3d | 4D | 3/2 | 2s2.2p4.(3P).4p | 4P* | 3/2 | | | L1406 | + 8886.170 | 8886.1617 | 22 | | | 290803.7989 - 302057.2516 | 2s2.2p4.(3P).4p | 4P* | 1/2 | 2s2.2p4.(3P).4d | 4D | 1/2 | | | L1406 | + 8900.046 | 8900.0534 | 22 | | | 280797.2387 - 292033.1263 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P).4p | 2D* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 8901.290 | 8901.2941 | 140 | | | 279137.6053 - 290371.9268 | 2s2.2p4.(3P).3d | 4D | 7/2 | 2s2.2p4.(3P).4p | 4P* | 5/2 | | | L5951 | + 8902.573 | 8902.5780 | 120 | | | 280262.3163 - 291495.0176 | 2s2.2p4.(3P).3d | 2F | 7/2 | 2s2.2p4.(3P).4p | 4D* | 5/2 | | | L1406 | + 8907.866 | 8907.8350 | 12 | | | 280268.9453 - 291495.0176 | 2s2.2p4.(3P).3d | 2D | 5/2 | 2s2.2p4.(3P).4p | 4D* | 5/2 | | | L1406 | + 8917.310 | 8917.2926 | 60 | | | 281332.521 - 292546.687 | 2s2.2p4.(3P).3d | 2P | 1/2 | 2s2.2p4.(3P).4p | 2S* | 1/2 | | | L1406 | + 8961.218 | 8961.2289 | 22 | | | 291801.858 - 302961.0417 | 2s2.2p4.(3P).4p | 4D* | 3/2 | 2s2.2p4.(3P).4d | 4F | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 8965.971 | 8965.9682 | 60 | | | 279218.6416 - 290371.9268 | 2s2.2p4.(3P).3d | 4D | 5/2 | 2s2.2p4.(3P).4p | 4P* | 5/2 | | | L1406 | + 9013.467 | 9013.478 | 100 | | | 306687.271 - 317781.767 | 2s2.2p4.(1D).3d | 2F | 5/2 | 2s2.2p4.(1D).4p | 2D* | 3/2 | | | L1406 | + 9014.098 | 9014.085 | 120 | | | 306687.654 - 317781.404 | 2s2.2p4.(1D).3d | 2F | 7/2 | 2s2.2p4.(1D).4p | 2D* | 5/2 | | | L1406 | + 9060.108 | 9060.0955 | 100 | | | 291801.858 - 302839.2693 | 2s2.2p4.(3P).4p | 4D* | 3/2 | 2s2.2p4.(3P).4d | 4F | 5/2 | | | L1406 | + 9066.985 | 9066.9853 | 160 | | | 280172.9854 - 291202.0096 | 2s2.2p4.(3P).3d | 4F | 9/2 | 2s2.2p4.(3P).4p | 4D* | 7/2 | | | L5951 | + | | | | | | | | | | | | | | | + 9073.563 | 9073.5545 | 100 | | | 280947.0768 - 291968.116 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P).4p | 4D* | 1/2 | | | L1406 | + 9081.9441 | 9081.9427 | 200 | | | 283322.3319 - 294333.192 | 2s2.2p4.(3P).4s | 2P | 3/2 | 2s2.2p4.(3P).4p | 2P* | 1/2 | | | L5951 | + 9084.981 | 9084.9678 | 22 | | | 281025.9326 - 292033.1263 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P).4p | 2D* | 5/2 | | | L1406 | + 9087.094 | 9087.093 | 120 | | | 280797.2387 - 291801.858 | 2s2.2p4.(3P).3d | 4F | 5/2 | 2s2.2p4.(3P).4p | 4D* | 3/2 | | | L1406 | + 9096.771 | 9096.7594 | 60 | | | 291968.116 - 302961.0417 | 2s2.2p4.(3P).4p | 4D* | 1/2 | 2s2.2p4.(3P).4d | 4F | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 9098.446 | 9098.4358 | 180 | | | 291202.0096 - 302192.9098 | 2s2.2p4.(3P).4p | 4D* | 7/2 | 2s2.2p4.(3P).4d | 4F | 9/2 | | | L1406 | + 9108.626 | 9108.6352 | 22 | | | 280989.5229 - 291968.116 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P).4p | 4D* | 1/2 | | | L1406 | + 9115.007 | 9114.995 | 22 | | | 301801.0594 - 312771.9922 | 2s2.2p4.(3P).4d | 4D | 7/2 | 2s2.2p4.(3P<2>).5f | 2[3]* | 7/2 | | | L1406 | + 9121.732 | 9121.729 | 80 | | | 301801.0594 - 312763.8929 | 2s2.2p4.(3P).4d | 4D | 7/2 | 2s2.2p4.(3P<2>).5f | 2[4]* | 9/2 | | | L1406 | + 9160.594 | 9160.595 | 12 | | | 301855.6707 - 312771.9922 | 2s2.2p4.(3P).4d | 4D | 5/2 | 2s2.2p4.(3P<2>).5f | 2[3]* | 7/2 | | | L1406 | + | | | | | | | | | | | | | | | + 9166.08 | 9166.061 | 12w | | | 301855.6707 - 312765.4821 | 2s2.2p4.(3P).4d | 4D | 5/2 | 2s2.2p4.(3P<2>).5f | 2[4]* | 7/2 | | | L1406 | + 9180.298 | 9180.30 | 80 | | | 316882.1781 - 327775.07 | 2s2.2p4.(1D).4p | 2F* | 5/2 | 2s2.2p4.(1D).4d | 2G | 7/2 | | | L1406 | + 9193.479 | 9193.48 | 80 | | | 316897.1647 - 327774.44 | 2s2.2p4.(1D).4p | 2F* | 7/2 | 2s2.2p4.(1D).4d | 2G | 9/2 | | | L1406 | + 9206.601 | 9206.6029 | 12 | | | 281171.3566 - 292033.1263 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P).4p | 2D* | 5/2 | | | L1406 | + 9210.309 | 9210.2992 | 60 | | | 292033.1263 - 302890.5369 | 2s2.2p4.(3P).4p | 2D* | 5/2 | 2s2.2p4.(3P).4d | 2F | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 9236.666 | 9236.671 | 120 | | | 281720.2755 - 292546.687 | 2s2.2p4.(3P).3d | 2P | 3/2 | 2s2.2p4.(3P).4p | 2S* | 1/2 | | | L1406 | + 9238.355 | 9238.343 | 40 | | | 301947.9789 - 312772.4309 | 2s2.2p4.(3P).4d | 4D | 3/2 | 2s2.2p4.(3P<2>).5f | 2[3]* | 5/2 | | | L1406 | + 9248.383 | 9248.3792 | 22 | | | 291495.0176 - 302307.7230 | 2s2.2p4.(3P).4p | 4D* | 5/2 | 2s2.2p4.(3P).4d | 2D | 5/2 | | | L1406 | + 9248.696 | 9248.696 | 60 | | | 280989.5229 - 291801.858 | 2s2.2p4.(3P).3d | 4P | 3/2 | 2s2.2p4.(3P).4p | 4D* | 3/2 | | | L1406 | + 9264.164 | 9264.1701 | 80 | | | 280700.7426 - 291495.0176 | 2s2.2p4.(3P).3d | 4F | 7/2 | 2s2.2p4.(3P).4p | 4D* | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 9290.112 | 9290.1154 | 400 | | | 283322.3319 - 294086.4608 | 2s2.2p4.(3P).4s | 2P | 3/2 | 2s2.2p4.(3P).4p | 2P* | 3/2 | | | L1406 | + 9295.150 | 9295.142 | 22 | | | 302961.0417 - 313719.3496 | 2s2.2p4.(3P).4d | 4F | 3/2 | 2s2.2p4.(3P<0>).5f | 2[3]* | 5/2 | | | L1406 | + 9299.9503 | 9299.9502 | 180 | | | 291495.0176 - 302247.7634 | 2s2.2p4.(3P).4p | 4D* | 5/2 | 2s2.2p4.(3P).4d | 2F | 7/2 | | | L5951 | + 9305.11 | 9305.117 | 22w | | | 302057.2516 - 312804.0268 | 2s2.2p4.(3P).4d | 4D | 1/2 | 2s2.2p4.(3P<2>).5f | 2[2]* | 3/2 | | | L1406 | + 9330.3985 | 9330.3969 | 160 | | | 292033.1263 - 302750.7840 | 2s2.2p4.(3P).4p | 2D* | 5/2 | 2s2.2p4.(3P).4d | 4F | 7/2 | | | L5951 | + | | | | | | | | | | | | | | | + 9353.487 | 9353.4806 | 40 | | | 292546.687 - 303237.8943 | 2s2.2p4.(3P).4p | 2S* | 1/2 | 2s2.2p4.(3P).4d | 2P | 3/2 | | | L1406 | + 9362.691 | 9362.695 | 100 | | | 302750.7840 - 313431.469 | 2s2.2p4.(3P).4d | 4F | 7/2 | 2s2.2p4.(3P<1>).5f | 2[4]* | 9/2 | | | L1406 | + 9392.191 | 9392.184 | 80 | | | 303071.8002 - 313718.9516 | 2s2.2p4.(3P).4d | 4P | 5/2 | 2s2.2p4.(3P<0>).5f | 2[3]* | 7/2 | | | L1406 | + 9408.562 | 9408.583 | 22 | | | 292651.298 - 303279.8915 | 2s2.2p4.(3P).4p | 4S* | 3/2 | 2s2.2p4.(3P).5s | 4P | 5/2 | | | L1406 | + 9412.880 | 9412.881 | 100 | | | 302192.9098 - 312816.6500 | 2s2.2p4.(3P).4d | 4F | 9/2 | 2s2.2p4.(3P<2>).5f | 2[5]* | 11/2 | | | L1406 | + | | | | | | | | | | | | | | | + 9416.154 | 9416.1595 | 140 | | | 292451.7592 - 303071.8002 | 2s2.2p4.(3P).4p | 2D* | 3/2 | 2s2.2p4.(3P).4d | 4P | 5/2 | | | L1406 | + 9431.186 | 9431.225 | 40 | | | 302816.9602 - 313420.0366 | 2s2.2p4.(3P).4d | 4P | 3/2 | 2s2.2p4.(3P<1>).5f | 2[2]* | 5/2 | | | L1406 | + 9434.805 | 9434.8080 | 100 | | | 291202.0096 - 301801.0594 | 2s2.2p4.(3P).4p | 4D* | 7/2 | 2s2.2p4.(3P).4d | 4D | 7/2 | | | L1406 | + 9440.203 | 9440.203 | 80 | | | 302839.2693 - 313432.262 | 2s2.2p4.(3P).4d | 4F | 5/2 | 2s2.2p4.(3P<1>).5f | 2[4]* | 7/2 | | | L1406 | + 9452.419 | 9452.362 | 60 | | | 302890.5369 - 313469.9028 | 2s2.2p4.(3P).4d | 2F | 5/2 | 2s2.2p4.(3P<1>).5f | 2[3]* | 7/2 | | | L1406 | + | | | | | | | | | | | | | | | + 9480.51 | 9480.5234 | 40w | | | 280947.0768 - 291495.0176 | 2s2.2p4.(3P).3d | 4F | 3/2 | 2s2.2p4.(3P).4p | 4D* | 5/2 | | | L1406 | + 9490.636 | 9490.63 | 22 | | | 317781.404 - 328318.11 | 2s2.2p4.(1D).4p | 2D* | 5/2 | 2s2.2p4.(1D).4d | 2F | 7/2 | | | L1406 | + 9491.123 | 9491.12 | 22 | | | 317781.767 - 328317.93 | 2s2.2p4.(1D).4p | 2D* | 3/2 | 2s2.2p4.(1D).4d | 2F | 5/2 | | | L1406 | + 9515.326 | 9515.324 | 22 | | | 302961.0417 - 313470.4054 | 2s2.2p4.(3P).4d | 4F | 3/2 | 2s2.2p4.(3P<1>).5f | 2[3]* | 5/2 | | | L1406 | + 9518.481 | 9518.493 | 22 | | | 291801.858 - 302307.7230 | 2s2.2p4.(3P).4p | 4D* | 3/2 | 2s2.2p4.(3P).4d | 2D | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 9540.671 | 9540.660 | 60 | | | 303237.8943 - 313719.3496 | 2s2.2p4.(3P).4d | 2P | 3/2 | 2s2.2p4.(3P<0>).5f | 2[3]* | 5/2 | | | L1406 | + 9551.929 | 9551.9331 | 22 | | | 281025.9326 - 291495.0176 | 2s2.2p4.(3P).3d | 2F | 5/2 | 2s2.2p4.(3P).4p | 4D* | 5/2 | | | L1406 | + 9556.320 | 9556.329 | 40 | | | 302307.7230 - 312771.9922 | 2s2.2p4.(3P).4d | 2D | 5/2 | 2s2.2p4.(3P<2>).5f | 2[3]* | 7/2 | | | L1406 | + 9562.26 | 9562.278 | 40w* | | | 302307.7230 - 312765.4821 | 2s2.2p4.(3P).4d | 2D | 5/2 | 2s2.2p4.(3P<2>).5f | 2[4]* | 7/2 | | | L1406 | + 9562.26 | 9562.334 | 40w* | | | 291968.116 - 302425.8139 | 2s2.2p4.(3P).4p | 4D* | 1/2 | 2s2.2p4.(3P).4d | 2D | 3/2 | | | L1406 | + | | | | | | | | | | | | | | | + 9579.640 | 9579.649 | 240* | | | 283894.3965 - 294333.192 | 2s2.2p4.(3P).4s | 2P | 1/2 | 2s2.2p4.(3P).4p | 2P* | 1/2 | | | L1406 | + 9579.640 | 9579.6656 | 240* | | | 292451.7592 - 302890.5369 | 2s2.2p4.(3P).4p | 2D* | 3/2 | 2s2.2p4.(3P).4d | 2F | 5/2 | | | L1406 | + 9602.123 | 9602.131 | 12 | | | 292546.687 - 302961.0417 | 2s2.2p4.(3P).4p | 2S* | 1/2 | 2s2.2p4.(3P).4d | 4F | 3/2 | | | L1406 | + 9658.335 | 9658.3381 | 40 | | | 294086.4608 - 304440.2089 | 2s2.2p4.(3P).4p | 2P* | 3/2 | 2s2.2p4.(3P).5s | 2P | 1/2 | | | L1406 | + 9766.327 | 9766.351 | 40 | | | 292651.298 - 302890.5369 | 2s2.2p4.(3P).4p | 4S* | 3/2 | 2s2.2p4.(3P).4d | 2F | 5/2 | | | L1406 | + | | | | | | | | | | | | | | | + 9794.767 | 9794.7887 | 40 | | | 306687.654 - 316897.1647 | 2s2.2p4.(1D).3d | 2F | 7/2 | 2s2.2p4.(1D).4p | 2F* | 7/2 | | | L1406 | + 9808.811 | 9808.819 | 40 | | | 306687.271 - 316882.1781 | 2s2.2p4.(1D).3d | 2F | 5/2 | 2s2.2p4.(1D).4p | 2F* | 5/2 | | | L1406 | + 9811.5554 | 9811.5551 | 200 | | | 283894.3965 - 294086.4608 | 2s2.2p4.(3P).4s | 2P | 1/2 | 2s2.2p4.(3P).4p | 2P* | 3/2 | | | L5951 | + 9969.447 | 9969.4407 | 60 | | | 281171.3566 - 291202.0096 | 2s2.2p4.(3P).3d | 4P | 5/2 | 2s2.2p4.(3P).4p | 4D* | 7/2 | | | L1406 | +#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/pypeit/data/arc_lines/convert_NIST_to_lists.py b/pypeit/data/arc_lines/convert_NIST_to_lists.py index d9b070f8ab..65524ef4cb 100644 --- a/pypeit/data/arc_lines/convert_NIST_to_lists.py +++ b/pypeit/data/arc_lines/convert_NIST_to_lists.py @@ -5,6 +5,7 @@ import pdb import datetime +from IPython import embed import astropy.table @@ -52,9 +53,8 @@ def init_line_list(): 'amplitude': 0, 'Source': dummy_src, } - # Return table - return astropy.table.Table(idict) + return astropy.table.Table(rows=[idict]) def load_line_list(line): @@ -104,9 +104,8 @@ def load_line_list(line): line_list.rename_column('Observed', 'wave') # Others # Grab ion name - i0 = line_file.rfind('/') - i1 = line_file.rfind('_') - ion = line_file[i0+1:i1] + i1 = line_file.name.rfind('_') + ion = line_file.name[:i1] line_list.add_column(astropy.table.Column([ion]*len(line_list), name='Ion', dtype='U5')) line_list.add_column(astropy.table.Column([1]*len(line_list), name='NIST')) return line_list @@ -158,12 +157,14 @@ def main(args=None): # Finally, sort the list by increasing wavelength linelist.sort('wave') + print(linelist) + # Write? if not pargs.write: print("=============================================================") print("Rerun with --write if you are happy with what you see.") print("=============================================================") - return + return linelist # Write the table to disk outfile = data.get_linelist_filepath(f'{line}_lines.dat') diff --git a/pypeit/data/arc_lines/lists/ArI_lines.dat b/pypeit/data/arc_lines/lists/ArI_lines.dat index ae4e1297af..67f8786419 100644 --- a/pypeit/data/arc_lines/lists/ArI_lines.dat +++ b/pypeit/data/arc_lines/lists/ArI_lines.dat @@ -11,7 +11,6 @@ # Creation Date: 2016-Dec-21 | ion | wave | NIST | Instr | amplitude | Source | | ArI | 3950.0970 | 1 | 128 | 10000 | NIST | -| ArI | 4045.5610 | 1 | 128 | 10000 | NIST | | ArI | 4159.7620 | 1 | 64 | 94 | GMOS_R400_blue_1_fit.json | | ArI | 4165.3540 | 1 | 128 | 10000 | NIST | | ArI | 4201.8580 | 1 | 128 | 10000 | NIST | @@ -160,4 +159,9 @@ | ArI | 9356.7870 | 1 | 89 | 3521 | lrisr_400_8500_PYPIT.json | | ArI | 9660.4350 | 1 | 72 | 35000 | deimos_830g_r_PYPIT.json | | ArI | 9787.1860 | 1 | 25 | 8197 | lrisr_400_8500_PYPIT.json | +| ArI | 10054.81 | 1 | 1 | 300 | keck_mosfire_webpage | +| ArI | 10335.55 | 1 | 1 | 200 | keck_mosfire_webpage | | ArI | 10472.9230 | 1 | 72 | 1600 | deimos_830g_pypeit_manual | +| ArI | 10676.49 | 1 | 1 | 11160 | keck_mosfire_webpage | +| ArI | 10703.90 | 1 | 1 | 130 | keck_mosfire_webpage | +| ArI | 10883.94 | 1 | 1 | 490 | keck_mosfire_webpage | \ No newline at end of file diff --git a/pypeit/data/arc_lines/lists/FeAr_lines.dat b/pypeit/data/arc_lines/lists/FeAr_lines.dat new file mode 100644 index 0000000000..a7038dc4da --- /dev/null +++ b/pypeit/data/arc_lines/lists/FeAr_lines.dat @@ -0,0 +1,124 @@ +# Creation Date: 2023-06-27 +# taken from https://www2.keck.hawaii.edu/inst/lris/arc_calibrations.html +# converted to vacuum, and matched to NIST, +# see dev_algorithms/wavelengths/lris_FeAr_linelist/create_lris_FeAr_list.py in PypeIt-development-suite +# NOTE: even if it is called FeAr (from lris header keyword), it is actually FeI, NeI, and NeII lines +| ion | wave | NIST | Instr | amplitude | Source | +| FeI | 3021.5186 | 1 | 0 | 0 | NIST | +| FeI | 3024.9130 | 1 | 0 | 123000 | NIST | +| FeI | 3038.2725 | 1 | 0 | 98000 | NIST | +| FeI | 3048.4908 | 1 | 0 | 0 | NIST | +| FeI | 3058.3346 | 1 | 0 | 120000 | NIST | +| FeI | 3059.9750 | 1 | 0 | 204000 | NIST | +| NeII | 3298.6749 | 1 | 0 | 300 | NIST | +| NeII | 3324.6913 | 1 | 0 | 2000 | NIST | +| NeII | 3346.4162 | 1 | 0 | 600 | NIST | +| NeII | 3398.8410 | 1 | 0 | 100 | NIST | +| NeI | 3418.8834 | 1 | 0 | 5000 | NIST | +| FeI | 3428.1020 | 1 | 0 | 138000 | NIST | +| FeI | 3441.5918 | 1 | 0 | 1350000 | NIST | +| FeI | 3476.4450 | 1 | 0 | 1200000 | NIST | +| FeI | 3477.6970 | 1 | 0 | 490000 | NIST | +| FeI | 3491.5727 | 1 | 0 | 1320000 | NIST | +| NeI | 3521.4780 | 1 | 0 | 10000 | NIST | +| FeI | 3542.0950 | 1 | 0 | 60000 | NIST | +| FeI | 3566.3969 | 1 | 0 | 760000 | NIST | +| FeI | 3582.2150 | 1 | 0 | 0 | NIST | +| NeI | 3601.1965 | 1 | 0 | 1000 | NIST | +| FeI | 3619.7996 | 1 | 0 | 830000 | NIST | +| FeI | 3632.4981 | 1 | 0 | 1150000 | NIST | +| FeI | 3680.9608 | 1 | 0 | 910000 | NIST | +| FeI | 3688.5061 | 1 | 0 | 355000 | NIST | +| FeI | 3706.6202 | 1 | 0 | 1290000 | NIST | +| FeI | 3710.3014 | 1 | 0 | 398000 | NIST | +| FeI | 3720.9926 | 1 | 0 | 0 | NIST | +| FeI | 3723.6215 | 1 | 0 | 1290000 | NIST | +| FeI | 3728.6789 | 1 | 0 | 355000 | NIST | +| FeI | 3734.3789 | 1 | 0 | 1000000 | NIST | +| FeI | 3735.9256 | 1 | 0 | 0 | NIST | +| FeI | 3738.1939 | 1 | 0 | 0 | NIST | +| FeI | 3746.6257 | 1 | 0 | 2510000 | NIST | +| FeI | 3749.3273 | 1 | 0 | 1910000 | NIST | +| FeI | 3750.5509 | 1 | 0 | 1150000 | NIST | +| FeI | 3759.3007 | 1 | 0 | 740000 | NIST | +| FeI | 3764.8583 | 1 | 0 | 590000 | NIST | +| FeI | 3768.2618 | 1 | 0 | 457000 | NIST | +| FeI | 3814.0465 | 1 | 0 | 398000 | NIST | +| FeI | 3816.9230 | 1 | 0 | 760000 | NIST | +| FeI | 3821.5091 | 1 | 0 | 0 | NIST | +| FeI | 3825.5287 | 1 | 0 | 1000000 | NIST | +| FeI | 3826.9665 | 1 | 0 | 760000 | NIST | +| FeI | 3828.9085 | 1 | 0 | 590000 | NIST | +| FeI | 3835.3100 | 1 | 0 | 590000 | NIST | +| FeI | 3857.4648 | 1 | 0 | 1100000 | NIST | +| FeI | 3861.0055 | 1 | 0 | 0 | NIST | +| FeI | 3879.6723 | 1 | 0 | 1290000 | NIST | +| FeI | 3896.7599 | 1 | 0 | 740000 | NIST | +| FeI | 3900.8119 | 1 | 0 | 1070000 | NIST | +| FeI | 3904.0511 | 1 | 0 | 302000 | NIST | +| FeI | 3907.5861 | 1 | 0 | 234000 | NIST | +| FeI | 3921.3680 | 1 | 0 | 650000 | NIST | +| FeI | 3924.0224 | 1 | 0 | 1000000 | NIST | +| FeI | 3931.4092 | 1 | 0 | 1150000 | NIST | +| FeI | 3970.3799 | 1 | 0 | 427000 | NIST | +| FeI | 4006.3740 | 1 | 0 | 209000 | NIST | +| FeI | 4064.7415 | 1 | 0 | 830000 | NIST | +| FeI | 4119.7067 | 1 | 0 | 47900 | NIST | +| FeI | 4199.4870 | 1 | 0 | 29500 | NIST | +| FeI | 4295.3326 | 1 | 0 | 58000 | NIST | +| FeI | 4309.1137 | 1 | 0 | 630000 | NIST | +| FeI | 4326.9780 | 1 | 0 | 830000 | NIST | +| FeI | 4377.1594 | 1 | 0 | 138000 | NIST | +| FeI | 4384.7763 | 1 | 0 | 141000 | NIST | +| FeI | 4405.9873 | 1 | 0 | 810000 | NIST | +| FeI | 4416.3621 | 1 | 0 | 288000 | NIST | +| FeI | 4490.9984 | 1 | 0 | 12900 | NIST | +| FeI | 4861.0986 | 1 | 0 | 24500 | NIST | +| FeI | 4892.8581 | 1 | 0 | 117000 | NIST | +| FeI | 4958.9800 | 1 | 0 | 295000 | NIST | +| FeI | 5196.3880 | 1 | 0 | 72000 | NIST | +| FeI | 5268.0207 | 1 | 0 | 51000 | NIST | +| FeI | 5271.0035 | 1 | 0 | 1020000 | NIST | +| FeI | 5285.0910 | 1 | 0 | 33900 | NIST | +| FeI | 5303.7740 | 1 | 0 | 25 | NIST | +| FeI | 5325.6598 | 1 | 0 | 78000 | NIST | +| NeI | 5332.2603 | 1 | 0 | 6000 | NIST | +| FeI | 5342.5093 | 1 | 0 | 81000 | NIST | +| FeI | 5372.9830 | 1 | 0 | 389000 | NIST | +| NeI | 5402.0631 | 1 | 0 | 20000 | NIST | +| FeI | 5588.3067 | 1 | 0 | 25700 | NIST | +| FeI | 5617.2024 | 1 | 0 | 37200 | NIST | +| NeI | 5691.3950 | 1 | 0 | 1500 | NIST | +| NeI | 5749.8929 | 1 | 0 | 5000 | NIST | +| NeI | 5766.0175 | 1 | 0 | 7000 | NIST | +| NeI | 5821.7694 | 1 | 0 | 5000 | NIST | +| NeI | 5854.1101 | 1 | 0 | 20000 | NIST | +| NeI | 5883.5252 | 1 | 0 | 10000 | NIST | +| NeI | 5946.4810 | 1 | 0 | 5000 | NIST | +| NeI | 5977.1895 | 1 | 0 | 6000 | NIST | +| NeI | 6076.0193 | 1 | 0 | 10000 | NIST | +| NeI | 6097.8506 | 1 | 0 | 3000 | NIST | +| NeI | 6130.1460 | 1 | 0 | 1000 | NIST | +| NeI | 6144.7629 | 1 | 0 | 10000 | NIST | +| NeI | 6165.2994 | 1 | 0 | 10000 | NIST | +| NeI | 6219.0013 | 1 | 0 | 10000 | NIST | +| NeI | 6268.2285 | 1 | 0 | 10000 | NIST | +| NeI | 6306.5329 | 1 | 0 | 1000 | NIST | +| NeI | 6336.1791 | 1 | 0 | 10000 | NIST | +| NeI | 6384.7560 | 1 | 0 | 10000 | NIST | +| NeI | 6508.3255 | 1 | 0 | 15000 | NIST | +| NeI | 6534.6872 | 1 | 0 | 1000 | NIST | +| NeI | 6600.7754 | 1 | 0 | 10000 | NIST | +| NeI | 6718.8974 | 1 | 0 | 700 | NIST | +| NeI | 6931.3787 | 1 | 0 | 100000 | NIST | +| NeI | 7034.3520 | 1 | 0 | 85000 | NIST | +| NeI | 7175.9154 | 1 | 0 | 77000 | NIST | +| NeI | 7247.1631 | 1 | 0 | 77000 | NIST | +| NeI | 7490.9335 | 1 | 0 | 32000 | NIST | +| NeI | 7537.8488 | 1 | 0 | 28000 | NIST | +| NeI | 7546.1211 | 1 | 0 | 13000 | NIST | +| NeI | 8379.9093 | 1 | 0 | 76000 | NIST | +| NeI | 8593.6184 | 1 | 0 | 41000 | NIST | +| NeI | 8637.0190 | 1 | 0 | 35000 | NIST | +| NeI | 8656.7599 | 1 | 0 | 64000 | NIST | +| NeI | 8921.9496 | 1 | 0 | 6400 | NIST | diff --git a/pypeit/data/arc_lines/lists/ZnI_lines.dat b/pypeit/data/arc_lines/lists/ZnI_lines.dat index 306fa610c2..36ce82109d 100644 --- a/pypeit/data/arc_lines/lists/ZnI_lines.dat +++ b/pypeit/data/arc_lines/lists/ZnI_lines.dat @@ -1,8 +1,28 @@ # Creation Date: 2016-Dec-21 -| ion | wave | NIST | Instr | amplitude | Source | -| ZnI | 3283.2800 | 1 | 2 | 528 | lrisb_600_4000_PYPIT.json | -| ZnI | 3303.5300 | 1 | 2 | 1209 | lrisb_600_4000_PYPIT.json | -| ZnI | 3345.9800 | 1 | 2 | 2198 | lrisb_600_4000_PYPIT.json | -| ZnI | 4681.4500 | 1 | 2 | 22288 | lrisb_600_4000_PYPIT.json | -| ZnI | 4723.4700 | 1 | 2 | 35865 | lrisb_600_4000_PYPIT.json | -| ZnI | 4811.8700 | 1 | 2 | 49870 | lrisb_600_4000_PYPIT.json | +| ion | wave | NIST | Instr | amplitude | Source | +| ZnI | 3019.24 | 1 | 2 | 125 | keck_lris_webpage | +| ZnI | 3036.66 | 1 | 2 | 200 | keck_lris_webpage | +| ZnI | 3072.95 | 1 | 2 | 200 | keck_lris_webpage | +| ZnI | 3076.79 | 1 | 2 | 150 | keck_lris_webpage | +| ZnI | 3283.28 | 1 | 2 | 528 | lrisb_600_4000_PYPIT.json | +| ZnI | 3303.53 | 1 | 2 | 1209 | lrisb_600_4000_PYPIT.json | +| ZnI | 3303.89 | 1 | 2 | 700 | keck_lris_webpage | +| ZnI | 3345.98 | 1 | 2 | 2198 | lrisb_600_4000_PYPIT.json | +| ZnI | 3346.53 | 1 | 2 | 500 | keck_lris_webpage | +| ZnI | 3346.90 | 1 | 2 | 150 | keck_lris_webpage | +| ZnI | 3884.44 | 1 | 2 | 50 | keck_lris_webpage | +| ZnI | 3966.55 | 1 | 2 | 15 | keck_lris_webpage | +| ZnI | 4114.37 | 1 | 2 | 10 | keck_lris_webpage | +| ZnI | 4294.09 | 1 | 2 | 25 | keck_lris_webpage | +| ZnI | 4299.54 | 1 | 2 | 25 | keck_lris_webpage | +| ZnI | 4631.11 | 1 | 2 | 35 | keck_lris_webpage | +| ZnI | 4681.45 | 1 | 2 | 22288 | lrisb_600_4000_PYPIT.json | +| ZnI | 4723.47 | 1 | 2 | 35865 | lrisb_600_4000_PYPIT.json | +| ZnI | 4811.87 | 1 | 2 | 49870 | lrisb_600_4000_PYPIT.json | +| ZnI | 5070.99 | 1 | 2 | 15 | keck_lris_webpage | +| ZnI | 5183.42 | 1 | 2 | 200 | keck_lris_webpage | +| ZnI | 5778.71 | 1 | 2 | 10 | keck_lris_webpage | +| ZnI | 6364.10 | 1 | 2 | 1000 | keck_lris_webpage | +| ZnI | 6480.97 | 1 | 2 | 10 | keck_lris_webpage | +| ZnI | 6930.23 | 1 | 2 | 15 | keck_lris_webpage | +| ZnI | 7801.51 | 1 | 2 | 10 | keck_lris_webpage | \ No newline at end of file diff --git a/pypeit/data/arc_lines/reid_arxiv/gemini_gnirs_lrifu_H.fits b/pypeit/data/arc_lines/reid_arxiv/gemini_gnirs_lrifu_H.fits new file mode 100644 index 0000000000..c347b9ca14 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/gemini_gnirs_lrifu_H.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/gemini_gnirs_lrifu_K.fits b/pypeit/data/arc_lines/reid_arxiv/gemini_gnirs_lrifu_K.fits new file mode 100644 index 0000000000..e2ed76cf88 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/gemini_gnirs_lrifu_K.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_esi_ECH.fits b/pypeit/data/arc_lines/reid_arxiv/keck_esi_ECH.fits new file mode 100644 index 0000000000..fbae7e7350 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_esi_ECH.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_kcrm_RH3.fits b/pypeit/data/arc_lines/reid_arxiv/keck_kcrm_RH3.fits new file mode 100644 index 0000000000..93d64a0a39 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_kcrm_RH3.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_kcrm_RM1.fits b/pypeit/data/arc_lines/reid_arxiv/keck_kcrm_RM1.fits new file mode 100644 index 0000000000..3c2c0ec775 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_kcrm_RM1.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_kcrm_RM2.fits b/pypeit/data/arc_lines/reid_arxiv/keck_kcrm_RM2.fits new file mode 100644 index 0000000000..77286b4963 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_kcrm_RM2.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_B1200_3400_d560_ArCdHgNeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_B1200_3400_d560_ArCdHgNeZn.fits new file mode 100644 index 0000000000..29a4335891 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_B1200_3400_d560_ArCdHgNeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_B300_5000_d680_ArCdHgKrNeXeZnFeAr.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_B300_5000_d680_ArCdHgKrNeXeZnFeAr.fits new file mode 100644 index 0000000000..8d4bf10b7b Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_B300_5000_d680_ArCdHgKrNeXeZnFeAr.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_B400_3400_d560_ArCdHgNeZnFeAr.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_B400_3400_d560_ArCdHgNeZnFeAr.fits new file mode 100644 index 0000000000..8e8128cb89 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_B400_3400_d560_ArCdHgNeZnFeAr.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_600_d560.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_B600_4000_d560_ArCdHgKrNeXeZn.fits similarity index 100% rename from pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_600_d560.fits rename to pypeit/data/arc_lines/reid_arxiv/keck_lris_blue_B600_4000_d560_ArCdHgKrNeXeZn.fits diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R1200_7500_ArCdHgKrNeXeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R1200_7500_ArCdHgKrNeXeZn.fits new file mode 100644 index 0000000000..aea1fd0550 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R1200_7500_ArCdHgKrNeXeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_1200_9000.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R1200_9000.fits similarity index 100% rename from pypeit/data/arc_lines/reid_arxiv/keck_lris_red_1200_9000.fits rename to pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R1200_9000.fits diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R150_7500_ArCdHgNeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R150_7500_ArCdHgNeZn.fits new file mode 100644 index 0000000000..37568c0518 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R150_7500_ArCdHgNeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R300_5000_ArCdHgKrNeXeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R300_5000_ArCdHgKrNeXeZn.fits new file mode 100644 index 0000000000..791e00aec5 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R300_5000_ArCdHgKrNeXeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R400_8500_ArCdHgKrNeXeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R400_8500_ArCdHgKrNeXeZn.fits new file mode 100644 index 0000000000..49e75e4ec2 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R400_8500_ArCdHgKrNeXeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R600_10000_ArCdHgKrNeXeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R600_10000_ArCdHgKrNeXeZn.fits new file mode 100644 index 0000000000..0f018d8d39 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R600_10000_ArCdHgKrNeXeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R600_5000_ArCdHgKrNeXeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R600_5000_ArCdHgKrNeXeZn.fits new file mode 100644 index 0000000000..a5d74ca8d1 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R600_5000_ArCdHgKrNeXeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R600_7500_ArCdHgKrNeXeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R600_7500_ArCdHgKrNeXeZn.fits new file mode 100644 index 0000000000..0753d737be Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R600_7500_ArCdHgKrNeXeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R831_8200_ArCdHgKrNeXeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R831_8200_ArCdHgKrNeXeZn.fits new file mode 100644 index 0000000000..8f9ceb8ce5 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R831_8200_ArCdHgKrNeXeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R900_5500_ArCdHgNeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R900_5500_ArCdHgNeZn.fits new file mode 100644 index 0000000000..4a9baa3a35 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_R900_5500_ArCdHgNeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R1200_7500_ArCdHgNeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R1200_7500_ArCdHgNeZn.fits new file mode 100644 index 0000000000..1364a9572c Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R1200_7500_ArCdHgNeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R150_7500_ArHgNe.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R150_7500_ArHgNe.fits new file mode 100644 index 0000000000..583756785e Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R150_7500_ArHgNe.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R300_5000_ArCdHgNeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R300_5000_ArCdHgNeZn.fits new file mode 100644 index 0000000000..7f6f691c87 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R300_5000_ArCdHgNeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R400_8500_ArCdHgNeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R400_8500_ArCdHgNeZn.fits new file mode 100644 index 0000000000..c6c94cee31 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R400_8500_ArCdHgNeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R600_10000_ArCdHgNeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R600_10000_ArCdHgNeZn.fits new file mode 100644 index 0000000000..f504fd8635 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R600_10000_ArCdHgNeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R600_5000_ArCdHgNeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R600_5000_ArCdHgNeZn.fits new file mode 100644 index 0000000000..dc9cd24cb0 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R600_5000_ArCdHgNeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R600_7500_ArCdHgNeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R600_7500_ArCdHgNeZn.fits new file mode 100644 index 0000000000..9f69f48134 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R600_7500_ArCdHgNeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R831_8200_ArCdHgNeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R831_8200_ArCdHgNeZn.fits new file mode 100644 index 0000000000..6d49d46cc3 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R831_8200_ArCdHgNeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R900_5500_ArCdHgNeZn.fits b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R900_5500_ArCdHgNeZn.fits new file mode 100644 index 0000000000..9b562eabac Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_lris_red_orig_R900_5500_ArCdHgNeZn.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_1200_5100.fits b/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_1200_5100.fits new file mode 100644 index 0000000000..283cf4b232 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_1200_5100.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_echelle_Ar.fits b/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_echelle_Ar.fits new file mode 100644 index 0000000000..9e6748ae17 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_echelle_Ar.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_echelle_Ar_Only.fits b/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_echelle_Ar_Only.fits new file mode 100644 index 0000000000..498c81e30a Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_echelle_Ar_Only.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_echelle_Ar_REAL.fits b/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_echelle_Ar_REAL.fits new file mode 100644 index 0000000000..7b32a9367f Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_echelle_Ar_REAL.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_echelle_NeXeAr.fits b/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_echelle_NeXeAr.fits new file mode 100644 index 0000000000..11dd5363b4 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/mdm_modspec_echelle_NeXeAr.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/shane_kast_red_600_7500.fits b/pypeit/data/arc_lines/reid_arxiv/shane_kast_red_600_7500.fits new file mode 100644 index 0000000000..79026d08b9 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/shane_kast_red_600_7500.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/wvarxiv_mdm_modspec_echelle_20220727T1201.fits b/pypeit/data/arc_lines/reid_arxiv/wvarxiv_mdm_modspec_echelle_20220727T1201.fits new file mode 100644 index 0000000000..99e22945fb Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/wvarxiv_mdm_modspec_echelle_20220727T1201.fits differ diff --git a/pypeit/data/standards/xshooter/xshooter_info.txt b/pypeit/data/standards/xshooter/xshooter_info.txt index 089f8c95ed..959f9f8a78 100644 --- a/pypeit/data/standards/xshooter/xshooter_info.txt +++ b/pypeit/data/standards/xshooter/xshooter_info.txt @@ -10,6 +10,7 @@ #The flux distribution over the full X-shooter range 3000 – 25000A is tabulated. #The fluxes are sampled (at 0.1A [< 1000A] and 0.2A [>10000A]). #The flux tabluations are in units of (ergs cm-2 s-1 A-1). +#NOTE: The wavelengths are in air, as output by the ESO pipeline. #http://www.eso.org/sci/observing/tools/standards/spectra/Xshooterspec.html File Name RA_2000 DEC_2000 V_MAG TYPE fGD71.dat GD71 05:52:27.61 +15:53:13.8 13.03 DA1 diff --git a/pypeit/data/utils.py b/pypeit/data/utils.py index 7b20b34da8..f5f6c5e300 100644 --- a/pypeit/data/utils.py +++ b/pypeit/data/utils.py @@ -26,7 +26,7 @@ rather than pure strings, with all of the functionality therein contained. Most (by number) of the package data files here are distributed with the -``PypeIt`` package and are accessed via the :class:`~pypeit.data.Paths` +``PypeIt`` package and are accessed via the :class:`~pypeit.data.utils.Paths` class. For instance, the NIR spectrophotometry for Vega is accessed via: .. code-block:: python @@ -65,7 +65,7 @@ If new package-included data are added that are not very large (total directory size < a few MB), it is not necessary to use the AstroPy cache/download system. In this case, simply add the directory path to the -:class:`~pypeit.data.Paths` class and access the enclosed files similarly +:class:`~pypeit.data.utils.Paths` class and access the enclosed files similarly to the Vega example above. .. include:: ../include/links.rst @@ -206,8 +206,7 @@ def check_isdir(path): If yes, return the directory path, else raise an error message """ if not path.is_dir(): - msgs.error(f"Unable to find {path}. " - "Check your installation.") + msgs.error(f"Unable to find {path}. Check your installation.") return path @@ -308,7 +307,7 @@ def get_skisim_filepath(skisim_file: str) -> pathlib.Path: return skisim_path -def get_sensfunc_filepath(sensfunc_file: str, symlink_in_pkgdir=False) -> pathlib.Path: +def get_sensfunc_filepath(sensfunc_file: str, symlink_in_pkgdir: bool=False) -> pathlib.Path: """Return the full path to the ``sensfunc`` file In an attempt to reduce the size of the PypeIt package as distributed on @@ -338,7 +337,7 @@ def get_sensfunc_filepath(sensfunc_file: str, symlink_in_pkgdir=False) -> pathli pointing to the cached downloaded file. Defaults to False. Returns: - :obj:`pthlib.Path`: The full path to the ``sensfunc`` file + :obj:`pathlib.Path`: The full path to the ``sensfunc`` file """ # Full path within the package data structure: sensfunc_path = Paths.sensfuncs / sensfunc_file @@ -494,10 +493,10 @@ def get_extinctfile_filepath(extinction_file: str) -> pathlib.Path: def fetch_remote_file( filename: str, filetype: str, - remote_host='github', - install_script=False, - force_update=False, - full_url=None + remote_host: str='github', + install_script: bool=False, + force_update: bool=False, + full_url: str=None, ) -> pathlib.Path: """Use ``astropy.utils.data`` to fetch file from remote or cache @@ -549,13 +548,14 @@ def fetch_remote_file( sources=sources, timeout=10, cache="update" if force_update else True, - pkgname="pypeit" + pkgname="pypeit", ) except urllib.error.URLError as error: - if remote_host == "s3_cloud" and (requests.head(sources[0]).status_code in - [requests.codes.forbidden, requests.codes.not_found]): - + if remote_host == "s3_cloud" and ( + requests.head(sources[0]).status_code + in [requests.codes.forbidden, requests.codes.not_found] + ): err_msg = ( f"The file {filename}{msgs.newline()}" f"is not hosted in the cloud. Please download this file from{msgs.newline()}" @@ -593,8 +593,11 @@ def fetch_remote_file( # Raise the appropriate error message msgs.error(err_msg) + except TimeoutError as error: + msgs.error(f"Timeout Error enountered: {error}") + # If no error, return the pathlib object - return pathlib.Path(cache_fn) + return pathlib.Path(cache_fn).resolve() def search_cache(pattern_str: str) -> list[pathlib.Path]: @@ -619,7 +622,7 @@ def search_cache(pattern_str: str) -> list[pathlib.Path]: return [pathlib.Path(cache_dict[url]) for url in cache_dict if pattern_str in url] -def write_file_to_cache(filename, cachename, filetype, remote_host="github"): +def write_file_to_cache(filename: str, cachename: str, filetype: str, remote_host: str="github"): """Use ``astropy.utils.data`` to save local file to cache This function writes a local file to the PypeIt cache as if it came from a @@ -638,14 +641,14 @@ def write_file_to_cache(filename, cachename, filetype, remote_host="github"): The remote host scheme. Currently only 'github' and 's3_cloud' are supported. Defaults to 'github'. """ - # Build the `url_key` as if this file were in the remote location + # Build the `url_key` as if this file were in the remote location url_key, _ = _build_remote_url(cachename, filetype, remote_host=remote_host) # Use `import file_to_cache()` to place the `filename` into the cache astropy.utils.data.import_file_to_cache(url_key, filename, pkgname="pypeit") -def _build_remote_url(f_name, f_type, remote_host=""): +def _build_remote_url(f_name: str, f_type: str, remote_host: str=""): """Build the remote URL for the ``astropy.utils.data`` functions This function keeps the URL-creation in one place. In the event that files @@ -678,9 +681,10 @@ def _build_remote_url(f_name, f_type, remote_host=""): # Hard-wire the URL based on PypeIt Version data_url = ( "https://raw.githubusercontent.com/pypeit/PypeIt/" - f"{'develop' if '.dev' in __version__ else __version__}/pypeit/data" + f"{'develop' if '.dev' in __version__ else __version__}" + f"/pypeit/data/{f_type}/{f_name}" ) - return f"{data_url}/{f_type}/{f_name}", None + return data_url, None if remote_host == "s3_cloud": # Build up the (permanent, fake) `remote_url` and (fluid, real) `sources` for S3 Cloud @@ -690,7 +694,7 @@ def _build_remote_url(f_name, f_type, remote_host=""): msgs.error(f"Remote host type {remote_host} is not supported for package data caching.") -def _get_s3_hostname(): +def _get_s3_hostname() -> str: """Get the current S3 hostname from the package file Since the S3 server hostname used to hold package data such as telluric @@ -723,7 +727,8 @@ def _get_s3_hostname(): requests.exceptions.ConnectionError, requests.exceptions.RequestException, urllib.error.URLError, - github.GithubException + github.GithubException, + TimeoutError, ): filepath = Paths.data / "s3_url.txt" @@ -743,7 +748,7 @@ def load_telluric_grid(filename: str): The filename (NO PATH) of the telluric atmospheric grid to use. Returns: - (:obj:`astropy.io.fits.HDUList`): Telluric Grid FITS HDU list + (`astropy.io.fits.HDUList`_): Telluric Grid FITS HDU list """ # Check for existance of file parameter if not filename: @@ -774,12 +779,12 @@ def load_thar_spec(): The filename (NO PATH) of the telluric atmospheric grid to use. Returns: - (:obj:`astropy.io.fits.HDUList`): ThAr Spectrum FITS HDU list + (`astropy.io.fits.HDUList`_): ThAr Spectrum FITS HDU list """ return io.fits_open(Paths.arclines / 'thar_spec_MM201006.fits') -def load_sky_spectrum(sky_file): +def load_sky_spectrum(sky_file: str) -> xspectrum1d.XSpectrum1D: """Load a sky spectrum into an XSpectrum1D object NOTE: This is where the path to the data directory is added! @@ -793,6 +798,6 @@ def load_sky_spectrum(sky_file): The filename (NO PATH) of the sky file to use. Returns: - (:obj:`XSpectrum1D`): Sky spectrum + (`linetools.spectra.xspectrum1d.XSpectrum1D`_): Sky spectrum """ return xspectrum1d.XSpectrum1D.from_file(str(Paths.sky_spec / sky_file)) diff --git a/pypeit/datamodel.py b/pypeit/datamodel.py index 550ac3feda..9101420aa5 100644 --- a/pypeit/datamodel.py +++ b/pypeit/datamodel.py @@ -17,13 +17,13 @@ - Define a class attribute called ``datamodel``. See the examples below for their format. - - Provide an :func:`__init__` method that defines the + - Provide an ``__init__`` method that defines the instantiation calling sequence and passes the relevant dictionary to this base-class instantiation. - - Provide a :func:`_validate` method, if necessary, that - processes the data provided in the `__init__` into a complete + - Provide a ``_validate`` method, if necessary, that + processes the data provided in the ``__init__`` into a complete instantiation of the object. This method and the - :func:`__init__` method are the *only* places where attributes + ``__init__`` method are the *only* places where attributes can be added to the class. - Provide a :func:`_bundle` method that reorganizes the datamodel into partitions that can be written to one or more fits @@ -39,7 +39,7 @@ The attributes of the class are *not required* to be a part of the ``datamodel``; however, it makes the object simpler when they are. Any attributes that are not part of the ``datamodel`` must - be defined in either the :func:`__init__` or :func:`_validate` + be defined in either the ``__init__`` or ``_validate`` methods; otherwise, the class with throw an ``AttributeError``. Here are some examples of how to and how not to use them. @@ -424,7 +424,7 @@ def _validate(self): - The following instantiation is fine because ``DubiousInitContainer`` handles the fact that some of the - arguments to :func:`__init__` are not part of the datamodel:: + arguments to ``__init__`` are not part of the datamodel:: data = DubiousInitContainer(x,y) print(np.array_equal(data.out, data.inp1+data.inp2)) @@ -453,8 +453,6 @@ def _validate(self): print(data.func == _data.func) # True ----- - .. include common links, assuming primary doc root is up one directory .. include:: ../include/links.rst @@ -493,13 +491,13 @@ class DataContainer: Derived classes must do the following: - Define a datamodel - - Provide an :func:`__init__` method that defines the + - Provide an ``__init__`` method that defines the instantiation calling sequence and passes the relevant dictionary to this base-class instantiation. - - Provide a :func:`_validate` method, if necessary, that - processes the data provided in the `__init__` into a + - Provide a ``_validate`` method, if necessary, that + processes the data provided in the ``__init__`` into a complete instantiation of the object. This method and the - :func:`__init__` method are the *only* places where + ``__init__`` method are the *only* places where attributes can be added to the class. - Provide a :func:`_bundle` method that reorganizes the datamodel into partitions that can be written to one or @@ -515,8 +513,8 @@ class DataContainer: The attributes of the class are *not required* to be a part of the ``datamodel``; however, it makes the object simpler when they are. Any attributes that are not part of the - ``datamodel`` must be defined in either the :func:`__init__` - or :func:`_validate` methods; otherwise, the class with throw + ``datamodel`` must be defined in either the ``__init__`` + or ``_validate`` methods; otherwise, the class with throw an ``AttributeError``. .. todo:: @@ -528,8 +526,8 @@ class DataContainer: Dictionary to copy to the internal attribute dictionary. All of the keys in the dictionary *must* be elements of the ``datamodel``. Any attributes that are not part of - the ``datamodel`` can be set in the :func:`__init__` or - :func:`_validate` methods of a derived class. If None, + the ``datamodel`` can be set in the ``__init__`` or + ``_validate`` methods of a derived class. If None, the object is instantiated with all of the relevant data model attributes but with all of those attributes set to None. diff --git a/pypeit/deprecated/coadd.py b/pypeit/deprecated/coadd.py index 37038677da..953a35b120 100644 --- a/pypeit/deprecated/coadd.py +++ b/pypeit/deprecated/coadd.py @@ -47,8 +47,125 @@ plt.rcParams["axes.labelsize"] = 17 -# TODO: merge with wavegrid routine in wvutils +# JFH This code should probably be deprecated since it is not used anywhere. +def order_median_scale(waves, fluxes, ivars, masks, min_good=0.05, maxiters=5, + max_factor=10., sigrej=3, debug=False, show=False): + ''' + Function to scaling different orders by the median S/N + + + Args: + waves (`numpy.ndarray`_): wavelength array of your spectra with the shape of (nspec, norder) + fluxes (`numpy.ndarray`_): flux array of your spectra with the shape of (nspec, norder) + ivars (`numpy.ndarray`_): ivar array of your spectra with the shape of (nspec, norder) + masks (`numpy.ndarray`_): mask for your spectra with the shape of (nspec, norder) + min_good (float, optional): minimum fraction of the total number of good pixels needed for estimate the median ratio + maxiters (int or float, optional): maximum iterations for rejecting outliers + max_factor (float, optional): maximum scale factor + sigrej (float, optional): sigma used for rejecting outliers + debug (bool, optional): if True show intermediate QA + show (bool, optional): if True show the final QA + + Returns: + tuple: (1) fluxes_new (`numpy.ndarray`_): re-scaled fluxes with the shape + of (nspec, norder). (2) ivars_new (`numpy.ndarray`_): re-scaled ivars + with the shape of (nspec, norder) (3) order_ratios (`numpy.ndarray`_): an + array of scale factor with the length of norder + ''' + + norder = np.shape(waves)[1] + order_ratios = np.ones(norder) + + ## re-scale bluer orders to match the reddest order. + # scaling spectrum order by order. We use the reddest order as the reference since slit loss in redder is smaller + for ii in range(norder - 1): + iord = norder - ii - 1 + wave_blue, flux_blue, ivar_blue, mask_blue = waves[:, iord-1], fluxes[:, iord-1],\ + ivars[:, iord-1], masks[:, iord-1] + + wave_red_tmp, flux_red_tmp = waves[:, iord], fluxes[:, iord]*order_ratios[iord] + ivar_red_tmp, mask_red_tmp = ivars[:, iord]*1.0/order_ratios[iord]**2, masks[:, iord] + wave_mask = wave_red_tmp>1.0 + wave_red, flux_red, ivar_red, mask_red = wave_red_tmp[wave_mask], flux_red_tmp[wave_mask], \ + ivar_red_tmp[wave_mask], mask_red_tmp[wave_mask], + + # interpolate iord-1 (bluer) to iord-1 (redder) + flux_blue_inter, ivar_blue_inter, mask_blue_inter = interp_spec(wave_red, wave_blue, flux_blue, ivar_blue, mask_blue) + + npix_overlap = np.sum(mask_blue_inter & mask_red) + percentile_iord = np.fmax(100.0 * (npix_overlap / np.sum(mask_red)-0.05), 10) + + mask_both = mask_blue_inter & mask_red + snr_median_red = np.median(flux_red[mask_both]*np.sqrt(ivar_red[mask_both])) + snr_median_blue = np.median(flux_blue_inter[mask_both]*np.sqrt(ivar_blue_inter[mask_both])) + + ## TODO: we set the SNR to be minimum of 300 to turn off the scaling but we need the QA plot + ## need to think more about whether we need to scale different orders, it seems make the spectra + ## much bluer than what it should be. + if (snr_median_blue>300.0) & (snr_median_red>300.0): + order_ratio_iord = robust_median_ratio(flux_blue_inter, ivar_blue_inter, flux_red, ivar_red, mask=mask_blue_inter, + mask_ref=mask_red, ref_percentile=percentile_iord, min_good=min_good, + maxiters=maxiters, max_factor=max_factor, sigrej=sigrej) + order_ratios[iord - 1] = np.fmax(np.fmin(order_ratio_iord, max_factor), 1.0/max_factor) + msgs.info('Scaled {}th order to {}th order by {:}'.format(iord-1, iord, order_ratios[iord-1])) + else: + if ii>0: + order_ratios[iord - 1] = order_ratios[iord] + msgs.warn('Scaled {}th order to {}th order by {:} using the redder order scaling ' + 'factor'.format(iord-1, iord, order_ratios[iord-1])) + else: + msgs.warn('The SNR in the overlapped region is too low or there is not enough overlapped pixels.'+ msgs.newline() + + 'Median scale between order {:} and order {:} was not attempted'.format(iord-1, iord)) + + if debug: + plt.figure(figsize=(12, 8)) + plt.plot(wave_red[mask_red], flux_red[mask_red], 'k-', label='reference spectrum') + plt.plot(wave_blue[mask_blue], flux_blue[mask_blue],color='dodgerblue', lw=3, label='raw spectrum') + plt.plot(wave_blue[mask_blue], flux_blue[mask_blue]*order_ratios[iord-1], color='r', + alpha=0.5, label='re-scaled spectrum') + ymin, ymax = get_ylim(flux_blue, ivar_blue, mask_blue) + plt.ylim([ymin, ymax]) + plt.xlim([np.min(wave_blue[mask_blue]), np.max(wave_red[mask_red])]) + plt.legend() + plt.xlabel('wavelength') + plt.ylabel('Flux') + plt.show() + + # Update flux and ivar + fluxes_new = np.zeros_like(fluxes) + ivars_new = np.zeros_like(ivars) + for ii in range(norder): + fluxes_new[:, ii] *= order_ratios[ii] + ivars_new[:, ii] *= 1.0/order_ratios[ii]**2 + + if show: + plt.figure(figsize=(12, 8)) + ymin = [] + ymax = [] + for ii in range(norder): + wave_stack_iord = waves[:, ii] + flux_stack_iord = fluxes_new[:, ii] + ivar_stack_iord = ivars_new[:, ii] + mask_stack_iord = masks[:, ii] + med_width = (2.0 * np.ceil(0.1 / 10.0 * np.size(wave_stack_iord[mask_stack_iord])) + 1).astype(int) + flux_med, ivar_med = median_filt_spec(flux_stack_iord, ivar_stack_iord, mask_stack_iord, med_width) + plt.plot(wave_stack_iord[mask_stack_iord], flux_med[mask_stack_iord], alpha=0.7) + #plt.plot(wave_stack_iord[mask_stack_iord], flux_stack_iord[mask_stack_iord], alpha=0.5) + # plt.plot(wave_stack_iord[mask_stack_iord],1.0/np.sqrt(ivar_stack_iord[mask_stack_iord])) + ymin_ii, ymax_ii = get_ylim(flux_stack_iord, ivar_stack_iord, mask_stack_iord) + ymax.append(ymax_ii) + ymin.append(ymin_ii) + plt.xlim([np.min(waves[masks]), np.max(waves[masks])]) + plt.ylim([-0.15*np.median(ymax), 1.5*np.median(ymax)]) + plt.xlabel('Wavelength ($\\rm\\AA$)') + plt.ylabel('Flux') + plt.show() + + return fluxes_new, ivars_new, order_ratios + + +# TODO: merge with wavegrid routine in wvutils def sensfunc_weights_old(sensfile, waves, masks, debug=False): ''' diff --git a/pypeit/deprecated/datacube.py b/pypeit/deprecated/datacube.py index 5fd01e07b6..56b3c1cc27 100644 --- a/pypeit/deprecated/datacube.py +++ b/pypeit/deprecated/datacube.py @@ -20,7 +20,7 @@ def generate_cube_resample(outfile, frame_wcs, slits, fluximg, ivarimg, raimg, d Args: outfile (`str`): Filename to be used to save the datacube - frame_wcs (`astropy.wcs.wcs.WCS`_): + frame_wcs (`astropy.wcs.WCS`_): World coordinate system for this frame. slits (:class:`pypeit.slittrace.SlitTraceSet`_) Information stored about the slits @@ -47,7 +47,7 @@ def generate_cube_resample(outfile, frame_wcs, slits, fluximg, ivarimg, raimg, d when evaluating the voxel geometry in detector coordinates overwrite (bool, optional): If the output file exists, it will be overwritten if this parameter is True. - output_wcs (`astropy.wcs.wcs.WCS`_, optional): + output_wcs (`astropy.wcs.WCS`_, optional): World coordinate system for the output datacube. If None, frame_wcs will be used. blaze_wave (`numpy.ndarray`_, optional): Wavelength array of the spectral blaze function @@ -214,3 +214,77 @@ def generate_cube_resample(outfile, frame_wcs, slits, fluximg, ivarimg, raimg, d msgs.info("Saving datacube as: {0:s}".format(outfile)) final_cube = DataCube(datcube.T, varcube.T, specname, blaze_wave, blaze_spec, sensfunc=sensfunc, fluxed=fluxcal) final_cube.to_file(outfile, hdr=hdr, overwrite=overwrite) + + +def generate_cube_ngp(outfile, hdr, all_sci, all_ivar, all_wghts, vox_coord, bins, + overwrite=False, blaze_wave=None, blaze_spec=None, fluxcal=False, + sensfunc=None, specname="PYP_SPEC", debug=False): + """ + TODO :: Deprecate this routine once subpixellate works for combining cubes + + Save a datacube using the Nearest Grid Point (NGP) algorithm. + + Args: + outfile (`str`): + Filename to be used to save the datacube + hdr (`astropy.io.fits.header_`): + Header of the output datacube (must contain WCS) + all_sci (`numpy.ndarray`_): + 1D flattened array containing the counts of each pixel from all spec2d files + all_ivar (`numpy.ndarray`_): + 1D flattened array containing the inverse variance of each pixel from all spec2d files + all_wghts (`numpy.ndarray`_): + 1D flattened array containing the weights of each pixel to be used in the combination + vox_coord (`numpy.ndarray`_): + The voxel coordinates of each pixel in the spec2d frames. vox_coord is returned by the + function `astropy.wcs.WCS.wcs_world2pix_` once a WCS is setup and every spec2d detector + pixel has an RA, DEC, and WAVELENGTH. + bins (tuple): + A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial and z wavelength coordinates + overwrite (`bool`): + If True, the output cube will be overwritten. + blaze_wave (`numpy.ndarray`_): + Wavelength array of the spectral blaze function + blaze_spec (`numpy.ndarray`_): + Spectral blaze function + fluxcal (bool): + Are the data flux calibrated? + sensfunc (`numpy.ndarray`_, None): + Sensitivity function that has been applied to the datacube + specname (str): + Name of the spectrograph + debug (bool): + Debug the code by writing out a residuals cube? + """ + # Add the unit of flux to the header + if fluxcal: + hdr['FLUXUNIT'] = (flux_calib.PYPEIT_FLUX_SCALE, "Flux units -- erg/s/cm^2/Angstrom/arcsec^2") + else: + hdr['FLUXUNIT'] = (1, "Flux units -- counts/s/Angstrom/arcsec^2") + + # Use NGP to generate the cube - this ensures errors between neighbouring voxels are not correlated + datacube, edges = np.histogramdd(vox_coord, bins=bins, weights=all_sci * all_wghts) + normcube, edges = np.histogramdd(vox_coord, bins=bins, weights=all_wghts) + nc_inverse = utils.inverse(normcube) + datacube *= nc_inverse + # Create the variance cube, including weights + msgs.info("Generating variance cube") + all_var = utils.inverse(all_ivar) + var_cube, edges = np.histogramdd(vox_coord, bins=bins, weights=all_var * all_wghts ** 2) + var_cube *= nc_inverse**2 + bpmcube = (normcube == 0).astype(np.uint8) + + # Save the datacube + if debug: + datacube_resid, edges = np.histogramdd(vox_coord, bins=bins, weights=all_sci * np.sqrt(all_ivar)) + normcube, edges = np.histogramdd(vox_coord, bins=bins) + nc_inverse = utils.inverse(normcube) + outfile_resid = "datacube_resid.fits" + msgs.info("Saving datacube as: {0:s}".format(outfile_resid)) + hdu = fits.PrimaryHDU((datacube_resid*nc_inverse).T, header=hdr) + hdu.writeto(outfile_resid, overwrite=overwrite) + + msgs.info("Saving datacube as: {0:s}".format(outfile)) + final_cube = DataCube(datacube.T, np.sqrt(var_cube.T), bpmcube.T, specname, blaze_wave, blaze_spec, + sensfunc=sensfunc, fluxed=fluxcal) + final_cube.to_file(outfile, hdr=hdr, overwrite=overwrite) diff --git a/pypeit/deprecated/flux.py b/pypeit/deprecated/flux.py index b57626f67c..925b09f55d 100644 --- a/pypeit/deprecated/flux.py +++ b/pypeit/deprecated/flux.py @@ -31,6 +31,10 @@ SN2_MAX = (20.0) ** 2 PYPEIT_FLUX_SCALE = 1e-17 + + + + def apply_sensfunc(spec_obj, sens_dict, airmass, exptime, extinct_correct=True, telluric_correct = False, longitude=None, latitude=None): """ Apply the sensitivity function to the data diff --git a/pypeit/deprecated/fluxcalibrate.py b/pypeit/deprecated/fluxcalibrate.py new file mode 100644 index 0000000000..5ec7636691 --- /dev/null +++ b/pypeit/deprecated/fluxcalibrate.py @@ -0,0 +1,128 @@ +# Module for flux calibrating spectra +import numpy as np +import os +import matplotlib.pyplot as plt +from astropy.io import fits + +from pypeit import msgs +from pypeit.spectrographs.util import load_spectrograph +from pypeit import specobjs +from pypeit.history import History +from astropy import table +from IPython import embed + + + +class FluxCalibrate: + """ + Class for flux calibrating spectra. + + Args: + spec1dfiles (list): + List of PypeIt spec1d files that you want to flux calibrate + sensfiles (list): + List of sensitivity function files to use to flux calibrate the spec1d files. This list and the sensfiles + list need to have the same length and be aligned + par (pypeit.par.pypeitpar.FluxCalibrate, optional): + Parset object containing parameters governing the flux calibration. + outfiles (list, optional): + Names of the output files. If None, this is set to spec1dfiles and those are overwritten + """ + def __init__(self, spec1dfiles, sensfiles, par=None, outfiles=None, debug=False): + + from pypeit import sensfunc + + self.spec1dfiles = spec1dfiles + self.sensfiles = sensfiles + + # Output file names + self.outfiles = spec1dfiles if outfiles is None else outfiles + + # Load the spectrograph + header = fits.getheader(spec1dfiles[0]) + self.spectrograph = load_spectrograph(header['PYP_SPEC']) + self.par = self.spectrograph.default_pypeit_par()['fluxcalib'] if par is None else par + self.debug = debug + + sensf_last = None + for spec1, sensf, outfile in zip(self.spec1dfiles, self.sensfiles, self.outfiles): + # Read in the data + sobjs = specobjs.SpecObjs.from_fitsfile(spec1) + history = History(sobjs.header) + if sensf != sensf_last: + sens = sensfunc.SensFunc.from_file(sensf) + sensf_last = sensf + history.append(f'PypeIt Flux calibration "{sensf}"') + apply_flux_calib(self.par, self.spectrograph, sobjs, sens) + sobjs.write_to_fits(sobjs.header, outfile, history=history, overwrite=True) + + +def apply_flux_calib(par, spectrograph, sobjs, sens): + """ + Flux calibrate the provided object spectra (``sobjs``) using the + provided sensitivity function (``sens``). + + + Args: + sobjs (:class:`~pypeit.specobjs.SpecObjs`): + Object spectra + sens (:class:`~pypeit.sensfunc.SensFunc`): + Sensitivity function + """ + + _extinct_correct = (True if sens.algorithm == 'UVIS' else False) \ + if par['extinct_correct'] is None else par['extinct_correct'] + + if spectrograph.pypeline == 'MultiSlit': + for ii, sci_obj in enumerate(sobjs): + if sens.wave.shape[1] == 1: + sci_obj.apply_flux_calib(sens.wave[:, 0], sens.zeropoint[:, 0], + sobjs.header['EXPTIME'], + extinct_correct=_extinct_correct, + longitude=spectrograph.telescope['longitude'], + latitude=spectrograph.telescope['latitude'], + extinctfilepar=par['extinct_file'], + extrap_sens=par['extrap_sens'], + airmass=float(sobjs.header['AIRMASS'])) + elif sens.wave.shape[1] > 1 and sens.splice_multi_det: + # This deals with the multi detector case where the sensitivity function is spliced. Note that + # the final sensitivity function written to disk is the spliced one. This functionality is only + # used internal to sensfunc.py for fluxing the standard for the QA plot. + sci_obj.apply_flux_calib(sens.wave[:, ii], sens.zeropoint[:, ii], + sobjs.header['EXPTIME'], + extinct_correct=_extinct_correct, + longitude=spectrograph.telescope['longitude'], + latitude=spectrograph.telescope['latitude'], + extinctfilepar=par['extinct_file'], + extrap_sens=par['extrap_sens'], + airmass=float(sobjs.header['AIRMASS'])) + else: + msgs.error('This should not happen, there is a problem with your sensitivity function.') + + + elif spectrograph.pypeline == 'Echelle': + # Flux calibrate the orders that are mutually in the meta_table and in + # the sobjs. This allows flexibility for applying to data for cases + # where not all orders are present in the data as in the sensfunc, etc., + # i.e. X-shooter with the K-band blocking filter. + ech_orders = np.array(sens.sens['ECH_ORDERS']).flatten() + for sci_obj in sobjs: + # JFH Is there a more elegant pythonic way to do this without looping over both orders and sci_obj? + indx = np.where(ech_orders == sci_obj.ECH_ORDER)[0] + if indx.size==1: + sci_obj.apply_flux_calib(sens.wave[:, indx[0]], sens.zeropoint[:,indx[0]], + sobjs.header['EXPTIME'], + extinct_correct=_extinct_correct, + extrap_sens = par['extrap_sens'], + longitude=spectrograph.telescope['longitude'], + latitude=spectrograph.telescope['latitude'], + extinctfilepar=par['extinct_file'], + airmass=float(sobjs.header['AIRMASS'])) + elif indx.size == 0: + msgs.info('Unable to flux calibrate order = {:} as it is not in your sensitivity function. ' + 'Something is probably wrong with your sensitivity function.'.format(sci_obj.ECH_ORDER)) + else: + msgs.error('This should not happen') + + else: + msgs.error('Unrecognized pypeline: {0}'.format(spectrograph.pypeline)) \ No newline at end of file diff --git a/pypeit/deprecated/orig_specobjs.py b/pypeit/deprecated/orig_specobjs.py index f14f9ab5b0..cf394c4969 100644 --- a/pypeit/deprecated/orig_specobjs.py +++ b/pypeit/deprecated/orig_specobjs.py @@ -380,7 +380,7 @@ def nobj(self): def get_std(self): """ - Return the standard star from this Specobjs. For MultiSlit this + Return the standard star from this SpecObjs. For MultiSlit this will be a single specobj in SpecObjs container, for Echelle it will be the standard for all the orders. diff --git a/pypeit/deprecated/procimg.py b/pypeit/deprecated/procimg.py index 2e1aaedb27..f2d3cccd6b 100644 --- a/pypeit/deprecated/procimg.py +++ b/pypeit/deprecated/procimg.py @@ -50,7 +50,7 @@ def lacosmic(sciframe, saturation, nonlinear, varframe=None, maxiter=1, grow=1.5 subsam = utils.subsample(scicopy) conved = signal.convolve2d(subsam, laplkernel, mode="same", boundary="symm") cliped = conved.clip(min=0.0) - lplus = utils.rebin_evlist(cliped, np.array(cliped.shape)/2.0) + lplus = utils.rebinND(cliped, np.array(cliped.shape)/2.0) msgs.info("Creating noise model") # Build a custom noise map, and compare this to the laplacian diff --git a/pypeit/scripts/ql_multislit.py b/pypeit/deprecated/ql_multislit.py similarity index 99% rename from pypeit/scripts/ql_multislit.py rename to pypeit/deprecated/ql_multislit.py index e409f7dba5..fb5780f138 100644 --- a/pypeit/scripts/ql_multislit.py +++ b/pypeit/deprecated/ql_multislit.py @@ -254,7 +254,7 @@ def reduce(files, caliBrate, spectrograph, parset, bkg_files=None, show=False, s global_sky=global_sky, waveTilts=caliBrate.wavetilts, wv_calib=caliBrate.wv_calib, bkg_redux=bkg_redux, return_negative=bkg_redux, show=show) - skymodel, objmodel, ivarmodel, outmask, sobjs, waveimg, tilts = extract.run() + skymodel, objmodel, ivarmodel, outmask, sobjs, waveimg, tilts, slits = extract.run() # TODO -- Do this upstream # Tack on detector @@ -304,7 +304,7 @@ def reduce(files, caliBrate, spectrograph, parset, bkg_files=None, show=False, s vel_corr=None, vel_type=parset['calibrations']['wavelengths']['refframe'], tilts=tilts, - slits=copy.deepcopy(caliBrate.slits), + slits=copy.deepcopy(slits), wavesol=caliBrate.wv_calib.wave_diagnostics(print_diag=False), maskdef_designtab=None) return spec2DObj, spec2DObj_bkg diff --git a/pypeit/display/display.py b/pypeit/display/display.py index 26e80f0f5b..6d3b574199 100644 --- a/pypeit/display/display.py +++ b/pypeit/display/display.py @@ -43,7 +43,7 @@ def connect_to_ginga(host='localhost', port=9000, raise_err=False, allow_new=Fal viewer if one is not already running. Returns: - RemoteClient: connection to ginga viewer. + `ginga.RemoteClient`_: connection to ginga viewer. """ # Start viewer = grc.RemoteClient(host, port) @@ -120,8 +120,8 @@ def show_image(inp, chname='Image', waveimg=None, mask=None, exten=0, cuts=None, image in other channels to it. Returns: - ginga.util.grc.RemoteClient, ginga.util.grc._channel_proxy: The - ginga remote client and the channel with the displayed image. + :obj:`tuple`: The ginga remote client object and the channel object with + the displayed image. Raises: ValueError: @@ -253,21 +253,21 @@ def show_points(viewer, ch, spec, spat, color='cyan', legend=None, legend_spec=N Parameters ---------- - viewer (ginga.util.grc.RemoteClient): + viewer : `ginga.RemoteClient`_ Ginga RC viewer - ch (ginga.util.grc._channel_proxy): + ch : ``ginga.util.grc._channel_proxy`` Ginga channel - spec (list): + spec : list List of spectral positions on image to plot - spat (list): + spat : list List of spatial positions on image to plot - color (str): + color : str Color for points - legend (str): - Label for a legeng - legend_spec (float): + legend : str + Label for a legend + legend_spec : float Spectral pixel loation for legend - legend_spat (float): + legend_spat : float Pixel loation for legend """ @@ -291,9 +291,9 @@ def show_slits(viewer, ch, left, right, slit_ids=None, left_ids=None, right_ids= Overplot slits on the image in Ginga in the given channel Args: - viewer (ginga.util.grc.RemoteClient): + viewer (`ginga.RemoteClient`_): Ginga RC viewer - ch (ginga.util.grc._channel_proxy): + ch (``ginga.util.grc._channel_proxy``): Ginga channel left (`numpy.ndarray`_): Array with spatial position of left slit edges. Shape must be :math:`(N_{\rm @@ -443,13 +443,13 @@ def show_slits(viewer, ch, left, right, slit_ids=None, left_ids=None, right_ids= def show_trace(viewer, ch, trace, trc_name=None, maskdef_extr=None, manual_extr=None, clear=False, - rotate=False, pstep=50, yval=None): + rotate=False, pstep=50, yval=None, color='blue'): r""" Args: - viewer (ginga.util.grc.RemoteClient): + viewer (`ginga.RemoteClient`_): Ginga RC viewer. - ch (ginga.util.grc._channel_proxy): + ch (``ginga.util.grc._channel_proxy``): Ginga channel. trace (`numpy.ndarray`_): Array with spatial position of the object traces on the detector. @@ -470,6 +470,8 @@ def show_trace(viewer, ch, trace, trc_name=None, maskdef_extr=None, manual_extr= Array with spectral position of the object traces. Shape must be :math:`(N_{\rm spec},)` or :math:`(N_{\rm spec}, N_{\rm trace})`. If not passed in, the default of np.arange(:math:`(N_{\rm spec},)`) will be used. + color (str, optional): + Color for the trace """ # Canvas @@ -480,6 +482,10 @@ def show_trace(viewer, ch, trace, trc_name=None, maskdef_extr=None, manual_extr= if trace.ndim == 1: trace = trace.reshape(-1,1) + ntrace = trace.shape[1] + _maskdef_extr = ntrace*[False] if maskdef_extr is None else maskdef_extr + _manual_extr = ntrace*[False] if manual_extr is None else manual_extr + # Show if yval is None: y = np.repeat(np.arange(trace.shape[0]).astype(float)[:, None], trace.shape[1], axis=1) @@ -488,22 +494,22 @@ def show_trace(viewer, ch, trace, trc_name=None, maskdef_extr=None, manual_extr= canvas_list = [] for i in range(trace.shape[1]): - if maskdef_extr[i]: - color = '#f0e442' - elif manual_extr[i]: - color = '#33ccff' + if _maskdef_extr[i]: + _color = '#f0e442' if color is not None else color + elif _manual_extr[i]: + _color = '#33ccff' if color is not None else color else: - color = 'orange' + _color = 'orange' if color is not None else color canvas_list += [dict(type=str('path'), args=(list(zip(y[::pstep,i].tolist(), trace[::pstep,i].tolist())),) if rotate else (list(zip(trace[::pstep,i].tolist(), y[::pstep,i].tolist())),), - kwargs=dict(color=color))] + kwargs=dict(color=_color))] # Text ohf = len(trace[:,i])//2 # Do it canvas_list += [dict(type='text',args=(float(y[ohf,i]), float(trace[ohf,i]), str(trc_name[i])) if rotate else (float(trace[ohf,i]), float(y[ohf,i]), str(trc_name[i])), - kwargs=dict(color=color, fontsize=17., rot_deg=90.))] + kwargs=dict(color=_color, fontsize=17., rot_deg=90.))] canvas.add('constructedcanvas', canvas_list) @@ -543,9 +549,9 @@ def show_tilts(viewer, ch, tilt_traces, yoff=0., xoff=0., points=True, nspec=Non Show the arc tilts on the input channel Args: - viewer (ginga.util.grc.RemoteClient): + viewer (`ginga.RemoteClient`_): Ginga RC viewer - ch (ginga.util.grc._channel_proxy): + ch (``ginga.util.grc._channel_proxy``): Ginga channel tilt_traces (`astropy.table.Table`_): Table containing the traced and fitted tilts diff --git a/pypeit/display/ginga_plugins.py b/pypeit/display/ginga_plugins.py index 42f3c93260..7560dda0a4 100644 --- a/pypeit/display/ginga_plugins.py +++ b/pypeit/display/ginga_plugins.py @@ -43,12 +43,12 @@ def info_xy(self, data_x, data_y, settings): image x coordinate data_y: float image y coordinate - settings: :class:`ginga.misc.Settings.SettingGroup` + settings: ``ginga.misc.Settings.SettingGroup`` ginga settings group Returns ------- - info : :class:`ginga.misc.Bunch.Bunch` + info : ``ginga.misc.Bunch.Bunch`` Metadata for this coordinate """ info = super(SlitImage, self).info_xy(data_x, data_y, settings) diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index b8455f3f64..45b0847f27 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -137,7 +137,6 @@ from pypeit.spectrographs.spectrograph import Spectrograph from pypeit.spectrographs.util import load_spectrograph - class EdgeTraceBitMask(BitMask): """ Mask bits used during slit tracing. @@ -176,9 +175,11 @@ def __init__(self): ('ABNORMALSLIT_LONG', 'Slit formed by left and right edge is abnormally long'), ('USERRMSLIT', 'Slit removed by user'), ('NOORDER', 'Unable to associate this trace with an echelle order (echelle ' - ' spectrographs only)'), + 'spectrographs only)'), ('ORDERMISMATCH', 'Slit traces are not well matched to any echelle order (echelle ' - ' spectrographs only)'), + 'spectrographs only)'), + ('ORDERINSERT', 'Trace was inserted as the expected location of an echelle ' + 'order missed by the automated tracing'), ('LARGELENGTHCHANGE', 'Large difference in the slit length as a function of ' 'wavelength.')]) super(EdgeTraceBitMask, self).__init__(list(mask.keys()), descr=list(mask.values())) @@ -196,7 +197,7 @@ def insert_flags(self): List of flags used to mark traces inserted for various reasons. """ - return ['USERINSERT', 'SYNCINSERT', 'MASKINSERT', 'ORPHANINSERT'] + return ['USERINSERT', 'SYNCINSERT', 'MASKINSERT', 'ORPHANINSERT', 'ORDERINSERT'] @property def order_flags(self): @@ -240,7 +241,7 @@ class EdgeTraceSet(calibframe.CalibFrame): used. The defaults are tuned for each spectrograph based on testing using data in the PypeIt development suite. See :ref:`pypeitpar` for the full documentation of the - :class:`pypeit.par.pypeitpar.EdgeTracePar` parameters. + :class:`~pypeit.par.pypeitpar.EdgeTracePar` parameters. Finally, note that the :attr:`design` and :attr:`object` data are currently empty, as part of a development path for matching slits @@ -284,7 +285,7 @@ class EdgeTraceSet(calibframe.CalibFrame): Attributes: traceimg - (:class:`~pypeit.images.buildcalibration.TraceImage`): + (:class:`~pypeit.images.buildimage.TraceImage`): See argument list. spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): @@ -295,7 +296,7 @@ class EdgeTraceSet(calibframe.CalibFrame): The list of raw files used to construct the trace image (:attr:`img`). Only defined if argument `img` in :func:`initial_trace` or :func:`auto_trace` is a - :class:`~pypeit.images.buildcalibration.TraceImage` object. + :class:`~pypeit.images.buildimage.TraceImage` object. img (`numpy.ndarray`_): Convenience for now. det (:obj:`int`): @@ -421,14 +422,14 @@ class EdgeTraceSet(calibframe.CalibFrame): 'omodel_tspat', # Right edges predicted by the optical model # (before x-correlation) 'cc_params_b', # Parameters of the x-correlation between LEFT edges - # predicted by the slitmask design and the one traced + # predicted by the slitmask design and the one traced # on the image. 'cc_params_t', # Parameters of the x-correlation between RIGHT edges # predicted by the slitmask design and the one traced # on the image. 'maskfile', # File used to slurp in slit-mask design 'slitmask', # SlitMask instance that hold info on slitmask design - 'success'] # Flag that the automatic edge tracing was successful + 'success'] # Flag that the automatic edge tracing was successful """ Attributes kept separate from the datamodel. """ @@ -674,7 +675,7 @@ def rectify(self, flux, bpm=None, extract_width=None, mask_threshold=0.5, side=' for more detail. Used parameters from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`) are + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are `left_right_pca`. Args: @@ -769,7 +770,7 @@ def auto_trace(self, bpm=None, debug=False, show_stages=False): user-provided lists in the :attr:`par`. Used parameters from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`) are + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are `use_maskdesign`. Args: @@ -838,6 +839,13 @@ def auto_trace(self, bpm=None, debug=False, show_stages=False): msgs.info('{0:^50}'.format('Matching traces to the slit-mask design')) msgs.info('-' * 50) self.maskdesign_matching(debug=debug) + if show_stages: + self.show(title='After matching to slit-mask design metadata.') + if np.all(self.bitmask.flagged(self.edge_msk, self.bitmask.bad_flags)): + msgs.error('All traces masked! Problem with mask-design matching, which may be ' + 'due to spurious edges. Try changing the edge detection threshold ' + '(edge_thresh) and troubleshooting the problem using the ' + 'pypeit_trace_edges script.') if self.par['auto_pca'] and not self.can_pca() and not self.is_empty and self.par['sync_predict'] == 'pca': # TODO: This causes the code to fault. Maybe there's a way @@ -866,6 +874,15 @@ def auto_trace(self, bpm=None, debug=False, show_stages=False): if show_stages: self.show(title='After synchronizing left-right traces into slits') + if not self.is_empty and self.par['add_missed_orders']: + self.order_refine(debug=debug) + # Check that the edges are still sinked (overkill?) + self.success = self.sync() + if not self.success: + return + if show_stages: + self.show(title='After adding in missing orders') + # First manually remove some traces, just in case a user # wishes to manually place a trace nearby a trace that # was automatically identified. One problem with adding @@ -1098,7 +1115,7 @@ def _base_header(self, hdr=None): """ Construct the baseline header for all HDU extensions. - This appends the :class:`EdgeTracePar` and + This appends the :class:`~pypeit.par.pypeitpar.EdgeTracePar` and :class:`EdgeTraceBitMask` data to the headers of all HDU extensions. This is overkill, but avoids overriding :func:`pypeit.datamodel.DataContainer.to_hdu` by just @@ -1995,7 +2012,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp :func:`trace_pixels_off_detector`. The only used parameter from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`) is ``match_tol``. + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) is ``match_tol``. .. warning:: @@ -2384,7 +2401,7 @@ def check_synced(self, rebuild_pca=False): synchronized partner. Used parameters from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`) are + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are `minimum_slit_gap`, `minimum_slit_length`, `minimum_slit_length_sci`, and `length_range`. @@ -2962,7 +2979,7 @@ def fit_refine(self, weighting='uniform', debug=False, idx=None): only traces that are *not* fully flagged are fit. Used parameters from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`) are + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are ``max_shift_abs``, ``max_spat_error``, ``fit_function``, ``fit_order``, ``fwhm_uniform``, ``fwhm_gaussian``, ``fit_maxdev``, ``fit_maxiter``, ``fit_niter``, and @@ -3079,11 +3096,12 @@ def can_pca(self): there are fewer than the minimum left *or* right edge traces. Used parameters from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`) are + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are ``fit_min_spec_length``, ``left_right_pca``, and ``pca_min_edges``. .. warning:: - This function calls :func:`check_trace` using + + This function calls :func:`check_traces` using `fit_min_spec_length` to flag short traces, meaning that :attr:`edge_msk` will can be altered by this call. @@ -3119,6 +3137,8 @@ def can_pca(self): and np.sum(good[self.is_right]) > self.par['pca_min_edges'] \ if self.par['left_right_pca'] else np.sum(good) > self.par['pca_min_edges'] + # TODO: Consolidate the options in `add_user_traces` with this function to + # enable more prediction options, not just the PCA. def predict_traces(self, edge_cen, side=None): """ Use the PCA decomposition to predict traces. @@ -3196,7 +3216,7 @@ def build_pca(self, use_center=False, debug=False): decompositions. Used parameters from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`) are + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are `fit_min_spec_length`, `left_right_pca`, `pca_n`, `pca_var_percent`, `pca_function`, `pca_order`, `pca_sigrej`, `pca_maxrej`, and `pca_maxiter`. @@ -3403,7 +3423,7 @@ def peak_refine(self, rebuild_pca=False, debug=False): :attr:`edge_fit`, and :attr:`fittype`. Used parameters from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`) are + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are ``left_right_pca``, ``edge_thresh``, ``smash_range``, ``edge_detect_clip``, ``trace_median_frac``, ``trace_thresh``, ``fit_function``, ``fit_order``, ``fwhm_uniform``, ``fwhm_uniform``, @@ -3622,7 +3642,7 @@ def _get_reference_locations(self, trace_cen, add_edge): by :func:`sync`. Used parameters from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`) are + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are ``sync_center``, ``sync_to_edge``, and ``gap_offset``. Args: @@ -3743,7 +3763,7 @@ def nudge_traces(self, trace_cen): (`max_nudge`), to be no closer than a minimum number (`det_buffer`) pixels from the detector edges. Both parameters are pulled from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`). No limit is + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`). No limit is imposed on the size of the shift if `max_nudge` is None. .. warning:: @@ -3837,7 +3857,7 @@ def sync(self, rebuild_pca=True, debug=False): exception is raised. Used parameters from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`) are + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are `det_buffer`, `left_right_pca`, and `sync_predict`. .. warning:: @@ -4078,7 +4098,7 @@ def insert_traces(self, side, trace_cen, loc=None, mode='user', resort=True, nud New traces to add are first nudged away from the detector edge (see :func:`nudge_traces`) according to parameters `max_nudge` and `det_buffer` from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`). They are then + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`). They are then inserted or appended to the existing traces and masked according to the provided `mode`. The traces are added to *both* the measured centroid list and the fitted model data. @@ -4121,6 +4141,8 @@ def insert_traces(self, side, trace_cen, loc=None, mode='user', resort=True, nud left and right traces. - ``'mask'``: Traces were generated based on the expected slit positions from mask design data. + - ``'order'``: Traces are the expected location of an + echelle order. resort (:obj:`bool`, optional): Resort the traces in the spatial dimension; see @@ -4131,6 +4153,14 @@ def insert_traces(self, side, trace_cen, loc=None, mode='user', resort=True, nud :func:`nudge_traces`. """ + # TODO: When inserting traces and echelle orders are already matched, + # the length of the orderid vector is not longer valid. For now, just + # remove any existing array and warn the user they they'll need to + # rematch the orders. + if self.orderid is not None: + msgs.warn('Inserting traces invalidates order matching. Removing.') + self.orderid = None + # Check input _side = np.atleast_1d(side) ntrace = _side.size @@ -4161,6 +4191,8 @@ def insert_traces(self, side, trace_cen, loc=None, mode='user', resort=True, nud mask = self.bitmask.turn_on(mask, 'SYNCINSERT') elif mode == 'mask': mask = self.bitmask.turn_on(mask, 'MASKINSERT') + elif mode == 'order': + mask = self.bitmask.turn_on(mask, 'ORDERINSERT') # Set the ID numbers for the new traces _traceid = np.empty(ntrace, dtype=int) @@ -4209,7 +4241,7 @@ def bound_detector(self): masked as user-inserted. Only used parameter from :attr:`par` - (:class:`pypeit.par.pypeitpar.EdgeTracePar`) is + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) is `det_buffer`. """ # find where sobelsig is not masked @@ -4231,7 +4263,7 @@ def fully_masked_traces(self, flag=None, exclude=None): Args: flag (:obj:`str`, :obj:`list`, optional): The bit mask flags to select. If None, any flags are - used. See :func:`pypeit.bitmask.Bitmask.flagged`. + used. See :func:`pypeit.bitmask.BitMask.flagged`. exclude (:obj:`str`, :obj:`list`, optional): A set of flags to explicitly exclude from consideration as a masked trace. I.e., if any @@ -4586,14 +4618,14 @@ def _fill_design_table(self, maskdef_id, cc_params_b, cc_params_t, omodel_bspat, and the one traced on the image. One value per each edge side Args: - maskdef_id (:obj:`numpy.array`): + maskdef_id (`numpy.ndarray`_): Slit ID number from slit-mask design matched to traced slits. cc_params_b, cc_params_t (:obj:`tuple`): Three parameters of the cross-correlation (2 coefficients and RMS) between slit-mask design and traced edges for the left and right edges. - omodel_bspat, omodel_tspat (:obj:`numpy.array`): + omodel_bspat, omodel_tspat (`numpy.ndarray`_): Left and right spatial position of the slit edges from optical model - spat_id (:obj:`numpy.array`): + spat_id (`numpy.ndarray`_): ID assigned by PypeIt to each slit. same as in `SlitTraceSet`. @@ -4664,7 +4696,7 @@ def _fill_objects_table(self, maskdef_id): Args: - maskdef_id (:obj:`numpy.array`): + maskdef_id (`numpy.ndarray`_): Slit ID number from slit-mask design matched to traced slits. """ # Check that slitmask is initiated @@ -4699,6 +4731,355 @@ def _fill_objects_table(self, maskdef_id): self.objects['TRACEID'] = utils.index_of_x_eq_y(self.objects['MASKDEF_ID'], self.design['MASKDEF_ID'], strict=True) +# NOTE: I'd like us to keep this commented mask_refine function around +# for the time being. + # def mask_refine(self, design_file=None, allow_resync=False, debug=False): + # """ + # Use the mask design data to refine the edge trace positions. + # + # Use of this method requires: + # - a PCA decomposition is available, + # - the traces are synchronized into left-right pairs, and + # - :attr:`spectrograph` has a viable `get_slitmask` method + # to read slit mask design data from a file. That file is + # either provided directly or pulled from one of the + # files used to construct the trace image; see + # `design_file`. The result of the `get_slitmask` method + # must provide a + # :class:`pypeit.spectrographs.slitmask.SlitMask` object + # with the slit-mask design data. + # + # TODO: Traces don't need to be synchronized... + # + # Also useful, but not required, is for :attr:`spectrograph` to + # have a viable `get_detector_map` method that provides a + # :class:`pypeit.spectrograph.opticalmodel.DetectorMap` object, + # which is used to provide a guess offset between the slit-mask + # focal-plane positions and the trace pixel positions. If no + # such `get_detector_method` exists, the guess offset is:: + # + # this + # + # and the match between expected and traced slit positions may + # be unstable. + # + # The method uses + # :class:`pypeit.spectrographs.slitmask.SlitRegister` to match + # the expected and traced position and identify both missing + # and erroneous trace locations. The former are used to add new + # traces and the latter are removed. The method also constructs + # the :attr:`design` and :attr:`objects` tables, depending on + # the data accessible via the + # :class:`pypeit.spectrographs.slitmask.SlitMask` instance. + # + # Used parameters from :attr:`par` + # (:class:`pypeit.par.pypeitpar.EdgeTracePar`) are + # `left_right_pca`, `mask_reg_maxiter`, `mask_reg_maxsep`, + # `mask_reg_sigrej`, and `ignore_alignment`. + # + # Args: + # design_file (:obj:`str`, optional): + # A file with the mask design data. If None, the method + # will use the first file in :attr:`files`; if + # :attr:`files` is also None, the method will raise an + # exception. + # debug (:obj:`bool`, optional): + # Run in debug mode. + # """ + # # Still not done with this function... + # raise NotImplementedError() + # + # # Check that there are traces to refine! + # if self.is_empty: + # msgs.error('No traces to refine.') + # + # # The PCA decomposition must have already been determined + # if self.pcatype is None: + # msgs.error('Must first run the PCA analysis for the traces; run build_pca.') + # + # # Get the file to use when parsing the mask design information + # _design_file = (None if self.traceimg.files is None else self.traceimg.files[0]) \ + # if design_file is None else design_file + # if _design_file is None or not os.path.isfile(_design_file): + # msgs.error('Slit-mask design file not found or none provided.') + # + # # Get the paramters to use + # maxiter = self.par['mask_reg_maxiter'] + # maxsep = self.par['mask_reg_maxsep'] + # sigma = self.par['mask_reg_sigrej'] + # ignore_alignment = self.par['ignore_alignment'] + # + # # TODO: Set allow_resync and design_file to be a parameters, as + # # well? + # + # # Read the design data + # msgs.info('Reading slit-mask design information from: {0}'.format(_design_file)) + # if self.spectrograph.get_slitmask(_design_file) is None: + # msgs.error('Unable to read design file or no slit-mask design reader ' + # 'defined for {0}.'.format(self.spectrograph.spectrograph)) + # + # # Match both left and right edges simultaneously + # x_design = np.array([self.spectrograph.slitmask.bottom[:, 0], + # self.spectrograph.slitmask.top[:, 0]]).T.ravel() + # reference_row = self.left_pca.reference_row if self.par['left_right_pca'] \ + # else self.pca.reference_row + # x_det = self.edge_fit[reference_row, :] + # + # # Mask traces that are fully masked, except if they were + # # specifically inserted in a previous step + # # TODO: Should the BOXSLITS also be included here? + # x_det_bpm = self.fully_masked_traces(flag=self.bitmask.bad_flags, + # exclude=self.bitmask.insert_flags) + # + # # x_design = np.amin(self.spectrograph.slitmask.corners[:,:,0], axis=1) + # # side = self.traceid < 0 + # # x_det = self.edge_fit[self.pca.reference_row,side] + # + # # x_design = np.amax(self.spectrograph.slitmask.corners[:,:,0], axis=1) + # # side = self.traceid > 0 + # # x_det = self.edge_fit[self.pca.reference_row,side] + # + # # Estimate the scale in pixels/mm as the telescope platescale + # # in arcsec/mm divided by the detector platescale in + # # arcsec/pixel + # pix_per_mm = self.spectrograph.telescope.platescale() \ + # / self.traceimg.detector['platescale'] + # # / self.spectrograph.detector[self.det - 1]['platescale'] + # + # # If the traces are synchronized, use the estimated scale to + # # first mask edges that yeild slits that are too small relative + # # to the range of slit lengths in the mask file. + # if self.is_synced: + # slit_len_det = np.diff(x_det.reshape(-1, 2), axis=1).ravel() + # slit_len_mask = np.diff(x_design.reshape(-1, 2), axis=1).ravel() * pix_per_mm + # indx = (slit_len_det < np.amin(slit_len_mask) / 1.1) \ + # | (slit_len_det > np.amax(slit_len_mask) * 1.1) + # if np.any(indx): + # msgs.info('Removing {0} edges that form (an) '.format(np.sum(indx) * 2) + # + 'errantly small or large slit(s) compared to the mask design data.') + # x_det_bpm[np.repeat(indx, 2)] = True + # + # # Initial guess for the offset + # try: + # raise NotImplementedError() + # # Try using the spectrograph detector map + # self.spectrograph.get_detector_map() + # # Set the offset based on the location of this detector + # offset = self.spectrograph.detector_map.image_coordinates( + # self.spectrograph.detector_map.npix[0] / 2, + # self.spectrograph.detector_map.npix[1] / 2, + # detector=self.traceimg.detector.det, + # in_mm=False)[0][0] - self.spectrograph.detector_map.npix[0] / 2 + # # Set the bounds to some nominal fraction of the detector + # # size and pix/mm scale; allow for a +/- 10% deviation in + # # the pixel scale + # # TODO: Is 10% generally enough (for any instrument)? Make + # # this a (spectrograph-specific) parameter? + # offset_rng = [offset - 0.1 * self.spectrograph.detector_map.npix[0], + # offset + 0.1 * self.spectrograph.detector_map.npix[0]] + # except: + # # No detector map + # msgs.warn('No detector map available for {0}'.format(self.spectrograph.spectrograph) + # + '; attempting to match to slit-mask design anyway.') + # # Set the guess offset such that two sets of coordinates + # # are offset to their mean + # offset = np.mean(x_det) - np.mean(pix_per_mm * x_design) + # # Set the offset range + # offset_rng = [offset - np.absolute(np.amin(x_det) - np.amin(pix_per_mm * x_design)) * 1.1, + # offset + np.absolute(np.amax(pix_per_mm * x_design) - np.amax(x_det)) * 1.1] + # + # # import pdb + # # pdb.set_trace() + # # + # # slitmask.xc_trace(x_det, x_design, pix_per_mm) + # # + # # pdb.set_trace() + # + # # The solution can be highly dependent on the initial guess for + # # the offset, so do an initial grid search to get close to the + # # solution. + # msgs.info('Running a grid search to try to find the best starting offset.') + # # Step by 2 pixels + # off = np.arange(offset_rng[0], offset_rng[1], 2).astype(float) + # rms = np.zeros_like(off, dtype=float) + # scl = np.zeros_like(off, dtype=float) + # par = np.array([0, pix_per_mm]) + # bounds = np.array([offset_rng, [pix_per_mm / 1.1, pix_per_mm * 1.1]]) + # register = slitmask.SlitRegister(x_det, x_design, trace_mask=x_det_bpm) + # + # # NOTE: The commented approach below gets the RMS at each + # # offset point just using the estimated scale. This is faster + # # than the approach taken, but results are sensitive to the + # # accuracy of the estimated scale, which can lead to problems + # # in corner cases. + # # for i in range(off.size): + # # print('Grid point: {0}/{1}'.format(i+1, off.size), end='\r') + # # par[0] = off[i] + # # register.par = par + # # minsep = register.match(unique=True)[1] + # # rms[i] = sigma_clipped_stats(minsep, sigma=5)[2] + # # print('Grid point: {0}/{0}'.format(off.size)) + # + # # For each grid point, keep the offset fixed and find the best + # # scale. No rejection iterations are performed. + # for i in range(off.size): + # print('Grid point: {0}/{1}'.format(i + 1, off.size), end='\r') + # par[0] = off[i] + # register.find_best_match(guess=par, fix=[True, False], bounds=bounds, penalty=False) + # minsep = register.match(unique=True)[1] + # scl[i] = register.par[1] + # rms[i] = sigma_clipped_stats(minsep, sigma=5)[2] + # print('Grid point: {0}/{0}'.format(off.size)) + # + # # Use the grid point with the best RMS + # minindx = np.argmin(rms) + # offset = off[minindx] + # best_rms = rms[minindx] + # msgs.info('Minimum RMS ({0:.2f}) found with offset = {1:.2f}'.format(best_rms, offset)) + # if debug: + # # Plot the result + # ax1 = plt.subplot(211) + # ax1.scatter(off, rms, color='k', marker='.', s=100, lw=0, zorder=0) + # ax1.scatter(offset, best_rms, color='C3', marker='x', s=50, zorder=1) + # ax1.set_xlabel('Trace Offset (pix)') + # ax1.set_ylabel('RMS (det-mask; pix)') + # ax1.set_title('Grid search for initial offset') + # ax2 = plt.subplot(212, sharex=ax1) + # ax2.scatter(off, scl, color='k', marker='.', s=100, lw=0, zorder=0) + # ax2.set_ylabel('Best-fit scale') + # plt.show() + # + # # Do the final fit with some rejection iterations + # register.find_best_match(guess=[offset, pix_per_mm], bounds=bounds, penalty=False, + # maxiter=maxiter, maxsep=maxsep, sigma=sigma, debug=debug) + # + # if debug: + # register.show(minmax=[0, self.nspat], synced=True) + # + # # Find the missing, bad, and masked traces + # missing, bad = register.trace_mismatch(minmax=[0, self.nspat], synced=True) + # # masked_by_registration = np.where(register.trace_mask & np.invert(x_det_bpm))[0] + # # bad = np.append(bad, masked_by_registration) + # bad = np.append(bad, np.where(register.trace_mask | x_det_bpm)[0]) + # + # # Ignore missing alignment boxes + # if ignore_alignment: + # missing = missing[np.invert(self.spectrograph.slitmask.alignment_slit[missing // 2])] + # found_alignment_slits = register.match_index[ + # self.spectrograph.slitmask.alignment_slit[register.match_index // 2]] + # bad = np.append(bad, found_alignment_slits) + # + # # Report + # msgs.info('Best-fitting offset and scale for mask coordinates: {0:.2f} {1:.2f}'.format( + # *register.par)) + # msgs.info('Traces will {0} alignment slits'.format('exclude' if ignore_alignment + # else 'include')) + # msgs.info('Number of missing mask traces to insert: {0}'.format(len(missing))) + # msgs.info('Number of bad or alignment traces to remove: {0}'.format(len(bad))) + # + # if self.is_synced and (len(missing) - len(bad)) % 2 != 0: + # if allow_resync: + # msgs.warning('Difference in added and removed traces is odd; will resync traces.') + # else: + # msgs.error('Difference in added and removed traces desyncronizes traces.') + # + # if len(bad) > 0: + # # Remove the bad traces and rebuild the pca + # rmtrace = np.zeros(self.ntrace, dtype=bool) + # rmtrace[bad] = True + # self.remove_traces(rmtrace, rebuild_pca=True) + # + # if len(missing) > 0: + # # Even indices are lefts, odd indices are rights + # side = missing % 2 * 2 - 1 + # # Predict the traces using the PCA + # missing_traces = self.predict_traces(register.match_coo[missing], side) + # # Insert them + # self.insert_traces(side, missing_traces, mode='mask') + # + # # import pdb + # # pdb.set_trace() + # + # if len(bad) > 0 or len(missing) > 0: + # # Traces were removed and/or inserted, resync or recheck that the edges are synced. + # if (len(missing) - len(bad)) % 2 != 0 and allow_resync: + # self.sync(rebuild_pca=True) + # else: + # self.check_synced(rebuild_pca=True) + # reference_row = self.left_pca.reference_row if self.par['left_right_pca'] \ + # else self.pca.reference_row + # # Reset the match after removing/inserting traces + # x_det = self.edge_fit[reference_row, :] + # # TODO: Should the BOXSLITS also be included here? + # x_det_bpm = self.fully_masked_traces(flag=self.bitmask.bad_flags, + # exclude=self.bitmask.insert_flags) + # register = slitmask.SlitRegister(x_det, x_design, trace_mask=x_det_bpm, + # guess=[offset, pix_per_mm], bounds=bounds, + # penalty=False, maxiter=maxiter, maxsep=maxsep, + # sigma=sigma, debug=debug, fit=True) + # + # # TODO: This fit should *never* result in missing or bad + # # traces! Keep this for a while until we feel like we've + # # vetted the code well enough. + # missing, bad = register.trace_mismatch(minmax=[0, self.nspat], synced=True) + # if len(missing) != 0 or len(bad) != 0: + # msgs.error('CODING ERROR: Should never find missing or bad traces in re-fit!') + # + # # Fill the slit-design and object tables + # self._fill_design_table(register, _design_file) + # self._fill_objects_table(register) + + def order_refine(self, debug=False): + """ + For echelle spectrographs, attempt to add any orders that are not + present in the current set of edges. + """ + if self.spectrograph.pypeline != 'Echelle': + msgs.warn('Parameter add_missed_orders only valid for Echelle spectrographs.') + return + + # TODO: What happens if *more* edges are detected than there are + # archived order positions? + + # First match the expected orders + spat_offset = self.match_order() + + available_orders = self.orderid[1::2] + missed_orders = np.setdiff1d(self.spectrograph.orders, available_orders) + if missed_orders.size == 0: + # No missing orders, we're done + return + + # TODO: Vet good traces + + # Update the PCA + # TODO: Check that the edges can be PCA'd somewhere before this? + self.build_pca() + + # Find the indices of the missing orders + missed_orders_indx = utils.index_of_x_eq_y(self.spectrograph.orders, missed_orders) + + # Get the spatial positions of the new left and right order edges + add_right_edges = (self.spectrograph.order_spat_pos[missed_orders_indx] + + self.spectrograph.order_spat_width[missed_orders_indx]/2. + + spat_offset) * self.nspat + + add_left_edges = (self.spectrograph.order_spat_pos[missed_orders_indx] + - self.spectrograph.order_spat_width[missed_orders_indx]/2. + + spat_offset) * self.nspat + + side = np.append(np.full(add_left_edges.size, -1, dtype=int), + np.full(add_right_edges.size, 1, dtype=int)) + + missed_traces = self.predict_traces(np.append(add_left_edges, add_right_edges), + side=side) + + # Insert the traces + self.insert_traces(side, missed_traces, mode='order', nudge=False) + + # Rematch the orders + self.match_order() + def slit_spatial_center(self, normalized=True, spec=None, use_center=False, include_box=False): """ @@ -4785,6 +5166,10 @@ def match_order(self): The result of this method is to instantiate :attr:`orderid`. + Returns: + :obj:`float`: The median offset in pixels between the archived order + positions and those measured via the edge tracing. + Raises: PypeItError: Raised if the number of orders or their spatial @@ -4806,8 +5191,9 @@ def match_order(self): if offset is None: offset = 0.0 - # This requires the slits to be synced! Masked elements in - # slit_cen are for bad slits. + # Get the order centers in fractions of the detector width. This + # requires the slits to be synced! Masked elements in slit_cen are for + # bad slits. slit_cen = self.slit_spatial_center() # Calculate the separation between the order and every @@ -4819,11 +5205,12 @@ def match_order(self): # keep the signed value for reporting, but used the absolute # value of the difference for vetting below. sep = sep[(np.arange(self.spectrograph.norders),slit_indx)] - min_sep = np.absolute(sep) + med_offset = np.median(sep) + min_sep = np.absolute(sep - med_offset) # Report - msgs.info('Before vetting, the echelle order, matching left-right trace pair, and ' - 'matching separation are:') + msgs.info(f'Median offset is {med_offset:.3f}.') + msgs.info('After offsetting, order-matching separations are:') msgs.info(' {0:>6} {1:>4} {2:>6}'.format('ORDER', 'PAIR', 'SEP')) msgs.info(' {0} {1} {2}'.format('-'*6, '-'*4, '-'*6)) for i in range(self.spectrograph.norders): @@ -4833,14 +5220,12 @@ def match_order(self): # Single slit matched to multiple orders uniq, cnts = np.unique(slit_indx.compressed(), return_counts=True) for u in uniq[cnts > 1]: - indx = slit_indx == u - # Keep the one with the smallest separation - indx[np.argmin(min_sep[indx]) + np.where(indx)[0][1:]] = False - # Disassociate the other order from any slit - slit_indx[np.logical_not(indx) & (slit_indx == u)] = np.ma.masked - - # Flag and remove orders separated by more than the provided - # threshold + # Find the unmasked and multiply-matched indices + indx = (slit_indx.data == u) & np.logical_not(np.ma.getmaskarray(slit_indx)) + # Keep the one with the smallest separation and mask the rest + slit_indx[np.setdiff1d(np.where(indx), [np.argmin(min_sep[indx])])] = np.ma.masked + + # Flag orders separated by more than the provided threshold if self.par['order_match'] is not None: indx = (min_sep > self.par['order_match']) \ & np.logical_not(np.ma.getmaskarray(min_sep)) @@ -4872,6 +5257,8 @@ def match_order(self): indx = (2*slit_indx.compressed()[:,None] + np.tile(np.array([0,1]), (nfound,1))).ravel() self.orderid[indx] = (np.array([-1,1])[None,:]*found_orders[:,None]).ravel() + return med_offset + def get_slits(self): """ Use the data to instatiate the relevant diff --git a/pypeit/extraction.py b/pypeit/extraction.py index b0e21dbe80..9b190cc4b5 100644 --- a/pypeit/extraction.py +++ b/pypeit/extraction.py @@ -13,11 +13,9 @@ from astropy import stats from abc import ABCMeta -from linetools import utils as ltu - from pypeit import msgs, utils from pypeit.display import display -from pypeit.core import skysub, extract, wave, flexure +from pypeit.core import skysub, extract, flexure from IPython import embed @@ -41,10 +39,10 @@ class Extract: Final output mask extractmask (`numpy.ndarray`_): Extraction mask - slits (:class:`pypeit.slittrace.SlitTraceSet`): - sobjs_obj (:class:`pypeit.specobjs.SpecObjs`): + slits (:class:`~pypeit.slittrace.SlitTraceSet`): + sobjs_obj (:class:`~pypeit.specobjs.SpecObjs`): Only object finding but no extraction - sobjs (:class:`pypeit.specobjs.SpecObjs`): + sobjs (:class:`~pypeit.specobjs.SpecObjs`): Final extracted object list with trace corrections applied spat_flexure_shift (float): tilts (`numpy.ndarray`_): @@ -68,14 +66,14 @@ def get_instance(cls, sciImg, slits, sobjs_obj, spectrograph, par, objtype, glob waveTilts=None, tilts=None, wv_calib=None, waveimg=None, bkg_redux=False, return_negative=False, std_redux=False, show=False, basename=None): """ - Instantiate the Reduce subclass appropriate for the provided + Instantiate the Extract subclass appropriate for the provided spectrograph. - The class must be subclassed from Reduce. See :class:`Reduce` for + The class must be subclassed from Extract. See :class:`Extract` for the description of the valid keyword arguments. Args: - sciImg (:class:`~pypeit.images.scienceimage.ScienceImage`): + sciImg (:class:`~pypeit.images.pypeitimage.PypeItImage`): Image to reduce. slits (:class:`~pypeit.slittrace.SlitTraceSet`): Slit trace set object @@ -96,7 +94,7 @@ def get_instance(cls, sciImg, slits, sobjs_obj, spectrograph, par, objtype, glob be provided. tilts (`numpy.ndarray`_, optional): Tilts image. Either a tilts image or waveTilts object (above) must be provided. - wv_calib (:class:`~pypeit.wavetilts.WaveCalib`, optional): + wv_calib (:class:`~pypeit.wavecalib.WaveCalib`, optional): This is the waveCalib object which is optional, but either wv_calib or waveimg must be provided. waveimg (`numpy.ndarray`_, optional): Wave image. Either a wave image or wv_calib object (above) must be provided @@ -161,12 +159,12 @@ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_ elif objtype == 'science_coadd2d': self.spat_flexure_shift = None - # Initialise the slits - msgs.info("Initialising slits") - self.initialise_slits(slits) + # Initialize the slits + msgs.info("Initializing slits") + self.initialize_slits(slits) # Internal bpm mask - self.extract_bpm = (self.slits.mask > 0) & (np.invert(self.slits.bitmask.flagged( + self.extract_bpm = (self.slits.mask > 0) & (np.logical_not(self.slits.bitmask.flagged( self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing))) self.extract_bpm_init = self.extract_bpm.copy() @@ -229,7 +227,14 @@ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_ self.wv_calib = wv_calib self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spat_flexure=self.spat_flexure_shift) elif wv_calib is None and waveimg is not None: - self.waveimg=waveimg + self.waveimg = waveimg + + msgs.info("Generating spectral FWHM image") + self.fwhmimg = None + if wv_calib is not None: + self.fwhmimg = wv_calib.build_fwhmimg(self.tilts, self.slits, initial=True, spat_flexure=self.spat_flexure_shift) + else: + msgs.warn("Spectral FWHM image could not be generated") # Now apply a global flexure correction to each slit provided it's not a standard star if self.par['flexure']['spec_method'] != 'skip' and not self.std_redux: @@ -266,9 +271,10 @@ def nsobj_to_extract(self): else: return 0 - def initialise_slits(self, slits, initial=False): + # TODO Make this a method possibly in slittrace.py. Almost identical code is in find_objects.py + def initialize_slits(self, slits, initial=False): """ - Gather all the :class:`SlitTraceSet` attributes + Gather all the :class:`~pypeit.slittrace.SlitTraceSet` attributes that we'll use here in :class:`Extract` Args @@ -281,8 +287,18 @@ def initialise_slits(self, slits, initial=False): # Slits self.slits = slits # Select the edges to use + + # TODO JFH: his is an ugly hack for the present moment until we get the slits object sorted out self.slits_left, self.slits_right, _ \ = self.slits.select_edges(initial=initial, flexure=self.spat_flexure_shift) + # This matches the logic below that is being applied to the slitmask. Better would be to clean up slits to + # to return a new slits object with the desired selection criteria which would remove the ambiguity + # about whether the slits and the slitmask are in sync. + #bpm = self.slits.mask.astype(bool) + #bpm &= np.logical_not(self.slits.bitmask.flagged(self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing)) + #gpm = np.logical_not(bpm) + #self.slits_left = slits_left[:, gpm] + #self.slits_right = slits_right[:, gpm] # Slitmask self.slitmask = self.slits.slit_img(initial=initial, flexure=self.spat_flexure_shift, @@ -291,8 +307,8 @@ def initialise_slits(self, slits, initial=False): # NOTE: this uses the par defined by EdgeTraceSet; this will # use the tweaked traces if they exist self.sciImg.update_mask_slitmask(self.slitmask) -# # For echelle -# self.spatial_coo = self.slits.spatial_coordinates(initial=initial, flexure=self.spat_flexure_shift) + + def extract(self, global_sky, model_noise=None, spat_pix=None): """ @@ -301,7 +317,7 @@ def extract(self, global_sky, model_noise=None, spat_pix=None): Args: global_sky (`numpy.ndarray`_): Sky estimate - sobjs_obj (:class:`pypeit.specobjs.SpecObjs`): + sobjs_obj (:class:`~pypeit.specobjs.SpecObjs`): List of SpecObj that have been found and traced model_noise (bool): If True, construct and iteratively update a model inverse variance image @@ -333,7 +349,7 @@ def extract(self, global_sky, model_noise=None, spat_pix=None): inmask = self.sciImg.select_flag(invert=True) & thismask # Do it extract.extract_boxcar(self.sciImg.image, self.sciImg.ivar, inmask, self.waveimg, - global_sky, sobj, base_var=self.sciImg.base_var, + global_sky, sobj, fwhmimg=self.fwhmimg, base_var=self.sciImg.base_var, count_scale=self.sciImg.img_scale, noise_floor=self.sciImg.noise_floor) @@ -369,9 +385,11 @@ def extract(self, global_sky, model_noise=None, spat_pix=None): if len(remove_idx) > 0: self.sobjs.remove_sobj(remove_idx) - # Add the S/N ratio for each extracted object + # Add the DETECTOR container, S/N ratio, and FWHM in ARCSEC for each extracted object for sobj in self.sobjs: + sobj.DETECTOR = self.sciImg.detector sobj.S2N = sobj.med_s2n() + sobj.SPAT_FWHM = sobj.med_fwhm() # Return return self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs @@ -397,7 +415,7 @@ def run(self, model_noise=None, spat_pix=None): Returns: tuple: skymodel (ndarray), objmodel (ndarray), ivarmodel (ndarray), outmask (ndarray), sobjs (SpecObjs), waveimg (`numpy.ndarray`_), - tilts (`numpy.ndarray`_). + tilts (`numpy.ndarray`_), slits (:class:`~pypeit.slittrace.SlitTraceSet`). See main doc string for description """ @@ -431,16 +449,14 @@ def run(self, model_noise=None, spat_pix=None): self.sobjs = self.sobjs_obj # Update the mask - # TODO avoid modifying arguments to a class or function in place. If slits is mutable, it should be a return - # value for the run function # TODO: change slits.mask > 2 to use named flags. - reduce_masked = np.where(np.invert(self.extract_bpm_init) & self.extract_bpm & (self.slits.mask > 2))[0] + reduce_masked = np.where(np.logical_not(self.extract_bpm_init) & self.extract_bpm & (self.slits.mask > 2))[0] if len(reduce_masked) > 0: self.slits.mask[reduce_masked] = self.slits.bitmask.turn_on( self.slits.mask[reduce_masked], 'BADEXTRACT') # Return - return self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs, self.waveimg, self.tilts + return self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs, self.waveimg, self.tilts, self.slits def local_skysub_extract(self, global_sky, sobjs, model_noise=True, spat_pix=None, show_profile=False, show_resids=False, show=False): @@ -460,7 +476,7 @@ def spec_flexure_correct(self, mode="local", sobjs=None): mode (str): "local" - Use sobjs to determine flexure correction "global" - Use waveimg and global_sky to determine flexure correction at the centre of the slit - sobjs (:class:`pypeit.specobjs.SpecObjs`, None): + sobjs (:class:`~pypeit.specobjs.SpecObjs`, None): Spectrally extracted objects """ if self.par['flexure']['spec_method'] == 'skip': @@ -528,51 +544,6 @@ def spec_flexure_correct(self, mode="local", sobjs=None): flexure.spec_flexure_qa(self.slits.slitord_id, self.extract_bpm, basename, flex_list, specobjs=sobjs, out_dir=out_dir) - def refframe_correct(self, ra, dec, obstime, sobjs=None): - """ Correct the calibrated wavelength to the user-supplied reference frame - - Args: - radec (astropy.coordiantes.SkyCoord): - Sky Coordinate of the observation - obstime (:obj:`astropy.time.Time`): - Observation time - sobjs (:class:`pypeit.specobjs.Specobjs`, None): - Spectrally extracted objects - - """ - # Correct Telescope's motion - refframe = self.par['calibrations']['wavelengths']['refframe'] - if refframe in ['heliocentric', 'barycentric'] \ - and self.par['calibrations']['wavelengths']['reference'] != 'pixel': - msgs.info("Performing a {0} correction".format(self.par['calibrations']['wavelengths']['refframe'])) - # Calculate correction - radec = ltu.radec_to_coord((ra, dec)) - vel, vel_corr = wave.geomotion_correct(radec, obstime, - self.spectrograph.telescope['longitude'], - self.spectrograph.telescope['latitude'], - self.spectrograph.telescope['elevation'], - refframe) - # Apply correction to objects - msgs.info('Applying {0} correction = {1:0.5f} km/s'.format(refframe, vel)) - if (sobjs is not None) and (sobjs.nobj != 0): - # Loop on slits to apply - gd_slitord = self.slits.slitord_id[np.logical_not(self.extract_bpm)] - for slitord in gd_slitord: - indx = sobjs.slitorder_indices(slitord) - this_specobjs = sobjs[indx] - # Loop on objects - for specobj in this_specobjs: - if specobj is None: - continue - specobj.apply_helio(vel_corr, refframe) - - # Apply correction to wavelength image - self.vel_corr = vel_corr - self.waveimg *= vel_corr - - else: - msgs.info('A wavelength reference frame correction will not be performed.') - def show(self, attr, image=None, showmask=False, sobjs=None, chname=None, slits=False,clear=False): """ @@ -674,7 +645,7 @@ def __repr__(self): class MultiSlitExtract(Extract): """ - Child of Reduce for Multislit and Longslit reductions + Child of Extract for Multislit and Longslit reductions See parent doc string for Args and Attributes @@ -726,7 +697,7 @@ def local_skysub_extract(self, global_sky, sobjs, spat_pix=None, model_noise=Tru self.global_sky = global_sky # get the good slits - gdslits = np.where(np.invert(self.extract_bpm))[0] + gdslits = np.where(np.logical_not(self.extract_bpm))[0] # Allocate the images that are needed # Initialize to mask in case no objects were found @@ -781,7 +752,7 @@ def local_skysub_extract(self, global_sky, sobjs, spat_pix=None, model_noise=Tru thismask, self.slits_left[:,slit_idx], self.slits_right[:, slit_idx], self.sobjs[thisobj], ingpm=ingpm, - spat_pix=spat_pix, + fwhmimg=self.fwhmimg, spat_pix=spat_pix, model_full_slit=model_full_slit, sigrej=sigrej, model_noise=model_noise, std=self.std_redux, bsp=bsp, @@ -811,7 +782,7 @@ def local_skysub_extract(self, global_sky, sobjs, spat_pix=None, model_noise=Tru class EchelleExtract(Extract): """ - Child of Reduce for Echelle reductions + Child of Extract for Echelle reductions See parent doc string for Args and Attributes @@ -820,12 +791,12 @@ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, **kwarg super(EchelleExtract, self).__init__(sciImg, slits, sobjs_obj, spectrograph, par, objtype, **kwargs) # JFH For 2d coadds the orders are no longer located at the standard locations - self.order_vec = spectrograph.orders if 'coadd2d' in self.objtype \ - else self.slits.ech_order -# else self.spectrograph.order_vec(self.spatial_coo) - if self.order_vec is None: - msgs.error('Unable to set Echelle orders, likely because they were incorrectly ' - 'assigned in the relevant SlitTraceSet.') + # TODO Need to test this for 2d coadds + #self.order_vec = spectrograph.orders if 'coadd2d' in self.objtype \ + # else self.slits.ech_order + #if self.order_vec is None: + # msgs.error('Unable to set Echelle orders, likely because they were incorrectly ' + # 'assigned in the relevant SlitTraceSet.') # JFH TODO Should we reduce the number of iterations for standards or near-IR redux where the noise model is not # being updated? @@ -869,49 +840,48 @@ def local_skysub_extract(self, global_sky, sobjs, """ self.global_sky = global_sky + # get the good slits + gdorders = np.logical_not(self.extract_bpm) + # Pulled out some parameters to make the method all easier to read bsp = self.par['reduce']['skysub']['bspline_spacing'] sigrej = self.par['reduce']['skysub']['sky_sigrej'] sn_gauss = self.par['reduce']['extraction']['sn_gauss'] model_full_slit = self.par['reduce']['extraction']['model_full_slit'] force_gauss = self.par['reduce']['extraction']['use_user_fwhm'] - - self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs \ - = skysub.ech_local_skysub_extract(self.sciImg.image, self.sciImg.ivar, - self.sciImg.fullmask, self.tilts, self.waveimg, - self.global_sky, self.slits_left, - self.slits_right, self.slitmask, sobjs, - self.order_vec, spat_pix=spat_pix, - std=self.std_redux, fit_fwhm=fit_fwhm, - min_snr=min_snr, bsp=bsp, sigrej=sigrej, - force_gauss=force_gauss, sn_gauss=sn_gauss, - model_full_slit=model_full_slit, - model_noise=model_noise, - show_profile=show_profile, - show_resids=show_resids, show_fwhm=show_fwhm, - base_var=self.sciImg.base_var, - count_scale=self.sciImg.img_scale, - adderr=self.sciImg.noise_floor) + = skysub.ech_local_skysub_extract(self.sciImg.image, self.sciImg.ivar, + self.sciImg.fullmask, self.tilts, self.waveimg, + self.global_sky, self.slits_left[:, gdorders], + self.slits_right[:, gdorders], self.slitmask, sobjs, + spat_pix=spat_pix, + std=self.std_redux, fit_fwhm=fit_fwhm, + min_snr=min_snr, bsp=bsp, sigrej=sigrej, + force_gauss=force_gauss, sn_gauss=sn_gauss, + model_full_slit=model_full_slit, + model_noise=model_noise, + show_profile=show_profile, + show_resids=show_resids, show_fwhm=show_fwhm, + base_var=self.sciImg.base_var, + count_scale=self.sciImg.img_scale, + adderr=self.sciImg.noise_floor) # Step self.steps.append(inspect.stack()[0][3]) if show: - self.show('local', sobjs = self.sobjs, slits= True, chname='ech_local') - self.show('resid', sobjs = self.sobjs, slits= True, chname='ech_resid') + self.show('local', sobjs=self.sobjs, slits=True, chname='ech_local') + self.show('resid', sobjs=self.sobjs, slits=True, chname='ech_resid') return self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs -# TODO Should this be removed? I think so. + class IFUExtract(MultiSlitExtract): """ - Child of Reduce for IFU reductions + Child of Extract for IFU reductions See parent doc string for Args and Attributes """ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, **kwargs): - super(IFUExtract, self).__init__(sciImg, slits, sobjs_obj, spectrograph, par, objtype, **kwargs) - self.initialise_slits(slits, initial=True) - - + # IFU doesn't extract, and there's no need for a super call here. + return diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index c9370d4a40..93ab69dfdf 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -13,14 +13,12 @@ from astropy import stats from abc import ABCMeta - from pypeit import specobjs from pypeit import msgs, utils from pypeit import flatfield from pypeit.display import display from pypeit.core import skysub, qa, parse, flat, flexure from pypeit.core import procimg -from pypeit.images import buildimage from pypeit.core import findobj_skymask from IPython import embed @@ -32,18 +30,18 @@ class FindObjects: science or standard-star exposures. Args: - sciImg (:class:`~pypeit.images.scienceimage.ScienceImage`): + sciImg (:class:`~pypeit.images.pypeitimage.PypeItImage`): Image to reduce. - slits (:class:`~pypeit.slittrace.SlitTracSet`): + slits (:class:`~pypeit.slittrace.SlitTraceSet`): Object providing slit traces for the image to reduce. spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): - PypeIt Sspectrograph class - par (:class:`~pypeit.par.pyepeitpar.PypeItPar`): + PypeIt Spectrograph class + par (:class:`~pypeit.par.pypeitpar.PypeItPar`): Reduction parameters class objtype (:obj:`str`): Specifies object being reduced. Should be 'science', 'standard', or 'science_coadd2d'. - wv_calib (:class:`~pypeit.wavetilts.WaveCalib`, optional): + wv_calib (:class:`~pypeit.wavecalib.WaveCalib`, optional): This is only used for the IFU child when a joint sky subtraction is requested. waveTilts (:class:`~pypeit.wavetilts.WaveTilts`, optional): @@ -71,7 +69,7 @@ class FindObjects: Clear the ginga window before showing the object finding results. basename (:obj:`str`, optional): Base name for output files - manual (:class:`~pypeit.manual_extract.ManualExtractObj`, optional): + manual (:class:`~pypeit.manual_extract.ManualExtractionObj`, optional): Object containing manual extraction instructions/parameters. Attributes: @@ -91,7 +89,7 @@ class FindObjects: Final output mask extractmask (`numpy.ndarray`_): Extraction mask - slits (:class:`pypeit.slittrace.SlitTraceSet`): + slits (:class:`~pypeit.slittrace.SlitTraceSet`): sobjs_obj (:class:`pypeit.specobjs.SpecObjs`): Objects found spat_flexure_shift (:obj:`float`): @@ -144,6 +142,7 @@ def __init__(self, sciImg, slits, spectrograph, par, objtype, wv_calib=None, wav self.manual = manual self.initial_skymask = initial_skymask self.wv_calib = wv_calib # TODO :: Ideally, we want to avoid this if possible. Find a better way to do joint_skysub fitting outside of the find_objects class. + self.waveimg = None # Parse # Slit pieces # WARNING -- It is best to unpack here then pass around self.slits @@ -161,8 +160,8 @@ def __init__(self, sciImg, slits, spectrograph, par, objtype, wv_calib=None, wav self.spat_flexure_shift = None # Initialise the slits - msgs.info("Initialising slits") - self.initialise_slits(slits) + msgs.info("Initializing slits") + self.initialize_slits(slits) # Internal bpm mask # We want to keep the 'BOXSLIT', which has bpm=2. But we don't want to keep 'BOXSLIT' @@ -268,9 +267,10 @@ def create_skymask(self, sobjs_obj): # Return return skymask - def initialise_slits(self, slits, initial=False): + # TODO Make this a method possibly in slittrace.py. Almost identical code is in extraction.py + def initialize_slits(self, slits, initial=False): """ - Gather all the :class:`SlitTraceSet` attributes + Gather all the :class:`~pypeit.slittrace.SlitTraceSet` attributes that we'll use here in :class:`FindObjects` Args: @@ -283,8 +283,18 @@ def initialise_slits(self, slits, initial=False): # Slits self.slits = slits # Select the edges to use + # TODO JFH: his is an ugly hack for the present moment until we get the slits object sorted out self.slits_left, self.slits_right, _ \ = self.slits.select_edges(initial=initial, flexure=self.spat_flexure_shift) + # This matches the logic below that is being applied to the slitmask. Better would be to clean up slits to + # to return a new slits object with the desired selection criteria which would remove the ambiguity + # about whether the slits and the slitmask are in sync. + #bpm = self.slits.mask.astype(bool) + #bpm &= np.invert(self.slits.bitmask.flagged(self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing + ['BOXSLIT'])) + #gpm = np.logical_not(bpm) + #self.slits_left = slits_left[:, gpm] + #self.slits_right = slits_right[:, gpm] + # Slitmask self.slitmask = self.slits.slit_img(initial=initial, flexure=self.spat_flexure_shift, @@ -296,6 +306,8 @@ def initialise_slits(self, slits, initial=False): # # For echelle # self.spatial_coo = self.slits.spatial_coordinates(initial=initial, flexure=self.spat_flexure_shift) + # TODO There are going to be problems with std_trace not being aligned with whatever orders are getting masked in + # this routine. def run(self, std_trace=None, show_peaks=False, show_skysub_fit=False): """ Primary code flow for object finding in PypeIt reductions @@ -327,18 +339,15 @@ def run(self, std_trace=None, show_peaks=False, show_skysub_fit=False): # Perform a first pass sky-subtraction. The mask is either empty or # uses the mask specified by the user. - + # TODO: Should we make this no_poly=True to have fewer degrees of freedom in # the with with-object global sky fits?? initial_sky0 = self.global_skysub(skymask=self.initial_skymask, update_crmask=False, objs_not_masked=True, show_fit=show_skysub_fit) # First pass object finding - save_objfindQA = self.par['reduce']['findobj']['skip_second_find'] or self.std_redux \ - or self.initial_skymask is not None sobjs_obj, self.nobj = \ self.find_objects(self.sciImg.image-initial_sky0, self.sciImg.ivar, std_trace=std_trace, - show_peaks=show_peaks, show=self.findobj_show and not self.std_redux, - save_objfindQA=save_objfindQA) + show_peaks=show_peaks, show=self.findobj_show and not self.std_redux) if self.nobj == 0 or self.initial_skymask is not None: # Either no objects were found, or the initial sky mask was provided by the user. @@ -482,11 +491,15 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg = (0, 0), skymask (`numpy.ndarray`_, None): A 2D image indicating sky regions (1=sky) update_crmask (bool, optional): + ?? trim_edg (tuple, optional): A two tuple of ints that specify the number of pixels to trim from the slit edges show_fit (bool, optional): + ?? show (bool, optional): + ?? show_objs (bool, optional): + ?? previous_sky (`numpy.ndarray`_, optional): Sky model estimate from a previous run of global_sky Used to generate an improved estimated of the variance @@ -590,25 +603,22 @@ def show(self, attr, image=None, global_sky=None, showmask=False, sobjs=None, Show one of the internal images .. todo:: - Should probably put some of these in ProcessImages + + - This docstring is incomplete! Parameters ---------- attr : str - global -- Sky model (global) - sci -- Processed science image - rawvar -- Raw variance image - modelvar -- Model variance image - crmasked -- Science image with CRs set to 0 - skysub -- Science image with global sky subtracted - image -- Input image - display : str, optional + String specifying the image to show. Options are: + - global -- Sky model (global) + - sci -- Processed science image + - rawvar -- Raw variance image + - modelvar -- Model variance image + - crmasked -- Science image with CRs set to 0 + - skysub -- Science image with global sky subtracted + - image -- Input image image : ndarray, optional - User supplied image to display - - Returns - ------- - + User supplied image to display """ mask_in = self.sciImg.fullmask if showmask else None @@ -716,7 +726,7 @@ def find_objects_pypeline(self, image, ivar, std_trace=None, Returns ------- - specobjs : :class:`~pypeot.specobjs.Specobjs` + specobjs : :class:`~pypeit.specobjs.SpecObjs` Container holding Specobj objects nobj : :obj:`int` Number of objects identified @@ -864,7 +874,7 @@ def find_objects_pypeline(self, image, ivar, std_trace=None, Returns ------- - specobjs : :class:`~pypeit.specobjs.Specobjs` + specobjs : :class:`~pypeit.specobjs.SpecObjs` Container holding Specobj objects nobj : :obj:`int` Number of objects identified @@ -896,16 +906,16 @@ def find_objects_pypeline(self, image, ivar, std_trace=None, nperorder = self.par['reduce']['findobj']['maxnumber_std'] if self.std_redux \ else self.par['reduce']['findobj']['maxnumber_sci'] + reduce_gpm = np.logical_not(self.reduce_bpm) sobjs_ech = findobj_skymask.ech_objfind( - image, ivar, self.slitmask, self.slits_left, self.slits_right, - self.order_vec, self.reduce_bpm, - self.slits.spat_id, - np.vstack((self.slits.specmin, self.slits.specmax)), + image, ivar, self.slitmask, self.slits_left[:, reduce_gpm], self.slits_right[:, reduce_gpm], + self.slits.spat_id[reduce_gpm], self.order_vec[reduce_gpm], + np.vstack((self.slits.specmin, self.slits.specmax))[:, reduce_gpm], det=self.det, inmask=inmask, ncoeff=self.par['reduce']['findobj']['trace_npoly'], manual_extract_dict=manual_extract_dict, - plate_scale=plate_scale, + plate_scale=plate_scale[reduce_gpm], std_trace=std_trace, specobj_dict=specobj_dict, snr_thresh=self.par['reduce']['findobj']['snr_thresh'], @@ -940,7 +950,7 @@ class IFUFindObjects(MultiSlitFindObjects): """ def __init__(self, sciImg, slits, spectrograph, par, objtype, **kwargs): super().__init__(sciImg, slits, spectrograph, par, objtype, **kwargs) - self.initialise_slits(slits, initial=True) + self.initialize_slits(slits, initial=True) def find_objects_pypeline(self, image, ivar, std_trace=None, show_peaks=False, show_fits=False, show_trace=False, @@ -975,12 +985,12 @@ def apply_relative_scale(self, scaleImg): self.sciImg.update_mask('BADSCALE', indx=_bpm) self.sciImg.ivar = utils.inverse(varImg) - # TODO :: THIS FUNCTION IS NOT CURRENTLY USED, BUT RJC REQUESTS TO KEEP THIS CODE HERE FOR THE TIME BEING. + # RJC :: THIS FUNCTION IS NOT CURRENTLY USED, BUT RJC REQUESTS TO KEEP THIS CODE HERE FOR THE TIME BEING. # def illum_profile_spatial(self, skymask=None, trim_edg=(0, 0), debug=False): # """ # Calculate the residual spatial illumination profile using the sky regions. # - # The redisual is calculated using the differential: + # The residual is calculated using the differential: # # .. code-block:: python # @@ -1096,13 +1106,13 @@ def illum_profile_spectral(self, global_sky, skymask=None): Mask of sky regions where the spectral illumination will be determined """ trim = self.par['calibrations']['flatfield']['slit_trim'] - ref_idx = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] + sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] smooth_npix = self.par['calibrations']['flatfield']['slit_illum_smooth_npix'] gpm = self.sciImg.select_flag(invert=True) # Note :: Need to provide wavelength to illum_profile_spectral (not the tilts) so that the # relative spectral sensitivity is calculated at a given wavelength for all slits simultaneously. scaleImg = flatfield.illum_profile_spectral(self.sciImg.image.copy(), self.waveimg, self.slits, - slit_illum_ref_idx=ref_idx, model=global_sky, gpmask=gpm, + slit_illum_ref_idx=sl_ref, model=global_sky, gpmask=gpm, skymask=skymask, trim=trim, flexure=self.spat_flexure_shift, smooth_npix=smooth_npix) # Now apply the correction to the science frame @@ -1116,7 +1126,7 @@ def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), msgs.info("Performing joint global sky subtraction") # Mask objects using the skymask? If skymask has been set by objfinding, and masking is requested, then do so skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) - global_sky = np.zeros_like(self.sciImg.image) + _global_sky = np.zeros_like(self.sciImg.image) thismask = (self.slitmask > 0) inmask = (self.sciImg.select_flag(invert=True) & thismask & skymask_now).astype(bool) # Convert the wavelength image to A/pixel, registered at pixel 0 (this gives something like @@ -1131,40 +1141,91 @@ def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), update_crmask = False if not self.par['reduce']['skysub']['global_sky_std']: msgs.info('Skipping global sky-subtraction for standard star.') - return global_sky - + return _global_sky + + # Use the FWHM map determined from the arc lines to convert the science frame + # to have the same effective spectral resolution. + fwhm_map = self.wv_calib.build_fwhmimg(self.tilts, self.slits, initial=True, spat_flexure=self.spat_flexure_shift) + thismask = thismask & (fwhm_map != 0.0) + # Need to include S/N for deconvolution + sciimg = skysub.convolve_skymodel(self.sciImg.image, fwhm_map, thismask) # Iterate to use a model variance image - numiter = 4 + numiter = 4 # This is more than enough, and will probably break earlier than this model_ivar = self.sciImg.ivar + sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] + # Prepare the slitmasks for the relative spectral illumination + slitmask = self.slits.slit_img(pad=0, initial=True, flexure=self.spat_flexure_shift) + slitmask_trim = self.slits.slit_img(pad=-3, initial=True, flexure=self.spat_flexure_shift) for nn in range(numiter): msgs.info("Performing iterative joint sky subtraction - ITERATION {0:d}/{1:d}".format(nn+1, numiter)) # TODO trim_edg is in the parset so it should be passed in here via trim_edg=tuple(self.par['reduce']['trim_edge']), - global_sky[thismask] = skysub.global_skysub(self.sciImg.image, model_ivar, tilt_wave, - thismask, self.slits_left, self.slits_right, inmask=inmask, - sigrej=sigrej, trim_edg=trim_edg, - bsp=self.par['reduce']['skysub']['bspline_spacing'], - no_poly=self.par['reduce']['skysub']['no_poly'], - pos_mask=not self.bkg_redux and not objs_not_masked, - max_mask_frac=self.par['reduce']['skysub']['max_mask_frac'], - show_fit=show_fit) + _global_sky[thismask] = skysub.global_skysub(sciimg, model_ivar, tilt_wave, + thismask, self.slits_left, self.slits_right, inmask=inmask, + sigrej=sigrej, trim_edg=trim_edg, + bsp=self.par['reduce']['skysub']['bspline_spacing'], + no_poly=self.par['reduce']['skysub']['no_poly'], + pos_mask=not self.bkg_redux and not objs_not_masked, + max_mask_frac=self.par['reduce']['skysub']['max_mask_frac'], + show_fit=show_fit) + + # Calculate the relative spectral illumination + scaleImg = flat.illum_profile_spectral_poly(sciimg, self.waveimg, slitmask, slitmask_trim, _global_sky, + slit_illum_ref_idx=sl_ref, gpmask=inmask, thismask=thismask) + # Apply this scale image to the temporary science frame + sciimg /= scaleImg + # Update the ivar image used in the sky fit msgs.info("Updating sky noise model") # Choose the highest counts out of sky and object - counts = global_sky + counts = _global_sky _scale = None if self.sciImg.img_scale is None else self.sciImg.img_scale[thismask] # NOTE: darkcurr must be a float for the call below to work. - var = procimg.variance_model(self.sciImg.base_var[thismask], counts=counts[thismask], - count_scale=_scale, noise_floor=adderr) - model_ivar[thismask] = utils.inverse(var) - # Redo the relative spectral illumination correction with the improved sky model - self.illum_profile_spectral(global_sky, skymask=thismask) + if not self.bkg_redux: + var = procimg.variance_model(self.sciImg.base_var[thismask], counts=counts[thismask], + count_scale=_scale, noise_floor=adderr) + model_ivar[thismask] = utils.inverse(var) + else: + model_ivar[thismask] = self.sciImg.ivar[thismask] + # RJC :: Recalculating the global sky and flexure is probably overkill... but please keep this code in for now + # Recalculate the sky on each individual slit and redetermine the spectral flexure + # global_sky_sep = super().global_skysub(skymask=skymask, update_crmask=update_crmask, + # trim_edg=trim_edg, show_fit=show_fit, show=show, + # show_objs=show_objs) + # self.calculate_flexure(global_sky_sep) + + # Check if the relative scaling isn't changing much after at least 4 iterations + minv, maxv = np.min(scaleImg[thismask]), np.max(scaleImg[thismask]) + if nn >= 3 and max(abs(1/minv), abs(maxv)) < 1.005: # Relative accuracy of 0.5% is sufficient + break if update_crmask: # Find CRs with sky subtraction # NOTE: There's no need to run `sciImg.update_mask_cr` after this. # This operation updates the mask directly! - self.sciImg.build_crmask(self.par['scienceframe']['process'], - subtract_img=global_sky) + self.sciImg.build_crmask(self.par['scienceframe']['process'], subtract_img=_global_sky) + + # Now we have a correct scale, apply it to the original science image + self.apply_relative_scale(scaleImg) + + # Recalculate the joint sky using the original image + _global_sky[thismask] = skysub.global_skysub(self.sciImg.image, model_ivar, tilt_wave, + thismask, self.slits_left, self.slits_right, inmask=inmask, + sigrej=sigrej, trim_edg=trim_edg, + bsp=self.par['reduce']['skysub']['bspline_spacing'], + no_poly=self.par['reduce']['skysub']['no_poly'], + pos_mask=not self.bkg_redux and not objs_not_masked, + max_mask_frac=self.par['reduce']['skysub']['max_mask_frac'], + show_fit=show_fit) + + # Update the ivar image used in the sky fit + msgs.info("Updating sky noise model") + # Choose the highest counts out of sky and object + counts = _global_sky + _scale = None if self.sciImg.img_scale is None else self.sciImg.img_scale[thismask] + # NOTE: darkcurr must be a float for the call below to work. + var = procimg.variance_model(self.sciImg.base_var[thismask], counts=counts[thismask], + count_scale=_scale, noise_floor=adderr) + model_ivar[thismask] = utils.inverse(var) # Step self.steps.append(inspect.stack()[0][3]) @@ -1172,8 +1233,8 @@ def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), if show: sobjs_show = None if show_objs else self.sobjs_obj # Global skysub is the first step in a new extraction so clear the channels here - self.show('global', global_sky=global_sky, slits=True, sobjs=sobjs_show, clear=False) - return global_sky + self.show('global', global_sky=_global_sky, slits=True, sobjs=sobjs_show, clear=False) + return _global_sky def global_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), previous_sky=None, show_fit=False, show=False, show_objs=False, objs_not_masked=False): @@ -1187,6 +1248,10 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), global_sky_sep = super().global_skysub(skymask=skymask, update_crmask=update_crmask, trim_edg=trim_edg, show_fit=show_fit, show=show, show_objs=show_objs) + # Check if any slits failed + if np.any(global_sky_sep[self.slitmask >= 0] == 0) and not self.bkg_redux: + # Cannot continue without a sky model for all slits + msgs.error("Global sky subtraction has failed for at least one slit.") # Check if flexure or a joint fit is requested if not self.par['reduce']['skysub']['joint_fit'] and self.par['flexure']['spec_method'] == 'skip': @@ -1194,18 +1259,14 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), if self.wv_calib is None: msgs.error("A wavelength calibration is needed (wv_calib) if a joint sky fit is requested.") msgs.info("Generating wavelength image") + self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spat_flexure=self.spat_flexure_shift) # Calculate spectral flexure method = self.par['flexure']['spec_method'] + # TODO :: Perhaps include a new label for IFU flexure correction - e.g. 'slitcen_relative' or 'slitcenIFU' or 'IFU' + # :: If a new label is introduced, change the other instances of 'method' (see below), and in flexure.spec_flexure_qa() if method in ['slitcen']: - trace_spat = 0.5 * (self.slits_left + self.slits_right) - gd_slits = np.ones(self.slits.nslits, dtype=bool) - flex_list = flexure.spec_flexure_slit_global(self.sciImg, self.waveimg, global_sky_sep, self.par, - self.slits, self.slitmask, trace_spat, gd_slits, - self.wv_calib, self.pypeline, self.det) - for sl in range(self.slits.nslits): - self.slitshift[sl] = flex_list[sl]['shift'][0] - msgs.info("Flexure correction of slit {0:d}: {1:.3f} pixels".format(1 + sl, self.slitshift[sl])) + self.calculate_flexure(global_sky_sep) # If the joint fit or spec/spat sensitivity corrections are not being performed, return the separate slits sky if not self.par['reduce']['skysub']['joint_fit']: @@ -1219,13 +1280,6 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), # global_sky_sep = Reduce.global_skysub(self, skymask=skymask, update_crmask=update_crmask, trim_edg=trim_edg, # show_fit=show_fit, show=show, show_objs=show_objs) - # Recalculate the wavelength image, and the global sky taking into account the spectral flexure - msgs.info("Generating wavelength image, accounting for spectral flexure") - self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spec_flexure=self.slitshift, - spat_flexure=self.spat_flexure_shift) - - self.illum_profile_spectral(global_sky_sep, skymask=skymask) - # Use sky information in all slits to perform a joint sky fit global_sky = self.joint_skysub(skymask=skymask, update_crmask=update_crmask, trim_edg=trim_edg, show_fit=show_fit, show=show, show_objs=show_objs, @@ -1233,3 +1287,71 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), return global_sky + def calculate_flexure(self, global_sky): + """ + Convenience function to calculate the flexure of the IFU + + Args: + global_sky (`numpy.ndarray`_): + Model of the sky + """ + sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] + box_rad = self.par['reduce']['extraction']['boxcar_radius'] + trace_spat = 0.5 * (self.slits_left + self.slits_right) + # Load archival sky spectrum for absolute correction + sky_spectrum, sky_fwhm_pix = flexure.get_archive_spectrum(self.par['flexure']['spectrum']) + # Get spectral FWHM (in Angstrom) if available + iwv = np.where(self.wv_calib.spat_ids == self.slits.spat_id[sl_ref])[0][0] + ref_fwhm_pix = self.wv_calib.wv_fits[iwv].fwhm + # Extract a spectrum of the sky + thismask = (self.slitmask == self.slits.spat_id[sl_ref]) + ref_skyspec = flexure.get_sky_spectrum(self.sciImg.image, self.sciImg.ivar, self.waveimg, thismask, + global_sky, box_rad, self.slits, trace_spat[:, sl_ref], + self.pypeline, self.det) + # Calculate the flexure + flex_dict_ref = flexure.spec_flex_shift(ref_skyspec, sky_spectrum, sky_fwhm_pix, spec_fwhm_pix=ref_fwhm_pix, + mxshft=self.par['flexure']['spec_maxshift'], + excess_shft=self.par['flexure']['excessive_shift'], + method="slitcen") + this_slitshift = np.zeros(self.slits.nslits) + if flex_dict_ref is not None: + msgs.warn("Only a relative spectral flexure correction will be performed") + this_slitshift = np.ones(self.slits.nslits) * flex_dict_ref['shift'] + # Now loop through all slits to calculate the additional shift relative to the reference slit + flex_list = [] + for slit_idx, slit_spat in enumerate(self.slits.spat_id): + thismask = (self.slitmask == slit_spat) + # Extract sky spectrum for this slit + this_skyspec = flexure.get_sky_spectrum(self.sciImg.image, self.sciImg.ivar, self.waveimg, thismask, + global_sky, box_rad, self.slits, trace_spat[:, slit_idx], + self.pypeline, self.det) + # Calculate the flexure + flex_dict = flexure.spec_flex_shift(this_skyspec, ref_skyspec, ref_fwhm_pix * 1.01, + spec_fwhm_pix=ref_fwhm_pix, + mxshft=self.par['flexure']['spec_maxshift'], + excess_shft=self.par['flexure']['excessive_shift'], + method="slitcen") + this_slitshift[slit_idx] += flex_dict['shift'] + flex_list.append(flex_dict.copy()) + # Replace the reference slit with the absolute shift + flex_list[sl_ref] = flex_dict_ref.copy() + # Add this flexure to the previous flexure correction + self.slitshift += this_slitshift + # Now report the flexure values + for slit_idx, slit_spat in enumerate(self.slits.spat_id): + msgs.info("Flexure correction, slit {0:d} (spat id={1:d}): {2:.3f} pixels".format(1+slit_idx, slit_spat, + self.slitshift[slit_idx])) + # Save QA + # TODO :: Need to implement QA + msgs.work("QA is not currently implemented for the flexure correction") + if False:#flex_list is not None: + basename = f'{self.basename}_global_{self.spectrograph.get_det_name(self.det)}' + out_dir = os.path.join(self.par['rdx']['redux_path'], 'QA') + slit_bpm = np.zeros(self.slits.nslits, dtype=bool) + flexure.spec_flexure_qa(self.slits.slitord_id, slit_bpm, basename, flex_list, out_dir=out_dir) + + # Recalculate the wavelength image, and the global sky taking into account the spectral flexure + msgs.info("Generating wavelength image, accounting for spectral flexure") + self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spec_flexure=self.slitshift, + spat_flexure=self.spat_flexure_shift) + return diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index b731050869..75096d4e72 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -16,6 +16,7 @@ from IPython import embed from pypeit import msgs +from pypeit.pypmsgs import PypeItError from pypeit import utils from pypeit import bspline @@ -480,8 +481,8 @@ class FlatField: The current slit traces. wavetilts (:class:`~pypeit.wavetilts.WaveTilts`): The current wavelength tilt traces; see - wv_calib (??): - ?? + wv_calib (:class:`~pypeit.wavecalib.WaveCalib`): + Wavelength calibration object spat_illum_only (bool, optional): Only perform the spatial illumination calculation, and ignore the 2D bspline fit. This should only be set to true if you @@ -489,8 +490,8 @@ class FlatField: simultaneously generate a pixel flat and a spatial illumination profile from the same input, this should be False (which is the default). - qa_path (??, optional): - ?? + qa_path (str, optional): + Path to QA directory Attributes: rawflatimg (:class:`~pypeit.images.pypeitimage.PypeItImage`): @@ -554,15 +555,12 @@ def run(self, doqa=False, debug=False, show=False): This is a simple wrapper for the main flat-field methods: - - Flat-field images are processed using :func:`build_pixflat`. - - Full 2D model, illumination flat, and pixel flat images are constructed by :func:`fit`. - The results can be shown in a ginga window using :func:`show`. - The method is a simple wrapper for :func:`build_pixflat`, :func:`fit`, - and :func:`show`. + The method is a simple wrapper for :func:`fit` and :func:`show`. Args: doqa (:obj:`bool`, optional): @@ -594,28 +592,31 @@ def run(self, doqa=False, debug=False, show=False): # If we're only doing the spatial illumination profile, the detector structure # has already been divided out by the pixel flat. No need to calculate structure if not self.spat_illum_only: - niter = 2 # Need two iterations, particularly for the fine spatial illumination correction. + niter = 1 # Just do one iteration... two is too long, and doesn't significantly improve the fine spatial illumination correction. det_resp_model = 1 # Initialise detector structure to a value of 1 (i.e. no detector structure) + onslits = self.slits.slit_img(pad=-self.flatpar['slit_trim'], initial=False) != -1 for ff in range(niter): # If we're only doing the spatial illumination profile, the detector structure # has already been divided out by the pixel flat. if self.spat_illum_only: break - msgs.info("Iteration {0:d} of 2D detector response extraction".format(ff+1)) + msgs.info("Iteration {0:d}/{1:d} of 2D detector response extraction".format(ff+1, niter)) # Extract a detector response image det_resp = self.extract_structure(rawflat_orig) - gpmask = (self.waveimg != 0.0) & gpm + # Trim the slits to avoid edge effects + gpmask = (self.waveimg != 0.0) & gpm & onslits # Model the 2D detector response in an instrument specific way det_resp_model = self.spectrograph.fit_2d_det_response(det_resp, gpmask) # Apply this model self.rawflatimg.image = rawflat_orig * utils.inverse(det_resp_model) - if doqa: - # TODO :: Probably need to pass in det when more spectrographs implement a structure correction... - outfile = qa.set_qa_filename("DetectorStructure_" + self.calib_key, 'detector_structure', - det="DET01", out_dir=self.qa_path) - detector_structure_qa(det_resp, det_resp_model, outfile=outfile) # Perform a 2D fit with the cleaned image self.fit(spat_illum_only=self.spat_illum_only, doqa=doqa, debug=debug) + # Save the QA, if requested + if doqa: + # TODO :: Probably need to pass in det when more spectrographs implement a structure correction... + outfile = qa.set_qa_filename("DetectorStructure_" + self.calib_key, 'detector_structure', + det="DET01", out_dir=self.qa_path) + detector_structure_qa(det_resp, det_resp_model, outfile=outfile) # Include the structure in the flat model and the pixelflat self.mspixelflat *= det_resp_model # Reset the rawimg @@ -652,7 +653,7 @@ def build_mask(self): Generate bad pixel mask. Returns: - :obj:`numpy.ndarray` : bad pixel mask + `numpy.ndarray`_ : bad pixel mask """ bpmflats = np.zeros_like(self.slits.mask, dtype=self.slits.bitmask.minimum_dtype()) for flag in ['SKIPFLATCALIB', 'BADFLATCALIB']: @@ -691,7 +692,7 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): Construct a model of the flat-field image. For this method to work, :attr:`rawflatimg` must have been - previously constructed; see :func:`build_pixflat`. + previously constructed. The method loops through all slits provided by the :attr:`slits` object, except those that have been masked (i.e., slits with @@ -730,7 +731,7 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): (see ``tweak_slits_thresh`` in :attr:`flatpar`), up to a maximum allowed shift from the existing slit edge (see ``tweak_slits_maxfrac`` in :attr:`flatpar`). See - :func:`pypeit.core.tweak_slit_edges`. If tweaked, the + :func:`~pypeit.core.flat.tweak_slit_edges`. If tweaked, the :func:`spatial_fit` is repeated to place it on the tweaked slits reference frame. - Use the bspline fit to construct the 2D illumination image @@ -755,7 +756,7 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): attributes are altered internally. If the slit edges are to be tweaked using the 1D illumination profile (``tweak_slits`` in :attr:`flatpar`), the tweaked slit edge arrays in the internal - :class:`~pypeit.edgetrace.SlitTraceSet` object, :attr:`slits`, + :class:`~pypeit.slittrace.SlitTraceSet` object, :attr:`slits`, are also altered. Used parameters from :attr:`flatpar` @@ -1367,7 +1368,7 @@ def spatial_fit(self, norm_spec, spat_coo, median_slit_width, spat_gpm, gpm, deb return exit_status, spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, \ spat_flat_fit, spat_flat_data_raw - def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gpm, doqa=False): + def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gpm, slit_trim=3, doqa=False): """ Generate a relative scaling image for a slit-based IFU. All slits are scaled relative to a reference slit, specified in @@ -1378,23 +1379,33 @@ def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gp spat_illum : `numpy.ndarray`_ An image containing the generated spatial illumination profile for all slits. onslit_tweak : `numpy.ndarray`_ - mask indicticating which pixels are on the slit (True = on slit) + mask indicating which pixels are on the slit (True = on slit) slit_idx : int Slit number (0-indexed) slit_spat : int Spatial ID of the slit gpm : `numpy.ndarray`_ Good pixel mask + slit_txt : str + if pypeline is "Echelle", then slit_txt should be set to "order", otherwise, use "slit" + slit_trim : int, optional + Trim the slit edges by this number of pixels during the fitting. Note that the + fit will be evaluated on the pixels indicated by onslit_tweak. + A positive number trims the slit edges, a negative number pads the slit edges. doqa : :obj:`bool`, optional: Save the QA? """ # TODO :: Include fit_order in the parset?? fit_order = np.array([3, 6]) - msgs.info("Performing a fine correction to the spatial illumination (slit={0:d})".format(slit_spat)) + slit_txt = self.slits.slitord_txt + slit_ordid = self.slits.slitord_id[slit_idx] + msgs.info(f"Performing a fine correction to the spatial illumination ({slit_txt} {slit_ordid})") # initialise illumflat_finecorr = np.ones_like(self.rawflatimg.image) + # Trim the edges by a few pixels to avoid edge effects + onslit_tweak_trim = self.slits.slit_img(pad=-slit_trim, slitidx=slit_idx, initial=False) == slit_spat # Setup - slitimg = (slit_spat+1) * onslit_tweak.astype(int) - 1 + slitimg = (slit_spat + 1) * onslit_tweak.astype(int) - 1 # Need to +1 and -1 so that slitimg=-1 when off the slit normed = self.rawflatimg.image.copy() ivarnrm = self.rawflatimg.ivar.copy() normed[onslit_tweak] *= utils.inverse(spat_illum) @@ -1404,46 +1415,52 @@ def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gp this_right = right[:, slit_idx] slitlen = int(np.median(this_right - this_left)) - # Prepare fitting coordinates - wgud = np.where(onslit_tweak & self.rawflatimg.select_flag(invert=True)) - cut = (wgud[0], wgud[1]) - thiswave = self.waveimg[cut] - ypos = (thiswave - thiswave.min()) / (thiswave.max() - thiswave.min()) + # Generate the coordinates to evaluate the fit + this_slit = np.where(onslit_tweak & self.rawflatimg.select_flag(invert=True) & (self.waveimg!=0.0)) + this_wave = self.waveimg[this_slit] xpos_img = self.slits.spatial_coordinate_image(slitidx=slit_idx, initial=True, slitid_img=slitimg, flexure_shift=self.wavetilts.spat_flexure) - xpos = xpos_img[cut] + # Generate the trimmed versions for fitting + this_slit_trim = np.where(onslit_tweak_trim & self.rawflatimg.select_flag(invert=True)) + this_wave_trim = self.waveimg[this_slit_trim] + wave_min, wave_max = this_wave_trim.min(), this_wave_trim.max() + ypos_fit = (this_wave_trim - wave_min) / (wave_max - wave_min) + xpos_fit = xpos_img[this_slit_trim] + # Evaluation coordinates + ypos = (this_wave - wave_min) / (wave_max - wave_min) # Need to use the same wave_min and wave_max as the fitting coordinates + xpos = xpos_img[this_slit] # Normalise the image delta = 0.5/self.slits.nspec # include the endpoints bins = np.linspace(0.0-delta, 1.0+delta, self.slits.nspec+1) - censpec, _ = np.histogram(ypos, bins=bins, weights=normed[cut]) - nrm, _ = np.histogram(ypos, bins=bins) + censpec, _ = np.histogram(ypos_fit, bins=bins, weights=normed[this_slit_trim]) + nrm, _ = np.histogram(ypos_fit, bins=bins) censpec *= utils.inverse(nrm) tiltspl = interpolate.interp1d(0.5*(bins[1:]+bins[:-1]), censpec, kind='linear', bounds_error=False, fill_value='extrapolate') - nrm_vals = tiltspl(ypos) - normed[wgud] *= utils.inverse(nrm_vals) - ivarnrm[wgud] *= nrm_vals**2 + nrm_vals = tiltspl(ypos_fit) + normed[this_slit_trim] *= utils.inverse(nrm_vals) + ivarnrm[this_slit_trim] *= nrm_vals**2 # Mask the edges and fit - gpmfit = gpm[cut] - # Trim by 5% of the slit length, or at least 3 pixels + gpmfit = gpm[this_slit_trim] + # Trim by 5% of the slit length, or at least slit_trim pixels xfrac = 0.05 - if xfrac * slitlen < 3: - xfrac = 3/slitlen - gpmfit[np.where((xpos < xfrac) | (xpos > 1-xfrac))] = False - fullfit = fitting.robust_fit(xpos, normed[cut], fit_order, x2=ypos, + if xfrac * slitlen < slit_trim: + xfrac = slit_trim/slitlen + gpmfit[np.where((xpos_fit < xfrac) | (xpos_fit > 1-xfrac))] = False + fullfit = fitting.robust_fit(xpos_fit, normed[this_slit_trim], fit_order, x2=ypos_fit, in_gpm=gpmfit, function='legendre2d', upper=2, lower=2, maxdev=1.0, minx=0.0, maxx=1.0, minx2=0.0, maxx2=1.0) # Generate the fine correction image and store the result if fullfit.success == 1: self.list_of_finecorr_fits[slit_idx] = fullfit - illumflat_finecorr[wgud] = fullfit.eval(xpos, ypos) + illumflat_finecorr[this_slit] = fullfit.eval(xpos, ypos) else: - msgs.warn("Fine correction to the spatial illumination failed for slit {0:d}".format(slit_spat)) + msgs.warn(f"Fine correction to the spatial illumination failed for {slit_txt} {slit_ordid}") return # Prepare QA @@ -1453,13 +1470,13 @@ def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gp prefix += "illumflat_" outfile = qa.set_qa_filename(prefix+self.calib_key, 'spatillum_finecorr', slit=slit_spat, out_dir=self.qa_path) - title = "Fine correction to spatial illumination (slit={0:d})".format(slit_spat) + title = f"Fine correction to spatial illumination ({slit_txt} {slit_ordid})" normed[np.logical_not(onslit_tweak)] = 1 # For the QA, make everything off the slit equal to 1 - spatillum_finecorr_qa(normed, illumflat_finecorr, this_left, this_right, ypos, cut, + spatillum_finecorr_qa(normed, illumflat_finecorr, this_left, this_right, ypos_fit, this_slit_trim, outfile=outfile, title=title, half_slen=slitlen//2) return - def extract_structure(self, rawflat_orig): + def extract_structure(self, rawflat_orig, slit_trim=3): """ Generate a relative scaling image for a slit-based IFU. All slits are scaled relative to a reference slit, specified in @@ -1469,6 +1486,10 @@ def extract_structure(self, rawflat_orig): ---------- rawflat_orig : `numpy.ndarray`_ The original raw image of the flatfield + slit_trim : int, optional + Trim the slit edges by this number of pixels during the fitting. Note that the + fit will be evaluated on the pixels indicated by onslit_tweak. + A positive number trims the slit edges, a negative number pads the slit edges. Returns ------- @@ -1502,15 +1523,17 @@ def extract_structure(self, rawflat_orig): model=None, gpmask=gpm, skymask=None, trim=self.flatpar['slit_trim'], flexure=self.wavetilts.spat_flexure, smooth_npix=self.flatpar['slit_illum_smooth_npix']) - # Construct a wavelength array + # Trim the edges by a few pixels to avoid edge effects + onslits_trim = gpm & (self.slits.slit_img(pad=-slit_trim, initial=False) != -1) onslits = (self.waveimg != 0.0) & gpm + # Construct a wavelength array minwv = np.min(self.waveimg[onslits]) maxwv = np.max(self.waveimg) wavebins = np.linspace(minwv, maxwv, self.slits.nspec) # Correct the raw flat for spatial illumination, then generate a spectrum rawflat_corr = rawflat * utils.inverse(scale_model) - hist, edge = np.histogram(self.waveimg[onslits], bins=wavebins, weights=rawflat_corr[onslits]) - cntr, edge = np.histogram(self.waveimg[onslits], bins=wavebins) + hist, edge = np.histogram(self.waveimg[onslits_trim], bins=wavebins, weights=rawflat_corr[onslits_trim]) + cntr, edge = np.histogram(self.waveimg[onslits_trim], bins=wavebins) cntr = cntr.astype(float) spec_ref = hist * utils.inverse(cntr) wave_ref = 0.5 * (wavebins[1:] + wavebins[:-1]) @@ -1629,8 +1652,10 @@ def spatillum_finecorr_qa(normed, finecorr, left, right, ypos, cut, outfile=None # Make the model offs = bb * sep model += offs - minmod = minmod if minmod < np.min(model) else np.min(model) - maxmod = maxmod if maxmod > np.max(model) else np.max(model) + nonzero = model != offs + if np.any(nonzero): + minmod = minmod if minmod < np.min(model[nonzero]) else np.min(model[nonzero]) + maxmod = maxmod if maxmod > np.max(model[nonzero]) else np.max(model[nonzero]) # Plot it! ax_spec.plot(spatmid, offs + cntr, linestyle='-', color=colors[bb]) ax_spec.plot(spatmid, model, linestyle='-', color=colors[bb], alpha=0.5, linewidth=3) @@ -1708,20 +1733,20 @@ def detector_structure_qa(det_resp, det_resp_model, outfile=None, title="Detecto gs = gridspec.GridSpec(1, 4, height_ratios=[1], width_ratios=[1.0, 1.0, 1.0, 0.05]) # Axes showing the observed detector response ax_data = plt.subplot(gs[0]) - ax_data.imshow(det_resp, vmin=vmin, vmax=vmax) + ax_data.imshow(det_resp, origin='lower', vmin=vmin, vmax=vmax) ax_data.set_xlabel("data", fontsize='medium') ax_data.axes.xaxis.set_ticks([]) ax_data.axes.yaxis.set_ticks([]) # Axes showing the model fit to the detector response ax_modl = plt.subplot(gs[1]) - im = ax_modl.imshow(det_resp_model, vmin=vmin, vmax=vmax) + im = ax_modl.imshow(det_resp_model, origin='lower', vmin=vmin, vmax=vmax) ax_modl.set_title(title, fontsize='medium') ax_modl.set_xlabel("model", fontsize='medium') ax_modl.axes.xaxis.set_ticks([]) ax_modl.axes.yaxis.set_ticks([]) # Axes showing the residual of the detector response fit ax_resd = plt.subplot(gs[2]) - ax_resd.imshow(det_resp-det_resp_model, vmin=vmin-1, vmax=vmax-1) + ax_resd.imshow(det_resp-det_resp_model, origin='lower', vmin=vmin-1, vmax=vmax-1) ax_resd.set_xlabel("data-model", fontsize='medium') ax_resd.axes.xaxis.set_ticks([]) ax_resd.axes.yaxis.set_ticks([]) @@ -1781,8 +1806,11 @@ def show_flats(image_list, wcs_match=True, slits=None, waveimg=None): clear = False -def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_npix=None, model=None, gpmask=None, skymask=None, trim=3, flexure=None): +def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_npix=None, + model=None, gpmask=None, skymask=None, trim=3, flexure=None): """ + TODO :: This could possibly be moved to core.flat + Determine the relative spectral illumination of all slits. Currently only used for image slicer IFUs. @@ -1895,7 +1923,7 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ scaleImg *= relscl_model #rawimg_copy /= relscl_model modelimg_copy /= relscl_model - if max(abs(1/minv), abs(maxv)) < 1.001: # Relative accruacy of 0.1% is sufficient + if max(abs(1/minv), abs(maxv)) < 1.005: # Relative accuracy of 0.5% is sufficient break debug = False if debug: diff --git a/pypeit/fluxcalibrate.py b/pypeit/fluxcalibrate.py index 9cff7e9c3e..e07819e0be 100644 --- a/pypeit/fluxcalibrate.py +++ b/pypeit/fluxcalibrate.py @@ -6,17 +6,16 @@ from pypeit import msgs from pypeit.spectrographs.util import load_spectrograph -from pypeit import sensfunc from pypeit import specobjs +from pypeit import sensfunc from pypeit.history import History from astropy import table from IPython import embed - -class FluxCalibrate: +def flux_calibrate(spec1dfiles, sensfiles, par=None, outfiles=None, chk_version=True): """ - Class for flux calibrating spectra. + Function for flux calibrating spectra. Args: spec1dfiles (list): @@ -24,141 +23,31 @@ class FluxCalibrate: sensfiles (list): List of sensitivity function files to use to flux calibrate the spec1d files. This list and the sensfiles list need to have the same length and be aligned - par (pypeit.par.pypeitpar.FluxCalibrate, optional): + par (:class:`~pypeit.par.pypeitpar.FluxCalibratePar`, optional): Parset object containing parameters governing the flux calibration. outfiles (list, optional): Names of the output files. If None, this is set to spec1dfiles and those are overwritten + chk_version (bool, optional): + Whether to check of the data model versions of spec1d and sens files. Defaults to True. """ - # Superclass factory method generates the subclass instance - @classmethod - def get_instance(cls, spec1dfiles, sensfiles, par=None, debug=False): - pypeline = fits.getheader(spec1dfiles[0])['PYPELINE'] + 'FC' - return next(c for c in cls.__subclasses__() if c.__name__ == pypeline)( - spec1dfiles, sensfiles, par=par, debug=debug) - - def __init__(self, spec1dfiles, sensfiles, par=None, debug=False, outfiles=None): - - self.spec1dfiles = spec1dfiles - self.sensfiles = sensfiles - - # Output file names - self.outfiles = spec1dfiles if outfiles is None else outfiles - - # Load the spectrograph - header = fits.getheader(spec1dfiles[0]) - self.spectrograph = load_spectrograph(header['PYP_SPEC']) - self.par = self.spectrograph.default_pypeit_par()['fluxcalib'] if par is None else par - self.debug = debug - self.algorithm = None - - sensf_last = None - for spec1, sensf, outfile in zip(self.spec1dfiles, self.sensfiles, self.outfiles): - # Read in the data - sobjs = specobjs.SpecObjs.from_fitsfile(spec1) - history = History(sobjs.header) - if sensf != sensf_last: - sens = sensfunc.SensFunc.from_file(sensf) - sensf_last = sensf - history.append(f'PypeIt Flux calibration "{sensf}"') - self.flux_calib(sobjs, sens) - sobjs.write_to_fits(sobjs.header, outfile, history=history, overwrite=True) - - def flux_calib(self, sobjs, sens): - """ - Flux calibrate the provided object spectra (``sobjs``) using the - provided sensitivity function (``sens``). - - This is an empty base-class method that must be overloaded by the - subclasses. - - Args: - sobjs (:class:`~pypeit.specobjs.SpecObjs`): - Object spectra - sens (:class:`~pypeit.sensfunc.SensFunc`): - Sensitivity function - """ - pass - - def _set_extinct_correct(self, extinct_correct, algorithm): - return (True if algorithm == 'UVIS' else False) if extinct_correct is None else extinct_correct - -class MultiSlitFC(FluxCalibrate): - """ - Child of FluxSpec for Multislit and Longslit reductions - """ - - def __init__(self, spec1dfiles, sensfiles, par=None, debug=False, outfiles=None): - super().__init__(spec1dfiles, sensfiles, par=par, debug=debug, outfiles=outfiles) - - - def flux_calib(self, sobjs, sens): - """ - Apply sensitivity function to all the spectra in an sobjs object. - - Args: - sobjs (:class:`~pypeit.specobjs.SpecObjs`): - Object spectra - sens (:class:`~pypeit.sensfunc.SensFunc`): - Sensitivity function - """ - # Run - for sci_obj in sobjs: - sci_obj.apply_flux_calib(sens.wave[:, 0], sens.zeropoint[:, 0], - sobjs.header['EXPTIME'], - extinct_correct=self._set_extinct_correct( - self.par['extinct_correct'], sens.algorithm), - longitude=self.spectrograph.telescope['longitude'], - latitude=self.spectrograph.telescope['latitude'], - extinctfilepar=self.par['extinct_file'], - extrap_sens=self.par['extrap_sens'], - airmass=float(sobjs.header['AIRMASS'])) - - - - -class EchelleFC(FluxCalibrate): - """ - Child of FluxSpec for Echelle reductions - """ - - def __init__(self, spec1dfiles, sensfiles, par=None, debug=False): - super().__init__(spec1dfiles, sensfiles, par=par, debug=debug) - - - def flux_calib(self, sobjs, sens): - """ - Apply sensitivity function to all the spectra in an sobjs object. - - Args: - sobjs (:class:`~pypeit.specobjs.SpecObjs`): - Object spectra - sens (:class:`~pypeit.sensfunc.SensFunc`): - Sensitivity function - """ - - # Flux calibrate the orders that are mutually in the meta_table and in - # the sobjs. This allows flexibility for applying to data for cases - # where not all orders are present in the data as in the sensfunc, etc., - # i.e. X-shooter with the K-band blocking filter. - ech_orders = np.array(sens.sens['ECH_ORDERS']).flatten() - #norders = ech_orders.size - for sci_obj in sobjs: - # JFH Is there a more elegant pythonic way to do this without looping over both orders and sci_obj? - indx = np.where(ech_orders == sci_obj.ECH_ORDER)[0] - if indx.size==1: - sci_obj.apply_flux_calib(sens.wave[:, indx[0]], sens.zeropoint[:,indx[0]], - sobjs.header['EXPTIME'], - extinct_correct=self._set_extinct_correct( - self.par['extinct_correct'], sens.algorithm), - extrap_sens = self.par['extrap_sens'], - longitude=self.spectrograph.telescope['longitude'], - latitude=self.spectrograph.telescope['latitude'], - extinctfilepar=self.par['extinct_file'], - airmass=float(sobjs.header['AIRMASS'])) - elif indx.size == 0: - msgs.info('Unable to flux calibrate order = {:} as it is not in your sensitivity function. ' - 'Something is probably wrong with your sensitivity function.'.format(sci_obj.ECH_ORDER)) - else: - msgs.error('This should not happen') + # Output file names + outfiles = spec1dfiles if outfiles is None else outfiles + + # Load the spectrograph + header = fits.getheader(spec1dfiles[0]) + spectrograph = load_spectrograph(header['PYP_SPEC']) + par = spectrograph.default_pypeit_par()['fluxcalib'] if par is None else par + + sensf_last = None + for spec1, sensf, outfile in zip(spec1dfiles, sensfiles, outfiles): + # Read in the data + sobjs = specobjs.SpecObjs.from_fitsfile(spec1, chk_version=chk_version) + history = History(sobjs.header) + if sensf != sensf_last: + sens = sensfunc.SensFunc.from_file(sensf, chk_version=chk_version) + sensf_last = sensf + history.append(f'PypeIt Flux calibration "{sensf}"') + sobjs.apply_flux_calib(par, spectrograph, sens) + sobjs.write_to_fits(sobjs.header, outfile, history=history, overwrite=True) diff --git a/pypeit/history.py b/pypeit/history.py index bbd803e832..9386eb69e2 100644 --- a/pypeit/history.py +++ b/pypeit/history.py @@ -6,6 +6,8 @@ """ import os.path +import numpy as np +from IPython import embed from astropy.time import Time from astropy.io import fits @@ -85,7 +87,7 @@ def add_reduce(self, calib_id, metadata, frames, bg_frames): for frame in calib_frames: self.append(f'{frame["frametype"]} "{frame["filename"]}"', add_date=False) - def add_coadd1d(self, spec1d_files, objids): + def add_coadd1d(self, spec1d_files, objids, gpm_exp=None): """ Add history entries for 1D coadding. @@ -104,27 +106,62 @@ def add_coadd1d(self, spec1d_files, objids): Args: spec1d_files (:obj:`list`): List of the spec1d files used for coadding. objids (:obj:`list`): List of the PypeIt object ids used in coadding. + gpm_exp (:obj:`list`, optional): List of boolean indicating which exposures were coadded. """ - combined_files_objids = list(zip(spec1d_files, objids)) - self.append(f'PypeIt Coadded {len(combined_files_objids)} objects from {len(set(spec1d_files))} spec1d files') - - current_spec1d = "" - for (spec1d, objid) in combined_files_objids: - if spec1d != current_spec1d: - current_spec1d = spec1d - - self.append(f'From "{os.path.basename(spec1d)}"', add_date=False) - header = fits.getheader(spec1d) - additional_info = None - if 'SEMESTER' in header: - additional_info = f"Semester: {header['SEMESTER']}" - if 'PROGID' in header: - additional_info += f" Program ID: {header['PROGID']}" - if additional_info is not None: - self.append(additional_info, add_date=False) - self.append(objid, add_date=False) + if gpm_exp is not None: + # Not coadded files and objids + notcoadded_spec1d_files = [spec1d_file for (spec1d_file, gpm_exp) in zip(spec1d_files, gpm_exp) if not gpm_exp] + notcoadded_objids = [objid for (objid, gpm_exp) in zip(objids, gpm_exp) if not gpm_exp] + combined_notcoadd_files_objids = list(zip(notcoadded_spec1d_files, notcoadded_objids)) + # Coadded files and objids + coadded_spec1d_files = [spec1d_file for (spec1d_file, gpm_exp) in zip(spec1d_files, gpm_exp) if gpm_exp] + coadded_objids = [objid for (objid, gpm_exp) in zip(objids, gpm_exp) if gpm_exp] + combined_files_objids = list(zip(coadded_spec1d_files, coadded_objids)) + else: + combined_files_objids = list(zip(spec1d_files, objids)) + combined_notcoadd_files_objids = None + + files_objids = [combined_files_objids, combined_notcoadd_files_objids] + # add history + for file_objid in files_objids: + if file_objid is None: + continue + elif file_objid == combined_files_objids: + self.append(f'PypeIt Coadded {len(file_objid)} objects ' + f'from {np.unique([f[0] for f in file_objid]).size} spec1d files') + elif file_objid == combined_notcoadd_files_objids and len(file_objid) > 0: + self.append(f'PypeIt DID NOT COADD {len(file_objid)} objects ' + f'from {np.unique([f[0] for f in file_objid]).size} spec1d files', add_date=False) + + current_spec1d = "" + for (spec1d, objid) in file_objid: + if spec1d != current_spec1d: + current_spec1d = spec1d + + self.append(f'From "{os.path.basename(spec1d)}"', add_date=False) + header = fits.getheader(spec1d) + additional_info = None + if 'SEMESTER' in header: + additional_info = f"Semester: {header['SEMESTER']}" + if 'PROGID' in header: + additional_info += f" Program ID: {header['PROGID']}" + if additional_info is not None: + self.append(additional_info, add_date=False) + obj_info = objid + # get extension names + hnames = [h.name for h in fits.open(spec1d)] + # find the extension name that include objid + ind_ext = np.where([objid in h for h in hnames])[0] + if ind_ext.size > 0: + # get the header for this extension + this_ext_header = fits.getheader(spec1d, ext=ind_ext[0]) + if 'MASKDEF_ID' in this_ext_header: + obj_info += f" {this_ext_header['MASKDEF_ID']}" + if 'MASKDEF_OBJNAME' in this_ext_header: + obj_info += f" {this_ext_header['MASKDEF_OBJNAME']}" + self.append(obj_info, add_date=False) def append(self, history, add_date=True): """Append a new history entry. diff --git a/pypeit/images/bitmaskarray.py b/pypeit/images/bitmaskarray.py index ee8ba13b76..db9c6898b6 100644 --- a/pypeit/images/bitmaskarray.py +++ b/pypeit/images/bitmaskarray.py @@ -6,8 +6,6 @@ .. include:: ../include/bitmaskarray_usage.rst ----- - .. include common links, assuming primary doc root is up one directory .. include:: ../include/links.rst """ diff --git a/pypeit/images/buildimage.py b/pypeit/images/buildimage.py index 2ce086f116..1d58581f55 100644 --- a/pypeit/images/buildimage.py +++ b/pypeit/images/buildimage.py @@ -111,7 +111,7 @@ def construct_file_name(cls, calib_key, calib_dir=None, basename=None): Args: calib_key (:obj:`str`): String identifier of the calibration group. See - :func:`construct_calib_key`. + :func:`~pypeit.calibframe.CalibFrame.construct_calib_key`. calib_dir (:obj:`str`, `Path`_, optional): If provided, return the full path to the file given this directory. @@ -164,7 +164,7 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm= The 1-indexed detector number(s) to process. If a tuple, it must include detectors viable as a mosaic for the provided spectrograph; see :func:`~pypeit.spectrographs.spectrograph.Spectrograph.allowed_mosaics`. - frame_par (:class:`~pypeit.par.pypeitpar.FramePar`): + frame_par (:class:`~pypeit.par.pypeitpar.FrameGroupPar`): Parameters that dictate the processing of the images. See :class:`~pypeit.par.pypeitpar.ProcessImagesPar` for the defaults. @@ -187,7 +187,7 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm= (``sigma_clip`` is True), this sets the maximum number of rejection iterations. If None, rejection iterations continue until no more data are rejected; see - :func:`~pypeit.core.combine.weighted_combine``. + :func:`~pypeit.core.combine.weighted_combine`. ignore_saturation (:obj:`bool`, optional): If True, turn off the saturation flag in the individual images before stacking. This avoids having such values set to 0, which diff --git a/pypeit/images/pypeitimage.py b/pypeit/images/pypeitimage.py index 50cd06642f..35bc7d9862 100644 --- a/pypeit/images/pypeitimage.py +++ b/pypeit/images/pypeitimage.py @@ -965,7 +965,7 @@ def from_pypeitimage(cls, pypeitImage, calib_dir=None, setup=None, calib_id=None detname (:obj:`str`): The identifier used for the detector or detector mosaic for the relevant instrument; see - :func:`~pypeit.spectrograph.spectrograph.Spectrograph.get_det_name`. + :func:`~pypeit.spectrographs.spectrograph.Spectrograph.get_det_name`. Returns: :class:`PypeItImage`: Image with the appropriate type and diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 7b3f1d8c07..aca870163c 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -60,7 +60,7 @@ class RawImage: Attributes: filename (:obj:`str`): Original file name with the data. - spectrograph (:class:`~pypeit.spectrograph.spectrographs.Spectrograph`): + spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): Spectrograph instance with the instrument-specific properties and methods. det (:obj:`int`, :obj:`tuple`): diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index eb84d24a2f..126ad2f2b2 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -83,8 +83,7 @@ def readlines(ifile:str): ifile (str): Name of the file to parse. Returns: - :obj:`numpy.ndarray`: Returns a list of the valid lines in the - files. + `numpy.ndarray`_: Returns a list of the valid lines in the files. """ # Check the files if not os.path.isfile(ifile): @@ -92,8 +91,7 @@ def readlines(ifile:str): # Read the input lines and replace special characters with open(ifile, 'r') as f: - lines = np.array([l.replace('\t', ' ').replace('\n', ' ').strip() \ - for l in f.readlines()]) + lines = np.array([l.replace('\t', ' ').replace('\n', ' ').strip() for l in f.readlines()]) # Remove empty or fully commented lines lines = lines[np.array([ len(l) > 0 and l[0] != '#' for l in lines ])] # Remove appended comments and return @@ -259,10 +257,9 @@ def _read_data_file_table(lines): List of lines *within the data* block read from the input file. Returns: - tuple: list, Table. A list of the paths provided (can be empty) - and a Table with the data provided in the input file. + tuple: A :obj:`list` with the paths provided (can be empty) and an + `astropy.table.Table`_ with the data provided in the input file. """ - # Allow for multiple paths paths = [] for l in lines: @@ -362,7 +359,7 @@ def find_block(lines, block): def path_and_files(self, key:str, skip_blank=False, check_exists=True): """Generate a list of the filenames with - the full path from the column of the data Table + the full path from the column of the data `astropy.table.Table`_ specified by `key`. The files must exist and be within one of the paths for this to succeed. @@ -488,8 +485,8 @@ def get_spectrograph(self): parameter. Raises: - :class:`~pypeit.pypmsgs.PypeItError`: Raised if the relevant - configuration parameter is not available. + :class:`~pypeit.pypmsgs.PypeItError`: + Raised if the relevant configuration parameter is not available. """ if 'rdx' not in self.config.keys() or 'spectrograph' not in self.config['rdx'].keys(): msgs.error('Cannot define spectrograph. Configuration file missing \n' @@ -676,6 +673,51 @@ def objids(self): # Return return oids + # TODO is the correct way to treat optional table entries? + + @property + def sensfiles(self): + """Generate a list of the sensitivity files with + the full path. The files must exist and be + within one of the paths (or the current + folder with not other paths specified) for this to succeed. + + Returns: + list: List of full path to each data file + or None if `filename` is not part of the data table + or there is no data table! + """ + + if 'sensfile' not in self.data.keys(): + return None + + # Grab em + sens_files = self.path_and_files('sensfile', skip_blank=True) + # Pad out + if len(sens_files) == 1 and len(self.filenames) > 1: + sens_files = sens_files * len(self.filenames) + # Return + return sens_files + + + @property + def setup_id(self): + + if 'setup_id' not in self.data.keys(): + return None + + # Generate list, scrubbing empty entries + sid = [str(item) for item in self.data['setup_id'] if str(item).strip() not in ['', 'none', 'None']] + + # Inflate as needed + if len(sid) == 1 and len(sid) < len(self.data): + sid = sid * len(self.data) + # Return + return sid + + + + class Coadd2DFile(InputFile): """Child class for coaddition in 2D @@ -720,7 +762,7 @@ def vet(self): msgs.error(f"Missing spectrograph in the Parameter block of your .coadd2d file. Add it!") # Done - msgs.info('.cube file successfully vetted.') + msgs.info('.coadd3d file successfully vetted.') @property def options(self): @@ -774,6 +816,27 @@ def options(self): elif len(skysub_frame) != 0: opts['skysub_frame'] = skysub_frame + # Load coordinate offsets for each file. This is "Delta RA cos(dec)" and "Delta Dec" + # Get the RA offset of each file + off_ra = self.path_and_files('ra_offset', skip_blank=False, check_exists=False) + if off_ra is None: + opts['ra_offset'] = None + elif len(off_ra) == 1 and len(self.filenames) > 1: + opts['ra_offset'] = off_ra*len(self.filenames) + elif len(off_ra) != 0: + opts['ra_offset'] = off_ra + # Get the DEC offset of each file + off_dec = self.path_and_files('dec_offset', skip_blank=False, check_exists=False) + if off_dec is None: + opts['dec_offset'] = None + elif len(off_dec) == 1 and len(self.filenames) > 1: + opts['dec_offset'] = off_dec*len(self.filenames) + elif len(off_dec) != 0: + opts['dec_offset'] = off_dec + # Check that both have been set + if (off_ra is not None and off_dec is None) or (off_ra is None and off_dec is not None): + msgs.error("You must specify both or neither of the following arguments: ra_offset, dec_offset") + # Return all options return opts diff --git a/pypeit/io.py b/pypeit/io.py index 2c4ddbae25..15e7380269 100644 --- a/pypeit/io.py +++ b/pypeit/io.py @@ -9,6 +9,7 @@ """ import os from pathlib import Path +import importlib import glob import sys import warnings @@ -880,3 +881,45 @@ def files_from_extension(raw_path, return numpy.concatenate([files_from_extension(p, extension=extension) for p in raw_path]).tolist() msgs.error(f"Incorrect type {type(raw_path)} for raw_path (must be str or list)") + + + +def load_object(module, obj=None): + """ + Load an abstracted module and object. + + Thanks to: https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path?rq=1 + + Args: + module (:obj:`str`): + The name of a global python module, the root name of a local file + with the object to import, or the full module + object type. If + ``obj`` is None, this *must* be the latter. + obj (:obj:`str`, optional): + The name of the object to import. If None, ``module`` must be the + full module + object type name. + + Return: + :obj:`type`: The imported object. + + Raises: + ImportError: + Raised if unable to import ``module``. + """ + if obj is None: + _module = '.'.join(module.split('.')[:-1]) + obj = module.split('.')[-1] + else: + _module = module + + try: + Module = importlib.import_module(_module) + except (ModuleNotFoundError, ImportError, TypeError) as e: + p = Path(module + '.py').resolve() + if not p.exists(): + raise ImportError(f'Unable to load module {_module}!') from e + spec = importlib.util.spec_from_file_location(_module, str(p)) + Module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(Module) + + return getattr(Module, obj) \ No newline at end of file diff --git a/pypeit/metadata.py b/pypeit/metadata.py index f8ec063a3e..22b4bd8890 100644 --- a/pypeit/metadata.py +++ b/pypeit/metadata.py @@ -59,9 +59,8 @@ class PypeItMetaData: The list of files to include in the table. data (table-like, optional): The data to include in the table. The type can be anything - allowed by the instantiation of - :class:`astropy.table.Table`. - usrdata (:obj:`astropy.table.Table`, optional): + allowed by the instantiation of `astropy.table.Table`_. + usrdata (`astropy.table.Table`_, optional): A user provided set of data used to supplement or overwrite metadata read from the file headers. The table must have a `filename` column that is used to match to the metadata @@ -69,28 +68,28 @@ class PypeItMetaData: `data` is also provided. This functionality is only used when building the metadata from the fits files. strict (:obj:`bool`, optional): - Function will fault if there is a problem with the reading - the header for any of the provided files; see - :func:`~pypeit.spectrographs.spectrograph.get_headarr`. Set - to False to instead report a warning and continue. + Function will fault if there is a problem with the reading the + header for any of the provided files; see + :func:`~pypeit.spectrographs.spectrograph.Spectrograph.get_headarr`. + Set to False to instead report a warning and continue. Attributes: spectrograph - (:class:`pypeit.spectrographs.spectrograph.Spectrograph`): + (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): The spectrograph used to collect the data save to each file. The class is used to provide the header keyword data to include in the table and specify any validation checks. - par (:class:`pypeit.par.pypeitpar.PypeItPar`): + par (:class:`~pypeit.par.pypeitpar.PypeItPar`): PypeIt parameters used to set the code behavior. If not provided, the default parameters specific to the provided spectrograph are used. configs (:obj:`dict`): A dictionary of the unique configurations identified. - type_bitmask (:class:`pypeit.core.framematch.FrameTypeBitMask`): + type_bitmask (:class:`~pypeit.core.framematch.FrameTypeBitMask`): The bitmask used to set the frame type of each fits file. - calib_bitmask (:class:`BitMask`): + calib_bitmask (:class:`~pypeit.bitmask.BitMask`): The bitmask used to keep track of the calibration group bits. - table (:class:`astropy.table.Table`): + table (`astropy.table.Table`_): The table with the relevant metadata for each fits file to use in the data reduction. """ @@ -104,6 +103,10 @@ def __init__(self, spectrograph, par, files=None, data=None, usrdata=None, # Initialize internals self.spectrograph = spectrograph + if files is not None: + # check if the spectrograph selected is correct for the data. NOTE: this is defined + # for each spectrograph independently, so it's currently not defined for all spectrographs + self.spectrograph.check_spectrograph(files if isinstance(files, str) else files[0]) self.par = par if not isinstance(self.par, PypeItPar): raise TypeError('Input parameter set must be of type PypeItPar.') @@ -155,10 +158,10 @@ def _build(self, files, strict=True, usrdata=None): files (:obj:`str`, :obj:`list`): One or more files to use to build the table. strict (:obj:`bool`, optional): - Function will fault if :func:`fits.getheader` fails to + Function will fault if `astropy.io.fits.getheader`_ fails to read any of the headers. Set to False to report a warning and continue. - usrdata (astropy.table.Table, optional): + usrdata (`astropy.table.Table`_, optional): Parsed for frametype for a few instruments (e.g. VLT) where meta data may not be required @@ -278,7 +281,7 @@ def merge(self, usrdata, match_type=True): this step by setting `match_type=False`. Args: - usrdata (:obj:`astropy.table.Table`): + usrdata (`astropy.table.Table`_): A user provided set of data used to supplement or overwrite metadata read from the file headers. The table must have a `filename` column that is used to @@ -289,14 +292,14 @@ def merge(self, usrdata, match_type=True): Raises: TypeError: - Raised if `usrdata` is not an `astropy.io.table.Table` + Raised if `usrdata` is not an `astropy.table.Table`_ KeyError: Raised if `filename` is not a key in the provided table. """ meta_data_model = meta.get_meta_data_model() # Check the input if not isinstance(usrdata, table.Table): - raise TypeError('Must provide an astropy.io.table.Table instance.') + raise TypeError('Must provide an astropy.table.Table instance.') if 'filename' not in usrdata.keys(): raise KeyError('The user-provided table must have \'filename\' column!') @@ -437,15 +440,12 @@ def construct_obstime(self, row): """ Construct the MJD of when the frame was observed. - .. todo:: - - Consolidate with :func:`convert_time` ? - Args: row (:obj:`int`): The 0-indexed row of the frame. Returns: - astropy.time.Time: The MJD of the observation. + `astropy.time.Time`_: The MJD of the observation. """ return time.Time(self['mjd'][row], format='mjd') @@ -456,7 +456,7 @@ def construct_basename(self, row, obstime=None): Args: row (:obj:`int`): The 0-indexed row of the frame. - obstime (:class:`astropy.time.Time`, optional): + obstime (`astropy.time.Time`_, optional): The MJD of the observation. If None, constructed using :func:`construct_obstime`. @@ -712,17 +712,11 @@ def unique_configurations(self, force=False, copy=False, rm_none=False): # If the frame types have been set, ignore anything listed in # the ignore_frames - indx = np.arange(len(self)) - ignore_frames = self.spectrograph.config_independent_frames() - if ignore_frames is not None: - if 'frametype' not in self.keys(): - msgs.error('To ignore frames, types must have been defined; run get_frame_types.') - ignore_frames = list(ignore_frames.keys()) - msgs.info('Unique configurations ignore frames with type: {0}'.format(ignore_frames)) - use = np.ones(len(self), dtype=bool) - for ftype in ignore_frames: - use &= np.logical_not(self.find_frames(ftype)) - indx = indx[use] + ignore_frames, ignore_indx = self.ignore_frames() + # Find the indices of the frames not to ignore + indx = np.arange(len(self.table)) + indx = indx[np.logical_not(np.in1d(indx, ignore_indx))] + if len(indx) == 0: msgs.error('No frames to use to define configurations!') @@ -816,12 +810,16 @@ def set_configurations(self, configs=None, force=False, fill=None): if len(set(cfg.keys()) - set(self.keys())) > 0: msgs.error('Configuration {0} defined using unavailable keywords!'.format(k)) + # Some frame types need to be ignored + ignore_frames, ignore_indx = self.ignore_frames() # define the column 'setup' in self.table nrows = len(self) col = table.Column(data=['None'] * nrows, name='setup', dtype='U25') self.table.add_column(col) is_science = self.find_frames('science') # Science frames can only have one configuration for i in range(nrows): + if i in ignore_indx: + continue for d, cfg in _configs.items(): # modify the configuration items only for specific frames. This is instrument dependent. mod_cfg = self.spectrograph.modify_config(self.table[i], cfg) @@ -840,10 +838,8 @@ def set_configurations(self, configs=None, force=False, fill=None): # All are set, so we're done return - # Some frame types may have been ignored - ignore_frames = self.spectrograph.config_independent_frames() + # If there's no frames to ignore, we can safely return if ignore_frames is None: - # Nope, we're still done return # At this point, we need the frame type to continue @@ -858,11 +854,21 @@ def set_configurations(self, configs=None, force=False, fill=None): for ftype, metakey in ignore_frames.items(): # TODO: For now, use this assert to check that the - # metakey is either not set or a string - assert metakey is None or isinstance(metakey, str), \ + # metakey is either not set, or is a string/list + assert metakey is None or isinstance(metakey, str) or isinstance(metakey, list), \ 'CODING ERROR: metadata keywords set by config_indpendent_frames are not ' \ 'correctly defined for {0}; values must be None or a string.'.format( self.spectrograph.__class__.__name__) + # If a list is input, check all elements of the list are strings + if isinstance(metakey, list): + for ll in metakey: + assert isinstance(ll, str), \ + 'CODING ERROR: metadata keywords set by config_indpendent_frames are not ' \ + 'correctly defined for {0}; values must be None or a string.'.format( + self.spectrograph.__class__.__name__) + elif isinstance(metakey, str): + # If metakey is a string, convert it to a one-element list + metakey = [metakey] # Get the list of frames of this type without a # configuration @@ -870,33 +876,53 @@ def set_configurations(self, configs=None, force=False, fill=None): if not np.any(indx): continue if metakey is None: - # No matching meta data defined, so just set all - # the frames to this (first) configuration - self.table['setup'][indx] = cfg_key + # No matching meta data defined, so just set all the frames to all of the configurations + new_cfg_key = np.full(len(self.table['setup'][indx]), 'None', dtype=object) + for c in range(len(self.table['setup'][indx])): + if cfg_key in self.table['setup'][indx][c]: + new_cfg_key[c] = self.table['setup'][indx][c] + elif self.table['setup'][indx][c] == 'None': + new_cfg_key[c] = cfg_key + else: + new_cfg_key[c] = self.table['setup'][indx][c] + ',{}'.format(cfg_key) + self.table['setup'][indx] = new_cfg_key continue - # Find the unique values of meta for this configuration - uniq_meta = np.unique(self.table[metakey][in_cfg].data) - # Warn the user that the matching meta values are not - # unique for this configuration. - if uniq_meta.size != 1: - msgs.warn('When setting the instrument configuration for {0} '.format(ftype) - + 'frames, configuration {0} does not have unique '.format(cfg_key) - + '{0} values.' .format(meta)) - # Find the frames of this type that match any of the - # meta data values - indx &= np.isin(self.table[metakey], uniq_meta) + # Loop through the meta keys + for mkey in metakey: + # Find the unique values of meta for this configuration + uniq_meta = np.unique(self.table[mkey][in_cfg].data) + # Warn the user that the matching meta values are not + # unique for this configuration. + if uniq_meta.size != 1: + msgs.warn('When setting the instrument configuration for {0} '.format(ftype) + + 'frames, configuration {0} does not have unique '.format(cfg_key) + + '{0} values.' .format(mkey)) + # Find the frames of this type that match any of the + # meta data values + indx &= np.isin(self.table[mkey], uniq_meta) + # assign new_cfg_key = np.full(len(self.table['setup'][indx]), 'None', dtype=object) for c in range(len(self.table['setup'][indx])): if cfg_key in self.table['setup'][indx][c]: new_cfg_key[c] = self.table['setup'][indx][c] - if self.table['setup'][indx][c] == 'None': + elif self.table['setup'][indx][c] == 'None': new_cfg_key[c] = cfg_key else: new_cfg_key[c] = self.table['setup'][indx][c] + ',{}'.format(cfg_key) self.table['setup'][indx] = new_cfg_key + # Check if still any of the configurations are not set. If yes, we want + # these frames to still be present in the .sorted file + not_setup = self.table['setup'] == 'None' + if np.any(not_setup): + cfg_gen = self.configuration_generator(start=len(np.unique(self.table['setup'][np.logical_not(not_setup)]))) + nw_setup = next(cfg_gen) + self.configs[nw_setup] = {} + msgs.warn('All files that did not match any setup are grouped into a single configuration.') + self.table['setup'][not_setup] = nw_setup + def clean_configurations(self): """ Ensure that configuration-defining keywords all have values @@ -954,7 +980,7 @@ def find_configuration(self, setup, index=False): boolean array. Returns: - numpy.ndarray: A boolean array, or an integer array if + `numpy.ndarray`_: A boolean array, or an integer array if ``index=True``, with the table rows associated with the requested setup/configuration. """ @@ -982,8 +1008,11 @@ def _set_calib_group_bits(self): # may not be integers instead of strings. self['calib'] = np.array([str(c) for c in self['calib']], dtype=object) # Collect and expand any lists - group_names = np.unique(np.concatenate( - [s.split(',') for s in self['calib'] if s not in ['all', 'None']])) + # group_names = np.unique(np.concatenate( + # [s.split(',') for s in self['calib'] if s not in ['all', 'None']])) + # DP changed to below because np.concatenate does not accept an empty list, + # which is the case when calib is None for all frames. This should avoid the code to crash + group_names = np.unique(sum([s.split(',') for s in self['calib'] if s not in ['all', 'None']], [])) # Expand any ranges keep_group = np.ones(group_names.size, dtype=bool) added_groups = [] @@ -1093,7 +1122,8 @@ def set_calibration_groups(self, global_frames=None, default=False, force=False) # The configuration must be present to determine the calibration # group if 'setup' not in self.keys(): - msgs.error('Must have defined \'setup\' column first; try running set_configurations.') + msgs.error('CODING ERROR: Must have defined \'setup\' column first; try running ' + 'set_configurations.') configs = np.unique(np.concatenate([_setup.split(',') for _setup in self['setup'].data])).tolist() if 'None' in configs: configs.remove('None') # Ignore frames with undefined configurations @@ -1106,7 +1136,9 @@ def set_calibration_groups(self, global_frames=None, default=False, force=False) # any changes to the strings will be truncated at 4 characters. self.table['calib'] = np.full(len(self), 'None', dtype=object) for i in range(n_cfg): - in_cfg = np.array([configs[i] in _set for _set in self.table['setup']]) & (self['framebit'] > 0) + in_cfg = np.array([configs[i] in _set for _set in self.table['setup']]) # & (self['framebit'] > 0) + if not any(in_cfg): + continue icalibs = np.full(len(self['calib'][in_cfg]), 'None', dtype=object) for c in range(len(self['calib'][in_cfg])): if self['calib'][in_cfg][c] == 'None': @@ -1133,6 +1165,32 @@ def set_calibration_groups(self, global_frames=None, default=False, force=False) # Check that the groups are valid self._check_calib_groups() + def ignore_frames(self): + """ + Construct a list of frame types to ignore, and the corresponding indices of these frametypes in the table. + + Returns: + :obj:`tuple`: Two objects are returned, (1) A dictionary where the + keys are the frame types that are configuration-independent and the + values are the metadata keywords that can be used to assign the + frames to a configuration group, and (2) an integer `numpy.ndarray` + with the table rows that should be ignored when defining the + configuration. + """ + ignore_indx = np.arange(len(self.table)) + ignore_frames = self.spectrograph.config_independent_frames() + ignmsk = np.zeros(len(self.table), dtype=bool) + if ignore_frames is not None: + if 'frametype' not in self.keys(): + msgs.error('To ignore frames, types must have been defined; run get_frame_types.') + list_ignore_frames = list(ignore_frames.keys()) + msgs.info('Unique configurations ignore frames with type: {0}'.format(list_ignore_frames)) + for ftype in list_ignore_frames: + ignmsk |= self.find_frames(ftype) + # Isolate the frames to be ignored + ignore_indx = ignore_indx[ignmsk] + return ignore_frames, ignore_indx + def find_frames(self, ftype, calib_ID=None, index=False): """ Find the rows with the associated frame type. @@ -1143,7 +1201,7 @@ def find_frames(self, ftype, calib_ID=None, index=False): Args: ftype (str): The frame type identifier. See the keys for - :class:`pypeit.core.framematch.FrameTypeBitMask`. If + :class:`~pypeit.core.framematch.FrameTypeBitMask`. If set to the string 'None', this returns all frames without a known type. calib_ID (:obj:`int`, optional): @@ -1154,7 +1212,7 @@ def find_frames(self, ftype, calib_ID=None, index=False): boolean array. Returns: - numpy.ndarray: A boolean array, or an integer array if + `numpy.ndarray`_: A boolean array, or an integer array if index=True, with the rows that contain the frames of the requested type. @@ -1186,7 +1244,7 @@ def find_frame_files(self, ftype, calib_ID=None): Args: ftype (str): The frame type identifier. See the keys for - :class:`pypeit.core.framematch.FrameTypeBitMask`. + :class:`~pypeit.core.framematch.FrameTypeBitMask`. calib_ID (:obj:`int`, optional): Index of the calibration group that it must match. If None, any row of the specified frame type is included. @@ -1218,7 +1276,7 @@ def set_frame_types(self, type_bits, merge=True): Set and return a Table with the frame types and bits. Args: - type_bits (numpy.ndarray): + type_bits (`numpy.ndarray`_): Integer bitmask with the frame types. The length must match the existing number of table rows. @@ -1227,7 +1285,7 @@ def set_frame_types(self, type_bits, merge=True): will *overwrite* any existing columns. Returns: - `astropy.table.Table`: Table with two columns, the frame + `astropy.table.Table`_: Table with two columns, the frame type name and bits. """ # Making Columns to pad string array @@ -1300,10 +1358,10 @@ def get_frame_types(self, flag_unknown=False, user=None, merge=True): Merge the frame typing into the exiting table. Returns: - :obj:`astropy.table.Table`: A Table with two columns, the - type names and the type bits. See - :class:`pypeit.core.framematch.FrameTypeBitMask` for the - allowed frame types. + `astropy.table.Table`_: A Table with two columns, the type names and + the type bits. See + :class:`~pypeit.core.framematch.FrameTypeBitMask` for the allowed + frame types. """ # Checks if 'frametype' in self.keys() or 'framebit' in self.keys(): @@ -1378,6 +1436,11 @@ def get_frame_types(self, flag_unknown=False, user=None, merge=True): msgs.info(f) if not flag_unknown: msgs.error("Check these files before continuing") + msgs.warn("These files are commented out and will be ignored during the reduction.") + # Comment out the frames that could not be identified + # first change the dtype of the filename column to be able to add a # + self['filename'] = self['filename'].value.astype(f" """ # Columns for output - columns = self.spectrograph.pypeit_file_keys() + ['calib'] + columns = self.spectrograph.pypeit_file_keys() - extras = [] + extras = ['calib'] # comb, bkg columns if write_bkg_pairs: @@ -1740,7 +1803,7 @@ def write(self, output=None, rows=None, columns=None, sort_col=None, overwrite=F file. Returns: - `astropy.table.Table`: The table object that would have been + `astropy.table.Table`_: The table object that would have been written/printed if ``output == 'table'``. Otherwise, the method always returns None. @@ -1850,7 +1913,7 @@ def find_calib_group(self, grp): The calibration group integer. Returns: - numpy.ndarray: Boolean array selecting those frames in the + `numpy.ndarray`_: Boolean array selecting those frames in the table included in the selected calibration group. Raises: diff --git a/pypeit/par/parset.py b/pypeit/par/parset.py index fb50eb6a4f..284917c799 100644 --- a/pypeit/par/parset.py +++ b/pypeit/par/parset.py @@ -317,7 +317,7 @@ def _data_table_string(data_table, delimeter='print'): columns and add a header (first row) and contents delimeter. Args: - data_table (:obj:`numpy.ndarray`): + data_table (`numpy.ndarray`_): Array of string representations of the data to print. Returns: @@ -415,7 +415,7 @@ def config_lines(par, section_name=None, section_comment=None, section_level=0, exclude_defaults=False, include_descr=True): """ Recursively generate the lines of a configuration file based on - the provided ParSet or dict (par). + the provided :class:`ParSet` or :obj:`dict` (see ``par``). Args: section_name (:obj:`str`, optional): @@ -488,8 +488,10 @@ def config_lines(par, section_name=None, section_comment=None, section_level=0, except: pass if not exclude_defaults or par[k] != par.default[k]: - lines += [ component_indent + k + ' = ' + ParSet._data_string(par[k]) ] - + argvalue = ParSet._data_string(par[k]) + if isinstance(par[k], list): + argvalue += ',' + lines += [ component_indent + k + ' = ' + argvalue ] # Then add the items that are ParSets as subsections for k in parset_keys: section_comment = None @@ -618,23 +620,22 @@ def to_config(self, cfg_file=None, section_name=None, section_comment=None, sect Args: cfg_file (:obj:`str`, optional): - The name of the file to write/append to. If None - (default), the function will just return the list of - strings that would have been written to the file. These - lines can be used to construct a :class:`ConfigObj` - instance. + The name of the file to write/append to. If None (default), the + function will just return the list of strings that would have + been written to the file. These lines can be used to construct + a `configobj`_ instance. section_name (:obj:`str`, optional): The top-level name for the config section. This must be provided if :attr:`cfg_section` is None or any of the - parameters are not also ParSet instances themselves. + parameters are not also :class:`ParSet` instances themselves. section_comment (:obj:`str`, optional): The top-level comment for the config section based on - this ParSet. + this :class:`ParSet`. section_level (:obj:`int`, optional): - The top level of this ParSet. Used for recursive output - of nested ParSets. + The top level of this :class:`ParSet`. Used for recursive output + of nested :class:`ParSet` instances. append (:obj:`bool`, optional): - Append this configuration output of this ParSet to the + Append this configuration output of this :class:`ParSet` to the file. False by default. If not appending and the file exists, the file is automatically overwritten. quiet (:obj:`bool`, optional): @@ -647,7 +648,7 @@ def to_config(self, cfg_file=None, section_name=None, section_comment=None, sect Raises: ValueError: - Raised if there are types other than ParSet in the + Raised if there are types other than :class:`ParSet` in the parameter list, :attr:`cfg_section` is None, and no section_name argument was provided. """ @@ -769,7 +770,7 @@ def to_header(self, hdr=None, prefix=None, quiet=False): """ Write the parameters to a fits header. - Any element that has a value of None or is a ParSet itself is + Any element that has a value of None or is a :class:`ParSet` itself is *not* written to the header. Args: @@ -819,10 +820,9 @@ def to_header(self, hdr=None, prefix=None, quiet=False): @classmethod def from_header(cls, hdr, prefix=None): """ - Instantiate the ParSet using data parsed from a fits header. + Instantiate the :class:`ParSet` using data parsed from a fits header. - This is a simple wrapper for - :func:`ParSet.parse_par_from_hdr` and + This is a simple wrapper for :func:`ParSet.parse_par_from_hdr` and :func:`ParSet.from_dict`. .. warning:: diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 0eb919114b..3dd0556bfb 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -40,9 +40,9 @@ def __init__(self, existing_par=None, foo=None): for pk in parkeys: kwargs[pk] = cfg[pk] if pk in k else None - - If the parameter is another ParSet or requires instantiation, - provide the instantiation. For example, see how the - :class:`ProcessImagesPar` parameter set is defined in the + - If the parameter is another :class:`~pypeit.par.parset.ParSet` or + requires instantiation, provide the instantiation. For example, see + how the :class:`ProcessImagesPar` parameter set is defined in the :class:`FrameGroupPar` class. E.g.:: pk = 'foo' @@ -54,8 +54,6 @@ def __init__(self, existing_par=None, foo=None): sets as a template, then add the parameter set to :class:`PypeItPar`, assuming you want it to be accessed throughout the code. ----- - .. include common links, assuming primary doc root is up one directory .. include:: ../include/links.rst @@ -201,8 +199,8 @@ class ProcessImagesPar(ParSet): The parameters needed to perform basic image processing. These parameters are primarily used by - :class:`pypeit.processimages.ProcessImages`, the base class of many - of the pypeit objects. + :func:`~pypeit.images.buildimage.buildimage_fromlist`, the main function + used to process and combine images. For a table with the current keywords, defaults, and descriptions, see :ref:`parameters`. @@ -939,7 +937,7 @@ def __init__(self, locations=None, trace_npoly=None, trim_edge=None, snr_thresh= # Fill out parameter specifications. Only the values that are # *not* None (i.e., the ones that are defined) need to be set - defaults['locations'] = [0.0, 0.5, 1.0] + defaults['locations'] = [0.0, 1.0] dtypes['locations'] = [list, np.ndarray] descr['locations'] = 'Locations of the bars, in a list, specified as a fraction of the slit width' @@ -996,9 +994,10 @@ class Coadd1DPar(ParSet): see :ref:`parameters`. """ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, - sn_smooth_npix=None, wave_method=None, dv=None, wave_grid_min=None, wave_grid_max=None, spec_samp_fact=None, ref_percentile=None, maxiter_scale=None, + sn_smooth_npix=None, sigrej_exp=None, wave_method=None, dv=None, dwave=None, dloglam=None, + wave_grid_min=None, wave_grid_max=None, spec_samp_fact=None, ref_percentile=None, maxiter_scale=None, sigrej_scale=None, scale_method=None, sn_min_medscale=None, sn_min_polyscale=None, maxiter_reject=None, - lower=None, upper=None, maxrej=None, sn_clip=None, nbest=None, sensfuncfile=None, coaddfile=None, + lower=None, upper=None, maxrej=None, sn_clip=None, nbests=None, coaddfile=None, mag_type=None, filter=None, filter_mag=None, filter_mask=None, chk_version=None): # Grab the parameter names and values from the function @@ -1030,7 +1029,6 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, dtypes['nmaskedge'] = int descr['nmaskedge'] = 'Number of edge pixels to mask. This should be removed/fixed.' - # Offsets defaults['sn_smooth_npix'] = None dtypes['sn_smooth_npix'] = [int, float] descr['sn_smooth_npix'] = 'Number of pixels to median filter by when computing S/N used to decide how to scale ' \ @@ -1038,8 +1036,12 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, 'number of good pixels per spectrum in the stack that is being co-added and use 10% of ' \ 'this neff.' + defaults['sigrej_exp'] = None + dtypes['sigrej_exp'] = [int, float] + descr['sigrej_exp'] = 'Rejection threshold used for rejecting exposures with S/N more than sigrej_exp*sigma ' \ + 'above the median S/N. If None (the default), no rejection is performed. Currently, ' \ + 'only available for multi-slit observations.' \ - # Offsets defaults['wave_method'] = 'linear' dtypes['wave_method'] = str descr['wave_method'] = "Method used to construct wavelength grid for coadding spectra. The routine that creates " \ @@ -1056,6 +1058,16 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, descr['dv'] = "Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), " \ "otherwise a median value is computed from the data." + defaults['dwave'] = None + dtypes['dwave'] = [int, float] + descr['dwave'] = "Dispersion in Angstroms in case you want to specify it in the get_wave_grid (for the 'linear' option), " \ + "otherwise a median value is computed from the data." + + defaults['dloglam'] = None + dtypes['dloglam'] = [int, float] + descr['dloglam'] = "Dispersion in units of log10(wave) in case you want to specify it in the get_wave_grid (for the 'velocity' or 'log10' options), " \ + "otherwise a median value is computed from the data." + defaults['wave_grid_min'] = None dtypes['wave_grid_min'] = [int, float] descr['wave_grid_min'] = "Used in case you want to specify the minimum wavelength in your wavelength grid, default=None computes from data" @@ -1130,10 +1142,10 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, 'prevents overly aggressive rejection in high S/N ratio spectrum which neverthless differ ' \ 'at a level greater than the formal S/N due to systematics.' - defaults['nbest'] = None - dtypes['nbest'] = int - descr['nbest'] = 'Number of orders to use for estimating the per exposure weights. Default is None, ' \ - 'which will just use one fourth of the total number of orders. This is only used for Echelle.' + defaults['nbests'] = None + dtypes['nbests'] = [list, int] + descr['nbests'] = 'Number of orders to use for estimating the per exposure weights. Default is None, ' \ + 'which will just use one fourth of the total number of orders. This is only used for Echelle' # For scaling to an input filter magnitude defaults['filter'] = 'none' @@ -1153,14 +1165,6 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, descr['filter_mask'] = 'List of wavelength regions to mask when doing the scaling (`i.e.`, occasional junk pixels). '\ 'Colon and comma separateed, e.g. 5552:5559,6010:6030' - - # JFH These last two are actually arguments and not parameters that are only here because there is no other easy - # way to parse .coadd1d files except with parsets. I would like to separate arguments from parameters. - defaults['sensfuncfile'] = None - dtypes['sensfuncfile'] = str - descr['sensfuncfile'] = 'File containing sensitivity function which is a requirement for echelle coadds. ' \ - 'This is only used for Echelle.' - defaults['coaddfile'] = None dtypes['coaddfile'] = str descr['coaddfile'] = 'Output filename' @@ -1183,10 +1187,11 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, @classmethod def from_dict(cls, cfg): k = np.array([*cfg.keys()]) - parkeys = ['ex_value', 'flux_value', 'nmaskedge', 'sn_smooth_npix', 'wave_method', 'dv', 'wave_grid_min', 'wave_grid_max', + parkeys = ['ex_value', 'flux_value', 'nmaskedge', 'sn_smooth_npix', 'sigrej_exp', + 'wave_method', 'dv', 'dwave', 'dloglam', 'wave_grid_min', 'wave_grid_max', 'spec_samp_fact', 'ref_percentile', 'maxiter_scale', 'sigrej_scale', 'scale_method', 'sn_min_medscale', 'sn_min_polyscale', 'maxiter_reject', 'lower', 'upper', - 'maxrej', 'sn_clip', 'nbest', 'sensfuncfile', 'coaddfile', 'chk_version', + 'maxrej', 'sn_clip', 'nbests', 'coaddfile', 'chk_version', 'filter', 'mag_type', 'filter_mag', 'filter_mask'] badkeys = np.array([pk not in parkeys for pk in k]) @@ -1219,8 +1224,8 @@ class Coadd2DPar(ParSet): For a table with the current keywords, defaults, and descriptions, see :ref:`parameters`. """ - def __init__(self, only_slits=None, offsets=None, spat_toler=None, weights=None, user_obj=None, - use_slits4wvgrid=None, manual=None): + def __init__(self, only_slits=None, exclude_slits=None, offsets=None, spat_toler=None, weights=None, user_obj=None, + use_slits4wvgrid=None, manual=None, wave_method=None): # Grab the parameter names and values from the function # arguments @@ -1233,18 +1238,24 @@ def __init__(self, only_slits=None, offsets=None, spat_toler=None, weights=None, dtypes = OrderedDict.fromkeys(pars.keys()) descr = OrderedDict.fromkeys(pars.keys()) - # Offsets defaults['only_slits'] = None - dtypes['only_slits'] = [int, list] - descr['only_slits'] = 'Slit ID, or list of slit IDs that the user want to restrict the coadd to. ' \ - 'I.e., only this/these slit/s will be coadded.' + dtypes['only_slits'] = [str, list] + descr['only_slits'] = 'Restrict coaddition to one or more of slits. Example syntax -- ' \ + 'DET01:175,DET02:205 or MSC02:2234. This and ``exclude_slits`` ' \ + 'are mutually exclusive. If both are provided, ``only_slits`` takes precedence.' + + defaults['exclude_slits'] = None + dtypes['exclude_slits'] = [str, list] + descr['exclude_slits'] = 'Exclude one or more slits from the coaddition. Example syntax -- ' \ + 'DET01:175,DET02:205 or MSC02:2234. This and ``only_slits`` ' \ + 'are mutually exclusive. If both are provided, ``only_slits`` takes precedence.' defaults['offsets'] = 'auto' dtypes['offsets'] = [str, list] descr['offsets'] = 'Offsets for the images being combined (spat pixels). Options are: ' \ '``maskdef_offsets``, ``header``, ``auto``, and a list of offsets. ' \ - 'Use ``maskdef_offsets`` to use the offsets computed during the slitmask ' \ - 'design matching (currently available for DEIMOS and MOSFIRE only). If equal ' \ + 'Use ``maskdef_offsets`` to use the offsets computed during the slitmask design matching ' \ + '(currently available for these :ref:`slitmask_info_instruments` only). If equal ' \ 'to ``header``, the dither offsets recorded in the header, when available, will be used. ' \ 'If ``auto`` is chosen, PypeIt will try to compute the offsets using a reference object ' \ 'with the highest S/N, or an object selected by the user (see ``user_obj``). ' \ @@ -1274,13 +1285,14 @@ def __init__(self, only_slits=None, offsets=None, spat_toler=None, weights=None, defaults['user_obj'] = None dtypes['user_obj'] = [int, list] descr['user_obj'] = 'Object that the user wants to use to compute the weights and/or the ' \ - 'offsets for coadding images. For slit spectroscopy, provide the ' \ + 'offsets for coadding images. For longslit/multislit spectroscopy, provide the ' \ '``SLITID`` and the ``OBJID``, separated by comma, of the selected object. ' \ 'For echelle spectroscopy, provide the ``ECH_OBJID`` of the selected object. ' \ 'See :doc:`out_spec1D` for more info about ``SLITID``, ``OBJID`` and ``ECH_OBJID``. ' \ 'If this parameter is not ``None``, it will be used to compute the offsets ' \ 'only if ``offsets = auto``, and it will used to compute ' \ 'the weights only if ``weights = auto``.' + # TODO For echelle coadds this should just default to 1 # manual extraction defaults['manual'] = None @@ -1290,18 +1302,34 @@ def __init__(self, only_slits=None, offsets=None, spat_toler=None, weights=None, 'and spat,spec are in the pseudo-image generated by COADD2D.' \ 'boxcar_radius is optional and in pixels (not arcsec!).' + # wave method + defaults['wave_method'] = None + dtypes['wave_method'] = str + descr['wave_method'] = "Argument to :func:`~pypeit.core.wavecal.wvutils.get_wave_grid` method, which determines how " \ + "the 2d coadd wavelength grid is constructed. The default is None, which will use a linear grid" \ + "for longslit/multislit coadds and a log10 grid for echelle coadds. " \ + "Currently supported options with 2d coadding are:" \ + "* 'iref' -- Use one of the exposures (the first) as the reference for the wavelength grid " \ + "* 'velocity' -- Grid is uniform in velocity" \ + "* 'log10' -- Grid is uniform in log10(wave). This is the same as velocity." \ + "* 'linear' -- Grid is uniform in wavelength" \ + + # Instantiate the parameter set super(Coadd2DPar, self).__init__(list(pars.keys()), values=list(pars.values()), defaults=list(defaults.values()), dtypes=list(dtypes.values()), descr=list(descr.values())) + + self.validate() @classmethod def from_dict(cls, cfg): k = np.array([*cfg.keys()]) - parkeys = ['only_slits', 'offsets', 'spat_toler', 'weights', 'user_obj', 'use_slits4wvgrid', 'manual'] + parkeys = ['only_slits', 'exclude_slits', 'offsets', 'spat_toler', 'weights', 'user_obj', 'use_slits4wvgrid', + 'manual', 'wave_method'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -1312,11 +1340,15 @@ def from_dict(cls, cfg): kwargs[pk] = cfg[pk] if pk in k else None return cls(**kwargs) + def validate(self): """ Check the parameters are valid for the provided method. """ - pass + allowed_wave_methods = ['iref', 'velocity', 'log10', 'linear'] + if self.data['wave_method'] is not None and self.data['wave_method'] not in allowed_wave_methods: + raise ValueError("If 'wave_method' is not None it must be one of:\n"+", ".join(allowed_wave_methods)) + class CubePar(ParSet): @@ -1328,8 +1360,8 @@ class CubePar(ParSet): see :ref:`parameters`. """ - def __init__(self, slit_spec=None, relative_weights=None, combine=None, output_filename=None, - standard_cube=None, reference_image=None, save_whitelight=None, method=None, + def __init__(self, slit_spec=None, relative_weights=None, align=None, combine=None, output_filename=None, + standard_cube=None, reference_image=None, save_whitelight=None, whitelight_range=None, method=None, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, spatial_delta=None, wave_delta=None, astrometric=None, grating_corr=None, scale_corr=None, skysub_frame=None, spec_subpixel=None, spat_subpixel=None): @@ -1362,6 +1394,14 @@ def __init__(self, slit_spec=None, relative_weights=None, combine=None, output_f 'view of all input observations, and is generally only required if high ' \ 'relative precision is desired.' + defaults['align'] = False + dtypes['align'] = [bool] + descr['align'] = 'If set to True, the input frames will be spatially aligned by cross-correlating the ' \ + 'whitelight images with either a reference image (see ``reference_image``) or the whitelight ' \ + 'image that is generated using the first spec2d listed in the coadd3d file. Alternatively, ' \ + 'the user can specify the offsets (i.e. Delta RA x cos(dec) and Delta Dec, both in arcsec) ' \ + 'in the spec2d block of the coadd3d file. See the documentation for examples of this usage.' + defaults['combine'] = False dtypes['combine'] = [bool] descr['combine'] = 'If set to True, the input frames will be combined. Otherwise, a separate ' \ @@ -1372,7 +1412,7 @@ def __init__(self, slit_spec=None, relative_weights=None, combine=None, output_f dtypes['output_filename'] = str descr['output_filename'] = 'If combining multiple frames, this string sets the output filename of ' \ 'the combined datacube. If combine=False, the output filenames will be ' \ - 'prefixed with "spec3d_*"' + 'prefixed with ``spec3d_*``' defaults['standard_cube'] = None dtypes['standard_cube'] = str @@ -1394,8 +1434,19 @@ def __init__(self, slit_spec=None, relative_weights=None, combine=None, output_f 'will be given by the "output_filename" variable with a suffix "_whitelight". ' \ 'Note that the white light image collapses the flux along the wavelength axis, ' \ 'so some spaxels in the 2D white light image may have different wavelength ' \ - 'ranges. If combine=False, the individual spec3d files will have a suffix ' \ - '"_whitelight".' + 'ranges. To set the wavelength range, use the "whitelight_range" parameter. ' \ + 'If combine=False, the individual spec3d files will have a suffix "_whitelight".' + + defaults['whitelight_range'] = [None, None] + dtypes['whitelight_range'] = list + descr['whitelight_range'] = 'A two element list specifying the wavelength range over which to generate the ' \ + 'white light image. The first (second) element is the minimum (maximum) ' \ + 'wavelength to use. If either of these elements are None, PypeIt will ' \ + 'automatically use a wavelength range that ensures all spaxels have the ' \ + 'same wavelength coverage. Note, if you are using a reference_image to align ' \ + 'all frames, it is preferable to use the same white light wavelength range ' \ + 'for all white light images. For example, you may wish to use an emission ' \ + 'line map to register two frames.' \ defaults['method'] = "subpixel" dtypes['method'] = str @@ -1501,7 +1552,8 @@ def __init__(self, slit_spec=None, relative_weights=None, combine=None, output_f 'for the sky subtraction, specify the relative path+file to the spec2D file that you ' \ 'would like to use for the sky subtraction. The model fit to the sky of the specified ' \ 'frame will be used. Note, the sky and science frames do not need to have the same ' \ - 'exposure time.' + 'exposure time; the sky model will be scaled to the science frame based on the ' \ + 'relative exposure time.' # Instantiate the parameter set super(CubePar, self).__init__(list(pars.keys()), @@ -1519,8 +1571,8 @@ def from_dict(cls, cfg): # Basic keywords parkeys = ['slit_spec', 'output_filename', 'standard_cube', 'reference_image', 'save_whitelight', 'method', 'spec_subpixel', 'spat_subpixel', 'ra_min', 'ra_max', 'dec_min', 'dec_max', - 'wave_min', 'wave_max', 'spatial_delta', 'wave_delta', 'relative_weights', 'combine', - 'astrometric', 'grating_corr', 'scale_corr', 'skysub_frame'] + 'wave_min', 'wave_max', 'spatial_delta', 'wave_delta', 'relative_weights', 'align', 'combine', + 'astrometric', 'grating_corr', 'scale_corr', 'skysub_frame', 'whitelight_range'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -1535,6 +1587,8 @@ def validate(self): allowed_methods = ["subpixel", "NGP"]#, "resample" if self.data['method'] not in allowed_methods: raise ValueError("The 'method' must be one of:\n"+", ".join(allowed_methods)) + if len(self.data['whitelight_range']) != 2: + raise ValueError("The 'whitelight_range' must be a two element list of either NoneType or float") class FluxCalibratePar(ParSet): @@ -1628,7 +1682,7 @@ class SensFuncPar(ParSet): For a table with the current keywords, defaults, and descriptions, see :ref:`parameters`. """ - def __init__(self, extrap_blu=None, extrap_red=None, samp_fact=None, multi_spec_det=None, algorithm=None, UVIS=None, + def __init__(self, flatfile=None, extrap_blu=None, extrap_red=None, samp_fact=None, multi_spec_det=None, algorithm=None, UVIS=None, IR=None, polyorder=None, star_type=None, star_mag=None, star_ra=None, star_dec=None, mask_hydrogen_lines=None, mask_helium_lines=None, hydrogen_mask_wid=None): # Grab the parameter names and values from the function arguments @@ -1641,6 +1695,12 @@ def __init__(self, extrap_blu=None, extrap_red=None, samp_fact=None, multi_spec_ dtypes = OrderedDict.fromkeys(pars.keys()) descr = OrderedDict.fromkeys(pars.keys()) + defaults['flatfile'] = None + dtypes['flatfile'] = str + descr['flatfile'] = 'Flat field file to be used if the sensitivity function model will utilize the blaze ' \ + 'function computed from a flat field file in the Calibrations directory, e.g.' \ + 'Calibrations/Flat_A_0_DET01.fits' + defaults['extrap_blu'] = 0.1 dtypes['extrap_blu'] = float descr['extrap_blu'] = 'Fraction of minimum wavelength coverage to grow the wavelength coverage of the ' \ @@ -1737,7 +1797,7 @@ def from_dict(cls, cfg): k = np.array([*cfg.keys()]) # Single element parameters - parkeys = ['extrap_blu', 'extrap_red', 'samp_fact', 'multi_spec_det', 'algorithm', + parkeys = ['flatfile', 'extrap_blu', 'extrap_red', 'samp_fact', 'multi_spec_det', 'algorithm', 'polyorder', 'star_type', 'star_mag', 'star_ra', 'star_dec', 'mask_hydrogen_lines', 'mask_helium_lines', 'hydrogen_mask_wid'] @@ -1956,8 +2016,8 @@ def __init__(self, obj_toler=None, assign_obj=None, snr_thrshd=None, defaults['bright_maskdef_id'] = None dtypes['bright_maskdef_id'] = int - descr['bright_maskdef_id'] = '`maskdef_id` (corresponding to `dSlitId` and `Slit_Number` in the DEIMOS ' \ - 'and MOSFIRE slitmask design, respectively) of a ' \ + descr['bright_maskdef_id'] = '`maskdef_id` (corresponding e.g., to `dSlitId` and `Slit_Number` ' \ + 'in the DEIMOS/LRIS and MOSFIRE slitmask design, respectively) of a ' \ 'slit containing a bright object that will be used to compute the ' \ 'slitmask offset. This parameter is optional and is ignored ' \ 'if ``slitmask_offset`` is provided.' @@ -2090,7 +2150,7 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ dtypes['minmax_coeff_bounds'] = tuple descr['minmax_coeff_bounds'] = "Parameters setting the polynomial coefficient bounds for sensfunc optimization. " \ "Bounds are currently determined as follows. We compute an initial fit to the " \ - "sensfunc in the pypeit.core.telluric.init_sensfunc_model function. That deterines " \ + "sensfunc in the :func:`~pypeit.core.telluric.init_sensfunc_model` function. That deterines " \ "a set of coefficients. The bounds are then determined according to: " \ "[(np.fmin(np.abs(this_coeff)*obj_params['delta_coeff_bounds'][0], " \ "obj_params['minmax_coeff_bounds'][0]), " \ @@ -2354,7 +2414,7 @@ def __init__(self, spectrograph=None, detnum=None, sortroot=None, calwin=None, s 'Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of ' \ 'tuples of the detector indices that are mosaiced together. ' \ 'E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for ' \ - 'Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]``' + 'Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]``' dtypes['slitspatnum'] = [str, list] descr['slitspatnum'] = 'Restrict reduction to a set of slit DET:SPAT values (closest slit is used). ' \ @@ -2445,13 +2505,13 @@ class WavelengthSolutionPar(ParSet): see :ref:`parameters`. """ def __init__(self, reference=None, method=None, echelle=None, ech_nspec_coeff=None, ech_norder_coeff=None, ech_sigrej=None, lamps=None, - sigdetect=None, fwhm=None, fwhm_fromlines=None, reid_arxiv=None, - nreid_min=None, cc_thresh=None, cc_local_thresh=None, nlocal_cc=None, + sigdetect=None, fwhm=None, fwhm_fromlines=None, fwhm_spat_order=None, fwhm_spec_order=None, + reid_arxiv=None, nreid_min=None, cc_thresh=None, cc_local_thresh=None, nlocal_cc=None, rms_threshold=None, match_toler=None, func=None, n_first=None, n_final=None, sigrej_first=None, sigrej_final=None, numsearch=None, nfitpix=None, refframe=None, nsnippet=None, use_instr_flag=None, wvrng_arxiv=None, - ech_separate_2d=None): + ech_separate_2d=None, redo_slits=None, qa_log=None): # Grab the parameter names and values from the function # arguments @@ -2554,7 +2614,7 @@ def __init__(self, reference=None, method=None, echelle=None, ech_nspec_coeff=No # 'be accurately centroided' defaults['sigdetect'] = 5. - dtypes['sigdetect'] = [int, float, list, np.ndarray] + dtypes['sigdetect'] = [int, float, list, np.ndarray] descr['sigdetect'] = 'Sigma threshold above fluctuations for arc-line detection. Arcs ' \ 'are continuum subtracted and the fluctuations are computed after ' \ 'continuum subtraction. This can be a single number or a vector ' \ @@ -2573,8 +2633,18 @@ def __init__(self, reference=None, method=None, echelle=None, ech_nspec_coeff=No 'the determination of the wavelength solution (`i.e.`, not in '\ 'WaveTilts).' + defaults['fwhm_spat_order'] = 0 + dtypes['fwhm_spat_order'] = int + descr['fwhm_spat_order'] = 'This parameter determines the spatial polynomial order to use in the ' \ + '2D polynomial fit to the FWHM of the arc lines. See also, fwhm_spec_order.' + + defaults['fwhm_spec_order'] = 1 + dtypes['fwhm_spec_order'] = int + descr['fwhm_spec_order'] = 'This parameter determines the spectral polynomial order to use in the ' \ + '2D polynomial fit to the FWHM of the arc lines. See also, fwhm_spat_order.' + # These are the parameters used for reidentification - defaults['reid_arxiv']=None + defaults['reid_arxiv'] = None dtypes['reid_arxiv'] = str descr['reid_arxiv'] = 'Name of the archival wavelength solution file that will be used ' \ 'for the wavelength reidentification. Only used if ``method`` is ' \ @@ -2629,10 +2699,11 @@ def __init__(self, reference=None, method=None, echelle=None, ech_nspec_coeff=No # These are the parameters used for the iterative fitting of the arc lines defaults['rms_threshold'] = 0.15 - dtypes['rms_threshold'] = [float, list, np.ndarray] - descr['rms_threshold'] = 'Minimum RMS for keeping a slit/order solution. This can be a ' \ - 'single number or a list/array providing the value for each slit. ' \ - 'Only used if ``method`` is either \'holy-grail\' or \'reidentify\'' + dtypes['rms_threshold'] = float + descr['rms_threshold'] = 'Maximum RMS (in binned pixels) for keeping a slit/order solution. ' \ + 'Used for echelle spectrographs, the \'reidentify\' method, and when re-analyzing a slit with the redo_slits parameter.' \ + 'In a future PR, we will refactor the code to always scale this threshold off the measured FWHM of the arc lines.' + defaults['match_toler'] = 2.0 dtypes['match_toler'] = float @@ -2687,6 +2758,17 @@ def __init__(self, reference=None, method=None, echelle=None, ech_nspec_coeff=No descr['refframe'] = 'Frame of reference for the wavelength calibration. ' \ 'Options are: {0}'.format(', '.join(options['refframe'])) + dtypes['redo_slits'] = [int, list] + descr['redo_slits'] = 'Redo the input slit(s) [multislit] or order(s) [echelle]' + + defaults['qa_log'] = True + dtypes['qa_log'] = bool + descr['qa_log'] = 'Governs whether the wavelength solution arc line QA plots will have log or linear scaling'\ + 'If True, the scaling will be log, if False linear' + + + + # Instantiate the parameter set super(WavelengthSolutionPar, self).__init__(list(pars.keys()), values=list(pars.values()), @@ -2701,11 +2783,12 @@ def from_dict(cls, cfg): k = np.array([*cfg.keys()]) parkeys = ['reference', 'method', 'echelle', 'ech_nspec_coeff', 'ech_norder_coeff', 'ech_sigrej', 'ech_separate_2d', 'lamps', 'sigdetect', - 'fwhm', 'fwhm_fromlines', 'reid_arxiv', 'nreid_min', 'cc_thresh', 'cc_local_thresh', + 'fwhm', 'fwhm_fromlines', 'fwhm_spat_order', 'fwhm_spec_order', + 'reid_arxiv', 'nreid_min', 'cc_thresh', 'cc_local_thresh', 'nlocal_cc', 'rms_threshold', 'match_toler', 'func', 'n_first','n_final', 'sigrej_first', 'sigrej_final', 'numsearch', 'nfitpix', - 'refframe', 'nsnippet', 'use_instr_flag', - 'wvrng_arxiv'] + 'refframe', 'nsnippet', 'use_instr_flag', 'wvrng_arxiv', + 'redo_slits', 'qa_log'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -2736,7 +2819,7 @@ def valid_lamps(): """ Return the valid lamp ions """ - return ['ArI', 'CdI', 'HgI', 'HeI', 'KrI', 'NeI', 'XeI', 'ZnI', 'ThAr'] + return ['ArI', 'CdI', 'HgI', 'HeI', 'KrI', 'NeI', 'XeI', 'ZnI', 'ThAr', 'FeAr'] @staticmethod def valid_media(): @@ -2774,13 +2857,12 @@ def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enha trace_median_frac=None, trace_thresh=None, fwhm_uniform=None, niter_uniform=None, fwhm_gaussian=None, niter_gaussian=None, det_buffer=None, max_nudge=None, sync_predict=None, sync_center=None, gap_offset=None, sync_to_edge=None, - bound_detector=None, minimum_slit_dlength=None, dlength_range=None, + bound_detector=None, minimum_slit_dlength=None, dlength_range=None, minimum_slit_length=None, minimum_slit_length_sci=None, length_range=None, minimum_slit_gap=None, clip=None, order_match=None, - order_offset=None, overlap=None, use_maskdesign=None, maskdesign_maxsep=None, + order_offset=None, add_missed_orders=None, overlap=None, use_maskdesign=None, maskdesign_maxsep=None, maskdesign_step=None, maskdesign_sigrej=None, pad=None, add_slits=None, - add_predict=None, rm_slits=None, - maskdesign_filename=None): + add_predict=None, rm_slits=None, maskdesign_filename=None): # Grab the parameter names and values from the function # arguments @@ -2882,7 +2964,7 @@ def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enha defaults['fit_niter'] = 1 dtypes['fit_niter'] = int descr['fit_niter'] = 'Number of iterations of re-measuring and re-fitting the edge ' \ - 'data; see :func:`pypeit.core.trace.fit_trace`.' + 'data; see :func:`~pypeit.core.trace.fit_trace`.' # TODO: Allow this to be a list so that it can be detector specific? defaults['fit_min_spec_length'] = 0.6 @@ -2981,27 +3063,27 @@ def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enha defaults['fwhm_uniform'] = 3.0 dtypes['fwhm_uniform'] = [int, float] descr['fwhm_uniform'] = 'The `fwhm` parameter to use when using uniform weighting in ' \ - ':func:`pypeit.core.trace.fit_trace` when refining the PCA ' \ + ':func:`~pypeit.core.trace.fit_trace` when refining the PCA ' \ 'predictions of edges. See description of ' \ - ':func:`pypeit.core.trace.peak_trace`.' + ':func:`~pypeit.core.trace.peak_trace`.' defaults['niter_uniform'] = 9 dtypes['niter_uniform'] = int descr['niter_uniform'] = 'The number of iterations of ' \ - ':func:`pypeit.core.trace.fit_trace` to use when using ' \ + ':func:`~pypeit.core.trace.fit_trace` to use when using ' \ 'uniform weighting.' defaults['fwhm_gaussian'] = 3.0 dtypes['fwhm_gaussian'] = [int, float] descr['fwhm_gaussian'] = 'The `fwhm` parameter to use when using Gaussian weighting in ' \ - ':func:`pypeit.core.trace.fit_trace` when refining the PCA ' \ + ':func:`~pypeit.core.trace.fit_trace` when refining the PCA ' \ 'predictions of edges. See description ' \ - ':func:`pypeit.core.trace.peak_trace`.' + ':func:`~pypeit.core.trace.peak_trace`.' defaults['niter_gaussian'] = 6 dtypes['niter_gaussian'] = int descr['niter_gaussian'] = 'The number of iterations of ' \ - ':func:`pypeit.core.trace.fit_trace` to use when using ' \ + ':func:`~pypeit.core.trace.fit_trace` to use when using ' \ 'Gaussian weighting.' defaults['det_buffer'] = 5 @@ -3139,6 +3221,12 @@ def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enha 'fraction of the detector spatial scale. If None, no offset ' \ 'is applied.' + defaults['add_missed_orders'] = False + dtypes['add_missed_orders'] = bool + descr['add_missed_orders'] = 'If orders are not detected by the automated edge tracing, ' \ + 'attempt to add them based on their expected positions on ' \ + 'on the detector. Echelle spectrographs only.' + defaults['overlap'] = False dtypes['overlap'] = bool descr['overlap'] = 'Assume slits identified as abnormally short are actually due to ' \ @@ -3250,12 +3338,11 @@ def from_dict(cls, cfg): 'smash_range', 'edge_detect_clip', 'trace_median_frac', 'trace_thresh', 'fwhm_uniform', 'niter_uniform', 'fwhm_gaussian', 'niter_gaussian', 'det_buffer', 'max_nudge', 'sync_predict', 'sync_center', 'gap_offset', - 'sync_to_edge', 'bound_detector', 'minimum_slit_dlength', 'dlength_range', + 'sync_to_edge', 'bound_detector', 'minimum_slit_dlength', 'dlength_range', 'minimum_slit_length', 'minimum_slit_length_sci', 'length_range', - 'minimum_slit_gap', 'clip', 'order_match', 'order_offset', 'overlap', - 'use_maskdesign', 'maskdesign_maxsep', 'maskdesign_step', - 'maskdesign_sigrej', 'maskdesign_filename', - 'pad', 'add_slits', 'add_predict', 'rm_slits'] + 'minimum_slit_gap', 'clip', 'order_match', 'order_offset', 'add_missed_orders', + 'overlap', 'use_maskdesign', 'maskdesign_maxsep', 'maskdesign_step', 'maskdesign_sigrej', + 'maskdesign_filename', 'pad', 'add_slits', 'add_predict', 'rm_slits'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -3367,8 +3454,8 @@ def __init__(self, idsonly=None, tracethresh=None, sig_neigh=None, nfwhm_neigh=N defaults['spat_order'] = 3 dtypes['spat_order'] = [int, float, list, np.ndarray] - descr['spat_order'] = 'Order of the legendre polynomial to be fit to the the tilt of an arc line. This parameter determines ' \ - 'both the orer of the *individual* arc line tilts, as well as the order of the spatial direction of the ' \ + descr['spat_order'] = 'Order of the legendre polynomial to be fit to the tilt of an arc line. This parameter determines ' \ + 'both the order of the *individual* arc line tilts, as well as the order of the spatial direction of the ' \ '2d legendre polynomial (spatial, spectral) that is fit to obtain a global solution for the tilts across the ' \ 'slit/order. This can be a single number or a list/array providing the value for each slit' @@ -3688,7 +3775,7 @@ def __init__(self, trace_npoly=None, snr_thresh=None, find_trim_edge=None, defaults['find_min_max'] = None dtypes['find_min_max'] = list - descr['find_min_max'] = 'It defines the minimum and maximum of your object in the spectral direction on the ' \ + descr['find_min_max'] = 'It defines the minimum and maximum of your object in pixels in the spectral direction on the ' \ 'detector. It only used for object finding. This parameter is helpful if your object only ' \ 'has emission lines or at high redshift and the trace only shows in part of the detector.' @@ -3807,7 +3894,9 @@ def __init__(self, bspline_spacing=None, sky_sigrej=None, global_sky_std=None, n defaults['joint_fit'] = False dtypes['joint_fit'] = bool descr['joint_fit'] = 'Perform a simultaneous joint fit to sky regions using all available slits. ' \ - 'Currently, this parameter is only used for IFU data reduction.' + 'Currently, this parameter is only used for IFU data reduction. Note that the ' \ + 'current implementation does not account for variations in the instrument FWHM ' \ + 'in different slits. This will be addressed by Issue #1660.' defaults['max_mask_frac'] = 0.80 dtypes['max_mask_frac'] = float @@ -3990,8 +4079,8 @@ def __init__(self, calib_dir=None, bpm_usebias=None, biasframe=None, darkframe=N dtypes['calib_dir'] = str descr['calib_dir'] = 'The name of the directory for the processed calibration frames. ' \ 'The host path for the directory is set by the redux_path (see ' \ - ':class:`ReduxPar`). Beware that success when changing the ' \ - 'default value is not well tested!' + ':class:`~pypeit.par.pypeitpar.ReduxPar`). Beware that success ' \ + 'when changing the default value is not well tested!' defaults['raise_chk_error'] = True dtypes['raise_chk_error'] = bool @@ -4216,7 +4305,7 @@ class PypeItPar(ParSet): merge_with=user_cfg_lines) To write the configuration of a given instance of :class:`PypeItPar`, - use the :func:`to_config` function:: + use the :func:`~pypeit.par.parset.ParSet.to_config` function:: par.to_config('mypypeitpar.cfg') @@ -4349,7 +4438,7 @@ def from_cfg_file(cls, cfg_file=None, merge_with=None, evaluate=True): When `evaluate` is true, the function runs `eval()` on all the entries in the `ConfigObj` dictionary, done using - :func:`_recursive_dict_evaluate`. This has the potential to + :func:`~pypeit.par.util.recursive_dict_evaluate`. This has the potential to go haywire if the name of a parameter unintentionally happens to be identical to an imported or system-level function. Of course, this can be useful by allowing one to @@ -4357,13 +4446,13 @@ def from_cfg_file(cls, cfg_file=None, merge_with=None, evaluate=True): one has to be careful with the values that the parameters should be allowed to have. The current way around this is to provide a list of strings that should be ignored during - the evaluation, done using :func:`_eval_ignore`. + the evaluation, done using :func:`~pypeit.par.util._eval_ignore`. .. todo:: Allow the user to add to the ignored strings. Returns: - :class:`pypeit.par.core.PypeItPar`: The instance of the + :class:`~pypeit.par.pypeitpar.PypeItPar`: The instance of the parameter set. """ # Get the base parameters in a ConfigObj instance @@ -4431,7 +4520,7 @@ def from_cfg_lines(cls, cfg_lines=None, merge_with=None, evaluate=True): When `evaluate` is true, the function runs `eval()` on all the entries in the `ConfigObj` dictionary, done using - :func:`_recursive_dict_evaluate`. This has the potential to + :func:`~pypeit.par.util.recursive_dict_evaluate`. This has the potential to go haywire if the name of a parameter unintentionally happens to be identical to an imported or system-level function. Of course, this can be useful by allowing one to @@ -4439,13 +4528,13 @@ def from_cfg_lines(cls, cfg_lines=None, merge_with=None, evaluate=True): one has to be careful with the values that the parameters should be allowed to have. The current way around this is to provide a list of strings that should be ignored during - the evaluation, done using :func:`_eval_ignore`. + the evaluation, done using :func:`~pypeit.par.util._eval_ignore`. .. todo:: Allow the user to add to the ignored strings. Returns: - :class:`pypeit.par.core.PypeItPar`: The instance of the + :class:`~pypeit.par.pypeitpar.PypeItPar`: The instance of the parameter set. """ # Get the base parameters in a ConfigObj instance @@ -4560,7 +4649,7 @@ def reset_all_processimages_par(self, **kwargs): Examples: To turn off the slit-illumination correction for all frames: - >>> from pypeit.spectrographs import load_spectrograph + >>> from pypeit.spectrographs.util import load_spectrograph >>> spec = load_spectrograph('shane_kast_blue') >>> par = spec.default_pypeit_par() >>> par.reset_all_processimages_par(use_illumflat=False) @@ -4731,7 +4820,7 @@ def valid_telescopes(): Return the valid telescopes. """ return [ 'GEMINI-N','GEMINI-S', 'KECK', 'SHANE', 'WHT', 'APF', 'TNG', 'VLT', 'MAGELLAN', 'LBT', 'MMT', - 'KPNO', 'NOT', 'P200', 'BOK', 'GTC', 'SOAR', 'NTT', 'LDT', 'JWST'] + 'KPNO', 'NOT', 'P200', 'BOK', 'GTC', 'SOAR', 'NTT', 'LDT', 'JWST', 'HILTNER'] def validate(self): pass @@ -4765,7 +4854,7 @@ class Collate1DPar(ParSet): For a table with the current keywords, defaults, and descriptions, see :ref:`parameters`. """ - def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, match_using=None, exclude_slit_trace_bm=[], exclude_serendip=False, wv_rms_thresh=None, outdir=None, spec1d_outdir=None, refframe=None): + def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, match_using=None, exclude_slit_trace_bm=[], exclude_serendip=False, wv_rms_thresh=None, outdir=None, spec1d_outdir=None, refframe=None, chk_version=False): # Grab the parameter names and values from the function # arguments @@ -4844,6 +4933,10 @@ def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, ma descr['refframe'] = 'Perform reference frame correction prior to coadding. ' \ 'Options are: {0}'.format(', '.join(options['refframe'])) + defaults['chk_version'] = False + dtypes['chk_version'] = bool + descr['chk_version'] = "Whether to check the data model versions of spec1d files and sensfunc files." + # Instantiate the parameter set super(Collate1DPar, self).__init__(list(pars.keys()), values=list(pars.values()), @@ -4855,7 +4948,7 @@ def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, ma @classmethod def from_dict(cls, cfg): k = [*cfg.keys()] - parkeys = ['tolerance', 'dry_run', 'ignore_flux', 'flux', 'match_using', 'exclude_slit_trace_bm', 'exclude_serendip', 'outdir', 'spec1d_outdir', 'wv_rms_thresh', 'refframe'] + parkeys = ['tolerance', 'dry_run', 'ignore_flux', 'flux', 'match_using', 'exclude_slit_trace_bm', 'exclude_serendip', 'outdir', 'spec1d_outdir', 'wv_rms_thresh', 'refframe', 'chk_version'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): diff --git a/pypeit/par/util.py b/pypeit/par/util.py index a985279854..cba7b6eb45 100644 --- a/pypeit/par/util.py +++ b/pypeit/par/util.py @@ -61,11 +61,11 @@ def recursive_dict_evaluate(d): Recursively run :func:`eval` on each element of the provided dictionary. - A raw read of a configuration file with `ConfigObj` results in a + A raw read of a configuration file with `configobj`_ results in a dictionary that contains strings or lists of strings. However, when assigning the values for the various ParSets, the `from_dict` methods expect the dictionary values to have the appropriate type. - E.g., the ConfigObj will have something like d['foo'] = '1', when + E.g., the `configobj`_ will have something like d['foo'] = '1', when the `from_dict` method expects the value to be an integer (d['foo'] = 1). @@ -77,8 +77,9 @@ def recursive_dict_evaluate(d): raises an exception is returned as the original string. - This is currently only used in :func:`PypitPar.from_cfg_file`; see - further comments there. + This is currently only used in + :func:`~pypeit.par.pypeitpar.PypitPar.from_cfg_file`; see further comments + there. Args: d (dict): @@ -122,7 +123,7 @@ def recursive_dict_evaluate(d): return d - +# TODO: I don't think this is used. We should deprecate it. def get_parset_list(cfg, pk, parsetclass): """ Create a list of ParSets based on a root keyword for a set of @@ -143,7 +144,7 @@ def get_parset_list(cfg, pk, parsetclass): kwargs['detector'] = get_parset_list(cfg, 'detector', DetectorPar) Args: - cfg (:class:`ConfigObj`, :obj:`dict`): + cfg (`configobj`_, :obj:`dict`): The top-level configuration that defines a list of sub-ParSets. pk (str): diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index 29dae9a77d..c733d0b1e1 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -21,7 +21,7 @@ from pypeit import io from pypeit import inputfiles from pypeit.calibframe import CalibFrame -from pypeit.core import parse +from pypeit.core import parse, wave, qa from pypeit import msgs from pypeit import calibrations from pypeit.images import buildimage @@ -29,7 +29,6 @@ from pypeit import find_objects from pypeit import extraction from pypeit import spec2dobj -from pypeit.core import qa from pypeit import specobjs #from pypeit.spectrographs.util import load_spectrograph from pypeit import slittrace @@ -627,6 +626,14 @@ def reduce_exposure(self, frames, bg_frames=None, std_outfile=None): # Hold em if tmp_sobjs.nobj > 0: all_specobjs_extract.add_sobj(tmp_sobjs) + + # Add calibration associations to the SpecObjs object + all_specobjs_extract.calibs = calibrations.Calibrations.get_association( + self.fitstbl, self.spectrograph, self.calibrations_path, + self.fitstbl[frames[0]]['setup'], + self.fitstbl.find_frame_calib_groups(frames[0])[0], self.det, + must_exist=True, proc_only=True) + # JFH TODO write out the background frame? # TODO -- Save here? Seems like we should. Would probably need to use update_det=True @@ -646,7 +653,7 @@ def get_sci_metadata(self, frame, det): 5 objects are returned:: - str: Object type; science or standard - str: Setup/configuration string - - astropy.time.Time: Time of observation + - `astropy.time.Time`_: Time of observation - str: Basename of the frame - str: Binning of the detector @@ -716,7 +723,7 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): List of frames to use as the background. Can be empty. std_outfile : :obj:`str`, optional Filename for the standard star spec1d file. Passed directly to - :func:`get_std_trace`. + :func:`~pypeit.specobjs.get_std_trace`. Returns ------- @@ -907,7 +914,7 @@ def extract_one(self, frames, det, sciImg, objFind, initial_sky, sobjs_obj): is provided det (:obj:`int`): Detector number (1-indexed) - sciImg (:class:`PypeItImage`): + sciImg (:class:`~pypeit.images.pypeitimage.PypeItImage`): Data container that holds a single image from a single detector its related images (e.g. ivar, mask) objFind : :class:`~pypeit.find_objects.FindObjects` @@ -961,46 +968,53 @@ def extract_one(self, frames, det, sciImg, objFind, initial_sky, sobjs_obj): slits.mask[flagged_slits] = \ slits.bitmask.turn_on(slits.mask[flagged_slits], 'BADSKYSUB') - msgs.info("Extraction begins for {} on det={}".format(self.basename, det)) - - # Instantiate Reduce object - # Required for pypeline specific object - # At instantiaton, the fullmask in self.sciImg is modified - # TODO Are we repeating steps in the init for FindObjects and Extract?? - self.exTract = extraction.Extract.get_instance( - sciImg, slits, sobjs_obj, self.spectrograph, - self.par, self.objtype, global_sky=final_global_sky, waveTilts=self.caliBrate.wavetilts, wv_calib=self.caliBrate.wv_calib, - bkg_redux=self.bkg_redux, return_negative=self.par['reduce']['extraction']['return_negative'], - std_redux=self.std_redux, basename=self.basename, show=self.show) - if not self.par['reduce']['extraction']['skip_extraction']: + msgs.info(f"Extraction begins for {self.basename} on det={det}") + # Instantiate Reduce object + # Required for pipeline specific object + # At instantiation, the fullmask in self.sciImg is modified + # TODO Are we repeating steps in the init for FindObjects and Extract?? + self.exTract = extraction.Extract.get_instance( + sciImg, slits, sobjs_obj, self.spectrograph, + self.par, self.objtype, global_sky=final_global_sky, waveTilts=self.caliBrate.wavetilts, + wv_calib=self.caliBrate.wv_calib, + bkg_redux=self.bkg_redux, return_negative=self.par['reduce']['extraction']['return_negative'], + std_redux=self.std_redux, basename=self.basename, show=self.show) # Perform the extraction - skymodel, objmodel, ivarmodel, outmask, sobjs, waveImg, tilts = self.exTract.run() - # Apply a reference frame correction to each object and the waveimg - self.exTract.refframe_correct(self.fitstbl["ra"][frames[0]], self.fitstbl["dec"][frames[0]], self.obstime, - sobjs=self.exTract.sobjs) + skymodel, objmodel, ivarmodel, outmask, sobjs, waveImg, tilts, slits = self.exTract.run() + slitgpm = np.logical_not(self.exTract.extract_bpm) + slitshift = self.exTract.slitshift else: + msgs.info(f"Extraction skipped for {self.basename} on det={det}") # Since the extraction was not performed, fill the arrays with the best available information - self.exTract.refframe_correct(self.fitstbl["ra"][frames[0]], self.fitstbl["dec"][frames[0]], self.obstime) - skymodel = final_global_sky - objmodel = np.zeros_like(self.exTract.sciImg.image) - ivarmodel = np.copy(self.exTract.sciImg.ivar) - outmask = self.exTract.sciImg.fullmask - sobjs = sobjs_obj - waveImg = self.exTract.waveimg - tilts = self.exTract.tilts + skymodel, objmodel, ivarmodel, outmask, sobjs, waveImg, tilts = \ + final_global_sky, \ + np.zeros_like(objFind.sciImg.image), \ + np.copy(objFind.sciImg.ivar), \ + objFind.sciImg.fullmask, \ + sobjs_obj, \ + objFind.waveimg, \ + objFind.tilts + slitgpm = (slits.mask == 0) + slitshift = objFind.slitshift + # If waveImg has not yet been created, make it now + if waveImg is None: + waveImg = self.caliBrate.wv_calib.build_waveimg(tilts, slits, spat_flexure=objFind.spat_flexure_shift) + + # Apply a reference frame correction to each object and the waveimg + vel_corr, waveImg = self.refframe_correct(slits, self.fitstbl["ra"][frames[0]], self.fitstbl["dec"][frames[0]], + self.obstime, slitgpm=slitgpm, waveimg=waveImg, sobjs=sobjs) # TODO -- Do this upstream - # Tack on detector and wavelength RMS + # Tack on wavelength RMS for sobj in sobjs: - sobj.DETECTOR = sciImg.detector iwv = np.where(self.caliBrate.wv_calib.spat_ids == sobj.SLITID)[0][0] sobj.WAVE_RMS =self.caliBrate.wv_calib.wv_fits[iwv].rms # Construct table of spectral flexure spec_flex_table = Table() spec_flex_table['spat_id'] = slits.spat_id - spec_flex_table['sci_spec_flexure'] = self.exTract.slitshift + spec_flex_table['sci_spec_flexure'] = slitshift # Construct the Spec2DObj spec2DObj = spec2dobj.Spec2DObj(sciimg=sciImg.image, @@ -1014,7 +1028,7 @@ def extract_one(self, frames, det, sciImg, objFind, initial_sky, sobjs_obj): detector=sciImg.detector, sci_spat_flexure=sciImg.spat_flexure, sci_spec_flexure=spec_flex_table, - vel_corr=self.exTract.vel_corr, + vel_corr=vel_corr, vel_type=self.par['calibrations']['wavelengths']['refframe'], tilts=tilts, slits=slits, @@ -1034,6 +1048,65 @@ def extract_one(self, frames, det, sciImg, objFind, initial_sky, sobjs_obj): # Return return spec2DObj, sobjs + # TODO :: Should this be moved outside of this class? + def refframe_correct(self, slits, ra, dec, obstime, slitgpm=None, waveimg=None, sobjs=None): + """ Correct the calibrated wavelength to the user-supplied reference frame + + Args: + slits (:class:`~pypeit.slittrace.SlitTraceSet`): + Slit trace set object + ra (float, str): + Right Ascension + dec (float, str): + Declination + obstime (`astropy.time.Time`_): + Observation time + slitgpm (`numpy.ndarray`_, None, optional): + 1D boolean array indicating the good slits (True). If None, the gpm will be taken from slits + waveimg (`numpy.ndarray`_, optional) + Two-dimensional image specifying the wavelength of each pixel + sobjs (:class:`~pypeit.specobjs.SpecObjs`, None, optional): + Spectrally extracted objects + + """ + if slitgpm is None: + slitgpm = (slits.mask == 0) + # Correct Telescope's motion + refframe = self.par['calibrations']['wavelengths']['refframe'] + vel_corr = 0.0 + if refframe in ['heliocentric', 'barycentric'] \ + and self.par['calibrations']['wavelengths']['reference'] != 'pixel': + msgs.info("Performing a {0} correction".format(self.par['calibrations']['wavelengths']['refframe'])) + # Calculate correction + radec = ltu.radec_to_coord((ra, dec)) + vel, vel_corr = wave.geomotion_correct(radec, obstime, + self.spectrograph.telescope['longitude'], + self.spectrograph.telescope['latitude'], + self.spectrograph.telescope['elevation'], + refframe) + # Apply correction to objects + msgs.info('Applying {0} correction = {1:0.5f} km/s'.format(refframe, vel)) + if (sobjs is not None) and (sobjs.nobj != 0): + # Loop on slits to apply + gd_slitord = slits.slitord_id[slitgpm] + for slitord in gd_slitord: + indx = sobjs.slitorder_indices(slitord) + this_specobjs = sobjs[indx] + # Loop on objects + for specobj in this_specobjs: + if specobj is None: + continue + specobj.apply_helio(vel_corr, refframe) + + # Apply correction to wavelength image + if waveimg is not None: + waveimg *= vel_corr + else: + msgs.info('A wavelength reference frame correction will not be performed.') + + # Return the value of the correction and the corrected wavelength image + return vel_corr, waveimg + def save_exposure(self, frame, all_spec2d, all_specobjs, basename, history=None): """ Save the outputs from extraction for a given exposure diff --git a/pypeit/pypeitsetup.py b/pypeit/pypeitsetup.py index 5aa7d2004a..89ff8c0207 100644 --- a/pypeit/pypeitsetup.py +++ b/pypeit/pypeitsetup.py @@ -37,8 +37,9 @@ class PypeItSetup: file name without the full path) to a specific frame type (e.g., arc, bias, etc.). The file name and type are expected to be the key and value of the dictionary, respectively. If None, this is - determined by the :func:`get_frame_types` method. - usrdata (:obj:`astropy.table.Table`, optional): + determined by the + :func:`~pypeit.metadata.PypeItMetaData.get_frame_types` method. + usrdata (`astropy.table.Table`_, optional): A user provided set of data used to supplement or overwrite metadata read from the file headers. The table must have a `filename` column that is used to match to the metadata @@ -167,8 +168,8 @@ def from_pypeit_file(cls, filename): @classmethod def from_file_root(cls, root, spectrograph, extension='.fits'): """ - Instantiate the :class:`PypeItSetup` object by providing a file - root. + Instantiate the :class:`~pypeit.pypeitsetup.PypeItSetup` object by + providing a file root. Args: root (:obj:`str`): @@ -176,9 +177,8 @@ def from_file_root(cls, root, spectrograph, extension='.fits'): :func:`~pypeit.io.files_from_extension`. spectrograph (:obj:`str`): The PypeIt name of the spectrograph used to take the - observations. This should be one of the available - options in - :func:`~pypeit.spectrographs.available_spectrographs`. + observations. This should be one of the available options in + :attr:`~pypeit.spectrographs.available_spectrographs`. extension (:obj:`str`, optional): The extension common to all the fits files to reduce; see :func:`~pypeit.io.files_from_extension`. @@ -190,7 +190,9 @@ def from_file_root(cls, root, spectrograph, extension='.fits'): @classmethod def from_rawfiles(cls, data_files:list, spectrograph:str, frametype=None): - """ Instantiate the :class:`PypeItSetup` object by providing a list of raw files. + """ + Instantiate the :class:`~pypeit.pypeitsetup.PypeItSetup` object by + providing a list of raw files. Args: data_files (list): @@ -202,10 +204,11 @@ def from_rawfiles(cls, data_files:list, spectrograph:str, frametype=None): file name without the full path) to a specific frame type (e.g., arc, bias, etc.). The file name and type are expected to be the key and value of the dictionary, respectively. If None, this is - determined by the :func:`get_frame_types` method. + determined by the + :func:`~pypeit.metadata.PypeItMetaData.get_frame_types` method. Returns: - :class:`PypeItSetup`: The instance of the class. + :class:`~pypeit.pypeitsetup.PypeItSetup`: The instance of the class. """ # Configure me @@ -229,14 +232,13 @@ def build_fitstbl(self, strict=True): """ Construct the table with metadata for the frames to reduce. - Largely a wrapper for :func:`pypeit.core.load.create_fitstbl`. + Largely a wrapper for :class:`~pypeit.metadata.PypeItMetaData`. Args: strict (:obj:`bool`, optional): - Function will fault if :func:`fits.getheader` fails to - read the headers of any of the files in - :attr:`file_list`. Set to False to only report a - warning and continue. + Function will fault if `astropy.io.fits.getheader`_ fails to + read the headers of any of the files in :attr:`file_list`. Set + to False to only report a warning and continue. Returns: `astropy.table.Table`_: Table with the metadata for each fits file @@ -259,7 +261,7 @@ def get_frame_types(self, flag_unknown=False): Include the frame types in the metadata table. This is mainly a wrapper for - :func:`PypeItMetaData.get_frame_types`. + :func:`~pypeit.metadata.PypeItMetaData.get_frame_types`. .. warning:: diff --git a/pypeit/pypmsgs.py b/pypeit/pypmsgs.py index d703ba953b..d927a427e5 100644 --- a/pypeit/pypmsgs.py +++ b/pypeit/pypmsgs.py @@ -43,16 +43,17 @@ class Messages: Parameters ---------- - log : str or None - Name of saved log file (no log will be saved if log=="") - verbosity : int (0,1,2) - Level of verbosity: - 0 = No output - 1 = Minimal output - 2 = All output (default) + log : str, optional + Name of saved log file (no log will be saved if log==""). If None, no + log is saved. + verbosity : int + Level of verbosity. Options are + - 0 = No output + - 1 = Minimal output + - 2 = All output (default) colors : bool - If true, the screen output will have colors, otherwise - normal screen output will be displayed + If true, the screen output will have colors, otherwise normal screen + output will be displayed """ def __init__(self, log=None, verbosity=None, colors=True): @@ -243,15 +244,23 @@ def pypeitpar_text(self, msglist): Parameters ---------- - msglist: list of str - A list containing the pypeit parameters. The last element of the list - must be the argument and the variable. For example, to print: + msglist: list + A list containing the pypeit parameter strings. The last element of + the list must be the argument and the variable. For example, to + print: - [sensfunc] - [[UVIS]] - polycorrect = False + .. code-block:: ini - you should set msglist = ['sensfunc', 'UVIS', 'polycorrect = False'] + [sensfunc] + [[UVIS]] + polycorrect = False + + you should set ``msglist = ['sensfunc', 'UVIS', 'polycorrect = False']``. + + Returns + ------- + parstring : str + The parameter string """ parstring = '\n' premsg = ' ' @@ -270,15 +279,19 @@ def pypeitpar(self, msglist): Parameters ---------- - msglist: list of str - A list containing the pypeit parameters. The last element of the list - must be the argument and the variable. For example, to print: + msglist: list + A list containing the pypeit parameter strings. The last element of + the list must be the argument and the variable. For example, to + print: + + .. code-block:: ini - [sensfunc] - [[UVIS]] - polycorrect = False + [sensfunc] + [[UVIS]] + polycorrect = False + + you should set ``msglist = ['sensfunc', 'UVIS', 'polycorrect = False']``. - you should set msglist = ['sensfunc', 'UVIS', 'polycorrect = False'] """ premsg = ' ' for ll, lin in enumerate(msglist): @@ -392,3 +405,4 @@ def set_logfile_and_verbosity(self, scriptname, verbosity): logname = f"{scriptname}_{timestamp}.log" if verbosity == 2 else None # Set the verbosity in msgs self.reset(log=logname, verbosity=verbosity) + diff --git a/pypeit/sampling.py b/pypeit/sampling.py index 3f9d306a7f..7225eced59 100644 --- a/pypeit/sampling.py +++ b/pypeit/sampling.py @@ -23,7 +23,7 @@ def spectral_coordinate_step(wave, log=False, base=10.0): of the wavelength; otherwise, return the linear step in angstroms. Args: - wave (numpy.ndarray): Wavelength coordinates of each spectral + wave (`numpy.ndarray`_): Wavelength coordinates of each spectral channel in angstroms. log (bool): (**Optional**) Input spectrum has been sampled geometrically. @@ -51,7 +51,7 @@ def spectrum_velocity_scale(wave): log(angstrom). Args: - wave (numpy.ndarray): Wavelength coordinates of each spectral + wave (`numpy.ndarray`_): Wavelength coordinates of each spectral channel in angstroms. It is expected that the spectrum has been sampled geometrically @@ -74,9 +74,9 @@ def angstroms_per_pixel(wave, log=False, base=10.0, regular=True): as determined by assuming the coordinate is at its center. Args: - wave (`numpy.ndarray`): + wave (`numpy.ndarray`_): (Geometric) centers of the spectrum pixels in angstroms. - log (`numpy.ndarray`, optional): + log (`numpy.ndarray`_, optional): The vector is geometrically sampled. base (:obj:`float`, optional): Base of the logarithm used in the geometric sampling. @@ -84,7 +84,7 @@ def angstroms_per_pixel(wave, log=False, base=10.0, regular=True): The vector is regularly sampled. Returns: - numpy.ndarray: The angstroms per pixel. + `numpy.ndarray`_: The angstroms per pixel. """ if regular: ang_per_pix = spectral_coordinate_step(wave, log=log, base=base) @@ -103,7 +103,7 @@ def _pixel_centers(xlim, npix, log=False, base=10.0): sampled vector given first, last and number of pixels Args: - xlim (numpy.ndarray) : (Geometric) Centers of the first and last + xlim (`numpy.ndarray`_) : (Geometric) Centers of the first and last pixel in the vector. npix (int) : Number of pixels in the vector. log (bool) : (**Optional**) The input range is (to be) @@ -113,7 +113,7 @@ def _pixel_centers(xlim, npix, log=False, base=10.0): natural logarithm. Returns: - numpy.ndarray, float: A vector with the npix centres of the + `numpy.ndarray`_, float: A vector with the npix centres of the pixels and the sampling rate. If logarithmically binned, the sampling is the step in :math`\log x`. """ @@ -133,7 +133,7 @@ def _pixel_borders(xlim, npix, log=False, base=10.0): number of pixels Args: - xlim (numpy.ndarray) : (Geometric) Centers of the first and last + xlim (`numpy.ndarray`_) : (Geometric) Centers of the first and last pixel in the vector. npix (int) : Number of pixels in the vector. log (bool) : (**Optional**) The input range is (to be) @@ -143,7 +143,7 @@ def _pixel_borders(xlim, npix, log=False, base=10.0): natural logarithm. Returns: - numpy.ndarray, float: A vector with the (npix+1) borders of the + `numpy.ndarray`_, float: A vector with the (npix+1) borders of the pixels and the sampling rate. If logarithmically binned, the sampling is the step in :math`\log x`. """ @@ -163,7 +163,7 @@ def resample_vector_npix(outRange=None, dx=None, log=False, base=10.0, default=N Determine the number of pixels needed to resample a vector given first, last pixel and dx Args: - outRange (list or numpy.ndarray) : Two-element array with the + outRange (list, `numpy.ndarray`_) : Two-element array with the starting and ending x coordinate of the pixel centers to divide into pixels of a given width. If *log* is True, this must still be the linear value of the x coordinate, not @@ -176,7 +176,7 @@ def resample_vector_npix(outRange=None, dx=None, log=False, base=10.0, default=N returned if either *outRange* or *dx* are not provided. Returns: - int, numpy.ndarray: Returns two objects: The number of pixels to + :obj:`tuple`: Returns two objects: The number of pixels to cover *outRange* with pixels of width *dx* and the adjusted range such that number of pixels of size dx is the exact integer. @@ -203,9 +203,8 @@ class Resample: Resample regularly or irregularly sampled data to a new grid using integration. - This is a generalization of the routine - :func:`ppxf.ppxf_util.log_rebin` provided by Michele Cappellari in - the pPXF package. + This is a generalization of the routine ``ppxf_util.log_rebin`` from from + the `ppxf`_ package by Michele Cappellari. The abscissa coordinates (`x`) or the pixel borders (`xBorders`) for the data (`y`) should be provided for irregularly sampled data. If @@ -263,24 +262,24 @@ class Resample: - Allow for a covariance matrix calculation. Args: - y (numpy.ndarray): + y (`numpy.ndarray`_): Data values to resample. Can be a numpy.ma.MaskedArray, and the shape can be 1 or 2D. If 1D, the shape must be :math:`(N_{\rm pix},)`; otherwise, it must be :math:`(N_y,N_{\rm pix})`. I.e., the length of the last axis must match the input coordinates. - e (numpy.ndarray, optional): + e (`numpy.ndarray`_, optional): Errors in the data that should be resampled. Can be a numpy.ma.MaskedArray, and the shape must match the input `y` array. These data are used to perform a nominal calculation of the error in the resampled array. - mask (numpy.ndarray, optional): + mask (`numpy.ndarray`_, optional): A boolean array (masked values are True) indicating values in `y` that should be ignored during the resampling. The mask used during the resampling is the union of this object and the masks of `y` and `e`, if they are provided as numpy.ma.MaskedArrays. - x (numpy.ndarray, optional): + x (`numpy.ndarray`_, optional): Abcissa coordinates for the data, which do not need to be regularly sampled. If the pixel borders are not provided, they are assumed to be half-way between adjacent pixels, and @@ -293,7 +292,7 @@ class Resample: A two-element array with the starting and ending value for the coordinates of the centers of the first and last pixels in y. Default is :math:`[0,N_{\rm pix}-1]`. - xBorders (numpy.ndarray, optional): + xBorders (`numpy.ndarray`_, optional): An array with the borders of each pixel that must have a length of :math:`N_{\rm pix}+1`. inLog (:obj:`bool`, optional): @@ -337,30 +336,30 @@ class Resample: interpolation between pixel samples. Attributes: - x (numpy.ndarray): + x (`numpy.ndarray`_): The coordinates of the function on input. - xborders (numpy.ndarray): + xborders (`numpy.ndarray`_): The borders of the input pixel samples. - y (numpy.ndarray): + y (`numpy.ndarray`_): The function to resample. - e (numpy.ndarray): + e (`numpy.ndarray`_): The 1-sigma errors in the function to resample. - m (numpy.ndarray): + m (`numpy.ndarray`_): The boolean mask for the input function. - outx (numpy.ndarray): + outx (`numpy.ndarray`_): The coordinates of the function on output. - outborders (numpy.ndarray): + outborders (`numpy.ndarray`_): The borders of the output pixel samples. - outy (numpy.ndarray): + outy (`numpy.ndarray`_): The resampled function. - oute (numpy.ndarray): + oute (`numpy.ndarray`_): The resampled 1-sigma errors. - outf (numpy.ndarray): + outf (`numpy.ndarray`_): The fraction of each output pixel that includes valid data from the input function. Raises: - ValueError: Raised if *y* is not of type numpy.ndarray, if *y* + ValueError: Raised if *y* is not of type `numpy.ndarray`_, if *y* is not one-dimensional, or if *xRange* is not provided and the input vector is logarithmically binned (see *inLog* above). diff --git a/pypeit/scripts/__init__.py b/pypeit/scripts/__init__.py index e5acdffcd1..410cc4f6d7 100644 --- a/pypeit/scripts/__init__.py +++ b/pypeit/scripts/__init__.py @@ -31,7 +31,7 @@ from pypeit.scripts import parse_slits from pypeit.scripts import qa_html from pypeit.scripts import ql -from pypeit.scripts import ql_multislit +#from pypeit.scripts import ql_multislit from pypeit.scripts import run_pypeit from pypeit.scripts import sensfunc from pypeit.scripts import setup diff --git a/pypeit/scripts/arxiv_solution.py b/pypeit/scripts/arxiv_solution.py new file mode 100644 index 0000000000..a608d5e2dd --- /dev/null +++ b/pypeit/scripts/arxiv_solution.py @@ -0,0 +1,63 @@ +""" +This script enables the user to convert a MasterWaveCalib wavelength solution fits file +into a PypeIt arxiv solution that can be used with the full_template method. + +.. include common links, assuming primary doc root is up one directory +.. include:: ../include/links.rst +""" +import time +from pypeit import msgs +from pypeit import par +from pypeit import inputfiles +from pypeit import utils +from pypeit.scripts import scriptbase + + +class ArxivSolution(scriptbase.ScriptBase): + + @classmethod + def get_parser(cls, width=None): + parser = super().get_parser(description='Read in a MasterWaveCalib solution and convert it into the ' + 'format required for the PypeIt full template archive', width=width) + parser.add_argument('file', type = str, default=None, help='MasterWaveCalib file') + parser.add_argument('binning', type=int, help="Spectral binning") + parser.add_argument('-s', '--slit', default=0, type=int, help='Slit number to use') + parser.add_argument('-v', '--verbosity', type=int, default=1, + help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' + 'Level 2 writes a log with filename make_arxiv_solution_YYYYMMDD-HHMM.log') + return parser + + @staticmethod + def main(args): + import os + from pypeit.wavecalib import WaveCalib + from pypeit.core.wavecal import wvutils + + # Set the verbosity, and create a logfile if verbosity == 2 + msgs.set_logfile_and_verbosity('arxiv_solution', args.verbosity) + + # Check that a file has been provided + if args.file is None: + msgs.error('You must input a MasterWaveCalib file') + elif not os.path.exists(args.file): + msgs.error("The following MasterWaveCalib file does not exist:" + msgs.newline() + args.file) + + # Load the wavelength calibration file + wv_calib = WaveCalib.from_file(args.file) + # Check if a wavelength solution exists + if wv_calib['wv_fits'][args.slit]['wave_soln'] is None: + gd_slits = [] + for slit in range(len(wv_calib['wv_fits'])): + if wv_calib['wv_fits'][slit]['wave_soln'] is not None: + gd_slits.append(f"{slit}") + # Prepare the message + thismsg = f"A wavelength solution does not exist for slit {args.slit}. " + if len(gd_slits) == 0: + thismsg += "There are no good slits - the WaveCalib file is bad." + else: + thismsg += "Try one of the following slits, instead: " + msgs.newline() + ", ".join(gd_slits) + msgs.error(thismsg) + wave = wv_calib['wv_fits'][args.slit]['wave_soln'].flatten() + spec = wv_calib['wv_fits'][args.slit]['spec'].flatten() + outname = args.file.replace(".fits", "_arXiv.fits") + wvutils.write_template(wave, spec, args.binning, './', outname, cache=True) diff --git a/pypeit/scripts/chk_for_calibs.py b/pypeit/scripts/chk_for_calibs.py index 9d7317dc7c..b7b16994e0 100644 --- a/pypeit/scripts/chk_for_calibs.py +++ b/pypeit/scripts/chk_for_calibs.py @@ -35,7 +35,7 @@ def main(args): args: Returns: - astropy.table.Table: + `astropy.table.Table`_: """ diff --git a/pypeit/scripts/chk_noise_1dspec.py b/pypeit/scripts/chk_noise_1dspec.py index 7501af7d21..33123e5533 100644 --- a/pypeit/scripts/chk_noise_1dspec.py +++ b/pypeit/scripts/chk_noise_1dspec.py @@ -4,6 +4,7 @@ .. include common links, assuming primary doc root is up one directory .. include:: ../include/links.rst + """ import numpy as np diff --git a/pypeit/scripts/chk_noise_2dspec.py b/pypeit/scripts/chk_noise_2dspec.py index 5f95b8ecdf..bb37af3452 100644 --- a/pypeit/scripts/chk_noise_2dspec.py +++ b/pypeit/scripts/chk_noise_2dspec.py @@ -4,6 +4,7 @@ .. include common links, assuming primary doc root is up one directory .. include:: ../include/links.rst + """ import numpy as np @@ -31,7 +32,7 @@ def plot(image:np.ndarray, chi_select:np.ndarray, flux_select:np.ndarray, """ Generate the plot Args: - image (np.ndarray): + image (`numpy.ndarray`_): Image of the slit to plot. chi_select (`numpy.ndarray`_): 2D array of the chi value for a selected part of the slit. @@ -39,7 +40,7 @@ def plot(image:np.ndarray, chi_select:np.ndarray, flux_select:np.ndarray, Array of wavelength of spectral features to be plotted. line_names (`numpy.ndarray`_): Array of names of spectral features to be plotted. - lbda_1d (np.ndarray): + lbda_1d (`numpy.ndarray`_): 1D array of the wavelength lbda_min (float): Minimum wavelength of the select pat of the slit diff --git a/pypeit/scripts/chk_wavecalib.py b/pypeit/scripts/chk_wavecalib.py index 36640c0c87..9874b3af56 100644 --- a/pypeit/scripts/chk_wavecalib.py +++ b/pypeit/scripts/chk_wavecalib.py @@ -29,8 +29,7 @@ def main(args): try: # Load - waveCalib = wavecalib.WaveCalib.from_file(args.input_file) - #, chk_version=(not args.try_old)) + waveCalib = wavecalib.WaveCalib.from_file(args.input_file, chk_version=False) # TODO: Should this specify the type of exception to pass? except: pass diff --git a/pypeit/scripts/coadd_1dspec.py b/pypeit/scripts/coadd_1dspec.py index e5ce75b198..e6e2c604c7 100644 --- a/pypeit/scripts/coadd_1dspec.py +++ b/pypeit/scripts/coadd_1dspec.py @@ -124,11 +124,15 @@ def get_parser(cls, width=None): width=width, formatter=scriptbase.SmartFormatter) parser.add_argument('coadd1d_file', type=str, - help="R|File to guide coadding process. This file must have the " - "following format (see docs for further details including the use of paths): \n\n" + help="R|File to guide coadding process.\n\n" + "------------------------\n" + " MultiSlit \n" + "------------------------\n\n" + "For coadding Multislit spectra the file must have the " + "following format (see docs for further details including the " + "use of paths): \n\n" "F|[coadd1d]\n" "F| coaddfile='output_filename.fits' # Optional\n" - "F| sensfuncfile = 'sensfunc.fits' # Required only for Echelle\n" "\n" "F| coadd1d read\n" "F| filename | obj_id\n" @@ -146,15 +150,51 @@ def get_parser(cls, width=None): "F| ... \n" "F| coadd1d end\n" "\n" - "That is the coadd1d block must be a two column list of " - "spec1dfiles and objids, but you can specify only a single objid for " - "all spec1dfiles on the first line\n\n" + "That is the coadd1d block must be a two column list of\n" + "spec1dfiles and objids, but you can specify only a single " + "objid for all spec1dfiles on the first line\n\n" "Where: \n" "\n" "spec1dfile: full path to a PypeIt spec1dfile\n\n" "objid: the object identifier. To determine the objids inspect " "the spec1d_*.txt files or run pypeit_show_1dspec spec1dfile " "--list\n\n" + "------------------------\n" + " Echelle \n" + "------------------------\n\n" + "For coadding Echelle spectra the file must have the following " + "format (see docs for further details): \n\n" + "F|[coadd1d]\n" + "F| coaddfile='output_filename.fits' # Optional\n" + "\n" + "F| coadd1d read\n" + "F| filename | obj_id | sensfile | setup_id \n" + "F| spec1dfile1 | objid1 | sensfile1 | setup_id1\n" + "F| spec1dfile2 | objid2 | sensfile2 | setup_id2\n" + "F| spec1dfile3 | objid3 | sensfile3 | setup_id3\n" + "F| ... \n" + "F| coadd1d end\n" + "\n OR the coadd1d read/end block can look like \n\n" + "F| coadd1d read\n" + "F| filename | obj_id | sensfile | setup_id\n" + "F| spec1dfile1 | objid1 | sensfile | setup_id\n" + "F| spec1dfile2 | | | \n" + "F| spec1dfile3 | | | \n" + "F| ... \n" + "F| coadd1d end\n" + "\n" + "That is the coadd1d block is a four column list of\n" + "spec1dfiles, objids, sensitivity function files, and setup_ids, " + "but you can specify only a single objid, sensfile, and setup_id " + "for all spec1dfiles on the first line\n\n" + "Here: \n" + "\n" + "spec1dfile: full path to a PypeIt spec1dfile\n\n" + "objid: the object identifier (see details above)\n\n" + "sensfile: full path to a PypeIt sensitivity function file for " + "the echelle setup in question\n\n" + "setup_id: string identifier for the echelle setup in question, " + "i.e. 'VIS', 'NIR', or 'UVB'\n\n" "If the coaddfile is not given the output file will be placed " "in the same directory as the first spec1d file.\n\n") parser.add_argument("--debug", default=False, action="store_true", help="show debug plots?") @@ -191,10 +231,14 @@ def main(args): spectrograph_def_par = spectrograph.default_pypeit_par() par = pypeitpar.PypeItPar.from_cfg_lines(cfg_lines=spectrograph_def_par.to_config(), merge_with=(coadd1dFile.cfg_lines,)) + # Check that sensfunc column is populated if this is echelle + if spectrograph.pypeline == 'Echelle' and coadd1dFile.sensfiles is None: + msgs.error("To coadd echelle spectra, the 'sensfile' column must present in your .coadd1d file") + # Write the par to disk print("Writing the parameters to {}".format(args.par_outfile)) par.to_config(args.par_outfile) - sensfile = par['coadd1d']['sensfuncfile'] + # TODO This needs to come out of the parset coaddfile = par['coadd1d']['coaddfile'] # Testing? @@ -203,8 +247,6 @@ def main(args): # sensfile = os.path.join(args.test_spec_path, sensfile) # coaddfile = os.path.join(args.test_spec_path, coaddfile) - if spectrograph.pypeline == 'Echelle' and sensfile is None: - msgs.error('You must specify the sensfuncfile in the .coadd1d file for Echelle coadds') if coaddfile is None: coaddfile = build_coadd_file_name(coadd1dFile.filenames, spectrograph) @@ -213,7 +255,9 @@ def main(args): coAdd1d = coadd1d.CoAdd1D.get_instance(coadd1dFile.filenames, coadd1dFile.objids, spectrograph=spectrograph, - par=par['coadd1d'], sensfile=sensfile, + par=par['coadd1d'], + sensfuncfile=coadd1dFile.sensfiles, + setup_id=coadd1dFile.setup_id, debug=args.debug, show=args.show) # Run coAdd1d.run() diff --git a/pypeit/scripts/coadd_2dspec.py b/pypeit/scripts/coadd_2dspec.py index 7c0a62c79a..0c61829161 100644 --- a/pypeit/scripts/coadd_2dspec.py +++ b/pypeit/scripts/coadd_2dspec.py @@ -5,6 +5,7 @@ .. include:: ../include/links.rst """ from pypeit.scripts import scriptbase +from pypeit.core import parse class CoAdd2DSpec(scriptbase.ScriptBase): @@ -103,7 +104,7 @@ def main(args): # Get the paths coadd_scidir, qa_path = map(lambda x : Path(x).resolve(), - coadd2d.CoAdd2D.output_paths(spec2d_files, par)) + coadd2d.CoAdd2D.output_paths(spec2d_files, par, coadd_dir=par['rdx']['redux_path'])) # Get the output basename head2d = fits.getheader(spec2d_files[0]) @@ -112,6 +113,7 @@ def main(args): # Write the par to disk par_outfile = coadd_scidir.parent / f'{basename}_coadd2d.par' + print(f'Writing full parameter set to {par_outfile}.') par.to_config(par_outfile, exclude_defaults=True, include_descr=False) @@ -137,11 +139,9 @@ def main(args): sci_dict['meta']['find_negative'] = find_negative # Find the detectors to reduce - # TODO: Allow slitspatnum to be specified? E.g.: -# detectors = spectrograph.select_detectors( -# subset=par['rdx']['detnum'] if par['rdx']['slitspatnum'] is None else -# par['rdx']['slitspatnum']) - detectors = spectrograph.select_detectors(subset=par['rdx']['detnum']) + detectors = spectrograph.select_detectors(subset=par['rdx']['detnum'] if par['coadd2d']['only_slits'] is None + else par['coadd2d']['only_slits']) + msgs.info(f'Detectors to work on: {detectors}') # container for specobjs @@ -152,14 +152,31 @@ def main(args): all_spec2d['meta']['bkg_redux'] = bkg_redux all_spec2d['meta']['find_negative'] = find_negative + # get only_slits and exclude_slits if they are set + only_dets, only_spat_ids, exclude_dets, exclude_spat_ids = None, None, None, None + if par['coadd2d']['only_slits'] is not None: + only_dets, only_spat_ids = parse.parse_slitspatnum(par['coadd2d']['only_slits']) + if par['coadd2d']['exclude_slits'] is not None: + if par['coadd2d']['only_slits'] is not None: + msgs.warn('Both `only_slits` and `exclude_slits` are provided. They are mutually exclusive. ' + 'Using `only_slits` and ignoring `exclude_slits`') + else: + exclude_dets, exclude_spat_ids = parse.parse_slitspatnum(par['coadd2d']['exclude_slits']) + # Loop on detectors for det in detectors: msgs.info("Working on detector {0}".format(det)) + detname = spectrograph.get_det_name(det) + this_only_slits = only_spat_ids[only_dets == detname] if np.any(only_dets == detname) else None + this_exclude_slits = exclude_spat_ids[exclude_dets == detname] if np.any(exclude_dets == detname) else None + # Instantiate Coadd2d coadd = coadd2d.CoAdd2D.get_instance(spec2d_files, spectrograph, par, det=det, offsets=par['coadd2d']['offsets'], weights=par['coadd2d']['weights'], + only_slits=this_only_slits, + exclude_slits=this_exclude_slits, spec_samp_fact=args.spec_samp_fact, spat_samp_fact=args.spat_samp_fact, bkg_redux=bkg_redux, find_negative=find_negative, @@ -168,7 +185,7 @@ def main(args): # TODO Add this stuff to a run method in coadd2d # Coadd the slits - coadd_dict_list = coadd.coadd(only_slits=par['coadd2d']['only_slits']) + coadd_dict_list = coadd.coadd() # Create the pseudo images pseudo_dict = coadd.create_pseudo_image(coadd_dict_list) # Reduce @@ -221,6 +238,8 @@ def main(args): vel_type=None, maskdef_designtab=maskdef_designtab) + all_spec2d['meta']['effective_exptime'] = coadd.exptime_coadd + # SAVE TO DISK # THE FOLLOWING MIMICS THE CODE IN pypeit.save_exposure() diff --git a/pypeit/scripts/collate_1d.py b/pypeit/scripts/collate_1d.py index f886c6e423..f2a342f938 100644 --- a/pypeit/scripts/collate_1d.py +++ b/pypeit/scripts/collate_1d.py @@ -38,35 +38,40 @@ def get_report_metadata(object_header_keys, spec_obj_keys, file_info): """ - Gets the metadata from a SourceObject instance used building a report - on the results of collation. It is intended to be wrapped in by functools - partial object that passes in object_header_keys and spec_obj_keys. file_info - is then passed as in by the :obj:`pypeit.archive.ArchiveMetadata` object. - Unlike the other get_*_metadata functions, this is not used for archiving; it is - used for reporting on the results of collating. - - If another type of file is added to the ArchiveMetadata object, the file_info - argument will not be a SourceObject, In this case, a list of ``None`` values are - returned. + Gets the metadata from a :class:`~pypeit.core.collate.SourceObject` instance + used building a report on the results of collation. It is intended to be + wrapped in by functools partial object that passes in object_header_keys and + spec_obj_keys. file_info is then passed as in by the + :class:`~pypeit.archive.ArchiveMetadata` object. Unlike the other + get_*_metadata functions, this is not used for archiving; it is used for + reporting on the results of collating. + + If another type of file is added to the ArchiveMetadata object, the + file_info argument will not be a :class:`~pypeit.core.collate.SourceObject`, + In this case, a list of ``None`` values are returned. Args: object_header_keys (list of str): The keys to read fom the spec1d headers from the SourceObject. spec_obj_keys (list of str): - The keys to read from the (:obj:`pypeit.specobj.SpecObj`) objects in the SourceObject. + The keys to read from the (:class:`~pypeit.specobj.SpecObj`) objects in + the SourceObject. - file_info (:obj:`pypeit.scripts.collate_1d.SourceObject`)): - The source object containing the headers, filenames and SpecObj information for a coadd output file. + file_info (:class:`~pypeit.core.collate.SourceObject`): + The source object containing the headers, filenames and SpecObj + information for a coadd output file. Returns: tuple: A tuple of two lists:. - **data_rows** (:obj:`list` of :obj:`list`): The metadata rows built from the source object. + - **data_rows** (:obj:`list` of :obj:`list`): The metadata rows + built from the source object. + + - **files_to_copy** (iterable): An list of tuples of files to copy. + Because this function is not used for archving data, this is + always ``None``. - **files_to_copy** (iterable): - An list of tuples of files to copy. Because this function is not used for - archving data, this is always ``None``. """ if not isinstance(file_info, SourceObject): @@ -119,7 +124,7 @@ def find_slits_to_exclude(spec2d_files, par): exclude_map = dict() for spec2d_file in spec2d_files: - allspec2d = AllSpec2DObj.from_fits(spec2d_file) + allspec2d = AllSpec2DObj.from_fits(spec2d_file, chk_version=par['collate1d']['chk_version']) for sobj2d in [allspec2d[det] for det in allspec2d.detectors]: for (slit_id, mask, slit_mask_id) in sobj2d['slits'].slit_info: for flag in exclude_flags: @@ -151,11 +156,11 @@ def exclude_source_objects(source_objects, exclude_map, par): Returns: tuple: Tuple containing two lists: - **filtered_objects** (:obj:`list`): A list of :class:`~pypeit.core.collate.SourceObject` - with any excluded ones removed. + - **filtered_objects** (:obj:`list`): A list of + :class:`~pypeit.core.collate.SourceObject` with any excluded ones + removed. - **missing_archive_msgs** (:obj:`list`): A list of messages explaining why some source - objects were excluded. + - **excluded_messages** (:obj:`list`): A list of messages explaining why some source objects were excluded. """ filtered_objects = [] excluded_messages= [] @@ -188,29 +193,80 @@ def exclude_source_objects(source_objects, exclude_map, par): excluded_messages.append(msg) continue - if par['coadd1d']['ex_value'] == 'OPT' and sobj.OPT_COUNTS is None: - msg = f'Excluding {sobj.NAME} in {spec1d_file} because of missing OPT_COUNTS. Consider changing ex_value to "BOX".' - msgs.warn(msg) - excluded_messages.append(msg) - continue + if par['coadd1d']['ex_value'] == 'OPT': + msg = None + if sobj.OPT_COUNTS is None: + msg = f'Excluding {sobj.NAME} in {spec1d_file} because of missing OPT_COUNTS. Consider changing ex_value to "BOX".' + elif sobj.OPT_MASK is None: + msg = f'Excluding {sobj.NAME} in {spec1d_file} because of missing OPT_MASK. Consider changing ex_value to "BOX".' + else: + if len(sobj.OPT_COUNTS[sobj.OPT_MASK]) == 0: + msg = f'Excluding {sobj.NAME} in {spec1d_file} because all of OPT_COUNTS was masked out. Consider changing ex_value to "BOX".' + + if msg is not None: + msgs.warn(msg) + excluded_messages.append(msg) + continue - if par['coadd1d']['ex_value'] == 'BOX' and sobj.BOX_COUNTS is None: - msg = f'Excluding {sobj.NAME} in {spec1d_file} because of missing BOX_COUNTS. Consider changing ex_value to "OPT".' - msgs.warn(msg) - excluded_messages.append(msg) - continue + if par['coadd1d']['ex_value'] == 'BOX': + msg = None + if sobj.BOX_COUNTS is None: + msg = f'Excluding {sobj.NAME} in {spec1d_file} because of missing BOX_COUNTS. Consider changing ex_value to "OPT".' + elif sobj.BOX_MASK is None: + msg = f'Excluding {sobj.NAME} in {spec1d_file} because of missing BOX_MASK. Consider changing ex_value to "OPT".' + else: + if len(sobj.BOX_COUNTS[sobj.BOX_MASK]) == 0: + msg = f'Excluding {sobj.NAME} in {spec1d_file} because all of BOX_COUNTS was masked out. Consider changing ex_value to "OPT".' + + if msg is not None: + msgs.warn(msg) + excluded_messages.append(msg) + continue filtered_objects.append(source_object) return (filtered_objects, excluded_messages) +def read_spec1d_files(par, spec1d_files, failure_msgs): + """ + Read spec1d files. + + Args: + par (`obj`:pypeit.par.pypeitpar.PypeItPar): + Parameters for collating, fluxing, and coadding. + spec1d_files (list of str): + List of spec1d files to read. + failure_msgs(list of str): + Return parameter describing any failures that occurred when reading. + + Returns: + list of str: The SpecObjs objects that were successfully read. + list of str: The spec1d files that were successfully read. + """ + + specobjs_list = [] + good_spec1d_files = [] + for spec1d_file in spec1d_files: + try: + sobjs = SpecObjs.from_fitsfile(spec1d_file, chk_version = par['collate1d']['chk_version']) + specobjs_list.append(sobjs) + good_spec1d_files.append(spec1d_file) + except Exception as e: + formatted_exception = traceback.format_exc() + msgs.warn(formatted_exception) + msgs.warn(f"Failed to read {spec1d_file}, skipping it.") + failure_msgs.append(f"Failed to read {spec1d_file}, skipping it.") + failure_msgs.append(formatted_exception) + + return specobjs_list, good_spec1d_files + def flux(par, spectrograph, spec1d_files, failed_fluxing_msgs): """ Flux calibrate spec1d files using archived sens func files. Args: - par (`obj`:pypeit.par.pypeitpar.PypeItPar): + par (:class:`~pypeit.par.pypeitpar.PypeItPar`): Parameters for collating, fluxing, and coadding. - spectrograph (`obj`:pypeit.spectrographs.spectrograph): + spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): Spectrograph for the files to flux. spec1d_files (list of str): List of spec1d files to flux calibrate. @@ -244,8 +300,7 @@ def flux(par, spectrograph, spec1d_files, failed_fluxing_msgs): # Flux calibrate the spec1d file try: msgs.info(f"Running flux calibrate on {spec1d_file}") - FxCalib = fluxcalibrate.FluxCalibrate.get_instance([spec1d_file], [sens_file], - par=par['fluxcalib']) + FxCalib = fluxcalibrate.flux_calibrate([spec1d_file], [sens_file], par=par['fluxcalib'], chk_version=par['collate1d']['chk_version']) flux_calibrated_files.append(spec1d_file) except Exception: @@ -359,13 +414,18 @@ def coadd(par, coaddfile, source): """coadd the spectra for a given source. Args: - par (`obj`:Collate1DPar): Paramters for the coadding - source (`obj`:SourceObject): The SourceObject with information on - which files and spectra to coadd. + par (:class:`~pypeit.par.pypeitpar.Collate1DPar`): + Parameters for the coadding + source (:class:`~pypeit.core.collate.SourceObject`): + The SourceObject with information on which files and spectra to + coadd. """ # Set destination file for coadding par['coadd1d']['coaddfile'] = coaddfile + # Whether to be forgiving of data model versions + par['coadd1d']['chk_version'] = par['collate1d']['chk_version'] + # Determine if we should coadd flux calibrated data flux_key = par['coadd1d']['ex_value'] + "_FLAM" @@ -434,7 +494,7 @@ def write_warnings(par, excluded_obj_msgs, failed_source_msgs, spec1d_failure_ms failed_source_msgs (:obj:`list` of :obj:`str`): Messages about which objects failed coadding and why. - spec1d_failure_msgs (:obj:)`list` of :obj:`str`): + spec1d_failure_msgs (:obj:`list` of :obj:`str`): Messages about failures with spec1d files and why. """ @@ -547,6 +607,9 @@ def build_parameters(args): if args.refframe is not None: params['collate1d']['refframe'] = args.refframe + if args.chk_version is True: + params['collate1d']['chk_version'] = True + return params, spectrograph, spec1d_files def create_report_archive(par): @@ -570,8 +633,8 @@ def create_report_archive(par): COADDED_SPEC1D_HEADER_KEYS = ['DISPNAME', 'DECKER', 'BINNING', 'MJD', 'AIRMASS', 'EXPTIME','GUIDFWHM', 'PROGPI', 'SEMESTER', 'PROGID'] COADDED_SPEC1D_COLUMN_NAMES = ['dispname', 'slmsknam', 'binning', 'mjd', 'airmass', 'exptime','guidfwhm', 'progpi', 'semester', 'progid'] - COADDED_SOBJ_KEYS = ['MASKDEF_OBJNAME', 'MASKDEF_ID', 'NAME', 'DET', 'RA', 'DEC', 'S2N', 'MASKDEF_EXTRACT', 'WAVE_RMS'] - COADDED_SOBJ_COLUMN_NAMES = ['maskdef_objname', 'maskdef_id', 'pypeit_name', 'det', 'objra', 'objdec', 's2n', 'maskdef_extract', 'wave_rms'] + COADDED_SOBJ_KEYS = ['MASKDEF_OBJNAME', 'MASKDEF_ID', 'NAME', 'DET', 'RA', 'DEC', 'MASKDEF_OBJMAG', 'MASKDEF_OBJMAG_BAND', 'S2N', 'MASKDEF_EXTRACT', 'WAVE_RMS'] + COADDED_SOBJ_COLUMN_NAMES = ['maskdef_objname', 'maskdef_id', 'pypeit_name', 'det', 'objra', 'objdec', 'maskdef_objmag', 'maskdef_objmag_band', 's2n', 'maskdef_extract', 'wave_rms'] report_names = ['filename'] + \ COADDED_SOBJ_COLUMN_NAMES + \ @@ -629,6 +692,8 @@ def get_parser(cls, width=None): 'F| value are skipped, else all wavelength rms values are accepted.\n' 'F| refframe Perform reference frame correction prior to coadding.\n' f'F| Options are {pypeitpar.WavelengthSolutionPar.valid_reference_frames()}. Defaults to None.\n' + 'F| chk_version If true, spec1ds and archival sensfuncs must match the currently\n' + 'F| supported versions. If false (the default) version numbers are not checked.\n' '\n' 'F|spec1d read\n' 'F|\n' @@ -655,6 +720,7 @@ def get_parser(cls, width=None): parser.add_argument("--wv_rms_thresh", type=float, default = None, help=blank_par.descr['wv_rms_thresh']) parser.add_argument("--refframe", type=str, default = None, choices = pypeitpar.WavelengthSolutionPar.valid_reference_frames(), help=blank_par.descr['refframe']) + parser.add_argument('--chk_version', action = 'store_true', help=blank_par.descr['chk_version']) parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename collate_1d_YYYYMMDD-HHMM.log') @@ -725,12 +791,14 @@ def main(args): refframe_correction(par, spectrograph, spec1d_files, spec1d_failure_msgs) + # Read in the spec1d files + specobjs_to_coadd, spec1d_files = read_spec1d_files(par, spec1d_files, spec1d_failure_msgs) + # Build source objects from spec1d file, this list is not collated - source_objects = SourceObject.build_source_objects(spec1d_files, + source_objects = SourceObject.build_source_objects(specobjs_to_coadd, spec1d_files, par['collate1d']['match_using']) - # Filter based on the coadding ex_value, and the exclude_serendip - # boolean + # Filter out unwanted SpecObj objects based on parameters (objects_to_coadd, excluded_obj_msgs) = exclude_source_objects(source_objects, exclude_map, par) # Collate the spectra diff --git a/pypeit/scripts/compare_sky.py b/pypeit/scripts/compare_sky.py index eabc26f2e7..c9cbb5b48e 100644 --- a/pypeit/scripts/compare_sky.py +++ b/pypeit/scripts/compare_sky.py @@ -6,6 +6,7 @@ .. include:: ../include/links.rst """ +import argparse from pypeit.scripts import scriptbase @@ -25,18 +26,16 @@ def get_parser(cls, width=None): parser.add_argument('--scale_user', default=1., type=float, help='Scale user spectrum by a factor') parser.add_argument('--test', default=False, action='store_true', - help='Load files but do not show plot') + help=argparse.SUPPRESS) return parser # Script to run XSpec from the command line or ipython @staticmethod def main(args): - import os + import matplotlib.pyplot as plt - from matplotlib import pyplot as plt - - from linetools.spectra.io import readspec + import linetools.spectra.io from pypeit import data @@ -57,7 +56,7 @@ def main(args): ikwargs['sig_tag'] = 'BOX_COUNTS_SIG' # Load user file - user_sky = readspec(args.file, exten=exten, **ikwargs) + user_sky = linetools.spectra.io.readspec(args.file, exten=exten, **ikwargs) # Load sky spec arx_sky = data.load_sky_spectrum(args.skyfile) diff --git a/pypeit/scripts/flux_calib.py b/pypeit/scripts/flux_calib.py index 05fabf4ab9..2405fd0cdc 100644 --- a/pypeit/scripts/flux_calib.py +++ b/pypeit/scripts/flux_calib.py @@ -57,8 +57,8 @@ def get_parser(cls, width=None): "specify no sensfiles and use an archived one.\n" "Archived sensfiles are available for the following spectrographs: " + ",".join(SensFileArchive.supported_spectrographs()) + "\n\n") - parser.add_argument("--debug", default=False, action="store_true", - help="show debug plots?") +# parser.add_argument("--debug", default=False, action="store_true", +# help="show debug plots?") parser.add_argument("--par_outfile", default='fluxing.par', action="store_true", help="Output to save the parameters") parser.add_argument('-v', '--verbosity', type=int, default=1, @@ -109,9 +109,7 @@ def main(args): 'Run pypeit_flux_calib --help for information on the format') # Instantiate - FxCalib = fluxcalibrate.FluxCalibrate.get_instance( - fluxFile.filenames, sensfiles, - par=par['fluxcalib'], debug=args.debug) + fluxcalibrate.flux_calibrate(fluxFile.filenames, sensfiles, par=par['fluxcalib']) msgs.info('Flux calibration complete') return 0 diff --git a/pypeit/scripts/flux_setup.py b/pypeit/scripts/flux_setup.py index ef6a79baeb..153bfe9288 100644 --- a/pypeit/scripts/flux_setup.py +++ b/pypeit/scripts/flux_setup.py @@ -6,7 +6,7 @@ """ import os -import time +from pathlib import Path import numpy as np @@ -16,17 +16,44 @@ from pypeit import io from pypeit.scripts import scriptbase from pypeit import inputfiles +from pypeit.spectrographs.util import load_spectrograph -# TODO -- We need a test of this script +def match_spec1ds_to_sensfuncs(spectrograph_name, spec1dfiles, sensfiles): + """ + This needs a docstring + """ + result_map = {} + spectrograph = load_spectrograph(spectrograph_name) + + # Read configurations of each sensfile + sens_configs = [] + for sensfile in sensfiles: + with io.fits_open(sensfile) as hdul: + header = hdul[0].header + + sens_configs.append({config_key: header[config_key] for config_key in spectrograph.configuration_keys() if config_key in header}) + + # Read the configurations of each spec1d file, and try to find a matching sensfunc file + for spec1dfile in spec1dfiles: + with io.fits_open(spec1dfile) as hdul: + header = hdul[0].header + + spec1d_config = {config_key: header[config_key] for config_key in spectrograph.configuration_keys() if config_key in header} + for i, sens_config in enumerate(sens_configs): + if spectrograph.same_configuration([spec1d_config, sens_config],check_keys=False): + result_map[spec1dfile.name] = sensfiles[i] + break + + return result_map class FluxSetup(scriptbase.ScriptBase): @classmethod def get_parser(cls, width=None): - parser = super().get_parser(description='Setup to perform flux calibration', + parser = super().get_parser(description='Setup configuration files to perform flux calibration, 1D coadding, and telluric correction.', width=width, formatter=scriptbase.SmartFormatter) - parser.add_argument("sci_path", type=str, help="Path for Science folder") - + parser.add_argument("paths", type=str, nargs='+', help="One or more paths for Science folders or sensitivity functions. Sensitivity functions must start with 'sens_' to be detected.") + parser.add_argument("--name", type=str, default=None, help="The base name to use for the output files. Defaults to the instrument name is used.") parser.add_argument("--objmodel", type=str, default='qso', choices=['qso', 'star', 'poly'], help='R|science object model used in the telluric fitting. The ' 'options are:\n\n' @@ -46,59 +73,73 @@ def main(args): """ This setups PypeIt input files for fluxing, coadding, and telluric corrections. It will produce three files named as - your_spectragraph.flux, your_spectragraph.coadd1d, and - your_spectragraph.tell. + name.flux, name.coadd1d, and name.tell. "name" defaults to the + spectrograph name but can be overriden on the command line. + """ - allfiles = os.listdir(args.sci_path) - allfiles = np.sort(allfiles) + allfiles = [] + for path in args.paths: + allfiles += Path(path).iterdir() spec1dfiles = [] spec2dfiles = [] spec1dinfos = [] + unique_paths = set() + sensfiles = [] for ifile in allfiles: - if ('spec1d' in ifile) and ('.fits' in ifile): + if ('spec1d' in ifile.name) and ('.fits' in ifile.name): spec1dfiles.append(ifile) - elif ('spec2d' in ifile) and ('.fits' in ifile): + unique_paths.add(str(ifile.parent)) + elif ('spec2d' in ifile.name) and ('.fits' in ifile.name): spec2dfiles.append(ifile) - elif ('spec1d' in ifile) and ('.txt' in ifile): + elif ('spec1d' in ifile.name) and ('.txt' in ifile.name): spec1dinfos.append(ifile) + elif ifile.name.startswith('sens_') and ('.fits' in ifile.name): + sensfiles.append(ifile) + unique_paths.add(str(ifile.parent)) else: - msgs.warn('{:} is not a standard PypeIt output.'.format(ifile)) + msgs.info('{:} is not a standard PypeIt output, skipping.'.format(ifile)) if len(spec2dfiles) > len(spec1dfiles): msgs.warn('The following exposures do not have 1D extractions:') for ii in range(len(spec2dfiles)): - if not os.path.exists(os.path.join(args.sci_path, - spec2dfiles[ii].replace('spec2d','spec1d'))): + if (spec2dfiles[ii].parent / spec2dfiles[ii].name.replace("spec2d", "spec1d")).exists(): msgs.info('\t {:}'.format(spec2dfiles[ii])) if len(spec1dfiles) > 0: - par = io.fits_open(os.path.join( - args.sci_path, spec1dfiles[0])) + with io.fits_open(spec1dfiles[0]) as hdul: - ## fluxing pypeit file - spectrograph = par[0].header['PYP_SPEC'] - pypeline = par[0].header['PYPELINE'] + # Get basic configuration info from first spec1d + spectrograph = hdul[0].header['PYP_SPEC'] + pypeline = hdul[0].header['PYPELINE'] + if args.name is None: + output_basename = spectrograph + else: + output_basename = args.name + + # Determine how to map sensitivity functions to spec1d files + sensfile_mapping=match_spec1ds_to_sensfuncs(spectrograph, spec1dfiles, sensfiles) + + ## fluxing pypeit file # Build the bits and pieces cfg_lines = ['[fluxcalib]'] cfg_lines += [' extinct_correct = False # Set to True if your SENSFUNC derived with the UVIS algorithm\n'] cfg_lines += ['# Please add your SENSFUNC file name below before running pypeit_flux_calib'] data = Table() - data['filename'] = spec1dfiles - data['sensfile'] = '' + data['filename'] = [x.name for x in spec1dfiles] + data['sensfile'] = ['' if x.name not in sensfile_mapping else sensfile_mapping[x.name].name for x in spec1dfiles] # Instantiate fluxFile = inputfiles.FluxFile( config=cfg_lines, - file_paths = [args.sci_path], + file_paths = unique_paths, data_table=data) # Write - flux_file = f'{spectrograph}.flux' + flux_file = f'{output_basename}.flux' fluxFile.write(flux_file) ## coadd1d pypeit file cfg_lines = ['[coadd1d]'] cfg_lines += [' coaddfile = YOUR_OUTPUT_FILE_NAME # Please set your output file name'] - cfg_lines += [' sensfuncfile = YOUR_SENSFUNC_FILE # Please set your SENSFUNC file name. Only required for Echelle'] if pypeline == 'Echelle': cfg_lines += [' wave_method = velocity # creates a uniformly space grid in log10(lambda)\n'] else: @@ -106,11 +147,14 @@ def main(args): cfg_lines += ['# This file includes all extracted objects. You need to figure out which object you want to \n'+\ '# coadd before running pypeit_coadd_1dspec!!!'] + if pypeline == 'Echelle': + cfg_lines += ['# For Echelle spectrographs, please double check the sensfunc file and setup id\n'] all_specfiles, all_obj = [], [] for ii in range(len(spec1dfiles)): - meta_tbl = Table.read(os.path.join(args.sci_path, spec1dfiles[ii]).replace('.fits', '.txt'), + txtinfofile = spec1dfiles[ii].parent / (spec1dfiles[ii].stem + ".txt") + meta_tbl = Table.read(txtinfofile, format='ascii.fixed_width') _, indx = np.unique(meta_tbl['name'],return_index=True) objects = meta_tbl[indx] @@ -118,15 +162,49 @@ def main(args): all_specfiles.append(spec1dfiles[ii]) all_obj.append(objects['name'][jj]) data = Table() - data['filename'] = all_specfiles + data['filename'] = [str(x.name) for x in all_specfiles] data['obj_id'] = all_obj + if pypeline == 'Echelle': + + if len(sensfiles) > 1: + # If there are multiple sensfunc files, try to set sensibile values + # for the 'sensfile' and 'setup_id' columns + all_sensfiles = [] + all_setup_ids = [] + for spec1d in data['filename']: + if spec1d in sensfile_mapping: + sensfile = sensfile_mapping[spec1d] + # Use the index of the sensfile in sensfiles as the setup id, + # converted to a letter. Hopefully we don't get more than 26 + setup_id = chr(ord('A') + sensfiles.index(sensfile)) + else: + sensfile = 'SENSFUNC FILE' + setup_id = 'default' + + all_sensfiles.append(sensfile.name) + all_setup_ids.append(setup_id) + data['sensfile'] = all_sensfiles + data['setup_id'] = all_setup_ids + + else: + # Just use one default sensfunc file and one setup. + if len(sensfiles) == 1: + default_sensfile = sensfiles[0].name # Use the first sensfunc and only sensfunc + else: + default_sensfile = 'SENSFUNC FILE' # Use a dummy sensfunc filename + + data['sensfile'] = [default_sensfile] + ([''] * (len(all_obj)-1)) + data['setup_id'] = ['A'] + ([''] * (len(all_obj)-1)) + + + # Instantiate coadd1dFile = inputfiles.Coadd1DFile( config=cfg_lines, - file_paths = [args.sci_path], + file_paths = unique_paths, data_table=data) # Write - coadd1d_file = '{:}.coadd1d'.format(spectrograph) + coadd1d_file = '{:}.coadd1d'.format(output_basename) coadd1dFile.write(coadd1d_file) ## tellfit pypeit file @@ -147,7 +225,7 @@ def main(args): tellFile = inputfiles.TelluricFile( config=cfg_lines) # Write - tellfit_file = f'{spectrograph}.tell' + tellfit_file = f'{output_basename}.tell' tellFile.write(tellfit_file) diff --git a/pypeit/scripts/identify.py b/pypeit/scripts/identify.py index e70e28af7d..50f0a45261 100644 --- a/pypeit/scripts/identify.py +++ b/pypeit/scripts/identify.py @@ -39,6 +39,9 @@ def get_parser(cls, width=None): help="Save the solutions, despite the RMS") parser.add_argument('--rescale_resid', default=False, action='store_true', help="Rescale the residual plot to include all points?") + parser.add_argument('-v', '--verbosity', type=int, default=1, + help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' + 'Level 2 writes a log with filename identify_YYYYMMDD-HHMM.log') return parser @staticmethod @@ -55,6 +58,9 @@ def main(args): from pypeit import slittrace from pypeit.images.buildimage import ArcImage + # Set the verbosity, and create a logfile if verbosity == 2 + msgs.set_logfile_and_verbosity('identify', args.verbosity) + # Load the Arc file msarc = ArcImage.from_file(args.arc_file) diff --git a/pypeit/scripts/parse_slits.py b/pypeit/scripts/parse_slits.py index 5bcc981889..07e887081b 100644 --- a/pypeit/scripts/parse_slits.py +++ b/pypeit/scripts/parse_slits.py @@ -9,13 +9,25 @@ from pypeit.scripts import scriptbase from pypeit import slittrace +from pypeit import msgs + +from IPython import embed def print_slits(slits): # bitmask bitmask = slittrace.SlitTraceBitMask() - print(f'{"SpatID":<8} {"MaskID":<8} {"MaskOFF (pix)":<14} {"Flags":<20}') - for slit_idx, slit_spat in enumerate(slits.spat_id): + if slits.pypeline in ['MultiSlit', 'IFU']: + slitord_id = slits.spat_id + slit_label = 'SpatID' + elif slits.pypeline == 'Echelle': + slitord_id = slits.slitord_id + slit_label = 'Order' + else: + msgs.error('Not ready for this pypeline: {0}'.format(slits.pypeline)) + print(f'{slit_label:<8} {"MaskID":<8} {"MaskOFF (pix)":<14} {"Flags":<20}') + # TODO JFH No need to print out the MaskID and MaskOFF for echelle + for slit_idx, slit_spat in enumerate(slitord_id): maskdefID = 0 if slits.maskdef_id is None else slits.maskdef_id[slit_idx] maskoff = 0 if slits.maskdef_offset is None else slits.maskdef_offset # Flags @@ -48,18 +60,33 @@ def main(pargs): from pypeit import spec2dobj - try: - slits = slittrace.SlitTraceSet.from_file(pargs.input_file)#, chk_version=False) - # TODO: Should this specify the type of exception to pass? - except: - pass + + # What kind of file are we?? + hdul = fits.open(pargs.input_file) + head0 = hdul[0].header + head1 = hdul[1].header + if 'MSTRTYP' in head0.keys() and head0['MSTRTYP'].strip() == 'Slits': + file_type = 'Slits' + elif 'CALIBTYP' in head1.keys() and head1['CALIBTYP'].strip() == 'Slits': + file_type = 'Slits' + elif 'PYP_CLS' in head0.keys() and head0['PYP_CLS'].strip() == 'AllSpec2DObj': + file_type = 'AllSpec2D' else: - print_slits(slits) - return + raise IOError("Bad file type input!") + + if file_type == 'Slits': + try: + slits = slittrace.SlitTraceSet.from_file(pargs.input_file, chk_version=False) + # TODO: Should this specify the type of exception to pass? + except: + pass + else: + print_slits(slits) + return try: allspec2D = spec2dobj.AllSpec2DObj.from_fits(pargs.input_file, chk_version=False) - # TODO: Should this specify the type of exception to pass? + # TODO: Should this specify the type of exception to pass? except: pass else: diff --git a/pypeit/scripts/ql.py b/pypeit/scripts/ql.py index 8346a3aacd..f1a2224d3b 100644 --- a/pypeit/scripts/ql.py +++ b/pypeit/scripts/ql.py @@ -32,6 +32,7 @@ #. Consider not writing out but return instead .. include:: ../include/links.rst + """ from pathlib import Path import time @@ -143,7 +144,7 @@ def quicklook_regroup(fitstbl): **This function directly alters the input object!** Args: - fitstbl (:class:~pypeit.metadata.PypeItMetaData`): + fitstbl (:class:`~pypeit.metadata.PypeItMetaData`): Metadata table for frames to be processed. """ comb_strt = 0 @@ -228,7 +229,8 @@ def generate_sci_pypeitfile(redux_path:str, ref_calib_dir (`Path`_): Path with the pre-processed calibration frames. A symlink will be created to this directory from within ``redux_path`` to mimic the - location of the calibrations expected by :class:`~pypeit.PypeIt`. + location of the calibrations expected by + :class:`~pypeit.pypeit.PypeIt`. ps_sci (:class:`~pypeit.pypeitsetup.PypeItSetup`): Setup object for the science frame(s) only. det (:obj:`str`, optional): @@ -353,7 +355,10 @@ def generate_sci_pypeitfile(redux_path:str, detname = slitTrace.detname # Mosaic? mosaic = True if detname[0:3] == 'MSC' else False - det_id = np.where(ps_sci.spectrograph.list_detectors(mosaic=mosaic) == detname)[0] + # if mosaic is False list_detectors() returns a 2D array for some spectrographs, therefore we flatten it + spec_list_dets = ps_sci.spectrograph.list_detectors(mosaic=mosaic) if mosaic \ + else ps_sci.spectrograph.list_detectors().flatten() + det_id = np.where(spec_list_dets == detname)[0] if len(det_id) == 0: detname = None # Reset continue # TODO: or fault? diff --git a/pypeit/scripts/scriptbase.py b/pypeit/scripts/scriptbase.py index 4c935d75a6..0e40eef31d 100644 --- a/pypeit/scripts/scriptbase.py +++ b/pypeit/scripts/scriptbase.py @@ -3,6 +3,7 @@ .. include common links, assuming primary doc root is up one directory .. include:: ../include/links.rst + """ from IPython import embed diff --git a/pypeit/scripts/sensfunc.py b/pypeit/scripts/sensfunc.py index 9d881d6b65..d6325f7a03 100644 --- a/pypeit/scripts/sensfunc.py +++ b/pypeit/scripts/sensfunc.py @@ -57,6 +57,15 @@ def get_parser(cls, width=None): 'provided but with .fits trimmed off if it is in the filename.') parser.add_argument("-s", "--sens_file", type=str, help='Configuration file with sensitivity function parameters') + parser.add_argument("-f", "--flatfile", type=str, + help="R|Use the flat file for computing the sensitivity " + "function. Note that it is not possible to set --flatfile and " + "simultaneously use a .sens file with the --sens_file option. If " + "you are using a .sens file, set the flatfile there via e.g.:\n\n" + "F| [sensfunc]\n" + "F| flatfile = Calibrations/Flat_A_0_DET01.fits\n" + "\nWhere Flat_A_0_DET01.fits is the flat file in your Calibrations directory\n") + parser.add_argument("--debug", default=False, action="store_true", help="show debug plots?") parser.add_argument("--par_outfile", default='sensfunc.par', @@ -72,8 +81,6 @@ def main(args): import os - from astropy.io import fits - from pypeit import msgs from pypeit import inputfiles from pypeit import io @@ -93,6 +100,14 @@ def main(args): " [sensfunc]\n" " algorithm = IR\n" "\n") + if args.flatfile is not None and args.sens_file is not None: + msgs.error("It is not possible to set --flatfile and simultaneously use a .sens " + "file via the --sens_file option. If you are using a .sens file set the " + "flatfile there via:\n" + "\n" + " [sensfunc]\n" + " flatfile = Calibrations/Flat_A_0_DET01.fits'\n" + "\n") if args.multi is not None and args.sens_file is not None: msgs.error("It is not possible to set --multi and simultaneously use a .sens file via " @@ -103,21 +118,21 @@ def main(args): " multi_spec_det = 3,7\n" "\n") - # Determine the spectrograph - hdul = io.fits_open(args.spec1dfile) - spectrograph = load_spectrograph(hdul[0].header['PYP_SPEC']) - spectrograph_config_par = spectrograph.config_specific_par(hdul) - - # Construct a primary FITS header that includes the spectrograph's - # config keys for inclusion in the output sensfunc file - primary_hdr = io.initialize_header() - add_keys = ( - ['PYP_SPEC', 'DATE-OBS', 'TELESCOP', 'INSTRUME', 'DETECTOR'] - + spectrograph.configuration_keys() + spectrograph.raw_header_cards() - ) - for key in add_keys: - if key.upper() in hdul[0].header.keys(): - primary_hdr[key.upper()] = hdul[0].header[key.upper()] + # Determine the spectrograph and generate the primary FITS header + with io.fits_open(args.spec1dfile) as hdul: + spectrograph = load_spectrograph(hdul[0].header['PYP_SPEC']) + spectrograph_config_par = spectrograph.config_specific_par(hdul) + + # Construct a primary FITS header that includes the spectrograph's + # config keys for inclusion in the output sensfunc file + primary_hdr = io.initialize_header() + add_keys = ( + ['PYP_SPEC', 'DATE-OBS', 'TELESCOP', 'INSTRUME', 'DETECTOR'] + + spectrograph.configuration_keys() + spectrograph.raw_header_cards() + ) + for key in add_keys: + if key.upper() in hdul[0].header.keys(): + primary_hdr[key.upper()] = hdul[0].header[key.upper()] # If the .sens file was passed in read it and overwrite default parameters @@ -134,6 +149,11 @@ def main(args): if args.algorithm is not None: par['sensfunc']['algorithm'] = args.algorithm + # If flatfile was provided override defaults. Note this does undo .sens + # file since they cannot both be passed + if args.flatfile is not None: + par['sensfunc']['flatfile'] = args.flatfile + # If multi was set override defaults. Note this does undo .sens file # since they cannot both be passed if args.multi is not None: diff --git a/pypeit/scripts/setup.py b/pypeit/scripts/setup.py index 901680f652..257577c08a 100644 --- a/pypeit/scripts/setup.py +++ b/pypeit/scripts/setup.py @@ -43,8 +43,9 @@ def get_parser(cls, width=None): help='Include the background-pair columns for the user to edit') parser.add_argument('-m', '--manual_extraction', default=False, action='store_true', help='Include the manual extraction column for the user to edit') - parser.add_argument('-v', '--verbosity', type=int, default=2, - help='Level of verbosity from 0 to 2.') + parser.add_argument('-v', '--verbosity', type=int, default=1, + help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' + 'Level 2 writes a log with filename setup_YYYYMMDD-HHMM.log') parser.add_argument('-k', '--keep_bad_frames', default=False, action='store_true', help='Keep all frames, even if they are identified as having ' 'bad/unrecognized configurations that cannot be reduced by ' @@ -68,9 +69,13 @@ def main(args): import time from pathlib import Path + from pypeit import msgs from pypeit.pypeitsetup import PypeItSetup from pypeit.calibrations import Calibrations + # Set the verbosity, and create a logfile if verbosity == 2 + msgs.set_logfile_and_verbosity('setup', args.verbosity) + if args.spectrograph is None: raise IOError('spectrograph is a required argument. Use the -s, --spectrograph ' 'command-line option.') diff --git a/pypeit/scripts/setup_coadd2d.py b/pypeit/scripts/setup_coadd2d.py index c534399a94..957fb6e52c 100644 --- a/pypeit/scripts/setup_coadd2d.py +++ b/pypeit/scripts/setup_coadd2d.py @@ -18,8 +18,9 @@ def get_parser(cls, width=None): input_group = parser.add_mutually_exclusive_group(required=True) input_group.add_argument('-f', '--pypeit_file', type=str, default=None, help='PypeIt reduction file') - input_group.add_argument('-d', '--science_dir', type=str, default=None, - help='Directory with spec2d files to stack') + input_group.add_argument('-d', '--science_dir', type=str, nargs='+', default=None, + help='One or more directories with spec2d files to stack ' + '(use wildcard to specify multiple directories).') parser.add_argument('--keep_par', dest='clean_par', default=True, action='store_false', help='Propagate all parameters from the pypeit file to the coadd2d ' @@ -43,8 +44,20 @@ def get_parser(cls, width=None): 'use --det 1 5; to coadd mosaics made up of detectors 1,5 and ' '3,7, you would use --det 1,5 3,7') parser.add_argument('--only_slits', type=str, nargs='+', - help='A space-separated set of slits to coadd. If not provided, all ' - 'slits are coadded.') + help='A space-separated set of slits to coadd. Example syntax for ' + 'argument is DET01:175,DET02:205 or MSC02:2234. If not ' + 'provided, all slits are coadded. If both --det and ' + '--only_slits are provided, --det will be ignored. This and ' + '--exclude_slits are mutually exclusive. If both are provided, ' + '--only_slits takes precedence.') + parser.add_argument('--exclude_slits', type=str, nargs='+', + help='A space-separated set of slits to exclude in the coaddition. ' + 'This and --only_slits are mutually exclusive. ' + 'If both are provided, --only_slits takes precedence.') + parser.add_argument('--spat_toler', type=int, default=None, + help='Desired tolerance in spatial pixel used to identify ' + 'slits in different exposures. If not provided, the default ' + 'value for the specific instrument/configuration is used.') parser.add_argument('--offsets', type=str, default=None, help='Spatial offsets to apply to each image; see the ' '[coadd2d][offsets] parameter. Options are restricted here to ' @@ -82,7 +95,7 @@ def main(args): pypeitFile = None par = None spec_name = None - sci_dir = Path(args.science_dir).resolve() + sci_dirs = [Path(sc).resolve() for sc in args.science_dir] else: # Read the pypeit file pypeitFile = inputfiles.PypeItFile.from_file(args.pypeit_file) @@ -97,17 +110,22 @@ def main(args): # based on the parameter value, then try to base it on the parent # directory of the provided pypeit file. The latter is critical to the # vet_test in the dev-suite. - sci_dir = Path(par['rdx']['redux_path']).resolve() / par['rdx']['scidir'] - if not sci_dir.exists(): - sci_dir = Path(args.pypeit_file).resolve().parent / par['rdx']['scidir'] + sci_dirs = [Path(par['rdx']['redux_path']).resolve() / par['rdx']['scidir']] + if not sci_dirs[0].exists(): + sci_dirs = [Path(args.pypeit_file).resolve().parent / par['rdx']['scidir']] - if not sci_dir.exists(): - msgs.error(f'Science directory not found: {sci_dir}\n') + sci_dirs_exist = [sc.exists() for sc in sci_dirs] + if not np.all(sci_dirs_exist): + msgs_string = 'The following science directories do not exist:' + msgs.newline() + for s in np.array(sci_dirs)[np.logical_not(sci_dirs_exist)]: + msgs_string += f'{s}' + msgs.newline() + msgs.error(msgs_string) # Find all the spec2d files: - spec2d_files = sorted(sci_dir.glob('spec2d*')) + spec2d_files = np.concatenate([sorted(sci_dir.glob('spec2d*')) for sci_dir in sci_dirs]).tolist() + if len(spec2d_files) == 0: - msgs.error(f'No spec2d files found in {sci_dir}.') + msgs.error(f'No spec2d files.') if spec_name is None: with io.fits_open(spec2d_files[0]) as hdu: @@ -146,31 +164,33 @@ def main(args): cfg = {} if args.clean_par or pypeitFile is None else dict(pypeitFile.config) if par is None: # This is only used to get the default offsets and weights (see below) - par = load_spectrograph(spec_name).default_pypeit_par() - utils.add_sub_dict(cfg, 'rdx') - cfg['rdx']['scidir'] = sci_dir.name - else: - utils.add_sub_dict(cfg, 'rdx') - cfg['rdx']['redux_path'] = par['rdx']['redux_path'] - cfg['rdx']['scidir'] = par['rdx']['scidir'] - cfg['rdx']['qadir'] = par['rdx']['qadir'] - utils.add_sub_dict(cfg, 'calibrations') - cfg['calibrations']['calib_dir'] = par['calibrations']['calib_dir'] + par = load_spectrograph(spec_name).config_specific_par(spec2d_files[0]) + utils.add_sub_dict(cfg, 'rdx') + cfg['rdx']['redux_path'] = par['rdx']['redux_path'] + cfg['rdx']['scidir'] = par['rdx']['scidir'] + cfg['rdx']['qadir'] = par['rdx']['qadir'] + utils.add_sub_dict(cfg, 'calibrations') + cfg['calibrations']['calib_dir'] = par['calibrations']['calib_dir'] utils.add_sub_dict(cfg, 'coadd2d') cfg['coadd2d']['offsets'] = par['coadd2d']['offsets'] \ if args.offsets is None else args.offsets cfg['coadd2d']['weights'] = par['coadd2d']['weights'] \ if args.weights is None else args.weights + cfg['coadd2d']['spat_toler'] = par['coadd2d']['spat_toler'] \ + if args.spat_toler is None else args.spat_toler # Build the default parameters - cfg = CoAdd2D.default_par(spec_name, inp_cfg=cfg, det=args.det, slits=args.only_slits) + cfg = CoAdd2D.default_par(spec_name, inp_cfg=cfg, det=args.det, only_slits=args.only_slits, + exclude_slits=args.exclude_slits) # Create a coadd2D file for each object - # NOTE: Below expect all spec2d files have the same path for obj, files in object_spec2d_files.items(): tbl = Table() tbl['filename'] = [f.name for f in files] - ofile = args.pypeit_file.replace('.pypeit', f'_{obj}.coadd2d') - inputfiles.Coadd2DFile(config=cfg, file_paths=[str(sci_dir)], + ofile_name = f'{spec_name}_{obj}.coadd2d' if args.pypeit_file is None \ + else Path(args.pypeit_file).name.replace('.pypeit', f'_{obj}.coadd2d') + ofile = str(Path(cfg['rdx']['redux_path']) / ofile_name) + + inputfiles.Coadd2DFile(config=cfg, file_paths=[str(sc) for sc in sci_dirs], data_table=tbl).write(ofile) diff --git a/pypeit/scripts/skysub_regions.py b/pypeit/scripts/skysub_regions.py index 5ff9254a36..eb254f308e 100644 --- a/pypeit/scripts/skysub_regions.py +++ b/pypeit/scripts/skysub_regions.py @@ -14,7 +14,7 @@ class SkySubRegions(scriptbase.ScriptBase): @classmethod def get_parser(cls, width=None): - parser = super().get_parser(description='Display a Raw science image and interactively ' + parser = super().get_parser(description='Display a spec2d frame and interactively ' 'define the sky regions using a GUI. Run in the ' 'same folder as your .pypeit file', width=width) diff --git a/pypeit/scripts/tellfit.py b/pypeit/scripts/tellfit.py index 9e2ec8f2d7..20ee30ae75 100644 --- a/pypeit/scripts/tellfit.py +++ b/pypeit/scripts/tellfit.py @@ -16,7 +16,7 @@ def get_parser(cls, width=None): parser = super().get_parser(description='Telluric correct a spectrum', width=width, formatter=scriptbase.SmartFormatter) parser.add_argument("spec1dfile", type=str, - help="spec1d file that will be used for telluric correction.") + help="spec1d or coadd file that will be used for telluric correction.") parser.add_argument("--objmodel", type=str, default=None, choices=['qso', 'star', 'poly'], help='R|science object model used in the fitting. The options are:\n' '\n' diff --git a/pypeit/scripts/view_fits.py b/pypeit/scripts/view_fits.py index 9cebd3ffcf..a98d12f273 100644 --- a/pypeit/scripts/view_fits.py +++ b/pypeit/scripts/view_fits.py @@ -25,8 +25,7 @@ def get_parser(cls, width=None): help='List the extensions only?') parser.add_argument('--proc', default=False, action='store_true', help='Process the image (i.e. orient, overscan subtract, multiply by ' - 'gain) using pypeit.images.buildimage. Note det=mosaic will not ' - 'work with this option') + 'gain) using pypeit.images.buildimage.') parser.add_argument('--bkg_file', type=str, default=None, help='FITS file to be subtracted from the image in file.' '--proc must be set in order for this option to work.') @@ -40,6 +39,8 @@ def get_parser(cls, width=None): 'spectrograph. Using "mosaic" for gemini_gmos, keck_deimos, or ' 'keck_lris will show the mosaic of all detectors.') parser.add_argument('--chname', type=str, default='Image', help='Name of Ginga tab') + parser.add_argument('--showmask', default=False, help='Overplot masked pixels', + action='store_true') parser.add_argument('--embed', default=False, action='store_true', help='Upon completion embed in ipython shell') return parser @@ -64,9 +65,6 @@ def main(args): if args.proc and args.exten is not None: msgs.error('You cannot specify --proc and --exten, since --exten shows the raw image') -# if args.proc and args.det == 'mosaic': -# msgs.error('You cannot specify --proc and --det mosaic, since --mosaic can only ' -# 'display the raw image mosaic') if args.exten is not None and args.det == 'mosaic': msgs.error('You cannot specify --exten and --det mosaic, since --mosaic displays ' 'multiple extensions by definition') @@ -95,7 +93,6 @@ def main(args): mosaic = len(_det) > 1 if not mosaic: _det = _det[0] - if args.proc: # Use the biasframe processing parameters because processing # these frames is independent of any other frames (ie., does not @@ -115,6 +112,8 @@ def main(args): except Exception as e: msgs.error(bad_read_message + f' Original exception -- {type(e).__name__}: {str(e)}') + + Img = Img.sub(bkgImg) img = Img.image @@ -129,6 +128,11 @@ def main(args): display.connect_to_ginga(raise_err=True, allow_new=True) display.show_image(img, chname=args.chname) + if args.showmask: + if not args.proc: + msgs.info("You need to use --proc with --showmask to show the mask. Ignoring your argument") + else: + viewer, ch_mask = display.show_image(Img.bpm, chname="BPM") if args.embed: embed(header=utils.embed_header()) diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index 7b24777f78..33937efba5 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -24,9 +24,13 @@ from pypeit.core import fitting from pypeit.core.wavecal import wvutils from pypeit.core import meta -from pypeit.spectrographs.util import load_spectrograph +from pypeit.core import flat +from pypeit.core.moment import moment1d +from pypeit.spectrographs.util import load_spectrograph from pypeit import datamodel +from pypeit import flatfield + # TODO Add the data model up here as a standard thing using DataContainer. @@ -53,8 +57,12 @@ class SensFunc(datamodel.DataContainer): PypeIt spec1d file for the standard file. sensfile (:obj:`str`): File name for the sensitivity function data. - par (:class:`~pypeit.par.pypeitpar.SensFuncPar`, optional): + par (:class:`~pypeit.par.pypeitpar.SensFuncPar`): The parameters required for the sensitivity function computation. + par_fluxcalib (:class:`~pypeit.par.pypeitpar.FluxCalibratePar`, optional): + The parameters required for flux calibration. These are only used + for flux calibration of the standard star spectrum for the QA plot. + If None, defaults will be used. debug (:obj:`bool`, optional): Run in debug mode, sending diagnostic information to the screen. """ @@ -101,13 +109,17 @@ class SensFunc(datamodel.DataContainer): internals = ['sensfile', 'spectrograph', 'par', + 'par_fluxcalib', 'qafile', 'thrufile', + 'fstdfile', 'debug', + 'sobjs_std', 'wave_cnts', 'counts', 'counts_ivar', 'counts_mask', + 'log10_blaze_function', 'nspec_in', 'norderdet', 'wave_splice', @@ -123,7 +135,7 @@ class SensFunc(datamodel.DataContainer): """Algorithm used for the sensitivity calculation.""" @staticmethod - def empty_sensfunc_table(norders, nspec, ncoeff=1): + def empty_sensfunc_table(norders, nspec, nspec_in, ncoeff=1): """ Construct an empty `astropy.table.Table`_ for the sensitivity function. @@ -132,7 +144,12 @@ def empty_sensfunc_table(norders, nspec, ncoeff=1): norders (:obj:`int`): The number of slits/orders on the detector. nspec (:obj:`int`): - The number of spectral pixels on the detector. + The number of spectral pixels for the zeropoint arrays. + nspec_in (:obj:`int`): + The number of spectral pixels on the detector for the input + standard star spectrum. + ncoeff (:obj:`int`, optional): + Number of coefficients for smooth model fit to zeropoints Returns: `astropy.table.Table`_: Instance of the empty sensitivity @@ -143,6 +160,8 @@ def empty_sensfunc_table(norders, nspec, ncoeff=1): description='Wavelength vector'), table.Column(name='SENS_COUNTS_PER_ANG', dtype=float, length=norders, shape=(nspec,), description='Flux in counts per angstrom'), + table.Column(name='SENS_LOG10_BLAZE_FUNCTION', dtype=float, length=norders, shape=(nspec,), + description='Log10 of the blaze function for each slit/order'), table.Column(name='SENS_ZEROPOINT', dtype=float, length=norders, shape=(nspec,), description='Measured sensitivity zero-point data'), table.Column(name='SENS_ZEROPOINT_GPM', dtype=bool, length=norders, shape=(nspec,), @@ -160,20 +179,30 @@ def empty_sensfunc_table(norders, nspec, ncoeff=1): table.Column(name='WAVE_MIN', dtype=float, length=norders, description='Minimum wavelength included in the fit'), table.Column(name='WAVE_MAX', dtype=float, length=norders, - description='Maximum wavelength included in the fit')]) + description='Maximum wavelength included in the fit'), + table.Column(name='SENS_FLUXED_STD_WAVE', dtype=float, length=norders, shape=(nspec_in,), + description='The wavelength array for the fluxed standard star spectrum'), + table.Column(name='SENS_FLUXED_STD_FLAM', dtype=float, length=norders, shape=(nspec_in,), + description='The F_lambda for the fluxed standard star spectrum'), + table.Column(name='SENS_FLUXED_STD_FLAM_IVAR', dtype=float, length=norders, shape=(nspec_in,), + description='The inverse variance of F_lambda for the fluxed standard star spectrum'), + table.Column(name='SENS_FLUXED_STD_MASK', dtype=bool, length=norders, shape=(nspec_in,), + description='The good pixel mask for the fluxed standard star spectrum ')]) + + # Superclass factory method generates the subclass instance @classmethod - def get_instance(cls, spec1dfile, sensfile, par, debug=False): + def get_instance(cls, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): """ Instantiate the relevant subclass based on the algorithm provided in ``par``. """ return next(c for c in cls.__subclasses__() - if c.__name__ == f"{par['algorithm']}SensFunc")(spec1dfile, sensfile, par=par, - debug=debug) + if c.__name__ == f"{par['algorithm']}SensFunc")(spec1dfile, sensfile, par, + par_fluxcalib=par_fluxcalib, debug=debug) - def __init__(self, spec1dfile, sensfile, par=None, debug=False): + def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): # Instantiate as an empty DataContainer super().__init__() @@ -181,7 +210,7 @@ def __init__(self, spec1dfile, sensfile, par=None, debug=False): # Input and Output files self.spec1df = spec1dfile self.sensfile = sensfile - + self.par = par # Spectrograph header = fits.getheader(self.spec1df) self.PYP_SPEC = header['PYP_SPEC'] @@ -191,10 +220,7 @@ def __init__(self, spec1dfile, sensfile, par=None, debug=False): # spectrograph objects with configuration specific information from # spec1d files. self.spectrograph.dispname = header['DISPNAME'] - - # Get the algorithm parameters - self.par = self.spectrograph.default_pypeit_par()['sensfunc'] if par is None else par - # TODO: Check the type of the parameter object? + self.par_fluxcalib = self.spectrograph.default_pypeit_par()['fluxcalib'] if par_fluxcalib is None else par_fluxcalib # Set the algorithm in the datamodel self.algorithm = self.__class__._algorithm @@ -202,6 +228,7 @@ def __init__(self, spec1dfile, sensfile, par=None, debug=False): # QA and throughput plot filenames self.qafile = sensfile.replace('.fits', '') + '_QA.pdf' self.thrufile = sensfile.replace('.fits', '') + '_throughput.pdf' + self.fstdfile = sensfile.replace('.fits', '') + '_fluxed_std.pdf' # Other self.debug = debug @@ -211,23 +238,25 @@ def __init__(self, spec1dfile, sensfile, par=None, debug=False): self.splice_multi_det = True if self.par['multi_spec_det'] is not None else False # Read in the Standard star data - sobjs_std = specobjs.SpecObjs.from_fitsfile(self.spec1df).get_std( - multi_spec_det=self.par['multi_spec_det']) + self.sobjs_std = specobjs.SpecObjs.from_fitsfile(self.spec1df).get_std(multi_spec_det=self.par['multi_spec_det']) - if sobjs_std is None: + if self.sobjs_std is None: msgs.error('There is a problem with your standard star spec1d file: {:s}'.format(self.spec1df)) + # Unpack standard - wave, counts, counts_ivar, counts_mask, self.meta_spec, header = sobjs_std.unpack_object(ret_flam=False) + wave, counts, counts_ivar, counts_mask, trace_spec, trace_spat, self.meta_spec, header = self.sobjs_std.unpack_object(ret_flam=False) - # Perform any instrument tweaks - wave_twk, counts_twk, counts_ivar_twk, counts_mask_twk \ - = self.spectrograph.tweak_standard(wave, counts, counts_ivar, counts_mask, - self.meta_spec) + # Compute the blaze function + # TODO Make the blaze function optional + log10_blaze_function = self.compute_blaze(wave, trace_spec, trace_spat, par['flatfile']) if par['flatfile'] is not None else None + # Perform any instrument tweaks + wave_twk, counts_twk, counts_ivar_twk, counts_mask_twk, log10_blaze_function_twk = \ + self.spectrograph.tweak_standard(wave, counts, counts_ivar, counts_mask, self.meta_spec, log10_blaze_function=log10_blaze_function) # Reshape to 2d arrays - self.wave_cnts, self.counts, self.counts_ivar, self.counts_mask, self.nspec_in, \ - self.norderdet \ - = utils.spec_atleast_2d(wave_twk, counts_twk, counts_ivar_twk, counts_mask_twk) + self.wave_cnts, self.counts, self.counts_ivar, self.counts_mask, self.log10_blaze_function, self.nspec_in, \ + self.norderdet = utils.spec_atleast_2d(wave_twk, counts_twk, counts_ivar_twk, counts_mask_twk, + log10_blaze_function=log10_blaze_function_twk) # If the user provided RA and DEC use those instead of what is in meta star_ra = self.meta_spec['RA'] if self.par['star_ra'] is None else self.par['star_ra'] @@ -240,6 +269,65 @@ def __init__(self, spec1dfile, sensfile, par=None, debug=False): star_mag=self.par['star_mag'], ra=star_ra, dec=star_dec) + def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, min_blaze_value=1e-3, debug=False): + """ + Compute the blaze function from a flat field image. + + Args: + wave (`numpy.ndarray`_): + Wavelength array. Shape = (nspec, norddet) + trace_spec (`numpy.ndarray`_): + Spectral pixels for the trace of the spectrum. Shape = (nspec, norddet) + trace_spat (`numpy.ndarray`_): + Spatial pixels for the trace of the spectrum. Shape = (nspec, norddet) + flatfile (:obj:`str`): + Filename for the flat field calibration image + box_radius (:obj:`float`, optional): + Radius of the boxcar extraction region used to extract the blaze function in pixels + min_blaze_value (:obj:`float`, optional): + Minimum value of the blaze function. Values below this are clipped and set to this value. Default=1e-3 + debug (:obj:`bool`, optional): + Show plots useful for debugging. Default=False + + Returns: + `numpy.ndarray`_: The log10 blaze function. Shape = (nspec, norddet) + if norddet > 1, else shape = (nspec,) + """ + + + flatImages = flatfield.FlatImages.from_file(flatfile) + + pixelflat_raw = flatImages.pixelflat_raw + pixelflat_norm = flatImages.pixelflat_norm + pixelflat_proc, flat_bpm = flat.flatfield(pixelflat_raw, pixelflat_norm) + + flux_box = moment1d(pixelflat_proc * np.logical_not(flat_bpm), trace_spat, 2 * box_radius, row=trace_spec)[0] + + pixtot = moment1d(pixelflat_proc*0 + 1.0, trace_spat, 2 * box_radius, row=trace_spec)[0] + pixmsk = moment1d(flat_bpm, trace_spat, 2 * box_radius, row=trace_spec)[0] + + mask_box = (pixmsk != pixtot) & np.isfinite(wave) & (wave > 0.0) + + # TODO This is ugly and redundant with spec_atleast_2d, but the order of operations compels me to do it this way + blaze_function = (np.clip(flux_box*mask_box, 1e-3, 1e9)).reshape(-1,1) if flux_box.ndim == 1 else flux_box*mask_box + wave_debug = wave.reshape(-1,1) if wave.ndim == 1 else wave + log10_blaze_function = np.zeros_like(blaze_function) + norddet = log10_blaze_function.shape[1] + for iorddet in range(norddet): + blaze_function_smooth = utils.fast_running_median(blaze_function[:, iorddet], 5) + blaze_function_norm = blaze_function_smooth/blaze_function_smooth.max() + log10_blaze_function[:, iorddet] = np.log10(np.clip(blaze_function_norm, min_blaze_value, None)) + if debug: + plt.plot(wave_debug[:, iorddet], log10_blaze_function[:,iorddet]) + if debug: + plt.show() + + + # TODO It would probably better to just return an array of shape (nspec, norddet) even if norddet = 1, i.e. + # to get rid of this .squeeze() + return log10_blaze_function.squeeze() + + def _bundle(self): """ Bundle the object for writing using @@ -357,12 +445,36 @@ def run(self): if self.splice_multi_det: self.wave_splice, self.zeropoint_splice = self.splice() + # Flux the standard star with this sensitivity function and add it to the output table + self.flux_std() + # Compute the throughput self.throughput, self.throughput_splice = self.compute_throughput() # Write out QA and throughput plots self.write_QA() + def flux_std(self): + """ + Flux the standard star and add it to the sensitivity function table + + """ + # Now flux the standard star + self.sobjs_std.apply_flux_calib(self.par_fluxcalib, self.spectrograph, self) + # TODO assign this to the data model + + # Unpack the fluxed standard + _wave, _flam, _flam_ivar, _flam_mask, _, _, _, _ = self.sobjs_std.unpack_object(ret_flam=True) + # Reshape to 2d arrays + wave, flam, flam_ivar, flam_mask, _, _, _ = utils.spec_atleast_2d(_wave, _flam, _flam_ivar, _flam_mask) + # Store in the sens table + self.sens['SENS_FLUXED_STD_WAVE'] = wave.T + self.sens['SENS_FLUXED_STD_FLAM'] = flam.T + self.sens['SENS_FLUXED_STD_FLAM_IVAR'] = flam_ivar.T + self.sens['SENS_FLUXED_STD_MASK'] = flam_mask.T + + + def eval_zeropoint(self, wave, iorddet): """ Dummy method, overloaded by subclasses @@ -519,7 +631,7 @@ def write_QA(self): # Plot QA for zeropoint if 'Echelle' in self.spectrograph.pypeline: - order_or_det = self.spectrograph.orders[np.arange(self.norderdet)] + order_or_det = self.meta_spec['ECH_ORDERS'] order_or_det_str = 'order' else: order_or_det = np.arange(self.norderdet) + 1 @@ -644,6 +756,41 @@ def write_QA(self): axis.set_title('PypeIt Throughput for' + spec_str) fig.savefig(self.thrufile) + # Plot fluxed standard star for all orders/det + fig = plt.figure(figsize=(12,8)) + axis = fig.add_axes([0.1, 0.1, 0.8, 0.8]) + axis.plot(self.std_dict['wave'].value, self.std_dict['flux'].value, color='green',linewidth=3.0, + label=self.std_dict['name'], zorder=100, alpha=0.7) + for iorddet in range(self.sens['SENS_FLUXED_STD_WAVE'].shape[0]): + # define the color + rr = (np.max(order_or_det) - order_or_det[iorddet]) \ + / np.maximum(np.max(order_or_det) - np.min(order_or_det), 1) + gg = 0.0 + bb = (order_or_det[iorddet] - np.min(order_or_det)) \ + / np.maximum(np.max(order_or_det) - np.min(order_or_det), 1) + wave_gpm = self.sens['SENS_FLUXED_STD_WAVE'][iorddet] > 1.0 + axis.plot(self.sens['SENS_FLUXED_STD_WAVE'][iorddet][wave_gpm], self.sens['SENS_FLUXED_STD_FLAM'][iorddet][wave_gpm], + color=(rr, gg, bb), drawstyle='steps-mid', linewidth=1.0, + label=thru_title[iorddet], zorder=idet, alpha=0.7) + + + wave_gpm_global = self.sens['SENS_FLUXED_STD_WAVE'] > 1.0 + wave_min = 0.98*(self.sens['SENS_FLUXED_STD_WAVE'][wave_gpm_global]).min() + wave_max = 1.02*(self.sens['SENS_FLUXED_STD_WAVE'][wave_gpm_global]).max() + pix_wave_std = (self.std_dict['wave'].value >= wave_min) & (self.std_dict['wave'].value <= wave_max) + flux_min = -1.0 + flux_max = 1.10*self.std_dict['flux'][pix_wave_std].value.max() + axis.set_xlim((wave_min, wave_max)) + axis.set_ylim((flux_min, flux_max)) + axis.legend() + axis.set_xlabel('Wavelength (Angstroms)') + axis.set_ylabel(r'$f_{{\lambda}}~~~(10^{{-17}}~{{\rm erg~s^{-1}~cm^{{-2}}~\AA^{{-1}}}})$') + axis.set_title('Fluxed Std Compared to True Spectrum:' + spec_str) + fig.savefig(self.fstdfile) + + + + @classmethod def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): """ @@ -652,13 +799,14 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): Args: sensfile (str): the name of your fits format sensfile - waves (ndarray): (nspec, norders, nexp) or (nspec, norders) - wavelength grid for your output weights + waves (`numpy.ndarray`_): + wavelength grid for your output weights. Shape is (nspec, + norders, nexp) or (nspec, norders). debug (bool): default=False show the weights QA Returns: - ndarray: sensfunc weights evaluated on the input waves + `numpy.ndarray`_: sensfunc weights evaluated on the input waves wavelength grid """ sens = cls.from_file(sensfile) @@ -688,7 +836,8 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): weights_stack[:,iord,iexp] = utils.inverse(sensfunc_iord) if debug: - coadd.weights_qa(waves_stack, weights_stack, (waves_stack > 1.0), title='sensfunc_weights') + coadd.weights_qa(utils.echarr_to_echlist(waves_stack)[0], utils.echarr_to_echlist(weights_stack)[0], + utils.echarr_to_echlist(waves_stack > 1.0)[0], title='sensfunc_weights') if waves.ndim == 2: weights_stack = np.reshape(weights_stack, (nspec, norder)) @@ -731,6 +880,7 @@ def compute_zeropoint(self): self.counts_mask, self.meta_spec['EXPTIME'], self.meta_spec['AIRMASS'], self.std_dict, self.par['IR']['telgridfile'], + log10_blaze_function=self.log10_blaze_function, polyorder=self.par['polyorder'], ech_orders=self.meta_spec['ECH_ORDERS'], resln_guess=self.par['IR']['resln_guess'], @@ -758,7 +908,7 @@ def compute_zeropoint(self): self.exptime = self.telluric.exptime # Instantiate the main output data table - self.sens = self.empty_sensfunc_table(self.telluric.norders, self.telluric.wave_grid.size, + self.sens = self.empty_sensfunc_table(self.telluric.norders, self.telluric.wave_grid.size, self.nspec_in, ncoeff=self.telluric.max_ntheta_obj) # For stupid reasons related to how astropy tables will let me store @@ -785,6 +935,11 @@ def compute_zeropoint(self): # Compute and assign the zeropint_data from the input data and the # best-fit telluric model self.sens['SENS_WAVE'][i,s[i]:e[i]] = self.telluric.wave_grid[s[i]:e[i]] + if self.log10_blaze_function is not None: + log10_blaze_function_iord = self.telluric.log10_blaze_func_arr[s[i]:e[i],i] + self.sens['SENS_LOG10_BLAZE_FUNCTION'][i,s[i]:e[i]] = self.telluric.log10_blaze_func_arr[s[i]:e[i],i] + else: + log10_blaze_function_iord = None self.sens['SENS_ZEROPOINT_GPM'][i,s[i]:e[i]] = self.telluric.mask_arr[s[i]:e[i],i] self.sens['SENS_COUNTS_PER_ANG'][i,s[i]:e[i]] = self.telluric.flux_arr[s[i]:e[i],i] N_lam = self.sens['SENS_COUNTS_PER_ANG'][i,s[i]:e[i]] / self.exptime @@ -796,11 +951,11 @@ def compute_zeropoint(self): # TODO: func is always 'legendre' because that is what's set by # sensfunc_telluric self.sens['SENS_ZEROPOINT_FIT'][i,s[i]:e[i]] \ - = fitting.evaluate_fit(self.sens['SENS_COEFF'][ - i,:self.sens['POLYORDER_VEC'][i]+2], - self.telluric.func, self.sens['SENS_WAVE'][i,s[i]:e[i]], - minx=self.sens['WAVE_MIN'][i], - maxx=self.sens['WAVE_MAX'][i]) + = flux_calib.eval_zeropoint( + self.sens['SENS_COEFF'][i,:self.sens['POLYORDER_VEC'][i]+2], + self.telluric.func, self.sens['SENS_WAVE'][i,s[i]:e[i]], + self.sens['WAVE_MIN'][i], self.sens['WAVE_MAX'][i], + log10_blaze_func_per_ang=log10_blaze_function_iord) self.sens['SENS_ZEROPOINT_FIT_GPM'][i,s[i]:e[i]] = self.telluric.outmask_list[i] def eval_zeropoint(self, wave, iorddet): @@ -818,10 +973,21 @@ def eval_zeropoint(self, wave, iorddet): ------- zeropoint : `numpy.ndarray`_, shape is (nspec,) """ - return fitting.evaluate_fit(self.sens['SENS_COEFF'][ - iorddet,:self.telluric.model['POLYORDER_VEC'][iorddet]+2], - self.telluric.func, wave, minx=self.sens['WAVE_MIN'][iorddet], - maxx=self.sens['WAVE_MAX'][iorddet]) + s = self.telluric.model['IND_LOWER'] + e = self.telluric.model['IND_UPPER']+1 + # TODO: Not sure what else to do here + if self.log10_blaze_function is not None: + log10_blaze_function = scipy.interpolate.interp1d( + self.sens['SENS_WAVE'][iorddet,s[iorddet]:e[iorddet]], + self.sens['SENS_LOG10_BLAZE_FUNCTION'][iorddet,s[iorddet]:e[iorddet]], + kind='linear', bounds_error=False, fill_value='extrapolate')(wave) + else: + log10_blaze_function = None + + return flux_calib.eval_zeropoint( + self.sens['SENS_COEFF'][iorddet,:self.telluric.model['POLYORDER_VEC'][iorddet]+2], + self.telluric.func, wave, self.sens['WAVE_MIN'][iorddet], self.sens['WAVE_MAX'][iorddet], + log10_blaze_func_per_ang=log10_blaze_function) class UVISSensFunc(SensFunc): @@ -843,8 +1009,8 @@ class UVISSensFunc(SensFunc): _algorithm = 'UVIS' """Algorithm used for the sensitivity calculation.""" - def __init__(self, spec1dfile, sensfile, par=None, debug=False): - super().__init__(spec1dfile, sensfile, par=par, debug=debug) + def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): + super().__init__(spec1dfile, sensfile, par, par_fluxcalib=par_fluxcalib, debug=debug) # Add some cards to the meta spec. These should maybe just be added # already in unpack object @@ -884,7 +1050,7 @@ def compute_zeropoint(self): norder, nspec = out_table['SENS_ZEROPOINT'].shape # Instantiate the main output data table - self.sens = self.empty_sensfunc_table(norder, nspec) + self.sens = self.empty_sensfunc_table(norder, nspec, self.nspec_in) # Copy the relevant data # NOTE: SENS_COEFF is empty! diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 425b8c71de..7ff941178c 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -72,8 +72,10 @@ class SlitTraceSet(calibframe.CalibFrame): .. include:: ../include/class_datamodel_slittraceset.rst Attributes: - left_flexure (`numpy.ndarray`_): Convenient spot to hold flexure corrected left - right_flexure (`numpy.ndarray`_): Convenient spot to hold flexure corrected right + left_flexure (`numpy.ndarray`_): + Convenient spot to hold flexure corrected left + right_flexure (`numpy.ndarray`_): + Convenient spot to hold flexure corrected right """ calib_type = 'Slits' """Name for type of calibration frame.""" @@ -340,6 +342,21 @@ def slitord_id(self): return self.ech_order msgs.error(f'Unrecognized Pypeline {self.pypeline}') + @property + def slitord_txt(self): + """ + Return string indicating if the logs/QA should use "slit" (MultiSlit, IFU) or "order" (Echelle) + + Returns: + str: Either 'slit' or 'order' + + """ + if self.pypeline in ['MultiSlit', 'IFU']: + return 'slit' + if self.pypeline == 'Echelle': + return 'order' + msgs.error(f'Unrecognized Pypeline {self.pypeline}') + def spatid_to_zero(self, spat_id): """ Convert input spat_id into a zero-based index @@ -405,12 +422,12 @@ def get_radec_image(self, wcs, alignSplines, tilts, initial=True, flexure=None): Parameters ---------- - wcs : astropy.wcs + wcs : `astropy.wcs.WCS`_ The World Coordinate system of a science frame alignSplines : :class:`pypeit.alignframe.AlignmentSplines` An instance of the AlignmentSplines class that allows one to build and transform between detector pixel coordinates and WCS pixel coordinates. - tilts : `numpy.ndarray` + tilts : `numpy.ndarray`_ Spectral tilts. initial : bool Select the initial slit edges? @@ -419,12 +436,16 @@ def get_radec_image(self, wcs, alignSplines, tilts, initial=True, flexure=None): Returns ------- - tuple : There are three elements in the tuple. The first two are 2D numpy arrays - of shape (nspec, nspat), where the first ndarray is the RA image, and the - second ndarray is the DEC image. RA and DEC are in units degrees. The third - element of the tuple stores the minimum and maximum difference (in pixels) - between the WCS reference (usually the centre of the slit) and the edges of - the slits. The third array has a shape of (nslits, 2). + raimg : `numpy.ndarray`_ + Image with the RA coordinates of each pixel in degrees. Shape is + (nspec, nspat). + decimg : `numpy.ndarray`_ + Image with the DEC coordinates of each pixel in degrees. Shape is + (nspec, nspat). + minmax : `numpy.ndarray`_ + The minimum and maximum difference (in pixels) between the WCS + reference (usually the centre of the slit) and the edges of the + slits. Shape is (nslits, 2). """ # Initialise the output raimg = np.zeros((self.nspec, self.nspat)) @@ -1285,16 +1306,18 @@ def get_maskdef_extract_fwhm(self, sobjs, platescale, fwhm_parset, find_fwhm): will be computed using the average fwhm of the detected objects. Args: - sobjs (:class:`pypeit.specobjs.SpecObjs`): + sobjs (:class:`~pypeit.specobjs.SpecObjs`): List of SpecObj that have been found and traced. platescale (:obj:`float`): Platescale. fwhm_parset (:obj:`float`, optional): - Parset that guides the determination of the fwhm of the maskdef_extract objects. - If None (default) the fwhm are computed as the averaged from the detected objects, + :class:`~pypeit.par.parset.Parset` that guides the determination + of the fwhm of the maskdef_extract objects. If None (default) + the fwhm are computed as the averaged from the detected objects, if it is a number it will be adopted as the fwhm. find_fwhm (:obj:`float`): - Initial guess of the objects fwhm in pixels (used in object finding) + Initial guess of the objects fwhm in pixels (used in object + finding) Returns: :obj:`float`: FWHM in pixels to be used in the optimal extraction @@ -1366,10 +1389,10 @@ def user_mask(self, det, user_slits): def mask_flats(self, flatImages): """ - Mask from a :class:`pypeit.flatfield.Flats` object + Mask based on a :class:`~pypeit.flatfield.FlatImages` object. Args: - flatImages (:class:`pypeit.flatfield.FlatImages`): + flatImages (:class:`~pypeit.flatfield.FlatImages`): """ # Loop on all the FLATFIELD BPM keys diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index 41c3500b35..fd6906288d 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -44,7 +44,7 @@ class Spec2DObj(datamodel.DataContainer): Args: Attributes: - head0 (`astropy.fits.Header`): + head0 (`astropy.io.fits.Header`_): Primary header if instantiated from a FITS file """ @@ -259,7 +259,7 @@ def _base_header(self, hdr=None): the header. Args: - hdr (`astropy.io.fits.Header`, optional): + hdr (`astropy.io.fits.Header`_, optional): Header object to update. The object is modified in-place and also returned. If None, an empty header is instantiated, edited, and returned. @@ -605,9 +605,9 @@ def write_to_fits(self, outfile, pri_hdr=None, update_det=None, Args: outfile (:obj:`str`, `Path`_): Output filename - pri_hdr (:class:`astropy.io.fits.Header`, optional): - Header to be used in lieu of default - Usually generated by :func:`pypeit,spec2dobj.AllSpec2DObj.build_primary_hdr` + pri_hdr (`astropy.io.fits.Header`_, optional): + Header to be used in lieu of default Usually generated by + :func:`~pypeit.spec2dobj.AllSpec2DObj.build_primary_hdr` slitspatnum (:obj:`str` or :obj:`list`, optional): Restricted set of slits for reduction If provided, do not clobber the existing file but only update @@ -615,7 +615,7 @@ def write_to_fits(self, outfile, pri_hdr=None, update_det=None, pri_hdr(): Baseline primary header. If None, initial primary header is empty. Usually generated by - :func:`pypeit,spec2dobj.AllSpec2DObj.build_primary_hdr` + :func:`~pypeit.spec2dobj.AllSpec2DObj.build_primary_hdr` update_det (:obj:`list`, optional): If the output file already exists, this sets the list of detectors/mosaics to update with the data in this object. If diff --git a/pypeit/specobj.py b/pypeit/specobj.py index 2aa4912fe9..a7b428aa4c 100644 --- a/pypeit/specobj.py +++ b/pypeit/specobj.py @@ -18,6 +18,7 @@ from pypeit import msgs from pypeit.core import flexure from pypeit.core import flux_calib +from pypeit.core import parse from pypeit import utils from pypeit import datamodel from pypeit.images.detector_container import DetectorContainer @@ -54,7 +55,7 @@ class SpecObj(datamodel.DataContainer): Running index for the order. """ - version = '1.1.8' + version = '1.1.10' """ Current datamodel version number. """ @@ -62,8 +63,9 @@ class SpecObj(datamodel.DataContainer): datamodel = {'TRACE_SPAT': dict(otype=np.ndarray, atype=float, descr='Object trace along the spec (spatial pixel)'), 'FWHM': dict(otype=float, descr='Spatial FWHM of the object (pixels)'), - 'FWHMFIT': dict(otype=np.ndarray, + 'FWHMFIT': dict(otype=np.ndarray, atype=float, descr='Spatial FWHM across the detector (pixels)'), + 'SPAT_FWHM': dict(otype=float, descr='Spatial FWHM of the object (arcsec)'), 'smash_peakflux': dict(otype=float, descr='Peak value of the spectral direction collapsed spatial profile'), 'smash_snr': dict(otype=float, @@ -87,6 +89,8 @@ class SpecObj(datamodel.DataContainer): 'noise only (counts^2)'), 'OPT_MASK': dict(otype=np.ndarray, atype=np.bool_, descr='Mask for optimally extracted flux. True=good'), + 'OPT_FWHM': dict(otype=np.ndarray, atype=float, + descr='Spectral FWHM (in Angstroms) at every pixel of the optimally extracted flux.'), 'OPT_COUNTS_SKY': dict(otype=np.ndarray, atype=float, descr='Optimally extracted sky (counts)'), 'OPT_COUNTS_SIG_DET': dict(otype=np.ndarray, atype=float, @@ -118,6 +122,8 @@ class SpecObj(datamodel.DataContainer): 'only (counts^2)'), 'BOX_MASK': dict(otype=np.ndarray, atype=np.bool_, descr='Mask for boxcar extracted flux. True=good'), + 'BOX_FWHM': dict(otype=np.ndarray, atype=float, + descr='Spectral FWHM (in Angstroms) at every pixel of the boxcar extracted flux.'), 'BOX_COUNTS_SKY': dict(otype=np.ndarray, atype=float, descr='Boxcar extracted sky (counts)'), 'BOX_COUNTS_SIG_DET': dict(otype=np.ndarray, atype=float, @@ -162,7 +168,7 @@ class SpecObj(datamodel.DataContainer): 'trace_spec': dict(otype=np.ndarray, atype=(int,np.integer), descr='Array of pixels along the spectral direction'), 'maskwidth': dict(otype=(float, np.floating), - descr='Size (in units of fwhm) of the region used for local sky subtraction'), + descr='Size (in units of spatial fwhm) of the region used for local sky subtraction'), # Slit and Object 'WAVE_RMS': dict(otype=(float, np.floating), descr='RMS (pix) for the wavelength solution for this slit.'), @@ -189,6 +195,8 @@ class SpecObj(datamodel.DataContainer): descr='Object ID for echelle data. Each object is given an ' 'index in the order it appears increasing from from left ' 'to right. These are one based.'), + # TODO ECH_ORDERINDX should be purged. It is not reliable for anything given masking. Instead + # one needs to use SLITID or ECH_ORDER 'ECH_ORDERINDX': dict(otype=(int, np.integer), descr='Order indx, analogous to SLITID for echelle. ' 'Zero based.'), @@ -351,6 +359,18 @@ def med_s2n(self): break return SN + def med_fwhm(self): + """Return median spatial FWHM of the spectrum + + Returns: + float + """ + FWHM = 0. + if self['FWHMFIT'] is not None and self['OPT_COUNTS'] is not None: + _, binspatial = parse.parse_binning(self['DETECTOR']['binning']) + FWHM = np.median(self['FWHMFIT']) * binspatial * self['DETECTOR']['platescale'] + return FWHM + def set_name(self): """ Construct the ``PypeIt`` name for this object. @@ -467,31 +487,32 @@ def update_flex_shift(self, shift, flex_type='local'): # TODO This should be a wrapper calling a core algorithm. def apply_flux_calib(self, wave_zp, zeropoint, exptime, tellmodel=None, extinct_correct=False, - airmass=None, longitude=None, latitude=None, extinctfilepar=None, extrap_sens=False): + airmass=None, longitude=None, latitude=None, extinctfilepar=None, + extrap_sens=False): """ Apply a sensitivity function to our spectrum FLAM, FLAM_SIG, and FLAM_IVAR are generated Args: - wave_zp (float array) + wave_zp (`numpy.ndarray`_): Zeropoint wavelength array - zeropoint (float array): + zeropoint (`numpy.ndarray`_): zeropoint array exptime (float): Exposure time - tellmodel: + tellmodel (?): Telluric correction - extinct_correct: + extinct_correct (?): If True, extinction correct airmass (float, optional): Airmass longitude (float, optional): longitude in degree for observatory - latitude: + latitude (float, optional): latitude in degree for observatory Used for extinction correction - extinctfilepar (str): + extinctfilepar (str, optional): [sensfunc][UVIS][extinct_file] parameter Used for extinction correction extrap_sens (bool, optional): diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index 780cec1aa2..7d72fb81d5 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -5,6 +5,7 @@ .. include:: ../include/links.rst """ import os +from pathlib import Path import re from typing import List @@ -42,9 +43,12 @@ class SpecObjs: Args: specobjs (`numpy.ndarray`_, list, optional): One or more :class:`~pypeit.specobj.SpecObj` objects + header (`astropy.io.fits.Header`_, optional): + Baseline header to use Attributes: - summary (astropy.table.Table): + summary (`astropy.table.Table`_): + Summary table (?) """ version = '1.0.0' @@ -72,49 +76,57 @@ def from_fitsfile(cls, fits_file, det=None, chk_version=True): provided fits file. """ # HDUList - hdul = io.fits_open(fits_file) - # Init - slf = cls() - # Add on the header - slf.header = hdul[0].header - # Keep track of HDUList for closing later - - # Catch common error of trying to read a OneSpec file - if 'DMODCLS' in hdul[1].header and hdul[1].header['DMODCLS'] == 'OneSpec': - msgs.error('This is a OneSpec file. You are treating it like a SpecObjs file.') + with io.fits_open(fits_file) as hdul: + # Init + slf = cls() + # Add on the header + slf.header = hdul[0].header + + # Catch common error of trying to read a OneSpec file + if 'DMODCLS' in hdul[1].header and hdul[1].header['DMODCLS'] == 'OneSpec': + msgs.error('This is a OneSpec file. You are treating it like a SpecObjs file.') + + # Load the calibration association into the instance attribute `calibs` + if 'CLBS_DIR' in slf.header: + slf.calibs = {} + slf.calibs['DIR'] = slf.header['CLBS_DIR'] + for key in slf.header.keys(): + if key.startswith('CLBS_') \ + and (Path(slf.calibs['DIR']).resolve() / slf.header[key]).exists(): + slf.calibs['_'.join(key.split('_')[1:])] = slf.header[key] + + detector_hdus = {} + # Loop for Detectors first as we need to add these to the objects + for hdu in hdul[1:]: + if 'DETECTOR' not in hdu.name: + continue + if 'DMODCLS' not in hdu.header: + msgs.error('HDUs with DETECTOR in the name must have DMODCLS in their header.') + try: + dmodcls = eval(hdu.header['DMODCLS']) + except: + msgs.error(f"Unknown detector type datamodel class: {hdu.header['DMODCLS']}") + # NOTE: This requires that any "detector" datamodel class has a + # from_hdu method, and the name of the HDU must have a known format + # (e.g., 'DET01-DETECTOR'). + _det = hdu.name.split('-')[0] + detector_hdus[_det] = dmodcls.from_hdu(hdu) + + # Now the objects + for hdu in hdul[1:]: + if 'DETECTOR' in hdu.name: + continue + sobj = specobj.SpecObj.from_hdu(hdu, chk_version=chk_version) + # Restrict on det? + if det is not None and sobj.DET != det: + continue + # Check for detector + if sobj.DET in detector_hdus.keys(): + sobj.DETECTOR = detector_hdus[sobj.DET] + # Append + slf.add_sobj(sobj) - detector_hdus = {} - # Loop for Detectors first as we need to add these to the objects - for hdu in hdul[1:]: - if 'DETECTOR' not in hdu.name: - continue - if 'DMODCLS' not in hdu.header: - msgs.error('HDUs with DETECTOR in the name must have DMODCLS in their header.') - try: - dmodcls = eval(hdu.header['DMODCLS']) - except: - msgs.error(f"Unknown detector type datamodel class: {hdu.header['DMODCLS']}") - # NOTE: This requires that any "detector" datamodel class has a - # from_hdu method, and the name of the HDU must have a known format - # (e.g., 'DET01-DETECTOR'). - _det = hdu.name.split('-')[0] - detector_hdus[_det] = dmodcls.from_hdu(hdu) - - # Now the objects - for hdu in hdul[1:]: - if 'DETECTOR' in hdu.name: - continue - sobj = specobj.SpecObj.from_hdu(hdu, chk_version=chk_version) - # Restrict on det? - if det is not None and sobj.DET != det: - continue - # Check for detector - if sobj.DET in detector_hdus.keys(): - sobj.DETECTOR = detector_hdus[sobj.DET] - # Append - slf.add_sobj(sobj) # Return - hdul.close() return slf @@ -130,6 +142,7 @@ def __init__(self, specobjs=None, header=None): self.header = header self.hdul = None + self.calibs = None # Turn off attributes from here # Anything else set will be on the individual specobj objects in the specobjs array @@ -176,7 +189,7 @@ def unpack_object(self, ret_flam=False, extract_type='OPT'): """ Utility function to unpack the sobjs for one object and return various numpy arrays describing the spectrum and meta - data. The user needs to already have trimmed the Specobjs to + data. The user needs to already have trimmed the :class:`SpecObjs` to the relevant indices for the object. Args: @@ -215,6 +228,9 @@ def unpack_object(self, ret_flam=False, extract_type='OPT'): flux = np.zeros((nspec, norddet)) flux_ivar = np.zeros((nspec, norddet)) flux_gpm = np.zeros((nspec, norddet), dtype=bool) + trace_spec = np.zeros((nspec, norddet)) + trace_spat = np.zeros((nspec, norddet)) + detector = [None]*norddet ech_orders = np.zeros(norddet, dtype=int) @@ -227,6 +243,8 @@ def unpack_object(self, ret_flam=False, extract_type='OPT'): ech_orders[iorddet] = self[iorddet].ECH_ORDER flux[:, iorddet] = getattr(self, flux_key)[iorddet] flux_ivar[:, iorddet] = getattr(self, flux_key+'_IVAR')[iorddet] #OPT_FLAM_IVAR + trace_spat[:, iorddet] = self[iorddet].TRACE_SPAT + trace_spec[:, iorddet] = self[iorddet].trace_spec # Populate meta data spectrograph = load_spectrograph(self.header['PYP_SPEC']) @@ -242,14 +260,14 @@ def unpack_object(self, ret_flam=False, extract_type='OPT'): if self[0].PYPELINE in ['MultiSlit', 'IFU'] and self.nobj == 1: meta_spec['ECH_ORDERS'] = None return wave.reshape(nspec), flux.reshape(nspec), flux_ivar.reshape(nspec), \ - flux_gpm.reshape(nspec), meta_spec, self.header + flux_gpm.reshape(nspec), trace_spec.reshape(nspec), trace_spat.reshape(nspec), meta_spec, self.header else: meta_spec['ECH_ORDERS'] = ech_orders - return wave, flux, flux_ivar, flux_gpm, meta_spec, self.header + return wave, flux, flux_ivar, flux_gpm, trace_spec, trace_spat, meta_spec, self.header def get_std(self, multi_spec_det=None): """ - Return the standard star from this Specobjs. For MultiSlit this + Return the standard star from this :class:`SpecObjs`. For MultiSlit this will be a single specobj in SpecObjs container, for Echelle it will be the standard for all the orders. @@ -537,6 +555,78 @@ def ready_for_fluxing(self): chk &= (sub_box or sub_opt) return chk + def apply_flux_calib(self, par, spectrograph, sens): + """ + Flux calibrate the object spectra (``sobjs``) using the provided + sensitivity function (``sens``). + + Args: + par (:class:`~pypeit.par.pypeitpar.FluxCalibratePar`): + Parset object containing parameters governing the flux calibration. + spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): + PypeIt Spectrograph class + sens (:class:`~pypeit.sensfunc.SensFunc`): + PypeIt Sensitivity function class + """ + + _extinct_correct = (True if sens.algorithm == 'UVIS' else False) \ + if par['extinct_correct'] is None else par['extinct_correct'] + + if spectrograph.pypeline == 'MultiSlit': + for ii, sci_obj in enumerate(self.specobjs): + if sens.wave.shape[1] == 1: + sci_obj.apply_flux_calib(sens.wave[:, 0], sens.zeropoint[:, 0], + self.header['EXPTIME'], + extinct_correct=_extinct_correct, + longitude=spectrograph.telescope['longitude'], + latitude=spectrograph.telescope['latitude'], + extinctfilepar=par['extinct_file'], + extrap_sens=par['extrap_sens'], + airmass=float(self.header['AIRMASS'])) + elif sens.wave.shape[1] > 1 and sens.splice_multi_det: + # This deals with the multi detector case where the sensitivity function is spliced. Note that + # the final sensitivity function written to disk is the spliced one. This functionality is only + # used internal to sensfunc.py for fluxing the standard for the QA plot. + sci_obj.apply_flux_calib(sens.wave[:, ii], sens.zeropoint[:, ii], + self.header['EXPTIME'], + extinct_correct=_extinct_correct, + longitude=spectrograph.telescope['longitude'], + latitude=spectrograph.telescope['latitude'], + extinctfilepar=par['extinct_file'], + extrap_sens=par['extrap_sens'], + airmass=float(self.header['AIRMASS'])) + else: + msgs.error('This should not happen, there is a problem with your sensitivity function.') + + + elif spectrograph.pypeline == 'Echelle': + # Flux calibrate the orders that are mutually in the meta_table and in + # the sobjs. This allows flexibility for applying to data for cases + # where not all orders are present in the data as in the sensfunc, etc., + # i.e. X-shooter with the K-band blocking filter. + ech_orders = np.array(sens.sens['ECH_ORDERS']).flatten() + for sci_obj in self.specobjs: + # JFH Is there a more elegant pythonic way to do this without looping over both orders and sci_obj? + indx = np.where(ech_orders == sci_obj.ECH_ORDER)[0] + if indx.size == 1: + sci_obj.apply_flux_calib(sens.wave[:, indx[0]], sens.zeropoint[:, indx[0]], + self.header['EXPTIME'], + extinct_correct=_extinct_correct, + extrap_sens=par['extrap_sens'], + longitude=spectrograph.telescope['longitude'], + latitude=spectrograph.telescope['latitude'], + extinctfilepar=par['extinct_file'], + airmass=float(self.header['AIRMASS'])) + elif indx.size == 0: + msgs.info('Unable to flux calibrate order = {:} as it is not in your sensitivity function. ' + 'Something is probably wrong with your sensitivity function.'.format(sci_obj.ECH_ORDER)) + else: + msgs.error('This should not happen') + + else: + msgs.error('Unrecognized pypeline: {0}'.format(spectrograph.pypeline)) + + def copy(self): """ Generate a copy of self @@ -672,6 +762,10 @@ def write_to_fits(self, subheader, outfile, overwrite=True, update_det=None, header[key.upper()] = line else: header[key.upper()] = subheader[key] + # Add calibration associations to Header + if self.calibs is not None: + for key, val in self.calibs.items(): + header[f'CLBS_{key}'] = val # Init prihdu = fits.PrimaryHDU(header=header) @@ -794,11 +888,8 @@ def write_info(self, outfile, pypeline): boxsize.append(0.) # Optimal profile (FWHM) + opt_fwhm.append(specobj.SPAT_FWHM) # S2N -- default to boxcar - if specobj.FWHMFIT is not None and specobj.OPT_COUNTS is not None: - opt_fwhm.append(np.median(specobj.FWHMFIT) * binspatial * platescale) - else: # Optimal is not required to occur - opt_fwhm.append(0.) # NOTE: Below requires that S2N not be None, otherwise the code will # fault. If the code gets here and S2N is None, check that 1D # extractions have been performed. @@ -917,10 +1008,11 @@ def get_std_trace(detname, std_outfile, chk_version=True): 1-indexed detector(s) to process. std_outfile (:obj:`str`): Filename with the standard star spec1d file. Can be None. + Returns: - `numpy.ndarray`_: Trace of the standard star on input detector. - Will be None if ``std_outfile`` is None, or if the selected detector/mosaic is not available - in the provided spec1d file. + `numpy.ndarray`_: Trace of the standard star on input detector. Will + be None if ``std_outfile`` is None, or if the selected detector/mosaic + is not available in the provided spec1d file. """ sobjs = SpecObjs.from_fitsfile(std_outfile, chk_version=chk_version) @@ -958,12 +1050,12 @@ def lst_to_array(lst, mask=None): Args: lst : list - Should be number or Quantities - mask (ndarray of bool, optional): Limit to a subset of the list. True=good + Should be number or Quantities + mask (`numpy.ndarray`_, optional): + Boolean array used to limit to a subset of the list. True=good Returns: - ndarray or Quantity array: Converted list - + `numpy.ndarray`_, `astropy.units.Quantity`_: Converted list """ if mask is None: mask = np.array([True]*len(lst)) diff --git a/pypeit/spectrographs/__init__.py b/pypeit/spectrographs/__init__.py index b96c98e2ba..3377156931 100644 --- a/pypeit/spectrographs/__init__.py +++ b/pypeit/spectrographs/__init__.py @@ -8,6 +8,7 @@ from pypeit.spectrographs import gemini_flamingos from pypeit.spectrographs import gemini_gmos from pypeit.spectrographs import gemini_gnirs +from pypeit.spectrographs import keck_esi from pypeit.spectrographs import keck_deimos from pypeit.spectrographs import keck_hires from pypeit.spectrographs import keck_kcwi @@ -20,6 +21,7 @@ from pypeit.spectrographs import ldt_deveny from pypeit.spectrographs import magellan_fire from pypeit.spectrographs import magellan_mage +from pypeit.spectrographs import mdm_modspec from pypeit.spectrographs import mdm_osmos from pypeit.spectrographs import mmt_binospec from pypeit.spectrographs import mmt_bluechannel diff --git a/pypeit/spectrographs/bok_bc.py b/pypeit/spectrographs/bok_bc.py index f81b32031b..11a9c642df 100644 --- a/pypeit/spectrographs/bok_bc.py +++ b/pypeit/spectrographs/bok_bc.py @@ -171,7 +171,7 @@ def get_detector_par(self, det, hdu=None): spatflip = False, #platescale = 15.0/18.0, platescale = 0.2, - darkcurr = 5.4, + darkcurr = 5.4, # e-/hour/unbinned pixel saturation = 65535., nonlinear = 1.0, mincounts = -1e10, @@ -243,7 +243,7 @@ def default_pypeit_par(cls): # Do not flux calibrate par['fluxcalib'] = None # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 1] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['arcframe']['exprng'] = [None, 120] @@ -278,11 +278,6 @@ def bpm(self, filename, det, shape=None, msbias=None): """ Generate a default bad-pixel mask. - Even though they are both optional, either the precise shape for - the image (``shape``) or an example file that can be read to get - the shape (``filename`` using :func:`get_image_shape`) *must* be - provided. - Args: filename (:obj:`str` or None): An example file to use to get the image shape. diff --git a/pypeit/spectrographs/gemini_flamingos.py b/pypeit/spectrographs/gemini_flamingos.py index bdfc336bdf..52dd3400b3 100644 --- a/pypeit/spectrographs/gemini_flamingos.py +++ b/pypeit/spectrographs/gemini_flamingos.py @@ -79,7 +79,7 @@ def get_detector_par(self, det, hdu=None): specflip = True, spatflip = False, platescale = 0.1787, - darkcurr = 0.5, + darkcurr = 1800.0, # e-/pixel/hour (=0.5 e-/pixel/s) saturation = 700000., #155400., nonlinear = 1.0, mincounts = -1e10, @@ -246,7 +246,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.15, - darkcurr = 0.01, + darkcurr = 1080.0, # e-/hour/pixel (=0.3 e-/pixel/s) saturation = 320000., #155400., nonlinear = 0.875, mincounts = -1e10, diff --git a/pypeit/spectrographs/gemini_gmos.py b/pypeit/spectrographs/gemini_gmos.py index 67ba21d66e..e64a3ca707 100644 --- a/pypeit/spectrographs/gemini_gmos.py +++ b/pypeit/spectrographs/gemini_gmos.py @@ -154,6 +154,26 @@ def compound_meta(self, headarr, meta_key): msgs.error('Binning not found') return binning + def config_independent_frames(self): + """ + Define frame types that are independent of the fully defined + instrument configuration. + + This method returns a dictionary where the keys of the dictionary are + the list of configuration-independent frame types. The value of each + dictionary element can be set to one or more metadata keys that can + be used to assign each frame type to a given configuration group. See + :func:`~pypeit.metadata.PypeItMetaData.set_configurations` and how it + interprets the dictionary values, which can be None. + + Returns: + :obj:`dict`: Dictionary where the keys are the frame types that + are configuration-independent and the values are the metadata + keywords that can be used to assign the frames to a configuration + group. + """ + return {'bias': 'datasec', 'dark': 'datasec'} + def configuration_keys(self): """ Return the metadata keys that define a unique instrument @@ -796,7 +816,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.080, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 129000., nonlinear = 0.95, mincounts = -1e10, @@ -813,7 +833,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.080, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 123000., nonlinear = 0.95, mincounts = -1e10, @@ -830,7 +850,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.080, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 125000., nonlinear = 0.95, mincounts = -1e10, @@ -1029,7 +1049,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.0807, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 129000., nonlinear = 0.95, mincounts = -1e10, @@ -1046,7 +1066,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.0807, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 123000., nonlinear = 0.95, mincounts = -1e10, @@ -1063,7 +1083,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.0807, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 125000., nonlinear = 0.95, mincounts = -1e10, @@ -1260,7 +1280,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.0728, # arcsec per pixel - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 110900., nonlinear = 0.95, mincounts = -1e10, @@ -1277,7 +1297,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.0728, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 115500., nonlinear = 0.95, mincounts = -1e10, @@ -1294,7 +1314,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.0728, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 116700., nonlinear = 0.95, mincounts = -1e10, diff --git a/pypeit/spectrographs/gemini_gnirs.py b/pypeit/spectrographs/gemini_gnirs.py index c6ec98a032..eca3e77cc9 100644 --- a/pypeit/spectrographs/gemini_gnirs.py +++ b/pypeit/spectrographs/gemini_gnirs.py @@ -4,10 +4,13 @@ .. include:: ../include/links.rst """ import numpy as np +from astropy import wcs, units +from astropy.coordinates import SkyCoord, EarthLocation +from astropy.time import Time from pypeit import msgs from pypeit import telescopes -from pypeit.core import framematch +from pypeit.core import framematch, parse from pypeit.images import detector_container from pypeit.spectrographs import spectrograph @@ -17,14 +20,16 @@ class GeminiGNIRSSpectrograph(spectrograph.Spectrograph): Child to handle Gemini/GNIRS specific code """ ndet = 1 - name = 'gemini_gnirs' camera = 'GNIRS' url = 'https://www.gemini.edu/instrumentation/gnirs' header_name = 'GNIRS' telescope = telescopes.GeminiNTelescopePar() - pypeline = 'Echelle' - ech_fixed_format = True - supported = True + + def __init__(self): + super().__init__() + + # TODO :: Might consider changing TelescopePar to use the astropy EarthLocation. + self.location = EarthLocation.of_site('Gemini North') def get_detector_par(self, det, hdu=None): """ @@ -50,7 +55,7 @@ def get_detector_par(self, det, hdu=None): specflip=True, spatflip=True, platescale = 0.15, - darkcurr = 0.15, + darkcurr = 540.0, # e-/hour/pixel (=0.15 e-/pixel/s) saturation = 150000., nonlinear = 0.71, mincounts = -1e10, @@ -62,150 +67,6 @@ def get_detector_par(self, det, hdu=None): ) return detector_container.DetectorContainer(**detector_dict) - @classmethod - def default_pypeit_par(cls): - """ - Return the default parameters to use for this instrument. - - Returns: - :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by - all of PypeIt methods. - """ - par = super().default_pypeit_par() - - # Image processing steps - turn_off = dict(use_illumflat=False, use_biasimage=False, use_overscan=False, - use_darkimage=False) - par.reset_all_processimages_par(**turn_off) - - # Flats - par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.90 - par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.10 - - # Reduce parameters - #par['reduce']['findobj']['snr_thresh'] = 5.0 # Object finding threshold - par['reduce']['findobj']['find_trim_edge'] = [2,2] # Slit is too short to trim 5,5 especially - par['reduce']['skysub']['bspline_spacing'] = 0.8 - par['reduce']['skysub']['global_sky_std'] = False # Do not perform global sky subtraction for standard stars - # TODO: JFH: Is this the correct behavior? (Is why we have sky-subtraction problems for GNIRS?) - par['reduce']['skysub']['no_poly'] = True # Do not use polynomial degree of freedom for global skysub - par['reduce']['extraction']['model_full_slit'] = True # local sky subtraction operates on entire slit - par['reduce']['findobj']['maxnumber_sci'] = 2 # Slit is narrow so allow one object per order - par['reduce']['findobj']['maxnumber_std'] = 1 # Slit is narrow so allow one object per order - # Standards - par['calibrations']['standardframe']['process']['mask_cr'] = False # Do not mask_cr standards - - # Do not correct for flexure - par['flexure']['spec_method'] = 'skip' - - # Set the default exposure time ranges for the frame typing - par['calibrations']['pixelflatframe']['exprng'] = [None, 30] - par['calibrations']['traceframe']['exprng'] = [None, 30] - par['calibrations']['standardframe']['exprng'] = [None, 30] - par['scienceframe']['exprng'] = [30, None] - - # Sensitivity function parameters - par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['polyorder'] = 6 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' - return par - - def config_specific_par(self, scifile, inp_par=None): - """ - Modify the PypeIt parameters to hard-wired values used for - specific instrument configurations. - - Args: - scifile (:obj:`str`): - File to use when determining the configuration and how - to adjust the input parameters. - inp_par (:class:`~pypeit.par.parset.ParSet`, optional): - Parameter set used for the full run of PypeIt. If None, - use :func:`default_pypeit_par`. - - Returns: - :class:`~pypeit.par.parset.ParSet`: The PypeIt parameter set - adjusted for configuration specific parameter values. - """ - par = super().config_specific_par(scifile, inp_par=inp_par) - - # TODO This is a hack for now until we figure out how to set dispname - # and other meta information in the spectrograph class itself - self.dispname = self.get_meta_value(scifile, 'dispname') - # 32/mmSB_G5533 setup, covering XYJHK with short blue camera - if '32/mm' in self.dispname: - # Edges - par['calibrations']['slitedges']['edge_thresh'] = 20. - par['calibrations']['slitedges']['trace_thresh'] = 10. - par['calibrations']['slitedges']['fit_order'] = 5 - par['calibrations']['slitedges']['max_shift_adj'] = 0.5 - par['calibrations']['slitedges']['fit_min_spec_length'] = 0.5 - par['calibrations']['slitedges']['left_right_pca'] = True - par['calibrations']['slitedges']['pca_order'] = 3 - - # Wavelengths - par['calibrations']['wavelengths']['rms_threshold'] = 1.0 # Might be grating dependent.. - par['calibrations']['wavelengths']['sigdetect'] = [4.0, 5.0, 5.0, 5.0, 5.0, 5.0] #5.0 - par['calibrations']['wavelengths']['lamps'] = ['OH_GNIRS'] - #par['calibrations']['wavelengths']['nonlinear_counts'] = self.detector[0]['nonlinear'] * self.detector[0]['saturation'] - par['calibrations']['wavelengths']['n_first'] = 2 - par['calibrations']['wavelengths']['n_final'] = [1, 3, 3, 3, 3, 3] - - # Reidentification parameters - par['calibrations']['wavelengths']['method'] = 'reidentify' - par['calibrations']['wavelengths']['cc_thresh'] = 0.6 - par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gnirs.fits' -# par['calibrations']['wavelengths']['ech_fix_format'] = True - # Echelle parameters - # JFH This is provisional these IDs should be checked. - par['calibrations']['wavelengths']['echelle'] = True - par['calibrations']['wavelengths']['ech_nspec_coeff'] = 3 - par['calibrations']['wavelengths']['ech_norder_coeff'] = 5 - par['calibrations']['wavelengths']['ech_sigrej'] = 3.0 - - # Tilts - par['calibrations']['tilts']['tracethresh'] = [5.0, 10, 10, 10, 10, 10] - par['calibrations']['tilts']['sig_neigh'] = 5.0 - par['calibrations']['tilts']['nfwhm_neigh'] = 2.0 - # 10/mmLBSX_G5532 setup, covering YJHK with the long blue camera and SXD prism - elif '10/mmLBSX' in self.dispname: - # Edges - par['calibrations']['slitedges']['edge_thresh'] = 20. - par['calibrations']['slitedges']['trace_thresh'] = 10. - par['calibrations']['slitedges']['fit_order'] = 2 - par['calibrations']['slitedges']['max_shift_adj'] = 0.5 - par['calibrations']['slitedges']['det_min_spec_length'] = 0.20 - par['calibrations']['slitedges']['fit_min_spec_length'] = 0.20 - par['calibrations']['slitedges']['left_right_pca'] = True # Actually we need a parameter to disable PCA entirely - par['calibrations']['slitedges']['sync_predict'] = 'nearest' - - # Wavelengths - par['calibrations']['wavelengths']['rms_threshold'] = 1.0 # Might be grating dependent.. - par['calibrations']['wavelengths']['sigdetect'] = 5.0 - par['calibrations']['wavelengths']['lamps'] = ['Ar_IR_GNIRS'] - #par['calibrations']['wavelengths']['nonlinear_counts'] = self.detector[0]['nonlinear'] * self.detector[0]['saturation'] - par['calibrations']['wavelengths']['n_first'] = 2 - par['calibrations']['wavelengths']['n_final'] = [3, 3, 3, 3] - # Reidentification parameters - par['calibrations']['wavelengths']['method'] = 'reidentify' - par['calibrations']['wavelengths']['cc_thresh'] = 0.6 - par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gnirs_10mm_LBSX.fits' -# par['calibrations']['wavelengths']['ech_fix_format'] = True - # Echelle parameters - par['calibrations']['wavelengths']['echelle'] = True - par['calibrations']['wavelengths']['ech_nspec_coeff'] = 3 - par['calibrations']['wavelengths']['ech_norder_coeff'] = 3 - par['calibrations']['wavelengths']['ech_sigrej'] = 3.0 - - # Tilts - par['calibrations']['tilts']['tracethresh'] = [10, 10, 10, 10] - par['calibrations']['tilts']['sig_neigh'] = 5.0 - par['calibrations']['tilts']['nfwhm_neigh'] = 2.0 - else: - msgs.error('Unrecognized GNIRS dispname') - - return par - def init_meta(self): """ Define how metadata are derived from the spectrograph files. @@ -230,6 +91,8 @@ def init_meta(self): self.meta['dithoff'] = dict(card=None, compound=True) # Extras for config and frametyping + self.meta['filter1'] = dict(ext=0, card='FILTER2') + self.meta['slitwid'] = dict(ext=0, compound=True, card=None) self.meta['dispname'] = dict(ext=0, card='GRATING') self.meta['hatch'] = dict(ext=0, card='COVER') self.meta['dispangle'] = dict(ext=0, card='GRATTILT', rtol=1e-4) @@ -255,6 +118,39 @@ def compound_meta(self, headarr, meta_key): return headarr[0].get('QOFFSET') else: return 0.0 + elif meta_key == 'slitwid': + deckname = headarr[0].get('DECKER') + if 'LR-IFU' in deckname: + return 0.15/3600.0 # divide by 3600 for degrees + elif 'HR-IFU' in deckname: + return 0.05/3600.0 # divide by 3600 for degrees + else: + # TODO :: Need to provide a more complete set of options here + return None + elif meta_key == 'obstime': + try: + return Time(headarr[0]['DATE-OBS'] + "T" + headarr[0]['TIME-OBS']) + except KeyError: + msgs.warn("Time of observation is not in header") + return 0.0 + elif meta_key == 'pressure': + try: + return headarr[0]['PRESSURE'] * 0.001 # Must be in astropy.units.bar + except KeyError: + msgs.warn("Pressure is not in header") + return 0.0 + elif meta_key == 'temperature': + try: + return headarr[0]['TAMBIENT'] # Must be in astropy.units.deg_C + except KeyError: + msgs.warn("Temperature is not in header") + return 0.0 + elif meta_key == 'humidity': + try: + return headarr[0]['HUMIDITY'] + except KeyError: + msgs.warn("Humidity is not in header") + return 0.0 else: msgs.error("Not ready for this compound meta") @@ -296,7 +192,7 @@ def raw_header_cards(self): def pypeit_file_keys(self): """ - Define the list of keys to be output into a standard PypeIt file. + Define the list of keys to be output into a standard ``PypeIt`` file. Returns: :obj:`list`: The list of keywords in the relevant @@ -345,6 +241,250 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool) + @classmethod + def default_pypeit_par(cls): + """ + Return the default parameters to use for this instrument. + + Returns: + :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by + all of PypeIt methods. + """ + par = super().default_pypeit_par() + + # Image processing steps + turn_off = dict(use_illumflat=False, use_biasimage=False, use_overscan=False, + use_darkimage=False) + par.reset_all_processimages_par(**turn_off) + + # Flats + par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.90 + par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.10 + + # Relatively short slit, so keep the spatial tilt order low + par['calibrations']['tilts']['spat_order'] = 1 + + # Reduce parameters + # par['reduce']['findobj']['snr_thresh'] = 5.0 # Object finding threshold + par['reduce']['findobj']['find_trim_edge'] = [2, 2] # Slit is too short to trim 5,5 especially + par['reduce']['skysub']['bspline_spacing'] = 0.8 + par['reduce']['skysub']['global_sky_std'] = False # Do not perform global sky subtraction for standard stars + par['reduce']['skysub']['no_poly'] = True # Do not use polynomial degree of freedom for global skysub + par['reduce']['extraction']['model_full_slit'] = True # local sky subtraction operates on entire slit + par['reduce']['findobj']['maxnumber_sci'] = 2 # Slit is narrow so allow two objects per order + par['reduce']['findobj']['maxnumber_std'] = 1 # Slit is narrow so allow one object per order + # Standards + par['calibrations']['standardframe']['process']['mask_cr'] = False # Do not mask_cr standards + + # Do not correct for flexure + par['flexure']['spec_method'] = 'skip' + + # Set the default exposure time ranges for the frame typing + par['calibrations']['pixelflatframe']['exprng'] = [None, 30] + par['calibrations']['traceframe']['exprng'] = [None, 30] + par['calibrations']['standardframe']['exprng'] = [None, 30] + par['scienceframe']['exprng'] = [30, None] + + # Sensitivity function parameters + par['sensfunc']['algorithm'] = 'IR' + par['sensfunc']['polyorder'] = 6 + par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + return par + + def config_specific_par(self, scifile, inp_par=None): + """ + Modify the PypeIt parameters to hard-wired values used for + specific instrument configurations. + + Args: + scifile (:obj:`str`): + File to use when determining the configuration and how + to adjust the input parameters. + inp_par (:class:`~pypeit.par.parset.ParSet`, optional): + Parameter set used for the full run of PypeIt. If None, + use :func:`default_pypeit_par`. + + Returns: + :class:`~pypeit.par.parset.ParSet`: The PypeIt parameter set + adjusted for configuration specific parameter values. + """ + par = super().config_specific_par(scifile, inp_par=inp_par) + # TODO This is a hack for now until we figure out how to set dispname + # and other meta information in the spectrograph class itself + self.dispname = self.get_meta_value(scifile, 'dispname') + # 32/mmSB_G5533 setup, covering XYJHK with short blue camera + if '32/mm' in self.dispname: + # Edges + par['calibrations']['slitedges']['edge_thresh'] = 20. + par['calibrations']['slitedges']['trace_thresh'] = 10. + par['calibrations']['slitedges']['fit_order'] = 5 + par['calibrations']['slitedges']['max_shift_adj'] = 0.5 + par['calibrations']['slitedges']['fit_min_spec_length'] = 0.5 + + # Wavelengths + par['calibrations']['wavelengths']['rms_threshold'] = 1.0 # Might be grating dependent.. + par['calibrations']['wavelengths']['sigdetect'] = 5.0 + par['calibrations']['wavelengths']['lamps'] = ['OH_GNIRS'] + # par['calibrations']['wavelengths']['nonlinear_counts'] = self.detector[0]['nonlinear'] * self.detector[0]['saturation'] + par['calibrations']['wavelengths']['n_first'] = 2 + par['calibrations']['wavelengths']['n_final'] = 3 + + # Reidentification parameters + par['calibrations']['wavelengths']['method'] = 'reidentify' + par['calibrations']['wavelengths']['cc_thresh'] = 0.6 + par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gnirs.fits' + + # Tilts + par['calibrations']['tilts']['tracethresh'] = 10 + par['calibrations']['tilts']['sig_neigh'] = 5.0 + par['calibrations']['tilts']['nfwhm_neigh'] = 2.0 + + # Coadding. Not for longslit data this might be problematic but that is not yet supported. + par['coadd1d']['wave_method'] = 'log10' + + # 10/mmLBSX_G5532 setup, covering YJHK with the long blue camera and SXD prism + elif '10/mmLBSX' in self.dispname: + # Edges + par['calibrations']['slitedges']['edge_thresh'] = 20. + par['calibrations']['slitedges']['trace_thresh'] = 10. + par['calibrations']['slitedges']['fit_order'] = 2 + par['calibrations']['slitedges']['max_shift_adj'] = 0.5 + par['calibrations']['slitedges']['det_min_spec_length'] = 0.20 + par['calibrations']['slitedges']['fit_min_spec_length'] = 0.20 + par['calibrations']['slitedges']['sync_predict'] = 'nearest' + + # Wavelengths + par['calibrations']['wavelengths']['rms_threshold'] = 1.0 # Might be grating dependent.. + par['calibrations']['wavelengths']['sigdetect'] = 5.0 + par['calibrations']['wavelengths']['lamps'] = ['Ar_IR_GNIRS'] + # par['calibrations']['wavelengths']['nonlinear_counts'] = self.detector[0]['nonlinear'] * self.detector[0]['saturation'] + par['calibrations']['wavelengths']['n_first'] = 2 + par['calibrations']['wavelengths']['n_final'] = 3 + # Reidentification parameters + par['calibrations']['wavelengths']['method'] = 'reidentify' + par['calibrations']['wavelengths']['cc_thresh'] = 0.6 + par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gnirs_10mm_LBSX.fits' + + # Tilts + par['calibrations']['tilts']['tracethresh'] = 10 + par['calibrations']['tilts']['sig_neigh'] = 5.0 + par['calibrations']['tilts']['nfwhm_neigh'] = 2.0 + elif '10/mmLBHR_G5532' in self.dispname: + # TODO :: Need to fill this in + pass + else: + msgs.error(f'Unrecognized GNIRS dispname: {self.dispname}') + + return par + + def bpm(self, filename, det, shape=None, msbias=None): + """ + Generate a default bad-pixel mask. + + Even though they are both optional, either the precise shape for + the image (``shape``) or an example file that can be read to get + the shape (``filename`` using :func:`get_image_shape`) *must* be + provided. + + Args: + filename (:obj:`str` or None): + An example file to use to get the image shape. + det (:obj:`int`): + 1-indexed detector number to use when getting the image + shape from the example file. + shape (tuple, optional): + Processed image shape + Required if filename is None + Ignored if filename is not None + msbias (`numpy.ndarray`_, optional): + Processed bias frame used to identify bad pixels + + Returns: + `numpy.ndarray`_: An integer array with a masked value set + to 1 and an unmasked value set to 0. All values are set to + 0. + """ + msgs.info("Custom bad pixel mask for GNIRS") + # Call the base-class method to generate the empty bpm + bpm_img = super().bpm(filename, det, shape=shape, msbias=msbias) + + # JFH Changed. Dealing with detector scratch + if det == 1: + bpm_img[687:765,12:16] = 1. + bpm_img[671:687,8:13] = 1. + # bpm_img[:, 1000:] = 1. + + return bpm_img + + +class GeminiGNIRSEchelleSpectrograph(GeminiGNIRSSpectrograph): + """ + Child to handle Gemini/GNIRS echelle specific code + """ + name = 'gemini_gnirs_echelle' + pypeline = 'Echelle' + ech_fixed_format = True + + def config_specific_par(self, scifile, inp_par=None): + """ + Modify the ``PypeIt`` parameters to hard-wired values used for + specific instrument configurations. + + Args: + scifile (:obj:`str`): + File to use when determining the configuration and how + to adjust the input parameters. + inp_par (:class:`~pypeit.par.parset.ParSet`, optional): + Parameter set used for the full run of PypeIt. If None, + use :func:`default_pypeit_par`. + + Returns: + :class:`~pypeit.par.parset.ParSet`: The PypeIt parameter set + adjusted for configuration specific parameter values. + """ + par = super().config_specific_par(scifile, inp_par=inp_par) + # TODO This is a hack for now until we figure out how to set dispname + # and other meta information in the spectrograph class itself + self.dispname = self.get_meta_value(scifile, 'dispname') + # 32/mmSB_G5533 setup, covering XYJHK with short blue camera + if '32/mm' in self.dispname: + # Edges + par['calibrations']['slitedges']['left_right_pca'] = True + par['calibrations']['slitedges']['pca_order'] = 3 + + # Wavelengths + par['calibrations']['wavelengths']['sigdetect'] = [4.0, 5.0, 5.0, 5.0, 5.0, 5.0] #5.0 + par['calibrations']['wavelengths']['n_final'] = [1, 3, 3, 3, 3, 3] + + # Echelle parameters + # JFH This is provisional these IDs should be checked. + par['calibrations']['wavelengths']['echelle'] = True + par['calibrations']['wavelengths']['ech_nspec_coeff'] = 3 + par['calibrations']['wavelengths']['ech_norder_coeff'] = 5 + par['calibrations']['wavelengths']['ech_sigrej'] = 3.0 + + # Tilts + par['calibrations']['tilts']['tracethresh'] = [5.0, 10, 10, 10, 10, 10] + # 10/mmLBSX_G5532 setup, covering YJHK with the long blue camera and SXD prism + elif '10/mmLBSX' in self.dispname: + # Edges + par['calibrations']['slitedges']['left_right_pca'] = True # Actually we need a parameter to disable PCA entirely + + # Wavelengths + par['calibrations']['wavelengths']['n_final'] = [3, 3, 3, 3] + # Echelle parameters + par['calibrations']['wavelengths']['echelle'] = True + par['calibrations']['wavelengths']['ech_nspec_coeff'] = 3 + par['calibrations']['wavelengths']['ech_norder_coeff'] = 3 + par['calibrations']['wavelengths']['ech_sigrej'] = 3.0 + + # Tilts + par['calibrations']['tilts']['tracethresh'] = [10, 10, 10, 10] + else: + msgs.error('Unrecognized GNIRS dispname') + + return par + def order_platescale(self, order_vec, binning=None): """ Return the platescale for each echelle order. @@ -431,44 +571,247 @@ def spec_min_max(self): else: msgs.error('Unrecognized disperser') - def bpm(self, filename, det, shape=None, msbias=None): - """ - Generate a default bad-pixel mask. - Even though they are both optional, either the precise shape for - the image (``shape``) or an example file that can be read to get - the shape (``filename`` using :func:`get_image_shape`) *must* be - provided. +class GNIRSIFUSpectrograph(GeminiGNIRSSpectrograph): + # TODO :: A list of steps that could improve the reduction + # * Have a high threshold for detecting slit edges (par['calibrations']['slitedges']['edge_thresh'] = 100.), and have an option when inserting new traces to be the median of all other slit lengths (or a fit to the slit lengths). + # * Need to store a wavelength solution for different grating options (Note, the Holy Grail algorithm works pretty well, most of the time) + name = 'gemini_gnirs_ifu' + pypeline = 'IFU' + + def init_meta(self): + super().init_meta() + self.meta['obstime'] = dict(card=None, compound=True, required=False) + self.meta['pressure'] = dict(card=None, compound=True, required=False) + self.meta['temperature'] = dict(card=None, compound=True, required=False) + self.meta['humidity'] = dict(card=None, compound=True, required=False) + + @classmethod + def default_pypeit_par(cls): + par = super().default_pypeit_par() + + # LACosmics parameters + par['scienceframe']['process']['sigclip'] = 4.0 + par['scienceframe']['process']['objlim'] = 1.5 + par['scienceframe']['process']['use_illumflat'] = False # illumflat is applied when building the relative scale image in reduce.py, so should be applied to scienceframe too. + par['scienceframe']['process']['use_specillum'] = False # apply relative spectral illumination + par['scienceframe']['process']['spat_flexure_correct'] = False # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames + par['scienceframe']['process']['use_biasimage'] = False + par['scienceframe']['process']['use_darkimage'] = False + par['calibrations']['flatfield']['slit_illum_finecorr'] = False + # Don't do 1D extraction for 3D data - it's meaningless because the DAR correction must be performed on the 3D data. + par['reduce']['extraction']['skip_extraction'] = True # Because extraction occurs before the DAR correction, don't extract + + #par['calibrations']['flatfield']['tweak_slits'] = False # Do not tweak the slit edges (we want to use the full slit) + par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.0 # Make sure the full slit is used (i.e. when the illumination fraction is > 0.5) + par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.0 # Make sure the full slit is used (i.e. no padding) + par['calibrations']['flatfield']['slit_trim'] = 2 # Trim the slit edges + par['calibrations']['slitedges']['pad'] = 2 # Need to pad out the tilts for the astrometric transform when creating a datacube. + + # Decrease the wave tilts order, given the shorter slits of the IFU + par['calibrations']['tilts']['spat_order'] = 1 + par['calibrations']['tilts']['spec_order'] = 1 + + # Make sure that this is reduced as a slit (as opposed to fiber) spectrograph + par['reduce']['cube']['slit_spec'] = True + par['reduce']['cube']['grating_corr'] = False + par['reduce']['cube']['combine'] = False # Make separate spec3d files from the input spec2d files + + # Sky subtraction parameters + par['reduce']['skysub']['no_poly'] = True + par['reduce']['skysub']['bspline_spacing'] = 0.6 + par['reduce']['skysub']['joint_fit'] = False + + # Don't correct flexure by default since the OH lines are used for wavelength calibration + # If someone does want to do a spectral flexure correction, you should use slitcen, + # because this is a slit-based IFU where no objects are extracted. + par['flexure']['spec_method'] = 'skip' + par['flexure']['spec_maxshift'] = 0 # The sky lines are used for calibration - don't allow flexure + + # Flux calibration parameters + par['sensfunc']['UVIS']['extinct_correct'] = False # This must be False - the extinction correction is performed when making the datacube + + return par + + def config_specific_par(self, scifile, inp_par=None): + """ + Modify the ``PypeIt`` parameters to hard-wired values used for + specific instrument configurations. Args: - filename (:obj:`str` or None): - An example file to use to get the image shape. - det (:obj:`int`): - 1-indexed detector number to use when getting the image - shape from the example file. - shape (tuple, optional): - Processed image shape - Required if filename is None - Ignored if filename is not None - msbias (`numpy.ndarray`_, optional): - Processed bias frame used to identify bad pixels + scifile (:obj:`str`): + File to use when determining the configuration and how + to adjust the input parameters. + inp_par (:class:`~pypeit.par.parset.ParSet`, optional): + Parameter set used for the full run of PypeIt. If None, + use :func:`default_pypeit_par`. Returns: - `numpy.ndarray`_: An integer array with a masked value set - to 1 and an unmasked value set to 0. All values are set to - 0. + :class:`~pypeit.par.parset.ParSet`: The PypeIt parameter set + adjusted for configuration specific parameter values. """ - msgs.info("Custom bad pixel mask for GNIRS") - # Call the base-class method to generate the empty bpm - bpm_img = super().bpm(filename, det, shape=shape, msbias=msbias) + par = super().config_specific_par(scifile, inp_par=inp_par) + # Obtain a header keyword to determine which range is being used + filter = self.get_meta_value(scifile, 'filter1') + par['calibrations']['slitedges']['edge_thresh'] = 30. + # TODO :: The following wavelength solutions are not general enough - need to implement a solution for each setup+grating + # TODO BEFORE PR MERGE :: The full_template solutions below were generated (quickly!) from holy-grail... might want to redo this... + if filter == 'X_G0518': # H band + par['calibrations']['wavelengths']['method'] = 'holy-grail' + elif filter == 'J_G0517': # K band + par['calibrations']['wavelengths']['method'] = 'holy-grail' + elif filter == 'H_G0516': # H band + par['calibrations']['wavelengths']['method'] = 'full_template' + par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gnirs_lrifu_H.fits' + elif filter == 'K_G0515': # K band + par['calibrations']['wavelengths']['method'] = 'full_template' + par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gnirs_lrifu_K.fits' + else: + par['calibrations']['wavelengths']['method'] = 'holy-grail' - # JFH Changed. Dealing with detector scratch - if det == 1: - bpm_img[687:765,12:16] = 1. - bpm_img[671:687,8:13] = 1. - # bpm_img[:, 1000:] = 1. + return par - return bpm_img + def get_wcs(self, hdr, slits, platescale, wave0, dwv, spatial_scale=None): + """ + Construct/Read a World-Coordinate System for a frame. + Args: + hdr (`astropy.io.fits.Header`_): + The header of the raw frame. The information in this + header will be extracted and returned as a WCS. + slits (:class:`~pypeit.slittrace.SlitTraceSet`): + Slit traces. + platescale (:obj:`float`): + The platescale of an unbinned pixel in arcsec/pixel (e.g. + detector.platescale). + wave0 (:obj:`float`): + The wavelength zeropoint. + dwv (:obj:`float`): + Change in wavelength per spectral pixel. + Returns: + `astropy.wcs.WCS`_: The world-coordinate system. + """ + msgs.info("Calculating the WCS") + # Get the x and y binning factors, and the typical slit length + binspec, binspat = parse.parse_binning(self.get_meta_value([hdr], 'binning')) + + # Get the pixel and slice scales + pxscl = platescale * binspat / 3600.0 # Need to convert arcsec to degrees + msgs.work("NEED TO WORK OUT SLICER SCALE AND PIXEL SCALE") + slscl = self.get_meta_value([hdr], 'slitwid') + if spatial_scale is not None: + if pxscl > spatial_scale / 3600.0: + msgs.warn("Spatial scale requested ({0:f}'') is less than the pixel scale ({1:f}'')".format(spatial_scale, pxscl*3600.0)) + # Update the pixel scale + pxscl = spatial_scale / 3600.0 # 3600 is to convert arcsec to degrees + + # Get the typical slit length (this changes by ~0.3% over all slits, so a constant is fine for now) + slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) + + # Get RA/DEC + raval = self.get_meta_value([hdr], 'ra') + decval = self.get_meta_value([hdr], 'dec') + + # Create a coordinate + coord = SkyCoord(raval, decval, unit=(units.deg, units.deg)) + + # Get rotator position + msgs.warn("CURRENTLY A HACK --- NEED TO FIGURE OUT RPOS and RREF FOR HRIFU FROM HEADER INFO") + if 'ROTPOSN' in hdr: + rpos = hdr['ROTPOSN'] + else: + rpos = 0. + if 'ROTREFAN' in hdr: + rref = hdr['ROTREFAN'] + else: + rref = 0. + # Get the offset and PA + rotoff = 0.0 # IFU-SKYPA offset (degrees) + skypa = rpos + rref # IFU position angle (degrees) + crota = np.radians(-(skypa + rotoff)) + + # Calculate the fits coordinates + cdelt1 = -slscl + cdelt2 = pxscl + if coord is None: + ra = 0. + dec = 0. + crota = 1 + else: + ra = coord.ra.degree + dec = coord.dec.degree + # Calculate the CD Matrix + cd11 = cdelt1 * np.cos(crota) # RA degrees per column + cd12 = abs(cdelt2) * np.sign(cdelt1) * np.sin(crota) # RA degrees per row + cd21 = -abs(cdelt1) * np.sign(cdelt2) * np.sin(crota) # DEC degress per column + cd22 = cdelt2 * np.cos(crota) # DEC degrees per row + # Get reference pixels (set these to the middle of the FOV) + crpix1 = 11 # i.e. see get_datacube_bins (11 is used as the reference point - somewhere in the middle of the FOV) + crpix2 = slitlength / 2. + crpix3 = 1. + # Get the offset + msgs.warn("HACK FOR HRIFU --- Need to obtain offset from header?") + off1 = 0. + off2 = 0. + off1 /= binspec + off2 /= binspat + crpix1 += off1 + crpix2 += off2 + + # Create a new WCS object. + msgs.info("Generating GNIRS IFU WCS") + w = wcs.WCS(naxis=3) + w.wcs.equinox = hdr['EQUINOX'] + w.wcs.name = 'GNIRS IFU' + w.wcs.radesys = 'FK5' + # Insert the coordinate frame + w.wcs.cname = ['RA', 'DEC', 'Wavelength'] + w.wcs.cunit = [units.degree, units.degree, units.Angstrom] + w.wcs.ctype = ["RA---TAN", "DEC--TAN", "WAVE"] # Note, WAVE is in vacuum + w.wcs.crval = [ra, dec, wave0] # RA, DEC, and wavelength zeropoints + w.wcs.crpix = [crpix1, crpix2, crpix3] # RA, DEC, and wavelength reference pixels + w.wcs.cd = np.array([[cd11, cd12, 0.0], [cd21, cd22, 0.0], [0.0, 0.0, dwv]]) + w.wcs.lonpole = 180.0 # Native longitude of the Celestial pole + w.wcs.latpole = 0.0 # Native latitude of the Celestial pole + + return w + + def get_datacube_bins(self, slitlength, minmax, num_wave): + r""" + Calculate the bin edges to be used when making a datacube. + Args: + slitlength (:obj:`int`): + Length of the slit in pixels + minmax (`numpy.ndarray`_): + An array with the minimum and maximum pixel locations on each + slit relative to the reference location (usually the centre + of the slit). Shape must be :math:`(N_{\rm slits},2)`, and is + typically the array returned by + :func:`~pypeit.slittrace.SlitTraceSet.get_radec_image`. + num_wave (:obj:`int`): + Number of wavelength steps. Given by:: + int(round((wavemax-wavemin)/delta_wave)) + + Args: + :obj:`tuple`: Three 1D `numpy.ndarray`_ providing the bins to use + when constructing a histogram of the spec2d files. The elements + are :math:`(x,y,\lambda)`. + """ + # TODO :: The HRIFU might have 25 slits with 13 being the reference + xbins = np.arange(1 + 21) - 11.0 - 0.5 # 21 is for 21 slices, and 11 is the reference slit + ybins = np.linspace(np.min(minmax[:, 0]), np.max(minmax[:, 1]), 1+slitlength) - 0.5 + spec_bins = np.arange(1+num_wave) - 0.5 + return xbins, ybins, spec_bins + + def pypeit_file_keys(self): + """ + Define the list of keys to be output into a standard ``PypeIt`` file. + + Returns: + :obj:`list`: The list of keywords in the relevant + :class:`~pypeit.metadata.PypeItMetaData` instance to print to the + :ref:`pypeit_file`. + """ + return super().pypeit_file_keys() + ['filter'] diff --git a/pypeit/spectrographs/gtc_osiris.py b/pypeit/spectrographs/gtc_osiris.py index cead6e8dbb..9c8777746d 100644 --- a/pypeit/spectrographs/gtc_osiris.py +++ b/pypeit/spectrographs/gtc_osiris.py @@ -65,7 +65,7 @@ def get_detector_par(self, det, hdu=None): specflip = True, spatflip = False, platescale = 0.125, # arcsec per pixel - darkcurr = 5.0, + darkcurr = 5.0, #e-/hr/pixel saturation = 65535., # ADU nonlinear = 0.95, mincounts = 0, @@ -100,11 +100,11 @@ def default_pypeit_par(cls): par['calibrations']['pixelflatframe']['process']['combine'] = 'median' # Wavelength calibration methods par['calibrations']['wavelengths']['method'] = 'full_template' - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI,NeI,ArI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI','NeI','ArI'] # Set the default exposure time ranges for the frame typing par['scienceframe']['exprng'] = [90, None] - par['calibrations']['biasframe']['exprng'] = [None, 1] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['arcframe']['exprng'] = [None, None] # Long arc exposures @@ -288,7 +288,7 @@ def config_independent_frames(self): keywords that can be used to assign the frames to a configuration group. """ - return {'standard': 'dispname', 'bias': None, 'dark': None} + return {'standard': 'dispname', 'bias': 'binning', 'dark': 'binning'} def config_specific_par(self, scifile, inp_par=None): """ @@ -320,7 +320,7 @@ def config_specific_par(self, scifile, inp_par=None): # Wavelength calibration and setup-dependent parameters if self.get_meta_value(scifile, 'dispname') == 'R300B': - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R300B.fits' par['reduce']['findobj']['find_min_max'] = [750, 2051] par['calibrations']['slitedges']['det_min_spec_length'] = 0.25 @@ -330,7 +330,7 @@ def config_specific_par(self, scifile, inp_par=None): par['reduce']['cube']['wave_min'] = 3600.0 par['reduce']['cube']['wave_max'] = 7200.0 elif self.get_meta_value(scifile, 'dispname') == 'R300R': - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R300R.fits' par['reduce']['findobj']['find_min_max'] = [750, 2051] par['calibrations']['slitedges']['det_min_spec_length'] = 0.25 @@ -340,38 +340,38 @@ def config_specific_par(self, scifile, inp_par=None): par['reduce']['cube']['wave_min'] = 4800.0 par['reduce']['cube']['wave_max'] = 10000.0 elif self.get_meta_value(scifile, 'dispname') == 'R500B': - par['calibrations']['wavelengths']['lamps'] = ['HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R500B.fits' par['reduce']['findobj']['find_min_max'] = [500, 2051] par['reduce']['cube']['wave_min'] = 3600.0 par['reduce']['cube']['wave_max'] = 7200.0 elif self.get_meta_value(scifile, 'dispname') == 'R500R': - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R500R.fits' par['reduce']['findobj']['find_min_max'] = [450, 2051] par['reduce']['cube']['wave_min'] = 4800.0 par['reduce']['cube']['wave_max'] = 10000.0 elif self.get_meta_value(scifile, 'dispname') == 'R1000B': - par['calibrations']['wavelengths']['lamps'] = ['ArI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['ArI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R1000B.fits' elif self.get_meta_value(scifile, 'dispname') == 'R1000R': - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R1000R.fits' elif self.get_meta_value(scifile, 'dispname') == 'R2000B': par['calibrations']['wavelengths']['fwhm'] = 15.0 - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2000B.fits' elif self.get_meta_value(scifile, 'dispname') == 'R2500U': - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2500U.fits' elif self.get_meta_value(scifile, 'dispname') == 'R2500V': par['calibrations']['wavelengths']['lamps'] = ['HgI','NeI','XeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2500V.fits' elif self.get_meta_value(scifile, 'dispname') == 'R2500R': - par['calibrations']['wavelengths']['lamps'] = ['ArI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['ArI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2500R.fits' elif self.get_meta_value(scifile, 'dispname') == 'R2500I': - par['calibrations']['wavelengths']['lamps'] = ['ArI,XeI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['ArI','XeI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2500I.fits' par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['IR']['telgridfile'] = "TelFit_MaunaKea_3100_26100_R20000.fits" @@ -500,7 +500,7 @@ def get_wcs(self, hdr, slits, platescale, wave0, dwv, spatial_scale=None): Change in wavelength per spectral pixel. Returns: - `astropy.wcs.wcs.WCS`_: The world-coordinate system. + `astropy.wcs.WCS`_: The world-coordinate system. """ msgs.info("Calculating the WCS") # Get the x and y binning factors, and the typical slit length @@ -657,7 +657,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.127, # arcsec per pixel - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 65535., # ADU nonlinear = 0.95, mincounts = 0, @@ -676,7 +676,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.127, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 65535., # ADU nonlinear = 0.95, mincounts = 0, @@ -711,11 +711,11 @@ def default_pypeit_par(cls): par['calibrations']['pixelflatframe']['process']['combine'] = 'median' # Wavelength calibration methods par['calibrations']['wavelengths']['method'] = 'full_template' - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI,NeI,ArI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI','NeI','ArI'] # Set the default exposure time ranges for the frame typing par['scienceframe']['exprng'] = [90, None] - par['calibrations']['biasframe']['exprng'] = [None, 1] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['arcframe']['exprng'] = [None, None] # Long arc exposures @@ -874,7 +874,7 @@ def config_independent_frames(self): keywords that can be used to assign the frames to a configuration group. """ - return {'standard': 'dispname','bias': None, 'dark': None} + return {'standard': 'dispname','bias': 'binning', 'dark': 'binning'} def config_specific_par(self, scifile, inp_par=None): """ @@ -906,41 +906,41 @@ def config_specific_par(self, scifile, inp_par=None): # Wavelength calibrations if self.get_meta_value(scifile, 'dispname') == 'R300B': - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R300B.fits' par['reduce']['findobj']['find_min_max']=[750,2051] elif self.get_meta_value(scifile, 'dispname') == 'R300R': - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R300R.fits' par['reduce']['findobj']['find_min_max']=[750,2051] elif self.get_meta_value(scifile, 'dispname') == 'R500B': - par['calibrations']['wavelengths']['lamps'] = ['HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R500B.fits' par['reduce']['findobj']['find_min_max']=[500,2051] elif self.get_meta_value(scifile, 'dispname') == 'R500R': - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R500R.fits' par['reduce']['findobj']['find_min_max']=[450,2051] elif self.get_meta_value(scifile, 'dispname') == 'R1000B': - par['calibrations']['wavelengths']['lamps'] = ['ArI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['ArI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R1000B.fits' elif self.get_meta_value(scifile, 'dispname') == 'R1000R': - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R1000R.fits' elif self.get_meta_value(scifile, 'dispname') == 'R2000B': - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2000B.fits' elif self.get_meta_value(scifile, 'dispname') == 'R2500U': - par['calibrations']['wavelengths']['lamps'] = ['XeI,HgI'] + par['calibrations']['wavelengths']['lamps'] = ['XeI','HgI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2500U.fits' elif self.get_meta_value(scifile, 'dispname') == 'R2500V': par['calibrations']['wavelengths']['lamps'] = ['HgI','NeI','XeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2500V.fits' elif self.get_meta_value(scifile, 'dispname') == 'R2500R': - par['calibrations']['wavelengths']['lamps'] = ['ArI,HgI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['ArI','HgI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2500R.fits' elif self.get_meta_value(scifile, 'dispname') == 'R2500I': - par['calibrations']['wavelengths']['lamps'] = ['ArI,XeI,NeI'] + par['calibrations']['wavelengths']['lamps'] = ['ArI','XeI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2500I.fits' par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['IR']['telgridfile'] = "TelFit_MaunaKea_3100_26100_R20000.fits" diff --git a/pypeit/spectrographs/jwst_nircam.py b/pypeit/spectrographs/jwst_nircam.py index d38e3f8dc4..bf0272fafc 100644 --- a/pypeit/spectrographs/jwst_nircam.py +++ b/pypeit/spectrographs/jwst_nircam.py @@ -67,7 +67,7 @@ def get_detector_par(self, det, hdu=None): ygap=0., ysize=1., platescale=0.063, - darkcurr=0.0335, # electron/s + darkcurr=120.6, # e-/pixel/hour (=0.0335 e-/pixel/s) saturation=59200., nonlinear=0.95, # need to look up and update mincounts=-1e10, @@ -83,7 +83,7 @@ def get_detector_par(self, det, hdu=None): detector_dict2.update(dict( det=2, dataext=0, - darkcurr=0.035, + darkcurr=126.0, # e-/pixel/hour (=0.035 e-/pixel/s) saturation=58500., gain=np.atleast_1d(1.80), ronoise=np.atleast_1d(8.57), diff --git a/pypeit/spectrographs/jwst_nirspec.py b/pypeit/spectrographs/jwst_nirspec.py index 2369092d90..e84d673fa9 100644 --- a/pypeit/spectrographs/jwst_nirspec.py +++ b/pypeit/spectrographs/jwst_nirspec.py @@ -5,12 +5,11 @@ """ import glob import numpy as np -from astropy.io import fits -from astropy.time import Time from pypeit import msgs from pypeit import telescopes from pypeit.core import framematch +from pypeit import io from pypeit.par import pypeitpar from pypeit.spectrographs import spectrograph from pypeit.core import parse @@ -50,15 +49,15 @@ def get_detector_par(self, det, hdu=None): detector_dict1 = dict( binning='1,1', det=1, - dataext=0, + dataext=1, specaxis=1, specflip=False, spatflip=False, - xgap=0., + xgap=180., ygap=0., ysize=1., platescale=0.1, - darkcurr=0.0092, + darkcurr=33.12, # e-/pixel/hour (=0.0092 e-/pixel/s) saturation=55100., nonlinear=0.95, # need to look up and update mincounts=-1e10, @@ -73,8 +72,8 @@ def get_detector_par(self, det, hdu=None): detector_dict2 = detector_dict1.copy() detector_dict2.update(dict( det=2, - dataext=0, - darkcurr=0.0057, + dataext=1, + darkcurr=20.52, # e-/pixel/hour, (=0.0057 e-/pixel/s) saturation=60400., gain=np.atleast_1d(1.137), ronoise=np.atleast_1d(6.60), @@ -82,34 +81,6 @@ def get_detector_par(self, det, hdu=None): detector_dicts = [detector_dict1, detector_dict2] return detector_container.DetectorContainer(**detector_dicts[det-1]) - def init_meta(self): - """ - Define how metadata are derived from the spectrograph files. - - That is, this associates the PypeIt-specific metadata keywords - with the instrument-specific header cards using :attr:`meta`. - """ - self.meta = {} - # Required (core) - self.meta['ra'] = dict(ext=0, card='RA') - self.meta['dec'] = dict(ext=0, card='DEC') - self.meta['target'] = dict(ext=0, card='OBJECT') - self.meta['decker'] = dict(ext=0, card='APERTURE') - self.meta['dichroic'] = dict(ext=0, card='INSFILTE') - self.meta['binning'] = dict(ext=0, card=None, compound=True) - self.meta['mjd'] = dict(ext=0, card=None, compound=True) - self.meta['exptime'] = dict(ext=0, card='EXPTIME') - self.meta['airmass'] = dict(ext=0, card='AIRMASS') - - # Extras for config and frametyping - self.meta['dispname'] = dict(ext=0, card='DISPERSE') - self.meta['idname'] = dict(ext=0, card='IMAGETYP') - - # used for arc and continuum lamps - self.meta['lampstat01'] = dict(ext=0, card=None, compound=True) - self.meta['instrument'] = dict(ext=0, card='INSTRUME') - - @classmethod def default_pypeit_par(cls): """ @@ -142,7 +113,7 @@ def default_pypeit_par(cls): # Extraction par['reduce']['extraction']['model_full_slit'] = True - par['reduce']['extraction']['sn_gauss'] = 6.0 + par['reduce']['extraction']['sn_gauss'] = 5.0 par['reduce']['extraction']['boxcar_radius'] = 0.2 # extent in calwebb is 0.55" source and on NIRSpec website par['reduce']['extraction']['use_2dmodel_mask'] = False # Don't use 2d mask in local skysub @@ -154,10 +125,185 @@ def default_pypeit_par(cls): # Skip reference frame correction for now. par['calibrations']['wavelengths']['refframe'] = 'observed' + return par + def init_meta(self): + """ + Define how metadata are derived from the spectrograph files. + + That is, this associates the ``PypeIt``-specific metadata keywords + with the instrument-specific header cards using :attr:`meta`. + """ + self.meta = {} + # Required (core) + self.meta['ra'] = dict(ext=0, card='TARG_RA') + self.meta['dec'] = dict(ext=0, card='TARG_DEC') + self.meta['target'] = dict(ext=0, card='TARGPROP') + self.meta['mode'] = dict(ext=0, card='EXP_TYPE') + self.meta['decker'] = dict(ext=0, card='APERNAME') + + self.meta['binning'] = dict(ext=0, card=None, default='1,1') + self.meta['mjd'] = dict(ext=0, card='EXPMID') + self.meta['exptime'] = dict(ext=0, card='EFFEXPTM') + self.meta['airmass'] = dict(ext=0, card=None, compound=True) + + # Extras for config and frametyping + self.meta['dispname'] = dict(ext=0, card='GRATING') + self.meta['filter1'] = dict(ext=0, card='FILTER') + self.meta['idname'] = dict(ext=0, card=None, compound=True) + self.meta['dithpat'] = dict(ext=0, card=None, compound=True) + self.meta['dithpos'] = dict(ext=0, card='YOFFSET') + + # used for arc and continuum lamps + self.meta['lampstat01'] = dict(ext=0, card=None, compound=True) + self.meta['instrument'] = dict(ext=0, card='INSTRUME') + + + + def compound_meta(self, headarr, meta_key): + """ + Methods to generate metadata requiring interpretation of the header + data, instead of simply reading the value of a header card. + + Args: + headarr (:obj:`list`): + List of `astropy.io.fits.Header`_ objects. + meta_key (:obj:`str`): + Metadata keyword to construct. + + Returns: + object: Metadata value read from the header(s). + """ + + if meta_key == 'dithpat': + exp_type = headarr[0].get('EXP_TYPE') + if exp_type == 'NRS_MSASPEC': + return headarr[0].get('NOD_TYPE') + elif exp_type == 'NRS_FIXEDSLIT': + return headarr[0].get('PATTTYPE') + + + def configuration_keys(self): + """ + Return the metadata keys that define a unique instrument + configuration. + + This list is used by :class:`~pypeit.metadata.PypeItMetaData` to + identify the unique configurations among the list of frames read + for a given reduction. + + Returns: + :obj:`list`: List of keywords of data pulled from file headers + and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` + object. + """ + return ['dispname', 'filter1', 'decker'] + + def pypeit_file_keys(self): + """ + Define the list of keys to be output into a standard ``PypeIt`` file. + + Returns: + :obj:`list`: The list of keywords in the relevant + :class:`~pypeit.metadata.PypeItMetaData` instance to print to the + :ref:`pypeit_file`. + """ + pypeit_keys = super().pypeit_file_keys() + pypeit_keys.remove('airmass') + pypeit_keys.remove('binning') + return pypeit_keys + + + def check_frame_type(self, ftype, fitstbl, exprng=None): + """ + Check for frames of the provided type. + + Args: + ftype (:obj:`str`): + Type of frame to check. Must be a valid frame type; see + frame-type :ref:`frame_type_defs`. + fitstbl (`astropy.table.Table`_): + The table with the metadata for one or more frames to check. + exprng (:obj:`list`, optional): + Range in the allowed exposure time for a frame of type + ``ftype``. See + :func:`pypeit.core.framematch.check_frame_exptime`. + + Returns: + `numpy.ndarray`_: Boolean array with the flags selecting the + exposures in ``fitstbl`` that are ``ftype`` type frames. + """ + + if ftype == 'science': + return np.ones(len(fitstbl), dtype=bool) + msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) + return np.zeros(len(fitstbl), dtype=bool) + + + + @property + def allowed_mosaics(self): + """ + Return the list of allowed detector mosaics. + + JWST/NIRSpec only allows for mosaicing the NRS1 and NRS2 detectors. + + Returns: + :obj:`list`: List of tuples, where each tuple provides the 1-indexed + detector numbers that can be combined into a mosaic and processed by + ``PypeIt``. + """ + return [(1,2)] + + def get_rawimage(self, raw_file, det): + """ + Read raw images and generate a few other bits and pieces + that are key for image processing. + + Based on readmhdufits.pro + + Parameters + ---------- + raw_file : :obj:`str` + File to read + det : :obj:`int` + 1-indexed detector to read + + Returns + ------- + detector_par : :class:`pypeit.images.detector_container.DetectorContainer` + Detector metadata parameters. + raw_img : `numpy.ndarray`_ + Raw image for this detector. + hdu : `astropy.io.fits.HDUList`_ + Opened fits file + exptime : :obj:`float` + Exposure time read from the file header + rawdatasec_img : `numpy.ndarray`_ + Data (Science) section of the detector as provided by setting the + (1-indexed) number of the amplifier used to read each detector + pixel. Pixels unassociated with any amplifier are set to 0. + oscansec_img : `numpy.ndarray`_ + Overscan section of the detector as provided by setting the + (1-indexed) number of the amplifier used to read each detector + pixel. Pixels unassociated with any amplifier are set to 0. + """ + # Check for file; allow for extra .gz, etc. suffix + fil = glob.glob(raw_file + '*') + if len(fil) != 1: + msgs.error("Found {:d} files matching {:s}".format(len(fil))) - return par + # Read + msgs.info("Reading JWST/NIRSpec file: {:s}".format(fil[0])) + hdu = io.fits_open(fil[0]) + head0 = hdu[0].header + detector = self.get_detector_par(det if det is not None else 1, hdu=hdu) + raw_img = hdu[detector['dataext']].data.astype(float) + # Need the exposure time + exptime = hdu[self.meta['exptime']['ext']].header[self.meta['exptime']['card']] + # Return + return detector, raw_img.T, hdu, exptime, None, None diff --git a/pypeit/spectrographs/keck_deimos.py b/pypeit/spectrographs/keck_deimos.py index 3eedd90a98..7dc87a1054 100644 --- a/pypeit/spectrographs/keck_deimos.py +++ b/pypeit/spectrographs/keck_deimos.py @@ -30,7 +30,7 @@ from pypeit.core import wave from pypeit import specobj, specobjs from pypeit.spectrographs import spectrograph -from pypeit.images import detector_container +from pypeit.images.detector_container import DetectorContainer from pypeit import data from pypeit.images.mosaic import Mosaic from pypeit.core.mosaic import build_image_mosaic_transform @@ -137,7 +137,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.1185, - darkcurr = 3.30, # changed by DP. Taken from WMKO measurements on 2022-Apr-22 + darkcurr = 3.30, # units are e-/pixel/hour... NOTE : changed by DP. Taken from WMKO measurements on 2022-Apr-22 saturation = 65535., # ADU nonlinear = 0.95, # Changed by JFH from 0.86 to 0.95 mincounts = -1e10, @@ -150,7 +150,7 @@ def get_detector_par(self, det, hdu=None): detector_dict2.update(dict( det=2, dataext=2, - darkcurr=3.60, + darkcurr=3.60, # e-/pixel/hour gain=np.atleast_1d(1.188), ronoise=np.atleast_1d(2.491), )) @@ -159,7 +159,7 @@ def get_detector_par(self, det, hdu=None): detector_dict3.update(dict( det=3, dataext=3, - darkcurr=3.50, + darkcurr=3.50, # e-/pixel/hour gain=np.atleast_1d(1.248), ronoise=np.atleast_1d(2.618), )) @@ -168,7 +168,7 @@ def get_detector_par(self, det, hdu=None): detector_dict4.update(dict( det=4, dataext=4, - darkcurr=3.70, + darkcurr=3.70, # e-/pixel/hour gain=np.atleast_1d(1.220), ronoise=np.atleast_1d(2.557), )) @@ -177,7 +177,7 @@ def get_detector_par(self, det, hdu=None): detector_dict5.update(dict( det=5, dataext=5, - darkcurr=2.70, + darkcurr=2.70, # e-/pixel/hour gain=np.atleast_1d(1.184), ronoise=np.atleast_1d(2.482), )) @@ -186,7 +186,7 @@ def get_detector_par(self, det, hdu=None): detector_dict6.update(dict( det=6, dataext=6, - darkcurr=3.80, + darkcurr=3.80, # e-/pixel/hour gain=np.atleast_1d(1.177), ronoise=np.atleast_1d(2.469), )) @@ -195,7 +195,7 @@ def get_detector_par(self, det, hdu=None): detector_dict7.update(dict( det=7, dataext=7, - darkcurr=3.30, + darkcurr=3.30, # e-/pixel/hour gain=np.atleast_1d(1.201), ronoise=np.atleast_1d(2.518), )) @@ -204,7 +204,7 @@ def get_detector_par(self, det, hdu=None): detector_dict8.update(dict( det=8, dataext=8, - darkcurr=3.70, + darkcurr=3.70, # e-/pixel/hour gain=np.atleast_1d(1.230), ronoise=np.atleast_1d(2.580), )) @@ -251,7 +251,7 @@ def get_detector_par(self, det, hdu=None): detectors = [detector_dict1, detector_dict2, detector_dict3, detector_dict4, detector_dict5, detector_dict6, detector_dict7, detector_dict8] # Return - return detector_container.DetectorContainer(**detectors[det-1]) + return DetectorContainer(**detectors[det-1]) @classmethod def default_pypeit_par(cls): @@ -561,7 +561,7 @@ def config_independent_frames(self): keywords that can be used to assign the frames to a configuration group. """ - return {'bias': 'dateobs', 'dark': 'dateobs'} + return {'bias': ['dateobs', 'binning', 'amp'], 'dark': ['dateobs', 'binning', 'amp']} def pypeit_file_keys(self): """ @@ -1524,11 +1524,8 @@ def list_detectors(self, mosaic=False): the array is 2D, there are detectors separated along the dispersion axis. """ - if mosaic: - return np.array([self.get_det_name(_det) for _det in self.allowed_mosaics]) - else: - return np.array([detector_container.DetectorContainer.get_name(i+1) - for i in range(self.ndet)]).reshape(2,-1) + dets = super().list_detectors(mosaic=mosaic) + return dets if mosaic else dets.reshape(2,-1) def spec1d_match_spectra(self, sobjs): """ @@ -1549,7 +1546,7 @@ def spec1d_match_spectra(self, sobjs): # MATCH RED TO BLUE VIA RA/DEC # mb = sobjs['DET'] <=4 # mr = sobjs['DET'] >4 - det = np.array([detector_container.DetectorContainer.parse_name(d) for d in sobjs.DET]) + det = np.array([DetectorContainer.parse_name(d) for d in sobjs.DET]) mb = det <= 4 mr = det > 4 diff --git a/pypeit/spectrographs/keck_esi.py b/pypeit/spectrographs/keck_esi.py new file mode 100644 index 0000000000..a8fbd33ea1 --- /dev/null +++ b/pypeit/spectrographs/keck_esi.py @@ -0,0 +1,417 @@ +""" +Module for Keck/ESI specific methods. + +.. include:: ../include/links.rst +""" +import os + +from IPython import embed + +import numpy as np + +from astropy.time import Time + +from pypeit import msgs +from pypeit import telescopes +from pypeit import io +from pypeit.core import framematch +from pypeit.core import parse +from pypeit.spectrographs import spectrograph +from pypeit.images import detector_container + +class KeckESISpectrograph(spectrograph.Spectrograph): + """ + Child to handle Keck/ESI specific code + """ + ndet = 1 + name = 'keck_esi' + camera = 'ESI' + header_name = 'ESI' + #url = 'https://www.lco.cl/?epkb_post_type_1=mage' + ech_fixed_format = True + telescope = telescopes.KeckTelescopePar() + pypeline = 'Echelle' + supported = True + #comment = 'See :doc:`mage`' + + def get_detector_par(self, det, hdu=None): + """ + Return metadata for the selected detector. + + Args: + det (:obj:`int`): + 1-indexed detector number. + hdu (`astropy.io.fits.HDUList`_, optional): + The open fits file with the raw image of interest. If not + provided, frame-dependent parameters are set to a default. + + Returns: + :class:`~pypeit.images.detector_container.DetectorContainer`: + Object with the detector metadata. + """ + # Binning + # TODO: Could this be detector dependent?? + binning = '1,1' if hdu is None else self.get_meta_value(self.get_headarr(hdu), 'binning') + + # Detector 1 + detector_dict = dict( + binning = binning, + det = 1, + dataext = 0, + specaxis = 0, + specflip = False, + spatflip = False, + # plate scale in arcsec/pixel + platescale = 0.1542, + # electrons/pixel/hour. + darkcurr = 2.10, # e/pixel/hour... Note : Could be updated + saturation = 65535., + # CCD is linear to better than 0.5 per cent up to digital saturation (65,536 DN including bias) in the Fast readout mode. + nonlinear = 0.99, + mincounts = -1e10, + numamplifiers = 2, + gain = np.atleast_1d([1.3, 1.3]), + ronoise = np.atleast_1d([2.5,2.5]), + ) + return detector_container.DetectorContainer(**detector_dict) + + @classmethod + def default_pypeit_par(cls): + """ + Return the default parameters to use for this instrument. + + Returns: + :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by + all of ``PypeIt`` methods. + """ + par = super().default_pypeit_par() + + # Bias + #par['calibrations']['biasframe']['useframe'] = 'overscan' + # Wavelengths + # 1D wavelength solution + # This is for 1x1 + par['calibrations']['wavelengths']['rms_threshold'] = 0.30 + par['calibrations']['wavelengths']['fwhm'] = 2.9 + par['calibrations']['wavelengths']['fwhm_fromlines'] = True + # + par['calibrations']['wavelengths']['sigdetect'] = 5.0 + par['calibrations']['wavelengths']['lamps'] = ['CuI', 'ArI', 'NeI', 'HgI', 'XeI', 'ArII'] + + par['calibrations']['wavelengths']['method'] = 'reidentify' + par['calibrations']['wavelengths']['cc_thresh'] = 0.50 + par['calibrations']['wavelengths']['cc_local_thresh'] = 0.50 + + # Reidentification parameters + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_esi_ECH.fits' + #par['calibrations']['wavelengths']['ech_fix_format'] = True + # Echelle parameters + par['calibrations']['wavelengths']['echelle'] = True + par['calibrations']['wavelengths']['ech_nspec_coeff'] = 4 + par['calibrations']['wavelengths']['ech_norder_coeff'] = 4 + par['calibrations']['wavelengths']['ech_sigrej'] = 3.0 + + par['scienceframe']['process']['sigclip'] = 20.0 + par['scienceframe']['process']['satpix'] = 'nothing' + + # Set slits and tilts parameters + par['calibrations']['tilts']['tracethresh'] = 10. #[10]*self.norders + par['calibrations']['slitedges']['fit_order'] = 5 + par['calibrations']['slitedges']['max_shift_adj'] = 3. + par['calibrations']['slitedges']['left_right_pca'] = True + + par['calibrations']['slitedges']['edge_thresh'] = 5.0 + par['calibrations']['slitedges']['det_min_spec_length'] = 0.2 + par['calibrations']['slitedges']['fit_min_spec_length'] = 0.4 + par['calibrations']['slitedges']['pca_sigrej'] = 1.5 + par['calibrations']['slitedges']['pca_order'] = 3 + par['calibrations']['slitedges']['add_missed_orders'] = True + # Find object parameters + par['reduce']['findobj']['find_trim_edge'] = [4,4] # Slit is too short to trim 5,5 especially with 2x binning + par['reduce']['findobj']['maxnumber_sci'] = 2 # Slit is narrow so allow one object per order + par['reduce']['findobj']['maxnumber_std'] = 1 # Slit is narrow so allow one object per order + par['reduce']['extraction']['model_full_slit'] = True # local sky subtraction operates on entire slit + + + # Always flux calibrate, starting with default parameters + # Do not correct for flexure + par['flexure']['spec_method'] = 'skip' + # Set the default exposure time ranges for the frame typing + par['calibrations']['standardframe']['exprng'] = [None, 60] + par['calibrations']['arcframe']['exprng'] = [300, None] # Allow for CuAr which can be quite long + par['calibrations']['darkframe']['exprng'] = [1, None] + par['scienceframe']['exprng'] = [60, None] + return par + + def init_meta(self): + """ + Define how metadata are derived from the spectrograph files. + + That is, this associates the ``PypeIt``-specific metadata keywords + with the instrument-specific header cards using :attr:`meta`. + """ + self.meta = {} + # Required (core) + self.meta['ra'] = dict(ext=0, card='RA') + self.meta['dec'] = dict(ext=0, card='DEC') + self.meta['target'] = dict(ext=0, card='TARGNAME') + + self.meta['decker'] = dict(ext=0, card='SLMSKNAM') + self.meta['binning'] = dict(card=None, compound=True) + self.meta['mjd'] = dict(ext=0, card='MJD-OBS') + self.meta['exptime'] = dict(ext=0, card='ELAPTIME') + self.meta['airmass'] = dict(ext=0, card='AIRMASS') + # Extras for config and frametyping + self.meta['dispname'] = dict(card=None, compound=True) + self.meta['idname'] = dict(ext=0, card='OBSTYPE') + self.meta['instrument'] = dict(ext=0, card='INSTRUME') + + # Lamps -- Have varied in time.. + self.meta['lampstat01'] = dict(ext=0, card='LAMPAR1') + self.meta['lampstat02'] = dict(ext=0, card='LAMPCU1') + self.meta['lampstat03'] = dict(ext=0, card='LAMPNE1') + self.meta['lampstat04'] = dict(ext=0, card='LAMPNE2') + self.meta['lampstat05'] = dict(ext=0, card='LAMPQTZ1') + self.meta['lampstat06'] = dict(ext=0, card='FLIMAGIN') + self.meta['lampstat07'] = dict(ext=0, card='FLSPECTR') + + # Hatch + self.meta['hatch'] = dict(ext=0, card='HATCHPOS') + + def compound_meta(self, headarr, meta_key): + """ + Methods to generate metadata requiring interpretation of the header + data, instead of simply reading the value of a header card. + + Args: + headarr (:obj:`list`): + List of `astropy.io.fits.Header`_ objects. + meta_key (:obj:`str`): + Metadata keyword to construct. + + Returns: + object: Metadata value read from the header(s). + """ + if meta_key == 'binning': + binspatial, binspec = parse.parse_binning(headarr[0]['BINNING']) + return parse.binning2string(binspec, binspatial) + elif meta_key == 'dispname': + if headarr[0]['PRISMNAM'] == 'in': + dname = 'Echellette' + else: # TODO -- Figure out prism and imaging modes + dname = 'UNKNWN' + return dname + else: + msgs.error("Not ready for this compound meta") + + def configuration_keys(self): + """ + Return the metadata keys that define a unique instrument + configuration. + + This list is used by :class:`~pypeit.metadata.PypeItMetaData` to + identify the unique configurations among the list of frames read + for a given reduction. + + Returns: + :obj:`list`: List of keywords of data pulled from file headers + and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` + object. + """ + return [] + + def check_frame_type(self, ftype, fitstbl, exprng=None): + """ + Check for frames of the provided type. + + Args: + ftype (:obj:`str`): + Type of frame to check. Must be a valid frame type; see + frame-type :ref:`frame_type_defs`. + fitstbl (`astropy.table.Table`_): + The table with the metadata for one or more frames to check. + exprng (:obj:`list`, optional): + Range in the allowed exposure time for a frame of type + ``ftype``. See + :func:`pypeit.core.framematch.check_frame_exptime`. + + Returns: + `numpy.ndarray`_: Boolean array with the flags selecting the + exposures in ``fitstbl`` that are ``ftype`` type frames. + """ + good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) + if ftype in ['pinhole', 'dark']: + # No pinhole or pinhole or dark frames + return np.zeros(len(fitstbl), dtype=bool) + if ftype in ['bias']: + return fitstbl['idname'] == 'Bias' + if ftype in ['pixelflat', 'trace', 'illumflat']: + ans = np.zeros(len(fitstbl), dtype=bool) + for kk, idnm in enumerate(fitstbl['idname']): + if idnm in ['DmFlat', 'IntFlat', 'SkyFlat']: + ans[kk] = True + return ans + if ftype in ['arc', 'tilt']: + return fitstbl['idname'] == 'Line' + if ftype == 'science': + return good_exp & (fitstbl['idname'] == 'Object') + if ftype == 'standard': + return good_exp & (fitstbl['idname'] == 'Object') + msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) + return np.zeros(len(fitstbl), dtype=bool) + + def bpm(self, filename, det, shape=None, msbias=None): + """ + Generate a default bad-pixel mask. + + Even though they are both optional, either the precise shape for + the image (``shape``) or an example file that can be read to get + the shape (``filename`` using :func:`get_image_shape`) *must* be + provided. + + Args: + filename (:obj:`str` or None): + An example file to use to get the image shape. + det (:obj:`int`): + 1-indexed detector number to use when getting the image + shape from the example file. + shape (tuple, optional): + Processed image shape + Required if filename is None + Ignored if filename is not None + msbias (`numpy.ndarray`_, optional): + Master bias frame used to identify bad pixels + + Returns: + `numpy.ndarray`_: An integer array with a masked value set + to 1 and an unmasked value set to 0. All values are set to + 0. + """ + # Call the base-class method to generate the empty bpm + bpm_img = super().bpm(filename, det, shape=shape, msbias=msbias) + + + # Get the binning + msgs.info("Custom bad pixel mask for ESI") + hdu = io.fits_open(filename) + binspatial, binspec = parse.parse_binning(hdu[0].header['BINNING']) + hdu.close() + + # Binning Independent Masking + bpm_img[:,0:(2//binspatial)] = 1 + # Mask out the 'hotspot' on the upper left coner + bpm_img[(3842//binspec):(3944//binspec), (19//binspatial):(161//binspatial)] = 1 + # Mask out the 'bad columns' on the upper left coner + bpm_img[(2642//binspec):, (418//binspatial):(442//binspatial)] = 1 + + # Return + return bpm_img + + @property + def norders(self): + """ + Number of orders for this spectograph. Should only defined for + echelle spectrographs, and it is undefined for the base class. + """ + return 10 # 15-6 + + @property + def order_spat_pos(self): + """ + Return the expected spatial position of each echelle order. + """ + return np.array([0.115, 0.245, 0.362, 0.465, 0.558, 0.642, 0.719, 0.791, 0.861, 0.933]) + + @property + def order_spat_width(self): + """ + Return the expected spatial width of each slit trace for each order, + relative to the spatial size of the detector. + """ + return np.array([0.0879, 0.0818, 0.0779, 0.0747, 0.0720, 0.0696, 0.0676, 0.0658, 0.0640, + 0.0617]) + + @property + def orders(self): + """ + Return the order number for each echelle order. + """ + return np.arange(15, 5, -1, dtype=int) + + @property + def spec_min_max(self): + """ + Return the minimum and maximum spectral pixel expected for the + spectral range of each order. + """ + spec_max = np.full(self.norders, np.inf) + spec_min = np.full(self.norders, -np.inf) + return np.vstack((spec_min, spec_max)) + + def order_platescale(self, order_vec, binning=None): + """ + Return the platescale for each echelle order. + + This routine is only defined for echelle spectrographs, and it is + undefined in the base class. + + Args: + order_vec (`numpy.ndarray`_): + The vector providing the order numbers. + binning (:obj:`str`, optional): + The string defining the spectral and spatial binning. + + Returns: + `numpy.ndarray`_: An array with the platescale for each order + provided by ``order``. + """ + norders = len(order_vec) + binspatial, binspec = parse.parse_binning(binning) + # Plate scales + unbinned_pscale = [0.120, #15 + 0.127, 0.134, 0.137, 0.144, 0.149, 0.153, + 0.158, 0.163, + 0.168, # 6 + ] + + return np.array(unbinned_pscale)*binspatial + + + + def get_rawimage(self, raw_file, det, spectrim=None): + """ Read the image + """ + # Check for file; allow for extra .gz, etc. suffix + if not os.path.isfile(raw_file): + msgs.error(f'{raw_file} not found!') + hdu = io.fits_open(raw_file) + head0 = hdu[0].header + + # Number of AMPS + namp = head0['NUMAMPS'] + + # Get post, pre-pix values + prepix = head0['PREPIX'] + postpix = head0['POSTPIX'] + preline = head0['PRELINE'] + postline = head0['POSTLINE'] + + # Grab the data + full_image = hdu[0].data.astype(float) + rawdatasec_img = np.zeros_like(full_image, dtype=int) + oscansec_img = np.zeros_like(full_image, dtype=int) + + # + nspat = int(head0['WINDOW'].split(',')[3]) // namp + for amp in range(namp): + col0 = prepix*2 + nspat*amp + # Data + rawdatasec_img[:, col0:col0+nspat] = amp+1 + # Overscan + o0 = prepix*2 + nspat*namp + postpix*amp + oscansec_img[:, o0:o0+postpix] = amp+1 + + #embed(header='435 of keck_esi.py') + + return self.get_detector_par(1, hdu=hdu), \ + full_image, hdu, head0['ELAPTIME'], rawdatasec_img, oscansec_img \ No newline at end of file diff --git a/pypeit/spectrographs/keck_hires.py b/pypeit/spectrographs/keck_hires.py index 61d6178561..df4c8eab03 100644 --- a/pypeit/spectrographs/keck_hires.py +++ b/pypeit/spectrographs/keck_hires.py @@ -150,8 +150,12 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['polyorder'] = 11 #[9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7] + par['sensfunc']['polyorder'] = 5 #[9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7] par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + + # Coadding + par['coadd1d']['wave_method'] = 'log10' + return par def init_meta(self): @@ -207,7 +211,6 @@ def compound_meta(self, headarr, meta_key): else: msgs.error("Not ready for this compound meta") - def configuration_keys(self): """ Return the metadata keys that define a unique instrument @@ -222,7 +225,7 @@ def configuration_keys(self): and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` object. """ - return ['filter1', 'echangle', 'xdangle'] + return ['filter1', 'echangle', 'xdangle', 'binning'] def raw_header_cards(self): """ @@ -506,7 +509,6 @@ def allowed_mosaics(self): def default_mosaic(self): return self.allowed_mosaics[0] - def get_detector_par(self, det, hdu=None): """ Return metadata for the selected detector. @@ -535,7 +537,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.135, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 65535., nonlinear = 0.7, # Website says 0.6, but we'll push it a bit mincounts = -1e10, diff --git a/pypeit/spectrographs/keck_kcwi.py b/pypeit/spectrographs/keck_kcwi.py index cad8feb98c..f917f15011 100644 --- a/pypeit/spectrographs/keck_kcwi.py +++ b/pypeit/spectrographs/keck_kcwi.py @@ -26,23 +26,22 @@ from pypeit.images import detector_container -class KeckKCWISpectrograph(spectrograph.Spectrograph): +class KeckKCWIKCRMSpectrograph(spectrograph.Spectrograph): """ - Child to handle Keck/KCWI specific code + Parent to handle Keck/KCWI+KCRM specific code .. todo:: - Need to apply spectral flexure and heliocentric correction to waveimg - + * Need to apply spectral flexure and heliocentric correction to waveimg -- done? + * Copy fast_histogram code into PypeIt? + * Re-write flexure code with datamodel + implement spectral flexure QA in find_objects.py + * When making the datacube, add an option to apply a spectral flexure correction from a different frame? + * Write some detailed docs about the corrections that can be used when making a datacube + * Consider introducing a new method (par['flexure']['spec_method']) for IFU flexure corrections (see find-objects.py) """ ndet = 1 - name = 'keck_kcwi' telescope = telescopes.KeckTelescopePar() - camera = 'KCWI' - url = 'https://www2.keck.hawaii.edu/inst/kcwi/' - header_name = 'KCWI' pypeline = 'IFU' supported = True - comment = 'Supported setups: BM, BH2; see :doc:`keck_kcwi`' def __init__(self): super().__init__() @@ -55,69 +54,53 @@ def __init__(self): # EarthLocation. KBW: Fine with me! self.location = EarthLocation.of_site('Keck Observatory') - def get_detector_par(self, det, hdu=None): + def init_meta(self): """ - Return metadata for the selected detector. + Define how metadata are derived from the spectrograph files. - .. warning:: + That is, this associates the PypeIt-specific metadata keywords + with the instrument-specific header cards using :attr:`meta`. + """ + self.meta = {} - Many of the necessary detector parameters are read from the file - header, meaning the ``hdu`` argument is effectively **required** for - KCWI. The optional use of ``hdu`` is only viable for automatically - generated documentation. + # Required (core) + self.meta['ra'] = dict(ext=0, card=None, compound=True) + self.meta['dec'] = dict(ext=0, card=None, compound=True) + self.meta['target'] = dict(ext=0, card='TARGNAME') + self.meta['decker'] = dict(ext=0, card='IFUNAM') + self.meta['binning'] = dict(card=None, compound=True) - Args: - det (:obj:`int`): - 1-indexed detector number. - hdu (`astropy.io.fits.HDUList`_, optional): - The open fits file with the raw image of interest. + self.meta['mjd'] = dict(ext=0, card='MJD') + self.meta['exptime'] = dict(card=None, compound=True) + self.meta['airmass'] = dict(ext=0, card='AIRMASS') - Returns: - :class:`~pypeit.images.detector_container.DetectorContainer`: - Object with the detector metadata. - """ - if hdu is None: - binning = '2,2' - specflip = None - numamps = None - gainarr = None - ronarr = None -# dsecarr = None -# msgs.error("A required keyword argument (hdu) was not supplied") - else: - # Some properties of the image - binning = self.compound_meta(self.get_headarr(hdu), "binning") - numamps = hdu[0].header['NVIDINP'] - specflip = True if hdu[0].header['AMPID1'] == 2 else False - gainmul, gainarr = hdu[0].header['GAINMUL'], np.zeros(numamps) - ronarr = np.zeros(numamps) # Set this to zero (determine the readout noise from the overscan regions) -# dsecarr = np.array(['']*numamps) + # Extras for config and frametyping + self.meta['hatch'] = dict(ext=0, card='HATPOS') + # self.meta['idname'] = dict(ext=0, card='CALXPOS') + self.meta['idname'] = dict(ext=0, card='IMTYPE') + self.meta['calpos'] = dict(ext=0, card='CALMNAM') + self.meta['slitwid'] = dict(card=None, compound=True) - for ii in range(numamps): - # Assign the gain for this amplifier - gainarr[ii] = hdu[0].header["GAIN{0:1d}".format(ii + 1)]# * gainmul + # Get atmospheric conditions (note, these are the conditions at the end of the exposure) + self.meta['obstime'] = dict(card=None, compound=True, required=False) + self.meta['pressure'] = dict(card=None, compound=True, required=False) + self.meta['temperature'] = dict(card=None, compound=True, required=False) + self.meta['humidity'] = dict(card=None, compound=True, required=False) + self.meta['instrument'] = dict(ext=0, card='INSTRUME') - detector = dict(det = det, - binning = binning, - dataext = 0, - specaxis = 0, - specflip = specflip, - spatflip = False, - platescale = 0.145728, # arcsec/pixel - darkcurr = None, # <-- TODO : Need to set this - mincounts = -1e10, - saturation = 65535., - nonlinear = 0.95, # For lack of a better number! - numamplifiers = numamps, - gain = gainarr, - ronoise = ronarr, -# TODO: These are never used because the image reader sets these up using the -# file headers data. -# datasec = dsecarr, #.copy(), # <-- This is provided in the header -# oscansec = dsecarr, #.copy(), # <-- This is provided in the header - ) - # Return - return detector_container.DetectorContainer(**detector) + # Lamps + lamp_names = ['LMP0', 'LMP1', 'LMP2', 'LMP3'] # FeAr, ThAr, Aux, Continuum + for kk, lamp_name in enumerate(lamp_names): + self.meta['lampstat{:02d}'.format(kk + 1)] = dict(ext=0, card=lamp_name + 'STAT') + for kk, lamp_name in enumerate(lamp_names): + if lamp_name == 'LMP3': + # There is no shutter on LMP3 + self.meta['lampshst{:02d}'.format(kk + 1)] = dict(ext=0, card=None, default=1) + continue + self.meta['lampshst{:02d}'.format(kk + 1)] = dict(ext=0, card=lamp_name + 'SHST') + # Add in the dome lamp + self.meta['lampstat{:02d}'.format(len(lamp_names) + 1)] = dict(ext=0, card='FLSPECTR') + self.meta['lampshst{:02d}'.format(len(lamp_names) + 1)] = dict(ext=0, card=None, default=1) def config_specific_par(self, scifile, inp_par=None): """ @@ -149,7 +132,16 @@ def config_specific_par(self, scifile, inp_par=None): par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_kcwi_BM.fits' elif self.get_meta_value(headarr, 'dispname') == 'BL': par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_kcwi_BL.fits' - + elif self.get_meta_value(headarr, 'dispname') == 'RM1': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_kcrm_RM1.fits' + elif self.get_meta_value(headarr, 'dispname') == 'RM2': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_kcrm_RM2.fits' + elif self.get_meta_value(headarr, 'dispname') == 'RH3': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_kcrm_RH3.fits' + else: + msgs.warn("Full template solution is unavailable") + msgs.info("Adopting holy-grail algorithm - Check the wavelength solution!") + par['calibrations']['wavelengths']['method'] = 'holy-grail' # FWHM # binning = parse.parse_binning(self.get_meta_value(headarr, 'binning')) # par['calibrations']['wavelengths']['fwhm'] = 6.0 / binning[1] @@ -157,54 +149,21 @@ def config_specific_par(self, scifile, inp_par=None): # Return return par - def init_meta(self): - """ - Define how metadata are derived from the spectrograph files. - - That is, this associates the PypeIt-specific metadata keywords - with the instrument-specific header cards using :attr:`meta`. + def configuration_keys(self): """ - self.meta = {} - # Required (core) - self.meta['ra'] = dict(ext=0, card=None, compound=True) - self.meta['dec'] = dict(ext=0, card=None, compound=True) - self.meta['target'] = dict(ext=0, card='TARGNAME') - self.meta['dispname'] = dict(ext=0, card='BGRATNAM') - self.meta['decker'] = dict(ext=0, card='IFUNAM') - self.meta['binning'] = dict(card=None, compound=True) - - self.meta['mjd'] = dict(ext=0, card='MJD') - self.meta['exptime'] = dict(card=None, compound=True) - self.meta['airmass'] = dict(ext=0, card='AIRMASS') - - # Extras for config and frametyping - self.meta['hatch'] = dict(ext=0, card='HATNUM') -# self.meta['idname'] = dict(ext=0, card='CALXPOS') - self.meta['idname'] = dict(ext=0, card='IMTYPE') - self.meta['calpos'] = dict(ext=0, card='CALMNAM') - self.meta['dispangle'] = dict(ext=0, card='BGRANGLE', rtol=0.01) - self.meta['slitwid'] = dict(card=None, compound=True) + Return the metadata keys that define a unique instrument + configuration. - # Get atmospheric conditions (note, these are the conditions at the end of the exposure) - self.meta['obstime'] = dict(card=None, compound=True, required=False) - self.meta['pressure'] = dict(card=None, compound=True, required=False) - self.meta['temperature'] = dict(card=None, compound=True, required=False) - self.meta['humidity'] = dict(card=None, compound=True, required=False) - self.meta['instrument'] = dict(ext=0, card='INSTRUME') + This list is used by :class:`~pypeit.metadata.PypeItMetaData` to + identify the unique configurations among the list of frames read + for a given reduction. - # Lamps - lamp_names = ['LMP0', 'LMP1', 'LMP2', 'LMP3'] # FeAr, ThAr, Aux, Continuum - for kk, lamp_name in enumerate(lamp_names): - self.meta['lampstat{:02d}'.format(kk + 1)] = dict(ext=0, card=lamp_name+'STAT') - for kk, lamp_name in enumerate(lamp_names): - if lamp_name == 'LMP3': - # There is no shutter on LMP3 - self.meta['lampshst{:02d}'.format(kk + 1)] = dict(ext=0, card=None, default=1) - continue - self.meta['lampshst{:02d}'.format(kk + 1)] = dict(ext=0, card=lamp_name+'SHST') - # Add in the dome lamp - self.meta['lampstat{:02d}'.format(len(lamp_names) + 1)] = dict(ext=0, card='FLSPECTR') - self.meta['lampshst{:02d}'.format(len(lamp_names) + 1)] = dict(ext=0, card=None, default=1) + Returns: + :obj:`list`: List of keywords of data pulled from file headers + and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` + object. + """ + return ['dispname', 'decker', 'binning', 'dispangle'] def compound_meta(self, headarr, meta_key): """ @@ -255,59 +214,27 @@ def compound_meta(self, headarr, meta_key): return headarr[0]['WXPRESS'] * 0.001 # Must be in astropy.units.bar except KeyError: msgs.warn("Pressure is not in header") - return 0.0 + msgs.info("The default pressure will be assumed: 0.611 bar") + return 0.611 elif meta_key == 'temperature': try: return headarr[0]['WXOUTTMP'] # Must be in astropy.units.deg_C except KeyError: msgs.warn("Temperature is not in header") - return 0.0 + msgs.info("The default temperature will be assumed: 1.5 deg C") + return 1.5 # van Kooten & Izett, arXiv:2208.11794 elif meta_key == 'humidity': try: return headarr[0]['WXOUTHUM'] / 100.0 except KeyError: msgs.warn("Humidity is not in header") - return 0.0 + msgs.info("The default relative humidity will be assumed: 20 %") + return 0.2 # van Kooten & Izett, arXiv:2208.11794 elif meta_key == 'obstime': return Time(headarr[0]['DATE-END']) else: msgs.error("Not ready for this compound meta") - def configuration_keys(self): - """ - Return the metadata keys that define a unique instrument - configuration. - - This list is used by :class:`~pypeit.metadata.PypeItMetaData` to - identify the unique configurations among the list of frames read - for a given reduction. - - Returns: - :obj:`list`: List of keywords of data pulled from file headers - and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` - object. - """ - return ['dispname', 'decker', 'binning', 'dispangle'] - - def raw_header_cards(self): - """ - Return additional raw header cards to be propagated in - downstream output files for configuration identification. - - The list of raw data FITS keywords should be those used to populate - the :meth:`~pypeit.spectrographs.spectrograph.Spectrograph.configuration_keys` - or are used in :meth:`~pypeit.spectrographs.spectrograph.Spectrograph.config_specific_par` - for a particular spectrograph, if different from the name of the - PypeIt metadata keyword. - - This list is used by :meth:`~pypeit.spectrographs.spectrograph.Spectrograph.subheader_for_spec` - to include additional FITS keywords in downstream output files. - - Returns: - :obj:`list`: List of keywords from the raw data files that should - be propagated in output files. - """ - return ['BGRATNAM', 'IFUNAM', 'BGRANGLE'] @classmethod def default_pypeit_par(cls): @@ -320,47 +247,9 @@ def default_pypeit_par(cls): """ par = super().default_pypeit_par() - # Subtract the detector pattern from certain frames. - # NOTE: The pattern subtraction is time-consuming, meaning we don't - # perform it (by default) for the high S/N pixel flat images but we do - # for everything else. - par['calibrations']['biasframe']['process']['use_pattern'] = True - par['calibrations']['darkframe']['process']['use_pattern'] = True - par['calibrations']['pixelflatframe']['process']['use_pattern'] = False - par['calibrations']['illumflatframe']['process']['use_pattern'] = True - par['calibrations']['standardframe']['process']['use_pattern'] = True - par['scienceframe']['process']['use_pattern'] = True - - # Correct the illumflat for pixel-to-pixel sensitivity variations - par['calibrations']['illumflatframe']['process']['use_pixelflat'] = True - - # Make sure the overscan is subtracted from the dark - par['calibrations']['darkframe']['process']['use_overscan'] = True - - # Set the slit edge parameters - par['calibrations']['slitedges']['fit_order'] = 4 - par['calibrations']['slitedges']['pad'] = 2 # Need to pad out the tilts for the astrometric transform when creating a datacube. - - # Alter the method used to combine pixel flats - par['calibrations']['pixelflatframe']['process']['combine'] = 'median' - par['calibrations']['flatfield']['spec_samp_coarse'] = 20.0 - #par['calibrations']['flatfield']['tweak_slits'] = False # Do not tweak the slit edges (we want to use the full slit) - par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.0 # Make sure the full slit is used (i.e. when the illumination fraction is > 0.5) - par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.0 # Make sure the full slit is used (i.e. no padding) - par['calibrations']['flatfield']['slit_trim'] = 3 # Trim the slit edges - # Relative illumination correction - par['calibrations']['flatfield']['slit_illum_relative'] = True # Calculate the relative slit illumination - par['calibrations']['flatfield']['slit_illum_ref_idx'] = 14 # The reference index - this should probably be the same for the science frame - par['calibrations']['flatfield']['slit_illum_smooth_npix'] = 5 # Sufficiently small value so less structure in relative weights - par['calibrations']['flatfield']['fit_2d_det_response'] = True # Include the 2D detector response in the pixelflat. - # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 0.01] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [0.01, None] -# par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames -# par['calibrations']['pixelflatframe']['exprng'] = [None, 30] -# par['calibrations']['traceframe']['exprng'] = [None, 30] -# par['scienceframe']['exprng'] = [30, None] # Set the number of alignments in the align frames par['calibrations']['alignment']['locations'] = [0.1, 0.3, 0.5, 0.7, 0.9] # TODO:: Check this - is this accurate enough? @@ -369,7 +258,7 @@ def default_pypeit_par(cls): par['scienceframe']['process']['sigclip'] = 4.0 par['scienceframe']['process']['objlim'] = 1.5 par['scienceframe']['process']['use_illumflat'] = True # illumflat is applied when building the relative scale image in reduce.py, so should be applied to scienceframe too. - par['scienceframe']['process']['use_specillum'] = False # apply relative spectral illumination + par['scienceframe']['process']['use_specillum'] = True # apply relative spectral illumination par['scienceframe']['process']['spat_flexure_correct'] = False # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames par['scienceframe']['process']['use_biasimage'] = False par['scienceframe']['process']['use_darkimage'] = False @@ -427,37 +316,35 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): exposures in ``fitstbl`` that are ``ftype`` type frames. """ good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) - # hatch=1,0=open,closed if ftype == 'science': return good_exp & (fitstbl['idname'] == 'OBJECT') & (fitstbl['calpos'] == 'Sky') \ - & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == '1') + & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'Open') if ftype == 'bias': return good_exp & (fitstbl['idname'] == 'BIAS') if ftype == 'pixelflat': # Use internal lamp return good_exp & (fitstbl['idname'] == 'FLATLAMP') & (fitstbl['calpos'] == 'Mirror') \ - & self.lamps(fitstbl, 'cont_noarc') & (fitstbl['hatch'] == '0') + & self.lamps(fitstbl, 'cont_noarc') if ftype in ['illumflat', 'trace']: # Use dome flats return good_exp & (fitstbl['idname'] == 'DOMEFLAT') & (fitstbl['calpos'] == 'Sky') \ - & self.lamps(fitstbl, 'dome_noarc') & (fitstbl['hatch'] == '1') + & self.lamps(fitstbl, 'dome_noarc') & (fitstbl['hatch'] == 'Open') if ftype == 'dark': # Dark frames return good_exp & (fitstbl['idname'] == 'DARK') & self.lamps(fitstbl, 'off') \ - & (fitstbl['hatch'] == '0') + & (fitstbl['hatch'] == 'Closed') if ftype == 'align': # Alignment frames # NOTE: Different from previous versions, this now only warns the user if everyth is_align = good_exp & (fitstbl['idname'] == 'CONTBARS') \ - & (fitstbl['calpos'] == 'Mirror') & self.lamps(fitstbl, 'cont') \ - & (fitstbl['hatch'] == '0') + & (fitstbl['calpos'] == 'Mirror') & self.lamps(fitstbl, 'cont') if np.any(is_align & np.logical_not(self.lamps(fitstbl, 'cont_noarc'))): msgs.warn('Alignment frames have both the continuum and arc lamps on (although ' 'arc-lamp shutter might be closed)!') return is_align if ftype in ['arc', 'tilt']: return good_exp & (fitstbl['idname'] == 'ARCLAMP') & (fitstbl['calpos'] == 'Mirror') \ - & self.lamps(fitstbl, 'arcs') & (fitstbl['hatch'] == '0') + & self.lamps(fitstbl, 'arcs') if ftype == 'pinhole': # Don't type pinhole frames return np.zeros(len(fitstbl), dtype=bool) @@ -556,9 +443,12 @@ def get_lamps_status(self, headarr): kk += 1 return "_".join(lampstat) - def get_rawimage(self, raw_file, det): + + def calc_pattern_freq(self, frame, rawdatasec_img, oscansec_img, hdu): """ - Read a raw KCWI data frame + Calculate the pattern frequency using the overscan region that covers + the overscan and data sections. Using a larger range allows the + frequency to be pinned down with high accuracy. NOTE: The amplifiers are arranged as follows: @@ -568,116 +458,24 @@ def get_rawimage(self, raw_file, det): | | 1 | 2 | | (0,0) --------- (nx, 0) + .. todo:: + + PATTERN FREQUENCY ALGORITHM HAS NOT BEEN TESTED WHEN BINNING != 1x1 + Parameters ---------- - raw_file : :obj:`str` - File to read - det : :obj:`int` - 1-indexed detector to read - - Returns - ------- - detector_par : :class:`pypeit.images.detector_container.DetectorContainer` - Detector metadata parameters. - raw_img : `numpy.ndarray`_ - Raw image for this detector. - hdu : `astropy.io.fits.HDUList`_ - Opened fits file - exptime : :obj:`float` - Exposure time read from the file header - rawdatasec_img : `numpy.ndarray`_ - Data (Science) section of the detector as provided by setting the - (1-indexed) number of the amplifier used to read each detector - pixel. Pixels unassociated with any amplifier are set to 0. - oscansec_img : `numpy.ndarray`_ - Overscan section of the detector as provided by setting the - (1-indexed) number of the amplifier used to read each detector - pixel. Pixels unassociated with any amplifier are set to 0. - """ - # Check for file; allow for extra .gz, etc. suffix - fil = glob.glob(raw_file + '*') - if len(fil) != 1: - msgs.error("Found {:d} files matching {:s}".format(len(fil), raw_file)) - - # Read - msgs.info("Reading KCWI file: {:s}".format(fil[0])) - hdu = io.fits_open(fil[0]) - detpar = self.get_detector_par(det if det is not None else 1, hdu=hdu) - head0 = hdu[0].header - raw_img = hdu[detpar['dataext']].data.astype(float) - - # Some properties of the image - numamps = head0['NVIDINP'] - # Exposure time (used by ProcessRawImage) - headarr = self.get_headarr(hdu) - exptime = self.get_meta_value(headarr, 'exptime') - - # get the x and y binning factors... - #binning = self.get_meta_value(headarr, 'binning') - - # Always assume normal FITS header formatting - one_indexed = True - include_last = True - for section in ['DSEC', 'BSEC']: - - # Initialize the image (0 means no amplifier) - pix_img = np.zeros(raw_img.shape, dtype=int) - for i in range(numamps): - # Get the data section - sec = head0[section+"{0:1d}".format(i+1)] - - # Convert the data section from a string to a slice - # TODO :: RJC - I think something has changed here... and the BPM is flipped (or not flipped) for different amp modes. - # TODO :: RJC - Note, KCWI records binned sections, so there's no need to pass binning in as an argument - datasec = parse.sec2slice(sec, one_indexed=one_indexed, - include_end=include_last, require_dim=2)#, binning=binning) - # Flip the datasec - datasec = datasec[::-1] - - # Assign the amplifier - pix_img[datasec] = i+1 - - # Finish - if section == 'DSEC': - rawdatasec_img = pix_img.copy() - elif section == 'BSEC': - oscansec_img = pix_img.copy() - - # Return - return detpar, raw_img, hdu, exptime, rawdatasec_img, oscansec_img - - def calc_pattern_freq(self, frame, rawdatasec_img, oscansec_img, hdu): - """ - Calculate the pattern frequency using the overscan region that covers - the overscan and data sections. Using a larger range allows the - frequency to be pinned down with high accuracy. - - NOTE: The amplifiers are arranged as follows: - - | (0,ny) --------- (nx,ny) - | | 3 | 4 | - | --------- - | | 1 | 2 | - | (0,0) --------- (nx, 0) - - .. todo:: - - PATTERN FREQUENCY ALGORITHM HAS NOT BEEN TESTED WHEN BINNING != 1x1 - - Parameters - ---------- - frame : `numpy.ndarray`_ - Raw data frame to be used to estimate the pattern frequency. - rawdatasec_img : `numpy.ndarray`_ - Array the same shape as ``frame``, used as a mask to identify the - data pixels (0 is no data, non-zero values indicate the amplifier - number). - oscansec_img : `numpy.ndarray`_ - Array the same shape as ``frame``, used as a mask to identify the - overscan pixels (0 is no data, non-zero values indicate the - amplifier number). - hdu : `astropy.io.fits.HDUList`_ - Opened fits file. + frame : `numpy.ndarray`_ + Raw data frame to be used to estimate the pattern frequency. + rawdatasec_img : `numpy.ndarray`_ + Array the same shape as ``frame``, used as a mask to identify the + data pixels (0 is no data, non-zero values indicate the amplifier + number). + oscansec_img : `numpy.ndarray`_ + Array the same shape as ``frame``, used as a mask to identify the + overscan pixels (0 is no data, non-zero values indicate the + amplifier number). + hdu : `astropy.io.fits.HDUList`_ + Opened fits file. Returns ------- @@ -721,101 +519,6 @@ def calc_pattern_freq(self, frame, rawdatasec_img, oscansec_img, hdu): # Return the list of pattern frequencies return patt_freqs - def bpm(self, filename, det, shape=None, msbias=None): - """ - Generate a default bad-pixel mask. - - Even though they are both optional, either the precise shape for - the image (``shape``) or an example file that can be read to get - the shape (``filename`` using :func:`get_image_shape`) *must* be - provided. - - Args: - filename (:obj:`str` or None): - An example file to use to get the image shape. - det (:obj:`int`): - 1-indexed detector number to use when getting the image - shape from the example file. - shape (tuple, optional): - Processed image shape - Required if filename is None - Ignored if filename is not None - msbias (`numpy.ndarray`_, optional): - Processed bias frame used to identify bad pixels. **This is - ignored for KCWI.** - - Returns: - `numpy.ndarray`_: An integer array with a masked value set - to 1 and an unmasked value set to 0. All values are set to - 0. - """ - # Call the base-class method to generate the empty bpm; msbias is always set to None. - bpm_img = super().bpm(filename, det, shape=shape, msbias=None) - - # Extract some header info - #msgs.info("Reading AMPMODE and BINNING from KCWI file: {:s}".format(filename)) - head0 = fits.getheader(filename, ext=0) - ampmode = head0['AMPMODE'] - binning = head0['BINNING'] - - # Construct a list of the bad columns - # Note: These were taken from v1.1.0 (REL) Date: 2018/06/11 of KDERP (updated to be more conservative) - # KDERP store values and in the code (stage1) subtract 1 from the badcol data files. - # Instead of this, I have already pre-subtracted the values in the following arrays. - bc = None - if ampmode == 'ALL': - # TODO: There are several bad columns in this mode, but this is typically only used for arcs. - # It's the same set of bad columns seen in the TBO and TUP amplifier modes. - if binning == '1,1': - bc = [[3676, 3676, 2056, 2244]] - elif binning == '2,2': - bc = [[1838, 1838, 1028, 1121]] - elif ampmode == 'TBO': - if binning == '1,1': - bc = [[2622, 2622, 619, 687], - [2739, 2739, 1748, 1860], - [3295, 3300, 2556, 2560], - [3675, 3676, 2243, 4111]] - elif binning == '2,2': - bc = [[1311, 1311, 310, 354], - [1369, 1369, 876, 947], - [1646, 1650, 1278, 1280], - [1838, 1838, 1122, 2055]] - if ampmode == 'TUP': - if binning == '1,1': -# bc = [[2622, 2622, 3492, 3528], - bc = [[2622, 2622, 3492, 4111], # Extending this BPM, as sometimes the bad column is larger than this. - [3295, 3300, 1550, 1555], - [3676, 3676, 1866, 4111]] - elif binning == '2,2': -# bc = [[1311, 1311, 1745, 1788], - bc = [[1311, 1311, 1745, 2055], # Extending this BPM, as sometimes the bad column is larger than this. - [1646, 1650, 775, 777], - [1838, 1838, 933, 2055]] - if bc is None: - msgs.warn("Bad pixel mask is not available for ampmode={0:s} binning={1:s}".format(ampmode, binning)) - bc = [] - - # Apply these bad columns to the mask - for bb in range(len(bc)): - bpm_img[bc[bb][2]:bc[bb][3]+1, bc[bb][0]:bc[bb][1]+1] = 1 - - return np.flipud(bpm_img) - - @staticmethod - def is_nasmask(hdr): - """ - Determine if a frame used nod-and-shuffle. - - Args: - hdr (`astropy.io.fits.Header`_): - The header of the raw frame. - - Returns: - :obj:`bool`: True if NAS used. - """ - return 'Mask' in hdr['BNASNAM'] - def get_wcs(self, hdr, slits, platescale, wave0, dwv, spatial_scale=None): """ Construct/Read a World-Coordinate System for a frame. @@ -841,9 +544,9 @@ def get_wcs(self, hdr, slits, platescale, wave0, dwv, spatial_scale=None): the platescale will be used. Returns: - `astropy.wcs.wcs.WCS`_: The world-coordinate system. + `astropy.wcs.WCS`_: The world-coordinate system. """ - msgs.info("Calculating the WCS") + msgs.info(f"Generating {self.camera} WCS") # Get the x and y binning factors, and the typical slit length binspec, binspat = parse.parse_binning(self.get_meta_value([hdr], 'binning')) @@ -903,40 +606,38 @@ def get_wcs(self, hdr, slits, platescale, wave0, dwv, spatial_scale=None): porg = hdr['PONAME'] ifunum = hdr['IFUNUM'] if 'IFU' in porg: - if ifunum == 1: # Large slicer - off1 = 1.0 - off2 = 4.0 - elif ifunum == 2: # Medium slicer - off1 = 1.0 - off2 = 5.0 - elif ifunum == 3: # Small slicer - off1 = 0.05 - off2 = 5.6 - else: - msgs.warn("Unknown IFU number: {0:d}".format(ifunum)) - off1 = 0. - off2 = 0. + # if ifunum == 1: # Large slicer + # off1 = 1.0 + # off2 = 4.0 + # elif ifunum == 2: # Medium slicer + # off1 = 1.0 + # off2 = 5.0 + # elif ifunum == 3: # Small slicer + # off1 = 0.05 + # off2 = 5.6 + # else: + # msgs.warn("Unknown IFU number: {0:d}".format(ifunum)) + off1 = 0. + off2 = 0. off1 /= binspec off2 /= binspat crpix1 += off1 crpix2 += off2 # Create a new WCS object. - msgs.info("Generating KCWI WCS") w = wcs.WCS(naxis=3) w.wcs.equinox = hdr['EQUINOX'] - w.wcs.name = 'KCWI' + w.wcs.name = self.camera w.wcs.radesys = 'FK5' + w.wcs.lonpole = 180.0 # Native longitude of the Celestial pole + w.wcs.latpole = 0.0 # Native latitude of the Celestial pole # Insert the coordinate frame - w.wcs.cname = ['KCWI RA', 'KCWI DEC', 'KCWI Wavelength'] + w.wcs.cname = ['RA', 'DEC', 'Wavelength'] w.wcs.cunit = [units.degree, units.degree, units.Angstrom] w.wcs.ctype = ["RA---TAN", "DEC--TAN", "WAVE"] # Note, WAVE is vacuum wavelength w.wcs.crval = [ra, dec, wave0] # RA, DEC, and wavelength zeropoints w.wcs.crpix = [crpix1, crpix2, crpix3] # RA, DEC, and wavelength reference pixels w.wcs.cd = np.array([[cd11, cd12, 0.0], [cd21, cd22, 0.0], [0.0, 0.0, dwv]]) - w.wcs.lonpole = 180.0 # Native longitude of the Celestial pole - w.wcs.latpole = 0.0 # Native latitude of the Celestial pole - return w def get_datacube_bins(self, slitlength, minmax, num_wave): @@ -961,49 +662,675 @@ def get_datacube_bins(self, slitlength, minmax, num_wave): when constructing a histogram of the spec2d files. The elements are :math:`(x,y,\lambda)`. """ - xbins = np.arange(1 + 24) - 12.0 - 0.5 + xbins = np.arange(1 + 24) - 24/2 - 0.5 ybins = np.linspace(np.min(minmax[:, 0]), np.max(minmax[:, 1]), 1+slitlength) - 0.5 spec_bins = np.arange(1+num_wave) - 0.5 return xbins, ybins, spec_bins - def fit_2d_det_response(self, det_resp, gpmask): - r""" - Perform a 2D model fit to the KCWI detector response. - A few different setups were inspected (BH2 & BM with different - grating angles), and a very similar response pattern was found for all - setups, indicating that this structure is something to do with - the detector. The starting parameters and functional form are - assumed to be sufficient for all setups. + def bpm(self, filename, det, shape=None, msbias=None): + """ + Generate a default bad-pixel mask for KCWI and KCRM. + + Even though they are both optional, either the precise shape for + the image (``shape``) or an example file that can be read to get + the shape (``filename`` using :func:`get_image_shape`) *must* be + provided. Args: - det_resp (`numpy.ndarray`_): - An image of the flatfield structure. - gpmask (`numpy.ndarray`_): - Good pixel mask (True=good), the same shape as ff_struct. + filename (:obj:`str` or None): + An example file to use to get the image shape. + det (:obj:`int`): + 1-indexed detector number to use when getting the image + shape from the example file. + shape (tuple, optional): + Processed image shape + Required if filename is None + Ignored if filename is not None + msbias (`numpy.ndarray`_, optional): + Processed bias frame used to identify bad pixels. **This is + ignored for KCWI.** Returns: - `numpy.ndarray`_: A model fit to the flatfield structure. + `numpy.ndarray`_: An integer array with a masked value set + to 1 and an unmasked value set to 0. All values are set to + 0. """ - msgs.info("Performing a 2D fit to the detector response") + # Call the base-class method to generate the empty bpm; msbias is always set to None. + bpm_img = super().bpm(filename, det, shape=shape, msbias=None) - # Define a 2D sine function, which is a good description of KCWI data - def sinfunc2d(x, amp, scl, phase, wavelength, angle): - """ - 2D Sine function - """ - xx, yy = x - angle *= np.pi / 180.0 - return 1 + (amp + xx * scl) * np.sin( - 2 * np.pi * (xx * np.cos(angle) + yy * np.sin(angle)) / wavelength + phase) + # Extract some header info + head0 = fits.getheader(filename, ext=0) + ampmode = head0['AMPMODE'] + binning = head0['BINNING'] - x = np.arange(det_resp.shape[0]) - y = np.arange(det_resp.shape[1]) - xx, yy = np.meshgrid(x, y, indexing='ij') - # Prepare the starting parameters - amp = 0.02 # Roughly a 2% effect - scale = 0.0 # Assume the amplitude is constant over the detector - wavelength = np.sqrt(det_resp.shape[0] ** 2 + det_resp.shape[1] ** 2) / 31.5 # 31-32 cycles of the pattern from corner to corner - phase, angle = 0.0, -45.34 # No phase, and a decent guess at the angle - p0 = [amp, scale, phase, wavelength, angle] - popt, pcov = curve_fit(sinfunc2d, (xx[gpmask], yy[gpmask]), det_resp[gpmask], p0=p0) - return sinfunc2d((xx, yy), *popt) + # Construct a list of the bad columns + # KCWI --> AMPMODE = 'ALL', 'TBO', 'TUP' + # KCRM --> AMPMODE = 'L2U2', 'L2U2L1U1' + bc = None + if ampmode == 'ALL': + # TODO: There are several bad columns in this mode, but this is typically only used for arcs. + # It's the same set of bad columns seen in the TBO and TUP amplifier modes. + if binning == '1,1': + bc = [[3676, 3676, 2056, 2244]] + elif binning == '2,2': + bc = [[1838, 1838, 1028, 1121]] + elif ampmode == 'TBO': + if binning == '1,1': + bc = [[2622, 2622, 619, 687], + [2739, 2739, 1748, 1860], + [3295, 3300, 2556, 2560], + [3675, 3676, 2243, 4111]] + elif binning == '2,2': + bc = [[1311, 1311, 310, 354], + [1369, 1369, 876, 947], + [1646, 1650, 1278, 1280], + [1838, 1838, 1122, 2055]] + elif ampmode == 'TUP': + if binning == '1,1': +# bc = [[2622, 2622, 3492, 3528], + bc = [[2622, 2622, 3492, 4111], # Extending this BPM, as sometimes the bad column is larger than this. + [3295, 3300, 1550, 1555], + [3676, 3676, 1866, 4111]] + elif binning == '2,2': +# bc = [[1311, 1311, 1745, 1788], + bc = [[1311, 1311, 1745, 2055], # Extending this BPM, as sometimes the bad column is larger than this. + [1646, 1650, 775, 777], + [1838, 1838, 933, 2055]] + elif ampmode == 'L2U2': + if binning == '1,1': + bc = [[3458, 3462, 0, 613]] + elif binning == '2,2': + bc = [[1730, 1730, 0, 307]] + elif ampmode == "L2U2L1U1": + pass + # Currently unchecked... + # if binning == '1,1': + # bc = [[3460, 3460, 2064, 3520]] + # elif binning == '2,2': + # bc = [[1838, 1838, 1028, 1121]] + + # Check if the bad columns haven't been set + if bc is None: + msgs.warn("KCRM bad pixel mask is not available for ampmode={0:s} binning={1:s}".format(ampmode, binning)) + bc = [] + + # Apply these bad columns to the mask + for bb in range(len(bc)): + bpm_img[bc[bb][2]:bc[bb][3]+1, bc[bb][0]:bc[bb][1]+1] = 1 + + return np.flipud(bpm_img) + + +class KeckKCWISpectrograph(KeckKCWIKCRMSpectrograph): + """ + Child to handle Keck/KCWI specific code + """ + name = 'keck_kcwi' + camera = 'KCWI' + url = 'https://www2.keck.hawaii.edu/inst/kcwi/' + header_name = 'KCWI' + comment = 'Supported setups: BL, BM, BH2; see :doc:`keck_kcwi`' + + def get_detector_par(self, det, hdu=None): + """ + Return metadata for the selected detector. + + .. warning:: + + Many of the necessary detector parameters are read from the file + header, meaning the ``hdu`` argument is effectively **required** for + KCWI. The optional use of ``hdu`` is only viable for automatically + generated documentation. + + Args: + det (:obj:`int`): + 1-indexed detector number. + hdu (`astropy.io.fits.HDUList`_, optional): + The open fits file with the raw image of interest. + + Returns: + :class:`~pypeit.images.detector_container.DetectorContainer`: + Object with the detector metadata. + """ + if hdu is None: + binning = '2,2' + specflip = None + numamps = None + gainarr = None + ronarr = None +# dsecarr = None +# msgs.error("A required keyword argument (hdu) was not supplied") + else: + # Some properties of the image + binning = self.compound_meta(self.get_headarr(hdu), "binning") + numamps = hdu[0].header['NVIDINP'] + specflip = True if hdu[0].header['AMPID1'] == 2 else False + gainmul, gainarr = hdu[0].header['GAINMUL'], np.zeros(numamps) + ronarr = np.zeros(numamps) # Set this to zero (determine the readout noise from the overscan regions) +# dsecarr = np.array(['']*numamps) + + for ii in range(numamps): + # Assign the gain for this amplifier + gainarr[ii] = hdu[0].header["GAIN{0:1d}".format(ii + 1)]# * gainmul + + detector = dict(det = det, + binning = binning, + dataext = 0, + specaxis = 0, + specflip = specflip, + spatflip = False, + platescale = 0.145728, # arcsec/pixel + darkcurr = 1.0, # e-/hour/unbinned pixel + mincounts = -1e10, + saturation = 65535., + nonlinear = 0.95, # For lack of a better number! + numamplifiers = numamps, + gain = gainarr, + ronoise = ronarr, + # These are never used because the image reader sets these up using the file headers data. + # datasec = dsecarr, #.copy(), # <-- This is provided in the header + # oscansec = dsecarr, #.copy(), # <-- This is provided in the header + ) + # Return + return detector_container.DetectorContainer(**detector) + + def init_meta(self): + """ + Define how metadata are derived from the spectrograph files. + + That is, this associates the PypeIt-specific metadata keywords + with the instrument-specific header cards using :attr:`meta`. + """ + super().init_meta() + self.meta['dispname'] = dict(ext=0, card='BGRATNAM') + self.meta['dispangle'] = dict(ext=0, card='BGRANGLE', rtol=0.01) + + def raw_header_cards(self): + """ + Return additional raw header cards to be propagated in + downstream output files for configuration identification. + + The list of raw data FITS keywords should be those used to populate + the :meth:`~pypeit.spectrographs.spectrograph.Spectrograph.configuration_keys` + or are used in :meth:`~pypeit.spectrographs.spectrograph.Spectrograph.config_specific_par` + for a particular spectrograph, if different from the name of the + PypeIt metadata keyword. + + This list is used by :meth:`~pypeit.spectrographs.spectrograph.Spectrograph.subheader_for_spec` + to include additional FITS keywords in downstream output files. + + Returns: + :obj:`list`: List of keywords from the raw data files that should + be propagated in output files. + """ + return ['BGRATNAM', 'IFUNAM', 'BGRANGLE'] + + @classmethod + def default_pypeit_par(cls): + """ + Return the default parameters to use for this instrument. + + Returns: + :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by + all of PypeIt methods. + """ + par = super().default_pypeit_par() + + # Subtract the detector pattern from certain frames. + # NOTE: The pattern subtraction is time-consuming, meaning we don't + # perform it (by default) for the high S/N pixel flat images but we do + # for everything else. + par['calibrations']['biasframe']['process']['use_pattern'] = True + par['calibrations']['darkframe']['process']['use_pattern'] = True + par['calibrations']['pixelflatframe']['process']['use_pattern'] = False + par['calibrations']['illumflatframe']['process']['use_pattern'] = True + par['calibrations']['standardframe']['process']['use_pattern'] = True + par['scienceframe']['process']['use_pattern'] = True + + # Correct the illumflat for pixel-to-pixel sensitivity variations + par['calibrations']['illumflatframe']['process']['use_pixelflat'] = True + + # Make sure the overscan is subtracted from the dark + par['calibrations']['darkframe']['process']['use_overscan'] = True + + # Set the slit edge parameters + par['calibrations']['slitedges']['fit_order'] = 4 + par['calibrations']['slitedges']['pad'] = 2 # Need to pad out the tilts for the astrometric transform when creating a datacube. + par['calibrations']['slitedges']['edge_thresh'] = 5 # 5 works well with a range of setups tested by RJC (mostly 1x1 binning) + + # KCWI has non-uniform spectral resolution across the field-of-view + par['calibrations']['wavelengths']['fwhm_spec_order'] = 1 + par['calibrations']['wavelengths']['fwhm_spat_order'] = 2 + + # Alter the method used to combine pixel flats + par['calibrations']['pixelflatframe']['process']['combine'] = 'median' + par['calibrations']['flatfield']['spec_samp_coarse'] = 20.0 + #par['calibrations']['flatfield']['tweak_slits'] = False # Do not tweak the slit edges (we want to use the full slit) + par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.0 # Make sure the full slit is used (i.e. when the illumination fraction is > 0.5) + par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.0 # Make sure the full slit is used (i.e. no padding) + par['calibrations']['flatfield']['slit_trim'] = 3 # Trim the slit edges + # Relative illumination correction + par['calibrations']['flatfield']['slit_illum_relative'] = True # Calculate the relative slit illumination + par['calibrations']['flatfield']['slit_illum_ref_idx'] = 14 # The reference index - this should probably be the same for the science frame + par['calibrations']['flatfield']['slit_illum_smooth_npix'] = 5 # Sufficiently small value so less structure in relative weights + par['calibrations']['flatfield']['fit_2d_det_response'] = True # Include the 2D detector response in the pixelflat. + + return par + + @staticmethod + def is_nasmask(hdr): + """ + Determine if a frame used nod-and-shuffle. + + Args: + hdr (`astropy.io.fits.Header`_): + The header of the raw frame. + + Returns: + :obj:`bool`: True if NAS used. + """ + return 'Mask' in hdr['BNASNAM'] + + def get_rawimage(self, raw_file, det): + """ + Read a raw KCWI data frame + + NOTE: The amplifiers are arranged as follows: + + | (0,ny) --------- (nx,ny) + | | 3 | 4 | + | --------- + | | 1 | 2 | + | (0,0) --------- (nx, 0) + + Parameters + ---------- + raw_file : :obj:`str` + File to read + det : :obj:`int` + 1-indexed detector to read + + Returns + ------- + detector_par : :class:`pypeit.images.detector_container.DetectorContainer` + Detector metadata parameters. + raw_img : `numpy.ndarray`_ + Raw image for this detector. + hdu : `astropy.io.fits.HDUList`_ + Opened fits file + exptime : :obj:`float` + Exposure time read from the file header + rawdatasec_img : `numpy.ndarray`_ + Data (Science) section of the detector as provided by setting the + (1-indexed) number of the amplifier used to read each detector + pixel. Pixels unassociated with any amplifier are set to 0. + oscansec_img : `numpy.ndarray`_ + Overscan section of the detector as provided by setting the + (1-indexed) number of the amplifier used to read each detector + pixel. Pixels unassociated with any amplifier are set to 0. + """ + # Check for file; allow for extra .gz, etc. suffix + fil = glob.glob(raw_file + '*') + if len(fil) != 1: + msgs.error("Found {:d} files matching {:s}".format(len(fil), raw_file)) + + # Read + msgs.info("Reading KCWI file: {:s}".format(fil[0])) + hdu = io.fits_open(fil[0]) + detpar = self.get_detector_par(det if det is not None else 1, hdu=hdu) + head0 = hdu[0].header + raw_img = hdu[detpar['dataext']].data.astype(float) + + # Some properties of the image + numamps = head0['NVIDINP'] + # Exposure time (used by ProcessRawImage) + headarr = self.get_headarr(hdu) + exptime = self.get_meta_value(headarr, 'exptime') + + # Always assume normal FITS header formatting + one_indexed = True + include_last = True + for section in ['DSEC', 'BSEC']: + + # Initialize the image (0 means no amplifier) + pix_img = np.zeros(raw_img.shape, dtype=int) + for aa, ampid in enumerate(1+np.arange(numamps)): + # Get the data section + sec = head0[section+"{0:1d}".format(ampid)] + + # Convert the data section from a string to a slice + # TODO :: RJC - I think something has changed here... and the BPM is flipped (or not flipped) for different amp modes. + # RJC - Note, KCWI records binned sections, so there's no need to pass binning in as an argument + datasec = parse.sec2slice(sec, one_indexed=one_indexed, include_end=include_last, require_dim=2) + # Flip the datasec + datasec = datasec[::-1] + + # Assign the amplifier + pix_img[datasec] = aa+1 + + # Finish + if section == 'DSEC': + rawdatasec_img = pix_img.copy() + elif section == 'BSEC': + oscansec_img = pix_img.copy() + + # Return + return detpar, raw_img, hdu, exptime, rawdatasec_img, oscansec_img + + def fit_2d_det_response(self, det_resp, gpmask): + r""" + Perform a 2D model fit to the KCWI detector response. + A few different setups were inspected (BH2 & BM with different + grating angles), and a very similar response pattern was found for all + setups, indicating that this structure is something to do with + the detector. The starting parameters and functional form are + assumed to be sufficient for all setups. + + Args: + det_resp (`numpy.ndarray`_): + An image of the flatfield structure. + gpmask (`numpy.ndarray`_): + Good pixel mask (True=good), the same shape as ff_struct. + + Returns: + `numpy.ndarray`_: A model fit to the flatfield structure. + """ + msgs.info("Performing a 2D fit to the detector response") + + # Define a 2D sine function, which is a good description of KCWI data + def sinfunc2d(x, amp, scl, phase, wavelength, angle): + """ + 2D Sine function + """ + xx, yy = x + angle *= np.pi / 180.0 + return 1 + (amp + xx * scl) * np.sin( + 2 * np.pi * (xx * np.cos(angle) + yy * np.sin(angle)) / wavelength + phase) + + x = np.arange(det_resp.shape[0]) + y = np.arange(det_resp.shape[1]) + xx, yy = np.meshgrid(x, y, indexing='ij') + # Prepare the starting parameters + amp = 0.02 # Roughly a 2% effect + scale = 0.0 # Assume the amplitude is constant over the detector + wavelength = np.sqrt(det_resp.shape[0] ** 2 + det_resp.shape[1] ** 2) / 31.5 # 31-32 cycles of the pattern from corner to corner + phase, angle = 0.0, -45.34 # No phase, and a decent guess at the angle + p0 = [amp, scale, phase, wavelength, angle] + popt, pcov = curve_fit(sinfunc2d, (xx[gpmask], yy[gpmask]), det_resp[gpmask], p0=p0) + return sinfunc2d((xx, yy), *popt) + + +class KeckKCRMSpectrograph(KeckKCWIKCRMSpectrograph): + """ + Child to handle Keck/KCRM specific code + + """ + name = 'keck_kcrm' + camera = 'KCRM' + url = 'https://www2.keck.hawaii.edu/inst/kcwi/' # TODO :: Need to update this website + header_name = 'KCRM' + comment = 'Supported setups: RM1, RM2, RH3; see :doc:`keck_kcwi`' + + def get_detector_par(self, det, hdu=None): + """ + Return metadata for the selected detector. + + .. warning:: + + Many of the necessary detector parameters are read from the file + header, meaning the ``hdu`` argument is effectively **required** for + KCRM. The optional use of ``hdu`` is only viable for automatically + generated documentation. + + Args: + det (:obj:`int`): + 1-indexed detector number. + hdu (`astropy.io.fits.HDUList`_, optional): + The open fits file with the raw image of interest. + + Returns: + :class:`~pypeit.images.detector_container.DetectorContainer`: + Object with the detector metadata. + """ + if hdu is None: + binning = '2,2' + specflip = None + numamps = None + gainarr = None + ronarr = None +# dsecarr = None +# msgs.error("A required keyword argument (hdu) was not supplied") + else: + # Some properties of the image + binning = self.compound_meta(self.get_headarr(hdu), "binning") + nampsxy = hdu[0].header['NAMPSXY'].split() + numamps = int(nampsxy[0]) * int(nampsxy[1]) + amps = self.get_amplifiers(numamps) + + specflip = False + gainarr = np.zeros(numamps) + ronarr = np.zeros(numamps) # Set this to zero (determine the readout noise from the overscan regions) + for aa, amp in enumerate(amps): + # Assign the gain for this amplifier + gainarr[aa] = hdu[0].header["GAIN{0:1d}".format(amp)] + + detector = dict(det = det, + binning = binning, + dataext = 0, + specaxis = 0, + specflip = specflip, + spatflip = False, + platescale = 0.145728, # arcsec/pixel TODO :: Need to double check this + darkcurr = None, # e-/pixel/hour TODO :: Need to check this. + mincounts = -1e10, + saturation = 65535., + nonlinear = 0.95, # For lack of a better number! + numamplifiers = numamps, + gain = gainarr, + ronoise = ronarr, + # These are never used because the image reader sets these up using the file headers data. + # datasec = dsecarr, #.copy(), # <-- This is provided in the header + # oscansec = dsecarr, #.copy(), # <-- This is provided in the header + ) + # Return + return detector_container.DetectorContainer(**detector) + + def get_amplifiers(self, numamps): + """ + Obtain a list of the amplifier ID numbers + + Args: + numamps (:obj:`int`): + Number of amplifiers used for readout + + Returns: + :obj:`list`: + A list (of length numamps) containing the ID number of the amplifiers used for readout + """ + if numamps == 2: + return [1, 3] + elif numamps == 4: + return [0, 1, 2, 3] + else: + msgs.error("PypeIt only supports 2 or 4 amplifier readout of KCRM data") + + def init_meta(self): + """ + Define how metadata are derived from the spectrograph files. + + That is, this associates the PypeIt-specific metadata keywords + with the instrument-specific header cards using :attr:`meta`. + """ + super().init_meta() + self.meta['dispname'] = dict(ext=0, card='RGRATNAM') + self.meta['dispangle'] = dict(ext=0, card='RGRANGLE', rtol=0.01) + + def raw_header_cards(self): + """ + Return additional raw header cards to be propagated in + downstream output files for configuration identification. + + The list of raw data FITS keywords should be those used to populate + the :meth:`~pypeit.spectrographs.spectrograph.Spectrograph.configuration_keys` + or are used in :meth:`~pypeit.spectrographs.spectrograph.Spectrograph.config_specific_par` + for a particular spectrograph, if different from the name of the + PypeIt metadata keyword. + + This list is used by :meth:`~pypeit.spectrographs.spectrograph.Spectrograph.subheader_for_spec` + to include additional FITS keywords in downstream output files. + + Returns: + :obj:`list`: List of keywords from the raw data files that should + be propagated in output files. + """ + return ['RGRATNAM', 'IFUNAM', 'RGRANGLE'] + + @classmethod + def default_pypeit_par(cls): + """ + Return the default parameters to use for this instrument. + + Returns: + :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by + all of PypeIt methods. + """ + par = super().default_pypeit_par() + + # Subtract the detector pattern from certain frames. + # NOTE: The pattern subtraction is time-consuming, meaning we don't + # perform it (by default) for the high S/N pixel flat images but we do + # for everything else. + par['calibrations']['biasframe']['process']['use_pattern'] = False + par['calibrations']['darkframe']['process']['use_pattern'] = False + par['calibrations']['pixelflatframe']['process']['use_pattern'] = False + par['calibrations']['illumflatframe']['process']['use_pattern'] = False + par['calibrations']['standardframe']['process']['use_pattern'] = False + par['scienceframe']['process']['use_pattern'] = False + + # Correct the illumflat for pixel-to-pixel sensitivity variations + par['calibrations']['illumflatframe']['process']['use_pixelflat'] = True + + # Make sure the overscan is subtracted from the dark + par['calibrations']['darkframe']['process']['use_overscan'] = True + + # Set the slit edge parameters + par['calibrations']['slitedges']['fit_order'] = 4 + par['calibrations']['slitedges']['pad'] = 2 # Need to pad out the tilts for the astrometric transform when creating a datacube. + par['calibrations']['slitedges']['edge_thresh'] = 5 # 5 works well with a range of setups tested by RJC (mostly 1x1 binning) + + # KCWI has non-uniform spectral resolution across the field-of-view + par['calibrations']['wavelengths']['fwhm_spec_order'] = 1 + par['calibrations']['wavelengths']['fwhm_spat_order'] = 2 + + # Alter the method used to combine pixel flats + par['calibrations']['pixelflatframe']['process']['combine'] = 'median' + par['calibrations']['flatfield']['spec_samp_coarse'] = 20.0 + #par['calibrations']['flatfield']['tweak_slits'] = False # Do not tweak the slit edges (we want to use the full slit) + par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.0 # Make sure the full slit is used (i.e. when the illumination fraction is > 0.5) + par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.0 # Make sure the full slit is used (i.e. no padding) + par['calibrations']['flatfield']['slit_trim'] = 3 # Trim the slit edges + # Relative illumination correction + par['calibrations']['flatfield']['slit_illum_relative'] = True # Calculate the relative slit illumination + par['calibrations']['flatfield']['slit_illum_ref_idx'] = 14 # The reference index - this should probably be the same for the science frame + par['calibrations']['flatfield']['slit_illum_smooth_npix'] = 5 # Sufficiently small value so less structure in relative weights + par['calibrations']['flatfield']['fit_2d_det_response'] = True # Include the 2D detector response in the pixelflat. + + return par + + @staticmethod + def is_nasmask(hdr): + """ + Determine if a frame used nod-and-shuffle. + + Args: + hdr (`astropy.io.fits.Header`_): + The header of the raw frame. + + Returns: + :obj:`bool`: True if NAS used. + """ + return 'Mask' in hdr['RNASNAM'] + + + def get_rawimage(self, raw_file, det): + """ + Read a raw KCRM data frame + + Parameters + ---------- + raw_file : :obj:`str` + File to read + det : :obj:`int` + 1-indexed detector to read + + Returns + ------- + detector_par : :class:`pypeit.images.detector_container.DetectorContainer` + Detector metadata parameters. + raw_img : `numpy.ndarray`_ + Raw image for this detector. + hdu : `astropy.io.fits.HDUList`_ + Opened fits file + exptime : :obj:`float` + Exposure time read from the file header + rawdatasec_img : `numpy.ndarray`_ + Data (Science) section of the detector as provided by setting the + (1-indexed) number of the amplifier used to read each detector + pixel. Pixels unassociated with any amplifier are set to 0. + oscansec_img : `numpy.ndarray`_ + Overscan section of the detector as provided by setting the + (1-indexed) number of the amplifier used to read each detector + pixel. Pixels unassociated with any amplifier are set to 0. + """ + # Check for file; allow for extra .gz, etc. suffix + fil = glob.glob(raw_file + '*') + if len(fil) != 1: + msgs.error("Found {:d} files matching {:s}".format(len(fil), raw_file)) + + # Read + msgs.info("Reading KCWI file: {:s}".format(fil[0])) + hdu = io.fits_open(fil[0]) + detpar = self.get_detector_par(det if det is not None else 1, hdu=hdu) + head0 = hdu[0].header + raw_img = hdu[detpar['dataext']].data.astype(float) + + # Some properties of the image + nampsxy = head0['NAMPSXY'].split() + numamps = int(nampsxy[0]) * int(nampsxy[1]) + amps = self.get_amplifiers(numamps) + # Exposure time (used by ProcessRawImage) + headarr = self.get_headarr(hdu) + exptime = self.get_meta_value(headarr, 'exptime') + + # get the x and y binning factors... + #binning = self.get_meta_value(headarr, 'binning') + + # Always assume normal FITS header formatting + one_indexed = True + include_last = True + for section in ['DSEC', 'BSEC']: + + # Initialize the image (0 means no amplifier) + pix_img = np.zeros(raw_img.shape, dtype=int) + for aa, ampid in enumerate(amps): + # Get the data section + sec = head0[section+"{0:1d}".format(ampid)] + + # Convert the data section from a string to a slice + # TODO :: RJC - I think something has changed here... and the BPM is flipped (or not flipped) for different amp modes. + # RJC - Note, KCWI records binned sections, so there's no need to pass binning in as an argument + datasec = parse.sec2slice(sec, one_indexed=one_indexed, include_end=include_last, require_dim=2) + # Flip the datasec + datasec = datasec[::-1] + + # Assign the amplifier + pix_img[datasec] = aa+1 + + # Finish + if section == 'DSEC': + rawdatasec_img = pix_img.copy() + elif section == 'BSEC': + oscansec_img = pix_img.copy() + + # Return + return detpar, raw_img, hdu, exptime, rawdatasec_img, oscansec_img diff --git a/pypeit/spectrographs/keck_lris.py b/pypeit/spectrographs/keck_lris.py index 3999595587..8c5c6a8db8 100644 --- a/pypeit/spectrographs/keck_lris.py +++ b/pypeit/spectrographs/keck_lris.py @@ -36,6 +36,21 @@ class KeckLRISSpectrograph(spectrograph.Spectrograph): telescope = telescopes.KeckTelescopePar() url = 'https://www2.keck.hawaii.edu/inst/lris/' + def check_spectrograph(self, filename): + """ + Check that the selected spectrograph is the correct one for the input data. + + Args: + filename (:obj:`str`): File to use when determining if the input spectrograph is the correct one. + + """ + instrume = self.get_meta_value(filename, 'instrument') + + if 'keck_lris_red' in self.name and instrume != 'LRIS': + msgs.error('This is not the correct spectrograph. You may want to use keck_lris_blue instead.') + elif 'keck_lris_blue' in self.name and instrume == 'LRIS': + msgs.error('This is not the correct spectrograph. You may want to use keck_lris_red instead.') + @classmethod def default_pypeit_par(cls): """ @@ -55,19 +70,20 @@ def default_pypeit_par(cls): # Keck_LRIS_red/multi_1200_9000_d680_1x2/ . May need a # different solution given that this is binned data and most of # the data in the dev suite is unbinned. - # JXP -- Increased to 6 arcsec. I don't know how 2 (or 1!) could have worked. - par['calibrations']['slitedges']['minimum_slit_length_sci'] = 6 + # lris alignment boxes are typically 4 arcsec + par['calibrations']['slitedges']['minimum_slit_length_sci'] = 5. # Remove slits that are too short - par['calibrations']['slitedges']['minimum_slit_length'] = 4. + par['calibrations']['slitedges']['minimum_slit_length'] = 3. # 1D wavelengths par['calibrations']['wavelengths']['rms_threshold'] = 0.20 # Might be grism dependent # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 1] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['pixelflatframe']['exprng'] = [None, 60] par['calibrations']['traceframe']['exprng'] = [None, 60] - par['calibrations']['standardframe']['exprng'] = [None, 30] + par['calibrations']['illumflatframe']['exprng'] = [None, 60] + par['calibrations']['standardframe']['exprng'] = [1, 61] # Flexure # Always correct for spectral flexure, starting with default parameters @@ -78,7 +94,7 @@ def default_pypeit_par(cls): par['scienceframe']['process']['spat_flexure_correct'] = True par['calibrations']['standardframe']['process']['spat_flexure_correct'] = True - par['scienceframe']['exprng'] = [60, None] + par['scienceframe']['exprng'] = [61, None] # If telluric is triggered @@ -86,7 +102,6 @@ def default_pypeit_par(cls): return par - def config_specific_par(self, scifile, inp_par=None): """ Modify the PypeIt parameters to hard-wired values used for @@ -118,6 +133,13 @@ def config_specific_par(self, scifile, inp_par=None): if self.name == 'keck_lris_red': par['calibrations']['slitedges']['edge_thresh'] = 1000. + # Wave FWHM + binning = parse.parse_binning(self.get_meta_value(scifile, 'binning')) + par['calibrations']['wavelengths']['fwhm'] = 8.0 / binning[0] + par['calibrations']['wavelengths']['fwhm_fromlines'] = True + # Arc lamps list from header + par['calibrations']['wavelengths']['lamps'] = ['use_header'] + return par def init_meta(self): @@ -135,7 +157,7 @@ def init_meta(self): self.meta['decker'] = dict(ext=0, card='SLITNAME') self.meta['binning'] = dict(card=None, compound=True) # - self.meta['mjd'] = dict(ext=0, card='MJD-OBS') + self.meta['mjd'] = dict(card=None, compound=True) self.meta['exptime'] = dict(ext=0, card='ELAPTIME') self.meta['airmass'] = dict(ext=0, card='AIRMASS') # Extras for config and frametyping @@ -143,19 +165,17 @@ def init_meta(self): self.meta['hatch'] = dict(ext=0, card='TRAPDOOR') # Red only, but grabbing here self.meta['dispangle'] = dict(ext=0, card='GRANGLE', rtol=1e-2) - self.meta['cenwave'] = dict(ext=0, card='WAVELEN', rtol=2.0) + self.meta['cenwave'] = dict(card=None, compound=True, rtol=1e-3) self.meta['frameno'] = dict(ext=0, card='FRAMENO') self.meta['instrument'] = dict(ext=0, card='INSTRUME') # Extras for pypeit file - if self.name == 'keck_lris_red_mark4': - self.meta['amp'] = dict(ext=0, card='TAPLINES') - else: - self.meta['amp'] = dict(ext=0, card='NUMAMPS') + self.meta['dateobs'] = dict(card=None, compound=True) + self.meta['amp'] = dict(ext=0, card='NUMAMPS') - # Lamps -- Have varied in time.. - for kk in range(12): # This needs to match the length of LAMPS below - self.meta['lampstat{:02d}'.format(kk+1)] = dict(card=None, compound=True) + # Lamps + # similar approach to DEIMOS + self.meta['lampstat01'] = dict(card=None, compound=True) def compound_meta(self, headarr, meta_key): """ @@ -174,38 +194,67 @@ def compound_meta(self, headarr, meta_key): if meta_key == 'binning': binspatial, binspec = parse.parse_binning(headarr[0]['BINNING']) binning = parse.binning2string(binspec, binspatial) - return binning - elif 'lampstat' in meta_key: - idx = int(meta_key[-2:]) - try: - curr_date = time.Time(self.get_meta_value(headarr, 'mjd'), format='mjd') - except: - msgs.warn('No mjd in header. You either have bad headers, ' - 'or incorrectly specified the wrong spectrograph, ' - 'or are reading in other files from your directory. ' - 'Using 2022-01-01 as the date for parsing lamp info from headers') - curr_date = time.Time("2022-01-01", format='isot') - # Modern -- Assuming the change occurred with the new red detector - t_newlamp = time.Time("2014-02-15", format='isot') # LAMPS changed in Header - if curr_date > t_newlamp: - lamp_names = ['MERCURY', 'NEON', 'ARGON', 'CADMIUM', 'ZINC', 'KRYPTON', 'XENON', - 'FEARGON', 'DEUTERI', 'FLAMP1', 'FLAMP2', 'HALOGEN'] - return headarr[0][lamp_names[idx-1]] # Use this index is offset by 1 - else: # Original lamps - plamps = headarr[0]['LAMPS'].split(',') - # https: // www2.keck.hawaii.edu / inst / lris / instrument_key_list.html - old_lamp_names = ['MERCURY', 'NEON', 'ARGON', 'CADMIUM', 'ZINC', 'HALOGEN'] - if idx <= 5: # Arcs - return ('off' if plamps[idx - 1] == '0' else 'on') - elif idx == 10: # Current FLAMP1 - return headarr[0]['FLIMAGIN'].strip() - elif idx == 11: # Current FLAMP2 - return headarr[0]['FLSPECTR'].strip() - elif idx == 12: # Current Halogen slot - return ('off' if plamps[len(old_lamp_names)-1] == '0' else 'on') - else: # Lamp didn't exist. Set to None - return 'None' + elif meta_key == 'cenwave': + # it may happen that cenwave is zero in the header + return headarr[0]['WAVELEN'] if headarr[0]['WAVELEN'] > 0. else 0.1 + elif meta_key == 'mjd': + if headarr[0].get('MJD-OBS') is not None: + return headarr[0]['MJD-OBS'] + elif headarr[0].get('UTC') is not None or headarr[0].get('UT') is not None: + ut = headarr[0].get('UTC') if headarr[0].get('UTC') is not None else headarr[0].get('UT') + if headarr[0].get('DATE-OBS') is not None: + return time.Time('{}T{}'.format(headarr[0]['DATE-OBS'], ut)).mjd + elif headarr[0].get('DATE') is not None: + # LRIS sometime has a duplicate DATE card. The first one is the date of the + # file creation and the second one is the date of the observation. + # We want the second one. Find it by looking for the date string without "T" + dd = np.where([headarr[0].cards[i][0] == 'DATE' and 'T' not in headarr[0].cards[i][1] + for i in range(len(headarr[0].cards))])[0] + if dd.size > 0: + date = headarr[0].cards[dd[0]][1] + return time.Time('{}T{}'.format(date, ut)).mjd + else: + # this is most likely not the obs date+time, but the date+time the file + # was created, which should be very close to the obs time + return time.Time(headarr[0]['DATE']).mjd + elif meta_key == 'dateobs': + if headarr[0].get('DATE-OBS') is not None: + return headarr[0]['DATE-OBS'] + elif headarr[0].get('DATE') is not None: + return headarr[0]['DATE'].split('T')[0] + elif meta_key == 'lampstat01': + # lamp status header keywords + lamp_keys = ['MERCURY', 'NEON', 'ARGON', 'CADMIUM', 'ZINC', 'HALOGEN', + 'KRYPTON', 'XENON', 'FEARGON', 'DEUTERI'] + # pypeit lamp names + lamp_names = np.array(['HgI', 'NeI', 'ArI', 'CdI', 'ZnI', 'Halogen', 'KrI', 'XeI', 'FeAr', '2H']) + + # lamps header keywords changed over time. We know when the change happened ("2014-02-15"), + # but will try to not use the date and instead look for the latest keyword. + # old keyword looks like LAMPS = '0,0,0,0,0,1', which refers to 'MERCURY,NEON,ARGON,CADMIUM,ZINC,HALOGEN' + # from https://www2.keck.hawaii.edu/inst/lris/instrument_key_list.html + old_lamp_status = np.array(headarr[0].get('LAMPS').split(','), dtype=int).astype(bool) \ + if headarr[0].get('LAMPS') is not None else None + new_lamp_status = np.array([headarr[0].get(k) == 'on' for k in lamp_keys]) + where_on = np.where(new_lamp_status)[0] if old_lamp_status is None else np.where(old_lamp_status)[0] + if where_on.size > 0: + return ' '.join(lamp_names[where_on]) + + # dome flat header keywords changed over time, but it's not clear-cut the date of the change. + # We check if the latest keyword is present, and if not, we assume it's the old one. + elif headarr[0].get('FLAMP1') is not None or headarr[0].get('FLAMP2') is not None: + if headarr[0].get('FLAMP1') == 'on' or headarr[0].get('FLAMP2') == 'on': + return 'on' + else: + return 'off' + elif headarr[0].get('FLIMAGIN') is not None or headarr[0].get('FLSPECTR') is not None: + if headarr[0].get('FLIMAGIN') == 'on' or headarr[0].get('FLSPECTR') == 'on': + return 'on' + else: + return 'off' + else: + return 'off' else: msgs.error("Not ready for this compound meta") @@ -223,7 +272,25 @@ def configuration_keys(self): and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` object. """ - return super().configuration_keys() + ['binning'] + return super().configuration_keys() + ['amp', 'binning'] + + def config_independent_frames(self): + """ + Define frame types that are independent of the fully defined + instrument configuration. + + Bias and dark frames are considered independent of a configuration, + but the DATE-OBS keyword is used to assign each to the most-relevant + configuration frame group. See + :func:`~pypeit.metadata.PypeItMetaData.set_configurations`. + + Returns: + :obj:`dict`: Dictionary where the keys are the frame types that + are configuration independent and the values are the metadata + keywords that can be used to assign the frames to a configuration + group. + """ + return {'bias': ['amp', 'binning', 'dateobs'], 'dark': ['amp', 'binning', 'dateobs']} def pypeit_file_keys(self): """ @@ -234,7 +301,7 @@ def pypeit_file_keys(self): :class:`~pypeit.metadata.PypeItMetaData` instance to print to the :ref:`pypeit_file`. """ - return super().pypeit_file_keys() + ['frameno'] + return super().pypeit_file_keys() + ['hatch', 'lampstat01', 'dateobs', 'utc', 'frameno'] def check_frame_type(self, ftype, fitstbl, exprng=None): """ @@ -265,7 +332,7 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): if ftype in ['pixelflat', 'trace', 'illumflat']: # Allow for dome or internal good_dome = self.lamps(fitstbl, 'dome') & (fitstbl['hatch'] == 'open') - good_internal = self.lamps(fitstbl, 'halogen') & (fitstbl['hatch'] == 'closed') + good_internal = self.lamps(fitstbl, 'internal') & (fitstbl['hatch'] == 'closed') # Flats and trace frames are typed together return good_exp & (good_dome + good_internal) if ftype in ['pinhole', 'dark']: @@ -298,28 +365,41 @@ def lamps(self, fitstbl, status): """ if status == 'off': # Check if all are off - return np.all(np.array([ (fitstbl[k] == 'off') | (fitstbl[k] == 'None') - for k in fitstbl.keys() if 'lampstat' in k]), axis=0) + return fitstbl['lampstat01'] == 'off' elif status == 'arcs': # Check if any arc lamps are on - arc_lamp_stat = [ 'lampstat{0:02d}'.format(i) for i in range(1,9) ] - return np.any(np.array([ fitstbl[k] == 'on' for k in fitstbl.keys() - if k in arc_lamp_stat]), axis=0) + return np.array([lamp not in ['Halogen', '2H', 'on', 'off'] for lamp in fitstbl['lampstat01']]) elif status == 'dome': # Check if any dome lamps are on - # Warning 9, 10 are FEARGON and DEUTERI - dome_lamp_stat = [ 'lampstat{0:02d}'.format(i) for i in range(9,13) ] - return np.any(np.array([ fitstbl[k] == 'on' for k in fitstbl.keys() - if k in dome_lamp_stat]), axis=0) - elif status == 'halogen': - halo_lamp_stat = ['lampstat12'] - return np.any(np.array([ fitstbl[k] == 'on' for k in fitstbl.keys() - if k in halo_lamp_stat]), axis=0) + return fitstbl['lampstat01'] == 'on' + + elif status == 'internal': + return np.array([lamp in ['Halogen', '2H'] for lamp in fitstbl['lampstat01']]) else: msgs.error(f"Bad status option! {status}") raise ValueError('No implementation for status = {0}'.format(status)) + def get_lamps(self, fitstbl): + """ + Extract the list of arc lamps used from header + + Args: + fitstbl (`astropy.table.Table`_): + The table with the metadata for one or more arc frames. + + Returns: + lamps (:obj:`list`) : List used arc lamps + + """ + lamps = [f'{lamp}' for lamp in np.unique(np.concatenate([lname.split() for lname in fitstbl['lampstat01']]))] + # sometimes the flat lamps keyword are set ON for arc frames, remove them to avoid crashes in wavecalib + flat_lamps = ['Halogen', '2H'] + for fl in flat_lamps: + if fl in lamps: + lamps.remove(fl) + return lamps + def get_rawimage(self, raw_file, det): """ Read raw images and generate a few other bits and pieces @@ -599,14 +679,20 @@ def get_maskdef_slitedges(self, ccdnum=None, filename=None, debug=None, # Trim down by detector # TODO -- Deal with Mark4 - max_spat = 2048//bin_spat + if self.name == 'keck_lris_red_mark4': + max_spat = 4112//bin_spat + else: + max_spat = 2048//bin_spat if ccdnum == 1: if self.name == 'keck_lris_red': good = centers < 0. xstart = max_spat + 160//bin_spat # The 160 is for the chip gap elif self.name == 'keck_lris_blue': good = centers < 0. - xstart = max_spat + 30//bin_spat + xstart = max_spat + 30//bin_spat + elif self.name == 'keck_lris_red_mark4': + xstart = 2073//bin_spat + good = centers < max_spat # No chip gap else: msgs.error(f'Not ready to use slitmasks for {self.name}. Develop it!') else: @@ -652,7 +738,7 @@ class KeckLRISBSpectrograph(KeckLRISSpectrograph): camera = 'LRISb' header_name = 'LRISBLUE' supported = True - comment = 'Blue camera; see :doc:`lris`' + comment = 'Blue camera; Current FITS file format; used from May 2009, see :doc:`lris`' def get_detector_par(self, det, hdu=None): """ @@ -682,7 +768,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.135, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 65535., nonlinear = 0.86, mincounts = -1e10, @@ -725,6 +811,26 @@ def get_detector_par(self, det, hdu=None): # Return return detector + def check_spectrograph(self, filename): + """ + Check that the selected spectrograph is the correct one for the input data. + + Args: + filename (:obj:`str`): File to use when determining if the input spectrograph is the correct one. + + """ + + super().check_spectrograph(filename) + + # check that we are using the right spectrograph (keck_lris_blue or keck_lris_blue_orig) + _dateobs = time.Time(self.get_meta_value(filename, 'dateobs'), format='iso') + # last day of keck_lris_blue_orig + date_orig = time.Time('2009-04-30', format='iso') + if _dateobs <= date_orig and self.name in ['keck_lris_blue']: + msgs.error('This is not the correct spectrograph. Use keck_lris_blue_orig instead.') + elif _dateobs > date_orig and self.name in ['keck_lris_blue_orig']: + msgs.error('This is not the correct spectrograph. Use keck_lris_blue instead.') + @classmethod def default_pypeit_par(cls): """ @@ -743,15 +849,14 @@ def default_pypeit_par(cls): par['calibrations']['wavelengths']['rms_threshold'] = 0.20 # Might be grism dependent.. par['calibrations']['wavelengths']['sigdetect'] = 10.0 - par['calibrations']['wavelengths']['lamps'] = ['NeI', 'ArI', 'CdI', 'KrI', 'XeI', 'ZnI', 'HgI'] #par['calibrations']['wavelengths']['nonlinear_counts'] = self.detector[0]['nonlinear'] * self.detector[0]['saturation'] par['calibrations']['wavelengths']['n_first'] = 3 - par['calibrations']['wavelengths']['match_toler'] = 2.5 par['calibrations']['wavelengths']['method'] = 'full_template' # Allow for longer exposure times on blue side (especially if using the Dome lamps) par['calibrations']['pixelflatframe']['exprng'] = [None, 300] par['calibrations']['traceframe']['exprng'] = [None, 300] + par['calibrations']['illumflatframe']['exprng'] = [None, 300] return par @@ -777,22 +882,22 @@ def config_specific_par(self, scifile, inp_par=None): # Wavelength calibrations if self.get_meta_value(scifile, 'dispname') == '300/5000': - par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_blue_300_d680.fits' + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_blue_B300_5000_d680_ArCdHgKrNeXeZnFeAr.fits' + par['calibrations']['wavelengths']['n_final'] = 2 par['flexure']['spectrum'] = 'sky_LRISb_400.fits' elif self.get_meta_value(scifile, 'dispname') == '400/3400': - par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_blue_400_d560.fits' + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_blue_B400_3400_d560_ArCdHgNeZnFeAr.fits' + par['calibrations']['wavelengths']['n_final'] = 2 par['flexure']['spectrum'] = 'sky_LRISb_400.fits' elif self.get_meta_value(scifile, 'dispname') == '600/4000': - par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_blue_600_d560.fits' + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_blue_B600_4000_d560_ArCdHgKrNeXeZn.fits' + par['calibrations']['wavelengths']['sigdetect'] = 20. + par['calibrations']['wavelengths']['n_final'] = 6 par['flexure']['spectrum'] = 'sky_LRISb_600.fits' elif self.get_meta_value(scifile, 'dispname') == '1200/3400': - par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_blue_1200_d460.fits' + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_blue_B1200_3400_d560_ArCdHgNeZn.fits' par['flexure']['spectrum'] = 'sky_LRISb_600.fits' - # FWHM - binning = parse.parse_binning(self.get_meta_value(scifile, 'binning')) - par['calibrations']['wavelengths']['fwhm'] = 8.0 / binning[0] - # Slit tracing # Reduce the slit parameters because the flux does not span the full detector # It is primarily on the upper half of the detector (usually) @@ -879,21 +984,7 @@ class KeckLRISBOrigSpectrograph(KeckLRISBSpectrograph): name = 'keck_lris_blue_orig' camera = 'LRISb' supported = True # TODO: Is this true? - comment = 'Original detector; replaced in 20??; see :doc:`lris`' - - def init_meta(self): - """ - Define how metadata are derived from the spectrograph files. - - That is, this associates the PypeIt-specific metadata keywords - with the instrument-specific header cards using :attr:`meta`. - """ - super().init_meta() - # Remove the lamps - keys = list(self.meta.keys()) - for key in keys: - if 'lampstat' in key: - self.meta.pop(key) + comment = 'Blue camera; Original FITS file format; used until April 2009; see :doc:`lris`' def get_detector_par(self, det, hdu=None): """ @@ -1021,7 +1112,8 @@ class KeckLRISRSpectrograph(KeckLRISSpectrograph): header_name = 'LRIS' supported = True ql_supported = True - comment = 'Red camera; LBNL detector, 2kx4k; see :doc:`lris`' + comment = 'Red camera; Current FITS file format; ' \ + 'LBNL detector, 2kx4k; used from May 2009, see :doc:`lris`' def get_detector_par(self, det, hdu=None): """ @@ -1050,8 +1142,8 @@ def get_detector_par(self, det, hdu=None): specaxis=0, specflip=False, spatflip=False, - platescale=0.135, - darkcurr=0.0, + platescale=0.135, # confirmed from keck website at: https://www2.keck.hawaii.edu/inst/lris/lris-red-upgrade-notes.html#dewar3 + darkcurr=0.0, # e-/pixel/hour saturation=65535., nonlinear=0.76, mincounts=-1e10, @@ -1124,7 +1216,28 @@ def get_detector_par(self, det, hdu=None): # Return return detector + def check_spectrograph(self, filename): + """ + Check that the selected spectrograph is the correct one for the input data. + + Args: + filename (:obj:`str`): File to use when determining if the input spectrograph is the correct one. + + """ + super().check_spectrograph(filename) + # check that we are using the right spectrograph (keck_lris_red, keck_lris_red_orig, or keck_lris_red_mark4) + _dateobs = time.Time(self.get_meta_value(filename, 'dateobs'), format='iso') + # starting date for keck_lris_red_mark4 + date_mark4 = time.Time('2021-04-22', format='iso') + # last day of keck_lris_red_orig + date_orig = time.Time('2009-05-02', format='iso') + if _dateobs <= date_orig and self.name in ['keck_lris_red_mark4', 'keck_lris_red']: + msgs.error('This is not the correct spectrograph. Use keck_lris_red_orig instead.') + elif _dateobs >= date_mark4 and self.name in ['keck_lris_red_orig', 'keck_lris_red']: + msgs.error('This is not the correct spectrograph. Use keck_lris_red_mark4 instead.') + elif date_orig < _dateobs < date_mark4 and self.name in ['keck_lris_red_orig', 'keck_lris_red_mark4']: + msgs.error('This is not the correct spectrograph. Use keck_lris_red instead.') @classmethod def default_pypeit_par(cls): @@ -1140,7 +1253,6 @@ def default_pypeit_par(cls): par['calibrations']['slitedges']['edge_thresh'] = 20. # 1D wavelength solution - par['calibrations']['wavelengths']['lamps'] = ['NeI', 'ArI', 'CdI', 'KrI', 'XeI', 'ZnI', 'HgI'] #par['calibrations']['wavelengths']['nonlinear_counts'] = self.detector[0]['nonlinear'] * self.detector[0]['saturation'] par['calibrations']['wavelengths']['sigdetect'] = 10.0 # Tilts @@ -1215,35 +1327,51 @@ def config_specific_par(self, scifile, inp_par=None): par['scienceframe']['process']['objlim'] = objlim # Wavelength calibrations - if self.get_meta_value(scifile, 'dispname') == '400/8500': # This is basically a reidentify - if self.name == 'keck_lris_red_mark4': - par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_mark4_R400.fits' - else: - par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_400.fits' + if self.get_meta_value(scifile, 'dispname') == '150/7500': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_R150_7500_ArCdHgNeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + elif self.get_meta_value(scifile, 'dispname') == '300/5000': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_R300_5000_ArCdHgKrNeXeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + elif self.get_meta_value(scifile, 'dispname') == '400/8500': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_R400_8500_ArCdHgKrNeXeZn.fits' par['calibrations']['wavelengths']['method'] = 'full_template' par['calibrations']['wavelengths']['sigdetect'] = 20.0 - par['calibrations']['wavelengths']['nsnippet'] = 1 elif self.get_meta_value(scifile, 'dispname') == '600/5000': - par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_600_5000.fits' + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_R600_5000_ArCdHgKrNeXeZn.fits' par['calibrations']['wavelengths']['method'] = 'full_template' elif self.get_meta_value(scifile, 'dispname') == '600/7500': - par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_600_7500.fits' + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_R600_7500_ArCdHgKrNeXeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + par['calibrations']['wavelengths']['n_first'] = 3 + elif self.get_meta_value(scifile, 'dispname') == '600/10000': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_R600_10000_ArCdHgKrNeXeZn.fits' par['calibrations']['wavelengths']['method'] = 'full_template' - elif self.get_meta_value(scifile, 'dispname') == '600/10000': # d680 - par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_600_10000.fits' + par['calibrations']['wavelengths']['n_first'] = 3 + elif self.get_meta_value(scifile, 'dispname') == '831/8200': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_R831_8200_ArCdHgKrNeXeZn.fits' par['calibrations']['wavelengths']['method'] = 'full_template' + par['calibrations']['wavelengths']['sigdetect'] = 30.0 # lots of ghost lines + par['calibrations']['wavelengths']['n_first'] = 3 + elif self.get_meta_value(scifile, 'dispname') == '900/5500': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_R900_5500_ArCdHgNeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + par['calibrations']['wavelengths']['sigdetect'] = 30.0 # lots of ghost lines + par['calibrations']['wavelengths']['n_first'] = 3 + elif self.get_meta_value(scifile, 'dispname') == '1200/7500': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_R1200_7500_ArCdHgKrNeXeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + par['calibrations']['wavelengths']['n_first'] = 3 + par['calibrations']['wavelengths']['n_final'] = 5 elif self.get_meta_value(scifile, 'dispname') == '1200/9000': - par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_1200_9000.fits' + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_R1200_9000.fits' par['calibrations']['wavelengths']['method'] = 'full_template' - - # FWHM - binning = parse.parse_binning(self.get_meta_value(scifile, 'binning')) - par['calibrations']['wavelengths']['fwhm'] = 8.0 / binning[0] + par['calibrations']['wavelengths']['n_first'] = 3 + par['calibrations']['wavelengths']['n_final'] = 5 # Return return par - def init_meta(self): """ Define how metadata are derived from the spectrograph files. @@ -1269,7 +1397,7 @@ def configuration_keys(self): and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` object. """ - return super().configuration_keys() + ['dispangle', 'cenwave', 'amp', 'binning'] + return super().configuration_keys() + ['dispangle', 'cenwave'] def raw_header_cards(self): """ @@ -1348,13 +1476,14 @@ class KeckLRISRMark4Spectrograph(KeckLRISRSpectrograph): ndet = 1 name = 'keck_lris_red_mark4' supported = True - comment = 'New Mark4 detector, circa Spring 2021; Supported setups = R400' + comment = 'Red camera; New Mark4 detector, in operation since May 2021; see :doc:`lris`' def init_meta(self): super().init_meta() # Over-ride a pair self.meta['mjd'] = dict(ext=0, card='MJD') self.meta['exptime'] = dict(ext=0, card='TELAPSE') + self.meta['amp'] = dict(ext=0, card='TAPLINES') def get_detector_par(self, det, hdu=None): """ @@ -1381,9 +1510,9 @@ def get_detector_par(self, det, hdu=None): dataext=0, specaxis=0, specflip=True, - spatflip=False, - platescale=0.123, # From the web page - darkcurr=0.0, + spatflip=True, + platescale=0.135, # From the web page + darkcurr=0.0, # e-/pixel/hour saturation=65535., nonlinear=0.76, mincounts=-1e10, @@ -1528,15 +1657,16 @@ def get_rawimage(self, raw_file, det): # Note: There is no way we know to super super super return spectrograph.Spectrograph.get_rawimage(self, raw_file, det) + class KeckLRISROrigSpectrograph(KeckLRISRSpectrograph): """ - Child to handle the original LRISr detector (pre 01 JUL 2009) + Child to handle the original LRISr detector (up to 2009-05-02) """ ndet = 1 name = 'keck_lris_red_orig' camera = 'LRISr' supported = True - comment = 'Original detector; replaced in 2009; see :doc:`lris`' + comment = 'Red camera; Original FITS file format; used until April 2009; see :doc:`lris`' @classmethod def default_pypeit_par(cls): @@ -1549,9 +1679,58 @@ def default_pypeit_par(cls): """ par = super().default_pypeit_par() - # 1D wavelength solution - par['calibrations']['wavelengths']['lamps'] = ['NeI', 'ArI', 'KrI', 'XeI', 'HgI'] + return par + def config_specific_par(self, scifile, inp_par=None): + """ + Modify the PypeIt parameters to hard-wired values used for + specific instrument configurations. + + Args: + scifile (:obj:`str`): + File to use when determining the configuration and how + to adjust the input parameters. + inp_par (:class:`~pypeit.par.parset.ParSet`, optional): + Parameter set used for the full run of PypeIt. If None, + use :func:`default_pypeit_par`. + + Returns: + :class:`~pypeit.par.parset.ParSet`: The PypeIt parameter set + adjusted for configuration specific parameter values. + """ + # Start with instrument wide + par = super().config_specific_par(scifile, inp_par=inp_par) + + # Wavelength calibrations + if self.get_meta_value(scifile, 'dispname') == '150/7500': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_orig_R150_7500_ArHgNe.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + elif self.get_meta_value(scifile, 'dispname') == '300/5000': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_orig_R300_5000_ArCdHgNeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + elif self.get_meta_value(scifile, 'dispname') == '400/8500': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_orig_R400_8500_ArCdHgNeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + elif self.get_meta_value(scifile, 'dispname') == '600/5000': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_orig_R600_5000_ArCdHgNeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + elif self.get_meta_value(scifile, 'dispname') == '600/7500': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_orig_R600_7500_ArCdHgNeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + elif self.get_meta_value(scifile, 'dispname') == '600/10000': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_orig_R600_10000_ArCdHgNeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + elif self.get_meta_value(scifile, 'dispname') == '831/8200': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_orig_R831_8200_ArCdHgNeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + elif self.get_meta_value(scifile, 'dispname') == '900/5500': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_orig_R900_5500_ArCdHgNeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + elif self.get_meta_value(scifile, 'dispname') == '1200/7500': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_lris_red_orig_R1200_7500_ArCdHgNeZn.fits' + par['calibrations']['wavelengths']['method'] = 'full_template' + + # Return return par def get_detector_par(self, det, hdu=None): @@ -1581,8 +1760,8 @@ def get_detector_par(self, det, hdu=None): specaxis=1, specflip=False, spatflip=False, - platescale=0.21, # TO BE UPDATED!! - darkcurr=0.0, + platescale=0.211, # confirmed from keck website at: https://www2.keck.hawaii.edu/inst/lris/lris-red-upgrade-notes.html#dewar3 + darkcurr=0.0, # e-/pixel/hour saturation=65535., nonlinear=0.76, mincounts=-1e10, @@ -1600,20 +1779,6 @@ def get_detector_par(self, det, hdu=None): # Return return detector - def init_meta(self): - """ - Define how metadata are derived from the spectrograph files. - - That is, this associates the PypeIt-specific metadata keywords - with the instrument-specific header cards using :attr:`meta`. - """ - super().init_meta() - # Remove the lamps - keys = list(self.meta.keys()) - for key in keys: - if 'lampstat' in key: - self.meta.pop(key) - def get_rawimage(self, raw_file, det): """ Read raw images and generate a few other bits and pieces diff --git a/pypeit/spectrographs/keck_mosfire.py b/pypeit/spectrographs/keck_mosfire.py index 4b5a5cfc52..fe78ec576e 100644 --- a/pypeit/spectrographs/keck_mosfire.py +++ b/pypeit/spectrographs/keck_mosfire.py @@ -59,7 +59,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.1798, - darkcurr = 0.8, + darkcurr = 28.8, # e-/pixel/hour (=0.008 e-/pixel/s) saturation = 1e9, # ADU, this is hacked for now nonlinear = 1.00, # docs say linear to 90,000 but our flats are usually higher numamplifiers = 1, @@ -757,7 +757,7 @@ def parse_dither_pattern(self, file_list, ext=None): offset_arcsec[ifile] = hdr['YOFFSET'] return np.array(dither_pattern), np.array(dither_id), np.array(offset_arcsec) - def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table, debug=False): + def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table, log10_blaze_function=None, debug=False): """ This routine is for performing instrument/disperser specific tweaks to standard stars so that sensitivity @@ -779,6 +779,9 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table, Table containing meta data that is slupred from the :class:`~pypeit.specobjs.SpecObjs` object. See :meth:`~pypeit.specobjs.SpecObjs.unpack_object` for the contents of this table. + log10_blaze_function: `numpy.ndarray`_ or None + Input blaze function to be tweaked, optional. Default=None. + Returns ------- @@ -790,6 +793,8 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table, Output inverse variance of standard star counts (:obj:`float`, ``shape = (nspec,)``) gpm_out: `numpy.ndarray`_ Output good pixel mask for standard (:obj:`bool`, ``shape = (nspec,)``) + log10_blaze_function_out: `numpy.ndarray`_ or None + Output blaze function after being tweaked. """ # Could check the wavelenghts here to do something more robust to header/meta data issues @@ -840,8 +845,9 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table, second_order_region= (wave_in < wave_blue) | (wave_in > wave_red) wave = wave_in.copy() counts = counts_in.copy() - gpm = gpm_in.copy() counts_ivar = counts_ivar_in.copy() + gpm = gpm_in.copy() + wave[second_order_region] = 0.0 counts[second_order_region] = 0.0 counts_ivar[second_order_region] = 0.0 @@ -849,6 +855,15 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table, # over the valid wavelength region. While we could mask, this would still produce a wave_min and wave_max # for the zeropoint that includes the bad regions, and the polynomial fits will extrapolate crazily there gpm[second_order_region] = False + + if log10_blaze_function is not None: + log10_blaze_function_out = log10_blaze_function.copy() + log10_blaze_function_out[second_order_region] = 0.0 + else: + log10_blaze_function_out = None + + return wave, counts, counts_ivar, gpm, log10_blaze_function_out + #if debug: # from matplotlib import pyplot as plt # counts_sigma = np.sqrt(utils.inverse(counts_ivar_in)) @@ -859,41 +874,6 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table, # plt.axvline(wave_red, color='red') # plt.legend() # plt.show() - return wave, counts, counts_ivar, gpm - - def list_detectors(self, mosaic=False): - """ - List the *names* of the detectors in this spectrograph. - - This is primarily used :func:`~pypeit.slittrace.average_maskdef_offset` - to measure the mean offset between the measured and expected slit - locations. - - Detectors separated along the dispersion direction should be ordered - along the first axis of the returned array. For example, Keck/DEIMOS - returns: - - .. code-block:: python - - dets = np.array([['DET01', 'DET02', 'DET03', 'DET04'], - ['DET05', 'DET06', 'DET07', 'DET08']]) - - such that all the bluest detectors are in ``dets[0]``, and the slits - found in detectors 1 and 5 are just from the blue and red counterparts - of the same slit. - - Args: - mosaic (:obj:`bool`, optional): - Is this a mosaic reduction? - It is used to determine how to list the detector, i.e., 'DET' or 'MSC'. - - Returns: - `numpy.ndarray`_: The list of detectors in a `numpy.ndarray`_. If - the array is 2D, there are detectors separated along the dispersion - axis. - """ - return np.array([detector_container.DetectorContainer.get_name(i+1) - for i in range(self.ndet)]) def get_slitmask(self, filename): """ diff --git a/pypeit/spectrographs/keck_nires.py b/pypeit/spectrographs/keck_nires.py index f9bf2c4a8b..e4d5ef1747 100644 --- a/pypeit/spectrographs/keck_nires.py +++ b/pypeit/spectrographs/keck_nires.py @@ -55,7 +55,7 @@ def get_detector_par(self, det, hdu=None): specflip = True, spatflip=False, platescale = 0.15, - darkcurr = 0.01, + darkcurr = 468.0, # e-/pixel/hour (=0.13 e-/pixel/s) saturation = 1e6, # I'm not sure we actually saturate with the DITs??? nonlinear = 0.76, mincounts = -1e10, @@ -80,9 +80,10 @@ def default_pypeit_par(cls): # Wavelengths # 1D wavelength solution - par['calibrations']['wavelengths']['rms_threshold'] = 0.20 #0.20 # Might be grating dependent.. + par['calibrations']['wavelengths']['rms_threshold'] = 0.30 par['calibrations']['wavelengths']['sigdetect']=5.0 - par['calibrations']['wavelengths']['fwhm']= 5.0 + par['calibrations']['wavelengths']['fwhm']= 2.2 # Measured + par['calibrations']['wavelengths']['fwhm_fromlines'] = True par['calibrations']['wavelengths']['n_final']= [3,4,4,4,4] par['calibrations']['wavelengths']['lamps'] = ['OH_NIRES'] #par['calibrations']['wavelengths']['nonlinear_counts'] = self.detector[0]['nonlinear'] * self.detector[0]['saturation'] @@ -111,10 +112,18 @@ def default_pypeit_par(cls): use_darkimage=False) par.reset_all_processimages_par(**turn_off) - # Extraction + + # Reduce -- Sky-Subtraction par['reduce']['skysub']['bspline_spacing'] = 0.8 + # Reduce -- Extraction par['reduce']['extraction']['sn_gauss'] = 4.0 + # Reduce -- Object finding + #par['reduce']['findobj']['ech_find_nabove_min_snr'] = 1 + # Require detection in a single order since given only 5 orders and slitlosses for NIRES, often + # things are only detected in the K-band? Decided not to make this the default. + + # Flexure par['flexure']['spec_method'] = 'skip' @@ -134,10 +143,11 @@ def default_pypeit_par(cls): par['sensfunc']['IR']['maxiter'] = 2 par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' - # COADD2D - # set offsets for coadd2d + # Coadding + par['coadd1d']['wave_method'] = 'log10' par['coadd2d']['offsets'] = 'header' + return par def init_meta(self): diff --git a/pypeit/spectrographs/keck_nirspec.py b/pypeit/spectrographs/keck_nirspec.py index caf58dbf4c..07ac4fa3d7 100644 --- a/pypeit/spectrographs/keck_nirspec.py +++ b/pypeit/spectrographs/keck_nirspec.py @@ -45,7 +45,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.193, - darkcurr = 0.8, + darkcurr = 2520.0, # e-/pixel/hour (=0.7 e-/pixel/s) saturation = 100000., nonlinear = 1.00, # docs say linear to 90,000 but our flats are usually higher numamplifiers = 1, diff --git a/pypeit/spectrographs/lbt_luci.py b/pypeit/spectrographs/lbt_luci.py index 8bad08f673..0ac9d14a04 100644 --- a/pypeit/spectrographs/lbt_luci.py +++ b/pypeit/spectrographs/lbt_luci.py @@ -43,7 +43,7 @@ class LBTLUCISpectrograph(spectrograph.Spectrograph): # use_darkimage=False) # par.reset_all_processimages_par(**turn_off) # -# par['calibrations']['biasframe']['exprng'] = [None, 1] +# par['calibrations']['biasframe']['exprng'] = [None, 0.001] # par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames # par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames # par['calibrations']['pixelflatframe']['exprng'] = [0, None] @@ -288,7 +288,7 @@ def get_detector_par(self, det, hdu=None): platescale = 0.25, # Dark current nominally is < 360 electrons per hours # but the dark subtraction will effectively bring this to 0 - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour # Saturation is 55000, but will be set to dummy value for # now saturation = 1e+8, @@ -417,7 +417,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.25, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour # Saturation is 55000, but will be set to dummy value for # now saturation=1e+8, diff --git a/pypeit/spectrographs/lbt_mods.py b/pypeit/spectrographs/lbt_mods.py index ab9b7bcecb..026d727e2e 100644 --- a/pypeit/spectrographs/lbt_mods.py +++ b/pypeit/spectrographs/lbt_mods.py @@ -14,7 +14,7 @@ from pypeit.par import pypeitpar from pypeit.spectrographs import spectrograph from pypeit.core import parse -from pypeit.images import detector_container +from pypeit.images.detector_container import DetectorContainer # TODO: FW: test MODS1B and MODS2B @@ -43,7 +43,7 @@ def default_pypeit_par(cls): # Scienceimage default parameters # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 1] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['pixelflatframe']['exprng'] = [0, None] @@ -299,7 +299,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.123, - darkcurr = 0.4, + darkcurr = 0.4, # e-/pixel/hour saturation = 65535., nonlinear = 0.99, mincounts = -1e10, @@ -310,7 +310,7 @@ def get_detector_par(self, det, hdu=None): # datasec = np.atleast_1d('[:,:]'), # oscansec = np.atleast_1d('[:,:]') ) - return detector_container.DetectorContainer(**detector_dict) + return DetectorContainer(**detector_dict) @classmethod def default_pypeit_par(cls): @@ -463,7 +463,7 @@ def get_detector_par(self, det, hdu=None): specflip = True, spatflip = False, platescale = 0.120, - darkcurr = 0.5, + darkcurr = 0.5, # e-/pixel/hour saturation = 65535., nonlinear = 0.99, mincounts = -1e10, @@ -474,7 +474,7 @@ def get_detector_par(self, det, hdu=None): # datasec = np.atleast_1d('[:,:]'), # oscansec = np.atleast_1d('[:,:]') ) - return detector_container.DetectorContainer(**detector_dict) + return DetectorContainer(**detector_dict) @classmethod def default_pypeit_par(cls): @@ -620,7 +620,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.123, - darkcurr = 0.4, + darkcurr = 0.4, # e-/pixel/hour saturation = 65535., nonlinear = 0.99, mincounts = -1e10, @@ -631,7 +631,7 @@ def get_detector_par(self, det, hdu=None): # datasec = np.atleast_1d('[:,:]'), # oscansec = np.atleast_1d('[:,:]') ) - return detector_container.DetectorContainer(**detector_dict) + return DetectorContainer(**detector_dict) @classmethod def default_pypeit_par(cls): @@ -782,7 +782,7 @@ def get_detector_par(self, det, hdu=None): specflip = True, spatflip = False, platescale = 0.120, - darkcurr = 0.5, + darkcurr = 0.5, # e-/pixel/hour saturation = 65535., nonlinear = 0.99, mincounts = -1e10, @@ -793,7 +793,7 @@ def get_detector_par(self, det, hdu=None): # datasec = np.atleast_1d('[:,:]'), # oscansec = np.atleast_1d('[:,:]') ) - return detector_container.DetectorContainer(**detector_dict) + return DetectorContainer(**detector_dict) @classmethod def default_pypeit_par(cls): diff --git a/pypeit/spectrographs/ldt_deveny.py b/pypeit/spectrographs/ldt_deveny.py index 7dc76a154a..df2b44cc4e 100644 --- a/pypeit/spectrographs/ldt_deveny.py +++ b/pypeit/spectrographs/ldt_deveny.py @@ -94,7 +94,7 @@ def get_detector_par(self, det, hdu=None): specflip = True, # DeVeny CCD has blue at the right spatflip = False, platescale = 0.34, # Arcsec / pixel - darkcurr = 4.5, # Electrons per hour + darkcurr = 4.5, # e-/pixel/hour saturation = 65535., # 16-bit ADC nonlinear = 0.97, # Linear to ~97% of saturation mincounts = -1e10, @@ -638,7 +638,7 @@ def get_rawimage(self, raw_file, det): # Return return detector, raw_img, hdu, exptime, rawdatasec_img, oscansec_img - def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table): + def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table, log10_blaze_function=None): """ This routine is for performing instrument- and/or disperser-specific tweaks to standard stars so that sensitivity function fits will be @@ -660,6 +660,8 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table) Table containing meta data that is slupred from the :class:`~pypeit.specobjs.SpecObjs` object. See :meth:`~pypeit.specobjs.SpecObjs.unpack_object` for the contents of this table. + log10_blaze_function: `numpy.ndarray`_ or None + Input blaze function to be tweaked, optional. Default=None. Returns ------- @@ -671,6 +673,8 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table) Output inverse variance of standard star counts (:obj:`float`, ``shape = (nspec,)``) gpm_out: `numpy.ndarray`_ Output good pixel mask for standard (:obj:`bool`, ``shape = (nspec,)``) + log10_blaze_function_out: `numpy.ndarray`_ or None + Output blaze function after being tweaked. """ # First, simply chop off the wavelengths outside physical limits: valid_wave = (wave_in >= 2900.0) & (wave_in <= 11000.0) @@ -679,10 +683,17 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table) counts_ivar_out = counts_ivar_in[valid_wave] gpm_out = gpm_in[valid_wave] + if log10_blaze_function is not None: + log10_blaze_function_out = log10_blaze_function[valid_wave] + else: + log10_blaze_function_out = None + + # Next, build a gpm based on other reasonable wavelengths and filters edge_region = (wave_out < 3000.0) | (wave_out > 10200.0) neg_counts = counts_out <= 0 + # If an order-blocking filter was in use, mask blocked region # at "nominal" cutoff value if 'FILTER1' in meta_table.keys(): @@ -708,7 +719,7 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table) & np.logical_not(block_region) ) - return wave_out, counts_out, counts_ivar_out, gpm_out + return wave_out, counts_out, counts_ivar_out, gpm_out, log10_blaze_function_out @staticmethod def rotate_trimsections(section_string: str, nspecpix: int): diff --git a/pypeit/spectrographs/magellan_fire.py b/pypeit/spectrographs/magellan_fire.py index 6d20cc5f22..1f103d8cba 100644 --- a/pypeit/spectrographs/magellan_fire.py +++ b/pypeit/spectrographs/magellan_fire.py @@ -119,7 +119,7 @@ def get_detector_par(self, det, hdu=None): specflip = True, spatflip = False, platescale = 0.18, - darkcurr = 0.01, + darkcurr = 3.06, # e-/pixel/hour (=0.00085 e-/pixel/s) #saturation = 20000., # high gain is 20000 ADU, low gain is 32000 ADU saturation = 100000., # This is an arbitrary value. nonlinear = 1.0, # high gain mode, low gain is 0.875 @@ -203,6 +203,10 @@ def default_pypeit_par(cls): # place holder for telgrid file par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + # Coadding. I'm not sure what this should be for PRISM mode? + par['coadd1d']['wave_method'] = 'log10' + + return par def check_frame_type(self, ftype, fitstbl, exprng=None): @@ -362,7 +366,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.15, - darkcurr = 0.01, + darkcurr = 3.06, # e-/pixel/hour (=0.00085 e-/pixel/s) saturation = 320000., #32000 for low gain, I set to a higher value to keep data in K-band nonlinear = 0.875, mincounts = -1e10, diff --git a/pypeit/spectrographs/magellan_mage.py b/pypeit/spectrographs/magellan_mage.py index cc3f77f60d..fc8fdc9d1f 100644 --- a/pypeit/spectrographs/magellan_mage.py +++ b/pypeit/spectrographs/magellan_mage.py @@ -63,7 +63,7 @@ def get_detector_par(self, det, hdu=None): # plate scale in arcsec/pixel platescale = 0.3, # electrons/pixel/hour. From: http://www.lco.cl/telescopes-information/magellan/instruments/mage/the-mage-spectrograph-user-manual - darkcurr = 1.00, + darkcurr = 1.0, # e-/pixel/hour saturation = 65535., # CCD is linear to better than 0.5 per cent up to digital saturation (65,536 DN including bias) in the Fast readout mode. nonlinear = 0.99, @@ -94,7 +94,11 @@ def default_pypeit_par(cls): #par['calibrations']['biasframe']['useframe'] = 'overscan' # Wavelengths # 1D wavelength solution - par['calibrations']['wavelengths']['rms_threshold'] = 0.20 # Might be grating dependent.. + # The following is for 1x1 binning + par['calibrations']['wavelengths']['rms_threshold'] = 0.40 + par['calibrations']['wavelengths']['fwhm'] = 3.0 + par['calibrations']['wavelengths']['fwhm_fromlines'] = True + # par['calibrations']['wavelengths']['sigdetect'] = 5.0 par['calibrations']['wavelengths']['lamps'] = ['ThAr_MagE'] @@ -136,6 +140,11 @@ def default_pypeit_par(cls): par['calibrations']['arcframe']['exprng'] = [20, None] par['calibrations']['darkframe']['exprng'] = [20, None] par['scienceframe']['exprng'] = [20, None] + + # Coadding + par['coadd1d']['wave_method'] = 'log10' + + return par def init_meta(self): diff --git a/pypeit/spectrographs/mdm_modspec.py b/pypeit/spectrographs/mdm_modspec.py new file mode 100644 index 0000000000..68b1dbe2c6 --- /dev/null +++ b/pypeit/spectrographs/mdm_modspec.py @@ -0,0 +1,289 @@ +""" +Module for MDM/Modspec specific methods. + +.. include:: ../include/links.rst +""" + +import array as arr +import numpy as np + +from astropy.time import Time + +from pypeit import msgs +from pypeit import telescopes +from pypeit.core import framematch +from pypeit.spectrographs import spectrograph +from pypeit.core import parse +from pypeit.images import detector_container + +class MDMModspecEchelleSpectrograph(spectrograph.Spectrograph): + """ + Child to handle MDM Modspec Echelle instrument+detector + """ + ndet = 1 + name = 'mdm_modspec' + + telescope = telescopes.HiltnerTelescopePar() + + camera = 'Echelle' + header_name = 'Modspec' + pypeline = 'MultiSlit' + supported = True + comment = 'MDM Modspec spectrometer; Only 1200l/mm disperser (so far)' + + def get_detector_par(self, det, hdu=None): + """ + Return metadata for the selected detector. + + Args: + det (:obj:`int`): + 1-indexed detector number. + hdu (`astropy.io.fits.HDUList`_, optional): + The open fits file with the raw image of interest. If not + provided, frame-dependent parameters are set to a default. + + Returns: + :class:`~pypeit.images.detector_container.DetectorContainer`: + Object with the detector metadata. + """ + # Detector 1 + # See Echelle at 2.4m f/7.5 scale : http://mdm.kpno.noirlab.edu/mdm-ccds.html + gain = np.atleast_1d([1.3]) # Hardcoded in the header + ronoise = np.atleast_1d([7.90]) # Hardcoded in the header + + # Allowing hdu=None is only needed for the automated documentation. + # TODO: See if there's a better way to automatically create the detector + # table for the docs. + if hdu is None: + lenSpat = None + lenSpec = None + datasec = None + oscansec = None + binning = '1,1' # Most common use mode + else: + # length of spatial axis, including overscan. Horizontal axis of + # original .fits files + lenSpat = hdu[0].header['NAXIS1'] + # length of spectral axis. Vertical axis of original .fits files + lenSpec = hdu[0].header['NAXIS2'] + datasec = np.atleast_1d([f'[1:{lenSpec},1:3002]']) + oscansec = np.atleast_1d([f'[1:{lenSpec},308:{lenSpat}]']) + binning = self.compound_meta(self.get_headarr(hdu), 'binning') + + if binning != '1,1': + msgs.error("Not ready for any binning except 1x1; contact the developers") + + # Detector 1 continued + detector_dict = dict( + binning = binning, + det = 1, + dataext = 0, + specaxis = 0, # Native spectrum is along the x-axis + specflip = True, # Wavelength decreases as pixel number increases + spatflip = False, # Spatial position increases as pixel number increases + platescale = 0.28, # Arcsec / pixel + darkcurr = 0.0, # e-/pixel/hour + saturation = 65535., # 16-bit ADC + nonlinear = 0.97, # Linear to ~97% of saturation + mincounts = -1e10, + numamplifiers = 1, + gain = gain, # See above + ronoise = ronoise, # See above + # Data & Overscan Sections -- Edge tracing can handle slit edges + datasec = datasec, # See above + oscansec = oscansec # See above + ) + # Return + return detector_container.DetectorContainer(**detector_dict) + + @classmethod + + def default_pypeit_par(cls): + """ + Return the default parameters to use for this instrument. + + Returns: + :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by + all of ``PypeIt`` methods. + """ + par = super().default_pypeit_par() + + # Slit edge method + par['calibrations']['slitedges']['sync_predict'] = 'nearest' # Ignore PCA + par['calibrations']['slitedges']['bound_detector'] = True # Edges of slit fall off the detector, so assign edges of detector as the edges of the slit + + # Pixel flat method + par['calibrations']['pixelflatframe']['process']['combine'] = 'mean' + par['calibrations']['pixelflatframe']['process']['clip'] = True + par['calibrations']['pixelflatframe']['process']['comb_sigrej'] = 3.0 + par['calibrations']['pixelflatframe']['process']['n_lohi'] = [1, 1] + par['calibrations']['pixelflatframe']['process']['use_overscan'] = True + + # Wavelength calibration method + par['calibrations']['wavelengths']['method'] = 'full_template' + par['calibrations']['wavelengths']['lamps'] = ['ArI', 'XeI', 'NeI'] + par['calibrations']['wavelengths']['reid_arxiv'] = 'mdm_modspec_1200_5100.fits' + par['calibrations']['wavelengths']['sigdetect'] = 5.0 # Sigma threshold above fluctuations for arc-line detection + #par['calibrations']['wavelengths']['ech_fix_format'] = False + par['calibrations']['wavelengths']['n_final'] = 9 + + # Flat fielding + par['calibrations']['flatfield']['slit_illum_finecorr'] = False + + # Bias method + par['calibrations']['biasframe']['process']['overscan_method'] = 'median' + + # Arc method + par['calibrations']['arcframe']['process']['subtract_continuum'] = True + par['calibrations']['arcframe']['process']['clip'] = False + par['calibrations']['arcframe']['process']['combine'] = 'mean' + + # Tilt method + par['calibrations']['tiltframe']['process']['subtract_continuum'] = True + par['calibrations']['tiltframe']['process']['clip'] = False + par['calibrations']['tiltframe']['process']['combine'] = 'mean' + + + # Set the default exposure time ranges for the frame typing + par['calibrations']['biasframe']['exprng'] = [None, 0.001] + par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames + par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames + par['calibrations']['arcframe']['exprng'] = [None, None] # Long arc exposures on this telescope + par['scienceframe']['exprng'] = [10, 600] + + return par + + def init_meta(self): + """ + Define how metadata are derived from the spectrograph files. + + That is, this associates the ``PypeIt``-specific metadata keywords + with the instrument-specific header cards using :attr:`meta`. + """ + # see https://pypeit.readthedocs.io/en/latest/setup.html?highlight=decker#overview + self.meta = {} + # Required (core) + self.meta['ra'] = dict(ext=0, card='RA') + self.meta['dec'] = dict(ext=0, card='DEC') + self.meta['target'] = dict(ext=0, card='OBJECT') + self.meta['decker'] = dict(card=None, compound=True) + self.meta['binning'] = dict(card=None, compound=True) + + #self.meta['datasec'] = dict(ext=0, card='DATASEC') # possibly use local variable for this + self.meta['filter1'] = dict(ext=0, card='FILTER') + + + self.meta['mjd'] = dict(card=None, compound=True) + self.meta['exptime'] = dict(ext=0, card='EXPTIME') + self.meta['airmass'] = dict(ext=0, card='AIRMASS') + + # Extras for config and frametyping + self.meta['dispname'] = dict(card=None, compound=True) + self.meta['idname'] = dict(ext=0, card='IMAGETYP') + self.meta['cenwave'] = dict(card=None, compound=True, rtol=2.0) + + + # Lamps + self.meta['lampstat01'] = dict(ext=0, card='LAMPS') + self.meta['instrument'] = dict(ext=0, card='INSTRUME') + + # Mirror + self.meta['mirror'] = dict(ext=0, card='MIRROR') + + def compound_meta(self, headarr, meta_key): + """ + Methods to generate metadata requiring interpretation of the header + data, instead of simply reading the value of a header card. + + Args: + headarr (:obj:`list`): + List of `astropy.io.fits.Header`_ objects. + meta_key (:obj:`str`): + Metadata keyword to construct. + + Returns: + object: Metadata value read from the header(s). + """ + if meta_key == 'binning': + binspatial = headarr[0]['CCDBIN1'] + binspec = headarr[0]['CCDBIN2'] + return parse.binning2string(binspec, binspatial) + if meta_key == 'mjd': + return Time(headarr[0]['JD'], format='jd').mjd + if meta_key == 'decker': + return 'none' + if meta_key == 'dispname': + return '1200 l/mm' + if meta_key == 'cenwave': + return 5100.0 + else: + msgs.error("Not ready for this compound meta") + + def configuration_keys(self): + """ + Return the metadata keys that define a unique instrument + configuration. + + This list is used by :class:`~pypeit.metadata.PypeItMetaData` to + identify the unique configurations among the list of frames read + for a given reduction. + + Returns: + :obj:`list`: List of keywords of data pulled from file headers + and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` + object. + """ + return ['dispname', 'cenwave', 'filter1', 'binning'] + + def pypeit_file_keys(self): + """ + Define the list of keys to be output into a standard ``PypeIt`` file. + + Returns: + :obj:`list`: The list of keywords in the relevant + :class:`~pypeit.metadata.PypeItMetaData` instance to print to the + :ref:`pypeit_file`. + """ + return super().pypeit_file_keys() + ['slitwid'] + + def check_frame_type(self, ftype, fitstbl, exprng=None): + """ + Check for frames of the provided type. + + Args: + ftype (:obj:`str`): + Type of frame to check. Must be a valid frame type; see + frame-type :ref:`frame_type_defs`. + fitstbl (`astropy.table.Table`_): + The table with the metadata for one or more frames to check. + This table uses the Pypeit-specific metadata keywords, as defined + under def init_meta(self). + exprng (:obj:`list`, optional): + Range in the allowed exposure time for a frame of type + ``ftype``. See + :func:`pypeit.core.framematch.check_frame_exptime`. + + Returns: + `numpy.ndarray`_: Boolean array with the flags selecting the + exposures in ``fitstbl`` that are ``ftype`` type frames. + """ + good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) + if ftype in ['science']: # Standards and Sciences lumped together under 'science' + return good_exp & (fitstbl['idname'] == 'Object') & (fitstbl['mirror'] == 'OUT') + + if ftype == 'bias': + return good_exp & (fitstbl['idname'] == 'Bias') + + if ftype in ['arc','tilt']: # Lamps and Arcs + return good_exp & np.array([target in ['Comp','Object'] for target in fitstbl['idname']]) & (fitstbl['mirror'] == 'IN') + + if ftype in ['pixelflat']: # Internal Flats + return good_exp & (fitstbl['idname'] == 'Flat') & (fitstbl['mirror'] == 'IN') + + if ftype in ['illumflat', 'trace']: # Twilight Flats + return good_exp & (fitstbl['idname'] == 'Flat') & (fitstbl['mirror'] == 'OUT') + + msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) + + return np.zeros(len(fitstbl), dtype=bool) + \ No newline at end of file diff --git a/pypeit/spectrographs/mdm_osmos.py b/pypeit/spectrographs/mdm_osmos.py index f3f46795bb..8a3b220d35 100644 --- a/pypeit/spectrographs/mdm_osmos.py +++ b/pypeit/spectrographs/mdm_osmos.py @@ -19,7 +19,7 @@ class MDMOSMOSMDM4KSpectrograph(spectrograph.Spectrograph): """ ndet = 1 name = 'mdm_osmos_mdm4k' - telescope = telescopes.KPNOTelescopePar() + telescope = telescopes.HiltnerTelescopePar() camera = 'MDM4K' url = 'https://www.astronomy.ohio-state.edu/martini.10/osmos/' header_name = 'OSMOS' @@ -55,7 +55,7 @@ def get_detector_par(self, det, hdu=None): ysize = 1., platescale = 0.273, mincounts = -1e10, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 65535., nonlinear = 0.86, numamplifiers = 4, @@ -91,7 +91,7 @@ def default_pypeit_par(cls): par['calibrations']['wavelengths']['reid_arxiv'] = 'mdm_osmos_mdm4k.fits' par['calibrations']['wavelengths']['sigdetect'] = 10.0 # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 1] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['arcframe']['exprng'] = [None, None] # Long arc exposures on this telescope @@ -124,6 +124,8 @@ def init_meta(self): # Lamps self.meta['lampstat01'] = dict(ext=0, card='LAMPS') self.meta['instrument'] = dict(ext=0, card='INSTRUME') + # Mirror + self.meta['mirror'] = dict(ext=0, card='MIRROR') def compound_meta(self, headarr, meta_key): """ @@ -216,9 +218,13 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): if ftype in ['science', 'standard']: return good_exp & (fitstbl['idname'] == 'OBJECT') if ftype == 'bias': - return good_exp & (fitstbl['idname'] == 'zero') - if ftype in ['pixelflat', 'trace']: - return good_exp & (fitstbl['lampstat01'] == 'Flat') & (fitstbl['idname'] == 'FLAT') + return good_exp & (fitstbl['idname'] == 'Bias') + ####return good_exp & (fitstbl['idname'] == 'zero') + if ftype == 'pixelflat': #Internal Flats + return good_exp & (fitstbl['lampstat01'] == 'Flat') & (fitstbl['idname'] == 'FLAT') & (fitstbl['mirror'] == 'IN') + if ftype in ['trace', 'illumflat']: #Twilight Flats + return good_exp & (fitstbl['idname'] == 'FLAT') & (fitstbl['mirror'] == 'OUT') + if ftype in ['pinhole', 'dark']: # Don't type pinhole or dark frames return np.zeros(len(fitstbl), dtype=bool) diff --git a/pypeit/spectrographs/mmt_binospec.py b/pypeit/spectrographs/mmt_binospec.py index f3bc5d1983..5dfb82155a 100644 --- a/pypeit/spectrographs/mmt_binospec.py +++ b/pypeit/spectrographs/mmt_binospec.py @@ -58,7 +58,7 @@ def get_detector_par(self, det, hdu=None): ygap = 0., ysize = 1., platescale = 0.24, - darkcurr = 3.0, #ToDO: To Be update + darkcurr = 3.6, #e-/pixel/hour (=0.001 e-/pixel/s) -- pulled from the ETC saturation = 65535., nonlinear = 0.95, #ToDO: To Be update mincounts = -1e10, diff --git a/pypeit/spectrographs/mmt_bluechannel.py b/pypeit/spectrographs/mmt_bluechannel.py index 9980c8fed1..f6257758da 100644 --- a/pypeit/spectrographs/mmt_bluechannel.py +++ b/pypeit/spectrographs/mmt_bluechannel.py @@ -55,14 +55,14 @@ def get_detector_par(self, det, hdu=None): binning = '1,1' gain = None ronoise = None - darkcurr = None + darkcurr = None # e-/pixel/hour datasec = None oscansec = None else: binning = self.get_meta_value(self.get_headarr(hdu), 'binning') gain = np.atleast_1d(hdu[0].header['GAIN']) ronoise = np.atleast_1d(hdu[0].header['RDNOISE']) - darkcurr = hdu[0].header['DARKCUR'] + darkcurr = hdu[0].header['DARKCUR'] # units are e-/pixel/hour datasec = np.atleast_1d(hdu[0].header['DATASEC']) oscansec = np.atleast_1d(hdu[0].header['BIASSEC']) @@ -78,7 +78,7 @@ def get_detector_par(self, det, hdu=None): ygap = 0., ysize = 1., platescale = 0.3, - darkcurr = darkcurr, #header['DARKCUR'], + darkcurr = darkcurr, # units are e-/pixel/hour saturation = 65535., nonlinear = 0.95, # need to look up and update mincounts = -1e10, diff --git a/pypeit/spectrographs/mmt_mmirs.py b/pypeit/spectrographs/mmt_mmirs.py index 94705905ee..36ae51c027 100644 --- a/pypeit/spectrographs/mmt_mmirs.py +++ b/pypeit/spectrographs/mmt_mmirs.py @@ -123,7 +123,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.2012, - darkcurr = 0.01, + darkcurr = 36.0, # e-/pixel/hour (=0.01 e-/pixel/s) saturation = 700000., #155400., nonlinear = 1.0, mincounts = -1e10, diff --git a/pypeit/spectrographs/ntt_efosc2.py b/pypeit/spectrographs/ntt_efosc2.py index 81cbbb5d11..71c425c7db 100644 --- a/pypeit/spectrographs/ntt_efosc2.py +++ b/pypeit/spectrographs/ntt_efosc2.py @@ -101,6 +101,26 @@ def compound_meta(self, headarr, meta_key): else: msgs.error("Not ready for this compound meta") + def config_independent_frames(self): + """ + Define frame types that are independent of the fully defined + instrument configuration. + + This method returns a dictionary where the keys of the dictionary are + the list of configuration-independent frame types. The value of each + dictionary element can be set to one or more metadata keys that can + be used to assign each frame type to a given configuration group. See + :func:`~pypeit.metadata.PypeItMetaData.set_configurations` and how it + interprets the dictionary values, which can be None. + + Returns: + :obj:`dict`: Dictionary where the keys are the frame types that + are configuration-independent and the values are the metadata + keywords that can be used to assign the frames to a configuration + group. + """ + return {'bias': ['binning', 'datasec'], 'dark': ['binning', 'datasec']} + def configuration_keys(self): """ Return the metadata keys that define a unique instrument @@ -182,7 +202,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.12, # Manual 2.2 - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 65535, # Maual Table 8 nonlinear = 0.80, mincounts = -1e10, diff --git a/pypeit/spectrographs/p200_dbsp.py b/pypeit/spectrographs/p200_dbsp.py index e38255fcb5..caecea72f0 100644 --- a/pypeit/spectrographs/p200_dbsp.py +++ b/pypeit/spectrographs/p200_dbsp.py @@ -246,7 +246,7 @@ def get_detector_par(self, det: int, hdu: Optional[fits.HDUList] = None): specflip = True, spatflip = False, # check platescale = 0.389, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 65000., nonlinear = 62./65., mincounts = -1e10, # cross-check @@ -293,7 +293,7 @@ def default_pypeit_par(cls): # Do not flux calibrate par['fluxcalib'] = None # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 1] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['arcframe']['exprng'] = [None, 120] @@ -470,7 +470,7 @@ def get_detector_par(self, det: int, hdu: Optional[fits.HDUList] = None): specflip = False, spatflip = False, # check platescale = 0.293, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 45000., nonlinear = 40./45., mincounts = -1e10, # check @@ -516,7 +516,7 @@ def default_pypeit_par(cls): # Do not flux calibrate par['fluxcalib'] = None # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 1] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['arcframe']['exprng'] = [None, 120] diff --git a/pypeit/spectrographs/p200_tspec.py b/pypeit/spectrographs/p200_tspec.py index 2d579099a6..28da608420 100644 --- a/pypeit/spectrographs/p200_tspec.py +++ b/pypeit/spectrographs/p200_tspec.py @@ -133,7 +133,7 @@ def get_detector_par(self, det, hdu=None): specflip = True, spatflip=False, platescale = 0.37, - darkcurr = 0.085, + darkcurr = 306.0, # e-/pixel/hour (=0.085 e-/pixel/s) saturation = 28000, nonlinear = 0.9, mincounts = -1e10, @@ -160,7 +160,8 @@ def default_pypeit_par(cls): # 1D wavelength solution par['calibrations']['wavelengths']['rms_threshold'] = 0.3 par['calibrations']['wavelengths']['sigdetect']=5.0 - par['calibrations']['wavelengths']['fwhm']= 5.0 + par['calibrations']['wavelengths']['fwhm']= 2.9 # As measured in DevSuite + par['calibrations']['wavelengths']['fwhm_fromlines'] = True par['calibrations']['wavelengths']['n_final']= [3,4,4,4,4] par['calibrations']['wavelengths']['lamps'] = ['OH_NIRES'] par['calibrations']['wavelengths']['method'] = 'reidentify' @@ -217,6 +218,10 @@ def default_pypeit_par(cls): par['sensfunc']['polyorder'] = 8 par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + # Coadding + par['coadd1d']['wave_method'] = 'log10' + + return par def pypeit_file_keys(self): diff --git a/pypeit/spectrographs/shane_kast.py b/pypeit/spectrographs/shane_kast.py index 19b94656de..45d86f1282 100644 --- a/pypeit/spectrographs/shane_kast.py +++ b/pypeit/spectrographs/shane_kast.py @@ -48,7 +48,7 @@ def default_pypeit_par(cls): # Always correct for flexure, starting with default parameters par['flexure']['spec_method'] = 'boxcar' # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 1] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['pixelflatframe']['exprng'] = [0, None] @@ -57,6 +57,7 @@ def default_pypeit_par(cls): par['calibrations']['standardframe']['exprng'] = [1, 61] # par['scienceframe']['exprng'] = [61, None] + par['sensfunc']['IR']['telgridfile'] = 'TelFit_Lick_3100_11100_R10000.fits' return par def init_meta(self): @@ -237,7 +238,7 @@ def get_detector_par(self, det, hdu=None): xgap=0., ygap=0., ysize=1., - darkcurr=0.0, + darkcurr=0.0, # e-/pixel/hour # These are rows, columns on the raw frame, 1-indexed datasec=np.asarray(['[:, 1:1024]', '[:, 1025:2048]']), oscansec=np.asarray(['[:, 2050:2080]', '[:, 2081:2111]']), @@ -386,7 +387,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.43, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 65535., nonlinear = 0.76, mincounts = -1e10, @@ -460,7 +461,6 @@ def default_pypeit_par(cls): # TODO In case someone wants to use the IR algorithm for shane kast this is the telluric file. Note the IR # algorithm is not the default. par['sensfunc']['IR']['telgridfile'] = 'TelFit_Lick_3100_11100_R10000.fits' - return par def init_meta(self): @@ -544,6 +544,10 @@ def config_specific_par(self, scifile, inp_par=None): par['calibrations']['wavelengths']['reid_arxiv'] = 'shane_kast_red_300_7500.fits' # Add CdI par['calibrations']['wavelengths']['lamps'] = ['NeI', 'HgI', 'HeI', 'ArI', 'CdI'] + elif self.get_meta_value(scifile, 'dispname') == '600/7500': + par['calibrations']['wavelengths']['method'] = 'full_template' + par['calibrations']['wavelengths']['reid_arxiv'] = 'shane_kast_red_600_7500.fits' + par['calibrations']['wavelengths']['lamps'] = ['NeI', 'HgI', 'HeI', 'ArI', 'CdI'] elif self.get_meta_value(scifile, 'dispname') == '1200/5000': par['calibrations']['wavelengths']['method'] = 'full_template' par['calibrations']['wavelengths']['reid_arxiv'] = 'shane_kast_red_1200_5000.fits' @@ -554,7 +558,7 @@ def config_specific_par(self, scifile, inp_par=None): # Return return par - def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table): + def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table, log10_blaze_function=None): """ This routine is for performing instrument/disperser specific tweaks to standard stars so that sensitivity @@ -576,6 +580,8 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table) Table containing meta data that is slupred from the :class:`~pypeit.specobjs.SpecObjs` object. See :meth:`~pypeit.specobjs.SpecObjs.unpack_object` for the contents of this table. + log10_blaze_function: `numpy.ndarray`_ or None + Input blaze function to be tweaked, optional. Default=None. Returns ------- @@ -587,6 +593,8 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table) Output inverse variance of standard star counts (:obj:`float`, ``shape = (nspec,)``) gpm_out: `numpy.ndarray`_ Output good pixel mask for standard (:obj:`bool`, ``shape = (nspec,)``) + log10_blaze_function_out: `numpy.ndarray`_ or None + Output blaze function after being tweaked. """ # Could check the wavelenghts here to do something more robust to header/meta data issues @@ -594,10 +602,16 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table) # The blue edge and red edge of the detector have no throughput so mask by hand. edge_region= (wave_in < 5400.0) | (wave_in > 8785.0) gpm_out = gpm_in & np.logical_not(edge_region) + # TODO Is this correct? else: gpm_out = gpm_in - return wave_in, counts_in, counts_ivar_in, gpm_out + if log10_blaze_function is not None: + log10_blaze_function_out = log10_blaze_function * gpm_out + else: + log10_blaze_function_out = None + return wave_in, counts_in, counts_ivar_in, gpm_out, log10_blaze_function_out + @@ -641,7 +655,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.774, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 120000., # JFH adjusted to this level as the flat are otherwise saturated nonlinear = 0.76, mincounts = -1e10, @@ -670,6 +684,8 @@ def default_pypeit_par(cls): par['calibrations']['wavelengths']['rms_threshold'] = 0.20 par['calibrations']['wavelengths']['sigdetect'] = 5. par['calibrations']['wavelengths']['use_instr_flag'] = True + par['sensfunc']['IR']['telgridfile'] = 'TelFit_Lick_3100_11100_R10000.fits' + return par diff --git a/pypeit/spectrographs/slitmask.py b/pypeit/spectrographs/slitmask.py index 529e7ced36..47fa898b6b 100644 --- a/pypeit/spectrographs/slitmask.py +++ b/pypeit/spectrographs/slitmask.py @@ -102,7 +102,7 @@ class SlitMask: Attributes: corners (`numpy.ndarray`_): See above. - id (`numpy.ndarray`_): + slitid (`numpy.ndarray`_): See `slitid` above. mask (`numpy.ndarray`_): Mask bits selecting the type of slit. @@ -1028,20 +1028,18 @@ def load_keck_deimoslris(filename:str, instr:str): for index in indices: for cdim in ['X', 'Y']: slit_list.append(hdu['BluSlits'].data[f'slit{cdim}{index}']) - slitmask = SlitMask(numpy.array(slit_list).T.reshape(-1,4,2), - slitid=hdu['BluSlits'].data['dSlitId'], - align=hdu['DesiSlits'].data['slitTyp'][indx] == 'A', - science=hdu['DesiSlits'].data['slitTyp'][indx] == 'P', - onsky=numpy.array([hdu['DesiSlits'].data['slitRA'][indx], - hdu['DesiSlits'].data['slitDec'][indx], - hdu['DesiSlits'].data['slitLen'][indx], - hdu['DesiSlits'].data['slitWid'][indx], - slit_pas]).T, - objects=objects, - #object_names=hdu['ObjectCat'].data['OBJECT'], - mask_radec=(hdu['MaskDesign'].data['RA_PNT'][0], - hdu['MaskDesign'].data['DEC_PNT'][0]), - posx_pa=posx_pa) - # Return - return slitmask + + return SlitMask(numpy.array(slit_list).T.reshape(-1,4,2), + slitid=hdu['BluSlits'].data['dSlitId'], + align=hdu['DesiSlits'].data['slitTyp'][indx] == 'A', + science=hdu['DesiSlits'].data['slitTyp'][indx] == 'P', + onsky=numpy.array([hdu['DesiSlits'].data['slitRA'][indx], + hdu['DesiSlits'].data['slitDec'][indx], + hdu['DesiSlits'].data['slitLen'][indx], + hdu['DesiSlits'].data['slitWid'][indx], + slit_pas]).T, + objects=objects, + mask_radec=(hdu['MaskDesign'].data['RA_PNT'][0], + hdu['MaskDesign'].data['DEC_PNT'][0]), + posx_pa=posx_pa) diff --git a/pypeit/spectrographs/soar_goodman.py b/pypeit/spectrographs/soar_goodman.py index 58e5f97a8b..1c523674e5 100644 --- a/pypeit/spectrographs/soar_goodman.py +++ b/pypeit/spectrographs/soar_goodman.py @@ -251,7 +251,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.15, - darkcurr = 0.00008, # e-/s/pix + darkcurr = 0.0, # e-/pixel/hour saturation = 65535., nonlinear = 1.0, mincounts = -1e10, @@ -313,6 +313,7 @@ def default_pypeit_par(cls): par['calibrations']['wavelengths']['rms_threshold'] = 0.5 par['calibrations']['wavelengths']['sigdetect'] = 5. par['calibrations']['wavelengths']['fwhm']= 5.0 + par['calibrations']['flatfield']['slit_illum_finecorr'] = False #par['calibrations']['wavelengths']['n_first'] = 3 #par['calibrations']['wavelengths']['n_final'] = 5 @@ -321,7 +322,7 @@ def default_pypeit_par(cls): #par['calibrations']['wavelengths']['disp'] = 0.2 # Set the default exposure time ranges for the frame typing - #par['calibrations']['biasframe']['exprng'] = [None, 1] + #par['calibrations']['biasframe']['exprng'] = [None, 0.001] #par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames #par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['arcframe']['exprng'] = [None, 30] @@ -460,7 +461,7 @@ def get_detector_par(self, det, hdu=None): specflip=False, spatflip=False, platescale=0.15, - darkcurr=0.00008, # e-/s/pix + darkcurr=0.0, # e-/pixel/hour saturation=65535., nonlinear=1.0, mincounts=-1e10, @@ -518,7 +519,7 @@ def default_pypeit_par(cls): # par['calibrations']['wavelengths']['disp'] = 0.2 # Set the default exposure time ranges for the frame typing - # par['calibrations']['biasframe']['exprng'] = [None, 1] + # par['calibrations']['biasframe']['exprng'] = [None, 0.001] # par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames # par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['arcframe']['exprng'] = [None, 30] diff --git a/pypeit/spectrographs/spectrograph.py b/pypeit/spectrographs/spectrograph.py index 074c6b59a9..c6ebd35faf 100644 --- a/pypeit/spectrographs/spectrograph.py +++ b/pypeit/spectrographs/spectrograph.py @@ -293,6 +293,17 @@ def _check_telescope(self): raise TypeError('Telescope parameters must be one of those specified in' 'pypeit.telescopes.') + def check_spectrograph(self, filename): + """ + Check that the selected spectrograph is the correct one for the input data. + NOTE: Not defined for all the spectrographs. + + Args: + filename (:obj:`str`): File to use when determining if the input spectrograph is the correct one. + + """ + pass + def raw_is_transposed(self, detector_par): """ Check if raw image files are transposed with respect to the @@ -543,11 +554,6 @@ def bpm(self, filename, det, shape=None, msbias=None): """ Generate a default bad-pixel mask. - Even though they are both optional, either the precise shape for - the image (``shape``) or an example file that can be read to get - the shape (``filename`` using :func:`get_image_shape`) *must* be - provided. - Args: filename (:obj:`str`): An example file to use to get the image shape. Can be None. @@ -591,7 +597,7 @@ def list_detectors(self, mosaic=False): This is primarily used :func:`~pypeit.slittrace.average_maskdef_offset` to measure the mean offset between the measured and expected slit - locations. **This method is not defined for all spectrographs.** + locations. Detectors separated along the dispersion direction should be ordered along the first axis of the returned array. For example, Keck/DEIMOS @@ -616,7 +622,11 @@ def list_detectors(self, mosaic=False): the array is 2D, there are detectors separated along the dispersion axis. """ - return None + if mosaic and len(self.allowed_mosaics) == 0: + msgs.error(f'Spectrograph {self.name} does not have any defined detector mosaics.') + dets = self.allowed_mosaics if mosaic else range(1,self.ndet+1) + return np.array([self.get_det_name(det) for det in dets]) + def get_lamps(self, fitstbl): """ @@ -678,7 +688,7 @@ def configuration_keys(self): Returns: :obj:`list`: List of keywords of data pulled from file headers - and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` + and used to construct the :class:`~pypeit.metadata.PypeItMetaData` object. """ return ['dispname', 'dichroic', 'decker'] @@ -828,19 +838,11 @@ def vet_instrument(self, meta_tbl): f'expected one! Found {instr_names[0]}, expected {self.header_name}. ' 'You may have chosen the wrong PypeIt spectrograph name!') - def config_independent_frames(self): """ Define frame types that are independent of the fully defined instrument configuration. - By default, bias and dark frames are considered independent of a - configuration; however, at the moment, these frames can only be - associated with a *single* configuration. That is, you cannot take - afternoon biases, change the instrument configuration during the - night, and then use the same biases for both configurations. See - :func:`~pypeit.metadata.PypeItMetaData.set_configurations`. - This method returns a dictionary where the keys of the dictionary are the list of configuration-independent frame types. The value of each dictionary element can be set to one or more metadata keys that can @@ -854,7 +856,7 @@ def config_independent_frames(self): keywords that can be used to assign the frames to a configuration group. """ - return {'bias': None, 'dark': None} + return {'bias': 'binning', 'dark': 'binning'} def get_comb_group(self, fitstbl): """ @@ -1062,16 +1064,20 @@ def select_detectors(self, subset=None): if subset is None: return np.arange(1, self.ndet+1).tolist() - if isinstance(subset, str): - _subset = parse.parse_slitspatnum(subset)[0].tolist() + # Parse subset if it's a string (single slitspatnum) or a list of slitspatnums + if isinstance(subset, str) or \ + (isinstance(subset, list) and np.all([isinstance(ss, str) for ss in subset]) + and np.all([':' in ss for ss in subset])): + subset_list = [subset] if isinstance(subset, str) else subset # Convert detector to int/tuple new_dets = [] - for item in _subset: - if 'DET' in item: - idx = np.where(self.list_detectors() == item)[0][0] + for ss in subset_list: + parsed_det = parse.parse_slitspatnum(ss)[0][0] + if 'DET' in parsed_det: + idx = np.where(self.list_detectors().flatten() == parsed_det)[0][0] new_dets.append(idx+1) - elif 'MSC' in item: - idx = np.where(self.list_detectors(mosaic=True) == item)[0][0] + elif 'MSC' in parsed_det: + idx = np.where(self.list_detectors(mosaic=True) == parsed_det)[0][0] new_dets.append(self.allowed_mosaics[idx]) _subset = new_dets elif isinstance(subset, (int, tuple)): @@ -1477,7 +1483,7 @@ def get_wcs(self, hdr, slits, platescale, wave0, dwv, spatial_scale=None): the platescale will be used. Returns: - `astropy.wcs.wcs.WCS`_: The world-coordinate system. + `astropy.wcs.WCS`_: The world-coordinate system. """ msgs.warn("No WCS setup for spectrograph: {0:s}".format(self.name)) return None @@ -1819,7 +1825,7 @@ def spec1d_match_spectra(self, sobjs): msgs.error(f'Method to match slits across detectors not defined for {self.name}') - def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table): + def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table, log10_blaze_function=None): """ This routine is for performing instrument/disperser specific tweaks to standard stars so that sensitivity @@ -1841,6 +1847,8 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table) Table containing meta data that is slupred from the :class:`~pypeit.specobjs.SpecObjs` object. See :meth:`~pypeit.specobjs.SpecObjs.unpack_object` for the contents of this table. + log10_blaze_function: `numpy.ndarray`_ or None + Input blaze function to be tweaked, optional. Default=None. Returns ------- @@ -1852,8 +1860,10 @@ def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table) Output inverse variance of standard star counts (:obj:`float`, ``shape = (nspec,)``) gpm_out: `numpy.ndarray`_ Output good pixel mask for standard (:obj:`bool`, ``shape = (nspec,)``) + log10_blaze_function_out: `numpy.ndarray`_ or None + Output blaze function after being tweaked. """ - return wave_in, counts_in, counts_ivar_in, gpm_in + return wave_in, counts_in, counts_ivar_in, gpm_in, log10_blaze_function def calc_pattern_freq(self, frame, rawdatasec_img, oscansec_img, hdu): """ diff --git a/pypeit/spectrographs/tng_dolores.py b/pypeit/spectrographs/tng_dolores.py index 23df646021..01d24a3276 100644 --- a/pypeit/spectrographs/tng_dolores.py +++ b/pypeit/spectrographs/tng_dolores.py @@ -54,7 +54,7 @@ def get_detector_par(self, det, hdu=None): ygap = 0., ysize = 1., platescale = 0.252, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 65500., nonlinear = 0.99, mincounts = -1e10, @@ -78,7 +78,7 @@ def default_pypeit_par(cls): par = super().default_pypeit_par() # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 0.1] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['scienceframe']['exprng'] = [1, None] diff --git a/pypeit/spectrographs/vlt_fors.py b/pypeit/spectrographs/vlt_fors.py index fbe909f7cf..54e4a9772f 100644 --- a/pypeit/spectrographs/vlt_fors.py +++ b/pypeit/spectrographs/vlt_fors.py @@ -241,7 +241,7 @@ def get_detector_par(self, det, hdu=None): # These numbers are from the ESO FORS2 user manual at: 0 # http://www.eso.org/sci/facilities/paranal/instruments/fors/doc/VLT-MAN-ESO-13100-1543_P01.1.pdf # They are for the MIT CCD (which is the default detector) for the high-gain, 100 khZ readout mode used for - # spectroscpy. The other readout modes are not yet implemented. The E2V detector is not yet supported!! + # spectroscopy. The other readout modes are not yet implemented. The E2V detector is not yet supported!! # CHIP1 detector_dict1 = dict( @@ -252,7 +252,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.126, # average between order 11 & 30, see manual - darkcurr = 2.1, + darkcurr = 2.1, # e-/pixel/hour saturation = 2.0e5, # I think saturation may never be a problem here since there are many DITs nonlinear = 0.80, mincounts = -1e10, @@ -272,7 +272,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.126, # average between order 11 & 30, see manual - darkcurr = 1.4, + darkcurr = 1.4, # e-/pixel/hour saturation = 2.0e5, # I think saturation may never be a problem here since there are many DITs nonlinear = 0.80, mincounts = -1e10, @@ -336,6 +336,26 @@ def config_specific_par(self, scifile, inp_par=None): return par + def config_independent_frames(self): + """ + Define frame types that are independent of the fully defined + instrument configuration. + + This method returns a dictionary where the keys of the dictionary are + the list of configuration-independent frame types. The value of each + dictionary element can be set to one or more metadata keys that can + be used to assign each frame type to a given configuration group. See + :func:`~pypeit.metadata.PypeItMetaData.set_configurations` and how it + interprets the dictionary values, which can be None. + + Returns: + :obj:`dict`: Dictionary where the keys are the frame types that + are configuration-independent and the values are the metadata + keywords that can be used to assign the frames to a configuration + group. + """ + return {'bias': 'detector', 'dark': 'detector'} + def configuration_keys(self): """ Return the metadata keys that define a unique instrument diff --git a/pypeit/spectrographs/vlt_sinfoni.py b/pypeit/spectrographs/vlt_sinfoni.py index 569e8affbe..5bfbcf855c 100644 --- a/pypeit/spectrographs/vlt_sinfoni.py +++ b/pypeit/spectrographs/vlt_sinfoni.py @@ -51,7 +51,7 @@ def get_detector_par(self, det, hdu=None): specflip = True, spatflip = False, platescale = 0.0125, - darkcurr = 0.15, + darkcurr = 540.0, # e-/pixel/hour (=0.15 e-/pixel/s) saturation = 1e9, # ADU, this is hacked for now nonlinear = 1.00, # docs say linear to 90,000 but our flats are usually higher numamplifiers = 1, diff --git a/pypeit/spectrographs/vlt_xshooter.py b/pypeit/spectrographs/vlt_xshooter.py index 39c8660908..c9f7aa9a42 100644 --- a/pypeit/spectrographs/vlt_xshooter.py +++ b/pypeit/spectrographs/vlt_xshooter.py @@ -83,6 +83,26 @@ def compound_meta(self, headarr, meta_key): return parse.binning2string(binspec, binspatial) msgs.error("Not ready for this compound meta") + def config_independent_frames(self): + """ + Define frame types that are independent of the fully defined + instrument configuration. + + This method returns a dictionary where the keys of the dictionary are + the list of configuration-independent frame types. The value of each + dictionary element can be set to one or more metadata keys that can + be used to assign each frame type to a given configuration group. See + :func:`~pypeit.metadata.PypeItMetaData.set_configurations` and how it + interprets the dictionary values, which can be None. + + Returns: + :obj:`dict`: Dictionary where the keys are the frame types that + are configuration-independent and the values are the metadata + keywords that can be used to assign the frames to a configuration + group. + """ + return {} + def configuration_keys(self): """ Return the metadata keys that define a unique instrument @@ -94,7 +114,7 @@ def configuration_keys(self): Returns: :obj:`list`: List of keywords of data pulled from file headers - and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` + and used to construct the :class:`~pypeit.metadata.PypeItMetaData` object. """ return ['arm'] @@ -211,7 +231,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.197, # average between order 11 & 30, see manual - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 2.0e5, # I think saturation may never be a problem here since there are many DITs nonlinear = 0.86, mincounts = -1e10, @@ -275,7 +295,7 @@ def default_pypeit_par(cls): # 1D wavelength solution par['calibrations']['wavelengths']['lamps'] = ['OH_XSHOOTER'] - par['calibrations']['wavelengths']['rms_threshold'] = 0.25 + par['calibrations']['wavelengths']['rms_threshold'] = 0.60 par['calibrations']['wavelengths']['sigdetect'] = 10.0 par['calibrations']['wavelengths']['fwhm'] = 5.0 par['calibrations']['wavelengths']['n_final'] = 4 @@ -290,6 +310,7 @@ def default_pypeit_par(cls): par['calibrations']['wavelengths']['ech_nspec_coeff'] = 5 par['calibrations']['wavelengths']['ech_norder_coeff'] = 5 par['calibrations']['wavelengths']['ech_sigrej'] = 3.0 + par['calibrations']['wavelengths']['qa_log'] = False # Flats #par['calibrations']['standardframe']['process']['illumflatten'] = False @@ -325,6 +346,10 @@ def default_pypeit_par(cls): par['sensfunc']['polyorder'] = 8 par['sensfunc']['IR']['telgridfile'] = 'TelFit_Paranal_NIR_9800_25000_R25000.fits' + # Coadding + par['coadd1d']['wave_method'] = 'log10' + + return par @@ -598,7 +623,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.16, # average from order 17 and order 30, see manual - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 65535., nonlinear = 0.86, mincounts = -1e10, @@ -657,15 +682,14 @@ def default_pypeit_par(cls): # 1D wavelength solution par['calibrations']['wavelengths']['lamps'] = ['ThAr_XSHOOTER_VIS'] # The following is for 1x1 binning. TODO GET BINNING SORTED OUT!! - par['calibrations']['wavelengths']['rms_threshold'] = 0.50 + par['calibrations']['wavelengths']['rms_threshold'] = 1.2 + par['calibrations']['wavelengths']['fwhm'] = 8.0 + par['calibrations']['wavelengths']['fwhm_fromlines'] = True + # par['calibrations']['wavelengths']['sigdetect'] = 5.0 par['calibrations']['wavelengths']['n_final'] = [3] + 13*[4] + [3] - # This is for 1x1 binning. Needs to be divided by binning for binned data!! - par['calibrations']['wavelengths']['fwhm'] = 11.0 # Reidentification parameters par['calibrations']['wavelengths']['method'] = 'reidentify' - # TODO: the arxived solution is for 1x1 binning. It needs to be - # generalized for different binning! par['calibrations']['wavelengths']['reid_arxiv'] = 'vlt_xshooter_vis1x1.fits' par['calibrations']['wavelengths']['cc_thresh'] = 0.50 par['calibrations']['wavelengths']['cc_local_thresh'] = 0.50 @@ -675,6 +699,8 @@ def default_pypeit_par(cls): par['calibrations']['wavelengths']['ech_nspec_coeff'] = 4 par['calibrations']['wavelengths']['ech_norder_coeff'] = 4 par['calibrations']['wavelengths']['ech_sigrej'] = 3.0 + par['calibrations']['wavelengths']['qa_log'] = True + # Flats par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.90 @@ -693,8 +719,12 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['polyorder'] = [9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7] + par['sensfunc']['polyorder'] = 8 #[9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7] par['sensfunc']['IR']['telgridfile'] = 'TelFit_Paranal_VIS_4900_11100_R25000.fits' + + # Coadding + par['coadd1d']['wave_method'] = 'log10' + return par def init_meta(self): @@ -880,7 +910,7 @@ def get_detector_par(self, det, hdu=None): specflip = True, spatflip = True, platescale = 0.161, # average from order 14 and order 24, see manual - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 65000., nonlinear = 0.86, mincounts = -1e10, @@ -924,7 +954,11 @@ def default_pypeit_par(cls): # 1D wavelength solution par['calibrations']['wavelengths']['lamps'] = ['ThAr_XSHOOTER_UVB'] par['calibrations']['wavelengths']['n_final'] = [3] + 10*[4] - par['calibrations']['wavelengths']['rms_threshold'] = 0.60 + # This is for 1x1 + par['calibrations']['wavelengths']['rms_threshold'] = 0.70 + par['calibrations']['wavelengths']['fwhm'] = 3.8 + par['calibrations']['wavelengths']['fwhm_fromlines'] = True + # par['calibrations']['wavelengths']['sigdetect'] = 3.0 # Pretty faint lines in places # Reidentification parameters par['calibrations']['wavelengths']['method'] = 'reidentify' @@ -938,6 +972,11 @@ def default_pypeit_par(cls): par['calibrations']['wavelengths']['cc_thresh'] = 0.50 par['calibrations']['wavelengths']['cc_local_thresh'] = 0.50 + par['calibrations']['wavelengths']['qa_log'] = True + + # Flats + par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.90 + par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.10 # Right now we are using the overscan and not biases becuase the # standards are read with a different read mode and we don't yet have @@ -963,6 +1002,21 @@ def default_pypeit_par(cls): par['reduce']['findobj']['maxnumber_sci'] = 2 # Assume that there is a max of 2 objects on the slit par['reduce']['findobj']['maxnumber_std'] = 1 # Assume that there is only one object on the slit. + # Coadding + par['coadd1d']['wave_method'] = 'log10' + + + # Sensitivity function parameters + par['sensfunc']['algorithm'] = 'IR' + par['sensfunc']['polyorder'] = 8 + par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + # This is a hack until we we have a Paranal file generated that covers the UVB wavelength range. + + # Coadding + par['coadd1d']['wave_method'] = 'log10' + + + return par def init_meta(self): diff --git a/pypeit/spectrographs/wht_isis.py b/pypeit/spectrographs/wht_isis.py index 8572609400..e107264ac3 100644 --- a/pypeit/spectrographs/wht_isis.py +++ b/pypeit/spectrographs/wht_isis.py @@ -154,7 +154,7 @@ def get_detector_par(self, det, hdu=None): specflip = False, spatflip = False, platescale = 0.20, - darkcurr = 0.0, + darkcurr = 0.0, # e-/pixel/hour saturation = 65535., nonlinear = 0.76, mincounts = -1e10, @@ -195,7 +195,7 @@ def default_pypeit_par(cls): # Do not flux calibrate par['fluxcalib'] = None # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 1] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['arcframe']['exprng'] = [None, 120] @@ -302,7 +302,7 @@ def get_detector_par(self, det, hdu=None): specflip=False, spatflip=False, platescale=0.22, - darkcurr=0.0, + darkcurr=0.0, # e-/pixel/hour saturation=65535., nonlinear=0.76, mincounts=-1e10, @@ -340,7 +340,7 @@ def default_pypeit_par(cls): # Do not flux calibrate par['fluxcalib'] = None # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 1] + par['calibrations']['biasframe']['exprng'] = [None, 0.001] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['arcframe']['exprng'] = [None, 120] diff --git a/pypeit/specutils/pypeit_loaders.py b/pypeit/specutils/pypeit_loaders.py index b7cd629c99..3cd8cee4c8 100644 --- a/pypeit/specutils/pypeit_loaders.py +++ b/pypeit/specutils/pypeit_loaders.py @@ -101,7 +101,7 @@ def pypeit_spec1d_loader(filename, extract=None, fluxed=True, **kwargs): Returns ------- - spec : SpectrumList + spec : `specutils.SpectrumList`_ Contains all spectra in the PypeIt spec1d file """ # Try to load the file and ignoring any version mismatch @@ -152,7 +152,7 @@ def pypeit_onespec_loader(filename, grid=False, **kwargs): Returns ------- - spec : Spectrum1D + spec : `specutils.Spectrum1D`_ Spectrum in the PypeIt OneSpec file """ # Try to load the file and ignoring any version mismatch diff --git a/pypeit/telescopes.py b/pypeit/telescopes.py index d899bbf460..e8bc8c3d3a 100644 --- a/pypeit/telescopes.py +++ b/pypeit/telescopes.py @@ -147,6 +147,16 @@ def __init__(self): eff_aperture=11.2) # KPNO from https://en.wikipedia.org/wiki/Nicholas_U._Mayall_Telescope +class HiltnerTelescopePar(TelescopePar): + def __init__(self): + loc = EarthLocation.of_site('mdm') + super(HiltnerTelescopePar, self).__init__(name='HILTNER', + longitude=loc.lon.to(units.deg).value, + latitude=loc.lat.to(units.deg).value, + elevation=loc.height.to(units.m).value, + diameter=2.4) +# See https://noirlab.edu/public/programs/kitt-peak-national-observatory/the-hiltner-24m-telescope/ + class MMTTelescopePar(TelescopePar): def __init__(self): loc = EarthLocation.of_site('Whipple Observatory') diff --git a/pypeit/tests/files/b24.fits.gz b/pypeit/tests/files/b24.fits.gz new file mode 100644 index 0000000000..5f7726aad7 Binary files /dev/null and b/pypeit/tests/files/b24.fits.gz differ diff --git a/pypeit/tests/files/shane_kast_blue.coadd1d b/pypeit/tests/files/shane_kast_blue.coadd1d deleted file mode 100644 index 8311e9a816..0000000000 --- a/pypeit/tests/files/shane_kast_blue.coadd1d +++ /dev/null @@ -1,10 +0,0 @@ -# User-defined coadding parameters -[coadd1d] - flux_value = False - -# Read in the data -coadd1d read -filename | obj_id -files/spec1d_b27.fits | SPAT0176-SLIT0175-DET01 -files/spec1d_b28.fits | SPAT0175-SLIT0175-DET01 -coadd1d end diff --git a/pypeit/tests/test_arcimage.py b/pypeit/tests/test_arcimage.py index 62fb1af3d0..4eb52f0717 100644 --- a/pypeit/tests/test_arcimage.py +++ b/pypeit/tests/test_arcimage.py @@ -57,4 +57,3 @@ def test_io(): ofile.unlink() - diff --git a/pypeit/tests/test_calibframe.py b/pypeit/tests/test_calibframe.py index d8e12db330..ebfa57a646 100644 --- a/pypeit/tests/test_calibframe.py +++ b/pypeit/tests/test_calibframe.py @@ -5,6 +5,8 @@ from IPython import embed +import numpy as np + import pytest from pypeit.pypmsgs import PypeItError @@ -56,7 +58,7 @@ def test_init(): calib.set_paths(odir, 'A', ['1','2'], 'DET01') ofile = Path(calib.get_path()).name - assert ofile == 'Minimal_A_1-2_DET01.fits', 'Wrong file name' + assert ofile == 'Minimal_A_1+2_DET01.fits', 'Wrong file name' def test_io(): @@ -97,7 +99,7 @@ def test_construct_calib_key(): key = CalibFrame.construct_calib_key('A', '1', 'DET01') assert key == 'A_1_DET01', 'Key changed' key = CalibFrame.construct_calib_key('A', ['1','2'], 'DET01') - assert key == 'A_1-2_DET01', 'Key changed' + assert key == 'A_1+2_DET01', 'Key changed' key = CalibFrame.construct_calib_key('A', 'all', 'DET01') assert key == 'A_all_DET01', 'Key changed' @@ -115,6 +117,31 @@ def test_ingest_calib_id(): 'Bad ingest' +def test_construct_calib_id(): + assert CalibFrame.construct_calib_id(['all']) == 'all', 'Construction should simply return all' + assert CalibFrame.construct_calib_id(['1']) == '1', \ + 'Construction with one calib_id should just return it' + calib_id = np.arange(10).tolist() + assert CalibFrame.construct_calib_id(calib_id) == '0+9', 'Bad simple construction' + # rng = np.random.default_rng(99) + # calib_id = np.unique(rng.integers(20, size=15)).tolist() + calib_id = [3, 5, 6, 10, 11, 12, 15, 18, 19] + assert CalibFrame.construct_calib_id(calib_id) == '3-5+6-10+12-15-18+19', \ + 'Bad complex construction' + + +def test_parse_calib_id(): + assert CalibFrame.parse_calib_id('all') == ['all'], 'Parsing should simply return all' + assert CalibFrame.parse_calib_id('1') == ['1'], 'Parsing should simply return all' + assert np.array_equal(CalibFrame.parse_calib_id('0+9'), np.arange(10).astype(str).tolist()), \ + 'Bad simple construction' + # rng = np.random.default_rng(99) + # calib_id = np.unique(rng.integers(20, size=15)).tolist() + calib_id = np.sort(np.array([3, 5, 6, 10, 11, 12, 15, 18, 19]).astype(str)) + assert np.array_equal(np.sort(CalibFrame.parse_calib_id('3-5+6-10+12-15-18+19')), calib_id), \ + 'Bad complex construction' + + def test_parse_key_dir(): calib = MinimalCalibFrame() odir = Path(data_path('')).resolve() diff --git a/pypeit/tests/test_collate_1d.py b/pypeit/tests/test_collate_1d.py index 4b023b42fe..d948ce260f 100644 --- a/pypeit/tests/test_collate_1d.py +++ b/pypeit/tests/test_collate_1d.py @@ -23,7 +23,7 @@ class MockSpecObj: - def __init__(self, MASKDEF_OBJNAME, MASKDEF_ID, DET, RA, DEC, SPAT_PIXPOS, NAME, WAVE_RMS, OPT_FLAM=None, OPT_COUNTS=None, BOX_COUNTS=None, VEL_CORR=None): + def __init__(self, MASKDEF_OBJNAME, MASKDEF_ID, DET, RA, DEC, SPAT_PIXPOS, NAME, WAVE_RMS, OPT_FLAM=None, OPT_COUNTS=None, BOX_COUNTS=None, OPT_MASK=None, BOX_MASK=None, VEL_CORR=None): self.MASKDEF_OBJNAME = MASKDEF_OBJNAME self.MASKDEF_ID = MASKDEF_ID self.DET = DetectorContainer.get_name(DET) @@ -34,6 +34,9 @@ def __init__(self, MASKDEF_OBJNAME, MASKDEF_ID, DET, RA, DEC, SPAT_PIXPOS, NAME, self.OPT_FLAM = OPT_FLAM self.OPT_COUNTS = OPT_COUNTS self.BOX_COUNTS = BOX_COUNTS + self.OPT_MASK = OPT_MASK + self.BOX_MASK = BOX_MASK + self.WAVE_RMS = WAVE_RMS self.VEL_CORR = VEL_CORR @@ -100,14 +103,14 @@ def __init__(self, file): # object4 also has boxcar counts and no opt_counts if file == "spec1d_file1": - self.specobjs = [MockSpecObj(MASKDEF_OBJNAME='object1', MASKDEF_ID='1001', DET=1, RA=201.1517, DEC=27.3246, SPAT_PIXPOS=1234.0, NAME='SPAT1234_SLIT1234_DET01', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100), BOX_COUNTS=np.zeros(100)), + self.specobjs = [MockSpecObj(MASKDEF_OBJNAME='object1', MASKDEF_ID='1001', DET=1, RA=201.1517, DEC=27.3246, SPAT_PIXPOS=1234.0, NAME='SPAT1234_SLIT1234_DET01', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_MASK=np.zeros(100, dtype=bool), OPT_FLAM=np.zeros(100), BOX_COUNTS=np.zeros(100)), MockSpecObj(MASKDEF_OBJNAME='SERENDIP', MASKDEF_ID='1001', DET=1, RA=201.1522, DEC=27.3250, SPAT_PIXPOS=1334.0, NAME='SPAT1334_SLIT1234_DET01', WAVE_RMS=0.02, VEL_CORR=2.0, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='object2', MASKDEF_ID='3002', DET=2, RA=201.0051, DEC=27.2228, SPAT_PIXPOS=5334.0, NAME='SPAT5334_SLIT4934_DET02', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=3, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3233.0, NAME='SPAT3233_SLIT3235_DET03', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100)), + MockSpecObj(MASKDEF_OBJNAME='object2', MASKDEF_ID='3002', DET=2, RA=201.0051, DEC=27.2228, SPAT_PIXPOS=5334.0, NAME='SPAT5334_SLIT4934_DET02', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_MASK=np.ones(100, dtype=bool), OPT_FLAM=np.zeros(100)), + MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=3, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3233.0, NAME='SPAT3233_SLIT3235_DET03', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_MASK=np.ones(100, dtype=bool), OPT_FLAM=np.zeros(100)), MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=3, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3232.0, NAME='SPAT3232_SLIT3235_DET03', WAVE_RMS=0.03), - MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=5, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3236.0, NAME='SPAT3236_SLIT3245_DET05', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='object1', MASKDEF_ID='1001', DET=7, RA=201.1517, DEC=27.3246, SPAT_PIXPOS=1233.0, NAME='SPAT1233_SLIT1235_DET07', WAVE_RMS=0.11, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100), BOX_COUNTS=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='SERENDIP', MASKDEF_ID='1001', DET=7, RA=201.1520, DEC=27.3249, SPAT_PIXPOS=1336.0, NAME='SPAT1336_SLIT1235_DET07', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100))] + MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=5, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3236.0, NAME='SPAT3236_SLIT3245_DET05', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_MASK=np.ones(100, dtype=bool), OPT_FLAM=np.zeros(100)), + MockSpecObj(MASKDEF_OBJNAME='object1', MASKDEF_ID='1001', DET=7, RA=201.1517, DEC=27.3246, SPAT_PIXPOS=1233.0, NAME='SPAT1233_SLIT1235_DET07', WAVE_RMS=0.11, OPT_COUNTS=np.zeros(100), OPT_MASK=np.ones(100, dtype=bool), OPT_FLAM=np.zeros(100), BOX_COUNTS=np.zeros(100), BOX_MASK=np.ones(100,dtype=bool)), + MockSpecObj(MASKDEF_OBJNAME='SERENDIP', MASKDEF_ID='1001', DET=7, RA=201.1520, DEC=27.3249, SPAT_PIXPOS=1336.0, NAME='SPAT1336_SLIT1235_DET07', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_MASK=np.ones(100, dtype=bool), OPT_FLAM=np.zeros(100))] elif file == "spec1d_file4": self.specobjs = [MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=3, RA=None, DEC=None, SPAT_PIXPOS=3234.0, NAME='SPAT3234_SLIT3236_DET03', WAVE_RMS=0.01, VEL_CORR=2.0, OPT_FLAM=np.zeros(100), OPT_COUNTS=np.zeros(100)), MockSpecObj(MASKDEF_OBJNAME='object4', MASKDEF_ID='4004', DET=3, RA=None, DEC=None, SPAT_PIXPOS=6250.0, NAME='SPAT6250_SLIT6235_DET03', WAVE_RMS=0.02, BOX_COUNTS=np.zeros(100)), @@ -115,11 +118,11 @@ def __init__(self, file): MockSpecObj(MASKDEF_OBJNAME='SERENDIP', MASKDEF_ID='4004', DET=5, RA=None, DEC=None, SPAT_PIXPOS=6934.0, NAME='SPAT6934_SLIT6245_DET05', WAVE_RMS=0.20, BOX_COUNTS=np.zeros(100)), MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=5, RA=None, DEC=None, SPAT_PIXPOS=3237.0, NAME='SPAT3237_SLIT3246_DET05', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100))] else: - self.specobjs = [MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=3, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3234.0, NAME='SPAT3234_SLIT3236_DET03', WAVE_RMS=0.01, OPT_FLAM=np.zeros(100), OPT_COUNTS=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='object4', MASKDEF_ID='4004', DET=3, RA=201.0052, DEC=27.2418, SPAT_PIXPOS=6250.0, NAME='SPAT6250_SLIT6235_DET03', WAVE_RMS=0.02, BOX_COUNTS=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='object4', MASKDEF_ID='4004', DET=5, RA=201.0052, DEC=27.2418, SPAT_PIXPOS=6256.0, NAME='SPAT6256_SLIT6245_DET05', WAVE_RMS=0.01, BOX_COUNTS=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='SERENDIP', MASKDEF_ID='4004', DET=5, RA=201.0056, DEC=27.2419, SPAT_PIXPOS=6934.0, NAME='SPAT6934_SLIT6245_DET05', WAVE_RMS=0.20, BOX_COUNTS=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=5, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3237.0, NAME='SPAT3237_SLIT3246_DET05', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100))] + self.specobjs = [MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=3, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3234.0, NAME='SPAT3234_SLIT3236_DET03', WAVE_RMS=0.01, OPT_FLAM=np.zeros(100), OPT_COUNTS=np.zeros(100), OPT_MASK=np.ones(100, dtype=bool)), + MockSpecObj(MASKDEF_OBJNAME='object4', MASKDEF_ID='4004', DET=3, RA=201.0052, DEC=27.2418, SPAT_PIXPOS=6250.0, NAME='SPAT6250_SLIT6235_DET03', WAVE_RMS=0.02, BOX_COUNTS=np.zeros(100), BOX_MASK=np.zeros(100, dtype=bool)), + MockSpecObj(MASKDEF_OBJNAME='object4', MASKDEF_ID='4004', DET=5, RA=201.0052, DEC=27.2418, SPAT_PIXPOS=6256.0, NAME='SPAT6256_SLIT6245_DET05', WAVE_RMS=0.01, BOX_COUNTS=np.zeros(100), BOX_MASK=np.ones(100, dtype=bool)), + MockSpecObj(MASKDEF_OBJNAME='SERENDIP', MASKDEF_ID='4004', DET=5, RA=201.0056, DEC=27.2419, SPAT_PIXPOS=6934.0, NAME='SPAT6934_SLIT6245_DET05', WAVE_RMS=0.20, BOX_COUNTS=np.zeros(100), BOX_MASK=np.ones(100, dtype=bool)), + MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=5, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3237.0, NAME='SPAT3237_SLIT3246_DET05', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), BOX_MASK=np.zeros(100, dtype=bool))] def __getitem__(self, idx): return self.specobjs[idx] @@ -127,14 +130,14 @@ def __getitem__(self, idx): def write_to_fits(self, *args, **kwargs): pass -def mock_specobjs(file): +def mock_specobjs(file, chk_version=False): return MockSpecObjs(file) def test_group_spectra_by_radec(monkeypatch): - monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_specobjs) file_list = ['spec1d_file1', 'spec1d_file2'] - uncollated_list = SourceObject.build_source_objects(file_list, 'ra/dec') + specobjs_list = [mock_specobjs(file) for file in file_list] + uncollated_list = SourceObject.build_source_objects(specobjs_list=specobjs_list, spec1d_files=file_list, match_type='ra/dec') source_list = collate_spectra_by_source(uncollated_list, 0.0003, u.deg) assert len(source_list) == 6 @@ -178,9 +181,10 @@ def test_group_spectra_by_pixel(monkeypatch): monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_specobjs) file_list = ['spec1d_file1', 'spec1d_file4'] - spectrograph = load_spectrograph('keck_deimos') + # Test matching by pixel and that unit argument is ignored - uncollated_list = SourceObject.build_source_objects(file_list, 'pixel') + specobjs_list = [mock_specobjs(file) for file in file_list] + uncollated_list = SourceObject.build_source_objects(specobjs_list=specobjs_list, spec1d_files=file_list, match_type='pixel') source_list = collate_spectra_by_source(uncollated_list, 5.0, u.deg) assert len(source_list) == 7 @@ -363,21 +367,34 @@ def test_exclude_source_objects(monkeypatch): monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_specobjs) file_list = ['spec1d_file1', 'spec1d_file2'] - uncollated_list = SourceObject.build_source_objects(file_list, 'ra/dec') + specobjs_list = [mock_specobjs(file) for file in file_list] + # Add another object to spec1d_file2 for the case where OPT_MASK is NONE + specobjs_list[1].specobjs.append(MockSpecObj(MASKDEF_OBJNAME='object2', + MASKDEF_ID='3002', + DET=2, + RA=201.0051, + DEC=27.2228, + SPAT_PIXPOS=5335.0, + NAME='SPAT5335_SLIT4934_DET02', + WAVE_RMS=0.01, + OPT_COUNTS=np.zeros(100), + OPT_FLAM=np.zeros(100))) + + uncollated_list = SourceObject.build_source_objects(specobjs_list=specobjs_list, spec1d_files=file_list, match_type='ra/dec') par = pypeitpar.PypeItPar() par['collate1d']['exclude_serendip'] = True par['collate1d']['wv_rms_thresh'] = 0.1 filtered_list, excluded_msgs = exclude_source_objects(uncollated_list, {'3003': 'Test Exclude`'}, par) - assert [so.spec_obj_list[0].NAME for so in filtered_list] == ['SPAT1234_SLIT1234_DET01', 'SPAT5334_SLIT4934_DET02'] - assert [so.spec1d_file_list[0] for so in filtered_list] == ['spec1d_file1', 'spec1d_file1'] + assert [so.spec_obj_list[0].NAME for so in filtered_list] == ['SPAT5334_SLIT4934_DET02'] + assert [so.spec1d_file_list[0] for so in filtered_list] == ['spec1d_file1'] par['collate1d']['exclude_serendip'] = False par['coadd1d']['ex_value'] = 'BOX' par['collate1d']['wv_rms_thresh'] = None filtered_list, excluded_msgs = exclude_source_objects(uncollated_list, dict(), par) - assert [so.spec_obj_list[0].NAME for so in filtered_list] == ['SPAT1234_SLIT1234_DET01', 'SPAT1233_SLIT1235_DET07', 'SPAT6250_SLIT6235_DET03', 'SPAT6256_SLIT6245_DET05', 'SPAT6934_SLIT6245_DET05'] - assert [so.spec1d_file_list[0] for so in filtered_list] == ['spec1d_file1', 'spec1d_file1', 'spec1d_file2', 'spec1d_file2', 'spec1d_file2' ] + assert [so.spec_obj_list[0].NAME for so in filtered_list] == ['SPAT1233_SLIT1235_DET07', 'SPAT6256_SLIT6245_DET05', 'SPAT6934_SLIT6245_DET05'] + assert [so.spec1d_file_list[0] for so in filtered_list] == ['spec1d_file1', 'spec1d_file2', 'spec1d_file2' ] def test_find_spec2d_from_spec1d(tmp_path): @@ -456,7 +473,7 @@ def mock_get_header(file): else: return {"DISPNAME": "600ZD" } - def mock_get_flux_calib_instance(spec1d_files, sens_files, par): + def mock_flux_calib(spec1d_files, sens_files, par, chk_version=False): if spec1d_files[0] == "fail_flux.fits": raise PypeItError("Test failure") else: @@ -467,7 +484,7 @@ def mock_get_flux_calib_instance(spec1d_files, sens_files, par): # Test success with monkeypatch.context() as m: monkeypatch.setattr(fits, "getheader", mock_get_header) - monkeypatch.setattr(fluxcalibrate.FluxCalibrate, "get_instance", mock_get_flux_calib_instance) + monkeypatch.setattr(fluxcalibrate, "flux_calibrate", mock_flux_calib) spectrograph = load_spectrograph('keck_deimos') # This could break if we start supporting it diff --git a/pypeit/tests/test_fluxspec.py b/pypeit/tests/test_fluxspec.py index 4f2dc6a94f..9daa95c3f6 100644 --- a/pypeit/tests/test_fluxspec.py +++ b/pypeit/tests/test_fluxspec.py @@ -99,7 +99,7 @@ def mock_get_header(*args, **kwargs): return {"DISPNAME": "600ZD", "PYP_SPEC": "keck_deimos" } - def mock_get_flux_calib_instance(*args, **kwargs): + def mock_flux_calib(spec1d_files, sens_files, par): # The flux_calib caller doesn't use the output, it just # depends on the side effect of fluxing return None @@ -107,7 +107,7 @@ def mock_get_flux_calib_instance(*args, **kwargs): with monkeypatch.context() as m: monkeypatch.setattr(fits, "getheader", mock_get_header) - monkeypatch.setattr(fluxcalibrate.FluxCalibrate, "get_instance", mock_get_flux_calib_instance) + monkeypatch.setattr(fluxcalibrate, "flux_calibrate", mock_flux_calib) # Test with a flux file missing "flux end" @@ -195,7 +195,7 @@ def extinction_correction_tester(algorithm): sensobj = sensfunc.SensFunc.get_instance(spec1d_file, sens_file, par['sensfunc']) sensobj.wave = np.linspace(3000, 6000, 300).reshape((300, 1)) - sensobj.sens = sensobj.empty_sensfunc_table(*sensobj.wave.T.shape) + sensobj.sens = sensobj.empty_sensfunc_table(*sensobj.wave.T.shape, 0) # make the zeropoint such that the sensfunc is flat sensobj.zeropoint = 30 - np.log10(sensobj.wave ** 2) / 0.4 @@ -203,7 +203,7 @@ def extinction_correction_tester(algorithm): # now flux our N_lam = 1 specobj par['fluxcalib']['extinct_correct'] = None - fluxCalibrate = fluxcalibrate.MultiSlitFC([spec1d_file], [sens_file], par=par['fluxcalib']) + fluxCalibrate = fluxcalibrate.flux_calibrate([spec1d_file], [sens_file], par=par['fluxcalib']) # without extinction correction, we should get constant F_lam # with extinction correction, the spectrum will be blue diff --git a/pypeit/tests/test_history.py b/pypeit/tests/test_history.py index 8093bebfda..f993641a74 100644 --- a/pypeit/tests/test_history.py +++ b/pypeit/tests/test_history.py @@ -146,12 +146,28 @@ def test_add_coadd1d(monkeypatch): def mock_getheader(file, **kwargs): if file == "spec1d_file1.fits": return {"SEMESTER": "2021A", - "PROGID": "test_prog" } + "PROGID": "test_prog" , + 'MASKDEF_ID': 11111, + 'MASKDEF_OBJNAME': 55555} else: return dict() + def mock_fits_open(file,**kwargs): + objids = ['SPAT001-SLIT001-DET01', + 'SPAT101-SLIT002-DET02', + 'SPAT002-SLIT001-DET01', + 'SPAT003-SLIT001-DET01', + 'SPAT103-SLIT002-DET02', + 'SPAT113-SLIT002-DET03'] + hdul = fits.HDUList(fits.PrimaryHDU()) + for obj in objids: + hdul.append(fits.BinTableHDU(name=obj)) + return hdul + + with monkeypatch.context() as m: monkeypatch.setattr(fits, "getheader", mock_getheader) + monkeypatch.setattr(fits, "open", mock_fits_open) # First test when the spec1d_files and objids # match length wise @@ -175,8 +191,8 @@ def mock_getheader(file, **kwargs): expected_history = [' PypeIt Coadded 6 objects from 3 spec1d files', 'From "spec1d_file1.fits"', 'Semester: 2021A Program ID: test_prog', - 'SPAT001-SLIT001-DET01', - 'SPAT101-SLIT002-DET02', + 'SPAT001-SLIT001-DET01 11111 55555', + 'SPAT101-SLIT002-DET02 11111 55555', 'From "spec1d_file2.fits"', 'SPAT002-SLIT001-DET01', 'From "spec1d_file3.fits"', diff --git a/pypeit/tests/test_io.py b/pypeit/tests/test_io.py index 74a93da2c4..52a8ee44fe 100644 --- a/pypeit/tests/test_io.py +++ b/pypeit/tests/test_io.py @@ -42,6 +42,6 @@ def test_grab_rawfiles(): assert [str(f) for f in raw_files] == _raw_files, 'Bad list_of_files read' _raw_files = inputfiles.grab_rawfiles(raw_paths=[str(root)], extension='.fits.gz') - assert len(_raw_files) == 8, 'Found the wrong number of files' + assert len(_raw_files) == 9, 'Found the wrong number of files' assert all([str(root / f) in _raw_files for f in tbl['filename']]), 'Missing expected files' diff --git a/pypeit/tests/test_metadata.py b/pypeit/tests/test_metadata.py index 002691c712..2866459e97 100644 --- a/pypeit/tests/test_metadata.py +++ b/pypeit/tests/test_metadata.py @@ -1,5 +1,6 @@ from pathlib import Path import shutil +import os from IPython import embed @@ -8,7 +9,7 @@ import numpy as np #from pypeit.par.util import parse_pypeit_file -from pypeit.tests.tstutils import data_path +from pypeit.tests.tstutils import data_path, make_fake_fits_files from pypeit.metadata import PypeItMetaData from pypeit.spectrographs.util import load_spectrograph from pypeit.scripts.setup import Setup @@ -47,9 +48,13 @@ def test_read_combid(): files=pypeItFile.filenames, usrdata=pypeItFile.data, strict=False) - indx = pmd['filename'] == 'b27.fits.gz' - assert pmd['comb_id'][indx] == [1], 'Incorrect combination group ID' - assert pmd['comb_id'][np.where(~indx)[0]][0] == -1, 'Incorrect combination group ID' + b27_indx = pmd['filename'] == 'b27.fits.gz' + b24_indx = pmd['filename'] == 'b24.fits.gz' + assert pmd['comb_id'][b27_indx] > 0, 'Science file should have a combination group ID' + assert pmd['comb_id'][b24_indx] > 0, 'Standard file should have a combination group ID' + assert pmd['comb_id'][b27_indx] != pmd['comb_id'][b24_indx], 'Science and standard should not have same combination group ID' + no_combid_indx = np.logical_not(b27_indx | b24_indx) + assert pmd['comb_id'][np.where(no_combid_indx)[0]][0] == -1, 'Incorrect combination group ID' shutil.rmtree(config_dir) @@ -95,3 +100,21 @@ def test_setup_iter(): == PypeItMetaData.maximum_number_of_configurations(), \ 'Number of configuration identifiers changed' + +def test_multiple_setups(): + filelist = make_fake_fits_files() + spectrograph = load_spectrograph("shane_kast_blue") + # Set the metadata + fitstbl = PypeItMetaData(spectrograph, spectrograph.default_pypeit_par(), files=filelist, strict=True) + fitstbl.get_frame_types() + cfgs = fitstbl.unique_configurations() + fitstbl.set_configurations(configs=cfgs) + # Now do some checks + for ff in range(len(fitstbl)): + if fitstbl[ff]['frametype'] == 'bias': + assert(len(fitstbl[ff]['setup'].split(",")) == 2) # Two configurations for the bias frames + else: + assert (len(fitstbl[ff]['setup'].split(",")) == 1) # One configuration for everything else + # Remove the created files + for fil in filelist: + os.remove(fil) diff --git a/pypeit/tests/test_parse.py b/pypeit/tests/test_parse.py index 41496b939b..5563bb21af 100644 --- a/pypeit/tests/test_parse.py +++ b/pypeit/tests/test_parse.py @@ -1,8 +1,8 @@ """ Module to run tests on arparse """ +from IPython import embed import numpy as np -import pytest from pypeit.core import parse @@ -40,4 +40,19 @@ def test_str2list(): assert np.array_equal(parse.str2list('3,1:5,6', length=10), [1,2,3,4,6]) assert np.array_equal(parse.str2list('3,1:5,8:', length=10), [1,2,3,4,8,9]) + +def test_parse_slitspatnum(): + assert [x[0] for x in parse.parse_slitspatnum('DET01:224')] == ['DET01', 224], \ + 'Bad parsing for single pair' + assert [x[0] for x in parse.parse_slitspatnum(['DET01:224'])] == ['DET01', 224], \ + 'Bad parsing for single list pair' + assert [x.tolist() for x in parse.parse_slitspatnum('DET01:224,DET02:331')] \ + == [['DET01', 'DET02'], [224, 331]], 'Bad parsing of comma-separated pairs' + assert [x.tolist() for x in parse.parse_slitspatnum(['DET01:224,DET02:331'])] \ + == [['DET01', 'DET02'], [224, 331]], 'Bad parsing of comma-separated pairs list' + assert [x.tolist() for x in parse.parse_slitspatnum(['DET01:224', 'DET02:331'])] \ + == [['DET01', 'DET02'], [224, 331]], 'Bad parsing of list of pairs' + assert [x.tolist() for x in parse.parse_slitspatnum(['DET01:224,DET02:331', 'DET03:442'])] \ + == [['DET01', 'DET02', 'DET03'], [224, 331, 442]], 'Bad mixed parsing' + diff --git a/pypeit/tests/test_procimg.py b/pypeit/tests/test_procimg.py index d04fe98f35..d3a8f5ac98 100644 --- a/pypeit/tests/test_procimg.py +++ b/pypeit/tests/test_procimg.py @@ -143,5 +143,5 @@ def test_boxcar(): a = np.arange(100).reshape(10,10).astype(float) arep = procimg.boxcar_replicate(a, 2) assert np.array_equal(procimg.boxcar_average(arep, 2), a), 'Bad replicate/average' - assert np.array_equal(utils.rebin_evlist(arep, a.shape), a), 'Bad replicate/rebin' + assert np.array_equal(utils.rebinND(arep, a.shape), a), 'Bad replicate/rebin' diff --git a/pypeit/tests/test_pypeitpar.py b/pypeit/tests/test_pypeitpar.py index c6cdcae9e9..dfb71b59d2 100644 --- a/pypeit/tests/test_pypeitpar.py +++ b/pypeit/tests/test_pypeitpar.py @@ -8,6 +8,7 @@ import pytest from pypeit.par import pypeitpar +from pypeit.par import parset from pypeit.par import util from pypeit.spectrographs.util import load_spectrograph from pypeit.tests.tstutils import data_path @@ -154,7 +155,7 @@ def test_telescope(): pypeitpar.TelescopePar() def test_fail_badpar(): - p = load_spectrograph('gemini_gnirs').default_pypeit_par() + p = load_spectrograph('gemini_gnirs_echelle').default_pypeit_par() # Faults because there's no junk parameter cfg_lines = ['[calibrations]', '[[biasframe]]', '[[[process]]]', 'junk = True'] @@ -163,7 +164,7 @@ def test_fail_badpar(): merge_with=cfg_lines) # Once as list def test_fail_badlevel(): - p = load_spectrograph('gemini_gnirs').default_pypeit_par() + p = load_spectrograph('gemini_gnirs_echelle').default_pypeit_par() # Faults because process isn't at the right level (i.e., there's no # process parameter for CalibrationsPar) @@ -173,3 +174,26 @@ def test_fail_badlevel(): merge_with=(cfg_lines,)) #Once as tuple +def test_lists(): + # Initialise the parset + p = load_spectrograph('keck_kcwi').default_pypeit_par() + + # Test with a single element list + p['calibrations']['alignment']['locations'] = [0.5] + _p = pypeitpar.PypeItPar.from_cfg_lines(cfg_lines=p.to_config()) # Once as tuple + assert(isinstance(_p['calibrations']['alignment']['locations'], list)) + assert(len(_p['calibrations']['alignment']['locations']) == 1) + assert (_p['calibrations']['alignment']['locations'][0] == 0.5) + + # Test with a multi-element list + p['calibrations']['alignment']['locations'] = [0.0, 1.0] + _p = pypeitpar.PypeItPar.from_cfg_lines(cfg_lines=p.to_config()) # Once as tuple + assert(isinstance(_p['calibrations']['alignment']['locations'], list)) + assert(len(_p['calibrations']['alignment']['locations']) == 2) + assert (_p['calibrations']['alignment']['locations'][0] == 0.0) + assert (_p['calibrations']['alignment']['locations'][1] == 1.0) + + # Test something that should fail + with pytest.raises(TypeError): + p['calibrations']['alignment']['locations'] = 0.0 + _p = pypeitpar.PypeItPar.from_cfg_lines(cfg_lines=p.to_config()) # Once as tuple diff --git a/pypeit/tests/test_runpypeit.py b/pypeit/tests/test_runpypeit.py index f49c7b5437..eb1c157868 100644 --- a/pypeit/tests/test_runpypeit.py +++ b/pypeit/tests/test_runpypeit.py @@ -4,14 +4,17 @@ from pathlib import Path import shutil from IPython.terminal.embed import embed +import numpy as np import matplotlib matplotlib.use('agg') from pypeit.scripts.setup import Setup from pypeit.scripts.run_pypeit import RunPypeIt +from pypeit.scripts.sensfunc import SensFunc +from pypeit.scripts.flux_calib import FluxCalib from pypeit.tests.tstutils import data_path -from pypeit import specobjs +from pypeit import specobjs, sensfunc from pypeit.par import pypeitpar @@ -49,6 +52,40 @@ def test_run_pypeit(): spec1d_file = configdir / 'Science' / 'spec1d_b27-J1217p3905_KASTb_20150520T045733.560.fits' assert spec1d_file.exists(), 'spec1d file missing' + # .par file was written and loads + par_file = str(list(configdir.glob('shane_kast_blue*.par'))[0]) + par = pypeitpar.PypeItPar.from_cfg_file(par_file) + assert isinstance(par, pypeitpar.PypeItPar) + + # Now re-use those calibration files + pargs = RunPypeIt.parse_args([str(pyp_file), '-o', '-r', str(configdir)]) + RunPypeIt.main(pargs) + + # Generate a sensitivity function from the standard star spec1d + std_file = configdir / 'Science' / 'spec1d_b24-Feige66_KASTb_20150520T041246.960.fits' + assert std_file.exists(), 'std spec1d file missing' + + sens_file = outdir / "sens_shane_kast_blue.fits" + + pargs = SensFunc.parse_args(["--algorithm", "UVIS", "-o", str(sens_file), str(std_file)]) + SensFunc.main(pargs) + + # Assert sensfunc file exists and contains data + assert sens_file.exists(), "sensfunc file missing" + sf = sensfunc.SensFunc.from_file(str(sens_file)) + assert len(sf.wave) > 0 + assert len(sf.zeropoint) > 0 + + # Flux calibrate + with open(outdir / "fluxfile", "w") as f: + print("flux read", file=f) + print("filename | sensfile", file=f) + print(f"{spec1d_file} | {sens_file}", file=f) + print("flux end", file=f) + pargs = FluxCalib.parse_args([str(outdir / "fluxfile")]) + FluxCalib.main(pargs) + + # Verify spec 1d # spec1d specObjs = specobjs.SpecObjs.from_fitsfile(spec1d_file) @@ -61,14 +98,20 @@ def test_run_pypeit(): # Helio assert abs(specObjs[0].VEL_CORR - 0.9999251762866389) < 1.0E-10 - # .par file was written and loads - par_file = str(list(configdir.glob('shane_kast_blue*.par'))[0]) - par = pypeitpar.PypeItPar.from_cfg_file(par_file) - assert isinstance(par, pypeitpar.PypeItPar) - - # Now re-use those calibration files - pargs = RunPypeIt.parse_args([str(pyp_file), '-o', '-r', str(configdir)]) - RunPypeIt.main(pargs) + # Flux + mask = specObjs[0].OPT_MASK + assert len(np.nonzero(mask)) != 0 + + assert len(specObjs[0].OPT_FLAM[mask]) != 0 + assert len(specObjs[0].OPT_FLAM_IVAR[mask]) != 0 + assert len(specObjs[0].OPT_FLAM_SIG[mask]) != 0 + + mask = specObjs[0].BOX_MASK + assert len(np.nonzero(mask)) != 0 + + assert len(specObjs[0].BOX_FLAM[mask]) != 0 + assert len(specObjs[0].BOX_FLAM_IVAR[mask]) != 0 + assert len(specObjs[0].BOX_FLAM_SIG[mask]) != 0 # Clean-up shutil.rmtree(outdir) diff --git a/pypeit/tests/test_setups.py b/pypeit/tests/test_setups.py index dcf075c3fb..89b1faf9c2 100644 --- a/pypeit/tests/test_setups.py +++ b/pypeit/tests/test_setups.py @@ -74,7 +74,7 @@ def test_setup_made_pypeit_file(): pypeItFile = PypeItFile.from_file(pypeit_file) # Test - assert len(pypeItFile.filenames) == 8 + assert len(pypeItFile.filenames) == 9 assert sorted(pypeItFile.frametypes['b1.fits.gz'].split(',')) == ['arc', 'tilt'] assert pypeItFile.setup_name == 'A' diff --git a/pypeit/tests/test_spectrographs.py b/pypeit/tests/test_spectrographs.py index 45d0617477..a1ce0bd5ff 100644 --- a/pypeit/tests/test_spectrographs.py +++ b/pypeit/tests/test_spectrographs.py @@ -62,6 +62,44 @@ def test_select_detectors_mosaic(): assert spec.select_detectors(subset=[3,(1,2,3)]) == [3,(1,2,3)], 'Bad detector selection' +def test_list_detectors_deimos(): + deimos = load_spectrograph('keck_deimos') + dets = deimos.list_detectors() + assert dets.ndim == 2, 'DEIMOS has a 2D array of detectors' + assert dets.size == 8, 'DEIMOS has 8 detectors' + mosaics = deimos.list_detectors(mosaic=True) + assert mosaics.ndim == 1, 'Mosaics are listed as 1D arrays' + assert mosaics.size == 4, 'DEIMOS has 4 predefined mosaics' + + +def test_list_detectors_mosfire(): + mosfire = load_spectrograph('keck_mosfire') + dets = mosfire.list_detectors() + assert dets.ndim == 1, 'MOSFIRE has a 1D array of detectors' + assert dets.size == 1, 'MOSFIRE has 1 detector' + with pytest.raises(PypeItError): + mosaics = mosfire.list_detectors(mosaic=True) + + +def test_list_detectors_mods(): + mods = load_spectrograph('lbt_mods1r') + dets = mods.list_detectors() + assert dets.ndim == 1, 'MODS1R has a 1D array of detectors' + assert dets.size == 1, 'MODS1R has 1 detector' + with pytest.raises(PypeItError): + mosaics = mods.list_detectors(mosaic=True) + + +def test_list_detectors_hires(): + hires = load_spectrograph('keck_hires') + dets = hires.list_detectors() + assert dets.ndim == 1, 'HIRES has a 1D array of detectors' + assert dets.size == 3, 'HIRES has 3 detectors' + mosaics = hires.list_detectors(mosaic=True) + assert mosaics.ndim == 1, 'Mosaics are listed as 1D arrays' + assert mosaics.size == 1, 'HIRES has 1 predefined mosaic' + + def test_configs(): spec = load_spectrograph('keck_deimos') diff --git a/pypeit/tests/test_wvcalib.py b/pypeit/tests/test_wvcalib.py index c1782bad61..4e85aef2a4 100644 --- a/pypeit/tests/test_wvcalib.py +++ b/pypeit/tests/test_wvcalib.py @@ -71,7 +71,8 @@ def test_wavecalib(): waveCalib = wavecalib.WaveCalib(wv_fits=np.asarray([waveFit]), nslits=1, spat_ids=np.asarray([232]), - wv_fit2d=np.array([pypeitFit2])) + wv_fit2d=np.array([pypeitFit2]), + fwhm_map=np.array([pypeitFit2])) waveCalib.set_paths(data_path(''), 'A', '1', 'DET01') ofile = Path(waveCalib.get_path()).resolve() @@ -88,6 +89,7 @@ def test_wavecalib(): assert np.array_equal(waveCalib.wv_fits[0].pypeitfit.fitc, waveCalib2.wv_fits[0].pypeitfit.fitc), 'Bad fitc' assert np.array_equal(waveCalib.wv_fit2d[0].xval, waveCalib2.wv_fit2d[0].xval) + assert np.array_equal(waveCalib.fwhm_map[0].xval, waveCalib2.fwhm_map[0].xval) # Write again! waveCalib2.to_file(overwrite=True) @@ -95,11 +97,11 @@ def test_wavecalib(): # Finish ofile.unlink() - # With None (failed wave) + # With None (failed wave) and no FWHM mapping spat_ids = np.asarray([232, 949]) waveCalib3 = wavecalib.WaveCalib(wv_fits=np.asarray([waveFit, wv_fitting.WaveFit(949)]), - nslits=2, spat_ids=spat_ids, - wv_fit2d=np.array([pypeitFit2, pypeitFit2])) + nslits=2, spat_ids=spat_ids, + wv_fit2d=np.array([pypeitFit2, pypeitFit2])) waveCalib3.set_paths(data_path(''), 'A', '1', 'DET01') waveCalib3.to_file(overwrite=True) waveCalib4 = wavecalib.WaveCalib.from_file(ofile) diff --git a/pypeit/tests/tstutils.py b/pypeit/tests/tstutils.py index 8605c76d5b..a6393a2c15 100644 --- a/pypeit/tests/tstutils.py +++ b/pypeit/tests/tstutils.py @@ -10,7 +10,8 @@ import numpy as np from astropy import time -from astropy.table import Table +from astropy.table import Table +import astropy.io.fits as fits from pypeit import data from pypeit.spectrographs.spectrograph import Spectrograph @@ -165,3 +166,50 @@ def make_shane_kast_blue_pypeitfile(): # Return return PypeItFile(confdict, file_paths, data, setup_dict) + + +def make_fake_fits_files(): + """ Generate some raw files covering multiple setups + """ + spectrograph = load_spectrograph("shane_kast_blue") + filelist = [] + setups = ['grismA', 'grismB'] # GRISM_N + nframes = dict({'bias':3, 'flat':2, 'arc':2, 'sci':1}) + # Make some bias frames (one setup only) + for frmtyp in nframes.keys(): + for ss, setup in enumerate(setups): + # Only have one set of bias frames, independent of setup + if frmtyp == 'bias' and ss != 0: + continue + # Loop over frame types + for ff in range(nframes[frmtyp]): + frname = f"{frmtyp}_{ff+1}_{setup}.fits" + filelist.append(frname) + hdu = fits.PrimaryHDU(np.zeros((2,2))) # Small fake image + for key in spectrograph.meta.keys(): + card = spectrograph.meta[key]['card'] + if key == 'exptime': + if frmtyp == 'bias': + value = 0.0 + elif frmtyp == 'sci': + value = 1800.0 + else: + value = 60.0 + elif key == 'mjd': + card, value = 'DATE', '2023-03-27T01:27:44.03' + elif key == 'binning': + continue + elif key == 'dispname': + value = setup + else: + value = 'None' + hdu.header[card] = value + if frmtyp == 'flat': + hdu.header['LAMPSTA1'] = 'on' + elif frmtyp == 'arc': + hdu.header['LAMPSTAC'] = 'on' + # Save the fake fits file to disk + hdu.writeto(frname, overwrite=True) + + # Return the filelist so that it can be later deleted + return filelist \ No newline at end of file diff --git a/pypeit/tracepca.py b/pypeit/tracepca.py index a77b6fe3a2..2f59ff6e69 100644 --- a/pypeit/tracepca.py +++ b/pypeit/tracepca.py @@ -29,9 +29,9 @@ class TracePCA(DataContainer): Class to build and interact with PCA model of traces. This is primarily a container class for the results of - :func:`pypeit.core.pca.pca_decomposition`, - :func:`pypeit.core.pca.fit_pca_coefficients`, and - :func:`pypeit.core.pca.pca_predict`. + :func:`~pypeit.core.pca.pca_decomposition`, + :func:`~pypeit.core.pca.fit_pca_coefficients`, and + :func:`~pypeit.core.pca.pca_predict`. The datamodel attributes are: @@ -45,13 +45,13 @@ class TracePCA(DataContainer): other keyword arguments are ignored. npca (:obj:`bool`, optional): The number of PCA components to keep. See - :func:`pypeit.core.pca.pca_decomposition`. + :func:`~pypeit.core.pca.pca_decomposition`. pca_explained_var (:obj:`float`, optional): The percentage (i.e., not the fraction) of the variance in the data accounted for by the PCA used to truncate the number of PCA coefficients to keep (see `npca`). Ignored if `npca` is provided directly. See - :func:`pypeit.core.pca.pca_decomposition`. + :func:`~pypeit.core.pca.pca_decomposition`. reference_row (:obj:`int`, optional): The row (spectral position) in `trace_cen` to use as the reference coordinate system for the PCA. If None, set to @@ -62,7 +62,7 @@ class TracePCA(DataContainer): trace},)`. If None, the reference coordinate system is defined by the value of `trace_cen` at the spectral position defined by `reference_row`. See the `mean` - argument of :func:`pypeit.core.pca.pca_decomposition`. + argument of :func:`~pypeit.core.pca.pca_decomposition`. """ version = '1.1.0' @@ -132,13 +132,13 @@ def decompose(self, trace_cen, npca=None, pca_explained_var=99.0, reference_row= N_{\rm trace})`. Cannot be None. npca (:obj:`bool`, optional): The number of PCA components to keep. See - :func:`pypeit.core.pca.pca_decomposition`. + :func:`~pypeit.core.pca.pca_decomposition`. pca_explained_var (:obj:`float`, optional): The percentage (i.e., not the fraction) of the variance in the data accounted for by the PCA used to truncate the number of PCA coefficients to keep (see `npca`). Ignored if `npca` is provided directly. See - :func:`pypeit.core.pca.pca_decomposition`. + :func:`~pypeit.core.pca.pca_decomposition`. reference_row (:obj:`int`, optional): The row (spectral position) in `trace_cen` to use as the reference coordinate system for the PCA. If None, @@ -150,7 +150,7 @@ def decompose(self, trace_cen, npca=None, pca_explained_var=99.0, reference_row= coordinate system is defined by the value of `trace_cen` at the spectral position defined by `reference_row`. See the `mean` argument of - :func:`pypeit.core.pca.pca_decomposition`. + :func:`~pypeit.core.pca.pca_decomposition`. """ if trace_cen is None: raise ValueError('Must provide traces to construct the PCA.') @@ -197,7 +197,7 @@ def _reinit(self): def build_interpolator(self, order, ivar=None, weights=None, function='polynomial', lower=3.0, upper=3.0, maxrej=1, maxiter=25, minx=None, maxx=None, debug=False): """ - Wrapper for :func:`fit_pca_coefficients` that uses class + Wrapper for :func:`~pypeit.core.pca.fit_pca_coefficients` that uses class attributes and saves the input parameters. """ if self.is_empty: @@ -260,7 +260,7 @@ def _parse(cls, hdu, hdu_prefix=None, **kwargs): """ Parse the data from the provided HDU. - See :func:`pypeit.datamodel.DataContainer._parse` for the + See :func:`~pypeit.datamodel.DataContainer._parse` for the argument descriptions. """ # Run the default parser to get most of the data @@ -288,7 +288,7 @@ def from_dict(cls, d=None): Instantiate from a dictionary. This is a basic wrapper for - :class:`pypeit.datamodel.DataContainer.from_dict` that + :class:`~pypeit.datamodel.DataContainer.from_dict` that appropriately toggles :attr:`is_empty`. """ self = super(TracePCA, cls).from_dict(d=d) @@ -334,13 +334,13 @@ def pca_trace_object(trace_cen, order=None, trace_bpm=None, min_length=0.6, npca decomposition. npca (:obj:`bool`, optional): The number of PCA components to keep. See - :func:`pypeit.core.pca.pca_decomposition`. + :func:`~pypeit.core.pca.pca_decomposition`. pca_explained_var (:obj:`float`, optional): The percentage (i.e., not the fraction) of the variance in the data accounted for by the PCA used to truncate the number of PCA coefficients to keep (see `npca`). Ignored if `npca` is provided directly. See - :func:`pypeit.core.pca.pca_decomposition`. + :func:`~pypeit.core.pca.pca_decomposition`. reference_row (:obj:`int`, optional): The row (spectral position) in `trace_cen` to use as the reference coordinate system for the PCA. If None, set to @@ -356,26 +356,26 @@ def pca_trace_object(trace_cen, order=None, trace_bpm=None, min_length=0.6, npca Minimum and maximum values used to rescale the independent axis data. If None, the minimum and maximum values of `coo` are used. See - :func:`utils.robust_polyfit_djs`. + :func:`~pypeit.core.fitting.robust_fit`. trace_wgt (`numpy.ndarray`_, optional): Weights to apply to the PCA coefficient of each trace during the fit. Weights are independent of the PCA component. See `weights` parameter of - :func:`pypeit.core.pca.fit_pca_coefficients`. Shape must + :func:`~pypeit.core.pca.fit_pca_coefficients`. Shape must be :math:`(N_{\rm trace},)`. function (:obj:`str`, optional): Type of function used to fit the data. lower (:obj:`float`, optional): Number of standard deviations used for rejecting data **below** the mean residual. If None, no rejection is - performed. See :func:`utils.robust_polyfit_djs`. + performed. See :func:`~pypeit.core.fitting.robust_fit`. upper (:obj:`float`, optional): Number of standard deviations used for rejecting data **above** the mean residual. If None, no rejection is - performed. See :func:`utils.robust_polyfit_djs`. + performed. See :func:`~pypeit.core.fitting.robust_fit`. maxrej (:obj:`int`, optional): Maximum number of points to reject during fit iterations. - See :func:`utils.robust_polyfit_djs`. + See :func:`~pypeit.core.fitting.robust_fit`. maxiter (:obj:`int`, optional): Maximum number of rejection iterations allows. To force no rejection iterations, set to 0. diff --git a/pypeit/utils.py b/pypeit/utils.py index 249adde001..994b50e837 100644 --- a/pypeit/utils.py +++ b/pypeit/utils.py @@ -11,6 +11,7 @@ import pathlib import itertools import glob +import colorsys import collections.abc from IPython import embed @@ -19,6 +20,7 @@ from numpy.lib.stride_tricks import as_strided import scipy.ndimage +from scipy import signal import matplotlib import matplotlib.pyplot as plt @@ -30,6 +32,342 @@ from pypeit.move_median import move_median +def zero_not_finite(array): + """ + Set the elements of an array to zero which are inf or nan + + Parameters + ---------- + array : `numpy.ndarray`_ + An numpy array of arbitrary shape that potentially has nans or + infinities. + + Returns + ------- + new_array : `numpy.ndarray`_ + A copy of the array with the nans and infinities set to zero. + """ + not_finite = np.logical_not(np.isfinite(array)) + new_array = array.copy() + new_array[not_finite] = 0.0 + return new_array + + +def arr_setup_to_setup_list(arr_setup): + """ + This utility routine converts an arr_setup list to a setup_list. The + arr_setup list and setup_lists are defined as follows, for e.g. echelle + wavelengths waves. See :func:`~pypeit.core.coadd.coadd1d.ech_combspec` for + further details. + + - ``arr_setup`` is a list of length nsetups, one for each setup. Each + element is a numpy array with ``shape = (nspec, norder, nexp)``, which + is the data model for echelle spectra for an individual setup. The + utiltities :func:`~pypeit.utils.arr_setup_to_setup_list` and + :func:`~pypeit.utils.setup_list_to_arr` convert between ``arr_setup`` + and ``setup_list``. + + - ``setup_list`` is a list of length ``nsetups``, one for each setup. + Each element is a list of length ``norder*nexp`` elements, each of + which contains the ``shape = (nspec1,)`` , e.g., wavelength arrays for + the order/exposure in ``setup1``. The list is arranged such that the + ``nexp1`` spectra for ``iorder=0`` appear first, then come ``nexp1`` + spectra for ``iorder=1``, i.e. the outer or fastest varying dimension + in python array ordering is the exposure number. The utility functions + :func:`~pypeit.utils.echarr_to_echlist` and + :func:`~pypeit.utils.echlist_to_echarr` convert between the + multi-dimensional numpy arrays in the ``arr_setup`` and the lists of + numpy arrays in ``setup_list``. + + Parameters + ---------- + arr_setup : :obj:`list` + A list of length nsetups echelle output arrays of shape=(nspec, norders, + nexp). + + Returns + ------- + setup_list : :obj:`list` + List of length nsetups. Each element of the setup list is a list of + length norder*nexp elements, each of which contains the shape = + (nspec1,) wavelength arrays for the order/exposure in setup1. The list + is arranged such that the nexp1 spectra for iorder=0 appear first, then + come nexp1 spectra for iorder=1, i.e. the outer or fastest varying + dimension in python array ordering is the exposure number. + """ + return [echarr_to_echlist(arr)[0] for arr in arr_setup] + + +def setup_list_to_arr_setup(setup_list, norders, nexps): + """ + This utility routine converts an setup_list list to an arr_setup list. The arr_setup list and setup_lists are defined + as follows, for e.g. echelle wavelengths waves. See core.coadd.coadd1d.ech_combspec for further details. + + - ``arr_setup`` is a list of length nsetups, one for each setup. Each + element is a numpy array with ``shape = (nspec, norder, nexp)``, which + is the data model for echelle spectra for an individual setup. The + utiltities :func:`~pypeit.utils.arr_setup_to_setup_list` and + :func:`~pypeit.utils.setup_list_to_arr` convert between ``arr_setup`` + and ``setup_list``. + + - ``setup_list`` is a list of length ``nsetups``, one for each setup. + Each element is a list of length ``norder*nexp`` elements, each of + which contains the ``shape = (nspec1,)`` , e.g., wavelength arrays for + the order/exposure in ``setup1``. The list is arranged such that the + ``nexp1`` spectra for ``iorder=0`` appear first, then come ``nexp1`` + spectra for ``iorder=1``, i.e. the outer or fastest varying dimension + in python array ordering is the exposure number. The utility functions + :func:`~pypeit.utils.echarr_to_echlist` and + :func:`~pypeit.utils.echlist_to_echarr` convert between the + multi-dimensional numpy arrays in the ``arr_setup`` and the lists of + numpy arrays in ``setup_list``. + + Parameters + ---------- + setup_list : :obj:`list` + List of length nsteups. Each element of the setup list is a list of + length norder*nexp elements, each of which contains the shape = + (nspec1,) wavelength arrays for the order/exposure in setup1. The list + is arranged such that the nexp1 spectra for iorder=0 appear first, then + come nexp1 spectra for iorder=1, i.e. the outer or fastest varying + dimension in python array ordering is the exposure number. + norders : :obj:`list` + List containing the number of orders for each setup. + nexps : :obj:`list` + List containing the number of exposures for each setup + + Returns + ------- + arr_setup : :obj:`list` + List of length nsetups each element of which is a numpy array of + shape=(nspec, norders, nexp) which is the echelle spectra data model. + """ + nsetups = len(setup_list) + arr_setup = [] + for isetup in range(nsetups): + shape = (setup_list[isetup][0].size, norders[isetup], nexps[isetup]) + arr_setup.append(echlist_to_echarr(setup_list[isetup], shape)) + return arr_setup + + +def concat_to_setup_list(concat, norders, nexps): + r""" + This routine converts from a ``concat`` list to a ``setup_list`` list. The + ``concat`` list and ``setup_lists`` are defined as follows. See + :func:`~pypeit.core.coadd.coadd1d.ech_combspec` for further details. + + - ``concat`` is a list of length :math:`\Sum_i N_{{\rm order},i} N_{{\rm + exp},i}` where :math:`i` runs over the setups. The elements of the + list contains a numpy array of, e.g., wavelengths for the setup, + order, exposure in question. The utility routines + :func:`~pypeit.utils.setup_list_to_concat` and + :func:`~pypeit.utils.concat_to_setup_list` convert between + ``setup_lists`` and ``concat``. + + - ``setup_list`` is a list of length ``nsetups``, one for each setup. + Each element is a list of length ``norder*nexp`` elements, each of + which contains the ``shape = (nspec1,)`` , e.g., wavelength arrays for + the order/exposure in ``setup1``. The list is arranged such that the + ``nexp1`` spectra for ``iorder=0`` appear first, then come ``nexp1`` + spectra for ``iorder=1``, i.e. the outer or fastest varying dimension + in python array ordering is the exposure number. The utility functions + :func:`~pypeit.utils.echarr_to_echlist` and + :func:`~pypeit.utils.echlist_to_echarr` convert between the + multi-dimensional numpy arrays in the ``arr_setup`` and the lists of + numpy arrays in ``setup_list``. + + Parameters + ---------- + concat : :obj:`list` + List of length :math:`\Sum_i N_{{\rm orders},i} N_{{\rm exp},i}` of + numpy arrays describing an echelle spectrum where :math:`i` runs over + the number of setups. + norders : :obj:`list` + List of length nsetups containing the number of orders for each setup. + nexps : :obj:`list` + List of length nexp containing the number of exposures for each setup. + + Parameters + ---------- + setup_list : :obj:`list`, list of length nsetups + Each element of the setup list is a list of length norder*nexp elements, + each of which contains the shape = (nspec1,) wavelength arrays for the + order/exposure in setup1. The list is arranged such that the nexp1 + spectra for iorder=0 appear first, then come nexp1 spectra for iorder=1, + i.e. the outer or fastest varying dimension in python array ordering is + the exposure number. + """ + if len(norders) != len(nexps): + msgs.error('The number of elements in norders and nexps must match') + nsetups = len(norders) + setup_list = [] + ind_start = 0 + for isetup in range(nsetups): + ind_end = ind_start + norders[isetup] * nexps[isetup] + setup_list.append(concat[ind_start:ind_end]) + ind_start = ind_end + + return setup_list + + +def setup_list_to_concat(lst): + """ + Unravel a list of lists. + + Parameters + ---------- + lst : :obj:`list` + List to unravel. + + Returns + ------- + concat_list : :obj:`list` + A list of the elements of the input list, unraveled. + """ + return list(itertools.chain.from_iterable(lst)) + + +def echarr_to_echlist(echarr): + """ + Convert an echelle array to a list of 1d arrays. + + Parameters + ---------- + echarr : `numpy.ndarray`_ + An echelle array of shape (nspec, norder, nexp). + + Returns + ------- + echlist : :obj:`list` + A unraveled list of 1d arrays of shape (nspec,) where the norder + dimension is the fastest varying dimension and the nexp dimension is the + slowest varying dimension. + shape : :obj:`tuple` + The shape of the provided echelle array (see ``echarr``). + """ + shape = echarr.shape + nspec, norder, nexp = shape + echlist = [echarr[:, i, j] for i in range(norder) for j in range(nexp)] + return echlist, shape + + +def echlist_to_echarr(echlist, shape): + """ + Convert a list of 1d arrays to a 3d echelle array in the format in which echelle outputs are stored, i.e. + with shape (nspec, norder, nexp). + + Parameters + ---------- + echlist : :obj:`list` + A unraveled list of 1d arrays of shape (nspec,) where the norder + dimension is the fastest varying dimension and the nexp dimension is the + slowest varying dimension. + shape : :obj:`tuple` + The shape of the echelle array to be returned, i.e. a tuple containing (nspec, norder, nexp) + + Returns + ------- + echarr : `numpy.ndarray`_ + An echelle spectral format array of shape (nspec, norder, nexp). + """ + nspec, norder, nexp = shape + echarr = np.zeros(shape, dtype=echlist[0].dtype) + for i in range(norder): + for j in range(nexp): + echarr[:, i, j] = echlist[i * nexp + j] + return echarr + + +def explist_to_array(explist, pad_value=0.0): + """ + Embed a list of length nexp 1d arrays of arbitrary size in a 2d array. + + Parameters + ---------- + explist : :obj:`list` + List of length nexp containing 1d arrays of arbitrary size. + pad_value : scalar-like + Value to use for padding the missing locations in the 2d array. The data + type should match the data type of in the 1d arrays in nexp_list. + + Returns + ------- + array : `numpy.ndarray`_ + A 2d array of shape (nspec_max, nexp) where nspec_max is the maximum + size of any of the members of the input nexp_list. The data type is the + same as the data type in the original 1d arrays. + """ + + nexp = len(explist) + # Find the maximum array size in the list + nspec_list = [arr.size for arr in explist] + nspec_max = np.max(nspec_list) + array = np.full((nspec_max, nexp), pad_value, dtype=explist[0].dtype) + for i in range(nexp): + array[:nspec_list[i], i] = explist[i] + + return array, nspec_list + + +def array_to_explist(array, nspec_list=None): + """ + Unfold a padded 2D array into a list of length nexp 1d arrays with sizes set by nspec_list + + Parameters + ---------- + array : `numpy.ndarray`_ + A 2d array of shape (nspec_max, nexp) where nspec_max is the maximum + size of any of the spectra in the array. + nspec_list : :obj:`list`, optional + List containing the size of each of the spectra embedded in the array. + If None, the routine will assume that all the spectra are the same size + equal to array.shape[0] + + Returns + ------- + explist : :obj:`list` + A list of 1d arrays of shape (nspec_max, nexp) where nspec_max is the + maximum size of any of the members of the input nexp_list. The data type + is the same as the data type in the original 1d arrays. + """ + nexp = array.shape[1] + if nspec_list is None: + _nspec_list = [array.shape[0]] * nexp + else: + _nspec_list = nspec_list + + explist = [] + for i in range(nexp): + explist.append(array[:_nspec_list[i], i]) + + return explist + + +def distinct_colors(num_colors): + """ + Return n distinct colors from the specified matplotlib colormap. Taken + from: + + https://stackoverflow.com/questions/470690/how-to-automatically-generate-n-distinct-colors + + Args: + num_colors (int): + Number of colors to return. + + Returns: + `numpy.ndarray`_: An array with shape (n,3) with the RGB values for + the requested number of colors. + """ + + colors = [] + for i in np.arange(0., 360., 360. / num_colors): + hue = i / 360. + lightness = (50 + np.random.rand() * 10) / 100. + saturation = (90 + np.random.rand() * 10) / 100. + colors.append(colorsys.hls_to_rgb(hue, lightness, saturation)) + return colors + + def get_time_string(codetime): """ Utility function that takes the codetime and @@ -75,7 +413,7 @@ def all_subclasses(cls): intermediate base classes in the inheritance thread. """ return set(cls.__subclasses__()).union( - [s for c in cls.__subclasses__() for s in all_subclasses(c)]) + [s for c in cls.__subclasses__() for s in all_subclasses(c)]) def embed_header(): @@ -128,8 +466,8 @@ def to_string(data, use_repr=True, verbatim=False): return data if not verbatim else '``' + data + '``' if hasattr(data, '__len__'): return '[]' if isinstance(data, list) and len(data) == 0 \ - else ', '.join([to_string(d, use_repr=use_repr, verbatim=verbatim) - for d in data ]) + else ', '.join([to_string(d, use_repr=use_repr, verbatim=verbatim) + for d in data]) return data.__repr__() if use_repr else str(data) @@ -164,28 +502,28 @@ def string_table(tbl, delimeter='print', has_header=True): _nrows += 1 start += 1 - row_string = ['']*_nrows + row_string = [''] * _nrows - for i in range(start,nrows+start-1): - row_string[i] = ' '.join([tbl[1+i-start,j].ljust(col_width[j]) for j in range(ncols)]) + for i in range(start, nrows + start - 1): + row_string[i] = ' '.join([tbl[1 + i - start, j].ljust(col_width[j]) for j in range(ncols)]) if delimeter == 'print': # Heading row - row_string[0] = ' '.join([tbl[0,j].ljust(col_width[j]) for j in range(ncols)]) + row_string[0] = ' '.join([tbl[0, j].ljust(col_width[j]) for j in range(ncols)]) # Delimiter if has_header: - row_string[1] = '-'*len(row_string[0]) - return '\n'.join(row_string)+'\n' + row_string[1] = '-' * len(row_string[0]) + return '\n'.join(row_string) + '\n' # For an rst table - row_string[0] = ' '.join([ '='*col_width[j] for j in range(ncols)]) - row_string[1] = ' '.join([tbl[0,j].ljust(col_width[j]) for j in range(ncols)]) + row_string[0] = ' '.join(['=' * col_width[j] for j in range(ncols)]) + row_string[1] = ' '.join([tbl[0, j].ljust(col_width[j]) for j in range(ncols)]) if has_header: row_string[2] = row_string[0] row_string[-1] = row_string[0] - return '\n'.join(row_string)+'\n' + return '\n'.join(row_string) + '\n' -def spec_atleast_2d(wave, flux, ivar, gpm, copy=False): +def spec_atleast_2d(wave, flux, ivar, gpm, log10_blaze_function=None, copy=False): """ Force spectral arrays to be 2D. @@ -209,9 +547,10 @@ def spec_atleast_2d(wave, flux, ivar, gpm, copy=False): forces the returned arrays to be copies instead. Returns: - :obj:`tuple`: Returns 6 objects. The first four are the reshaped - wavelength, flux, inverse variance, and gpm arrays. The next two - give the length of each spectrum and the total number of spectra; + :obj:`tuple`: Returns 7 objects. The first four are the reshaped + wavelength, flux, inverse variance, and gpm arrays. Next is the + log10_blaze_function, which is None if not provided as an input argument. + The next two give the length of each spectrum and the total number of spectra; i.e., the last two elements are identical to the shape of the returned flux array. @@ -228,8 +567,12 @@ def spec_atleast_2d(wave, flux, ivar, gpm, copy=False): if flux.ndim == 1: # Input flux is 1D # NOTE: These reshape calls return copies of the arrays - return wave.reshape(-1, 1), flux.reshape(-1, 1), ivar.reshape(-1, 1), \ - gpm.reshape(-1, 1), flux.size, 1 + if log10_blaze_function is not None: + return wave.reshape(-1, 1), flux.reshape(-1, 1), ivar.reshape(-1, 1), \ + gpm.reshape(-1, 1), log10_blaze_function.reshape(-1, 1), flux.size, 1 + else: + return wave.reshape(-1, 1), flux.reshape(-1, 1), ivar.reshape(-1, 1), \ + gpm.reshape(-1, 1), None, flux.size, 1 # Input is 2D nspec, norders = flux.shape @@ -237,7 +580,11 @@ def spec_atleast_2d(wave, flux, ivar, gpm, copy=False): _flux = flux.copy() if copy else flux _ivar = ivar.copy() if copy else ivar _gpm = gpm.copy() if copy else gpm - return _wave, _flux, _ivar, _gpm, nspec, norders + if log10_blaze_function is not None: + _log10_blaze_function = log10_blaze_function.copy() if copy else log10_blaze_function + else: + _log10_blaze_function = None + return _wave, _flux, _ivar, _gpm, _log10_blaze_function, nspec, norders def nan_mad_std(data, axis=None, func=None): @@ -250,15 +597,15 @@ def nan_mad_std(data, axis=None, func=None): Args: data (array-like): Data array or object that can be converted to an array. - axis (int, sequence of int, None, optional): + axis (int, tuple, optional): Axis along which the robust standard deviations are computed. The default (`None`) is to compute the robust standard deviation of the flattened array. Returns: - float, `numpy.ndarray`: The robust standard deviation of the + float, `numpy.ndarray`_: The robust standard deviation of the input data. If ``axis`` is `None` then a scalar will be - returned, otherwise a `~numpy.ndarray` will be returned. + returned, otherwise a `numpy.ndarray`_ will be returned. """ return stats.mad_std(data, axis=axis, func=func, ignore_nan=True) @@ -294,19 +641,19 @@ def growth_lim(a, lim, fac=1.0, midpoint=None, default=[0., 1.]): # Set the starting and ending values based on a fraction of the # growth _lim = 1.0 if lim > 1.0 else lim - start, end = (len(_a)*(1.0+_lim*np.array([-1,1]))/2).astype(int) + start, end = (len(_a) * (1.0 + _lim * np.array([-1, 1])) / 2).astype(int) if end == len(_a): end -= 1 # Set the full range and multiply it by the provided factor srt = np.ma.argsort(_a) - Da = (_a[srt[end]] - _a[srt[start]])*fac + Da = (_a[srt[end]] - _a[srt[start]]) * fac # Set the midpoint - mid = _a[srt[len(_a)//2]] if midpoint is None else midpoint + mid = _a[srt[len(_a) // 2]] if midpoint is None else midpoint # Return the range centered on the midpoint - return [ mid - Da/2, mid + Da/2 ] + return [mid - Da / 2, mid + Da / 2] def nearest_unmasked(arr, use_indices=False): @@ -344,7 +691,7 @@ def nearest_unmasked(arr, use_indices=False): return nearest_unmasked(np.ma.MaskedArray(np.arange(arr.size), mask=arr.mask.copy())) # Get the difference of each element with every other element - nearest = np.absolute(arr[None,:]-arr.data[:,None]) + nearest = np.absolute(arr[None, :] - arr.data[:, None]) # Ignore the diagonal nearest[np.diag_indices(arr.size)] = np.ma.masked # Return the location of the minimum value ignoring the masked values @@ -418,14 +765,14 @@ def boxcar_smooth_rows(img, nave, wgt=None, mode='nearest', replace='original'): # Construct the kernel for mean calculation _nave = np.fmin(nave, img.shape[0]) - kernel = np.ones((_nave, 1))/float(_nave) + kernel = np.ones((_nave, 1)) / float(_nave) if wgt is None: # No weights so just smooth return scipy.ndimage.convolve(img, kernel, mode='nearest') # Weighted smoothing - cimg = scipy.ndimage.convolve(img*wgt, kernel, mode='nearest') + cimg = scipy.ndimage.convolve(img * wgt, kernel, mode='nearest') wimg = scipy.ndimage.convolve(wgt, kernel, mode='nearest') smoothed_img = np.ma.divide(cimg, wimg) if replace == 'original': @@ -437,6 +784,82 @@ def boxcar_smooth_rows(img, nave, wgt=None, mode='nearest', replace='original'): return smoothed_img.data +def convolve_fft(img, kernel, msk): + """ + Convolve img with an input kernel using an FFT. Following the FFT, + a slower convolution is used to estimate the convolved image near + the masked pixels. + + .. note:: + For images following the PypeIt convention, this smooths the + data in the spectral direction for each spatial position. + + Args: + img (`numpy.ndarray`_): + Image to convolve, shape = (nspec, nspat) + kernel (`numpy.ndarray`_): + 1D kernel to use when convolving the image in the spectral direction + msk (`numpy.ndarray`_): + Mask of good pixels (True=good pixel). This should ideally be a slit mask, + where a True value represents a pixel on the slit, and a False value is a + pixel that is not on the slit. Image shape should be the same as img + + Returns: + `numpy.ndarray`_: The convolved image, same shape as the input img + """ + # Check the kernel shape + if kernel.ndim == 1: + kernel = kernel.reshape((kernel.size, 1)) + # Start by convolving the image by the kernel + img_conv = signal.fftconvolve(img, kernel, mode='same', axes=0) + # Find the slit edge pixels in the spectral direction + rmsk = (msk != np.roll(msk, 1, axis=0)) + # Remove the edges of the detector + rmsk[0, :] = False + rmsk[-1, :] = False + # Separate into up edges and down edges + wup = np.where(rmsk & msk) + wdn = np.where(rmsk & np.logical_not(msk)) + + # Setup some of the variables used in the slow convolution + nspec = img.shape[0] + kernsize = kernel.size + hwid = (kernsize - 1) // 2 + harr = np.arange(-hwid, +hwid + 1) + + # Create an inner function that deals with the convolution near masked pixels + def inner_conv(idx_i, idx_j): + slc = idx_i + harr + slc = slc[(slc >= 0) & (slc < nspec)] + wsl = np.where(msk[slc, idx_j]) + return np.sum(img[slc[wsl], idx_j] * kernel[wsl]) / np.sum(kernel[wsl]) + + # Now calculate the convolution directly for these pixels + for ee in range(wup[0].size): + ii, jj = wup[0][ee], wup[1][ee] + for cc in range(ii, min(nspec - 1, ii + hwid + 1)): + img_conv[cc, jj] = inner_conv(cc, jj) + for ee in range(wdn[0].size): + ii, jj = wdn[0][ee], wdn[1][ee] + for cc in range(max(0, ii - hwid - 1), ii + 1): + img_conv[cc, jj] = inner_conv(cc, jj) + # Now recalculate the convolution near the spectral edges + rmsk = np.zeros(msk.shape, dtype=bool) + rmsk[:hwid + 1, :] = True + wlo = np.where(msk & rmsk) + rmsk[:hwid + 1, :] = False + rmsk[-hwid - 1:, :] = True + whi = np.where(msk & rmsk) + for ee in range(wlo[0].size): + ii, jj = wlo[0][ee], wlo[1][ee] + img_conv[ii, jj] = inner_conv(ii, jj) + for ee in range(whi[0].size): + ii, jj = whi[0][ee], whi[1][ee] + img_conv[ii, jj] = inner_conv(ii, jj) + # Return the convolved image + return img_conv + + # TODO: Could this use bisect? def index_of_x_eq_y(x, y, strict=False): """ @@ -472,25 +895,26 @@ def index_of_x_eq_y(x, y, strict=False): return x2y -def rebin(a, newshape): +def rebin_slice(a, newshape): """ Rebin an array to a new shape using slicing. This routine is taken from: https://scipy-cookbook.readthedocs.io/items/Rebinning.html. The image shapes need not be integer multiples of each other, but in this regime the transformation will not be reversible, i.e. if - a_orig = rebin(rebin(a,newshape), a.shape) then a_orig will not be - everywhere equal to a (but it will be equal in most places). + a_orig = rebin_slice(rebin_slice(a,newshape), a.shape) then a_orig will not be + everywhere equal to a (but it will be equal in most places). To rebin and + conserve flux, use the `pypeit.utils.rebinND()` function (see below). Args: - a (ndarray, any dtype): + a (`numpy.ndarray`_): Image of any dimensionality and data type newshape (tuple): Shape of the new image desired. Dimensionality must be the same as a. Returns: - ndarray: same dtype as input Image with same values as a + `numpy.ndarray`_: same dtype as input Image with same values as a rebinning to shape newshape """ if not len(a.shape) == len(newshape): @@ -501,28 +925,41 @@ def rebin(a, newshape): indices = coordinates.astype('i') # choose the biggest smaller integer index return a[tuple(indices)] -# TODO This function is only used by procimg.lacosmic. Can it be replaced by above? -def rebin_evlist(frame, newshape): - # This appears to be from - # https://scipy-cookbook.readthedocs.io/items/Rebinning.html - shape = frame.shape - lenShape = len(shape) - factor = (np.asarray(shape)/np.asarray(newshape)).astype(int) - evList = ['frame.reshape('] + \ - ['int(newshape[%d]),factor[%d],'% (i, i) for i in range(lenShape)] + \ - [')'] + ['.sum(%d)' % (i+1) for i in range(lenShape)] + \ - ['/factor[%d]' % i for i in range(lenShape)] - return eval(''.join(evList)) +def rebinND(img, shape): + """ + Rebin a 2D image to a smaller shape. For example, if img.shape=(100,100), + then shape=(10,10) would take the mean of the first 10x10 pixels into a + single output pixel, then the mean of the next 10x10 pixels will be output + into the next pixel. Note that img.shape must be an integer multiple of the + elements in the new shape. + + Args: + img (`numpy.ndarray`_): + A 2D input image + shape (:obj:`tuple`): + The desired shape to be returned. The elements of img.shape + should be an integer multiple of the elements of shape. + Returns: + `numpy.ndarray`_: The input image rebinned to shape + """ + # First check that the old shape is an integer multiple of the new shape + rem0, rem1 = img.shape[0] % shape[0], img.shape[1] % shape[1] + if rem0 != 0 or rem1 != 0: + # In this case, the shapes are not an integer multiple... need to slice + msgs.warn("Input image shape is not an integer multiple of the requested shape. Flux is not conserved.") + return rebin_slice(img, shape) + # Convert input 2D image into a 4D array to make the rebinning easier + sh = shape[0], img.shape[0] // shape[0], shape[1], img.shape[1] // shape[1] + # Rebin to the 4D array and then average over the second and last elements. + img_out = img.reshape(sh).mean(-1).mean(1) + return img_out def pyplot_rcparams(): """ params for pretty matplotlib plots - - Returns: - """ # set some plotting parameters plt.rcParams["xtick.top"] = True @@ -552,9 +989,6 @@ def pyplot_rcparams(): def pyplot_rcparams_default(): """ restore default rcparams - - Returns: - """ matplotlib.rcParams.update(matplotlib.rcParamsDefault) @@ -572,13 +1006,17 @@ def smooth(x, window_len, window='flat'): the window parameter could be the window itself if an array instead of a string Args: - x: the input signal - window_len: the dimension of the smoothing window; should be an odd integer - window: the type of window from 'flat', 'hanning', 'hamming', 'bartlett', 'blackman' - flat window will produce a moving average smoothing., default is 'flat' + x (`numpy.ndarray`_): + the input signal + window_len (:obj:`int`): + the dimension of the smoothing window; should be an odd integer + window (:obj:`str`, optional): + the type of window from 'flat', 'hanning', 'hamming', 'bartlett', + 'blackman' flat window will produce a moving average smoothing. + Default is 'flat'. Returns: - the smoothed signal, same shape as x + `numpy.ndarray`_: the smoothed signal, same shape as x Examples: @@ -616,7 +1054,7 @@ def smooth(x, window_len, window='flat'): y = np.convolve(w / w.sum(), s, mode='same') - return y[(window_len-1):(y.size-(window_len-1))] + return y[(window_len - 1):(y.size - (window_len - 1))] def fast_running_median(seq, window_size): @@ -634,36 +1072,39 @@ def fast_running_median(seq, window_size): scipy.ndimage.median_filter with the reflect boundary condition, but is ~ 100 times faster. - Args: - seq (list or 1-d numpy array of numbers): - window_size (int): size of running window. - - Returns: - ndarray: median filtered values - Code originally contributed by Peter Otten, made to be consistent with scipy.ndimage.median_filter by Joe Hennawi. Now makes use of the Bottleneck library https://pypi.org/project/Bottleneck/. + + Args: + seq (list, `numpy.ndarray`_): + 1D array of values + window_size (int): + size of running window. + + Returns: + `numpy.ndarray`_: median filtered values """ # Enforce that the window_size needs to be smaller than the sequence, otherwise we get arrays of the wrong size # upon return (very bad). Added by JFH. Should we print out an error here? - if (window_size > (len(seq)-1)): + if (window_size > (len(seq) - 1)): msgs.warn('window_size > len(seq)-1. Truncating window_size to len(seq)-1, but something is probably wrong....') if (window_size < 0): - msgs.warn('window_size is negative. This does not make sense something is probably wrong. Setting window size to 1') + msgs.warn( + 'window_size is negative. This does not make sense something is probably wrong. Setting window size to 1') - window_size = int(np.fmax(np.fmin(int(window_size), len(seq)-1),1)) + window_size = int(np.fmax(np.fmin(int(window_size), len(seq) - 1), 1)) # pad the array for the reflection - seq_pad = np.concatenate((seq[0:window_size][::-1],seq,seq[-1:(-1-window_size):-1])) + seq_pad = np.concatenate((seq[0:window_size][::-1], seq, seq[-1:(-1 - window_size):-1])) result = move_median.move_median(seq_pad, window_size) # This takes care of the offset produced by the original code deducec by trial and error comparison with # scipy.ndimage.medfilt - result = np.roll(result, -window_size//2 + 1) + result = np.roll(result, -window_size // 2 + 1) return result[window_size:-window_size] @@ -680,24 +1121,24 @@ def cross_correlate(x, y, maxlag): Edges are padded with zeros using ``np.pad(mode='constant')``. - Args: - x (ndarray): - First vector of the cross-correlation. - y (ndarray): - Second vector of the cross-correlation. `x` and `y` must be - one-dimensional numpy arrays with the same length. - maxlag (int): - The maximum lag for which to compute the cross-correlation. - The cross correlation is computed at integer lags from - (-maxlag, maxlag) + Parameters + ---------- + x : `numpy.ndarray`_ + First vector of the cross-correlation. + y : `numpy.ndarray`_ + Second vector of the cross-correlation. `x` and `y` must be + one-dimensional numpy arrays with the same length. + maxlag : :obj:`int` + The maximum lag for which to compute the cross-correlation. The cross + correlation is computed at integer lags from (-maxlag, maxlag) - Returns: - tuple: Returns are as follows: - - lags (ndarray): shape = (2*maxlag + 1); Lags for the - cross-correlation. Integer spaced values from (-maxlag, - maxlag) - - xcorr (ndarray): shape = (2*maxlag + 1); Cross-correlation - at the lags + Returns + ------- + lags : `numpy.ndarray`_, shape = (2*maxlag + 1) + Lags for the cross-correlation. Integer spaced values from (-maxlag, + maxlag). + xcorr : `numpy.ndarray`_, shape = (2*maxlag + 1) + Cross-correlation at the lags """ x = np.asarray(x) @@ -707,13 +1148,12 @@ def cross_correlate(x, y, maxlag): if y.ndim != 1: msgs.error('y must be one-dimensional.') - - #py = np.pad(y.conj(), 2*maxlag, mode=mode) - py = np.pad(y, 2*maxlag, mode='constant') - T = as_strided(py[2*maxlag:], shape=(2*maxlag+1, len(y) + 2*maxlag), + # py = np.pad(y.conj(), 2*maxlag, mode=mode) + py = np.pad(y, 2 * maxlag, mode='constant') + T = as_strided(py[2 * maxlag:], shape=(2 * maxlag + 1, len(y) + 2 * maxlag), strides=(-py.strides[0], py.strides[0])) px = np.pad(x, maxlag, mode='constant') - lags = np.arange(-maxlag, maxlag + 1,dtype=float) + lags = np.arange(-maxlag, maxlag + 1, dtype=float) return lags, T.dot(px) @@ -754,8 +1194,8 @@ def clip_ivar(flux, ivar, sn_clip, gpm=None, verbose=False): _gpm = ivar > 0. if gpm is not None: _gpm &= gpm - adderr = 1.0/sn_clip - ivar_cap = _gpm/(1.0/(ivar + np.logical_not(_gpm)) + adderr**2*(np.abs(flux))**2) + adderr = 1.0 / sn_clip + ivar_cap = _gpm / (1.0 / (ivar + np.logical_not(_gpm)) + adderr ** 2 * (np.abs(flux)) ** 2) return np.minimum(ivar, ivar_cap) @@ -777,7 +1217,7 @@ def inverse(array): Returns: `numpy.ndarray`_: Result of controlled ``1/array`` calculation. """ - return (array > 0.0)/(np.abs(array) + (array == 0.0)) + return (array > 0.0) / (np.abs(array) + (array == 0.0)) def calc_ivar(varframe): @@ -788,45 +1228,34 @@ def calc_ivar(varframe): Wrapper to inverse() Args: - varframe (ndarray): Variance image + varframe (`numpy.ndarray`_): Variance image Returns: - ndarray: Inverse variance image + `numpy.ndarray`_: Inverse variance image """ # THIS WILL BE DEPRECATED!! return inverse(varframe) - - def robust_meanstd(array): """ Determine a robust measure of the mean and dispersion of array Args: - array (ndarray): an array of values + array (`numpy.ndarray`_): an array of values Returns: tuple: Median of the array and a robust estimate of the standand deviation (assuming a symmetric distribution). """ med = np.median(array) - mad = np.median(np.abs(array-med)) - return med, 1.4826*mad - + mad = np.median(np.abs(array - med)) + return med, 1.4826 * mad def polyfitter2d(data, mask=None, order=2): """ 2D fitter - - Args: - data: - mask: - order: - - Returns: - """ x, y = np.meshgrid(np.linspace(0.0, 1.0, data.shape[1]), np.linspace(0.0, 1.0, data.shape[0])) if isinstance(mask, (float, int)): @@ -861,21 +1290,12 @@ def polyfitter2d(data, mask=None, order=2): def polyfit2d(x, y, z, order=3): """ Generate 2D polynomial - - Args: - x: - y: - z: - order: - - Returns: - """ - ncols = (order + 1)**2 + ncols = (order + 1) ** 2 G = np.zeros((x.size, ncols)) - ij = itertools.product(range(order+1), range(order+1)) - for k, (i,j) in enumerate(ij): - G[:,k] = x**i * y**j + ij = itertools.product(range(order + 1), range(order + 1)) + for k, (i, j) in enumerate(ij): + G[:, k] = x ** i * y ** j m, null, null, null = np.linalg.lstsq(G, z) return m @@ -883,20 +1303,12 @@ def polyfit2d(x, y, z, order=3): def polyval2d(x, y, m): """ Generate 2D polynomial - - Args: - x: - y: - m: - - Returns: - """ order = int(np.sqrt(len(m))) - 1 - ij = itertools.product(range(order+1), range(order+1)) + ij = itertools.product(range(order + 1), range(order + 1)) z = np.zeros_like(x) for a, (i, j) in zip(m, ij): - z += a * x**i * y**j + z += a * x ** i * y ** j return z @@ -905,14 +1317,15 @@ def subsample(frame): Used by LACosmic Args: - frame (ndarray): + frame (`numpy.ndarray`_): + Array of data to subsample. Returns: - ndarray: Sliced image + `numpy.ndarray`_: Sliced image """ - newshape = (2*frame.shape[0], 2*frame.shape[1]) - slices = [slice(0, old, float(old)/new) for old, new in zip(frame.shape, newshape)] + newshape = (2 * frame.shape[0], 2 * frame.shape[1]) + slices = [slice(0, old, float(old) / new) for old, new in zip(frame.shape, newshape)] coordinates = np.mgrid[slices] indices = coordinates.astype('i') return frame[tuple(indices)] @@ -923,14 +1336,15 @@ def find_nearest(array, values): Parameters ---------- - array : numpy.ndarray + array : `numpy.ndarray`_ Array of values - values : numpy.ndarray + values : `numpy.ndarray`_ Values to be compared with the elements of `array` - Return - ------ - numpy.ndarray : indices of `array` that are closest to each element of value + Returns + ------- + idxs : `numpy.ndarray`_ + indices of ``array`` that are closest to each element of value """ # Make sure the input is a numpy array array = np.array(array) @@ -968,10 +1382,10 @@ def yamlify(obj, debug=False): Returns ------- - obj: :class:`object` + obj : :class:`object` An object suitable for yaml serialization. For example - :class:`numpy.ndarray` is converted to :class:`list`, - :class:`numpy.int64` is converted to :class:`int`, etc. + `numpy.ndarray`_ is converted to :class:`list`, + ``numpy.int64`` is converted to :class:`int`, etc. """ # TODO: Change to np.floating? if isinstance(obj, (np.float64, np.float32)): @@ -981,12 +1395,12 @@ def yamlify(obj, debug=False): obj = int(obj) elif isinstance(obj, np.bool_): obj = bool(obj) -# elif isinstance(obj, bytes): -# obj = obj.decode('utf-8') + # elif isinstance(obj, bytes): + # obj = obj.decode('utf-8') elif isinstance(obj, (np.string_, str)): # Worry about colons! if ':' in obj: - obj = '"'+str(obj)+'"' + obj = '"' + str(obj) + '"' else: obj = str(obj) elif isinstance(obj, units.Quantity): @@ -1128,7 +1542,7 @@ def load_pickle(fname): ## Python version taken from https://pythonhosted.org/pyDOE/randomized.html by JFH -def lhs(n, samples=None, criterion=None, iterations=None): +def lhs(n, samples=None, criterion=None, iterations=None, seed_or_rng=12345): """ Generate a latin-hypercube design @@ -1151,7 +1565,7 @@ def lhs(n, samples=None, criterion=None, iterations=None): Returns ------- - H : 2d-array + H : `numpy.ndarray`_ An n-by-samples design matrix that has been normalized so factor values are uniformly spaced between zero and one. @@ -1198,6 +1612,7 @@ def lhs(n, samples=None, criterion=None, iterations=None): >>> lhs(4, samples=5, criterion='correlate', iterations=10) """ + rng = np.random.default_rng(seed_or_rng) H = None if samples is None: @@ -1208,7 +1623,7 @@ def lhs(n, samples=None, criterion=None, iterations=None): 'centermaximin', 'cm', 'correlation', 'corr'), 'Invalid value for "criterion": {}'.format(criterion) else: - H = _lhsclassic(n, samples) + H = _lhsclassic(rng, n, samples) if criterion is None: criterion = 'center' @@ -1218,93 +1633,102 @@ def lhs(n, samples=None, criterion=None, iterations=None): if H is None: if criterion.lower() in ('center', 'c'): - H = _lhscentered(n, samples) + H = _lhscentered(rng, n, samples) elif criterion.lower() in ('maximin', 'm'): - H = _lhsmaximin(n, samples, iterations, 'maximin') + H = _lhsmaximin(rng, n, samples, iterations, 'maximin') elif criterion.lower() in ('centermaximin', 'cm'): - H = _lhsmaximin(n, samples, iterations, 'centermaximin') + H = _lhsmaximin(rng, n, samples, iterations, 'centermaximin') elif criterion.lower() in ('correlate', 'corr'): - H = _lhscorrelate(n, samples, iterations) + H = _lhscorrelate(rng, n, samples, iterations) return H + ################################################################################ -def _lhsclassic(n, samples): +def _lhsclassic(rng, n, samples): # Generate the intervals cut = np.linspace(0, 1, samples + 1) # Fill points uniformly in each interval - u = np.random.rand(samples, n) + u = rng.random((samples, n)) + # u = np.random.rand(samples, n) a = cut[:samples] b = cut[1:samples + 1] rdpoints = np.zeros_like(u) for j in range(n): - rdpoints[:, j] = u[:, j ] *( b -a) + a + rdpoints[:, j] = u[:, j] * (b - a) + a # Make the random pairings H = np.zeros_like(rdpoints) for j in range(n): - order = np.random.permutation(range(samples)) + order = rng.permutation(range(samples)) + # order = np.random.permutation(range(samples)) H[:, j] = rdpoints[order, j] return H + ################################################################################ -def _lhscentered(n, samples): +def _lhscentered(rng, n, samples): # Generate the intervals cut = np.linspace(0, 1, samples + 1) # Fill points uniformly in each interval - u = np.random.rand(samples, n) + u = rng.random((samples, n)) + # u = np.random.rand(samples, n) a = cut[:samples] b = cut[1:samples + 1] - _center = (a + b ) /2 + _center = (a + b) / 2 # Make the random pairings H = np.zeros_like(u) for j in range(n): - H[:, j] = np.random.permutation(_center) + H[:, j] = rng.permutation(_center) + # H[:, j] = np.random.permutation(_center) return H + ################################################################################ -def _lhsmaximin(n, samples, iterations, lhstype): +def _lhsmaximin(rng, n, samples, iterations, lhstype): maxdist = 0 # Maximize the minimum distance between points for i in range(iterations): - if lhstype=='maximin': - Hcandidate = _lhsclassic(n, samples) + if lhstype == 'maximin': + Hcandidate = _lhsclassic(rng, n, samples) else: - Hcandidate = _lhscentered(n, samples) + Hcandidate = _lhscentered(rng, n, samples) d = _pdist(Hcandidate) - if maxdist pathlib.Path: """Find a single file matching a wildcard pattern. @@ -1384,6 +1809,7 @@ def find_single_file(file_pattern) -> pathlib.Path: msgs.warn(f'Found multiple files matching {file_pattern}; using the first one.') return pathlib.Path(files[0]) + def DFS(v: int, visited: list[bool], group: list[int], adj: np.ndarray): """ Depth-First Search of graph given by matrix `adj` starting from `v`. @@ -1398,7 +1824,7 @@ def DFS(v: int, visited: list[bool], group: list[int], adj: np.ndarray): visited in THIS CALL of DFS. After DFS returns, `group` contains all members of the connected component containing v. `i in group` is True iff vertex `i` has been visited in THIS CALL of DFS. - adj (np.ndarray): Adjacency matrix description of the graph. `adj[i,j]` + adj (`numpy.ndarray`_): Adjacency matrix description of the graph. `adj[i,j]` is True iff there is a vertex between `i` and `j`. """ stack = [] @@ -1408,48 +1834,49 @@ def DFS(v: int, visited: list[bool], group: list[int], adj: np.ndarray): if not visited[u]: visited[u] = True group.append(u) - neighbors = [i for i in range(len(adj[u])) if adj[u,i]] + neighbors = [i for i in range(len(adj[u])) if adj[u, i]] for neighbor in neighbors: stack.append(neighbor) + +# TODO: Describe returned arrays def list_of_spectral_lines(): """ Generate a list of spectral lines Returns: - tuple: np.ndarray, np.ndarray + tuple: Two `numpy.ndarray`_ objects. """ # spectral features - CIVnam1, CIVwav1='CIV', 1548. - CIVnam2, CIVwav2='CIV', 1550. + CIVnam1, CIVwav1 = 'CIV', 1548. + CIVnam2, CIVwav2 = 'CIV', 1550. - HeIInam0, HeIIwav0='HeII', 1640. + HeIInam0, HeIIwav0 = 'HeII', 1640. - OIIInam01, OIIIwav01='OIII]', 1661. - OIIInam02, OIIIwav02='OIII]', 1666. + OIIInam01, OIIIwav01 = 'OIII]', 1661. + OIIInam02, OIIIwav02 = 'OIII]', 1666. - SiIIInam1, SiIIIwav1='SiIII]', 1882. - SiIIInam2, SiIIIwav2='SiIII]', 1892. + SiIIInam1, SiIIIwav1 = 'SiIII]', 1882. + SiIIInam2, SiIIIwav2 = 'SiIII]', 1892. - CIIInam1, CIIIwav1='CIII]', 1907. - CIIInam2, CIIIwav2='CIII]', 1909. + CIIInam1, CIIIwav1 = 'CIII]', 1907. + CIIInam2, CIIIwav2 = 'CIII]', 1909. - Lyalphanam, Lyalphawav='Lyalpha', 1215.7 - OIInam1, OIIwav1='[OII]', 3726 - OIInam2, OIIwav2='[OII]', 3729 - OIIInam1, OIIIwav1='[OIII]', 5007. - OIIInam2, OIIIwav2='[OIII]', 4959. - OIIInam3, OIIIwav3='[OIII]', 4363. - Halphanam, Halphawav='Halpha', 6563. - Hbetanam, Hbetawav='Hbeta', 4861. - Hdeltanam, Hdeltawav='Hdelta', 4101. - Hgammanam, Hgammawav='Hgamma', 4341. + Lyalphanam, Lyalphawav = 'Lyalpha', 1215.7 + OIInam1, OIIwav1 = '[OII]', 3726 + OIInam2, OIIwav2 = '[OII]', 3729 + OIIInam1, OIIIwav1 = '[OIII]', 5007. + OIIInam2, OIIIwav2 = '[OIII]', 4959. + OIIInam3, OIIIwav3 = '[OIII]', 4363. + Halphanam, Halphawav = 'Halpha', 6563. + Hbetanam, Hbetawav = 'Hbeta', 4861. + Hdeltanam, Hdeltawav = 'Hdelta', 4101. + Hgammanam, Hgammawav = 'Hgamma', 4341. NeIIInam, NeIIIwav = '[NeIII]', 3869. NeVnam, NeVwav = '[NeV]', 3426. SIInam1, SIIwav1 = '[SII]', 6716. SIInam2, SIIwav2 = '[SII]', 6716. - ##absorption H13nam, H13wav = 'H13', 3734. H12nam, H12wav = 'H12', 3750. @@ -1464,16 +1891,16 @@ def list_of_spectral_lines(): Gbandnam, Gbandwav = 'Gband', 4305. - line_names=np.array([CIVnam1, CIVnam2, HeIInam0, OIIInam01, OIIInam02, SiIIInam1, SiIIInam2, - CIIInam1, CIIInam2, Lyalphanam, OIInam1, OIInam2, OIIInam1, OIIInam2, - OIIInam3, Halphanam, Hbetanam, Hdeltanam, Hgammanam, NeIIInam, NeVnam, - SIInam1, SIInam2, H13nam, H12nam, H11nam, H10nam, H9nam, H8nam, HeInam, - CAII_Knam, CAII_Hnam, Gbandnam]) + line_names = np.array([CIVnam1, CIVnam2, HeIInam0, OIIInam01, OIIInam02, SiIIInam1, SiIIInam2, + CIIInam1, CIIInam2, Lyalphanam, OIInam1, OIInam2, OIIInam1, OIIInam2, + OIIInam3, Halphanam, Hbetanam, Hdeltanam, Hgammanam, NeIIInam, NeVnam, + SIInam1, SIInam2, H13nam, H12nam, H11nam, H10nam, H9nam, H8nam, HeInam, + CAII_Knam, CAII_Hnam, Gbandnam]) line_wav = np.array([CIVwav2, CIVwav2, HeIIwav0, OIIIwav01, OIIIwav02, SiIIIwav1, SiIIIwav2, CIIIwav1, CIIIwav2, Lyalphawav, OIIwav1, OIIwav2, OIIIwav1, OIIIwav2, OIIIwav3, Halphawav, Hbetawav, Hdeltawav, Hgammawav, NeIIIwav, NeVwav, - SIIwav1,SIIwav2, H13wav, H12wav, H11wav, H10wav, H9wav, H8wav, HeIwav, + SIIwav1, SIIwav2, H13wav, H12wav, H11wav, H10wav, H9wav, H8wav, HeIwav, CaII_Kwav, CaII_Hwav, Gbandwav]) return line_names, line_wav diff --git a/pypeit/wavecalib.py b/pypeit/wavecalib.py index 6f1ef90fab..3f6fc47c47 100644 --- a/pypeit/wavecalib.py +++ b/pypeit/wavecalib.py @@ -17,7 +17,7 @@ from pypeit.core import arc, qa from pypeit.core import fitting from pypeit.core import parse -from pypeit.core.wavecal import autoid, wv_fitting +from pypeit.core.wavecal import autoid, wv_fitting, wvutils from pypeit.core.gui.identify import Identify from pypeit import datamodel from pypeit import calibframe @@ -38,7 +38,7 @@ class WaveCalib(calibframe.CalibFrame): .. include:: ../include/class_datamodel_wavecalib.rst """ - version = '1.1.0' + version = '1.1.1' # Calibration frame attributes calib_type = 'WaveCalib' @@ -58,6 +58,8 @@ class WaveCalib(calibframe.CalibFrame): descr='2D wavelength solution(s) (echelle). If there is more ' 'than one, they must be aligned to the separate detectors ' 'analyzed'), + 'fwhm_map': dict(otype=np.ndarray, atype=fitting.PypeItFit, + descr='A fit that determines the spectral FWHM at every location of every slit'), 'det_img': dict(otype=np.ndarray, atype=np.integer, descr='Detector image which indicates which pixel in the mosaic ' 'corresponds to which detector; used occasionally by ' @@ -73,7 +75,7 @@ class WaveCalib(calibframe.CalibFrame): 'lamps': dict(otype=str, descr='List of arc lamps used for the wavelength calibration')} - def __init__(self, wv_fits=None, nslits=None, spat_ids=None, PYP_SPEC=None, + def __init__(self, wv_fits=None, fwhm_map=None, nslits=None, spat_ids=None, PYP_SPEC=None, strpar=None, wv_fit2d=None, arc_spectra=None, lamps=None, det_img=None): # Parse @@ -107,7 +109,7 @@ def _bundle(self): continue # Array? if self.datamodel[key]['otype'] == np.ndarray and \ - key not in ['wv_fits', 'wv_fit2d']: + key not in ['wv_fits', 'wv_fit2d', 'fwhm_map']: _d.append({key: self[key]}) # TODO: Can we put all the WAVEFIT and PYPEITFIT at the end of the # list of HDUs? This would mean ARC_SPECTRA is always in the same @@ -135,6 +137,16 @@ def _bundle(self): for ss, wv_fit2d in enumerate(self[key]): dkey = f'WAVE2DFIT-{ss}' _d.append({dkey: wv_fit2d}) + elif key == 'fwhm_map': + for ss, fwhm_fit in enumerate(self[key]): + dkey = 'SPAT_ID-{}_FWHMFIT'.format(self.spat_ids[ss]) + # Generate a dummy? + if fwhm_fit is None: + _fwhm_fit = fitting.PypeItFit() + else: + _fwhm_fit = fwhm_fit + # Save + _d.append({dkey: _fwhm_fit}) else: # Add to header of the spat_id image _d[0][key] = self[key] # Return @@ -152,11 +164,14 @@ def _parse(cls, hdu, **kwargs): # Now the wave_fits list_of_wave_fits = [] list_of_wave2d_fits = [] + list_of_fwhm_fits = [] spat_ids = [] for ihdu in hdu: if 'WAVEFIT' in ihdu.name: # Allow for empty if len(ihdu.data) == 0: + # TODO: This is a hack. We shouldn't be writing empty HDUs, + # except for the primary HDU. iwavefit = wv_fitting.WaveFit(ihdu.header['SPAT_ID']) else: # TODO -- Replace the following with WaveFit._parse() and pass that back!! @@ -176,13 +191,23 @@ def _parse(cls, hdu, **kwargs): iwave2dfit = fitting.PypeItFit.from_hdu(ihdu) list_of_wave2d_fits.append(iwave2dfit) parsed_hdus += ihdu.name + elif 'FWHMFIT' in ihdu.name: + # TODO: This is a hack. We shouldn't be writing empty HDUs, + # except for the primary HDU. + ifwhmfit = fitting.PypeItFit() if len(ihdu.data) == 0 \ + else fitting.PypeItFit.from_hdu(ihdu) + list_of_fwhm_fits.append(ifwhmfit) + parsed_hdus += ihdu.name # Check if spat_ids != _d['spat_ids'].tolist(): + #embed(header="198 of wavecalib.py") msgs.error("Bad parsing of WaveCalib") # Finish _d['wv_fits'] = np.asarray(list_of_wave_fits) if len(list_of_wave2d_fits) > 0: _d['wv_fit2d'] = np.asarray(list_of_wave2d_fits) + if len(list_of_fwhm_fits) > 0: + _d['fwhm_map'] = np.asarray(list_of_fwhm_fits) return _d, dm_version_passed, dm_type_passed, parsed_hdus @property @@ -203,6 +228,38 @@ def chk_synced(self, slits): msgs.error('Your wavelength solutions are out of sync with your slits. Remove ' 'Calibrations and restart from scratch.') + def build_fwhmimg(self, tilts, slits, initial=False, spat_flexure=None): + """ + Generates an image of the instrument spectral FWHM (units=pixels) at every pixel on the detector. + + Args: + tilts (`numpy.ndarray`_): + Image holding tilts + slits (:class:`pypeit.slittrace.SlitTraceSet`): + Properties of the slits + initial (bool, optional): + If True, the initial slit locations will be used. Otherwise, the tweaked edges will be used. + spat_flexure (float, optional): + Spatial flexure correction in pixels. + + Returns: + `numpy.ndarray`_: The spectral FWHM image. + """ + # Check spatial flexure type + if (spat_flexure is not None) and (not isinstance(spat_flexure, float)): + msgs.error("Spatial flexure must be None or float") + # Generate the slit mask and slit edges - pad slitmask by 1 for edge effects + slitmask = slits.slit_img(pad=1, initial=initial, flexure=spat_flexure) + slits_left, slits_right, _ = slits.select_edges(initial=initial, flexure=spat_flexure) + # Build a map of the spectral FWHM + fwhmimg = np.zeros(tilts.shape) + for sl, spat_id in enumerate(slits.spat_id): + this_mask = slitmask == spat_id + spec, spat = np.where(this_mask) + spat_loc = (spat - slits_left[spec, sl]) / (slits_right[spec, sl] - slits_left[spec, sl]) + fwhmimg[this_mask] = self.fwhm_map[sl].eval(spec, spat_loc) + return fwhmimg + def build_waveimg(self, tilts, slits, spat_flexure=None, spec_flexure=None): """ Main algorithm to build the wavelength image @@ -214,6 +271,7 @@ def build_waveimg(self, tilts, slits, spat_flexure=None, spec_flexure=None): tilts (`numpy.ndarray`_): Image holding tilts slits (:class:`pypeit.slittrace.SlitTraceSet`): + Properties of the slits spat_flexure (float, optional): Spatial flexure correction in pixels. spec_flexure (float, `numpy.ndarray`_, optional): @@ -352,18 +410,18 @@ class BuildWaveCalib: Class to guide wavelength calibration Args: - msarc (:class:`pypeit.images.pypeitimage.PypeItImage`): + msarc (:class:`~pypeit.images.pypeitimage.PypeItImage`): Arc image, created by the ArcImage class - slits (:class:`pypeit.slittrace.SlitTraceSet`): + slits (:class:`~pypeit.slittrace.SlitTraceSet`): Slit edges - spectrograph (:class:`pypeit.spectrographs.spectrograph.Spectrograph`): + spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): The `Spectrograph` instance that sets the instrument used to take the observations. Used to set :attr:`spectrograph`. - par (:class:`pypeit.par.pypeitpar.WaveSolutionPar`): - The parameters used for the wavelength solution - Uses ['calibrations']['wavelengths'] - meta_dict (dict: optional): + par (:class:`~pypeit.par.pypeitpar.WavelengthSolutionPar`): + The parameters used for the wavelength solution. + Uses ``['calibrations']['wavelengths']``. + meta_dict (dict, optional): Dictionary containing meta information required for wavelength calibration. Specifically for non-fixed format echelles this dict must contain the following keys: @@ -374,20 +432,20 @@ class BuildWaveCalib: det (int, optional): Detector number - msbpm (ndarray, optional): + msbpm (`numpy.ndarray`_, optional): Bad pixel mask image qa_path (str, optional): For QA Attributes: - steps : list + steps (list): List of the processing steps performed - wv_calib : dict + wv_calib (dict): Primary output. Keys 0, 1, 2, 3 are solution for individual previously slit steps - arccen (ndarray): + arccen (`numpy.ndarray`_): (nwave, nslit) Extracted arc(s) down the center of the slit(s) - maskslits : ndarray (nslit); bool + maskslits (`numpy.ndarray`_): Slits to ignore because they were not extracted. WARNING: Outside of this Class, it is best to regenerate the mask using make_maskslits() @@ -397,14 +455,15 @@ class BuildWaveCalib: require that we write it to disk with self.msarc.image nonlinear_counts (float): Specifies saturation level for the arc lines - wvc_bpm (`numpy.ndarray`_): Mask for slits attempted to have a wv_calib solution + wvc_bpm (`numpy.ndarray`_): + Mask for slits attempted to have a wv_calib solution """ # TODO: Is this used anywhere? frametype = 'wv_calib' - def __init__(self, msarc, slits, spectrograph, par, lamps, meta_dict=None, det=1, qa_path=None, - msbpm=None): + def __init__(self, msarc, slits, spectrograph, par, lamps, + meta_dict=None, det=1, qa_path=None, msbpm=None): # TODO: This should be a stop-gap to avoid instantiation of this with # any Nones. @@ -448,9 +507,21 @@ def __init__(self, msarc, slits, spectrograph, par, lamps, meta_dict=None, det=1 # have a different binning then the trace images used to defined # the slits if self.slits is not None and self.msarc is not None: + # Redo? + if self.par['redo_slits'] is not None: + if self.par['echelle'] and self.slits.ech_order is not None: + idx = np.in1d(self.slits.ech_order, self.par['redo_slits']) + # Turn off mask + self.slits.mask[idx] = self.slits.bitmask.turn_off( + self.slits.mask[idx], 'BADWVCALIB') + else: + idx = np.in1d(self.slits.spat_id, self.par['redo_slits']) + self.slits.mask[idx] = self.slits.bitmask.turn_off( + self.slits.mask[idx], 'BADWVCALIB') + # Load up slits # TODO -- Allow for flexure - all_left, all_right, mask = self.slits.select_edges(initial=True, flexure=None) # Grabs all, init slits + flexure + slits_left, slits_right, mask = self.slits.select_edges(initial=True, flexure=None) # Grabs all, init slits + flexure self.orders = self.slits.ech_order # Can be None # self.spat_coo = self.slits.spatial_coordinates() # All slits, even masked # Internal mask for failed wv_calib analysis @@ -467,7 +538,10 @@ def __init__(self, msarc, slits, spectrograph, par, lamps, meta_dict=None, det=1 self.shape_science = self.slitmask_science.shape self.shape_arc = self.msarc.image.shape # slitcen is padded to include slits that may be masked, for convenience in coding downstream - self.slitcen = arc.resize_slits2arc(self.shape_arc, self.shape_science, (all_left+all_right)/2) + self.slits_left = arc.resize_slits2arc(self.shape_arc, self.shape_science, slits_left) + self.slits_right = arc.resize_slits2arc(self.shape_arc, self.shape_science, slits_right) + self.slitcen = (self.slits_left+self.slits_right)/2 + #self.slitcen = arc.resize_slits2arc(self.shape_arc, self.shape_science, (self.slits_left+self.slits_right)/2) self.slitmask = arc.resize_mask2arc(self.shape_arc, self.slitmask_science) # Mask # TODO: The bpm defined above is already a boolean and cannot be None. @@ -486,10 +560,13 @@ def __init__(self, msarc, slits, spectrograph, par, lamps, meta_dict=None, det=1 self.shape_science = None self.shape_arc = None self.slitcen = None + self.slits_left = None + self.slits_right = None self.slitmask = None self.gpm = None - def build_wv_calib(self, arccen, method, skip_QA=False): + def build_wv_calib(self, arccen, method, skip_QA=False, + prev_wvcalib=None): """ Main routine to generate the wavelength solutions in a loop over slits Wrapper to arc.simple_calib or arc.calib_with_arclines @@ -505,6 +582,8 @@ def build_wv_calib(self, arccen, method, skip_QA=False): 'identify' -- wavecal.identify.Identify 'full_template' -- wavecal.auotid.full_template skip_QA (bool, optional) + prev_wvcalib (WaveCalib, optional): + Previous wavelength calibration Returns: dict: self.wv_calib @@ -516,17 +595,28 @@ def build_wv_calib(self, arccen, method, skip_QA=False): if self.slits.maskdef_designtab is not None: msgs.info("Slit widths (arcsec): {}".format(np.round(self.slits.maskdef_designtab['SLITWID'].data, 2))) - # measure the FWHM of the arc lines + # Generate a map of the instrumental spectral FWHM + # TODO nsample should be a parameter + fwhm_map = autoid.map_fwhm(self.msarc.image, self.gpm, self.slits_left, self.slits_right, self.slitmask, + nsample=10, slit_bpm=self.wvc_bpm, specord=self.par['fwhm_spec_order'], + spatord=self.par['fwhm_spat_order']) + # Calculate the typical spectral FWHM down the centre of the slit measured_fwhms = np.zeros(arccen.shape[1], dtype=object) for islit in range(arccen.shape[1]): if islit not in ok_mask_idx: continue - measured_fwhms[islit] = autoid.measure_fwhm(arccen[:, islit]) + # Measure the spectral FWHM (in pixels) at the midpoint of the slit + # (i.e. the midpoint in both the spectral and spatial directions) + measured_fwhms[islit] = fwhm_map[islit].eval(self.msarc.image.shape[0]//2, 0.5) + + # Save for redo's + self.measured_fwhms = measured_fwhms # Obtain calibration for all slits if method == 'holy-grail': # Sometimes works, sometimes fails - arcfitter = autoid.HolyGrail(arccen, self.lamps, par=self.par, ok_mask=ok_mask_idx, + arcfitter = autoid.HolyGrail(arccen, self.lamps, par=self.par, + ok_mask=ok_mask_idx, nonlinear_counts=self.nonlinear_counts, spectrograph=self.spectrograph.name) patt_dict, final_fit = arcfitter.get_results() @@ -545,65 +635,98 @@ def build_wv_calib(self, arccen, method, skip_QA=False): elif method == 'reidentify': # Now preferred # Slit positions - arcfitter = autoid.ArchiveReid(arccen, self.lamps, self.par, - ech_fixed_format=self.spectrograph.ech_fixed_format, ok_mask=ok_mask_idx, - measured_fwhms=measured_fwhms, - orders=self.orders, - nonlinear_counts=self.nonlinear_counts) + arcfitter = autoid.ArchiveReid( + arccen, self.lamps, self.par, + ech_fixed_format=self.spectrograph.ech_fixed_format, + ok_mask=ok_mask_idx, + measured_fwhms=self.measured_fwhms, + orders=self.orders, + nonlinear_counts=self.nonlinear_counts) patt_dict, final_fit = arcfitter.get_results() + + # Grab arxiv for redo later? + if self.par['echelle']: + # Hold for later usage + self.wave_soln_arxiv, self.arcspec_arxiv = arcfitter.get_arxiv(self.orders) + self.arccen = arccen elif method == 'full_template': # Now preferred if self.binspectral is None: msgs.error("You must specify binspectral for the full_template method!") final_fit = autoid.full_template(arccen, self.lamps, self.par, ok_mask_idx, self.det, - self.binspectral, measured_fwhms=measured_fwhms, + self.binspectral, slit_ids=self.slits.slitord_id, + measured_fwhms=self.measured_fwhms, nonlinear_counts=self.nonlinear_counts, nsnippet=self.par['nsnippet']) #debug=True, debug_reid=True, debug_xcorr=True) elif self.par['method'] == 'echelle': - # TODO -- Merge this with reidentify for fixed echelle formats - # Echelle calibration files angle_fits_file, composite_arc_file = self.spectrograph.get_echelle_angle_files() # Identify the echelle orders msgs.info("Finding the echelle orders") order_vec, wave_soln_arxiv, arcspec_arxiv = echelle.identify_ech_orders( - arccen, self.meta_dict['echangle'], self.meta_dict['xdangle'], self.meta_dict['dispname'], - angle_fits_file, composite_arc_file, pad=3, debug=False) + arccen, self.meta_dict['echangle'], + self.meta_dict['xdangle'], + self.meta_dict['dispname'], + angle_fits_file, + composite_arc_file, + pad=3, debug=False) # Put the order numbers in the slit object self.slits.ech_order = order_vec msgs.info(f"The observation covers the following orders: {order_vec}") - # TODO: - # HACK!! - ok_mask_idx = ok_mask_idx[:-1] - patt_dict, final_fit = autoid.echelle_wvcalib(arccen, order_vec, arcspec_arxiv, wave_soln_arxiv, - self.lamps, self.par, ok_mask=ok_mask_idx, - nonlinear_counts=self.nonlinear_counts, - debug_all=False) + + #ok_mask_idx = ok_mask_idx[:-1] + patt_dict, final_fit = autoid.echelle_wvcalib( + arccen, order_vec, arcspec_arxiv, wave_soln_arxiv, + self.lamps, self.par, ok_mask=ok_mask_idx, + nonlinear_counts=self.nonlinear_counts, + debug_all=False, + redo_slits=np.atleast_1d(self.par['redo_slits']) if self.par['redo_slits'] is not None else None) + + # Save as internals in case we need to redo + self.wave_soln_arxiv = wave_soln_arxiv + self.arcspec_arxiv = arcspec_arxiv + self.arccen = arccen + else: msgs.error('Unrecognized wavelength calibration method: {:}'.format(method)) # Build the DataContainer - # Loop on WaveFit items - tmp = [] - for idx in range(self.slits.nslits): - item = final_fit.pop(str(idx)) - if item is None: # Add an empty WaveFit - tmp.append(wv_fitting.WaveFit(self.slits.spat_id[idx])) - else: - # This is for I/O naming - item.spat_id = self.slits.spat_id[idx] - # add measured fwhm - item['fwhm'] = measured_fwhms[idx] - tmp.append(item) - self.wv_calib = WaveCalib(wv_fits=np.asarray(tmp), - arc_spectra=arccen, - nslits=self.slits.nslits, - spat_ids=self.slits.spat_id, - PYP_SPEC=self.spectrograph.name, - lamps=','.join(self.lamps)) + if self.par['redo_slits'] is not None: + # If we are only redoing slits, we start from the + # previous wv_calib and update only the (good) redone slits + self.wv_calib = prev_wvcalib + # Update/reset items + self.wv_calib.arc_spectra = arccen + # Save the new fits (if they meet tolerance) + for key in final_fit.keys(): + if final_fit[key]['rms'] < self.par['rms_threshold']: + idx = int(key) + self.wv_calib.wv_fits[idx] = final_fit[key] + self.wv_calib.wv_fits[idx].spat_id = self.slits.spat_id[idx] + self.wv_calib.wv_fits[idx].fwhm = self.measured_fwhms[idx] + else: # Generate the DataContainer from scratch + # Loop on WaveFit items + tmp = [] + for idx in range(self.slits.nslits): + item = final_fit.pop(str(idx)) + if item is None: # Add an empty WaveFit + tmp.append(wv_fitting.WaveFit(self.slits.spat_id[idx])) + else: + # This is for I/O naming + item.spat_id = self.slits.spat_id[idx] + # add measured fwhm + item['fwhm'] = measured_fwhms[idx] + tmp.append(item) + self.wv_calib = WaveCalib(wv_fits=np.asarray(tmp), + fwhm_map=fwhm_map, + arc_spectra=arccen, + nslits=self.slits.nslits, + spat_ids=self.slits.spat_id, + PYP_SPEC=self.spectrograph.name, + lamps=','.join(self.lamps)) # Inherit the calibration frame naming from self.msarc # TODO: Should throw an error here if these calibration frame naming # elements are not defined by self.msarc... @@ -616,18 +739,122 @@ def build_wv_calib(self, arccen, method, skip_QA=False): if not skip_QA: ok_mask_idx = np.where(np.invert(self.wvc_bpm))[0] for slit_idx in ok_mask_idx: - outfile = qa.set_qa_filename(self.wv_calib.calib_key, 'arc_fit_qa', - slit=self.slits.slitord_id[slit_idx], - out_dir=self.qa_path) - # - autoid.arc_fit_qa(self.wv_calib.wv_fits[slit_idx], + msgs.info(f"Preparing wavelength calibration QA for slit {slit_idx+1}/{self.slits.nslits}") + # Obtain the output QA name for the wavelength solution + outfile = qa.set_qa_filename( + self.wv_calib.calib_key, 'arc_fit_qa', + slit=self.slits.slitord_id[slit_idx], + out_dir=self.qa_path) + # Save the wavelength solution fits + autoid.arc_fit_qa(self.wv_calib.wv_fits[slit_idx], + title=f'Arc Fit QA for slit/order: {self.slits.slitord_id[slit_idx]}', outfile=outfile) + # Obtain the output QA name for the spectral resolution map + outfile_fwhm = qa.set_qa_filename(self.wv_calib.calib_key, 'arc_fwhm_qa', + slit=self.slits.slitord_id[slit_idx], + out_dir=self.qa_path) + # Save the wavelength solution fits + autoid.arc_fwhm_qa(self.wv_calib.fwhm_map[slit_idx], + self.slits.slitord_id[slit_idx], self.slits.slitord_txt, + outfile=outfile_fwhm) + # Return self.steps.append(inspect.stack()[0][3]) return self.wv_calib + def redo_echelle_orders(self, bad_orders:np.ndarray, dets:np.ndarray, order_dets:np.ndarray): + """ Attempt to redo the wavelength calibration for a set + of bad echelle orders + + Args: + bad_orders (np.ndarray): Array of bad order numbers + dets (np.ndarray): detectors of the spectrograph + Multiple numbers for mosaic (typically) + order_dets (np.ndarray): Orders on the each detector + + Returns: + bool: True if any of the echelle orders were + successfully redone + """ + + # Make this outside the for loop.. + #bad_orders = self.slits.ech_order[np.where(bad_rms)[0]] + fixed = False + + for idet in range(len(dets)): + in_det = np.in1d(bad_orders, order_dets[idet]) + if not np.any(in_det): + continue + # Are there few enough? + # TODO -- make max_bad a parameter + max_bad = len(order_dets[idet])//10 + 1 + if np.sum(in_det) > max_bad: + msgs.warn(f"Too many bad orders in detector={dets[idet]} to attempt a refit.") + continue + # Loop + for order in bad_orders[in_det]: + iord = np.where(self.slits.ech_order == order)[0][0] + # Predict the wavelengths + nspec = self.arccen.shape[0] + spec_vec_norm = np.linspace(0,1,nspec) + wv_order_mod = self.wv_calib.wv_fit2d[idet].eval(spec_vec_norm, + x2=np.ones_like(spec_vec_norm)*order)/order + + # Link me + tcent, spec_cont_sub, patt_dict_slit, tot_llist = autoid.match_to_arxiv( + self.lamps, self.arccen[:,iord], wv_order_mod, + self.arcspec_arxiv[:, iord], self.wave_soln_arxiv[:,iord], + self.par['nreid_min'], + match_toler=self.par['match_toler'], + nonlinear_counts=self.nonlinear_counts, + sigdetect=wvutils.parse_param(self.par, 'sigdetect', iord), + fwhm=self.par['fwhm']) + + if not patt_dict_slit['acceptable']: + msgs.warn(f"Order {order} is still not acceptable after attempt to reidentify.") + continue + + # Fit me -- RMS may be too high again + n_final = wvutils.parse_param(self.par, 'n_final', iord) + # TODO - Make this cheaper + final_fit = wv_fitting.fit_slit( + spec_cont_sub, patt_dict_slit, tcent, tot_llist, + match_toler=self.par['match_toler'], + func=self.par['func'], + n_first=self.par['n_first'], + #n_first=3, + sigrej_first=self.par['sigrej_first'], + n_final=n_final, + sigrej_final=2.) + msgs.info(f"New RMS for redo of order={order}: {final_fit['rms']}") + + # Keep? + # TODO -- Make this a parameter? + increase_rms = 1.5 + if final_fit['rms'] < increase_rms*self.par['rms_threshold']* np.median(self.measured_fwhms)/self.par['fwhm']: + # TODO -- This is repeated from build_wv_calib() + # Would be nice to consolidate + # QA + outfile = qa.set_qa_filename( + self.wv_calib.calib_key, 'arc_fit_qa', + slit=order, + out_dir=self.qa_path) + autoid.arc_fit_qa(final_fit, + title=f'Arc Fit QA for slit/order: {order}', + outfile=outfile) + # This is for I/O naming + final_fit.spat_id = self.slits.spat_id[iord] + final_fit.fwhm = self.measured_fwhms[iord] + # Save the wavelength solution fits + self.wv_calib.wv_fits[iord] = final_fit + self.wvc_bpm[iord] = False + fixed = True + # + return fixed + + def echelle_2dfit(self, wv_calib, debug=False, skip_QA=False): """ Fit a two-dimensional wavelength solution for echelle data. @@ -635,18 +862,25 @@ def echelle_2dfit(self, wv_calib, debug=False, skip_QA=False): Primarily a wrapper for :func:`pypeit.core.arc.fit2darc`, using data unpacked from the ``wv_calib`` dictionary. - Args: - wv_calib (:class:`pypeit.wavecalib.WaveCalib`): - Wavelength calibration object - debug (:obj:`bool`, optional): - Show debugging info - skip_QA (:obj:`bool`, optional): - Flag to skip construction of the nominal QA plots. - - Returns: - list: list of :class:`pypeit.fitting.PypeItFit`: objects containing information from 2-d fit. - Frequently a list of 1 fit. The main exception is for - a mosaic when one sets echelle_separate_2d=True + Parameters + ---------- + wv_calib : :class:`pypeit.wavecalib.WaveCalib` + Wavelength calibration object + debug : :obj:`bool`, optional + Show debugging info + skip_QA : :obj:`bool`, optional + Flag to skip construction of the nominal QA plots. + + Returns + ------- + fit2ds : list of :class:`pypeit.fitting.PypeItFit` + Contains information from 2-d fit. Frequently a list of 1 fit. The + main exception is for a mosaic when one sets + ``echelle_separate_2d=True``. + dets : list + List of integers for the detector numbers. + save_order_dets: list + List of integer lists providing list of the orders. """ if self.spectrograph.pypeline != 'Echelle': msgs.error('Cannot execute echelle_2dfit for a non-echelle spectrograph.') @@ -670,7 +904,9 @@ def echelle_2dfit(self, wv_calib, debug=False, skip_QA=False): # Loop on detectors fit2ds = [] + save_order_dets = [] for idet in dets: + order_in_dets = [] msgs.info('Fitting detector {:d}'.format(idet)) # Init all_wave = np.array([], dtype=float) @@ -680,8 +916,6 @@ def echelle_2dfit(self, wv_calib, debug=False, skip_QA=False): # Loop to grab the good orders for ii in range(wv_calib.nslits): iorder = self.slits.ech_order[ii] - if iorder not in ok_mask_order: - continue # Separate detector analysis? if self.par['ech_separate_2d']: @@ -694,6 +928,13 @@ def echelle_2dfit(self, wv_calib, debug=False, skip_QA=False): if ordr_det != idet: continue + # Need to record this whether or not it is ok + order_in_dets.append(iorder) + + # Is it ok? + if iorder not in ok_mask_order: + continue + # Slurp mask_now = wv_calib.wv_fits[ii].pypeitfit.bool_gpm all_wave = np.append(all_wave, wv_calib.wv_fits[ii]['wave_fit'][mask_now]) @@ -702,10 +943,19 @@ def echelle_2dfit(self, wv_calib, debug=False, skip_QA=False): float(iorder))) # Fit + if len(all_order) < 2: + msgs.warn(f"Fewer than 2 orders to fit for detector {idet}. Skipping") + save_order_dets.append([]) + # Add a dummy fit + fit2ds.append(fitting.PypeItFit()) + continue + fit2d = arc.fit2darc(all_wave, all_pixel, all_order, nspec, nspec_coeff=self.par['ech_nspec_coeff'], norder_coeff=self.par['ech_norder_coeff'], sigrej=self.par['ech_sigrej'], debug=debug) + # Save + save_order_dets.append(order_in_dets) fit2ds.append(fit2d) self.steps.append(inspect.stack()[0][3]) @@ -734,7 +984,7 @@ def echelle_2dfit(self, wv_calib, debug=False, skip_QA=False): out_dir=self.qa_path) arc.fit2darc_orders_qa(fit2d, nspec, outfile=outfile_orders) - return fit2ds + return fit2ds, dets, save_order_dets # TODO: JFH this method is identical to the code in wavetilts. @@ -745,22 +995,25 @@ def extract_arcs(self, slitIDs=None): Wrapper to arc.get_censpec() - Args: - slitIDs (:obj:`list`, optional): - A list of the slit IDs to extract (if None, all slits will be extracted) - - Returns: - tuple: Returns the following: - - self.arccen: ndarray, (nspec, nslit): arc spectrum for - all slits - - self.arc_maskslit: ndarray, bool (nsit): boolean array - containing a mask indicating which slits are good - True = masked (bad) - + Parameters + ---------- + slitIDs : :obj:`list`, optional + A list of the slit IDs to extract (if None, all slits will be + extracted) + + Returns + ------- + arccen : `numpy.ndarray`_ + arc spectrum for all slits, shape is (nspec, nslit): + wvc_bpm : `numpy.ndarray`_ + boolean array containing a mask indicating which slits are good. True + = masked (bad). """ # Do it on the slits not masked in self.slitmask arccen, arccen_bpm, arc_maskslit = arc.get_censpec( - self.slitcen, self.slitmask, self.msarc.image, gpm=self.gpm, slit_bpm=self.wvc_bpm, slitIDs=slitIDs) + self.slitcen, self.slitmask, self.msarc.image, + gpm=self.gpm, slit_bpm=self.wvc_bpm, + slitIDs=slitIDs) # Step self.steps.append(inspect.stack()[0][3]) @@ -788,21 +1041,25 @@ def update_wvmask(self): self.wvc_bpm[kk] = True - def run(self, skip_QA=False, debug=False): + def run(self, skip_QA=False, debug=False, + prev_wvcalib=None): """ - Main driver for wavelength calibration + Main method for wavelength calibration Code flow: 1. Extract 1D arc spectra down the center of each unmasked slit/order - 2. Load the parameters guiding wavelength calibration - 3. Generate the 1D wavelength fits - 4. Generate a mask + 2. Generate the 1D wavelength fits + 3. If echelle, perform 2D fit(s). + 4. Deal with masking + 5. Return a WaveCalib object Args: - skip_QA : bool, optional + skip_QA (bool, optional): Skip QA? + prev_wvcalib (WaveCalib, optional): + Previous wavelength calibration object (from disk, typically) Returns: - dict: wv_calib dict + WaveCalib: wavelength calibration object """ ############### @@ -810,19 +1067,46 @@ def run(self, skip_QA=False, debug=False): self.arccen, self.wvc_bpm = self.extract_arcs() # Fill up the calibrations and generate QA - self.wv_calib = self.build_wv_calib(self.arccen, - self.par['method'], skip_QA=skip_QA) + self.wv_calib = self.build_wv_calib( + self.arccen, self.par['method'], + skip_QA=skip_QA, + prev_wvcalib=prev_wvcalib) # Fit 2D? if self.par['echelle']: + # Assess the fits + rms = np.array([999. if wvfit.rms is None else wvfit.rms for wvfit in self.wv_calib.wv_fits]) + # Test and scale by measured_fwhms + bad_rms = rms > (self.par['rms_threshold'] * np.median(self.measured_fwhms)/self.par['fwhm']) + #embed(header='line 975 of wavecalib.py') + if np.any(bad_rms): + self.wvc_bpm[bad_rms] = True + msgs.warn("Masking one or more bad orders (RMS)") # Fit - fit2ds = self.echelle_2dfit(self.wv_calib, skip_QA = skip_QA, debug=debug) + fit2ds, dets, order_dets = self.echelle_2dfit( + self.wv_calib, skip_QA = skip_QA, debug=debug) + # Save self.wv_calib.wv_fit2d = np.array(fit2ds) # Save det_img? if self.par['ech_separate_2d']: self.wv_calib.det_img = self.msarc.det_img.copy() + # Try a second attempt with 1D, if needed + if np.any(bad_rms): + bad_orders = self.slits.ech_order[np.where(bad_rms)[0]] + any_fixed = self.redo_echelle_orders(bad_orders, dets, order_dets) + + # Do another full 2D? + if any_fixed: + fit2ds, _, _ = self.echelle_2dfit(self.wv_calib, skip_QA = skip_QA, debug=debug) + # Save + self.wv_calib.wv_fit2d = np.array(fit2ds) + + # Check that we have at least one good 2D fit + if not np.any([fit2d.success for fit2d in self.wv_calib.wv_fit2d]): + msgs.error("No successful 2D Wavelength fits. Cannot proceed.") + # Deal with mask self.update_wvmask() diff --git a/pypeit/wavemodel.py b/pypeit/wavemodel.py index 7e7c0bbd10..58fed838b2 100644 --- a/pypeit/wavemodel.py +++ b/pypeit/wavemodel.py @@ -1,5 +1,8 @@ """ Module to create models of arc lines. + +.. include:: ../include/links.rst + """ import os @@ -31,7 +34,7 @@ def blackbody(wavelength, T_BB=250., debug=False): Parameters ---------- - wavelength : np.array + wavelength : `numpy.ndarray`_ wavelength vector in microns T_BB : float black body temperature in Kelvin. Default is set to: @@ -39,10 +42,10 @@ def blackbody(wavelength, T_BB=250., debug=False): Returns ------- - blackbody : np.array + blackbody : `numpy.ndarray`_ spectral radiance of the black body in cgs units: B_lambda = 2.*h*c^2/lambda^5.*(1./(exp(h*c/(lambda*k_b*T_BB))-1.) - blackbody_counts : np.array + blackbody_counts : `numpy.ndarray`_ Same as above but in flux density """ @@ -85,10 +88,12 @@ def addlines2spec(wavelength, wl_line, fl_line, resolution, Parameters ---------- - wavelength : np.array + wavelength : `numpy.ndarray`_ wavelength vector of the input spectrum - wl_line, fl_line : np.arrays - wavelength and flux of each individual line + wl_line : `numpy.ndarray`_ + wavelength of each individual line + fl_line : `numpy.ndarray`_ + flux of each individual line resolution : float resolution of the spectrograph. In other words, the lines will have a FWHM equal to: @@ -96,12 +101,12 @@ def addlines2spec(wavelength, wl_line, fl_line, resolution, scale_spec : float rescale all the normalization of the final spectrum. Default scale_spec=1. - debug : boolean + debug : bool If True will show debug plots Returns ------- - line_spec : np.array + line_spec : `numpy.ndarray`_ Spectrum with lines """ @@ -142,10 +147,11 @@ def oh_lines(): Returns ------- - wavelength, amplitude : np.arrays - Wavelength [in microns] and amplitude of the OH lines. + wavelength : `numpy.ndarray`_ + Wavelength [in microns] of the OH lines. + amplitude : `numpy.ndarray`_ + Amplitude of the OH lines. """ - msgs.info("Reading in the Rousselot (2000) OH line list") oh = np.loadtxt(data.get_skisim_filepath('rousselot2000.dat'), usecols=(0, 1)) @@ -158,16 +164,16 @@ def transparency(wavelength, debug=False): Parameters ---------- - wavelength : np.array + wavelength : `numpy.ndarray`_ wavelength vector in microns - debug : boolean + debug : bool If True will show debug plots Returns ------- - transparency : np.array - Transmission of the sky over the considered wavelength rage. - 1. means fully transparent and 0. fully opaque + transparency : `numpy.ndarray`_ + Transmission of the sky over the considered wavelength range. 1. means + fully transparent and 0. fully opaque """ msgs.info("Reading in the atmospheric transmission model") @@ -219,11 +225,11 @@ def h2o_lines(): Returns ------- - wavelength, flux : np.arrays - Wavelength [in microns] and flux of the H2O atmospheric - spectrum. + wavelength : `numpy.ndarray`_ + Wavelength [in microns] of the H2O atmospheric spectrum. + flux : `numpy.ndarray`_ + Flux of the H2O atmospheric spectrum. """ - msgs.info("Reading in the water atmsopheric spectrum") h2o = np.loadtxt(data.get_skisim_filepath('HITRAN.txt'), usecols=(0, 1)) @@ -239,11 +245,11 @@ def thar_lines(): Returns ------- - wavelength, flux : np.arrays - Wavelength [in angstrom] and flux of the ThAr lamp - spectrum. + wavelength : `numpy.ndarray`_ + Wavelength [in microns] of the ThAr lamp spectrum. + flux : `numpy.ndarray`_ + Flux of the ThAr lamp spectrum. """ - msgs.info("Reading in the ThAr spectrum") thar = data.load_thar_spec() @@ -280,7 +286,7 @@ def nearIR_modelsky(resolution, waveminmax=(0.8,2.6), dlam=40.0, If flgd='True' it is a bin in velocity (km/s). If flgd='False' it is a bin in linear space (microns). Default is: 40.0 (with flgd='True') - flgd : boolean + flgd : bool if flgd='True' (default) wavelengths are created with equal steps in log space. If 'False', wavelengths will be created wit equal steps in linear space. @@ -302,15 +308,16 @@ def nearIR_modelsky(resolution, waveminmax=(0.8,2.6), dlam=40.0, WAVE_WATER : float wavelength (in microns) at which the H2O are inclued. Default: WAVE_WATER = 2.3 - debug : boolean + debug : bool If True will show debug plots Returns ------- - wave, sky_model : np.arrays - wavelength (in Ang.) and flux of the final model of the sky. + wave : `numpy.ndarray`_ + Wavelength (in Ang.) of the final model of the sky. + sky_model : `numpy.ndarray`_ + Flux of the final model of the sky. """ - # Create the wavelength array: wv_min = waveminmax[0] wv_max = waveminmax[1] @@ -439,20 +446,22 @@ def optical_modelThAr(resolution, waveminmax=(3000.,10500.), dlam=40.0, If flgd='True' it is a bin in velocity (km/s). If flgd='False' it is a bin in linear space (microns). Default is: 40.0 (with flgd='True') - flgd : boolean + flgd : bool if flgd='True' (default) wavelengths are created with equal steps in log space. If 'False', wavelengths will be created wit equal steps in linear space. thar_outfile : str name of the fits file where the model sky spectrum will be stored. default is 'None' (i.e., no file will be written). - debug : boolean + debug : bool If True will show debug plots Returns ------- - wave, thar_model : np.arrays - wavelength (in Ang.) and flux of the final model of the ThAr lamp emission. + wave : `numpy.ndarray`_ + Wavelength (in Ang.) of the final model of the ThAr lamp emission. + thar_model : `numpy.ndarray`_ + Flux of the final model of the ThAr lamp emission. """ # Create the wavelength array: @@ -543,21 +552,21 @@ def conv2res(wavelength, flux, resolution, central_wl='midpt', Parameters ---------- - wavelength : np.array + wavelength : `numpy.ndarray`_ wavelength - flux : np.array + flux : `numpy.ndarray`_ flux resolution : float resolution of the spectrograph central_wl if 'midpt' the central pixel of wavelength is used, otherwise the central_wl will be used. - debug : boolean + debug : bool If True will show debug plots Returns ------- - flux_convolved :np.array + flux_convolved : `numpy.ndarray`_ Resulting flux after convolution px_sigma : float Size of the sigma in pixels at central_wl @@ -603,7 +612,8 @@ def conv2res(wavelength, flux, resolution, central_wl='midpt', def iraf_datareader(database_dir, id_file): - """Reads in a line identification database created with IRAF + """ + Reads in a line identification database created with IRAF identify. These are usually locate in a directory called 'database'. This read pixel location and wavelength of the lines that have been id with IRAF. Note that the first pixel in IRAF @@ -612,16 +622,18 @@ def iraf_datareader(database_dir, id_file): Parameters ---------- - database_dir : string + database_dir : str directory where the id files are located. - id_file : string + id_file : str filename that is going to be read. Returns ------- - pixel, line_id : np.arrays - Position of the line in pixel and ID of the line. - For IRAF output, these are usually in Ang. + pixel : `numpy.ndarray`_ + Position of the line in pixel of the line. For IRAF output, these are + usually in Ang. + line_id : `numpy.ndarray`_ + ID of the line. """ lines_database = [] @@ -657,9 +669,9 @@ def create_linelist(wavelength, spec, fwhm, sigdetec=2., Parameters ---------- - wavelength : np.array + wavelength : `numpy.ndarray`_ wavelength - spec : np.array + spec : `numpy.ndarray`_ spectrum fwhm : float fwhm in pixels used for filtering out arc lines that are too @@ -679,7 +691,7 @@ def create_linelist(wavelength, spec, fwhm, sigdetec=2., iraf_frmt : bool if True, the file is written in the IRAF format (i.e. wavelength, ion name, amplitude). - convert_air_to_vac (bool): + convert_air_to_vac : bool If True, convert the wavelengths of the created linelist from air to vacuum """ @@ -734,7 +746,7 @@ def create_OHlinelist(resolution, waveminmax=(0.8,2.6), dlam=40.0, flgd=True, ni If flgd='True' it is a bin in velocity (km/s). If flgd='False' it is a bin in linear space (microns). Default is: 40.0 (with flgd='True') - flgd : boolean + flgd : bool if flgd='True' (default) wavelengths are created with equal steps in log space. If 'False', wavelengths will be created wit equal steps in linear space. @@ -758,7 +770,7 @@ def create_OHlinelist(resolution, waveminmax=(0.8,2.6), dlam=40.0, flgd=True, ni iraf_frmt : bool if True, the file is written in the IRAF format (i.e. wavelength, ion name, amplitude). - debug : boolean + debug : bool If True will show debug plots """ @@ -808,7 +820,7 @@ def create_ThArlinelist(resolution, waveminmax=(3000.,10500.), dlam=40.0, flgd=T If flgd='True' it is a bin in velocity (km/s). If flgd='False' it is a bin in linear space (angstrom). Default is: 40.0 (with flgd='True') - flgd : boolean + flgd : bool if flgd='True' (default) wavelengths are created with equal steps in log space. If 'False', wavelengths will be created wit equal steps in linear space. @@ -832,9 +844,9 @@ def create_ThArlinelist(resolution, waveminmax=(3000.,10500.), dlam=40.0, flgd=T iraf_frmt : bool if True, the file is written in the IRAF format (i.e. wavelength, ion name, amplitude). - debug : boolean + debug : bool If True will show debug plots - convert_air_to_vac (bool): + convert_air_to_vac : bool If True, convert the wavelengths of the created linelist from air to vacuum """ diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index 18a9fa558c..7b4e805db9 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -79,7 +79,7 @@ class WaveTilts(calibframe.CalibFrame): 'find Tiltimg file when running pypeit_chk_tilts()'), 'tilt_traces': dict(otype=table.Table, descr='Table with the positions of the ' 'traced and fitted tilts for all the slits. ' - 'see :func:`make_tbl_tilt_traces` for more details. ') + 'see :func:`~pypeit.wavetilts.BuildWaveTilts.make_tbl_tilt_traces` for more details. ') } def __init__(self, coeffs, nslit, spat_id, spat_order, spec_order, func2d, bpmtilts=None, @@ -96,7 +96,7 @@ def _bundle(self): """ Bundle the data in preparation for writing to a fits file. - See :func:`pypeit.datamodel.DataContainer._bundle`. Data is + See :func:`~pypeit.datamodel.DataContainer._bundle`. Data is always written to a 'TILTS' extension. """ _d = super(WaveTilts, self)._bundle(ext='TILTS') @@ -116,7 +116,7 @@ def is_synced(self, slits): Barfs if not Args: - slits (:class:`pypeit.slittrace.SlitTraceSet`): + slits (:class:`~pypeit.slittrace.SlitTraceSet`): """ if not np.array_equal(self.spat_id, slits.spat_id): @@ -131,6 +131,7 @@ def fit2tiltimg(self, slitmask, flexure=None): Args: slitmask (`numpy.ndarray`_): + ?? flexure (float, optional): Spatial shift of the tilt image onto the desired frame (typically a science image) @@ -188,27 +189,32 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False): If True, show the traces of the tilts on the image. """ + # get tilt_img_dict + if (Path(self.calib_dir).resolve() / self.tiltimg_filename).exists(): + tilt_img_dict = buildimage.TiltImage.from_file(Path(self.calib_dir).resolve() / self.tiltimg_filename) + else: + msgs.error(f'Tilt image {str((Path(self.calib_dir).resolve() / self.tiltimg_filename))} NOT FOUND.') + # get slits + slitmask = None if (Path(self.calib_dir).resolve() / self.slits_filename).exists(): slits = slittrace.SlitTraceSet.from_file(Path(self.calib_dir).resolve() / self.slits_filename) - slitmask = slits.slit_img(initial=True, flexure=self.spat_flexure) - left, right, mask = slits.select_edges(flexure=self.spat_flexure) - gpm = mask == 0 + _slitmask = slits.slit_img(initial=True, flexure=self.spat_flexure) + _left, _right, _mask = slits.select_edges(flexure=self.spat_flexure) + gpm = _mask == 0 + # resize + slitmask = arc.resize_mask2arc(tilt_img_dict.image.shape, _slitmask) + left = arc.resize_slits2arc(tilt_img_dict.image.shape, _slitmask.shape, _left) + right = arc.resize_slits2arc(tilt_img_dict.image.shape, _slitmask.shape, _right) else: slits = None msgs.warn('Could not load slits to show with tilts image.') - # get tiltimg - if (Path(self.calib_dir).resolve() / self.tiltimg_filename).exists(): - tilt_img_dict = buildimage.TiltImage.from_file(Path(self.calib_dir).resolve() / self.tiltimg_filename) - tilt_img = tilt_img_dict.image * (slitmask > -1) if slits is not None else tilt_img_dict.image - else: - msgs.error('Tilt image not found.') - # get waveimg - if waveimg is None and in_ginga: + same_size = (slits.nspec, slits.nspat) == tilt_img_dict.image.shape + if waveimg is None and slits is not None and same_size and in_ginga: wv_calib_name = wavecalib.WaveCalib.construct_file_name(self.calib_key, calib_dir=self.calib_dir) - if Path(wv_calib_name).resolve().exists() and slits is not None: + if Path(wv_calib_name).resolve().exists(): wv_calib = wavecalib.WaveCalib.from_file(wv_calib_name) tilts = self.fit2tiltimg(slitmask, flexure=self.spat_flexure) waveimg = wv_calib.build_waveimg(tilts, slits, spat_flexure=self.spat_flexure) @@ -216,6 +222,8 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False): msgs.warn('Could not load Wave image to show with tilts image.') # Show + # tilt image + tilt_img = tilt_img_dict.image * (slitmask > -1) if slitmask is not None else tilt_img_dict.image # set cuts zmax = stats.sigma_clip(tilt_img, sigma=10, return_bounds=True)[2] zmin = stats.sigma_clip(tilt_img, sigma=5, return_bounds=True)[1] * 2 @@ -247,16 +255,16 @@ class BuildWaveTilts: Class to guide arc/sky tracing Args: - mstilt (:class:`pypeit.images.buildimage.TiltImage`): + mstilt (:class:`~pypeit.images.buildimage.TiltImage`): Tilt image. QA file naming inherits the calibration key (``calib_key``) from this object. - slits (:class:`pypeit.slittrace.SlitTraceSet`): + slits (:class:`~pypeit.slittrace.SlitTraceSet`): Slit edges - spectrograph (:class:`pypeit.spectrographs.spectrograph.Spectrograph`): + spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): Spectrograph object - par (:class:`pypeit.par.pypeitpar.WaveTiltsPar` or None): + par (:class:`~pypeit.par.pypeitpar.WaveTiltsPar` or None): The parameters used to fuss with the tilts - wavepar (:class:`pypeit.par.pypeitpar.WaveSolutionPar` or None): + wavepar (:class:`~pypeit.par.pypeitpar.WavelengthSolutionPar` or None): The parameters used for the wavelength solution det (int): Detector index qa_path (:obj:`str`, optional): @@ -267,22 +275,24 @@ class BuildWaveTilts: Attributes: - spectrograph (:class:`pypeit.spectrographs.spectrograph.Spectrograph`): - steps : list - mask : ndarray, bool - True = Ignore this slit - all_trcdict : list of dict - All trace dict's - tilts : ndarray - Tilts for a single slit/order - all_ttilts : list of tuples - Tuple of tilts ndarray's - final_tilts : ndarray - Final tilts image + spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): + ?? + steps (list): + ?? + mask (`numpy.ndarray`_): + boolean array; True = Ignore this slit + all_trcdict (list): + All trace dict's + tilts (`numpy.ndarray`_): + Tilts for a single slit/order + all_tilts (list): + Tuple of tilts `numpy.ndarray`_ objects + final_tilts (`numpy.ndarray`_): + Final tilts image gpm (`numpy.ndarray`_): - Good pixel mask - Eventually, we might attach this to self.mstilt although that would then - require that we write it to disk with self.mstilt.image + Good pixel mask. Eventually, we might attach this to self.mstilt + although that would then require that we write it to disk with + self.mstilt.image """ # TODO This needs to be modified to take an inmask @@ -375,12 +385,16 @@ def find_lines(self, arcspec, slit_cen, slit_idx, bpm=None, debug=False): Wrapper to tracewave.tilts_find_lines() Args: - arcspec: - slit_cen: + arcspec (): + ?? + slit_cen (): + ?? slit_idx (int): Slit index, zero-based bpm (`numpy.ndarray`_, optional): + ?? debug (bool, optional): + ?? Returns: tuple: 2 objectcs @@ -431,20 +445,18 @@ def fit_tilts(self, trc_tilt_dict, thismask, slit_cen, spat_order, spec_order, s Args: trc_tilt_dict (dict): Contains information from tilt tracing - slit_cen (ndarray): (nspec,) Central trace for this slit + slit_cen (`numpy.ndarray`_): (nspec,) Central trace for this slit spat_order (int): Order of the 2d polynomial fit for the spatial direction spec_order (int): Order of the 2d polytnomial fit for the spectral direction slit_idx (int): zero-based, integer index for the slit in question - - Optional Args: - show_QA: bool, default = False + show_QA (bool, optional): show the QA instead of writing it out to the outfile - doqa: bool, default = True + doqa (bool, optional): Construct the QA plot Returns: - `numpy.ndarray`_: coeff: ndarray (spat_order + 1, spec_order+1) - Array containing the coefficients for the 2d legendre polynomial fit + `numpy.ndarray`_: Array containing the coefficients for the 2d + legendre polynomial fit. Shape is (spat_order + 1, spec_order+1). """ # Index self.all_fit_dict[slit_idx], self.all_trace_dict[slit_idx] \ @@ -465,7 +477,6 @@ def trace_tilts(self, arcimg, lines_spec, lines_spat, thismask, slit_cen, Trace the tilts Args: - arcimg (`numpy.ndarray`_): Arc image. Shape is (nspec, nspat). lines_spec (`numpy.ndarray`_): @@ -484,7 +495,8 @@ def trace_tilts(self, arcimg, lines_spec, lines_spat, thismask, slit_cen, Integer index indicating the slit in question. Returns: - dict: Dictionary containing information on the traced tilts required to fit the filts. + dict: Dictionary containing information on the traced tilts required + to fit the filts. """ trace_dict = tracewave.trace_tilts(arcimg, lines_spec, lines_spat, thismask, slit_cen, @@ -505,7 +517,7 @@ def model_arc_continuum(self, debug=False): The method uses the arc spectra extracted using :attr:`extract_arcs` and fits a characteristic low-order continuum for each slit/order using - :func:`pypeit.util.robust_polyfit_djs` and the parameters + :func:`~pypeit.core.fitting.robust_fit` and the parameters `cont_function`, `cont_order`, and `cont_rej` from :attr:`par`. The characteristic continuum is then rescaled to match the continuum at each spatial position in the @@ -525,7 +537,7 @@ def model_arc_continuum(self, debug=False): Run the method in debug mode. Returns: - numpy.ndarray: Returns a 2D image with the same shape as + `numpy.ndarray`_: Returns a 2D image with the same shape as :attr:`mstilt` with the model continuum. """ # TODO: Should make this operation part of WaveTiltsPar ... @@ -651,9 +663,9 @@ def run(self, doqa=True, debug=False, show=False): self.arccen, self.arccen_bpm = self.extract_arcs() # TODO: Leave for now. Used for debugging -# self.par['rm_continuum'] = True -# debug = True -# show = True + #self.par['rm_continuum'] = True + #debug = True + #show = True # Subtract arc continuum _mstilt = self.mstilt.image.copy() @@ -690,11 +702,10 @@ def run(self, doqa=True, debug=False, show=False): # Loop on all slits for slit_idx, slit_spat in enumerate(self.slits.spat_id): if self.tilt_bpm[slit_idx]: - msgs.info('Skipping bad slit {0}/{1}'.format(slit_idx, self.slits.nslits)) + msgs.info(f'Skipping bad slit/order {self.slits.slitord_id[slit_idx]} ({slit_idx+1}/{self.slits.nslits})') self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADTILTCALIB') continue - #msgs.info('Computing tilts for slit {0}/{1}'.format(slit, self.slits.nslits-1)) - msgs.info('Computing tilts for slit {0}/{1}'.format(slit_idx, self.slits.nslits)) + msgs.info(f'Computing tilts for slit/order {self.slits.slitord_id[slit_idx]} ({slit_idx+1}/{self.slits.nslits})') # Identify lines for tracing tilts msgs.info('Finding lines for tilt analysis') self.lines_spec, self.lines_spat \ @@ -930,12 +941,15 @@ def show_tilts_mpl(tilt_img, tilt_traces, show_traces=False, left_edges=None, tilt_img (`numpy.ndarray`_): TiltImage tilt_traces (`astropy.table.Table`_): - Table containing the traced and fitted tilts. - See :func:`make_tbl_tilt_traces` for information on the table columns. + Table containing the traced and fitted tilts. See + :func:`~pypeit.wavetilts.BuiltWaveTilts.make_tbl_tilt_traces` for + information on the table columns. show_traces (bool, optional): Show the traced tilts - left_edges, right_edges (`numpy.ndarray`_, optional): - Left and right edges of the slits + left_edges (`numpy.ndarray`_, optional): + Left edges of the slits + right_edges (`numpy.ndarray`_, optional): + Right edges of the slits slit_ids (`numpy.ndarray`_, optional): Slit IDs cut (tuple, optional): diff --git a/setup.cfg b/setup.cfg index bbb6f42214..7b7b5c476c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,19 +32,19 @@ python_requires = >=3.9,<3.12 setup_requires = setuptools_scm include_package_data = True install_requires = - numpy>=1.21 + numpy>=1.22 astropy>=4.3 extension-helpers>=0.1 packaging>=0.19 scipy>=1.7 - matplotlib>=3.3 + matplotlib>=3.7 PyYAML>=5.1 configobj>=5.0.6 scikit-learn>=1.0 IPython>=7.10.0 - ginga>=3.2 + ginga>=4.1.1 linetools - qtpy>=1.9 + qtpy>=2.0.1 pygithub bottleneck pyqt6 @@ -67,20 +67,22 @@ test = pytest-cov coverage docs = - sphinx<6,>=1.6 - docutils<0.18 + sphinx<7,>=1.6 + docutils<0.19 sphinx-automodapi - sphinx_rtd_theme==1.1.1 + sphinx_rtd_theme==1.2.2 dev = + scikit-image + specutils pytest>=6.0.0 pytest-astropy tox pytest-cov coverage - sphinx<6,>=1.6 - docutils<0.18 + sphinx<7,>=1.6 + docutils<0.19 sphinx-automodapi - sphinx_rtd_theme==1.1.1 + sphinx_rtd_theme==1.2.2 [options.package_data] * = *.rst, *.txt, data/*, data/*/*, data/*/*/*, data/*/*/*/*, data/*/*/*/*/*, data/*/*/*/*/*/* @@ -89,6 +91,7 @@ dev = console_scripts = # non-GUI scripts + pypeit_arxiv_solution = pypeit.scripts.arxiv_solution:ArxivSolution.entry_point pypeit_cache_github_data = pypeit.scripts.cache_github_data:CacheGithubData.entry_point pypeit_chk_for_calibs = pypeit.scripts.chk_for_calibs:ChkForCalibs.entry_point pypeit_chk_noise_1dspec = pypeit.scripts.chk_noise_1dspec:ChkNoise1D.entry_point diff --git a/tox.ini b/tox.ini index 13cffacbb2..f568605e74 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] envlist = - py{39,310,311}-test{,-alldeps,-shapely,-specutils}{,-cov} - py{39,310,311}-test-numpy{120,121,122,123} - py{39,310,311}-test-{numpy,astropy,linetools,ginga}dev + {3.9,3.10,3.11}-test{,-alldeps,-shapely,-specutils}{,-cov} + {3.9,3.10,3.11}-test-numpy{122,123,124,125} + {3.9,3.10,3.11}-test-{numpy,astropy,linetools,ginga}dev codestyle requires = setuptools >= 30.3.0 @@ -36,21 +36,19 @@ description = devdeps: with the latest developer version of key dependencies oldestdeps: with the oldest supported version of key dependencies cov: and test coverage - numpy120: with numpy 1.20.* - numpy121: with numpy 1.21.* numpy122: with numpy 1.22.* numpy123: with numpy 1.23.* numpy124: with numpy 1.24.* + numpy125: with numpy 1.25.* # The following provides some specific pinnings for key packages deps = cov: coverage - numpy120: numpy==1.20.* - numpy121: numpy==1.21.* numpy122: numpy==1.22.* numpy123: numpy==1.23.* numpy124: numpy==1.24.* + numpy125: numpy==1.25.* numpydev: numpy>=0.0.dev0 astropydev: git+https://github.com/astropy/astropy.git#egg=astropy