diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index aba0c60f0..000000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore index 1e436d398..dd4a6aca2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ src/*.o */.DS_Store .DS_Store pro/.DS_Store +*.pyc diff --git a/Makefile b/Makefile index 2baa87b9e..01627a02e 100644 --- a/Makefile +++ b/Makefile @@ -44,8 +44,9 @@ all : # Used to (re)make the documentation files # doc : - $(MAKE) -C doc clean - $(MAKE) -C doc all + $(MAKE) -C $(MAKE) -C docs/sphinx/ clean + $(MAKE) -C $(MAKE) -C docs/sphinx/ html singlehtml latexpdf + $(MAKE) -C $(MAKE) -C docs/sphinx/ clean_tmp # # Install things in their proper places in $(INSTALL_DIR) # diff --git a/RELEASE_NOTES b/RELEASE_NOTES deleted file mode 100644 index fbb1fa849..000000000 --- a/RELEASE_NOTES +++ /dev/null @@ -1,5796 +0,0 @@ -v5_11_0 -Tag for final eBOSS pipeline -JEB (2018-09-24) - -Removing exposure 258988 of plate 9438 mjd 58125 because of trail in data -- pro/spec2d/spplan2d.pro - -Added iterative sky-subtraction. Basically we just perform the -sky subtraction two times. This reduces the average sky residuals. -Added nskyiter option to spreduce2d.pro -- pro/spec2d/spreduce2d.pro -- pro/spec2d/spreduce.pro -- pro/spec2d/extract_object.pro -- pro/spec2d/skysubtract_iter.pro - -Fixed bug on the djs_median calls where the reflect option was -not correctly implemented. This was causing some extra residuals -near the edges of the spectra. -- spflux_v5.pro -- spcoadd_v5.pro -- skysubtract.pro -- extract_bundle_image.pro -- combine1fiber.pro - - -------------------------------------------------------------------------------- -v5_10_11 -JEB (2018-07-24) - -Implemented DESI stellar templates for flux calibration -- pro/spec2d/spflux_read_desi.pro (new) -- pro/spec2d/spflux_v5.pro -Tests on this change are documented in -https://trac.sdss.org/attachment/wiki/eBOSS/Pipeline/desi_stars_in_eboss.pdf - - -JEB( 2018-07-09) - -Increased by one digit EXPID header keywords to solve -issue with plate 6138-56598 which contains more than 100 frames -- bolton_biasgen.pro -- spbiasgen.pro -- spcoadd_v5.pro -- spflatgen.pro -- spflatten2.pro -- bin/reformat_spectra.py - -------------------------------------------------------------------------------- -v5_10_10 -JEB (2018-04-15) - -Fixed bug (nPoly was still set to 2 while it should be 0 for the new -extraction) -- pro/spec2d/spcalib.pro - -Fixed MAG field in eboss23 ELG plates with FIBER2MAG from DECals data. -This modification happened directly in $SPECLOG_DIR. - - -------------------------------------------------------------------------------- -v5_10_9 - -JEB (2018-03-02) - -Implementation of background fit in extraction provided by Julien Guy. -The background contribution is removed from the full 2D image using -the flux between bundles. -- pro/spec2d/extract_bundle_image.pro -- pro/spec2d/extract_bundle_row.pro -- pro/spec2d/extract_object.pro - -Modified algorithm to compute plate S/N for ELGs. We are doing similar than -the SOS, using z-band fluxes free of emission lines. -We did not implement the artificial scaling of S/N values for the offline -reductions. -- pro/spec2d/platesn.pro - -This version will require use of version 1_29 of SPECFLAT -- etc/idlspec2d.module - -------------------------------------------------------------------------------- -v5_10_8 - -VM (2018-01-23) -Modifications for RM plates to have the original depth of B=10 and R=22 -Hardcoded plateids !! - -Changes in the following routines --pro/apo2d/quickextract.pro : Multiplied SN2 values by a scale factor=0.8 --pro/apo2d/apo_plotsn.pro : For RM plates, use plotsn_rm.pro - -Adding: --pro/spec2d/plotsn_rm.pro : Hardcoded scale factor = 0.8 -------------------------------------------------------------------------------- - -v5_10_7 - -VM (2017-10-30) --pro/spec2d/sdssproc.pro : Change in sdssproc_badformat routine to take care - of the shift in R2 camera transient location after electronics change on MJD 58055 - -JEB -- pro/spec2d/sdssproc.pro: updated gain values for all cameras - -------------------------------------------------------------------------------- -v5_10_6 - -VM (2017-10-03) --examples/opLimits.par : new SN2 thresholds 8 and 18 for blue and red cameras respectively. --pro/apo2d/quickextract_elg.pro : new ELG scalefactor 2.11 instead of 2.58 to match the new depth of 18 in red cameras --pro/spec2d/plotsn_elg.pro : new ELG scalefactor 2.11 instead of 2.58 to match the new eboss depth of 18 in red cameras - -JEB (2017-05-05) -- etc/lamphgcdne.dat : using MANGA list of lines that excludes faint ones. - - -JEB (2017-03-03) -- pro/spec2d/spcalib.pro: check if traces are too close only over valid pixels [#317] -- pro/spec2d/trace320crude.pro: use mpfitfun instead of djs_polyfit as suggested by David Law [#2068] - -VM(2016-12-21) Updating XMID & WAVEMID ranges for spectrograph 2 in examples/opLimits.par - -------------------------------------------------------------------------------- -v5_10_3 -VM (2016-11-19) SoS modifications --changed the scale factor in quickextract_elg.pro and plotsn_elg.pro - -JEB -- bin/fluxcorr_prior.py: fixed a bug (removing '.gz' from the framefile name) -- bin/reformat_spectra_daily: changed mkdir for makedirs to create spectra/full and spectra/lite -- bin/reformat_spectra_batch: changed mkdir for makedirs to create spectra/full and spectra/lite - -------------------------------------------------------------------------------- -v5_10_2 -VM (2016-09-09) SoS modifications to handle the ELG plates --Based on keyword 'programname' in plugmap files, ELG plates will be treated differently for SN2- - measurements. Use sky free wavelengths in z-band to measure meansn2. --Included a scale factor in quickextract_elg.pro and plotsn_elg.pro to tune the ELG depth. --Included a constant (=0.083 corresponding to the scalefactor) in the equation on SN plots; plotsn_elg.pro --Updated opSNlimits.par to include z-band with a slope 0.6 --Updated new gain measurements post 2016 Summer shutdown --Updated plotsn to exclude objtype='NA' - -Adding: -- pro/apo2d/quickextract_elg.pro -- pro/spec2d/plotsn_elg.pro - -Updating: -- pro/apo2d/aporeduce.pro -- pro/apo2d/quicktrace.pro -- pro/apo2d/apo_plotsn.pro -- pro/spec2d/sdssproc.pro -- opfiles/opSNlimits.par - -------------------------------------------------------------------------------- -v5_10_1 -JEB (2016-06-17) Post DR14 tag for daily reductions - -- Few bugs fixed after DR14 run - -------------------------------------------------------------------------------- -v5_10_0 - -JEB (2017-01-19) Fix for DR14 tag -- pro/spec1d/spreduce1d.pro: change of specobjid type from LONG64 to ULONG64 -- pro/spec1d/readspec.pro: adding /unsigned keyword to get correct values for SPECOBJID in zans structure -- bin/reformat_spectra.py: replace fitsio by astropy.io.fits (fitsio crashes with uint64 values) in tables - -JEB (2016-05-24) Tag for DR14. -New python fluxcorr code to avoid biases for low S/N spectra (by Stephen Bailey) -IDL code to perform ADR corrections on individual exposures (by Daniel Margala, S. Bailey and J. Bautista) - -Adding: -- bin/fluxcorr_prior.py -- pro/spec2d/xythrucorr.pro -- pro/spec2d/fiberfraction.pro -Updating: -- pro/spec2d/spcoadd_v5.pro -- pro/spec2d/spcombine_v5.pro -- pro/spec2d/spflux_v5.pro -- pro/spec2d/spfluxcorr_v5.pro -- pro/spec2d/spframe_read.pro - -- etc/idlspec2d.module : adding platedesign and yannytools modules, required by the new changes. - - VM (2016-02-29) --pro/apo2d/quicktrace.pro : Hack to avoid resetting the magnitudes to 0 for OBJTYE = NA for ELG test plates (8954 to 8959) - - -------------------------------------------------------------------------------- -v5_9_1 -Tag for post-DR13 daily runs - -VM (2015-11-13) --pro/apo2d/quicktrace.pro : Hack to avoid resetting the magnitudes to 0 for OBJTYE = NA for QSO test plates (8788 to 8793) --bin/guidermon_checkin.sh : updated the script to point to correct SPECLOG directory -v5_9_0 (2015-11-05) -Fix of the previous v5_9_0 for DR13 - - -JEB (2015-10-12) - -- pro/plate/plate_to_string.pro: - Correcting plate number definition for plates >= 10k -- pro/spec1d/platemerge.pro: - Correcting trailing blank spaces when generating spAll for some programs - -------------------------------------------------------------------------------- -v5_9_0 -Tag for DR13 - -JEB (2015-09-24) - -- bin/reformat_spectra_daily: - Script to generate spec files after the daily reductions -- bin/uurundaily_script: - Added reformat_spectra_daily command -- bin/reformat_spectra.py: - If update option on, look for existing spec files per plate-mjd-fiber - (not per plate only) -- bin/reformat_spectra_batch: - Include --update option for catch up of missing plates using the utah cluster -- pro/spec2d/spplan2d.pro: - Add dr13 keyword to exclude eboss plates and some RM plates. - -------------------------------------------------------------------------------- -v5_8_2 -Test tag for DR13 - -JEB (2015-09-18) - -platelist.pro: adding eBOSS LRGs and ELGs to the N_gal column and eBOSS quasars - to N_QSO column in platelist. - Added eBOSS LRGs to %LRG1 column in platequality. - Added eBOSS quasars to %QSO column in platequality. - Added a %ELG column in platequality. - - -JEB (2015-09-05) 6-digit plate number armaggedon fixed, but not tested yet with a - problematic plate - Added pro/plate/plate_to_string.pro to translate plate numbers into - strings. - -------------------------------------------------------------------------------- - -VM : v5_8_1 (2015-08-11) -Adding bin/speclog_svn_update.sh : script called by the cronjob to auto update the $SPECLOG_DIR every morning -Changes in the following files : -bin/sos_apostart : disabled the ssh agent -bin/sos_runnerd.py : disabled doBookKeeping -examples/opLimits.par : updated WAVEMID values post 2015 summershutdown -------------------------------------------------------------------------------- - -v5_8_0 (2015-06-12) -== New extraction algorithm by Julien Guy == - -For details, see notes on the new extraction on -https://trac.sdss.org/wiki/eBOSS/Pipeline/v5_8_0 - -sdssproc.pro : - - transmit the rdnoise vector - -extract_object.pro : - - reject highrej=4 lowrej=4 - -extract_bundle_image : - - new argument :use_image_ivar=use_image_ivar - - saves chi2pdf - - rejection of pixels based on chi2pdf per spectrum - -extract_bundle_row : - - new args rdnoise, chi2pdf=chi2pdf, use_image_ivar=use_image_ivar - - lots of changes (see notes) - -skysubtract.pro : - - create model of ivar (complex) - - fit - - put back variane - - compute error on sky model - -combine1fiber.pro : - - create model of ivar (simple) - - fit - - put back variane - - compute error on sky model - -spflux_v5.pro : - - outlier rejection when combining several exposures of std star - -spfluxcorr_v5.pro : - - don't fit spfluxcorr if very few valid data - -spcalib.pro : - - highrej = 100 lowrej = 100 (100 sigma!!) - - use_image_ivar=1 ; JG more robust to trace offsets for arcs - -=== New S/N calculations for plates == -Mode details on : -https://trac.sdss.org/wiki/eBOSS/Pipeline/PlateSN - -fitsn_jb: - - new formula that applies for a wider magnitude range - -general_sn: - - implementation of formula above S/N = a*flux/ sqrt( flux + b) - -plotsn_jb: - - modified to deal with new S/N fits - -opSNlimits.par: - - new magnitude limits: [16., 24.] for all bands - - - -------------------------------------------------------------------------------- -v5_7_13 (2015-05-08) JEB - -spreduce1d: fixed bug in the delta redshift to velocity conversion - - -------------------------------------------------------------------------------- -v5_7_12 - -VM 2015-04-21: -opLimits.par : Changed b1 camera XSIGMA/WSIGMA tolerances to >1.25 (yellow) and >1.30 (red) - -SJB 2015-04-13: -apofix: correct logic when SOS logfile doesn't exist and print more - helpful status/error messages. - -VM 2015-03-23: -quicktrace : hack to ignore OBJTYPE="NA" only for plateid > 7517 (eboss) - -------------------------------------------------------------------------------- -v5_7_9 (2015-03-10) JEB - -uuplotspec: improvements for visual inspection, new database. - People can now login with trac.sdss.org username and password and - save inspections. - We did this to visual inspect LRGs and obtain n(z). - -plotsn: coeffs keyword to recover fitsn results for each spectro and each band -plotsn: use dereddened magnitudes in all fits -platesn: same as plotsn - -opSnlimits: new slopes derived from the median distribution of slopes, when these were - let free in a test sample. The slopes are still fixed. - -VM 2015-02-23 -opSnlimits : g and i band sn fitting slope changed to -0.32 and -0.36 - - -QUICKTRACE: Hack to ignore the OBJTYPE='NA' targets to address ticket #2184. - -------------------------------------------------------------------------------- -v5_7_6 (2014-11-05) JEB - -VM 2014-12-06: -quickextract : adding 'splitsky' to r1 camera for MJD ge 56858 and r2 ge 55300 -readplugmap : edit for new extinctiion coefficients (Shalfy 2011) -reject_science : removed the last override 'qbad = 0' - -JEB 2014-11-05: -spreduce1d: change redshift range for eboss and sequels plates: - from [-0.01,1.] to [-0.01,2.] -spreduce2d: adding splitsky to r1 camera for MJD ge 56858 due to strange - behaviour -platemerge: added eBOSS mask bits -plotspec: added eBOSS mask bits -uuplotspec: added eBOSS mask bits - -uubatchpbs: added stderr in each plate folder in addition to stdout - -SJB+JEB 2014-10-25: -sdssproc: updated r1 gain for CCD that was replaced summer 2014. - -------------------------------------------------------------------------------- -v5_7_5 (2014-10-10) SJB -Missing one commit to trunk for yet another sdss3 -> sdss4 change. -Committing that and tagging again. - -------------------------------------------------------------------------------- -v5_7_4 (2014-10-10) SJB -Tag for SOS - -SJB 2014-09-22: -sos_runnerd: make robust to blank PLATETYP keyword for bias and darks. -guidermon_checkin: observer@sos3 -> eboss@sdss4-eboss changes; not yet working -loadSN2Value: BOSS -> eBOSS - -SJB 2014-08-28: -sos_runnerd: make more robust if munged data is missing NAME keyword. - -SJB 2014-08-25: -platelist: fix bug when creating platelist with multiple run2d -platemerge: (re)add ability to write a single spAll file with multiple RUN2D - versions in it. -reformat_spectra.py : add --mjd option - -------------------------------------------------------------------------------- -v5_7_3 (2014-08-26) SJB -Tag for new season of daily eBOSS data processing. - -SJB 2014-08-25: -loadSN2Value : update for new platedb connection methods. -apofix : change /not_sos option to /force, and check for eboss@sdss4-eboss - rather than checking for running on sos or sos3.apo.nmsu.edu - -DJS 2014-08-24 -apod: only reduce eBOSS files, not MaNGA. (This is the very much simplified -script for running SOS, without connecting to platedb or svn.) - -------------------------------------------------------------------------------- -v5_7_2 (2014-07-23 ?) SJB -Tag for final DR12 run of post-BOSS data - -JCR 2014-07-23: -extract_object: Looks up file spSplitSky.par in $SPECLOG_DIR/opfiles for -plate and camera to determine if splitsky should occur. If plate and -camera are in that file, then splitsky does not occur. - -SJB 2014-05-23: -spreduce1d: fix parsing bug when spPlate files have a different prefix -or plateids that have more than 4 digits. regex to the rescue. - -SJB 2014-04-28: -rundaily_riemann_script: make sure to svn update speclog and platelist trunk, -not some tagged version - -------------------------------------------------------------------------------- -v5_7_1 (2014-04-24) SJB -Tag for continuing with daily processing post BOSS LSS DR12. -Code is identical except addition of spx_to_sdss.pro - -DJS 2014-04-22: -spec1d/spx_to_sdss.pro : converter of spectroperfectionism output -to spPlate format for testing. - -------------------------------------------------------------------------------- -v5_7_0 (2014-04-10) SJB -Tag for BOSS main survey DR12 - -SJB 2014-04-09: -SPPLAN2D: typo! Exclude 'test' exposures. Only 'excellent'. Smithers. -SPHDRFIX: Include exposure number in the splog line when changing something. -SDSS_PLATE_SORT (ticket 1896): - - Don't propagate upstream bug where failed position matches could still - have thing_id>0. Replace with thing_id=-1. Photo data already 0. - - Replace ra,dec of failed matches with plug_ra, plug_dec. - - Final result: - thing_id=-1 means failed match and photo data=0 for those. - ra,dec = position match if successful, otherwise plug_ra,plug_dec - -SJB 2014-04-08: -SPPLAN2D: check PLATETYP keyword to reject non-BOSS non-EBOSS data. -bin/getPlugmap : update call to catPlPlugMapM to keep freaking SqlAlchemy - warning messages out of our plugmap files. - -------------------------------------------------------------------------------- -v5_6_12 (2014-04-04) SJB -Second pre-DR12 tag. - -BATCHMERGE: re-enabled pngs with a /makepng option - -------------------------------------------------------------------------------- -v5_6_11 (2014-04-24) SJB -Tag for pre-DR12 testing. Will move to v5_7_x series when we think we are -ready to go for real. - -SJB 2014-04-04: -PLATEMERGE: removing re-reading plugmap hack for propagating eboss_target0, - in preparation for pre-DR12 testing. -SPCOADD_V5: allow missing spectrographs. Missing data have flux=ivar=0 - but still have valid plugmap propagated. -ZCOMPUTE: if all ivar=0, don't bother trying to compute a redshift -SPREDUCE1D: if all ivar=0, flag ZWARNING with NODATA -PLUG2TSOBJ: bug fix in determining indir - -SJB 2014-04-01: -SPPLAN2D: tweaks to give more informative warning messages. - -SJB 2014-03-27: -opfiles/washers.par, spec2d/readplugmap.pro : -Added final entries for washer usage at the end of BOSS. - -SJB 2014-03-25: -SDSSPROC: refactor to separate options for pixflat from badpix mask. -Change default to using both. Use /nopixflat and /nopixmask to turn -them off. Updated multiple other codes to handle changed defaults. -apo2d/*.pro : start using pixflat and pixmask for apo2d routines. ---> Not yet tested with end-to-end pipeline run - -SJB 2014-03-25: -pro/testsuite/check_cmass_changes.scr.pro -Added hooks for comparing two productions for how many targets changed -from good to bad, etc. - -SJB 2014-02-06: -PLATELIST: added hooks for updating interactive platelist website. - -MDO 2014-02-05 - -COMBSMALLCOLLIMATE: Updated offsets from data on 56691 , second test, in both comsmallcollimate and in sosactor - for the record (offsets are now stored in Hartmann.__init__(),specifically the "fudge" dictionary that is defined at line 73. - https://trac.sdss3.org/browser/repo/ops/actors/sosActor/trunk/python - /sosActor/utils/boss_collimate.py (MDO) - - -SJB 2014-02-02: -sos_runnerd.py : add check for MaNGA plates and skip them for SOS. - -Make sure to run against a new version of specflat that includes badpixel mask and pixbias for r2 holiday 2013 problems (56649) MDO - -SJB 2014-01-22: -READPLUGMAP, PLATEMERGE : Added hooks to propagate eboss_target0 flags from - plateHoles files. Includes code in platemerge to override plugmap in - spPlate -- this should be removed for the DR12 tag. - -COMBSMALLCOLLIMATE: Updated offsets from data on 56528 for the record (offsets - are now stored in Hartmann.__init__(),specifically the "fudge" - dictionary that is defined at line 100. - https://trac.sdss3.org/browser/repo/ops/actors/sopActor/trunk/python - /sopActor/utils/boss_collimate.py (MDO) - -PLUGHISTORY: Report for cart #18, and don't crash when trying to read - data for MANGA data (ie, plate 6655 on cart 1). - Only do SDSS-I/II and BOSS plates by looking at "platetype". - Report fibers plugged less than 95% of the time. -READPLUGMAP: Fix typo in #906 PSF/fiber2flux corrections (CPL) -SPECTRO_SUMMARY: Merged with version in boss/papers/OneDee. Added entries - from DR9 publications which weren't calculated here. (SJB) -SPFLUX_V5: Wrote new SPFLUX_BALMERFIX routine to address ticket #1958 - in simplest possible manner. Called on line 1054 of main routine. - Comment that out to restore previous behavior. (ASB, 2014-Jan-17) - -------------------------------------------------------------------------------- -v5_6_5 (2013-06-24) ASB -Candidate tag for DR11 galaxy LSS analysis -Built against: - elodie/v1_3 - specflat/v1_20 - idlutils/v5_5_10 - -BOLTON_BIASSUB, SDSSPROC: New code to deal with r2 power-supply bias - instability over MJD range [56352, 56371] (Ticket #1759). -COMBSMALLCOLLIMATE: Change offsets because focus ring changes on 56434 (MDO) -PLATELIST: exclude LITTLE_COVERAGE, UNPLUGGED, and BAD_TARGET fibers from - statistics for platelist. Ticket #1727. -SPREDUCE1D: Add ability to override default redshift-fitting range for - galaxy templates by using a file $SPECLOG_DIR/opfiles/spPlateZrange.par - (ticket #1084). - -templates/spEigenGal-56436.fits: Copy of galaxy templates pEigenGal-55740.fits - with an extrapolation further to the red to logwave=3.0730, such that - these can be used to z=2. - -------------------------------------------------------------------------------- -v5_6_4 (2013-01-22) SJB -New tag for SOS for high read noise warning. - -FINDSPEC: Fix to work with new behaviour of PLATELIST looking in RUN2D - subdir names for the platelist.fits file -RDNOISE_HISTORY: Skip hartmanns -SDSSPROC: Add warning for high read noise in addition to low read noise. -SOS_LOGS_CONCAT, SOS_LOGS_PLOTSKY: New procs for looking at summary - SOS log statistics - -------------------------------------------------------------------------------- -v5_6_3 (2013-01-22) MDO -Changed collimate offsets and r1 focus to minimize xsig/wsig - -MDO 2013-01-21: -collimate.pro: add offset for r1 so focus includes minimizing -xsig+wsig in regions used for SOS -combsmallcollimate.pro: change offsets for xsig/wsig problems -from temperature and focus ring changes - -BAW 2013-01-16: -bin/knownMissing.py: script to search for missing files in $SPECTRO_REDUX. - -SJB 2012-01-11: -rundaily_riemann_script: always use trunk for platelist. -Turn off platemerge to reduce load on master node; this is -now done in the morning batchmerge job. - -SJB 2012-01-10: -PLATELIST: Added /alldirs option to call to get_mjd_dir when using it -to get a list of possible run2d and run1d dirs. This is required because -of recent change to get_mjd_dir to by default only return numeric subdirs. - -------------------------------------------------------------------------------- -v5_6_2 (2012-12-28) Tag for SoS with new offset values for XMID and WAVEMID - -MDO 2012-12-18: -combsmallcollimate: change offsets from tests on 12-12 and 12-13 due to temperature and focus ring changes. - -ASB 2012-12-17: -SPECTRO_SUMMARY: exclude BAD_TARGET spectra from effective Nspectra count. - -SJB 2012-12-17: -PLATEMERGE: gzip spAll*.dat file for faster downloads. Leave fits file as is. - -SJB 2012-12-05: -SPREDUCE1D : allow RUN1D strings which don't match vX_Y_Z. - -PLATELIST: Added columns PROGRAMNAME and TILEID to platelist.txt and .html - --> resolves ticket 1716. -PLATEMERGE: Don't include public bad plates in spAll (e.g. 3698). - -------------------------------------------------------------------------------- -v5_6_1 (2012-11-29) Tag for SoS to ignore cart<10 data - -SJB 2012-11-29: -bin/reformat_spectra.py: fix COEFF0 keyword when trimming ivar=0 leading - pixels. Back propagate by hand into v5_5_12. -bin/copy_spPlan.py : refactor to only copy plans for good plate-mjd - unless --bad option -BATCHPBS: if topdir option is used to override BOSS_SPECTRO_REDUX, - put that into the batch script as well. - -SJB 2012-11-28: -bin/sos_filesequencer: don't process arc/flat/science for cart<10 - to protect BOSS from MaNGA tests. Process dark/bias from any cart - to allow afternoon checkout tests while APOGEE carts are loaded. - -------------------------------------------------------------------------------- -v5_6_0 (2012-11-16) Tag to reprocess all data; starts a new season toward DR11 - -ASB 2012-11-15: Added MATCH_SPALL (routine) and - CHECK_CMASS_CHANGES (script) in pro/testsuite - -SJB 2012-11-02: - SDSSPROC : Added warning for readnoise<1.0 which could mean - CCD voltages are wrong - -SJB 2012-10-28: reformat_spectra.py - Added x column to exp HDUs of spec files. - -------------------------------------------------------------------------------- -v5_5_12 (2012-10-23) -- DR10 release candidate - -SJB 2012-10-23: -SPREDUCE2D: add photolog_version to log output -SPCOADD_V5: add EXTNAME keywords to spCFrame and spPlate files - -JB 2012-10-22: -PLATEMERGE, SPREDUCE1D: Add CAS-styled specobjid to spreduce1d output - -ASB 2012-10-20: -FLUX_DISTORTION, SPREDUCE1D: call SDSS_ASTROMBAD rather than looking -for CALIB_STATUS bit to flag bad fields. (Ticket #1583). - -MDO 2012-10-09: -SDSSPROC: add case to check if saturated pixel occurs within 3 pixels of top and bottom of image. (Ticket $1424) - -opfiles/washers.par: Remove duplicate entry 3537-55116 - -------------------------------------------------------------------------------- -v5_5_11 (2012-10-04) -- dry run for DR10 - -BATCHPBS: Added /skip1d option -BBSPEC_EXTRACT: bbspec ymodel moved to HDU 6 in extracted spectra. -BOSSLOG: New proc. -COMBINE1FIBER: only set NODATA in AND_MASK if there really is no data - on any input spectrum; resolves ticket 1301. -COMBINEBPM: updated hand edited bad columns (ticket 1424, MDO) -FLUX_DISTORTION: mask CALIB_STATUS->ASTROMBAD fluxes - (ticket #1583, ASB) -PLATELIST: Change defult input/output location to be in RUN2D subdir - to make it easier to work with multiple versions. Can be overriden - with TOPDIR and OUTDIR options, and multiple RUN2D versions can - still be combined into a single platelist using RUN2D='*' or - RUN2D=[x, y, z], in which case the platelist is written to - $BOSS_SPECTRO_REDUX (not in the RUN2D subdir). - Resolves ticket 1659. -PLATEMERGE: removed Z_PERSON, CLASS_PERSON, Z_CONF_PERSON, - COMMENTS_PERSON from spAll so that quasar catalogs can proceed on - timescale indepdenent of pipeline. DRxQ will be authoritative - source of FPG scans rather than also propagating to spAll. -PLATEMERGE: Added BOSSPRIMARY tag to replicate SPECPRIMARY but to be - immune from downstream modification; changed SPECOBJ_ID to - BOSS_SPECOBJ_ID (Ticket #1634). -READPLUGMAP: append CALIB_STATUS when /calibobj is passed - (ticket #1583, ASB) -SDSS_PLATE_SORT: Adding ERROR to error messages for easier grepping -SDSSPROC: added masking for hotpixel trails (MDO) -SN_MEDIAN: Added "sn_all" keyword for median S/N across all bands - (ticket #1634, ASB) -SPECTRO_SUMMARY: removed N_QSO_SCANNED, N_LYA_INCOMP, and N_LYA_IMPURE - statistics. -SPREDUCE1D: push CALIB_STATUS->ASTROMBAD to ZWARNING->BAD_TARGET - (ticket #1583, ASB) -SPREDUCE1D: Added SN_MEDIAN_ALL (median S/N across all bands) to - output structure (ticket #1634, ASB). -SPREDUCE1D: Added ZNUM_NOQSO (Brownstein) - -bin/copy_spPlan.py - New (SJB) - -------------------------------------------------------------------------------- -v5_5_10 (2012-08-21) - -BOLTON_BIASSUB: Bug fix for CRs in row-by-row kluge-processed overscan - region of wonky b2 quadrant (ASB, ticket #1651). -QUICKTRACE: Bug fix for SOS reductions, which now need to pass cartID - as a keyword to TRACE320CRUDE to be consistent with recent changes - (ticket #1656). - -------------------------------------------------------------------------------- -v5_5_9 (2012-08-15) -Tag for electronics bias drift on b2 UR quadrant. -Built against elodie/v1_3, specflat/v1_19, idlutils/v5_4_28+. - -APO_CHECKLIMITS: Bug fix to test against values of exactly zero, - although this wouldn't have been triggered. -BAD_BOSSFORMAT: Print mean and 95% of each sub-image. -BOLTON_BIASSUB: Adding kluge to do row-by-row bias subtraction for - upper right quadrant of b2 post-Summer 2012, due to crazy bias - phenomenon that has appeared there (ASB, 2012-Aug-15). -FITFLATWIDTH,SPCALIB,SPREDUCE,TRACE320CRUDE: Minor changes to use a - different number of traces for the SDSS-I spectra (ticket #541). -PLATELIST: Remove %Main from the HTML file, since those targets do - not exist in BOSS. -OPFIBER_GENERATE: Bug fix to work for SDSS-I plates with plate # < 1000 - and read traces from spFlat files for the old data model. -READPLUGMAP : reads opfiles/washer.par to get override status for - ZOFFSET. (SJB 2012-08-07) -SDSS_PLATE_SORT: The photoMatch,photoPlate,photoMatch files for the SDSS-I - plates are alread sorted to a particular plugging, and have MJD - in their names. Rather than assume these are sorted, take the first - matching file for a plate and re-sort, as we do for the BOSS files. -SPECTRO_SUMMARY: Summarize BOSS spectro totals from spAll file. - (ASB: motivated by DR9.) - -opfiles/opFibers.par: Add trace positions for SDSS-I spectra using - the following cart+plate+MJDs, but then edited to start at MJD=50000 - in this file: - 1 414 51901 - 2 400 51820 - 3 404 51877 - 4 418 51884 - 5 406 51869 - 6 431 51877 - 7 412 51931 - 8 410 51816 - 9 401 51788 - -------------------------------------------------------------------------------- -v5_5_8 (2012-04-05, ASB) -Tag for spectrophoto bug fix (Ticket #1509), as well as -various SOS-related changes, file cleanups, bug fixes, etc. -Built against elodie/v1_3, specflat/v1_18, idlutils/v5_4_28. - -Documentation: Fixed the formatting for IDL documentation in a number of files: - pro/spec1d/bandpassfilter.pro - pro/spec1d/filter_check_chi2.pro - pro/spec1d/filter_solve.pro - pro/spec1d/filter_check.pro - pro/spec1d/filter_select.pro - pro/spec1d/hogg_extinction.pro - pro/spec1d/bolton_mlpca.pro - pro/spec2d/extract_object.pro - pro/spec2d/bolton_sn_fit.pro - pro/spec2d/gausspix.pro - pro/spec2d/bbspec_test.pro - pro/spec2d/trace_sparse_crude.pro - pro/spec2d/smooth_halo.pro - pro/spec2d/smooth_halo2d.pro - pro/inspect/inspectgen.pro - pro/templates/pca_qso_dev.pro - pro/templates/pca_gal_dev.pro - pro/templates/mlpca_star.pro - pro/templates/align_rest.pro - pro/templates/mlpca_cvstar.pro -Fixing some files that don't need to be executable - examples/opECalib-51430.par - examples/opECalib-blank.par - examples/opECalib-51577.par - examples/opECalib-55026.par - -DECOLLIDE,RANDOM_DECOLLIDE: Fixed problem with duplicate function names. -EXTRACT_OBJECT: Commented out dispset in calls to SKYSUBSTRACT. - (ticket #1475, JCR). -GAMA_TO_SDSS: New proc for converting 2dF GAMA spectra files to spPlate - format for redshift-fitting with Spectro-1D. -PCA_QSO: Add /allatonce keyword. -SPFLUXCORR_V5: changed multiplicative polynomial fit parameters and - changed fit to depend on index of refraction of air (ASB, ticket #1509) - -Makefile (top-level): make doc is now not part of default install -Makefile (lower-level): All unnecessary Makefiles (that only did make install) - are removed. -.cvsignore: Removed unnecessary .cvsignore files -bin/idlspec2d_version: Change from bash to sh. -doc/mk_doc: Removed, document generation now handled directly by doc/Makefile -etc/emlines.par: The H_gamma line fit width was accidentally tied to the - forbidden lines rather than the Balmer lines. Fixed. -examples/opECalib-*.par: Removed executable tags on some files. -examples/opLimits.par: increased limits on r1 wsig by 0.05. - b1 xsig goes high for high fibers at middle pixel numbers but after - collimator swap still within old limits so didn't change - r1 wsig goes high for middle fibers at high pixel numbers -include/export.h: Remove old export.h and link to $IDL_DIR/external/export.h - This is the same as what is done in the idlutils product. -ups/idlspec2d.table: Remove explicit version dependency idlutils v5_4_26. - Remove outdated notation. - -------------------------------------------------------------------------------- -v5_5_7 (2012-01-18) ---> Installed at APO - -APO_PLOTSN: Line 92 changed to address ticket #1506: SOS not plotting - data from very low S/N exposures. - -bin/updatePluggingStatus: Remove to avoid conflict with version in - the platedb product. -ups/idlspec2d.table: Force explicit version dependency idlutils v5_4_26. - -------------------------------------------------------------------------------- -v5_5_4 - v5_5_6 (2012-1-10) -[several busted tag iterations sorting out dependencies] ---> Never installed at APO; skip to v5_5_7 - -BATCHPBS: Turned off calls to plate_spec_image: post-DR9 one should use - plotspec or the spectrawebapp for casual browsing of spectra. -BBSPEC_EXTRACT,FITSCATTER: Implement a naive scattered light model for - the bbspec extraction. -QUICKEXTRACT: Bug fix to sky-subtract in 25 bundles, not just 16. - Use 2-D sky-subtraction with NPOLY=3 in call to SKYSUBTRACT. -SKYSUBTRACT: Cleaned up use of DISPSET argument; needs test (ticket #1475). -VDISPFIT: Change to flatten probability in optional redshift marginalization, - may or may not be retained (ASB) -VDISP_PLATEFIT: New routine to facilitate re-batching of vdisp fitting code - (ASB) - -------------------------------------------------------------------------------- -v5_5_3 (2011-11-14) ---> Installed at APO -Tag for SOS with small changes after b2 triplet replacement. -Built against elodie/v1_3, specflat/v1_16, idlutils/v5_4_24. - -BOLTON_BIASSUB: add code to handle subarray readout bias subtraction - gracefully (ASB, re: ticket #1466) -COMBINEBPM : Added bad columns for 55800-55850 on r1. After putting the set - temperature for this CCD higher by 5 deg C, r1 looks good with no bad cols. -COMBSMALLCOLLIMATE: Changed offsets for b2 after triplet swap -SDSS_PLATE_SORT : Add MOD 360 to match_ra to prevent crashes on match_ra>360 - in photoMatchPlate files (affects plates 4405 and 4277). -SDSS_SPEC_IMAGE, PLATE_SPEC_IMAGE : Added /silent hooks - -bin/rundaily_cron, rundaily_riemann_script, rundaily_riemann: - Added silent hooks for sdss_spec_image and plate_spec_image - -------------------------------------------------------------------------------- -v5_5_2 (2010-10-19) -Tag for SOS bug fixes. -Built against elodie/v1_3, specflat/v1_16, idlutils/v5_4_24. - -COMBSMALLCOLLIMATE: Correctly apply offsets. -examples/opLimits.par: Tweaks to all XMID,WAVEMID values for year 3 (KSD). - -------------------------------------------------------------------------------- -v5_5_1 (2011-09-17) -Tag for SOS bug fixes. -Built against elodie/v1_3, specflat/v1_16, idlutils/v5_4_24. - -COMBSMALLCOLLIMATE: Update offsets using all hartmanns from 55800-817 - -examples/opLimits.par: Switch incorrectly-swapped the values for WAVEMID - for r1 & b1. Set error for EXPTIME < 300 sec. - Set error for SN2 less than 0.2. - -------------------------------------------------------------------------------- -v5_5_0 (2011-09-09) -Tag for full reductions using replacement r1 CCD. -Built against elodie/v1_3, specflat/v1_16, idlutils/v5_4_24. - -COLLIMATE: Add test keyword to help find offsets with SMALLCOLLIMATE. -COMBSMALLCOLLIMATE: Update offsets and add /TEST keyword. -PLATECOMPARE: Fix to work with change from SNMEDIAN from vector to array. -PLOTSPEC, UUPLOTSPEC: Show 3 digits after decimal in rchi2 (ASB) -SDSSPROC: Added new r1 CCD gains to sdssproc for data after MJD 55800: - amps 0-4 gains [1.9253, 1.5122, 1.4738, 1.5053] - -bin/reformat_spectra.py: New proc to reformat spectra into one-file-per-object - format (basically ticket #837) -bin/sos_apocommand : updatePluggingStatus no longer accepts -d -v options, - so remove them from the call. -bin/updatePluggingStatus - this has been replaced by - PLATEDB_DIR/bin/updatePluggingStatus. Rename this copy to .._old just - in case we need to get to it again. After next APO install, if things are - running smoothly and we don't need this, we can delete it for good (history - will still be in svn) - -------------------------------------------------------------------------------- -v5_4_45 (2011-08-27, ASB) -Bug fixes for DR9 data release. -Built against elodie/v1_3, specflat/v1_16, idlutils/v5_4_24. - -BUNDLETHRU: New proc to measure relative throughput of fiber bundles from - flat-fields -PLATESN,PLOTSN: Bug fix(es) for new calling sequence to FITSN, where we no - longer are passing the plugmap but instead the redden values. - -------------------------------------------------------------------------------- -v5_4_44 (2011-08-26, ASB) -Tag to catch that last change -If good, this tag will be the DR9 tag. -Built against elodie/v1_3, specflat/v1_16, idlutils/v5_4_24. - -EXTRACT_OBJECT: Compute the SKYCHI2 output in the science frame headers - only for good pixels (ticket #712) - -------------------------------------------------------------------------------- -v5_4_43 (2011-08-25, ASB) -Tag to clean up a bunch of bugs and straggler tickets. -If good, this tag will be the DR9 tag. -Built against elodie/v1_3, specflat/v1_16, idlutils/v5_4_24. - -BOLTON_BIASSUB, SDSSPROC: Fix to make use of SDSSPROC_BADFORMAT - image corrections (ASB, re: ticket # 1408) -CONFIGURATION__DEFINE: Remove some deprecated functions. -EXTRACT_BUNDLE_ROW, EXTRACT_BUNDLE_IMAGE: documentation cleanup - to reflect the bundle-wise extraction mods (ASB) -EXTRACT_OBJECT,SPCALIB,QUICKWAVE: Remove call to getDetectorFormat and - determine the image size from the image. -EXTRACT_OBJECT,FITSN,SPREDUCE: Pass the median reddening values for - the plate from the plugmap file (as "reddeningMed" in the header) - as a set of "REDDEN*" keywords in the output science frames. - Use these values for evaluating the reddening-corrected S/N values - in FITSN. (ticket #879) -FITSN: If there are fewer than 10 points in the mag fit range, - then extend that range to all brighter magnitudes. This will - not effect any proper survey plates. But it allows the S/N fit - to occur for some special plates, such as 5017-5019 which have - much fainter targets. For those plates, then the spectro-photo - stars are selected for the fit. (ticket #1351) -PLATEMERGE: Add DIMS0,DIMS1 keywords to spAllLine file (ticket #1031) -SDSSPROC: Add option to return CCDMASK pixel mask that sets the 'NODATA' - bit for regions of the BOSS CCD that are unused. This should allow - the option of including more mask bits at this stage to propogate - through the pipeline, such as CR masks. -SPBIASGEN: Add /NOPROC option for generating average bias frames - without overscan-trimming, etc. -SPCALIB: Remove some clutter of commented-out code. -SPECLINEFIT,LINEBACKFIT: Add new CONTRANGE keyword for setting the region - for the continuum-fitting for each line. A mean (unweighted) continuum - is evaluated in the blueward and redward bands, and then the average of - those two bands. This is used at a range of [300,600] km/s for stars - and galaxies. QSOs retain the default of selecting the continuum level - at the mid-point of the line, which is actually equivalent since the - background terms are all straight lines for QSOs. (ticket #1243) -SPREDUCE1D: Fix spurious "broadline" subclassing (ASB, ticket #1316) -SPREDUCE,EXTRACT_OBJECT: Make use of the CCDMASK from SDSSPROC, - and include those mask values through the pixel mask. A 'NODATA' - value is currently inherited from the initial SDSSPROC. - Erase several of the bits in the 'NODATA' regions: - ['NEARBADPIXEL','LOWFLAT','SCATTEREDLIGHT','NOSKY']. - This partially addresses ticket #1301. - -bin/loadSN2Value : fix platedb database connection method. -bin/sos_apostart: Correct instructions about which password to use for ssh - daemon. - -------------------------------------------------------------------------------- -v5_4_42 (2011-08-18, ASB) -Tag to fix 8000ang break feature with discontinuous sky model. -If good, this tag will be the DR9 tag. -Built against elodie/v1_3, specflat/v1_16, idlutils/v5_4_24. - -EXTRACT_OBJECT, SPREDUCE, SPREDUCE2D: Enabling split sky model between - spatial halves (ASB, chasing ticket #1388) - -Changed RAWDATA_DIR to BOSS_SPECTRO_DATA in 40 files (Ticket #1067). - Script that did this is in misc/rawdata2bossdata (ASB) - -------------------------------------------------------------------------------- -v5_4_41 (2011-08-08, ASB) -Tag for new bias recipe, new pixflats, misc 1d changes, misc 2d bugfixes. -Built against elodie/v1_3, specflat/v1_16, idlutils/v5_4_24. - -BOLTON_BIASGEN: Minimum-chi^2 master bias model-building routine (ASB) -BOLTON_BIASSUB: Bias-subtraction using bolton_biasgen outputs (ASB) -FITVACSET: Added handling of 2-phase discontinuity info (ASB) -SDSSPROC: Add measured readnoise for each amp as fits header keywords - RDNOISE0/1/2/3. We might want that info some day for noise modeling. -PLATEMERGE: Set SPECPRIMARY for GALAXY targets now based upon - ZWARNING_NOQSO (if present) rather than ZWARNING (ASB, ticket #1389) -PLATEMERGE: Test for inheritance of Z_NOQSO & related before computing - (see SPREDUCE1D changes) (ASB, ticket #1385). -SDSSPROC: Change to use BOLTON_BIASSUB for BOSS MJD >= 55170 (ASB) - (Will require coordinated tag with specflat.) -SPADD_GUIDERINFO: Limit reporting of seeing to in-focal-plane fibers only - (ticket #1391). -SPREDUCE1D: Compute Z_NOQSO and related values here, rather than in - PLATEMERGE (ASB, ticket #1385). - Also compute redshift-marginalized vdisp likelihood curves - with the Shu et al. 2011 parameter settings. -VDISPFIT: Subtract the # of invvar=0 pixels from vdispdof (ASB). - -------------------------------------------------------------------------------- -v5_4_40 (2011-07-05, ASB) -Tag for new bad-pixel masks, against specflat v1_14 - -COMBINEBPM: Update using bad pixel masks (for tickets 1363, 1368, 1369). -PLATEMERGE: Improved handling of best non-QSO redshifts (ASB) - No longer include the *NOQSO* entries unless /CALC_NOQSO is set. -SDSSPROC: When using /applypixflat, the (new) badpixel masks are applied - immediately after reading in the pixflat (ticket #1369). - -------------------------------------------------------------------------------- -v5_4_39 (2011-06-28, ASB) -Tag for a few 1D fiddles - -VDISPFIT: Bug fix when marginalizing over redshifts with new DZPIX keyword. -ZFIND: Change to disallow negative-model fits for single-component - template sets (i.e., most stars) upstream of ZWARNINGs (ASB) - -templates/spEigenGal-55740.fits: - Yanmei Chen's latest galaxy redshift templates, renamed to match - wildcarding, covering 1800 to 10500 Ang (ASB). - -------------------------------------------------------------------------------- -v5_4_38 (2011-06-21, ASB) -Tag for testing new redshift templates - -ATVSPEC: Add /verbose option. -BBSPEC_EXTRACT: New proc to act as wrapper to bbspec extraction code. - This first implementation only does a small sub-image for a single - fiber at a time. -BBSPEC_PIXPSF: Initial check-in for code that generates a pixelated PSF - from an arc frame. -EXTRACT_OBJECT: Add /bbspec option, and if set call BBSPEC_EXTRACT to replace - fluxes and errors. -FITDISPERSION: Change quadrupole region for /QUICK option for reporting - MEDWIDTH for different regions on the CCD (ticket #822). This - option used by QUICKWAVE. -FITFLATWIDTH: At least half the points should be good to fit the median - value within a bundle. -PCA_CVSTAR_BOSS: New version of PCA_CVSTAR for generating BOSS CV star templates -PCA_STAR_BOSS: New version of PCA_STAR for stellar templates - based upon extended/patched Indo-US archetypes resampled for BOSS, - plus some empirically derived templates of oddball stars from BOSS - spectroscopy (ASB). -PSOLVE: Initial check-in of routines used by BBSPEC_PIXPSF. -SDSSPROC: Remove the VARFILE option, and write the inverse variance - as HDU #1 of OUTFILE instead. Define a default naming scheme for OUTFILE. -SHIFT_TRACE: Remove this deprecated routine. -SPCALIB: Add /bbspec option, and if set then extract the flat-field with - the 2D PSF. -SPREDUCE: Add /bbspec option, and if set then generated sdProc and - spBasisPSFfiles for the best arc image. Pass keyword to EXTRACT_OBJECT. -SPREDUCE1D: Default for CHOP_DATA changed from [3650,9200] to [3600,10400] - (ticket #539). - Restore old NPOLY choices of 3 for galaxies, QSOs, CV stars, and - 4 for stars (ticket #851). - Change wildcard-matching for template files to explicitly have - a 5-digit MJD string. -SPREDUCE2D: Propogate all the test keywords to the recursive calling of - this routine. - Add /bbspec option. - -opfiles/lamplines.par: New file for lamp lines in bbspec PSF fitting. -templates/spArchIndoUS.fits,eigeninput_star_boss.dat,spEigenStar-55734.fits: - New stellar template files as input and output by PCA_STAR_BOSS. -templates/eigeninput_qso_boss.dat,spEigenQSO-55732.fits: - New QSO PCA redshift templates for BOSS (ASB) -templates/eigeninput_cvstar_boss.dat,spEigenCVstar-55734.fits: - New CV PCA redshift templates for BOSS (ASB). -templates/spEigenGal-55686.fits: - New GALAXY PCA redshift templates for BOSS, including - type II QSOs. Generated by BAW; checked in by ASB: -ups/idlspec2d.table: Add optional dependency on bbspec product. - -------------------------------------------------------------------------------- -v5_4_37 (2011-06-13) -Another tag for SOS addressing ticket #822. - -examples/opLimits.par: Switch the values for WSIGMA,XSIGMA - since those were in error (ticket #822). - -------------------------------------------------------------------------------- -v5_4_36 (2011-05-17) -Tag for SOS addressing ticket #822. - -etc/opLimits.par: Changed red and yellow limits on xsig,wsig for r1; - corrected order of r1 b1. - -------------------------------------------------------------------------------- -v5_4_35 (2011-04-22 by ASB) -Tag for further test of bundle-wise extraction and other fixes -Built against specflat v1_13, elodie v1_3, idlutils v5_4_23. - -CONFIGURATION__DEFINE: Increased ncoeff used for BOSS linewidth fits - to 5 (red) or 4 (blue) (ASB). -EXTRACT_ROW, EXTRACT_BUNDLE_ROW: moved separation check below "ngood" - check, to avoid pointless warnings on fully invvar=0 rows - (goes to ticket 1209) (ASB). -FITDISPERSION: Added maxdev=0.2 in traceset fit (ASB). -PLUGMAP_DIFF: New proc to find instances where the plPlugMapM file in - the speclog file differs from what was used in the reductions. - Used to see which fibers changed when fixing tickets #819,1262. -SPCALIB: new definition of "inmask" for trace-position fits, and - new value for "maxdev" (goes to ticket #1269) (ASB). -SPREDUCE1D: Force the SPEC2_* values in the ZANS structure to always - be type float (ticket #1290). - -------------------------------------------------------------------------------- -v5_4_34 (2011-04-21) -Tag for SOS errors/warnings. -Built against specflat v1_13, elodie v1_3, idlutils v5_4_23. - -APOREDUCE: Look for INFO as well as ABORT,WARNING messages in log file -COMBSMALLCOLLIMATE: Change default values of badres from 5 to 6 deg - (related to ticket #1266). -FITDISPERSION: Add /QUICK option for reporting MEDWIDTH for different - regions on the CCD (ticket #822). This option used by QUICKWAVE. -FITFLATWIDTH: Add /QUICK option to trim to the central region of CCD - and report MEDWIDTH in different regions. This option used by - QUICKTRACE (for flats) and QUICKEXTRACT (for science frames). -GAUSSPIX: New routine for pixel-integrated gaussians, similar to but - not to be confused with GAUSSPIX1D. -PLATEMERGE: Add XFOCAL,YFOCAL to spAll file (ticket #1271). - Change PLATE from an optional input to an optional keyword to allow spAll - generation for a subset of plates. -PLUGMAP_REPLACE: New proc for replacing plPlugMapM files in the speclog - product with new versions from re-running EVILREMAP on the mapper movies. -QUICKEXTRACT,QUICKTRACE: Use /QUICK flag in call to FITFLATWIDTH. -QUICKWAVE: Change reporting of dispersion warning messages to be consistent - with that reported by FITDISPERSION. Use /QUICK flag to FITDISPERSION. -READPLUGMAP: MOD the range of values for PLUG_RA to fall strictly in - the range [0,360) deg (tickets #1131, #1255). -SPADD_GUIDERINFO: Exclude NaN,Inf in guiderMon files when calculating - guider statistics (ticket #1276). -SPCALIB,EXTRACT_OBJECT,EXTRACT_BUNDLE_IMAGE,EXTRACT_BUNDLE_ROW: Replace calls - to EXTRACT_IMAGE with the newly-written bundle-by-bundle IDL extraction - code (ASB). -WRITESPEC, WRITE_UROS: Format file names for 1000-fiber (4 digit) fibers - (ticket #1275). - -bin/riemann_redux_snapshot: New script to make a snapshot of the BOSS - reductions of the platelist*,spAll* files. -examples/opLimits.par: Change red thresholds on XSIGMA from - 1.15,1.20,1.25,1.30 to 1.20,1.20,1.25,1.25 (with yellow lower by 0.05). - This is to be consistent with the changes to FITFLATWIDTH. - Change red thresholds on WSIGMA from 1.35,1.35,1.25,1.25 to - 1.25,1.30,1.20,1.25 (with yellow lower by 0.05) to be consistent - with changes in FITDISPERSION. -templates/eigeninput_typeIIqso.dat: Type II QSO training sample drawn - from BOSS galaxy targets, for GALAXY redshift templates (ticket #633). - Not yet used. - -------------------------------------------------------------------------------- -v5_4_33 (2011-03-30 by ASB) -Tag for testing bundle-wise IDL extraction -Built against specflat v1_13, elodie v1_3, idlutils v5_4_23. - -BATCHPBS: Change back to the original directory at end of proc. -EXTRACT_BUNDLE_ROW, EXTRACT_BUNDLE_IMAGE, GAUSSPIX (ASB, U. Utah): - Routines to enable pure IDL extraction, chunked up bundle-wise. -EXTRACT_OBJECT, SPCALIB (ASB, U. Utah): - Switched to pure IDL bundle-wise extraction. - Also addresses ticket #781, removing a redundant extraction step. -PLATEMERGE: The setting of SPECPRIMARY needs to address SN_MEDIAN - as a 5-element array rather than a scalar. Select based upon - this S/N in r-band. -READPLUGMAP: MOD the range of values for PLUG_RA to fall strictly in - the range [0,360) deg (ticket #1255). -VDISPFIT: Enable more sophisticated redshift-error marginalization. - Add DZPIX keyword. - -------------------------------------------------------------------------------- -v5_4_32 (2011-03-18) -Tag for testing spectrophotometry fixes. -Built against specflat v1_13, elodie v1_3, idlutils v5_4_23. - -BATCHPBS: Added -G -t GAL to runz_BOSS.sh call (ticket #1251). -CONFIGURATION__DEFINE,FITDISPERSION,QUICKWAVE,SPCALIB,TRACE320CRUDE: - Remove some unnecessary definitions from the configuration__define proc. -FITDISPERSION: In the call to XY2TRACESET, pass a mask of the badly - measured individual widths such that they do not throw the fits - (ticket #822). -FLUX_DISTORTION: For determining if this is a multiple-pointing plate, - don't look at missing plugmap entries. This change necessary for - the change with SPCOADD_V5 to include all zeros for a missing spectrograph. -PLATEMERGE: Modified to include minimal info on best non-QSO and - best non-galaxy redshifts and classifications by default. -SPCOADD_V5: Always set the number of output fibers as 2X the number on - a single spectrograph. Otherwise, if data from one BOSS spectrograph - is missing, then previous reductions made output files with 500 instead - of 1000 objects. Related to ticket #764. -SPFLUX_V5: In the internal function spflux_mratio_flatten, do a fit with - outlier rejection when fitting each star's mratio to the mean - (ticket #1249 and #1252). -SPPLAN2D: For any frames that are 'bad', enter them as 'UNKNOWN' - in the plan files so that we don't attempt to reduce them (ticket #764). - -etc/lamphgcdne.dat: Remove the Hg I lines at 4105 Ang, as it was the - culpret for ticket #822 (although the fix to FITDISPERSION should - make SOS robust to that anyway). -examples/opLimits.par: Change the S/N limits to be consistent with - the numbers hard-wired in PLATELIST. - Add yellow/red limits for %LRG1, %LRG2. - -------------------------------------------------------------------------------- -v5_4_31 (2011-03-02) -Tag for production 2D reductions. -Built against specflat v1_13, elodie v1_3, idlutils v5_4_23. - -BATCHPBS: Don't print run1d and run2d multiple times. -LOGSHEET: Also look for files that might be .fits instead of .fit, - as some were during the early BOSS commissioning. -PLATECOMPARE: Add support for RUN2D,RUN1D kewyords for comparing - different version reductions of the same plates. -PLATELIST: DeRedSN^2 -> DEREDSN2 in column name. - Shorten the SN2 column names for the html output. -SDSSPROC: The header values for the recently-added TWOPHASE keyword - were not properly boolian values, with the result that the blue CCDs - were having the 2-phase wavelength-correction accidentally applied - (ticket #1238). -SPFLUX_V5: Add another digit to fiber number (for printing fiber #1000). - -------------------------------------------------------------------------------- -v5_4_30 (2011-02-16) -Built against specflat v1_13, elodie v1_3, idlutils v5_4_23. - -BOLTON_STD_THROUGHPUT: new function to compute end-to-end - throughput on standard stars. -EXTRACT_OBJECT: Add DEREDSN2 keyword to spFrame output. -FITSN: If plugmap is provided, fill in dered_sn2 as dereddened - version of sn2 output (ticket #879). -PLATELIST: Output dered_sn2_* values to spAll files, using - those values instead of sn2_* for determining plate quality. - New keyword RAWSN2 to revert to previous behaviour. -PLATEMERGE: Make directory-concatenation OS-independent, and remove - undocumented use of a "spAll" subdir. - Optional keyword INDIR added. - Add DREDSN2 to the output file. -PLATESN: Add SN2EXT* keywords to spPlate output (ticket #879). -PLOTSN: Pass through dered_snplate from fitsn dered_sn2 output -SDSSPROC,SPCALIB: Move hard-coded logic of when the BOSS red CCDs switched - from 1-phase to 2-phase readout to be inside the SDSSPROC routine. -SN_MEDIAN: New proc to compute median spectroscopic S/N in the SDSS filters - (ticket #976). -SPREDUCE1D: Add call to SN_MEDIAN. -TRACE320CRUDE: Added condition on minimum fraction of traces for - application of polyfitw (ticket #1025) [ASB] - -bin/rundaily_riemann_script: Email Stephen instead of Gary when things go wrong - -------------------------------------------------------------------------------- -v5_4_29 (2011-02-01) -Built against specflat v1_12, elodie v1_3, idlutils v5_4_23. -Attempts by ASB to fix UPS dependencies. - -------------------------------------------------------------------------------- -v5_4_28 (2011-01-29) -Built against specflat v1_12, elodie v1_3, idlutils v5_4_23. -Attempts by ASB to fix UPS dependencies. - -------------------------------------------------------------------------------- -v5_4_27 (2011-01-29) -Tag for ticket #880. -Built against specflat v1_12, elodie v1_3, idlutils v5_4_23. - -SPCALIB, FITARCIMAGE: - Added code to handle 2-phase red-side BOSS readout discontinuity. - Coordinated with changes to XY2TRACESET, TRACESET2XY, TRACESET2PIX - in idlutils product, so need to update that dependent product. - Addresses ticket #880. - -------------------------------------------------------------------------------- -v5_4_26 (2011-01-20) - -COMBSMALLCOLLIMATE: Added offsets for focus ring moves from COLLIMATE - and COMBSMALLCOLLIMATE. - -------------------------------------------------------------------------------- -v5_4_25 (2011-01-20) -Tag for SOS collimation. - -BATCHPBS: Fix bug when passing plate numbers as a number rather than string. - Add /NOSUBMIT option. - Print name of script being submitted for later logfile grepping. - Change "script-XXX" to "redux-XXX" to fit within qstat display length. - Added plate_spec_image calls. -CCDTILT: Fix crash condition for /DOPLOT option. -COMBSMALLCOLLIMATE: Refine offsets to gives same results as - COLLIMATE using full Hartmanns taken from MJD 55440 to 55511 and tests - of -10000,0,10000 step offset and more tests from 55581. - b1 offset created using collimate, not collimate,/debug. - Setting slope equal to 1 for offset between collimate and - combsmallcollimate. - Minor change on b residual output -FINDSPEC: Works with SDSS-I,II data as well as BOSS data - Add /SDSS option that is hard-wired to run2d='26' -OPFIBER_GENERATE: New proc for generating the opFibers.par file - used for trace-finding in TRACE320CRUDE. -PLATELIST: Add option for SKIPCART to skip problematic carts (like the - current problems with cart 15). - Change S/N^2 thresholds to > 10 in B and > 22 in R. - Add function get_lastline to read the last line of log files rather - than making a call to SPAWN. -PLATEMERGE: Put spAll files into RUN2D subdirs instead of leaving at top - level $BOSS_SPECTRO_REDUX. -QUICKTRACE,SPCALIB,TRACE_CEN,TRACE320CEN: Changes to use new trace - parameters stored in the file opFibers.par . Ticket #817. -QUICKTRACE: Changes made to region used for quick_extract for - BOSS exposures to help with xsig. - Bug fix from hdr=flathdr to flathdr=flathdr in call to TRACE320CRUDE. -RUN_SDSSPROC: New proc for running SDSSPROC over many sdR files and - saving resulting image and invvar data to new files. -SDSSGUIDE: Guider images looks like they changed dimensions slightly - on MJD 55425. -UUBATCHPBS: Modified version of BATCHPBS to put many plates in a single job. -VDISPFIT: Fix bug for zobj=0. case - Add redshift marginalization and chi^2 map return options using - DZPIX and RETURN_CHISQ keywords. Do not use yet as default - behaviours for SPREDUCE1D. - -opfiles/opFibers.par: New file with trace parameters, generated by - OPFIBER_GENERATE and to be used by TRACE320CRUDE. - -------------------------------------------------------------------------------- -v5_4_20 (2010-10-27) -Tag for SOS S/N changes (ticket #1116). -Built against specflat v1_12, elodie v1_3, idlutils v5_4_19. - -TRACE320CRUDE: Added lines to ignore rows with invvar all zero (ticket #1025) - -bin/sos_apospammer: Added Stephen Bailey to SOS log spam list. -bin/updatePluggingStatus: Update help message. -examples/opLimts.par: Change S/N^2 limits to 10 (blue) and 22 (red cameras), - as per ticket #1116. - -------------------------------------------------------------------------------- -v5_4_19 (2010-09-13) -Tag to fix a number of bugs, esp. ticket #1043 -Built against specflat v1_12, elodie v1_3, idlutils v5_4_19. - -APO_APPENDLOG: Use COPY_STRUCT_INX to copy into an existing data structure, - rather than an assignment that could fail if structures are different. -APOD: Fix logic for re-reducing good arcs that precede good flats. -APOREDUCE: Last tag replaced one instance of 'gs' with '/usr/bin/convert', but - do the second instance too. - Print the stdout and sterr from the call to convert. -QUICKBIAS,QUICKEXTRACT,QUICKTRACE,QUICKWAVE: Explicitly set types of all - output structure elements, which may help address ticket #1043. -QUICKEXTRACT: Force data type of EXPTIME and SKYLEVEL to be floating-point. - The ICC changes can result in an EXPTIME that is either float or double, - and that gets propogated through and can result in a data-type mismatch - (ticket #1043). - to type double. It appears this is the variable that is k -READSPEC: Restore ability to read SDSS spectra, particularly the SDSS-I - photoPosPlate files. -SDSSPROC: Failed on line 363 if passing this routine an array of filenames, - even a 1-element array. Internally recast INFILE as a scalar to - protect against that case (although never encountered in the pipeline). - -bin/sos_apocontrol: stopKill now kills all child processes when called. This - will (hopefully) kill any stuck idl jobs. - -------------------------------------------------------------------------------- -v5_4_18 (Sep 11, 2010) idlspec2d -Tag for SOS critical bug (ticket #1034). -Built against specflat v1_12, elodie v1_3, idlutils v5_4_19. - -APO_APPENDLOG: Change timeout to 200 sec (was accidentally 400 sec). -APOREDUCE: Use 'convert' instead of 'gs' to convert PostScript plots - into JPEGs. Write to a JPEG file that is uniquely named per - exposure+camera, and then rename with a call to "mv" to make it - atomic and prevent 4 jobs from writing to the same JPEG file at once. - Put that job in the background instead of waiting for it to complete. - -------------------------------------------------------------------------------- -v5_4_17 (Sep 7, 2010) idlspec2d -Tag for SOS bugs (tickets #636, 797, 1017, 1022 addressed at some level). -Built against specflat v1_12, elodie v1_3, idlutils v5_4_19. - -APO_APPENDLOG: Timeout after 200 sec, at which point a frame might be - reduced but it will not be added to the log file or plots. - This might help address tickets #1017, #797, or at least provide - more info in the SOS logs about a possible problem. -APOD: Search for gzipped sdR files. - Add logic to reduce an arc preceding a flat (untested). -COMBSMALLCOLLIMATE: Remove test lines that accidentally changed the offsets. - Apodize the top and bottom of the image to be robust against an arc - line fallling near the edge of the active region. (Ticket #1022). - -bin/rundaily: Wait for the current MJD to show up in $DAILY_RAWDATA_DIR -bin/rundaily_riemann_script: Use environment variables RUN2D,RUN1D. -examples/opLimits.par: Define XMID to 2036 2045 2043 2044 based upon - Kaike Pan's posting on 7 Sep 2010 to be more consistent between - the blue and red cameras (ticket #636). - -------------------------------------------------------------------------------- -v5_4_16 (Sep 6, 2010) idlspec2d -Tag for critical bug fix for SOS (ticket #1012). -Built against specflat v1_12, elodie v1_3, idlutils v5_4_19. - -bin/loadSN2Value,updatePluggingStatus: Back out to previous version, - since changes were made for a new version of platedb that does - not yet exist. - -------------------------------------------------------------------------------- -v5_4_15 (Sep 4, 2010) idlspec2d -Tag for critical bug fix for SOS (ticket #1005). -Built against specflat v1_12, elodie v1_3, idlutils v5_4_19. - -APOALL,APOREDUCE,QUICKBIAS,QUICKEXTRACT,QUICKREDUCE,QUICKTRACE,QUICKWAVE: - Allow APOALL to run without generating lock files in the raw data - directories. Pass this as a /NO_LOCK keyword to APOREDUCE, and then - pass /DO_LOCK to the subroutines. -APOREDUCE: Do not send any Hartmann exposures (full frame or subarray) - to the QUICKWAVE routine. This is to address the problems - in ticket #1008. -COLLIMATE,COMBSMALLCOLLIMATE: Change offsets to gives same results as - COLLIMATE using data taken on MJD 55440 -COMBSMALLCOLLIMATE: Include calls to FITS_WAIT() before reading the file - to make sure that they're completely written (ticket #1005). - Test every 5 sec, and timeout after 30 sec. -SPCOMBINE_V5: Remove reading RUN2D from the spPlancomb file, since it - is unused. -QUICKEXTRACT: Increase the tolerance at which to report a flexure error - message from 0.5 pix to 1.0 pix. Ticket #1009. -ZPLOT: Change to work for BOSS targets. - -bin/loadSN2Value,updatePluggingStatus: Update to use the new version of platedb -examples/opLimits.par: Change the WAVEMID value for r2 from 8071 to 8053, - to better fit what Kaike Pan found to be consistent with b2 position. - -------------------------------------------------------------------------------- -v5_4_14 (Aug 26, 2010) idlspec2d -Tag for testing new galaxy templates in Spectro-1D. -Built against specflat v1_12, elodie v1_3, idlutils v5_4_19. - -SPREDUCE1D: changed back to NPOLY=3, optimal for new galaxy templates. -templates/spEigenGal-boss_v0_new1.fits: - New BOSS-based tempaltes from Yanmei Chen -INSPECTGEN: Update to be consistent with the suggested Data Model, which - is what is tested with INSPECT_VERIFY. -loadSN2Value, updatePlugginStatus: update to use new version of platedb -UPDATE_PLATELIST: The auto-generated "platequality" is used for plates - being added if they didn't already have a quality in the file. - -------------------------------------------------------------------------------- -v5_4_13 (Aug 5, 2010) idlspec2d -Tag for testing patches to Spectro-2D and npoly changes to Spectro-1D. -Built against specflat v1_12, elodie v1_3, idlutils v5_4_19. - -AESTHETICS: New routine for filling in bad values for COMBINE1FIBER. -ATVSPEC: New proc for displaying cutouts of raw images around specified - wavelengths. -COMBINE1FIBER: Call new routine AESTHETICS, but no change in default behaviour. -CONFIGURATION__DEFINE: Change spcalib_xy2traceset_ncoeff from 4 to 7 - for the polynomimial order of fitting the traces in the call - to XY2TRACESET in SPCALIB (ticket #845). - Change the initial guess for the sigma of the optimal extraction - in the spatial direction from 1.4 pix to 1.0 pix (spcalib_sigmaguess). - This is because there is only 1 iteration on this fitting, so it may - not otherwise converge to the correct result. - Change spcalib_sigmaguess to 1.15, a compromise BOSS value since - range is approx. 0.85 to 1.35. -FLUX_DISTORTION: Use only standard stars for the flux-distortion maps, - requiring at least 10 objects (instead of 50). This will not work - with SDSS-I data, but will remove the strangeness of using galaxies - on BOSS plates. - Increase the number of iterations in MPFIT from 50 to 200 (ticket #938). -GUIDERPLAY: Wrapper for IDL guider to simply allow play-back of images - for a specified night. -IDLSPEC2D_VERSION: Back out to the previous version, since this does not - report the svn version number in the case of untagged code. - For that, we call the idlspec2d_version bash script (with a spawn). -INSPECT_VERIFY: New proc for testing the format of spInspect files. -LOGSHEET: Add /COLLIMATOR option to print the collimator motor positions. -PCA_GAL: Call AESTHETICS for filling in bad values. - Add a table of input spectra to the eigenspectra files. -PLATEMERGE: Added EXCLUDE_CLASS and SKIP_LINE functionality. -PLOTSPEC: Fixing Ticket #926 /ALLEXP bug and odd feature. -PLUGHISTORY: Plot out to fiber #1000 for BOSS cartridges. -READPLUGMAP: Change PSF/fiber flux ratios from computed numbers to - fixed constants. For 2 arcsec fiber fluxes, these constants - are [2.085, 2.085, 2.116, 2.134, 3.135]. For 3 arcsec fiber - fluxes, they are [1.343, 1.336, 1.354, 1.363, 1.367]. Ticket #906. -SDSS_SPEC_IMAGE: Ignore greater than 1 micron for plot limits. -SPCALIB: Fix a bug introduced in v5_4_7 with the re-write of FITARCIMAGE, - where bad lines (marked as such in REJLINE) were passed along - to FITDISPERSION. This fixes ticket #823. - Change MAXDEV from 0.15 pix to 1.0 pix inthe call to TRACE320CRUDE. - This fixes ticket #845. -SPCOMBINE_V5: Modify the fix implemented in v5_4_8, and discard all data - from a given spectrograph if there is a camera with no good files. - This retains the behaviour of discarding sp2 data for MJD 55290-55299 - when r2 was dead (ticket #764). But it retains sp2 for plate - 3817/55276 when only two images are missing (ticket #787). -SPREDUCE1D: Lowered NPOLY from 3 to 0 for galaxies, and 3 to 0 for QSOs, - and 4 to 1 for stars. - Commented out MANY_OUTLIERS flag setting (ticket 955). - -bin/sos_runnerd.py: Write a version string a file so the running version - can be determined -opfiles/boss_cartridge15_gaps.txt: added file with measured bundle gaps - for BOSS cartridge 15 (see ticket #817). -opfiles/spInspect-0000-00000.par: Template for spInspect files. -templates/spEigenGal-boss_v0_new1.fits: Add Yanmei Chen's BOSS galaxy - eigentemplates -templates/spEigenGalNew-55399-baw-mean.fits: Test template file using the 'mean' - setting of aesthetics.pro. - Changed filename (from spEigenGal-55399-baw-mean.fits) so as not to - not be found automatically by the wildcarding in the 1D code. - -------------------------------------------------------------------------------- -v5_4_12 (May 31, 2010) idlspec2d -Tag for new S/N fits for SOS, and revert to SDSS-I templates in Spectro-2D. -Built against specflat v1_11, elodie v1_3, idlutils v5_4_13. - -APOALL,SLITHISTORY,SPECDB_CREATE,SPPLAN2D: Change references to - the now-defunct $ASTROLOG_DIR directory to $SPECLOG_DIR. -BOLTON_SN_FIT: new low-level routine for (hopefully) robust - estimation of exposure SN. -FITSN: Another major change to how the S/N lines are fit. Force the fit - to be a line with a fixed slope (as specified in opSNlimits.par), - and choose the offset such that half the points are below and half - above in the fitting magnitude range. Call this a "median line." - This addresses ticket #754, but really continues ticket #750. -PLATE_SPEC_IMAGE: Smoothly die if zans is missing. -PLOTSN: Sort the points randomly, such that not all the spec2 points - are plotted on top of the spec1 points, and make the points smaller. -PLUG2TSOBJ: Read photoPosPlate files instead of photoPlate files. - This addresses ticke #809. -QUICKWAVE: Always generate the fiber-flat vectors for the corresponding - flat-field image, even if that fiber-flat already exists. If it - exists, it gets over-written. This is confusing behaviour, and - might do bad things in some cases, but addresses ticket #831. -SDREPORTMAKE: Remove this now-unused piece of code. -SPCOADD_V5: Delete QUALITY,REQTIME keywords (ticket #789). -SPREDUCE1D: Back out last change to use Bolton's new spMLpca templates - and change the default of chop_data back. -TRACE_SPARSE_CRUDE: New semi-crude, semi-robust sparse flat tracing routine. - -opfiles/opSNlimits.par: Changes to be consistent with FITSN. Remove - the fiducial_coeff field which was unused. Add a slope which is - set to -0.30 for all cases. The fitting mag range is set to be - the range [snmag-1,snmag] for the new, median-line fits. - -------------------------------------------------------------------------------- -v5_4_11 (May 16, 2010) idlspec2d -Tag to test spMLpca templates (bug fixes to last tag) -Built against specflat v1_11, elodie v1_3, idlutils v5_4_13. - -BATCHPBS: Fix typo on naming script files with /skip2d option. -SDSS_SPEC_IMAGE: Change flat to target. -SPREDUCE1D: Modify to test Bolton's new spMLpca templates, and change - default of chop_data to 3600 to 10000 Ang. - -------------------------------------------------------------------------------- -v5_4_10 (May 15, 2010) idlspec2d -Tag to test spMLpca templates -Built against specflat v1_11, elodie v1_3, idlutils v5_4_13. - -BATCHPBS: Add /SKIP2D option. -BOLTON_MLPCA,BOLTON_MLPCA_DEMO: New proc with Bolton's maximum-likelihood PCA. -COMBSMALLCOLLIMATE: Document explicitly outputs (with /SIMPLE) to be read by - telescope control SOP code. -PLATE_SPEC_IMAGE,SDSS_SPEC_IMAGE: New code for making JPGs of spectra (Blanton). -PLOTSPEC: Along with printing the target flags, print whether it is 'SKY'. -PLUGHISTORY: Update to work with BOSS cartridges 10-17 with 1000 fibers. -SPTHROUGHPUT: Return correct number of vectors if INDX not set. -UUPLOTSPEC: Modified version of PLOTSPEC for interfacing with an online - database at http://boss.astro.utah.edu (Brownstein). - -pro/Makefile: Add templates directory. -pro/templates: Files (code + data files) for generating Bolton's new templates, - but need to clean this up. -templates/spMLpcaCVstar-55332.fits,spMLpcaQSO-55332.fits, - spMLpcaGal-55332.fits,spMLpcaStar-55332.fits: New templates - to test from Bolton. Galaxies span [1800,10020] Ang, QSOs [580,9963] Ang, - stars [3580,10318] Ang, CV stars [3581,10315] Ang. - -------------------------------------------------------------------------------- -v5_4_9c (May 5, 2010) idlspec2d -Tag for SOS with gzipped files. - -APOFIX: Remove references to $ASTROLOG_DIR, instead using $SPECLOG_DIR. - New keyword SOS_DIR? Put back in code to edit the logsheet-$MJD.fits - file when the quality of an exposure is changed. -APOREDUCE: Report "WARNING" message if the HARTMANN header keyword is - set to either "Left" or "Right". Note that this creates a lot - of unnecessary warning messages on the SOS web page. -BESTFOCUS: Allow images where the shift was more than 2.5 pix (those were - intentionally ignored previously). -COLLIMATE: Call COMBSMALLCOLLIMATE if the SUBFRAME keyword exists in - the header. -COMBSMALLCOLLIMATE: New fast collimate routine to focus - sp1 and sp2, uses subarray readouts, single region, and splits the - difference between red and blue offsets. Called by "sop doHartmann" - as part of the new field acquisition. -SMALLCOLLIMATE: New routine. Don't know how this differs from the other - collimation routines? - -bin/filecheck.py, sos_redorunner, sxpar.py: Read gz files -bin/getPlugmap: Get plugmaps for multiple-pointing plates, assume "A" - pointing if not specified. -bin/putils.py: Add openRead to open a normal or gz file for reading. -bin/run_collimate.csh: Print more blank lines. -bin/rundaily_riemann: Bug fix (typo). -bin/rundaily_riemann_script: Bug fix for env variables (typo). -bin/sos_apocommand: Add quality type to error message. Add directory - as well as files. -bin/sos_apocontrol: Added print statement of options. -bin/sos_aporedo,sos_apostart: Work with gz files and filesequencer -bin/sos_filesequencer: Handle cases where files need to go into aporeduce not - in the order the come out of the icc. Right now it special cases - arcs and flats. Then changed the code to allow all arcs, but switch - the order if the arc comes in before the flat. -bin/sos_runnerd.py: Display output of svn commands to log. -bin/sxpar.py: Work with gz files. -examples/opLimits.par: Enlarge all the ranges for XMID and WAVEMID. - Further analysis reveals that the scatter in wavemid is larger than - the original constraints and producing a lot of warnings that are being - ignored. Following figures from David Harris that show scatter, change - limits to have center at WAVEMID= 4908 4995 7987 8071 - Issue warning if more than 10 Ang from these best values, and error if - more than 20 Ang (red) and 15 Ang (blue). - -------------------------------------------------------------------------------- -v5_4_9a (Apr 30, 2010) idlspec2d -Tag before SOS changes for gzipped files. - -APOFIX: What changed ??? -BATCHPBS: Add QUEUE keyword to specify the queue for qsub -PCA_GAL: Add /FLUX and /REPLACE keywords. The /FLUX option loads in - a save set from a previous call with this option??? -PCA_SOLVE: Added FLUX keyword -PLATELIST: Tiny html fix. -PLATEMERGE: Target flags should be 64-bit (type LONG64). - Bug fix to select correct RUN2Ds when several exist in the platelist file. -SPCALIB, SPREDUCE, SPREDUCE2D: Add /writeflatflatmodel to call to - SPREDUCE2D to add the model image for flat-fields to the spFlat files. - -bin/filecheck.py: Test for quality=test|excellent fits files. Fixed bug - if there is no keyword. -bin/loadSN2Value,updatePluggingStatus: Now update quality on --update -bin/putils.py: Change method of executing commands and handling svn. - Now command output is displayed as the command executes, not all - when it finishes. Files are not added to svn as they are created, - but right before check-in. -bin/rundaily: Send output of getWikiMJD to the bitbucket. -bin/rundaily_riemann_script: Use env variables that are set in .bashrc . -bin/sos_apocommand: Skip all non-excellent images and test exposures. - (But not actually done, see ticket #684). -bin/sos_apocontrol: Run a status after start. -bin/sos_aporedo: Outout b1 in real time and wait for all processes to complete. -bin/sos_runnerd.py: Changed method of executing commands and handling svn. - Now command output is displayed as the command executes, not all - when it finishes. Files are not added to svn as they are created, - but right before check-in. - -------------------------------------------------------------------------------- -v5_4_9 (Apr 16, 2010) idlspec2d tested on IDL 7.0 - 7.1 -Tag to fix install problem. -Built against specflat v1_11, elodie v1_3, idlutils v5_4_13. - -opfiles/Makefile: Need Makefile for install. - -------------------------------------------------------------------------------- -v5_4_8 (Apr 16, 2010) idlspec2d tested on IDL 7.0 - 7.1 -Substantial bug fixes to SOS scripts. -Spectro-2D uses photoPlate files in new directory tree. -Built against specflat v1_10, elodie v1_3, idlutils v5_4_13. - -APO_LOG2HTML,APO_PLOTSN,APOREDUCE: Replace "APO Spectro" with "BOSS Spectro". -APO_PLOTSN,EXTRACT_OBJECT,FITSN,PLATESN,PLOTSN,QUICKEXTRACT: - Consolidate all the fitting magnitudes into one parameter file, - and use FITSN to reference that file. This is to address ticket #750 - and ticket #749. -APOFIX: Bug fix. Add /EXCELLENT keyword. -APOREDUCE: Copy the single-exposure S/N plot to the COPYDIR directory - (ticket #675). -BATCH1D,BATCH2D,DOSCATFIT,EXTRACT_ROW_SAFE,FIT_TSET,FITARCIMAGE_OLD, - FLATINTERP,FUNCCHEB,GAUSS_SMOOTH,LOCATESKYLINES_TEST,MONSTER, - MYFLUXCALIB,READ2DOUT,READSMEAR,SPCOADD_FRAMES,SPCOMBINE,SPFLUX,USHIFT: - Remove unused code. -BATCHPBS: Call new proc SDSS_PLATE_SORT to generate the photoPlate files. -FINDSPEC: Allow this to work with different plugmap structures by - just explicitly using fiberid,ra,dec. - Trim whitespace on run2d,run1d. -FITFLATWIDTH: Assume bundles have 20 fibers, and get rid of numbundles keyword. -LEDFLAT: Major changes to loop through each pixel, computing statistics - using a subset of the best images for each pixel. -PCA_GAL: Add headers and other minor fixes. -PLATELIST: Exclude runz subdirs from searches for run1d files. -PLOTSN: On right-hand plots, only plot the objects used in the fit range - and not all the faint, noisy points. - Add SNMIN keyword, and default to 0.5, to exclude all the very - low-signal fibers from the fitting. Addresses ticket #754, although - with the a simple solution rather than a well-motivated one. - Show the fit line for each spectrograph. -PLUG2TSOBJ: Only look for photoPlate files in plate redux directory, - and not in subdirectories. -QUICKEXTRACT: No need for config. - Issue flexure warning only if larger than 0.5 pix, not 0.25 pix. -QUICKTRACE: Remove "Argon in flats" message until its triggering can be - tracked down (ticket #755). -SDSS_PLATE_SORT: New proc for sorting photoPlate files to be the same - as a given plugmap. This addresses ticket #721. -SDSSPROC: Remove CHECKSUM,DATASUM keywords from headers so that they are - not propogated to downstream files (ticket #756). - Measure gain values for the new r2 CCD from LED flats #113197-113198, - to be used starting on MJD 55300 (13/14 April 2010). - The exact location, width, and counts of the serial transients - appear to be different on the new r2 CCD as compared to the old - r2 CCD. These values have been appropriately updated in - SDSSPROC->SDSSPROC_BADFORMAT for starting on MJD 55300 (ticket #766). -SPCALIB: Remove deprecated keyword in FITFLATWIDTH. -SPCOMBINE_V5,SPFLUX_V5,SPFLUXCORR_V5,SPCOADD_V5: Clean up code to not - pass the number of pixels in images, but compute based on the images. -SPCOMBINE_V5: Hack to limit the combine to either b1+r1 or b2+r2 in the - case where any images from one of the cameras is missing. - This addresses ticket #764, but in an inelegant way. -SPPLAN1D: Get rid of unused file name definitions. -SPREDUCE: Look for photoPlate files in current reduction directory, - rather than using env variables to determine. -SPTHROUGHPUT: Remove hard-wired number of fibers. - -bin/loadSN2Value,updatePluggingStatus: Better db error-handling. -bin/sos_aporedo,sos_redorunner,sos_runnerd.py: - Results have an option to now go into - http://sdss3.apo.nmsu.edu/~observer/sosredo/logfile-current.html and - /data/boss/sosredo. -etc/BG14.txt,FG3.txt,altbg14.txt,blue.bkpts,brg.08.gk.01,f8v.ab,f8vspline.dat, - fluxcorr.bkpts,intrinsic1.dat,intrinsicF2.dat,red.bkpts: - Remove unused files. -Makefile: Add opfiles subdirectory. -examples/opLimits.par: Increase the values for XSIGMA based upon the - best achievable on MJD 55300. Yellow at 1.10 1.15 1.10 1.25 (b1 b2 r1 r2) - and red if 0.05 worse than tose values. Decrease the values for - WSIGMA to yellow at 1.05 in all cameras, red at 1.15. -opfiles/opSNlimits.par: New file for tracking the S/N-fitting magnitudes. - -------------------------------------------------------------------------------- -v5_4_7 (Mar 31, 2010) idlspec2d tested on IDL 7.0 - 7.1 -Substantial changes to SOS scripts. -New Spectro-2D with rewrite of wavelength fits and use BOSS pixel flats. -Built against specflat v1_9, elodie v1_3, idlutils v5_4_13. - -APOD: New cheat script for running SOS at Apache Point (not documented). -APOFIX: Update default astrolog_dir. Remove all of the code to edit - the APO logfile, and instead spawn a call to sos_aporedo. -APO_LOG2HTML: Include links to S/N plot for each individual exposure. -APO_LOG2HTML,APOREDUCE: Include cartid in the last HDU of the logfile, - and display on the web page. This does change the data model for - the logfile, meaning that we cannot have this and the previous - version of SOS writing to the same file. -APO_PLOTSN,APOREDUCE: Generate S/N plots for individual exposures as well - as for the full plate. -ARCFIT_GUESS, EXTRACT_IMAGE, GEN_SCATTER, MYFLUXCORR, SKYSUBTRACT, -SMOOTH_HALO2D, SPFLUX_READ_KURUCZ, VDISPFIT, ZCOMPUTE, ZCOMPUTE_QSO: - Commented out some print updates that were making the PBS log files - large (all those with string(13b)). -APO2D,QUICKTRACE,SPREDUCE: Simplified call to READPLUGMAP, and remove - call to SORTPLUGMAP. -APOREDUCE: Get airtemp from the MC1TEMDN or MC2TEMDN keyword for BOSS, - since the AIRTEMP keyword does not exist. -APOREDUCE,APO_PLOTSN: No need to use the configuration file to determine - the number of fibers. -BATCHPBS: Add option to join stderr and stdout in log file (kushner) - Create a set of files per MJD observed, even though they should be - identical. -COLLIMATE: Report failure if Hartmann's not in proper sequence - KD -COMBINEBPM,FLATFIELD: New routines for generating BOSS pixel flats from LED - images (Olmstead). -CONFIGURATION__DEFINE: Increase spcalib_trace320cen_deltax by a small - amount for the r1 camera to be consistent with the median solution. -FIND_RESOLVE: Bug fix for 3-digit plate numbers. Deprecate, putting the - logic internal to PLUG2TSOBJ, which is the only place this was called. -FINDSPEC: Make consistent with platelist files. -FITARCIMAGE: Major changes to the iteration of the fitting and the - tracking of rejected arc lines. Iterate the full fits, rejecting at - most 1 arc line per iteration. Track the reason for rejecting lines - and return in REJLINE. Final fits are done where the higher-order - terms in the trace-set are replaced with polynomials (cubics). - Those final fits use the actual centroid locations + errors, rather - than the "replaced" positions (as was done in the last version of this - code). Using the actual centroids should be fine with the smoothed - higher-order terms. - Warning message if Cd I 3610 lamp not good (ticket #641). - Issue warning about big wavelength gap is bigger than log10=0.05 - instead of 0.04, such that a gap between 3650-4046 is barely OK. - Comment back in the failure if log10>0.10. - The fits were sometimes being effected by a fiber with very bad - fits, where loglam would be nonsense values. The floating point - precision was then making the fits to all of the fibers much noisier. - An example of this was on palte 3651/55246 on the frame - sdR-b2-00108656.fit . For such bad fibers, always interpolate over - those fiber coeffs before calling traceset2xy. -FLUX_DISTORTION: Bug fix where some values of 320 that represented the - plate radius in mm were replaced with the number of fibers. - Change those numbers to 325 mm to avoid future mis-interpretations. - Change MINFLUX to a 5-element array, with the minimum flux reduced - from 5,5,5 nmy in gri-bands to 2.5 nmy in r-band. -IDLSPEC2D_VERSION: Replace spawn to shell script bin/idlspec2d_version with IDL - (Weaver). -LEDFLAT: New proc for making pixel flats from Natalie's LED flats. -LOGSHEET: Print timestamp from TAI-BEG in header, since TAI keyword no - longer exists. -PCA_GAL: Updates from Weaver. Add optional BINSZ keyword. -PLATELIST: Include all of the GAL_CMASS* flags for the high-z LRGs. -PLATELIST,UPDATE_PLATELIST,UPDATE_PLATE_RELEASE: Move the spPlateList.par - files from this product to the speclog product. -PLATEMERGE: Changes to the outputs in the ASCII file, make shorter aliases - for some of those ASCII headers, and trim unneccessary whitespace - from the string values. - Include manual redshifts and comments from spInspect files. - Include more values from the new plugmaps (lambda_eff,bluefiber,zoffset). - Make the new target flags 64-bit (bug fix). - Add /INCLUDE_BAD option. -PLATESN,PLOTSN: Make the fiducial magnitudes all brigher by 0.80 mag, so that - one is analyzing w.r.t. the same magnitudes as SOS. PSF magnitudes - are used for stars here, whereas fiberflux2 are used for SOS. -PLOTSPEC: Smooth flux values for /allexp option. -PLUG2TSOBJ,READPLUGMAP,SPREDUCE: Fix and simplify finding photoPlate files, - insisting that one exist for each plate+MJD that a plate is observed. -QAPLOT_ARCLINE: Plot rejected arc lines (in red). Make sure all arc lines - show up in the lower plot by at least putting a point near the location. -QAPLOT_SKYSUB: Plot median |chi| rather than chi^2, and compare to Gaussian. -READONESPEC: Work with either the 2048-row SDSS-I spectra or the 4128-row - BOSS spectra (but still assume same number of rows in blue or red). -READPLUGMAP: Do the sorting internal to this routine, rather than a later - call to SORTPLUGMAP. Include SPECTROGRAPHID,FIBERMASK to call. - Fix crash condition in ticket #604. - Demand that FIBERMASK is not already set. -READSPEC: Add optional output ZMANUAL. - For cases where the data files cannot be found, default to either 640 - or 1000 blank rows in the outputs based upon MJD, with 1000 fibers - assumed for data starting on MJD 55025. (One-line change at line 206.) -SDSSPROC: Bug fix for ticket #610 in identifying the pixel shifts. - There was a bug in defining one of the overscan transients in the b2 CCD - (line 144). Aside from that, the transient is not always easily detected - in all rows. The somewhat hacky solution is to apply the same shift - (or no shift) to any row where the transient is not easily measured. - Re-organize to call much of the code, like bias and flat-subtraction, - for either SDSS or BOSS data. - Look for compressed pixel bias or pixel flat images. -SORTPLUGMAP: Remove this routine, and make it internal to READPLUGMAP. -SPADD_GUIDERINFO: If this is an old SDSS-I file, then the timstamps are all - offset and need correcting; if BOSS, then no corrections. - Test that FWHM values are not NaN. -SPCALIB: Print proftype. -SPCALIB,SPREDUCE: Add REJLINE to the ARCSTRUCT structure, allowing one to - include rejected arc lines in the QA plots. -SPCALIB,QUICKWAVE: Remove numberBundles in call to FITARCIMAGE. -SPCOADD_V5: Remove unneccesary keyword to FLUX_DISTORTION call. -SPMANUAL: New function to read manual inspection files (spInspect). -SPREDUCE1D: Replace spawn of Unix mkdir command with FILE_MKDIR. -SPTHROUGHPUT: Fix directory structure for BOSS. -SUPERFLAT: Change min + max wavelength ranges such that it spans the range - that includes at least one good pix. Bug fix in computing the - flux scaling between the fibers. - -bin/sos_apocommand: Process all images, even non-BOSS images -bin/rundaily, rundaily, rundaily_riemann_script: Always exit zero so as - not to upset cron. -bin/sos_redorunner, sos_aporedo, sos_aporedomakecmd: New re-do command -bin/sos_apostart: Don't sued nuce. -bin/rundaily_riemann_script: Added /zcode. -bin/sos_runnerd.py: Only do svn ci durring bookeeping. -bin/sos_runnerd.py, sos_classes.py: Add exposure number option. -bin/loadSN2Value, updatePluggingStatus: New code to load S/N^2 values - into platedb. -doc/www/spectroSOS.html: Update SOS stop / restart instructions. - Document warning message pertaining to ticket #641. -etc/lamphgcdne.par: Add back in the lines as 3650+3654 and do not mark - as blends, so that the 3650 line can be used when the 3610 line - is too faint (ticket #644). Also add back in 6506, 8408, 9313 lines - since the code is more robust against bad lines now. Add new - lines at 9657, 9784 (both faint Ar I lines). -etc/spPlateList.par: Move this file to $SPECLOG_DIR/opfiles. -examples/opLimits.par: Update WAVEMID values for sp2 to center up - the new collimator masks (ticket #636). -pro/Makefile: Add specflat subdirectory. -src/Makefile: Make compatible with GNU make (Weaver). -ups/idlspec2d.table: Added setupOptional("platedb") - -------------------------------------------------------------------------------- -v5_4_6 (Feb 9, 2010) idlspec2d tested on IDL 7.0 - 7.1 -Tagged to fix critical ticket #600 on SOS. -Built against specflat v1_8, elodie v1_3, idlutils v5_4_12. - -Major environment change: Global replace of SPECTRO_DATA -> BOSS_SPECTRO_REDUX -(B. A. Weaver, 2010-01-26) - -BATCHPBS: Typo fix for making match files. Add /ZCODE option to setup up - and run Zcode. -BESTFOCUS: Dawson and Olmstead code for solving for CCD focal plane. -CCDTILT: New proc to measure CCD tilt from a whole sequence of Hartmanns, - based upon the COLLIMATE routine. -CHUNKINFO: Read info from $PLATELIST_DIR/platePlans.par file. -FINDSPEC: Warning message for missing files. -PLATELIST: Fix bug finding the spDiagcomb file. - Fix umask for the PBS log files (Weaver). - Add interpretation of the BOSS target flags, and include - %LRG1,%LRG2 in the outputs. - Include SURVEY, PROGRAMNAME, CHUNK and remove PROGNAME,CHUNKNAME. - Also included are TILE, RACEN, DECCEN, EPOCH from platePlans.par - since those are probably more trusted than other sources. - Include N_TOTAL as total number of objects (640 or 1000). -PLATEMERGE: Optional calling sequence with RUN2D rather than with PUBLIC. - Read all input data from READSPEC instead of other various wrappers. -PLUG2TSOBJ: Simplify to only support photoPlate and calibPlateP files. - Remove PLUGMAP option and add RUN2D option. -READSPEC: Small changes to support change to PLUG2TSOBJ. -SPREDUCE: Read TILEID from the plugmap file, since it is no longer - in the sdR file headers (ticket #562). -TRACE320CRUDE: Bug fix for when only a few good pixels exist in a row - (ticket #600). - -bin/rundaily, rundaily, rundaily_riemann_script: Scripts for after daily - data transfers. -bin/sos_apospammer: Updated to roll log and reset error messages. -etc/spChunkList.par: Remove file, as this is deprecated by - $PLATELIST_DIR/platePlans.par -examples/opLimits.par: Remove the warnings for SUCCESS_QSO in the platelist. - -------------------------------------------------------------------------------- -v5_4_5 (Jan 23, 2010) idlspec2d tested on IDL 7.0 - 7.1 -Tagged to fix batch processing of BOSS data. -Built against specflat v1_8, elodie v1_3, idlutils v5_4_12. - -APO_CHECKLIMITS: Different font formatting (Weaver). -BATCHPBS: Add automatic plate matching (Weaver), but adds dependency - on the sas product. -EXTRACT_OBJECT: Fix bug where FRAMESN2 was being set to 1 instead of 0 - when there was no flux on an exposure. -GET_MJD_DIR: Use FILE_SEARCH() instead of calls to SPAWN, and only - return directories and not file names. -PLATELIST: Support new RUN2D,RUN1D directory structures. - Put RUN2D,RUN1D in the html files rather than PROG,CHUNK. - Switch from html to xthml (Weaver). - Only mark plates as 'bad' or 'good', not 'marginal' (too confusing). -PLOTSPEC,READSPEC,READONESPEC: Add support for RUN2D,RUN1D. -READPLUGMAP: Remove PSFFLUX*,FIBERFLUX*,FIBER2FLUX*, since those appear - to all be zeros in the platelist files. -SPCOMBINE_V5: Accidentally had removed setting the name of one plot file. -SPREDUCE2D: Fix mis-formatted names for output log and PostScript files. - -bin/sos_apostart,sos_apocontrol: Added a delete for the socket file. - Created more descriptive error and warning messages -bin/sos_apocontrol: Update stop messages. -pro/Makefile: Include all the pro/science subdirs. - -------------------------------------------------------------------------------- -v5_4_4 (Jan 18, 2010) idlspec2d tested on IDL 7.0 - 7.1 -Tagged to allow batch processing of BOSS data. -Built against specflat v1_8, elodie v1_3, idlutils v5_4_10. - -pro/Makefile: Turn this into a common makefile for all subdirs. -pro/*/Makefile: Remove these old makefiles. - -------------------------------------------------------------------------------- -v5_4_3 (Jan 17, 2010) idlspec2d tested on IDL 7.0 - 7.1 -Tagged to allow batch processing of BOSS data (but crashes). -Built against specflat v1_8, elodie v1_3, idlutils v5_4_10. - -BATCHPBS: Remove accidental return before calling qsub. -SPREDUCE2D,SPCOMBINE_V5: Fix typos. - -------------------------------------------------------------------------------- -v5_4_2 (Jan 17, 2010) idlspec2d tested on IDL 7.0 - 7.1 -Tagged to allow batch processing of BOSS data (but crashes). -Built against specflat v1_8, elodie v1_3, idlutils v5_4_10. - -READSPEC,SPREDUCE1D,READONESPEC,SPCOMBINE_V5,SPPLAN1D,SPREDUCE, - SPPLAN2D,SPREDUCE2D: Implement new data model for directory - structure that makes use of $RUN2D and $RUN1D subdirectories. - Remove some of the file name declarations in the plan files - in preference to generating those just based upon plate+MJD. -APO_COLOR2HEX: Change the yellow color from a bright yellow that's hard - to see to a darker yellow. -APO_LOG2HTML,APOREDUCE: Generate a JPEG file instead of a GIF. -APO_PLOTSN: Change fit magnitudes passed to PLOTSN from 20.33 to 22.0 in g-band - and from 20.06 to 20.0 in i-band, and set the other unused bands to zero. -APOREDUCE: Create output directory if it does not exist. -ARCFIT_GUESS: Allow more than 4 coefficients in initial wavelength solution. -BAD_BOSSFORMAT: New routine to search for shifted rows in raw images, - like on MJD 55176 exp #104648,104652 -BATCHPBS: New proc for batching both Spectro-2D and Spectro-1D with PBS. -BOSS_GAIN: New proc for computing gains on BOSS CCDs. -COLLIMATE: Display specific commands for observers to cut & paste. - Simplify the basic call to this routine to compute the Y offsets - in a hard-wired region in the center of each CCD. The offset maps - are produced only if the /DOPLOT option is set. - Tested against BOSS data on MJD 55113, and SDSS-I data on MJD 54932. - Changed calibrations that convert from pixel shifts to offsets. - Test for proper values of FFS, NE, HGCD in headers. -CORRECT_CROSSTALK: Include crosstalk correction in the spec2d processing chain -EXTRACT_OBJECT: Evaluate the S/N at fainter mags: change from g,i=20.2,19.9 - to 22.0,20.0. -FITARCIMAGE: Filtering of initial crude centroids was done with xy2traceset - with only 3 coefficients, which is too few for BOSS. Change to - using the value of NCOEFF (line 224). - Add more traceset fits at the end, rejecting arc lines that are too - noisy or too far offset from the solution. Do the final fit by - weighting each line by its scatter from the previous solution. -FITSN: Change magnitudes to fainter values. -FITDISPERSION,FITFLATWIDTH: Return the medians in each quadrant - excluding masked portions of the CCD. -FITFLATWIDTH: Generate a mask that is the same within each bundle - before sending the width to the trace-set fitting. This fixes a - crash condition on plate 3538/55124-r1, where there was a fiber - with only a few unmasked pixels that was getting a crazy fit. - For the case of no input mask (INMASK), then generate a mask - to use only good pixels. -GUIDERMONFILE: New procedure to create guiderMon file using proc-gimg files. -IDLSPEC2D_VERSION: Now have this just call the shell script idlspec2d_version. -LOCATESKYLINES: Do not use any sky lines that don't have at least 80% - good centroids. This mattered for 3790/55208-b1, where the line - at 6363 Ang was only being well-fit in a few fibers. -LOGSHEET: Default to 'b1' only. -PLATESN: Change fiducial mags from [20.2, 20.25, 19.9] to [22.0, 21.0, 21.0]. -PLOTSN: Change fiducial mags from [20.0, 20.2, 20.25, 19.9, 19.0] - to [20.0, 22.0, 21.0, 21.0, 20.0]. - Plotting range to include FITMAG and SNMAG. -PLOTSPEC: Add option to do on-the-fly template fits with "z". -PLUGMAP_VERIFY: New code to run some sanity checks on plate designs - (plPlugMapP files). -QUICKBIAS: Reject if >40% pixels bad, not 10%, since we are currently - masking 31% of the blue images and 12% of the red that are beyond - the useful wavelength ranges. -QUICKEXTRACT: Include more calling parameters to fitflatwidth. - Change mangitudes at which to evaluate S/N from 20.33 to 22.0 in g-band - and from 20.06 to 21.0 in i-band. - Remove the code that would issue the warning messages - to the SOS web page about bad chi^2 residuals. - No longer pass SKYCHI2SET to SKYSUBTRACT routine, and do not output - to the FITS file. - Trigger flexure warning message at 0.25 pix instead of 0.15. - Include the raw data header in the output FITS (sci-*) file, - including the "FRAMESN2" keyword. -QUICKTRACE: Fix typo. Include more calling params to trace320crude. - Consider a fiber bad only if any of the following mask bits are set, - but specifically not if BADTRACE is set. -QUICKTRACE,SPCALIB: Change call to XY2TRACESET to ignore values that - fall on bad parts of the CCD. -QUICKWAVE: Include more calling params to fiberflat and fitarcimage. -RDNOISE_HISTORY: New proc for computing read noise history of BOSS CCDs. -SDSSGUIDE: Comment-out the parts of code that reads the plugmap files - and that computes the telescope offsets. - Remove the camera offsets that were used by the old camera hardware; - the code will presently only work with the new camera. -SDSSPROC: Images are rotated 180 deg starting on MJD 55113. - So we no longer need to rotate those images. The gains have - been remapped, and the masked regions, to these rotated coordinates. - Declare Hartmann exposures (Left or Right) to be 'test'. - Update BOSS gain values based upon outputs of BOSS_GAIN flat-field tests. - Add keyword /APPLYCROSSTALK and call to cross-talk fixing code. - Pass /SILENT option to SPHDRFIX. - Only call WARN_DAYTIME if not called silently. - Saddlebags swapped between r1 and r2 on MJD 55132. This changes - the gain on both CCDs. The high noise on the lower-left of r1 - is now fixed, and we've moved a higher noise on r2 (but not as bad). - Measure gain from LED flats on MJD 55142 on exposures 103065+103066. - No longer splice in columns to the center of b1+b2 from the edges; - the central columns were supposedly fixed on MJD 55141, and data - predating that actually looks like it shouldn't have two extra - columns in there either. - Only call WARN_DAYTIME for science exposures. - Remove BZERO,BSCALE from the headers. These appear newly in - the BOSS headers, and otherwise propogate to the reductions. - This is ticket #400. - New internal proc SDSSPROC_BADFORMAT to identify and fix shifted - rows in the raw image (ticket #423). - Cache the code versions, since they involve calls to spawn that - are very slow. -SKYSUBTRACT: Only call REDMONSTER routine if RELCHI2SET keyword is passed, - thus disabling those error messages from appearing on the SOS - web page. -SORTPLUGMAP: Bug fix assigning dead fibers (as with plate 3674/55153). -SPADD_GUIDERINFO: Interpret the new guiderMon files generated by - GUIDERMONFILE, while being backwards-compatible with the old ones. - Guider offsets measured for all enabled fibers, and FWHM measured - from only those where focusoffset is <100 microns (in-focus fibers). -SPFLUX_V5: Fix plotting limits. -SPHDRFIX: Add /SILENT option. -SPREDUCE1D: Force the use of /CHOP_DATA until we fix the templates. -SUPERFLAT: Fix plotting range. - Better selection of which fibers to use (otherwise fails - on plate 3604/55146-b1), not getting thrown by a few fibers. -TRACE_CEN: In intial fit, explore a small range of fiberspace around - the nominal, as well as the cross-correlation. That scale range - is +/- 0.5%. - -Makefile: Add config to install directories. -bin/aporsync_alllog.sh, aporsync_blue.sh, aporsync_logs.sh, aporsync_red.sh, - data_rsync.sh, killdata.sh, sos.sh, sos_redo, sos_start, sos_status, - sos_stop, spamdup, spreduce_batch, spsyncrobot.sh, spsyncrobot_start, - spsyncrobot_stop, startapo.sh: Remove all these deprecated routines -bin/fb_classes.py, getPlugmap, sos_apoclog, sos_apocontrol, sos_apostart, - sos_batchd, sos_classes.py, sos_runnerd, sos_runnerd_processlist, - sxpar.py: New routines for Son-of-Spectro -bin/idlspec2d_version: Script to return SVN version of the idlspec2d product. - This is to replace the IDL version of this. -bin/run_collimate.csh: Script to run collimate in parallel processing (Dawson). -bin/sos_README: Updated documentation for Son-of-Spectro -doc/www/spectroSOS.html: Include warning message about "Electronics shifted". -examples/cartridgeInfo.par: Add entries for cartridges 10-17, with the - same values for all of them. This is for SDSSGUIDE. -examples/lamphgcdne.dat: Remove the Ne I line at 8118 Ang, since the - arc line at 8115 was being called 8118. - Remove the line 7943 which appears to get incorrectly identified. -examples/opLimits.par: Define the XMID,WAVEMID to be those observed - with plate 3552/55142 on cartridge 11. - Set S/N^2 limits as 16 in blue and 26 in red for "good", and - 13 + 22 as "marginal" (yellow). The red limits are based upon - an analysis posted by Peter Brown on Nov 30. -examples/plPlugMapP-verify.par: Data file of header keywords to check - using the PLUGMAP_VERIFY code. - -------------------------------------------------------------------------------- -v5_4_1 (Sep 25, 2009) idlspec2d running on IDL 5.4 - 7.1 -Tagged to begin development for BOSS spectrographs, backing -out SEGUE-specific development. -Built against specflat v1_8, elodie v1_3, idlutils v5_4_4. - -APOREDUCE,QUICKTRACE,QUICKWAVE: Use BOSS configuration. -CALCSCATIMAGE: Fix for non-square images! -COLLIMATE: Update to work on the BOSS data. Simplify the code to - only look for offsets in Y, and do so by looking at subpixel - shifts rather than fitting to the shift from integer-pixel shifts. - The calibration for the collimator and focus rings will be - different for BOSS, because the CCD pixels are smaller by 15/24. - Add /PRINT option to print the table of values used for the figure. -CONFIGURATION__DEFINE: Configuration file to set pipeline parameters - for either SDSS-I or BOSS. -EXTRACT_OBJECT: Simplify call to CALCSCATIMAGE. - Make calls to DIVIDEFLAT less verbose. - Bad columns only if 30% of pixels (not 10% previously for SDSS-I). -EXTRACT_ROW: Hack to keep the C code from failing, by setting masked - pixels to have values of invvar slightly above zero. - This is ticket 298. -FITARCIMAGE: Pass parameters (ACOEFF,DCOEFF) to ARCFIT_GUESS using _EXTRA. - New input keyword WRANGE to explicitly limit the wavelength range - of the arc lines used. - New input keyword NUMBERBUNDLES. - Get rid of /STICKY in all to XY2TRACESET, since that rejects too - many arc lines. -GET_TAI: Work if only TAI_BEG and EXPTIME are defined (which is all we - seem to have for BOSS commissioning). -PLOTSPEC: Support for plotting BOSS files with different numbers of fibers. - Print old or new target names. -QAPLOT_ARCLINE: Minor plot changes. -QUICKTRACE: - Call to XY2TRACESET using errors, which is much slower (by 10X), - but prevents crashes from extrapolating the traces to bad positions. -READPLUGMAP: Read the plateHoles file and append some tags to the plugmap - structure. -READSPEC: Support for reading BOSS files with different numbers of - fibers per plate. This should be backwards compatible, except - with different behaviour if one tries to read non-existent files. - Previous behaviour was to return 640-length vectors of zeros. - Now return 1-length vectors of zeros, since we in general don't - know how many objects there are per plate. - Optional return value of NFIBER to tell the number of fibers per plate. -SDSSPROC: Hack for reading the first BOSS test data on MJD 55052, - which are not yet properly-formatted images. -SKYSUBTRACT: Change calls to bspline_iterfit to pass all double-precision. -SPCALIB,SPCOMBINE_V5,SFLUX_V5,SPFLUXCORR_V5,SPCOADD_V5: - Changes to use Alex's new configuration definitions for SDSS-I/BOSS. -SPCALIB: Pass configuration values to FITARCIMAGE. -SMOOTH_SUPERFLAT: Return zeros for the returned image outside of the - wavelength range used for the fit. Also don't include the - crazy values outside the fit region in the plots. -SPECHEADERS: New routine for consolodating all of the spectro FITS headers - into a single FITS binary table. -SPPLAN1D,SPPLAN2D,SPREDUCE1D,SPREDUCE2D,SDSSPROC, - SPCOADD_FRAMES,SPCOMBINE,SPCOMBINE_V5: Use the /noshell option - for some calls to spawn (except for mkdir,uname,ls). -SPPLAN_FINDRAWDATA: Look for .fits files as well as .fit, since such - files were written on MJD 55052. -SPREDUCE1D: Add CHOP_DATA keyword for reducing the new BOSS spectra with - the old SDSS templates that do not have sufficient wavelength range. -SUPERFLAT: QA plot now only plots percentiles of the fiber values at - each wavelength, rather than a very large number of points. -TRACE_CEN: New routine for finding the initial X positions of the traces, - to replace TRACE320CEN. The logic is based upon the new code - in the fiber-mapper. -TRACE320CEN,TRACE320CRUDE: Fix error in guessing the position of the - first fiber. Add new keyword MTHRESHFACTOR, defined in configuration. -TRACE320CRUDE,SPCALIB: Use TRACE_CEN instead of TRACE320CEN. -ZFIND: Add /mask_emission option (Vaishali), although should this be - done external to this routine? - -etc/lamphgcdne.dat: New lines and their fluxes added in the UV - and the near-IR for the BOSS detectors. Remove the line - at 6506 Ang from the list because it gets confused with - a line at 2nd-order in the b1 spectrograph. -examples/opLimits.par: New parameters for BOSS. Not made to be - backwards-compatable with SDSS-I. -examples/opConfig-55025.par,opECalib-55025.par: CCD parameter - files for Lauren's original simulations. -examples/opConfig-55026.par,opECalib-55026.par: CCD parameter - files for real BOSS data based upon Gunn's document - "Arrangement of Serial Data for BOSS CCD Images Into Images". - We don't yet use these for real data, but use all the hacks - within SDSSPROC instead. -ups/idlspec2d.table: Add optional dependence on the platelist product, - which is used to add information from the plateHoles files to the - plPlugMapM files. - -------------------------------------------------------------------------------- -v5_4_0 (June 19, 2009) idlspec2d running on IDL 5.4 - 7.0 -Tagged to begin development for BOSS spectrographs, backing -out SEGUE-specific development. -Built against specflat v1_7, elodie v1_3, idlutils v5_4_2. - -ARCFIT_GUESS: For the red camera, extend the range of the last - polynomial term, which is necessary for some test data taken - where the wavelength ranges were shifted by large amounts. -BATCH1D,BATCH2D,NEBULAR_BATCH: Fix the priority option to call - '/bin/nice -19' rather than '/bin/nice -n 19' (for example). -CALCSCATIMAGE: Change default break points to NSCATBKPTS=16 (from 40). -EXTRACT_ASYMBOX: Variant of the boxcar-extraction routine that allows - for asymmetric and/or variable windows (Burles). -EXTRACT_BOXCAR: New /IDL keyword to call pure IDL boxcar extraction (Burles). -EXTRACT_OBJECT: Roll back to using RA,DEC from the headers, and try to fix - that upstream of this if it's ever a problem. - Don't return SIGMA from call to FITSN (since it's not used). - Output X structure as HDU #7 to be complient with data model. - Output superflat as HDU #8 (which is used by SPTHROUGHPUT). - Remove HDUs after that unless/until added to the data model. -FIBERFLAT: Fix test as to whether B-spline fit failed. -FLUX2MAGGIE: New simple wrapper for FILTER_THRU. -GET_PLATECENTER: Remove since we'll want this functionality upstream - if needed. -GUIDERMOVIE,GUIDERMPEG: Update to use the dark frames for the original - SDSS guide camera that are now checked into the etc directory, - rather than a hard-wired file name that only existed at sdsshost.apo. -PLUG2TSOBJ: Remove undocumented PLUGFILE keyword. -READSPEC_FOOTPRINT: New routine for reading all spectro objects - within the footprint of a specified plate. -SDSSGUIDE: New IDL guider for the Sloan Telescope. -SOLVEFILTER: Update to use current photometry files. -SPCOADD_V5: Re-remove the synthetic magnitude HDU and the S/N HDU - that should have never been at the end of the spPlate files. - Add superflat as HDU #8. -SPFLUX_V5,SPCOMBINE_V5: Revert to v5_2_0 version until we document - and go through v5_3 branch changes. -SPFLUX_BASESPLINE,SPFLUX_BYEXP,SPFLUX_FITTEMPLATES, - SPFLUX_MAKETELLURICTEMPLATES,SPFLUX_MASKLINES: - Remove from the main-line, and we'll presumably merge back in from v5_3. -SPFLUX_BASESPLINE: Bug fixes in case the wavelength range is 10s of Ang - away from the canonical setup. Our blue throughput tests on - plate 2587/54137 was such a case, where this code crashed. -SPPLAN2D: Get the plate ID from the final output structure, which has - excluded bad files. - Only define code versions once, since it involved a slow spawn command. - Crash if the output directory cannot be created or written to. -SUPERFLAT: Print warning message and return 0 if the wavelength mapping makes - no sense (this could have crashed before, and did for UV test data). - -bin/speclog_update: add -forcecvs option, to help recover from nights - which were copied in, but had CVS problems. -doc/www/spectro_home.html: Add instructions for downloading DR6. - Update Data Model for spCFrame files for superflat in HDU #8. -etc/dark-??-*.fits: Dark frames from the original SDSS guide camera, - copied from sdsshost.apo.nmsu.edu:/data/spectro/guiderDarks. - These files are now used by GUIDERMOVIE,GUIDERMPEG. -etc/emlines.par: Add 2 more nebular lines of interest to Andrew West: - [N_II] 5755 and He_I 5876. -etc/spChunkList.par: Add chunks 162-167,168,169-172. - Chunks 99, 99A and 100A should have been "main" rather than "special". - Add chunks 173-192 and 34A. -etc/spPlateList.par: Update from 2005 through summer 2007 shutdown - (MJD 54275). This adds a total of 226 new plates to the list, and two - plates have had their MJDs changed: 1026/52534 and 1028/52562 are - removed from the list and replaced with 1026/52532 and 1028/52559. - Denote 349 plates as DR6 plates, including 22 that are marked as bad - in the v5_1 reductions. There is just one inconsistency in MJDs on - the sdss.org web site, which is they include plate 1135/53018 instead - of 1135/53024 (which has one exposure from that later date). - This DR6 plate list is from http://www.sdss.org/dr6/coverage/alldr6spectro.par -examples/cartridgeInfo.par: Data file for SDSSGUIDE copied out of the IOP/SOP - product. - -------------------------------------------------------------------------------- -v5_2_0 (February 1, 2007) idlspec2d running on IDL 5.4 - 6.3 -Built against specflat v1_7, elodie v1_1, idlutils v5_2_0. - - Merge from v5_2_test branch, with the following changes: - -ELODIE_FILELIST, PCA_ELODIE: Handle v1_1 and later elodie products - with a template list and blue cutoff.par file. Handle Elodie v3 - catalogs, even though we don't use them. Update - spEigenElodie.fits to match. [This was mostly in by v5_1_7, but - there have been some bugfixes since. ] -PLUG2TSOBJ, BATHCH2D, READPLUMAP: If the TSOBJMAPROOT environment - variable is set, read Survey $TSOBJMAPROOT/tObj-$PLATE files - instead of $SPECTRO_DATA/calibObj/calibPlateP-$PLATE files. -CALCSCATIMAGE, EXTRACT_OBJECT, SPCALIB, DOSCATFIT, -DOSCATTER, GEN_SCATTER, examples/scattering-CAM-MJD.fits: Add - per-camera scattering models; modify spcalib and extract_object - to call doscatter to generate them. -SPFLUX_MASKLINES: Move out of splux_f5. Add /o2 and /h2o options. -SPFLUX_MAKETELLURICTEMPLATES, examples/spFluxTemplates-CAM.fits: - Create telluric template files. -SPFLUX_V5, SPFLUX_BASESPLINE, SPFLUX_BYEXP, SPFLUX_FITTEMPLATES: - Generate flux calibration vectors based on per-exposure - low-order splines plus scaled telluric templates. -FITARCIMAGE: Add /dofirst option to allow getting crude results from sparse - plates, etc. -SPCOADD_V5: Re-add the deprecated synthetic magnitude HDU and the - (ugh) S/N HDU. Appended to end of spPlate.fits - -doc/spphoto.ps: Added JEG notes on spectro-photometry. - -------------------------------------------------------------------------------- -v5_1_8 (January 9, 2007) idlspec2d running on IDL 5.4 - 6.3 -Tagged for Son-of-Spectro. -Built against specflat v1_7, elodie v1_0, idlutils v5_1_7. - -APOPLOT: Bug fix -- use the implied wavelength limits from fcalib_default - -------------------------------------------------------------------------------- -v5_1_7 (January 5, 2007) idlspec2d running on IDL 5.4 - 6.3 -Tagged for Son-of-Spectro. -Built against specflat v1_7, elodie v1_0, idlutils v5_1_7. - -ARCFIT_GUESS: Change search range for the central wavelength from - 0.05 in log-wavelength to 0.15 in log-wavelength. This is - for UV throughput tests that we want to do (see PR #7190). - Until now, the search would only cover +/-200 Ang from 3800 Ang - on the blue side. This is the default DCOEFF[0]=0.05. - The default DCOEFF[0] has been increased to 0.15, for a search - of approx 3200-4520 Ang for the blue side of the blue CCDs. -DESIGN_SDSS3TEST: New proc to design test plates for SDSS-3. -DESIGN_PLATE: Modified to not use most of the dumb TCL scripts in the - "plate" product. -ELODIE_BEST,PCA_ELODIE: Small change to call the new function ELODIE_FILELIST, - and pass MINLOGLAMCLIP to READ_ELODIE. -ELODIE_FILELIST: New function for returning only good Elodie files (Loomis). -EXTRACT_OBJECT,SPCOADD_V5,SPFRAME_READ: Add the spatial traces to the - spFrame and spCFrame files as the last HDU (#7), as per the request - of PR #7137. -EXTRACT_ROW: Catch divide by 0 in reduced chi^2. -LINEBACKFIT: Fix crash condition for when no BACKGROUND vectors are set. - Add /SILENT option. Add BACKGUESS keyword. - Make all loop variables type long (2 places where this could matter). -READ_ELODIE: Updates for the new Elodie-3 catalog. Add keyword MINLOGLAMCLIP - and an internal spline-interpolation over masked pixels. The code - should still return exactly the same results for Elodie-2. -READ1SPEC: Remove this deprecated routine. -READONESPEC: Add optional EXPNUM return value. Add support for reading - the spatial traces (XIMG) from the new outputs. -SIMPLE_PLATE: This is essentially a re-write of DESIGN_PLATE that - still calls all the dumb TCL scripts in the "plate" product. -SPREDUCE1D: Comment-out the STAR_DVELOCITY code for the time being, - since it is not yet debugged but takes many CPU cycles. -ZCOMPUTE: Add FIXED_TEMPLATE as optional input, and ZANS_FIXED output. -ZFIND: Add optional ZANS_FIXED return keyword from ZCOMPUTE. - -bin/spsyncrobot_start: Insist that the SSH keys we were told about can - actually do what we need them to (Loomis). -doc/www/spectro_home.html: Update for DR5. Add info to data model about - spatial traces (XIMG) for spCFrame files. -doc/www/specroSOS.html: Add warning message corresponding to the fix - for PR #6840: "Wrong number of elements for reddeningMed", - which was fixed in v5_1_2. -etc/emlines.par: Add 8 more nebular lines of interest to Andrew West: - [Ne_III] 3868, [Ne_III] 3970, H_epsilon, He_II 4685, He_II 5411, - [O_I] 5577, [S_III] 6312, [Ar_III] 7135 -etc/spChunkList.par: Add chunks 148-157, 153A, 158-162. -examples/opECalib-50000.par,opECalib-51430.par,opECalib-51577.par: - Raise the full well limit to 65535 from 35000, as the detectors - are not showing signs of saturation. - Make the (unused) linearity adjustements 1.0, just for clarity. - PR #6592 discusses these issues in detail. -lib/Makefile: Correctly copy built libraries for platforms other than Linux. - -------------------------------------------------------------------------------- -v5_1_4 (May 31, 2006) idlspec2d running on IDL 5.4 - 6.2 -Built against specflat v1_6, elodie v1_0, idlutils v5_1_4 - -EXTRACT_OBJECT: Print a WARNING statement if either RA or DEC are - missing from the header, since that will result in an invalid - AIRMASS calculation (see PR #6953). Also, report AIRMASS=0 - in the header if RA, DEC or TAI are zero. -FITARCIMAGE: Detector wavelength limits were not being accounted for - when checking for big wavelength gaps (Loomis). -LRGMODEL_SPECTRA: New routine, split out of LRGMODEL_PHOTOZ and/or LRGSIM, - for generating the LRG model spectra. -PLATELIST: Fix the bit of code that finds all input plate files to be - more robust to large number of plates, since a dumb use of - the FINDFILE() function will overflow. Instead, first get the - list of all directories, and then search within each of those. -PLOTSPECQA: Wrapper to PLOTSPEC to select only standard stars or skies. -SDSSPROC: In response to PR #6893 against Son-of-Spectro, don't print - any error messages if the spawn of 'speclog_version' fails. - That bash script only exists in the speclog product, which is - not required for Son-of-Spectro. -SKYSPEC_PARANAL: New function to return the sky spectrum at Paranal. -SKYSUBTRACT: The rescaling of the errors with RELCHI2FIT seems to fail - in the case where all sky fibers but 4 have been rejected. - This happened on plate 2333/53679 exposure b1-00034582, - which is a bright SEGUE plate with all sorts of S/N problems. -SPFRAME_READ: Add option to read superflat vectors from spFrame files. -SPFLUX_V5: We were writing calibration vectors in HDU #0 of the spFluxcalib - files that were double-precision. Recast those to single precision. -SPFLUXCORR_V5: Allow a range of 0.1-10 for the flux correction vectors, - which had been 0.2-5. There were perverse cases (like for plate 321, - c.f. PR #6774) where these large flux correction vectors were - necessary, although the correct solution in that case was to - reject three bad exposures. Also, print a Warning message if the - flux-correction vector is out-of-bounds and then ignored. -SPTEMPLATE_REBIN: New function to read and optionally rebin spectral - templates. This is for use for Monte Carlo simulations of - a spectroscopic survey at different resolutions. -SPTHROUGHPUT: New function to return the throughput and efficiency for - a given plate+camera+exposure - -bin/sprobot_start,sprobot_stop,sprobot.sh: Remove these scripts that - loaded the Spectro cron robot, which have now been split off - into the rsync and reduce cron jobs started with spsynrobot_start - and spredrobot_start. -bin/spsyncrobot_start,spsyncrobot_stop,spsyncrobot.sh, - spredrobot_start,spredrobot_stop,spredrobot.sh: - Split sprobot.sh script into an rsync task that will - be run as a cron job by the "dssdata" user, and a processing task - that will be run as a cron job by "dssrun" user. The intent is to - match the photoop userid logic, where data is copied as one - user and reduced as another. -doc/www/idlspec2d_install.html,spectro_inspect.html: The CVS repository - has been changed from spectro.princeton.edu to - sdsscvs.astro.princeton.edu . -etc/spChunkList.par: Add chunks 139-142, 143-144, 145-147. -examples/opECalib-blank.par: Files with gain, bias, linearity, set to 1,0,1 - to make it a bit easier to investigate raw frames with the standard - tools (Loomis). -examples/opHdrFix.par: Reject some exposures on plate 321 (M67) where the - telescope was clearly not guiding. This addresses PR #6957. -templates/bc03_padova1994_chab_z*_ssp.fit.gz: These are 4 FITS files with - the full BC03 SP unnormalized grids (ages in years, wavelengths - in Angstroms, and fluxes). These files came from Christy Tremonti - on 14 Feb 2006 with the following notes: "The ages are kind of - oddly spaced -- I'm not really sure why, that's just how they come. - Note that the wavelengths are not uniformly spaced outside of the - optical and extend to from 90 A - 1600000 A. (If you quickly plot - wave vs flux, you won't see anything b/c the optical is compressed - at the left edge of the plot!) I created these FITS files using - a handy IDL BC03 reader "im__read_bc03.pro". It was written by - John Moustakas." -templates/fluxed_sky_*.fits: These files contain fluxed sky spectra - from Paranal, all in units of 1e-16 erg/sec/cm^2/A/arcsec^2. - From http://www.eso.info/observing/dfo/quality/UVES/pipeline/sky_spectrum.html - -------------------------------------------------------------------------------- -v5_1_3 (Feb 14, 2006 idlspec2d running on IDL 5.4 - 6.2 -Built against specflat v1_6, elodie v1_0, idlutils v5_1_3 -for bug fix in fluxing. - -FLUX_DISTORTION: Add MAXDPERSTEP keyword with a default value of 0.03, - which is the maximum peak deviation in flux distortion image - allowed per iteration from a change in any one parameter. - This prevents the fit from jumping to crazy solutions with - high order quadratic terms (PR #6774). Note that an update - to the MPFIT code was also necessary for the correct behaviour - of the MPMAXSTEP parameter. -PLATEMERGE: Add EXCEPT_TAGS keyword, and default to excluding tags - with the name '*COVAR', since those are typically large - covariance matrices that I've added recently to the ZANS structure. -QUICKEXTRACT: Bug fix in print statement "Whopping fiber #..." - -etc/spChunkList.par: Add chunks 135-137. - -------------------------------------------------------------------------------- -v5_1_2 (Feb 2, 2006 idlspec2d running on IDL 5.4 - 6.2 -Built against specflat v1_6, elodie v1_0, idlutils v5_1_2 -improving robustness of B-spline routines. - -APOFLUXCALIB: Generalize this routine to work from any plates, and fix - to work with the Spectro v5 outputs. -APOPLOT: Break the flux-calibration code out into the FCALIB_DEFAULT function. -CALCSCATIMAGE,COMBINE1FIBER,FIBERFLAT,MYFLUXCALIB,SKYSUBTRACT, - SMOOTH_SUPERFLAT,SPFLATTEN2,SPFLUX_V5,SUPERFLAT,TELLURIC_CORR: - Include specification of REQUIREN in call to the B-spline - fitting code (BSPLINE_ITERFIT) for robustness, often just - setting this to 1 or 2. -FCALIB_DEFAULT: Function for default flux-calibration vectors, where this - code is based upon what was internal to APOPLOT. -FIND_NMINIMA: Recast some arguments as double-precision. -GET_TAI: Fix a WARNING message that was always spuriously triggered. -LINEBACKFIT: Fix crash condition when not specifing the background terms. - Add options ZLIMITS and SIGLIMITS. -PLATEMERGE: Bug fix whereby the SPECPRIMARY flag was not being set properly - for any object observed multiple times, as pointed out by Hennawi - in PR #6841. -READONESPEC: Add the CAMERAS option for specifying only blue or red CCDs. - Set default return values to zero. Make robust to missing Spectro-1D - reductions, and just return vectors zeros for SYNFLUX in those cases. - Propogate the /SILENT flag everywhere. - Add option to read LINEFLUX, resampled to the output wavelengths. -READPLUGMAP: Make robust to "reddeningMed" in the plug-map file header - being either missing or not having 5 elements, as happened with - the Orion plate 1245 on MJD 53742. -SPCOADD_V5: Print the dynamic range off the flux distortion image, and - print a warning and disable the correction in the dynamic range - exceeds 100. -SPCOMBINE_V5: Change some print statements (use PRELOG in SPLOG command). -SPFLUX_V5: Print statements. -SPFLUXCORR_V5: If flux-calibration failed (for ex, if no standard stars), - then issue a warning message and then use FCALIB_DEFAULT() to - apply a default flux-calibration vector. -SPREDUCE: If plug-map file not found, then print an ABORT message and return. - -etc/spChunkList.par: Add chunks 133-134. -examples/spFluxcalib-b1.fits,spFluxcalib-b2.fits, - spFluxcalib-r1.fits,spFluxcalib-r2.fits: Update thse default flux- - calibration vectors with the v5 reductions of plate 406/51817 - generated by APOFLUXCALIB. -lib/.cvsignore: Add appropriate files to ignore by "cvs update". - -------------------------------------------------------------------------------- -v5_1_0 (Dec 19, 2005 idlspec2d running on IDL 5.4 - 6.2 -Built against specflat v1_6, elodie v1_0, idlutils v5_1_0 for upgrading SOS; -fix double-precision on OSX and Linux64. - -ARCFIT_GUESS: Change the FIX() functions to LONG(), though I don't believe - this was causing any trouble. -APO_LOG2HTML,APOREDUCE,BATCH1D,BATCH2D,CATPLOT,SDSSPROC, -APO_PLOTSN,PLOTSN: There was a problem long ago whereby the fiber magnitudes - from photo changed, and we decided to change the magnitude at which - Son-of-Spectro computes the S/N for each exposure. This was fixed - in Oct 2003 as documented in PR #5642 and sdss-obs/4029. However, - the S/N plots from SOS still use the old magnitudes. I've changed - those g,r magnitude limits from 20.20,19.90 -> 20.33,20.06 to be - consistent with the SOS tables. Note that the old limits will still - be used by the full Spectro-2D reductions (since those are now tied - to PSF magnitudes in newly-generated calibObj files). -ATVRAWSPEC: Plot a red column over every bad column, not just a box around - each bad region. For ex, the 4 bad columns (1112-1115) on r1 will - not appear with a 4 red lines. -BATCH2D: Allow PLATENUMS to be integer-valued, in which case they are - recaste as zero-padded strings that are 4 characters long - (such that they match the directory names). -CHECKSN,MAKE2DMERGE,SPALLCOMBINE,SPALLREDUCE,SPMULTI,SPPLANCOMB,SPPLAN: - These are all deprecated Fermi-only routines that would no - longer work with the v5 versions of this code, so remove them. -COMBINE1FIBER: Make robust to the B-spline code failing (and return zeros). - Fix a rare crash condition that happened on plate 321: It is - possible for NUMINSIDE to be zero (on the inner loop), if the - input data points span an extremely small wavelength range, - within which there are no output wavelengths. -COMPUTECHI2: Add optional outputs COVAR,VAR. - Explicitly do all arithmetic in double-precision, due to the change - in behaviour of IDL on the Mac and 64-bit Linux where matrix multiplies - have less precision (as described in PR #6754). If the change is - only made here, then there is alot of recasting of arrays that - will slow the code. Therefore, in practice we'll want to recast - these arrays in the calling routines (like in ZCOMPUTE). - Fix the computation of VAR and COVAR in the case of one template - vector in AMATRIX. -ELODIE_BEST: Trim to stars not labelled as binary or triple stars, according - to the Q_VR keyword in the Elodie FITS headers. Rockosi brought - this to my attention on Sep 20, 2004. -ELODIE_BEST,FITREDSHIFT,FITMEANX,FITWITHMX,MYFLUXCALIB,SDSSPROC,TRACE_FIT: - Add /double keyword to STDDEV() call, such the behaviour will be - the same on 32-bit or 64-bit machines. -EXTRACT_BOXCAR,EXTRACT_PROFILE,EXTRACT_ROW,SSHIFTVEC: - Call IDLUTILS_SO_EXT() to get the suffix for dynamically - loaded libraries (rather than assuming ".so"). -EXTRACT_OBJECT: Write as HDU #8 an image of the relative chi^2, or an - image of all 1's if no such structure was ever made in the SKYSUBTRACT - code. Note that these values are not truncated to be >=1, as they - are when we re-scale the sky errors. - If there is a telluric image (which we've deprecated for the v5 code), - then it is now in HDU #9. -FIBERFLAT: Replace the /EACHGROUP keyword in BSPLINE_ITERFIT with appropriate - GROUPSIZE keywords. The former was a holdover from the days - of the Fortran Slatec code. -FIND_NMINIMA: Catch the status from calling MPFITPEAK, and deal with it - appropriately if the fit failed. -FINDSPEC: Complete re-write to match against long lists of objects quickly, - rather than the slow loop over one object at a time. Also, address - PR #6583 to deal with the crazy possibility of the same object (with - the same RA+DEC) appearing on the same plate. -FITARCIMAGE: Print the wavelengths of any rejected arc lines. -FITREDSHIFT,QAPLOT_FCALIBVEC,SOLVEFILTER,SPREDUCE1D,VELDISP, - FLUX_DISTORTION,SDSSPROC: Change calls from the deprecated STDEV() - to STDDEV(). -FITSN: If fewer than 20 good points within the default fitting mag range, - then redefine that fitting range to be [0,+1] mag from the median - of good magnitudes. This makes the plots look better from Spectro-2D - and from SOS. It should also help address PR #6325, by choosing - more reasonable fit ranges for SEGUE plates with bright stars. -FLUX_DISTORTION: There was a bug in the way objects were being rejected - in the main iteration loop. Basically, objects were being rejected, - but then had chi=0 in the next iteration, and then were put back - in the fits. I now have a /nomask option in flux_distort_fn() - that allows me to compute chi even for rejected points. - Also, always start the fitting routine with tiny values for - the coefficients. This is to prevent us from "walking away" from - the correction solution in the case where some initially bad points - effect the fit. - Add two more terms, which are quadratics in the wavelength parameter - (one for each spectrograph). This takes out curvature in the spectra - which I otherwise couldn't fit out -- though if the AB zeropoints are - not correct, then these terms enforce consistency but are not correct. - Add the functionality to have multiple offset pointings on the same - plate, which is true for SEGUE plates 2247 and 2255 (the latter was - already observed). Do the standard flux-distortion corrections from - the first offset (where OFFSETID=1). If there are repeated objects - between that offset and others (as determined by matching RA,Dec), - then solve for the mean flux-correction vector between those offsets. - Otherwise, scale the fluxes according to SCI_EXPTIME in the plugmap. - When fitting for the ratio vectors, use THISIVAR internally instead - of OBJIVAR (the former masks out bad sky-subtraction regions). -LINEBACKFIT: The calculation of LINEEW_ERR was incorrect, as noted in PR #6235. -LRGMODEL_PHOTOZ: Optionally compute and return FITFLUX. -PCA_CVSTAR,PCA_GAL,PCA_QSO: Insist that all requested spectra exist. -PCA_GAL: Make plots. -PCA_SOLVE: Make loop variables long ints. - Make the internal arrays double-precision instead of floats - before calling COMPUTECHI2 due to round-off errors on some - platforms (PR #6754). -PLATELIST: Use FINDFILE() function to see if files exist before calling - MRDFITS(); this should be a slight speed-improvement. - Since the v5 version of this code will combine data with as few as - 1 exposure in a given camera, set the plate quality to 'bad' if there - is only 1 exposure, or 'mariginal' if only 2 exposures in any given CCD. - Allow INFILE to be an array. -PLATEMERGE: Add the fields CALIBFLUX,CALIBFLUX_IVAR which are explicitly - copied over from the plug-map structures. - Get rid of the ZFILE option in place of the PLATE,MJD options. - Make robust to the spZLine structures being different for - different plates, for ex. if different code versions were used. - Re-write the bit of code that matches objects. This was necessary - to address PR #6583, since there are now some bizarre, special - plates where the same object appears twice. Note that this also - fixes the bug whereby some objects are not matched if the plate list - has incorrect central RA,DEC positions (such as plate 1274 on MJD 52995 - where those values are NaNs). The new implementation uses SPHEREGROUP. - Also, the logic for setting SPECPRIMARY was slightly changed. -PLOTSN: Choose which objects are "good" on a filter-by-filter basis, - to avoid stars with crazy magnitudes (like 0 or -9999) from - throwing the plots (for example, on SEGUE plate 2255). - Trivial bug fixes to prevent crashes when making plots with - a single point. -PLOTSPEC: Add capability to plot reconstructed frames (added by Blanton). - Fix units on Y axis to /cm^2 instead of /cm (was fixed in v4_9_13 - when plotting to files, but not when to the terminal). - Break out call to displaying photo images with a call to - the new routine PLOTSPEC_IMAGE. -PLOTSPEC_IMAGE: New routine called by PLOTSPEC for displaying images - with calls to the photoop product. -PLUGMAP_TYCHO: New proc for plotting bright Tycho stars on a plate. -QAPLOT_FCALIBVEC: Put plots for each spectrograph on its own page, and - put offsets between the plots for all objects, F stars, and QSOs - so that we can see the plots more easily. Also, smooth the plots - a little bit after the median-filtering. -QUICKEXTRACT: Remove the end of the warning message for "Whopping fiber" - which used to say "(may have adverse afffect on S/N)". This was - being interpreted as chaning the S/N on the whole plate, when in - fact it usually just affects a few fibers. - For the warning message "Large flexure...", append the shift - number to the message (in response to PR #6239). - Only print the fiber numbers for the first 5 whopping fibers. - Change cod to be consistent with SKYSUBTRACT, which now returns - RELCHI2SET rather than RELCHI2STRUCT. -READPLUGMAP: Fix bug when replacing unknown fluxes in the calibObj - structure with old values in the plug-map file. - Add OFFSETID and SCI_EXPTIME to the output structures, which - are obtained from an optional PLUGMAPTARG structure in the - plPlugMapM file. The plate where I've added that is SEGUE plate 2255, - which has multiple offsets in the same exposure. - Do not use magnitudes in the input file not in the range [0,50], - in order to avoid overflows when magnitudes are set to crazy - values like -999. - Add missing iobj, so that the call from QUICKTRACE with the /DEREDDEN - option works. - Add HDR return keyword, which is used by SPREDUCE routine. -READONESPEC: Return all headers in FRAMEHDR. -READSPEC: Change how the internal procedure READSPEC1 works, such that - arrays of zeros are returned if data is missing. This allows us - to return a good one-to-one list of objects from an input list - of plate,mjd,fiberid. However, I'm not positive that this is always - the desired behaviour. For ex, requesting a non-existent plate - will now return 640-element arrays of zeros. - Make this routine robust to missing data when reading in structures, - such as plugmap, zline, tsobj. -REDMONSTER: Make robust to the perverse case of the number of (good) pixels - in the spectrum being less than the median filter size of 25 pixels. - This happened for saturated data on plate 2333, but does not happen - on any other plates. -REDUCE_PLATE: Removed (old code). -SDSSPROC: Re-enable the call to 'speclog_version'. - Add call to FITS_PURGE_NANS in order to address PR #6032, 6084, - whereby the DA has written invalid FITS header keywords. - Use the measured read noise (rather than the value in the op file) - for constructing the inverse variance image. - Add three more possible warning messages from testing the statistics - of the bias region: - "expected read noise = ..., measured = ... DN", - and - "bias region difference at xxx-th-percentile =... DN", - and - "Amp #... way too many pixels (xxx%) below bias-5*sigma=... DN." - This replaces the warning message: - "Std. dev. in bias region for amp#... is ... DN (5-sig clip)". - These tests should be more robustly sensitive to PR #2335 - (when r2 was randomly flipping bits on 22/23 Aug 2000), and - the more recent PR #6143 (r2 electronics problems in Aug 2004). -SKYSUBTRACT: Return the trace-set RELCHI2SET instead of RELCHI2STRUCT, - for use by the EXTRACT_OBJECT code. - Fix a bug, where the "airmass_correction" term was not squared - when applied to the sky variance (on line 285). - Apply a ratio of "number of sky pixels in a breakpoint bin" to the - number of degrees of freedom in the fit to the correction to - local chi^2. - Recast "fullfit" to floating-point to avoid returning a double- - precision sky array. This was happening due to same changes to - the B-spline code to support double-precision. - Replace the /EACHGROUP keyword in BSPLINE_ITERFIT with appropriate - GROUPSIZE keywords. The former was a holdover from the days - of the Fortran Slatec code. -SLITHISTORY: Do not plot missing measurements (zeros). -SMOOTH_SUPERFLAT: Bug fix, whereby we could trigger a warning message when - the ends of the wavelength range did not have any good data points. - In such a case, the fits are meaningless, and any value could result. - This was triggering false warning messages on plate 1053/52468 in r2, - plate 1051/52468 on r2, etc. This should now be fixed by setting - the internal variable RATIO=1 for bad points. -SPBIASAVE,SPFLATAVE: Fix bug where filename matching is off-by-1. -SPBIASGEN: Trivial bug fix to mask bad pixels. -SPCALIB: Change call from SMOOTH_HALO() to SMOOTH_HALO2D() for fitting - the scattered light background. This should do a more proper treatment - of the scattering, but in practice does not seem to make much - difference. -SPCOADD_V5: Apply the flux-distortion image to the sky vectors, as well - as to OBJFLUX,OBJIVAR. - Match objects between exposures by XFOCAL,YFOCAL in the plugmap - rather than using RA,DEC. This is because there are the bizarre - plates like SEGUE plate 2255 that has the same object several times. - Add keywords NEXP_B1,NEXP_B2,NEXP_R1,NEXP_R2 to output header. -SPCOMBINE_V5: Make the (dangerous!) change of setting the minimum score - for an exposure to a S/N value of >0 rather than >1. This is - necessary for some SEGUE plates, but is dangerous for main survey - plates if some data is not marked bad. Hopefully, we'll track - all those bad data down with PR #6722. -SPFLATGEN: Discard exposures where the quality keyword is set to 'bad'. -SPFLATTEN2: Recast an internal variable to float to prevent double-precision - output files. -SPFLUX_READ_KURUCZ: This function was pulled out of SPFLUX_V5. -SPFLUX_V5: Pull out the SPFLUX_READ_KURUCZ() code as a separate function. - Comment-out the median/mean rescaling factor in SPFLUX_MRATIO_FLATTEN(), - since I think that could potentially be mis-matched between the blue - and red sides. For rejecting the F stars, start by rejecting any - stars with more than 20% bad points in any image. (This fixes a - problem with plate 662/52147 where a bad star throws the fits.) - Rescale the per-fiber plots of the spectro-photo vectors using - the FLATARR such that they should all lie on top of one another. - The "Flux-calib" plot for spectro-2 incorrectly listed fiber numbers - between 1 and 320 instead of 321 and 640 -- fixed. - Only select spectro-photo stars from OFFSETID=1, which will deal - with multiple-offset plates like SEGUE plate 2255. -SPFLUXCORR_V5: Return chi instead of sqrt(chi2) from the function FCORR_CHI_FN. - Add INPARAMS option to SPFLUXCORR_SOLVE2() function. - In the final loop, first do a fit using the SPFLUXCORR_SOLVE() code - that does not rescale the errors, then use those fits as the starting - point when calling SPFLUXCORR_SOLVE2(). - Test all solutions up to MAXPOLY terms, not necessarily stopping - when there is no improvement to chi^2. This is because one could - imaging that going from 1->2 terms is very little improvment, but - then 2->3 is substantial improvment. - Generalize the internal routines SPFLUXCORR_SOLVE,SPFLUXCORR_SOLVE2 - and rename it to SOLVE_POLY_RATIO in the idlutils product. -SPGAIN: Revamp this code, trying to measure the gain from a set of flats - taken on MJD 53114 for this purpose. -SPPLAN1D,SPPLAN_FINDRAWDATA: - Replace calls to the obsolete function RSTRPOS() - with STRPOS(/REVERSE_SEARCH). -SPREDUCE: Pass exposure time to READPLUGMAP, which now uses that for - multiple-offset plates (like SEGUE plate 2255). - Return plugmap-header from call to READPLUGMAP(), since it is - used to get the CARTID for the output files. -SPREDUCE1D: (Use the B-spline spectra for QSO-fitting). - When computing FRACNSIGMA,FRACNSIGHI,FRACNSIGLO, ignore points - blueward of rest-frame 1216 Ang, since these numbers would then - be dominated by LyA absorption in QSOs. (This is PR #2703.) - Add call to STAR_DVELOCITY, but only for stars, even though in - principle it should work for any objects. -SSHIFT,SSHIFT2D: Remove these routines that are in the idlutils product. -STAR_DVELOCITY: New proc to solve for velocity shifts between multiple - exposures. -SYNTHSPEC,ZFIND: If the eigenspectrum file does not contain in image in HDU #0, - then look for a B-spline in HDU #1 and assume that it describes - a B-spline. (This is what we're trying to use for QSOs now.) -TRACE320CRUDE: This routine was not actually setting the default value - of RADIUS, but was getting it from an internal call to TRACE_CRUDE. - This has been fixed (necessarily, after a similar fix to TRACE_CRUDE). -ZTWEAK_STAR: Add optional keywords. -ZCOMPUTE_QSO: A version of ZCOMPUTE that is basically the same code, but - using a B-spline composite spectrum that must be evaluated at each - redshift. Because there are multiplicative polynomials rather than - additive polynomial terms, this may fail to find weak em-line QSOs. -ZCOMPUTE,ZCOMPUTE_CHI2: Recast the fluxes and models in double-precision - before calling COMPUTECHI2 due to round-off errors on some - platforms (PR #6754). -ZCOMPUTE: Add THETA_COVAR to the output structure. Instead of storing all - of the coefficients for the fits at all lags, add a final call - to COMPUTECHI2 at the end to compute THETA and THETA_COVAR. - These changes relate to PR #6757. -ZFIND: Mask any pixels on the templates where the first template contains zeros. - This is useful, in particular, where the stellar templates have zeros - at the beginning or end of the spectral range due lack of wavelength - coverage. This is done by setting the STARMASK parameter that is - passed to ZCOMPUTE. This should fix PR #5903. - Return THETA_COVAR as computed by ZCOMPUTE (see PR #6757). -bin/aporsync_alllog.sh, aporsync_blue.sh, aporsync_logs.sh, aporsync_red.sh, - killdata.sh, sos_start: Pull the data (sdR files and log files) - from the machine specified by the environment variable - $RAWDATA_HOST rather than from the hardwired sdsshost.apo.nmsu.edu. - Set the default value to sdssfiles1.apo.nmsu.edu in the sos_start - script. -bin/aporsync_alllog.sh, aporsync_blue.sh, aporsync_logs.sh, aporsync_red.sh: - Remove the "--rsync-path" option to these scripts, which was used - to point to the only working version of rsync on sdsshost.apo. - Now that the IRIX machine sdsshost.apo has been replaced with - a Linux machine, that is no longer necessary. -bin/spreduce_batch: Script to batch a single Spectro-2D or Spectro-1D job. -bin/sprobotlist.sh: Only run the PLATEMERGE command for all data, but - not all the subsets such as DR1, etc. No need to recreate those - every day! -doc/www/idlspec2d_reduce.html,spectro_home.html: The machine alias name - sdssdata.princeton.edu should really be sdssdata.astro.princeton.edu -doc/www/spectro_home.html: Add info for downloading DR4. -etc/lamplist.dat: Remove this file, since we use "lamphgcdne.dat" instead. -etc/spChunkList.par: Add chunks 91 through 107, 99A, 100A, 108, 109, - 110, 111, 112, 113, 114, 115-118, 119, 120, 121, 122, 123, 124, 125, - 126, 127, 128, 129, 123A, 131. -etc/spPlateList.par: Add 436 new plates, through MJD 53360 (Dec 21, 2004). - Add the DR4 public plates, consistent with sdss-archive/2374. - (I posted some inconsistencies about these plate lists to - sdss-archive/2376.) Note that I have all the same plates being - marked as public, except there are four plates from the EDR - the FNAL has dropped from their lists: 314/51641, 357/51813, - 362/51999, 412/51871. - The following plates are public, but bad: - 504 52316 DR2 bad Bad spectrograph collimation - 704 52205 DR4 bad Bad spectrograph collimation - 721 52228 DR2 bad Bad spectrograph collimation - 761 52266 DR2 bad Bad spectrograph collimation - 769 52282 DR2 bad Bad spectrograph collimation - 770 52282 DR2 bad Bad spectrograph collimation - 775 52295 DR2 bad Bad spectrograph collimation - 778 52337 DR2 bad Bad spectrograph collimation - List each plate in one and only one data release (the first time - it is released). - The most recent list from Fermi (from March 15, 2005; - pointed to in sdss-archive/2439) misses the following plates - that I have in my list from earlier e-mails. All of these are - repeats of other observations of the same plates: - 293 51689 EDR - 352 51694 EDR - 360 51780 DR1 - 412 51931 EDR - 476 52027 DR4 - 496 51973 DR4 - 525 52029 DR4 - 574 52347 DR4 - 574 52366 DR4 - 581 52353 DR4 - 641 52176 DR4 - 722 52206 DR4 - 903 52385 DR4 - 1296 52738 DR4 - Add another 233 plates through MJD 53566 (July 14, 2005). - Marked plates for DR5, according to sdss-archive/2661 (226 plates) and - sdss-archive/2684 (89 plates). Note that I already had the following - two plates in DR4: 712/52179 and 1664/52965, and will keep them there. - I've also added 1138/53228 and 1144/53238 to DR5, since the Spectro v5 - reductions turn these plates into 'marginal' rather than 'bad. - Note that the following 18 'good' or 'marginal' plates could be - included (c.f. sdss-archive/26744): - 230 52251 special - 231 52221 special - 1244 52674 special - 1247 52677 special - 1250 52930 special - 1251 52964 special - 1252 52970 special - 1253 52974 special - 1254 52972 special - 1256 52902 special - 1257 52944 special - 1259 52931 special - 1637 52992 special - 1638 52999 special - 1641 53032 special - 1907 53265 special - 1913 53295 special - 1917 53295 special - Remove the plates 521/52325 and 574/52337, which were only - partial reductions that should not be in this list. -examples/opBC-50000.par,opBC-51809.par,opBC-51813.par: Added a partial - bad column on the b2 CCD that is responsible for some of the absorption - glitches seen in fibers 343+344 (c.f. PR #3681). -examples/opECalib-51577.par: Update the gains based upon work by Eric Switzer - and a more careful look at a recent set of identical flats taken - on the night MJD 53114. I've updated the numbers as follows for - the two gain and two read noise values per CCD: - b1: 1.13 1.13 3.9 3.3 -> 1.10 1.05 3.86 3.24 - b2: 1.32 1.32 3.7 3.26 -> 1.23 1.26 3.88 3.38 - r1: 1.15 1.00 3.7 3.7 -> 1.09 1.00 4.08 4.04 - r2: 1.00 1.05 3.7 3.7 -> 1.05 1.05 4.32 4.12 - Note that this is the opECalib file used for the vast majority of data. -include/export.h: New export.h file in the IDL 6.2 beta distribution. -lib/Makefile: For "make clean", also remove ".a" and ".dylib" files. -pro/Makefile: Remove reference to pro/fplan directory, since all those - deprecated routines have been removed. -pro/photoz/Makefile: Add this Makefile. -templates/eigeninput_star.par: - Shift all of the normal stellar templates (O through M) to - agree with the Elodie redshifts in the v5_0_0 reductions. - Shift the CarbonWD and Carbon star templates by +65 km/sec. - No change to the WD or WDmagnetic velocities (out of ignorance). -templates/spBsplineQSO-53115.fits: Fist attempt at a B-spline composite - for QSO spectra as a fn. of redshift (made by Burles). -templates/spEigenCVstar-53724.fits,spEigenGal-53724.fits,spEigenQSO-53724.fits, - spEigenStar-53724.fits: Rebuilt these templates using an untagged - version of the reductions essentially equivalent to v5_1_0, such - that the spectrophotometry will be improved. Note that we still - mean to improve these templates at some later date by improving - the lists of input objects, especially for stars. -templates/wfEigen*.fits: Add WFCCD templates from Prochaska - -------------------------------------------------------------------------------- -v5_0_0 (Feb 19, 2004 idlspec2d running on IDL 5.4 - 6.0 -Built against specflat v1_5, elodie v1_0, idlutils v5_0_0. -The speclog product was also tagged at v1_3, valid through MJD=53054 (today). - -Next major release of Spectro-2D, with the "Version 5" data model. -Only minimal changes to P-1D, but search galaxy redshifts to z=1. - -APO_PLOTSN,QUICKTRACE: Need the /APOTAGS in the call to READPLUGMAP(), - which has been re-written to be used by the full 2D reductions too. -BADFMAGS: Find bad F star photometry that might get used by Spectro-2D. -BADFSTARS: Find bad F stars used for spectro-photometry in the Spectro-2D. -BATCH2D: Call SPCOMBINE_V5 instead of SPCOMBINE. - Update the list of output files to correspond to the new v5 outputs. -COMBINE1FIBER: Add the skies by inputting SKYFLUX and outputting NEWSKY. - The skies are combined in the same simplistic manner as the - dispersion pixel map. - Make loop variables type long. - In this case of no good points, set the NODATA bit everywhere. - Also, if NOPLUG is set in the first input bit-mask, assume it - should be set everywhere in the output bit masks. No other mask bits - are set in this case of no data. This should address PR #3690. -DJS_FLUXCORR,FLUXCORR: Remove these deprecated routines, which are now - replaced with FLUXCORR_NEW (if using SPCOMBINE) or SPFLUXCORR_V5 - (if using SPCOMBINE_V5). -EXTRACT_OBJECT: Turn off telluric-corrections by default, which must now - be enabled with the /DO_TELLURIC keyword. Also, delete some old code. - Compress (with gzip) the output file. -FIBER_ROLLCALL: This code has been broken out of PLATESN. -FILTER_THRU: SDSS filter curves are now in the idlutils products under - the subdirectory data/filters. -FIND_NMINIMA: Fix bug when plotting. -FLUX_DISTORTION: New function for computing the overall flux-distortion - on an entire plate using the spectrophotometry compared to the - fluxes in the plug-map structure. -FSTARS_PLOT: New code to make color-color plots of spectro-photometric stars - as compared to the stellar locus and BD+17. This code actually dates - back to Sep 2002, but was never checked in. -GET_TAI: Report warning about exposures less than 2 minutes only - for science or smear exposures. -JEG_SPHOTO_COEF: Christy's spectrophotometry code using Jim Gunn's functional - form to correct using the PHOTO magnitudes. -KURUCZ_FITSFILE: New proc for generating a single FITS file from a list - of ASCII-formatted Kurucz models. -LINEBACKFIT: Consolodate some of the print statements from MPFIT onto 1 line. -LRGMODEL_TRAIN: Calling script for training the Bruzual-Charlot models - for photo-z's. -PCA_GAL: Set the starting wavelength to 1850 Ang so that we can go to z=1.0; - it was 2300 Ang, which only let us fit out to z=0.6. -PCA_STAR: Interpolate over bad pixels in the middle of any spectra, - but still set any bad end pixels to zero (as before), which should - make those pixels be ignored in the redshift-fitting code. -PLATELIST: Get rid of the SMEARUSE entry, since it is now deprecated. - Include any public plates in the default output file; this is - because there are some DR1,DR2 plates that are called 'bad', but - we probably still want them in the full merge files. - Read the spectro-photometry keywords [GRI]OFFSTD,[GRI]OFFRMS, etc. -PLATESN: Remove the code that adds the spectro-photometry keywords to - the output headers, since P-1D does this more carefully. - Split out the fiber "roll call" code into FIBER_ROLLCALL. - Add a bunch of spectro-photometry keywords to the FITS header, - such as GOFFSTD,GRMSSTD,... -- These will be read by PLATELIST. -PLOTSIGNAL: New routine for making diagnostic plots of the signal seen - by the spectrographs, unlike most of our routines that show S/N. -PLOTSN: Re-write of the code that makes all the plots beyond the first - page of S/N plots. Compare the spectro-photometry to the CALIBFLUX - entries in the plug-map structure if they exist. -PLOTSPEC: Add the /SKY option for over-plotting the sky. - Add the /ALLEXP option for plotting the fluxes from all exposures, - rather than the co-added spectrum. -PLUG2TSOBJ: Add /SILENT option. -PLUGFILE_RELABEL: Relabel the OBJTYPE's for specified objects in all - plPlugMapM files. -READONESPEC: New proc for reading single exposure spectra (from spCFrame files). -READPLUGMAP: Many changes to this code, which is now used by both SOS - and by Spectro-2D. Options /APOTAGS,/DEREDDEN are for the SOS code. - Option /CALIBOBJ is for the Spectro-2D reductions, and re-calibrates - the photometry to the Princeton calibObj files. - Also read SFD_EBV (reddening values) with the /CALIBOBJ option. -READSPEC: Add support for reading the sky (SKY=) from HDU #6 of the spPlate - files. - No longer name the returned plug-map structure, since we may be - adding tags to this structure, and I believe structures no longer - need to be named in order to concatenate them in IDL. -SKYSUBTRACT: Fix possible crash condition, whereby the re-fit to the - supersky is not done if the first fit failed and FULLBKPT does - not exist (line 243). -SPBIASGEN,SPFLATTEN2: In the output headers, start numbering the EXPID - keywords at EXPID01 instead of 00. Also, drop MJD from that string. - This makes it consistent with the keywords in the spPlate files, for ex. -SPCALIB: Change the way we spawn the gzip command. -SPCOADD_FLUXED_FRAMES: Christy's code that now de-reddens the spectra; - definitely do not use! -SPCOADD_FRAMES: Get rid of some unused lines of code. - Get rid of returned synthetic mags + S/N values from PLATESN, - since I've excised that code, and don't write the undocumented - last two HDUs to the spPlate files. -SPCOADD_FRAMES,SPCOADD_V5: For the EXPIDxx keywords in the output spPlate file, - start the numbering with EXPID01 (instead of 00) because that's more - like the FITS standard, and SXPAR('EXPID*') can then be used. - Also, drop the MJD from this identifying string, so that the first - 11 characters can be used to directly determine the names of the - individual spFrame or spCFrame files. - Call FIBER_ROLLCALL from this routine, rather than doing it in PLATESN. -SPCOADD_V5: New code to replace SPCOADD_FRAMES for version 5 outputs. - The sky vectors are now combined in some semi-rational way, - and output as HDU #6. -SPCOMBINE: Pass all arguments if this proc is called recursively. -SPCOMBINE_V5: New code to replace SPCOMBINE for version 5 outputs. - Add call to FLUX_DISTORTION to compute the 2-D spectro-photometric - corrections. This modifies the fluxes both in the spCFrame and - the spPlate files. This is a fairly major change, in that the - spectro-phometry is now explicitly tied to the plugmap-structure - magnitudes and colors (at least in the mean). - We currently hard-wire the rejection of all smears, all exposures - with any CCDs with (S/N)^2 < 1, and any with (S/N)^2 less than 20% of - the best exposure. -SPFLUX_V5: New code to replace SPFLUX for version 5 outputs. This computes - the spectro-photometry vectors on an exposure-by-exposure basis - (though doing the fit on all exposures at once). The low-order - per-object flux-correction vectors that used to be handled - (through a call to FLUCORR_NEW) must now be handled separately, - and *after* the call to this routine. -SPFLUXCORR_V5: New code to replace FLUXCORR_NEW. This will be called - by SPCOMBINE_V5. -SPFRAME_READ: Routine for reading specified HDUs and fibers from either spFrame - or spCFrame files. -SPREDUCE: Abort and return from this procedure if no good flats exist. - Call READPLUGMAP() to read the plug-map file, and get the photometric - calibrations from the calibObj files. -SPREDUCE1D: Dispose of COUNTS_SPECTRO,COUNTS_SYNTH in favour of the fields - SPECTROFLUX,SPECTROSYNFLUX which are in nano-maggies (1e9 larger). - Also compute SPECTROSKYFLUX,SPECTROFLUX_IVAR,SPECTROSYNFLUX_IVAR - from the sky vectors, which are now in the new spPlate files. - Log peak memory usage. - Fit galaxy redshifts out to z=1.0 (instead of 0.6). - Add the /verbose option to the zfind() function for the main loop - finding galaxy redshifts and QSO redshifts, since those are the two - loops where this code spends most of its time. -SPREDUCE2D: Report peak memory usage. -SPREDUCE,SPREDUCE2D: Pass the optional /DO_TELLURIC along to EXTRACT_IMAGE. -UPDATE_PLATE_RELEASE: Code to mark the DR2 plates (for example) from - the "etc/dr2spectro.par" file into "etc/spPlateList.par". -VDISPFIT: Change the SPLOG of "Using previously cached..." to a simple - print statement so it doesn't go in the log files so many times. -ZCOMPUTE: Do not be so verbose (printing a line to the log for every fit) - unless a /verbose keyword is set. This is because the log files - were being dominated by this one line! - -etc/dr2spectro.par: List of DR2 plates from the web site - "http://www.sdss.org/dr2/coverage/dr2spectro.par". -etc/spChunkList.par: Add chunks 87-89. -etc/spPlateList.par: Add list of DR2 plates. -etc/sdss_*.dat: Remove these SDSS filter curves, which are now in the - idlutils products under the subdirectory data/filters. -etc/skylines.par: Updates many of the sky lines to agree with the NIST 2.0 - database at "http://physics.nist.gov/cgi-bin/AtData/main_asd". - The changes are very minor. This does not update any of the OH lines. - Change the HgI 5460.74 wavelength to 5460.94 Ang; this is completely - ad hoc, simply because it *looks* like the line always ends up there - (c.f. PR #3259). This may be the actual centroid of the line since - it is pressure-broadened in a funny way, but this should definitely - be checked (the PR will not be closed). -include/export.h: Change to be consistent with the file by the same name - in the idlutils product. This has two minor changes for SGI 64-bit. -templates/eigeninput_star.par: Remove the 3 objects labelled Carbon_DQred, - since those were actually BAL quasars (c.f. PR #3953). - Also, rename the "Carbon_DQ" objects as "CarbonWD". -templates/kurucz_stds_raw_v5.fits: FITS file with Kurucz templates - generated by KURUCZ_FITSFILE. This is simply the set of Kurucz - ASCII files collapsed into a single FITS file, which is then - used by SPFLUX_V5. -templates/spEigenCVstar-53054.fits, spEigenGal-53054.fits, - spEigenQSO-53054.fits, spEigenStar-53054.fits: - These are a new set of templates built from almost exactly the - same list of objects as before, but using preliminary v5 reductions. - Also, some mis-classified stars removed from the star list. - -------------------------------------------------------------------------------- -v4_11 branch + v4_11_0 (Feb 6, 2004) - -This branch was created by Lupton for FNAL operations. -The branch point was v4_10_7 == v4_11_0, with the following changes: - -Four files copied from the mainline as they were on this date -which was a tag on the mainline called RHL-create-v4_11): - pro/fluxfix/jeg_sphoto_coef.pro : New routine - pro/fluxfix/spcoadd_fluxed_frames.pro : Only changed by Tremonti - pro/spec2d/platesn.pro : Only changed by Tremonti - pro/spec2d/plotsn.pro : Bug fixes by DJS and later changes by Tremonti. - -------------------------------------------------------------------------------- -v4_10_7 (Dec 11, 2003 idlspec2d running on IDL 5.3 - 5.6 -Built against specflat v1_5, elodie v1_0, idlutils v4_10_7. -Tagged for fixing SOS PR #5735. - -COLLIMATE: Since the DJS_SVDFIT() function was removed in v4_10_1, - explicitly compute the chi^2 minimum with a matrix inversion. -FIND_NMINIMA: If the fit value of YPEAK is less than zero, then change - it to zero. This code is always used for chi^2 minimization, - and a negative value for the chi^2 would not make sense. -LRG_PHOTOZ: New simple photo-z finder for LRGs using a single template. -LRG_PHOTOZ_TEMPLATE: Code to tweak the photo-z template used by LRG_PHOTOZ(). -LRG_PHOTOZ_TEST: Code to test the photo-z's. -LRGMODEL_PHOTOZ: Photo-z finder using Bruzual-Charlot elliptical galaxy models. -LRGMODEL_TWEAK_TEMPLATE: Code to tweak the Bruzual-Charlot model parameters - used for the templates in LRGMODEL_PHOTOZ(). -NEWSPCOMBINE: Christy's code: changed so that if one frame does not make - the S/N cut then all 4 frames of that exposure are rejected. -PLATELINKS: Call SPAWN with the /noshell option to greatly increase - the speed of this procedure. -PLATEMERGE: Major re-write to use less memory. All of the ZANS structures - are read into memory at once, but not all of the TSOBJ structures. - The output file are written one plate at a time, with the TSOBJ - structure for that plate read only when it is needed. - Functionally, the outputs are exactly identical. -PLOTSN: Christy's last page of plots calls GAUSSFIT(), which can crash - with too few points. This happened on the special plate 1180 - on MJD 52974. I've changed this to using DJS_ITERSTAT. -QUICKEXTRACT: The logic to trim away extra sky fibers when there are - more than 20 appeared to be incorrect, and could trim away most - or all sky fibers. I fixed this, and also now choose 40 skies - in the case of many skies. -SDSSPROC: Don't print bias noise level if /silent is set. -SPREDUCE2D: Print the version of "speclog". - -bin/sprobot_start: Insist that the cvs version of speclog be set up - for running this robot. -doc/www/idlspec2d_install.html: Change instructions to installing with EvilUPS. -doc/www/spectro_home.html: Include instructions to download EDR and DR1 - using wget. -etc/spChunkList.par: Add chunks 80-84, 85-86. -pro/Makefile: Add photoz directory -templates/spLRG-52928.fits: LRG templates for LRG_PHOTOZ function. - -------------------------------------------------------------------------------- -v4_10_6 (Oct 15, 2003 idlspec2d running on IDL 5.3 - 5.6 -Built against specflat v1_5, elodie v1_0, idlutils v4_10_6. - -ATMDISP_COR -DJS_SFIT_ITER -DR1_SPPLATE_COR (removed) -FLUXCALIB_RESID (removed) -FLUX_STANDARD (removed) -FRAME_FLUX_CALIB -FRAME_FLUX_TWEAK -KURUCZ_RESTORE -NEWSPCOMBINE, -PCA_FLUX_STANDARD -PLATE_FLUX_RECAL (removed) -RECTIFY -SMEAR_COMPARE -SPCOADD_FLUXED_FRAMES -SPDATA2MODEL_RATIO -SPHOTO_CALIB -SPMEDIAN_REBIN -SPPLATE_CORRECT (removed) -SPREAD_FRAMES -STYPE_STANDARD: More development by Christy. -UNDO_SMEAR (removed) -ZEROPT_COR (removed) - -APOPLOTGEOM: This is a plotting routine to visualize the geometry of the - arc lines on the CCD from the Son-of-Spectro outputs. It was - written in Nov 2002, but never checked in. -COMPUTECHI2: Do the chi^2 computation in double-precision. -DAYTIME_TEST: Proc to look for spectro exposures taken during the day - and not marked as test. -DR1LIST_BEST: Proc to determine which plates *should* have been in DR1 - that match the tiles of those actually chosen. -FILTER_THRU: Make silent in call to xy2traceset. -FIND_UNPLUGGED: Proc to find all unplugged fibers that still seemed - to get a spectrum in the reduced data. -FITSN: Changed call to the obsolete function POLYFITW, to the newer - but worse POLY_FIT. - Apply a work-around to the apparent change in PHOTO fiber-mags, - as discussed in sdss-obs/4029 and PR #5642. Shift the blue - and red fit magnitudes from: - [18.20,19.70] -> [18.33,19.83] in blue - [17.90,19.40] -> [18.06,19.56] in red -LISTEXPTIME: New simple proc for listing all exposure times. -PCA_SOLVE: Added /quiet keyword. -PHOTO_LOADER: New proc to write PHOTO outputs from SDSS_READOBJ() to an - ASCII stream for direct input into Postgres; written by Hogg - and Schlegel. -PLATECEN_TEST: Proc to check the consistency of the RA,DEC positions in - the output spPlate file headers. -PLATESN: New header keywords ROFFSET & RSIGMA store the offset and standard - deviation of the r-band difference in the spectro and photo mags. - GIOFF and GISIGMA store the offset and scatter in the (g-i) color. - Changed (g-i) to (g-r). -PLOTSN: Fixed histogram scales, changed colors for histograms. - Changed (g-i) to (g-r). - Tremonti has added return values for ROFFSET,RSIGMA,GROFFSET,GRSIGMA, - but these are not documented. Also, ignore objects with PLUG.MAG[2] - <= 0 (but what would those be?). Plotting ranges changed. - A 3rd set of plots for histograms. - (Revert this code back to v4_10_1 if need be.) -PLUGFILE_UNMAPPED: Proc to list which fibers in a plPlugMapM file are unmapped, - and the info that might help decide where those fibers should be. -QUICKEXTRACT: Apply a work-around to the apparent change in PHOTO fiber-mags, - as discussed in sdss-obs/4029 and PR #5642. Shift the mags at - which to evaluate the S/N from: - mag=20.20 -> 20.33 in blue - mag=19.90 -> 20.06 in red -SDSSPROC: Compute the standard deviation in the bias region again, - but using a weaker 5-sigma clipping. This is done solely - for the purpose of identifying any electronics problems, - such as that on the left half of r2 on MJD 51779. - Trigger a warning message if above 10 ADU. - This addresses PR #3700. -SMEARTIMES: Proc to list the TAI timestamps of 'smear' exposures in a log file. - The purpose of this is to get an exact list of times and determine - from the TPM logs whether the telescope moved in the way that it - was supposed to during these 'smear' exposures. -SPREDUCE1D: Add the "UNPLUGGED" value to the ZWARNING flag. - Add the fields SPECTROFLUX,SPECTROSYNTHFLUX,SPECTROSKYFLUX - which are the same as the COUNTS* fields but in nanomaggies - instead of maggies, which are the same units as our imaging outputs. -UPDATE_PLATELIST: Proc for easily updating the spPlateList.par file. - -doc/spectroSOS.html: Add the warning message about "Std. dev. in bias region". -etc/atmdisp_vec_am1.1.see2.0.fit - Tremonti's atmos. dispersion corrrection? -etc/atmdisp_vec_am1.2.see2.0.fit - Tremonti's atmos. dispersion corrrection? -etc/atmdisp_vec_am1.3.see2.0.fit - Tremonti's atmos. dispersion corrrection? -etc/atmdisp_vec_am1.4.see2.0.fit - Tremonti's atmos. dispersion corrrection? -etc/atmdisp_vec_am1.5.see2.0.fit - Tremonti's atmos. dispersion corrrection? -etc/emlines.par: The [O_II] lines were updated from Peter van Hoof's web - site at: http://www.pa.uky.edu/~peter/atomic/ -etc/kurucz_stds_v5.fit: Made with new SPECTRUM code. Use white dwarf - correction to colors. -etc/kurucz_model_fix.fit (removed) -etc/kurucz_stds_interp.fit (removed) -etc/spChunkList.par: Add chunks 28A, 71, 68A. Chunk 62 should be "special", - not "main". Add chunks 77,78,79. -etc/spPlateList.par: Update from 20 Nov 2002 to 11 Sep 2003. -pro/Makefile: Add the plan subdirectory. -ups/idlspec2d.table: Get rid of all the $UPS_PROD_FLAVOR references for - dependent products. - -------------------------------------------------------------------------------- -v4_10_2-2003/8/13 -This is an untested version of the code with the files in some -state between v4_10_1 and v4_10_6. It was created retro-actively -by RHL on 18 March 2004 to tag the code as is stood on 13 Aug 2003. - -------------------------------------------------------------------------------- -v4_10_1 (Jun 18, 2003 idlspec2d running on IDL 5.3 - 5.5 -Built against specflat v1_5, elodie v1_0, idlutils v4_10_1. - -Builds spAll summary files using Princeton calibObj files. - -DJS_SVDFIT: This proc removed, since it was a temporary fix to - be called for solving a singular matrix from FIND_NMINIMA. -FIND_NMINIMA: Remove use of DJS_SVDFIT (which didn't work on the Mac - anyway). Re-write the fitting to exactly three points - with an explicit solution of the quadratic equation. - Add another error code (-7L) for the case where all of the Y - values are identical, in which case the fit is impossible. -FINDSPEC: Rationalize the calling sequence to be more consistent - with SDSS_FINDIMAGE. Get rid of /DUPLICATE keyword, and - add /BEST keyword. Allow matching multiple objects per plate - if the search radius is large. -FITMEANX: Add MINSDEV and INMASK optional keywords. Both these are - used by LOCATESKYLINES to prevent the S/N of some lines getting - infinitely large. This happened for the r1+r2 exposures on - plate 336/51999, where the first sky line was off the edge - of the plate for most of the lines. -GET_TAI: Add /SILENT option. -LOCATESKYLINES: Change MAXSHIFT from 2.0 pixels to 3.5 pixels. The worst- - case real arc->science flexure that we've seen is 3.2 pix - for the r2 camera on plate 192 MJD 51461. - Use the INMASK keyword in the call of FITMEANX(). -PLATELIST: Use PLUG_RA,PLUG_DEC to match duplicate observations of - individual objects, rather than RA,DEC (which can be zeros - if there was no match to the photometry). -PLATEMERGE: Small changes to allow using the (new) calibObj files - instead of the (old) tsObj files. - For the ASCII output, write MODELFLUX instead of COUNTS_MODEL. - In call to PLUG2TSOBJ, pass PLUGMAP so that warnings of unmatched - sky fibers are not printed. - Add SPECOBJ_ID and NSPECOBS to the output structure. -PLUG2TSOBJ: Small changes to allow reading of the (new) calibObj files. - For now, assume these files are in $SPECTRO_DATA/calibobj. - Print a warning message for non-matched objects only if they - are not skies. -READSPEC: Add internal function RSPEC_ZLINE_APPEND() for appending two - zline structures, even if they are different structures or - have different number of lines measured. -SDSSPROC: Call the new proc WARN_DAYTIME. -SPCALIB: The correct TSET structure was not always being written - to the spFlat files (though those files aren't actually - used for anything), nor the FIBERMASK array. - Write those files directly from the FLATSTRUCT,ARCSTRUCT - structures instead. Also, gzip those two output files. -SPPLAN2D: Exclude files where the plate number does not match that - in the map name. This happens, for ex, for exp #14970 on - MJD 52405, which has the MAPNAME for plate 802 but a PLATEID - of 9999. -SPREDUCE: Major change to how the flat+arc calibrations are chosen. - Now choose the flat that is good and nearest in time to - the midpoint of all the science exposures. Select the arc - that was used for that flat (regardless of whether it was - the best arc). Previouly, we chose the flat that was nearest - in time to each exposure, and we chose the best arc regardless - of the time it was taken. This had the potential to choose - an arc taken during the afternoon (as happened for plate 820 - on MJD 52405, c.f., PR #3966) if it is not explicitly marked - as a test exposure. -WARN_DAYTIME: New proc to print a warning if an excellent-quality exposure - taken with the Sun up. For example, this is how the bad arc - exposure got in to ruin plate 820/52405 (PR #3966). - -bin/speclog_update: New bash script to update the "speclog" product for one - or more MJDs from the astrolog directory. -bin/sprobot_start: Put the environment variable $CVS_RSH into the cron file, - for use by the "speclog_update" script. - No longer set $SPECLOG_DIR=$ASTROLOG_DIR, since now they - really are different. -bin/sprobot.sh: After copying the astrolog files, call the "speclog_update" - script to make a CVS-versioned copy from $ASTROLOG_DIR->$SPECLOG_DIR. -doc/www/spectroSOS.html: Add warning message for "Sun above horizon." -examples/opHdrFix.par: Go through all the headers looking for flat, arc, - or science exposures taken while the sun was up. Discard them - in this file, which takes care of all corrections through - MJD 52426 (1-Jun-03), after which the sdHdrFix-$MJD.par files - are used. -ups/idlspec2d.table: Add dependency on "speclog" product. -ups/Makefile - Get rid of the confusing dependency on - $(SDSSTOOLS_DIR)/bin/install_ups_table. - -------------------------------------------------------------------------------- -v4_10_0 (Apr 8, 2003 idlspec2d running on IDL 5.3 - 5.5 -Built against specflat v1_5, elodie v1_0, idlutils v4_10_0. - -Test against new idlutils. - -APOHEADER: Allow an optional keyword MJD (to speed things up). -CREATE_LINESTRUCT,LINEBACKFIT,SPREDUCE1D: Replace LINENPIX with the number - of pixels to the left and right of the line center, LINENPIXLEFT - and LINENPIXRIGHT. Use both of those to trigger the warning - NEGATIVE_EMISSION from quasars, which was triggered sometimes - when there were only pixels to one side of the line center - (PR #3797). -FLUX_STANDARD: Added empirical correction to SPECTRUM-Kurucz models. - Then got rid of them! -GET_TAI: Trigger a Warning if (TAI-END) - (TAI-BEG) < EXPTIME + 20 sec. -PLATECOMPARE: Also plot any objects that have ZWARNING set to zero - in one observation and non-zero in another. -PLATELINKS: New proc for generating symbolic links to either all of the - interesting files in the $SPECTRO_DATA tree, or one of the - public subsets (e.g., "EDR" or "DR1"). -PLATEMERGE: Create spAllLine files. Allow PUBLIC to be a keyword specifying - any of the public subsets (e.g., "EDR" or "DR1"). -PLOTSPEC: Fix trivial bug (had "EXTRA" instead of "_EXTRA") in passing - additional plotting keywords to plots. Also, pass those - keywords along to the XYOUTS commands. -PLOTSN: The last plot was accidentally showing i-band residuals instead - of r-band. -SPHDRFIX: Test if the opHdrFix file (or the directory!) even exist at all. - Otherwise, if the directory does not exist, we'll get stuck in - an infinite loop. -READSPEC: Fix minor bug that would truncate the wavelength mapping - when spectra are read from multiple plates and /ALIGN not used. -SPPLATE_CORRECT: Fixed small bug so that it works with the smear left in. -SPREDUCE1D: In setting the NEGATIVE_EMISSION flag for QSOs, demand that - the line be negative by at least 3-sigma. -STYPE_STANDARD: Fixed error in smoothing box size. -ZEROPT_COR: Added range checks to get rid of seg faults. - -bin/sprobot.sh: Instead of looking for "/data/spectro/$MJD" to get - an MJD list, look for "/data/spectro/astrolog/$MJD". - This will get the log files for MJDs where there are no - spectroscopic data. -bin/sprobotlist.sh: Add calls to PLATEMERGE for the EDR and DR1 subsets. -etc/kurucz_model_fix.fit - New file for empirical corrections to - SPECTRUM-Kurucz models, used by FLUX_STANDARD. -examples/opHdrFix.par: More updates, in an attempt to identify - all exposures with wrong exposure times in their headers - due to various SOP and DA bugs. - Flag a bunch of old data as bad, where I've specifically searched - for short science exposures. - Change all FLAVOR='unknown' entries to QUALITY='bad'. -pro/fluxfix/Makefile - Forget to add this Makefile, which is needed - to prevent the "make install" from crashing on an IRIX platform. -ups/idlspec2d.table: Change flavor to "ANY". - -In idlutils: - -DUST_SDSSFILTER: New proc to integrate the extinction curve over - a selection of representative source function and the SDSS filters. -DUST_INTFILTER: New proc to integrate the extinction curve over a - given source function + filter curve. -GSC_READ_TABLE: New proc to read one table from the Guide Star Catalog. -HOGG_SCATTERPLOT: New proc to plot greyscale scatterplot with contours. -MWRFITS_CHUNKS - Do not always create a new file (unless /CREATE is set). -NONNEG_MULT_UPDATE,NONNEG_MULT_UPDATE_SOLVE: New code from M. Blanton - for non-negative least-squares solving. This is a special - case of box constraints in quadratic programming, as suggested - by Sha, Saul, & Lee (2002), "Multiplicative updates for - nonnegative quadratic programming in support vector machines" - (UPenn Tech Report MS-CIS-02-19) -STRUCT_PRINT: If /NO_HEAD is set, do not increase the width of a column - to occomodate the column name. New FORMATCODES option. -STRUCT_TRIMTAGS: Would not have worked if only the very 1st tag were selected. -YANNY_WRITE: New options /ALIGN and FORMATCODES, even though these options - make it possible to write invalid Yanny files. - -Various astrometric routines have been moved from the hoggpt product -into pro/astrom. The USNO routines have been modified for reading -the USNO-B1.0 catalog (as well as USNO-A1,USNO-A2). -LL2UV -USNO_CONE -USNO_READ -USNO_READZONE -USNOB10_EXTRACT -UV2LL - -Various "mangle" utilities added by M. Blanton in pro/mangle. - -evilinstall: Moved to the photoop product. -evilmake: Turn on optimization for all platforms except IRIX. - Previously, we tried to set -O3 on IRIX, which crashes - builds on the machine sdsshost.apo. -src/spheregroup/chunks.c - Minor bug fixed. -ups/idlutils.table: Change flavor to "ANY". - -------------------------------------------------------------------------------- -v4_9_13 (Nov 24, 2002 idlspec2d+idlutils) running on IDL 5.3 - 5.5 -Built against specflat v1_4. - -Support for Mac OS X platform. New features + minor fixes for SOS. -First check-in of Tremonti's flux-calibration code, though not integrated. -Check-in of dust map code. Re-write of YANNY_READ. - -APO_LOG2HTML: Default link to GIF plot, with another link to PostScript. -APOPLOTARC: New proc for plotting arc spectra from the Son-of-Spectro outputs. -APOPLOTSKY: New proc for plotting sky spectra from the Son-of-Spectro outputs. -APOREDUCE: Convert the PostScript plot to GIF format using "pstogif", - which is in idlutils. -EXTRACT_ROW: Be sure to pass "whoppingct" as type LONG to the C code. - If not, then it fails on the Mac OS X platform. -FIND_NMINIMA: Fix tiny bug for optional plots. -FINDSPEC: Add the optional keyword SEARCHRAD. -GUIDERMOVIE: The format for the date in the header of the guider images - has changed. This code should now comptabably read the old - or new date format. -HELIOCENTRIC: Moved to idlutils product. -LINEBACKFIT: Change how the error on equiv. width (LINEEW_ERR) is computed. - I believe it was incorrect before, and should simply be - computed from LINEAREA_ERR,LINECONTLEVEL_ERR. -PLATELIST: Replace the plate center RA,DEC with that of the object - closest to the center for those cases in which the spPlate - headers got the wrong coordinates (see PR #2419). When - these are replaced, print a WARNING statement. - Be more silent when calling MRDFITS(),HEADFITS(). -PLATEMERGE: Search for overlaps in plates with a somewhat larger radius, - just to be somewhat immune to errors in listed plate centers - (for ex, see PR #2419). Also, print some more informational - messages. -PLOTSPEC: Bug fix to allow one to specify the output file name with PSFILE=. - Fix units on Y axis to /cm^2 instead of /cm. -QUICKEXTRACT: Change print statement for whopping fibers to one per line. - Write SSET,RELCHI2STRUCT to the output "sci-" file. -READ_ELODIE: Use INTERPOL() instead of POPULATE_IMAGE for interpolating - the high-resolution Elodie spectra onto the log-wavelength binning. - I'm still uncertain as to the absolute units of the spectra. -READSPEC: Add PATH option, which can override TOPDIR. - Determine the MJD dynamically if set equal to 0. -SDSSPROC: Fix silly bug trying to set QUALITY='excellent' when undefined. -SLITHISTORY: New proc to plot history of spectro slit-head positions - based upon the (new) plSlitps files from the mapper. -SOLVEFILTER: New proc to determine the 2.5-m filters from the spectra. -SKYSUBTRACT: Optionally return SSET, which is the B-spline structure - for the supersky. This will be used by APOPLOTSKY. -SPCOADD_FRAMES: Change BUNIT header keywords in output files to have - the comment '1E-17 erg/cm^2/s/Ang' instead of the incorrect - comment '10E-17 erg/cm/s/Ang'. -WRITE_UROS: Add Flatvec,Rnois to output files. - -New procs from Tremonti in pro/fluxfix: - -DR1_SPPLATE_COR: Modify a list of spPlate files for improved spectrophotometry. -FLUX_STANDARD: Solve the flux-calib vectors for a single camera+spectrograph. -PLATE_FLUX_RECAL: Called by SPPLATE_CORRECT. -SPPLATE_CORRECT: Correct the flux calibration of the spPlate files for DR1. -STYPE_STANDARD: Spectral type the standard stars and save the result as - a FITS file. (This should probably be changed to return the result). -UNDO_SMEAR: Create correction vectors for 1 plate which can be used to - remove the the effects of "smear" from the spectra. -ZEROPT_COR: Compute a zeropoint correction to the flux calibration. - -aporsync_alllog.sh: New shell script for copying all of the log files - from sdsshost.apo:/astrolog/$MJD to the sos.apo machine. - This will be done once per day at 7:45. This rsync must - follow symbolic links on sdsshost.apo, since it now looks - like quite a few of the files there are now symbolic links. - (That's crazy, but what's a guy to do?) Copy the last 7 nights - of logs, whether or not there was spectroscopy. -aporsync_logs.sh: The last line of the first rsync command was accidentally - dropped in this last version - fix. Also, only copy sdHdrFix file - if it exists. This addresses PR #3968. - Also copy plSlitpos*.par,plPlugMap*.log files. -export.h: Update this file to agree with the IDL 5.5beta for Mac OS X. -Makefile: Support for new "fluxfix" subdirectory. -opHdrFix.par: Fix the RA,DEC positions in the raw file headers to fix - some remaining problems for PR #3009. Specifically, the following - plates were getting the wrong RA,DEC in the platelist files: - 351/51780, 355/51788, 356/51779, 360/51780, 385/51783, 391/51782, - 395/51783. - Discard exposures 14967-8 for plate 820/52404. -sos_start,sprobot_start: If the environment variable PATH has more than 1020 - characters than abort. This is because the PATH appears to be - truncated beyond that point on certain Linux systems. - Run the mail program at 8:10 in the morning, rather than 7:20. - This addresses PR #4194. -spEigenElodie.fits: Recreate with fixed READ_ELODIE(). New first eigenspectrum - now looks like a normal stellar spectrum. -sprobot_start: Fix tiny typo (extra quotation on line 141). - Start the main job at 10:30am instead of 10:00am. -sprobot.sh: Change the name of the astrolog directory on sos.apo.nmsu.edu - from /astrolog -> /data/spectro/astrolog (they changed it!) -sprobot.sh,sprobot1d.sh,sprobot2d.sh,sprobotlist.sh: - Output the environment variables $IDLSPEC2D_DIR,$IDLUTILS_DIR. -spChunkList.par: Added chunks 48-55. -spPlateList.par: Change yet again to agree with the latest posting from JEG - of 291 plates. About a dozen have changed since the Annis list - dated 13 Oct 02. - -In idlutils: - -BSPLINE_BKPTS: Integer roll-over bug fixed by DPF for more than 2^30 - data points (doesn't matter for SDSS pipelines, but it - does for DEIMOS). -DFPSPLOT: Added optional XSIZE keyword. -DJS_BATCH: Add optional keyword SELECTHOST. This will be used - for the HoggPT processing of 2.5-m data. - The inputs LOCALFILE,OUTFILE are now optional. -DJS_LOCKFILE: Add /append option. -DJS_LOCKFILE,DJS_UNLOCKFILE: These routines had a small chance that - more than one process could lock a file at once. This - small chance is now eliminated on Unix platforms running - IDL 5.4 or later by building a symbolic link as the lock file. -DJS_SFIT: New proc for surface fitting to tabulated data with errors. -EXT_CCM,EXT_ODONNELL: Move these old dust-related routines here since - they are general-purpose. They were not previously in any - repository. -FITS_WAIT: Allow DELTAT,TMAX to be zero. -HELIOCENTRIC: Moved from idlspec2d product, since this is of general use. -HOGG_ITERLINFIT: General-purpose sigma-clipping linear fitting routine - copied over from the hoggpt product. -HOGG_STRSPLIT: New support proc for YANNY_READ. -LOOKFORGZIP: Also look for ".Z" file if others not found. -POLY_ITER: Change "LT" logic to "LE" to handle case where residual - is 0 (e.g., a perfect fit) -POPULATE_IMAGE: Fix possible crash condition under Mac OS X when IMAGE - is 1-dimensional (and NY was not type LONG in the call to C-code). -RADEC_GREATCIRLCE: New proc to solve for the great circle for a set - of RA,DEC positions at a set of times. -SPLOT: Bug fix in the autoscale code (splot_autoscale_x) that crashed - on Mac OS X. -YANNY_READ: Make use of Hogg's procedure HOGG_STRSPLIT, which should be more - robust for parsing strings and speed things up. Not yet thoroughly - tested. - Pre-allocate a large array of length MAXLEN for each structure. - This should save a huge amount of time when reading large files, - since we used to concatenate structures with a=[a,b] every time - a new line was read. - Add support for compressed files, with file names ending with - either '.gz' or '.Z'. This is somewhat complicated by the fact - that we need to start by getting the number of lines in the file. - Return an errcode of 0 in the case of no file, which conforms - to the behaviour of previous versions of this code. -YANNY_READONE: New proc; wrapper for YANNY_READ. -YANNY_WRITE: Default to having the variable names in the typedef's in - lower-case (not that it matters). - -The pro/djsphot directory has been moved from the hoggpt product. -This includes: - -DJS_NEFF -DJS_PHOT -DJS_PHOTCEN -DJS_PHOTFRAC -DJS_PHOTSKY -FIND_SIMPLE -QUICK_PHOTFRAC -SQUAREPHOT - -These SFD dust routines have been moved here: -BH_RDFORT -DJS_PLANCK -DUST_GETVAL - And modified to read I12, I25, I60 maps also. -FAC_TEMP2FLUX -FAC_FLUX2TEMP -PLANCKCORR -PREDICT_SYNC -PREDICT_THERMAL -SETLOG -WCS_COORD2PIX -WCS_GETVAL - -Various astrometric routines have been moved from the hoggpt product -into pro/astrom: -ANGLE_FROM_PAIRS -ASTROM_ENGINE -ASTROM_TWEAK -OFFSET_FROM_PAIRS - -evilmake: Add "Darwin" (Mac OS X) as a possible platform. - Export F77=g77 for the platform. - This doesn't work yet for Blanton's spheregroup code or Fortran. -export.h: Update this file to agree with the IDL 5.5beta for Mac OS X. - (There are actually 5 copies of this file in this product!) -sprobot.sh: Comment-out the code that exits if there is no new data. - This way, we still launch BATCH2D,BATCH1D even if no new data - has been copied over from Apache Point. - -------------------------------------------------------------------------------- -v4_9_12 (Jun 26, 2002 idlspec2d+idlutils) running on IDL 5.3 - 5.5 -Built against specflat v1_4. - -Minor enhancements for SOS. For P-1D, now able to generate larger summary -files and improvements to the web site. - -APO_LOG2HTML: Change link from SOS web pages to new location at - "http://sdsshost.apo.nmsu.edu/idlspec2d/spectroSOS.html". - For the total S/N^2 on a plate, only include exposures - where quality='excellent', as per PR #3898. -CATPLOT: New proc. Modified version of SPLOT by Tremonti for inspecting spectra. -COLLIMATE: Clarify the output text to say "RMS across CCD", rather - than implying this is a measurement error (with "+/-"). -LINEBACKFIT: Fix typo for setting default value for SIGGUESS. -MAKE_SKYINVVAR: New proc. First attempt to reconstruct the sky background - with a v4 spPlate file. -PLATEHIST: Select "good" plates based upon "platequality" field - from PLATELIST. -PLATELIST: Use manually-assigned platequality from "spPlateList.par" - file when it exists. - Add fields N_TARGET_MAIN,N_TARGET_LRG,N_TARGET_QSO. - Add fields SUCCESS_MAIN,SUCCESS_LRG,SUCCESS_QSO. - Use STRUCT_PRINT to print all ASCII and HTML files. - Use APO_CHECKLIMITS() to check limits of tabulated values - and color-code them in the HTML files. -PLATEMERGE: Use MWRFITS_CHUNKS to allow us to write bigger files - (approximately up to your memory size). - Add PROGNAME,SPECPRIMARY to ASCII output. - If SUBCLASS contains several words, then use a plus sign between - the words in the ASCII file rather than a space (so that awk will - work). -PLOTSPEC: Added option to use Blanton + Hogg's java viewer (Blanton). -PLUG2TSOBJ: Slight change to find Blanton's files which are now named - tsObjPlate-*.fits rather than tsObj-*.fit. -SDSSPROC: If MJD <= 51813, then set QUALITY=excellent unless this is - over-written by SPHDRFIX. This is because for the early data, - this keyword was arbitrarily set to 'unknown', 'bad', - 'acceptable', or 'excellent'. Note that because of this, - early data will not have their plan files built properly for - versions v4_9_10 or v4_9_11. -WRITE_UROS: New proc to generate ASCII files of individual exposures - on spectra for Uros Seljak -ZPLOT: Change "brg" names to "lrg". - -aporsync_logs.sh: Add an rsync command to copy the "sdHdrFix*.par" file - from the sos.apo machine to sdsshost.apo. Do *not* rsync - this file from sdsshost.apo -> sos.apo. -mailhtml.sh: Make archival copies of the SOS log file (sos.log), naming - them sos.log.1dayold, sos.log.2dayold, .... -make_room.sh: Simple script to remove old MJD data directories on sos.apo - in order to bring free space to above 5%. Not called by any - other scripts -- could be dangerous! -opLimits.par: Use the SKYPERSEC limit for either science or smear exposures. - Add limits for use by the PLATELIST procedure. -spPlateList.par: Update to data through MJD 52413 (19 May 2002). - Add PLATEQUALITY,QUALCOMMENTS to Yanny structure. - Remove the following (bad) plates from public data releases: - 266/51602 (Internal LEDs), 360/51780 (b1 CCD warmed up). -sprobotlist.sh: Call the IDL commands ZPLOT and PLATEHIST each night - from the Spectro-Robot. -sprobot_start: Add a line to the cron job to run at 12:59 am that - copies the sprobot.log file to sprobot.log.1dayold and - sprobot.log.2dayold. - -In idlutils: - -FLOATCOMPRESS: Handle Infs and NaNs. -MWRFITS_CHUNKS: New proc to write a FITS binary table in chunks - (used by PLATEMERGE). -REPSTR: Change loop indices to type long, which was necessary to the - call from PLATEMERGE. We should be sure that this gets - into Landsman's code version. -STRUCT_PRINT: Add options TARRAY, /NO_HEAD, /HTML, FDIDIT, DDIGIT, ALIAS. -STRUCT_TRIMTAGS: New procedure. -SXASCII_READ: Fixes for sxascii_read, I think it might actually work?! -YANNY_READ: Hacked by Christy to fix the bug to allow semi-colons - within quotes. This adds the internal function YANNY_INQUOTES(). - I suspect there may be a better way to do this using REGEXP(). - If someone ever figures that out, the version before this was 1.31 - (which appeared in v4_9_10,v4_9_11). - -evilinstall: One line change to do one more check for proper sdsstools. - -------------------------------------------------------------------------------- -v4_9_11 (May 31, 2002 idlspec2d+idlutils) running on IDL 5.3 - 5.5 -Built against specflat v1_4. - -Fix "make install" bugs and crash condition for P-1D. - -LINEBACKFIT: Fix crash condition just introduced (testing for return - status from MPFITFUN). -LOGSHEET: Include UT time, NAME, CARTID, and QUALITY in outputs. - -doc/Makefile: Remove invalid copy line. -etc/Makefile: Change copy of "*" to "*.*" so the "make install" does - not crash on IRIX+6 platforms. -pro/Makefile: Remove invalid copy line. -pro/inspect/Makefile: Added this makefile. -pro/science/qso/Makefile: Fix Makefile. -templates/Makefile: Change copy of "*" to "*.*" so the "make install" does - not crash on IRIX+6 platforms. -spectroSOS.html: More comments on the QUALITY keyword. - -In idlutils: - -MJD2DATELIST: Make MJEND optional, so that this can convert one date - into a string. - -evilinstall: Use "awk" instead of "head -1c" so that this script will - run on an IRIX+6 platform. -bin/Makefile: For the "make install", explicitly copy file names - rather than just "*". Using the wildcard tries to copy - the CVS directory on an IRIX+6 platform, which then - crashes the install. -goddard/text/Makefile: We were missing this Makefile. -src/math/Makefile: Comment-out the build of the code that Hogg - just added but doesn't build: idl_nnls,nnls,diff,g1,g2,h12. - -------------------------------------------------------------------------------- -v4_9_10 (May 28, 2002 idlspec2d+idlutils) running on IDL 5.3 - 5.5 -Built against specflat v1_4. - -New version for Son-of-Spectro with more development work. - -APO_CHECKLIMITS,opLimits.par: Allow text limits in the opLimits.par file, - and allow the use of wildcards for flavor,field,camera. -APOFIX: Use QUALITY header keyword to declare exposures bad (as specified - by PR #3687). Enclose any string in single-quotes, so that - the SPHDRFIX routine can desriminate between strings and numbers. - Add lots of explicit checks for valid values for each keyword, - and only allow the modification of a small list of keywords. - Also add /test option. If the QUALITY card is edited, then - go through the logfile (logsheet-$MJD.fits) and explicitly - change the QUALITY fields in those FITS tables (but don't - regenerate the HTML table?). -APOHEADER: Use SDSSHEAD() instead of HEADFITS() to read the headers, such - that this printout includes any observer edits to the headers. - Also fix so that this proc works if both a ".fit" and ".fit.gz" - file exist on disk. - Also list the QUALITY keyword. -APOREDUCE: Add the following to the output structure: QUALITY,NAME,OBSCOMM - RADEG,DECDEG,AIRMASS. -APOREDUCE,QUICKBIAS,QUICKEXTRACT,QUICKTRACE,QUICKWAVE,REMOVE2REDO: Make - use of /do_lock keyword when calling SDSSHEAD() or SDSSPROC. -APO_LOG2HTML: Add QUALITY column to output table. -GET_MJD_DIR: Comment-out test that MJD directories contain files, since - that's so bloody slow. -GUIDERMOVIE,GUIDERMPEG: New procs for looking at the spectroscopic guider - images for a night. -INSPECFILES,INSPECTGEN: New routines for generating prototype files - for manual inspection. -LOGSHEET: Do not include both uncompressed and compressed files in a listing. - Remove the INDIR option, and add MJD and CAMERA keywords. -PLATEHIST: Use call to MJD2DATELIST. -PLATEMERGE: Include logic for setting the SPECPRIMARY field in the outputs. - Speed this up by pre-generating the exact output structure, - and use structure assignments rather than copy_struct_inx. - Exclude some tags from the output structure, spec. ROSAT*,FIRST*. -PLOTSPEC: Some major re-organizing of the main routine to allow PLATE,MJD - to be passed as vectors. Add optional ZHDR return value. - Add /RESTFRAME option. -QUICKEXTRACT: Add warning messages for "Median sky-residual chi2" - and "Max sky-residual chi2". -QUICKTRACE,QUICKWAVE: Do not write the tsetfile or wsetfile if the - quality of the exposure in not 'excellent' (and remove any - such file for that exposure if it already exists). -READSPEC: Fix the re-sorting of the ZLINE output structure, which is - complicated by being a multi-dimensional structure. -REMOVE2REDO: Only allow this proc to run on a machine named "sos", - unless /not_sos is set. This is the same logic used by APOFIX. -SDSSPROC: If the OBSCOMM keyword contains the words "dithered" or "focus", - then assume this is test data (set QUALITY='test'). -SDSSHEAD,SDSSPROC,SPHDRFIX: Add /do_lock option to lock the sdHdrFix files when - running Son-of-Spectro, which can dynamically alter those files. -SPPLAN2D: Exclude all files where the QUALITY header keyword is not 'excellent' - (as per change requested by PR 3687). -SPREDUCE1D: The output FRACNSIGLO was being computed completely wrong -- Fixed. -ZFIND: Bug fix for when NPOLY=0. - -aporsync_logs.sh: Include the "sdHdrFix*.par" files as the first ones - to copy over from sdsshost.apo -> sos.apo. -data_rsync.sh,killdata.sh: Two minor fixes: (1) gives proper syntax to - remove files in spectrolog directories, and (2) excludes - uncompressed guider files from being rsync-ed to rawcopy. -idlspec2d.table: Add optional dependency on "spinspect" product. -pullspectra: New bash script for pulling SDSS spectra from Princeton - to other institutions. -opHdrFix.par: Update through MJD 52413 (May 2002 dark run). -opLimits.par: Change exposure time limits to [599,1804] sec, in response - to PR #3653. -spectro_apodisplay.html: Include more links to spectro.princeton web site. -spectro_inspect.html: New file for instructions on manual inspection of spectra. -spectroSOS.html: Include more links to spectro.princeton web site. - Update to include comments regarding freeing up disk space (PR 3675). - Update to include changes to declaring exposures bad (PR 3687). - -In idlutils: - -BSPLINE_ITERFIT: Added /double to computation of INVVAR (line 107). -DJS_BATCH: Fix invalid return codes in a sub-proc. -NNLS: New non-negative least-squares fitting code from Hogg. -POPULATE_IMAGE: Do not recast if image already float (speeds up 4x) - -diff.f,g1.f,g2.f,h12.f,idl_nnls.c,nnls.f: C+Fortran code for NNLS proc. - -------------------------------------------------------------------------------- -v4_9_9 (Apr 26, 2002 idlspec2d+idlutils) running on IDL 5.3 - 5.5 -Built against specflat v1_4. - -New version for Son-of-Spectro with added functionality. - -APO_LOG2HTML: Add columns for EXPTIME and AIRTEMP to web page tables. - Comment-out lines that set FONT SIZE="+4", since that shows up - as huge on the mozilla web browser. -APOALL,APOREDUCE: Add the /no_diskcheck option. -APOHEADER: New proc for the observers' to print the subset of interesting - header keywords from the raw sdR files. -APOFIX: New proc for declaring exposures bad, or fixing other header files, - by using sdHdrFix-$MJD.par files (a new file type). -APOPLOT: Bug fix to prevent crash when specifying EXPNUM. -APOREDUCE: Add CARTID,AIRTEMP,CCDTEMP to all output structures. -BATCH1D: Add spZline file to list of output files. -EXTRACT_OBJECT: Mitigate possible crash condition if relchi2struct not set. -LINEBACKFIT: Fix crash condition if there is at least 1 data point, but - too few points to fit. When this happens, MPFITFUN() returns - with STATUS=0 and does not set PERROR, etc. This crashed - plate 230/52251. -PLATELIST: Add NGUIDE,FBADPIX,FBADPIX1,FBADPIX2 to structure. - No longer compute airmass from TAI, since TAI is now an average - time and may even correspond to the middle of the day if a plate - is observed over two nights. Trust the average airmass in the - spPlate header. Add PLATESN2,PLATEQUALITY fields. -PLATEMERGE: Limit output file to only 'good' or 'marginal' plates. - Get rid of /QSURVEY option. Add the following columns to - the output FITS file: progname, chunkname, platequality, - platesn2, specprimary (not using that last field yet). -QAPLOT_SKYSUB: Don't crash even in the case of a single sky fiber - (as for plate 218/51461). -QUICKEXTRACT: Return the relchi2struct from SKYSUBTRACT, and put SKYCHI2 - in the output structure. - For the warning message "Whopping", report the fiber numbers - as 1 to 640, rather than the 0-indexed fibers on each spectrograph. -SKYSUBTRACT: Merely clean out some commented-out code and update internal - comments. Avoid a possible crash condition in the case of - all (or most?) bad sky fibers. Version v4_9_8 would still crash - on the bad data for plate 296/51578 on exposure b2-00002076. - Insist that if we use "everyn" keyword for b-spline fits, - that it's >=1. -SPHEADERFIX: Use the new file "sdHdrFix-$MJD.par" that can be written - by the observers' at APO using APOFIX, rather than using the - same mechanism through the sdReport file (which was never used). - -opHdrFix.par: Declare exposures #14537,14571,14638,14653 bad, which were - previously declared bad in the sdReport files. But we're no - longer using the sdReport files for reductions. -opLimits.par: Impose limits on EXPTIME for bias,arc,flat,smear. - Impose limits on (S/N)^2 for smear exposures. -spChunkList.par: Add chunks 40-42. -spPlateList.par: Update to include list of DR1 plates. -spectroSOS.html: Document the new procs APOHEADER,APOFIX. -sprobotlist.sh: Get rid of "platemerge, /qsurvey" since that option - no longer exists. - -In idlutils: - -BSPLINE_FIT: The YFIT vector was not always being computed, depending upon - how this routine completed. Also, some Schlegelization of the code. -DJS_BATCH: Quit if REMOTEHOST is not set, or if any entries are null strings. - This allows us to disable the Spectro robots by commenting out - all lines in the batch2d.par,batch1d.par files. -DJS_MEDIAN: Allow ARRAY to be a vector, and pass DIMENSION=1; this is the - degenerate case where a scalar is returned. -FITSDIR,MRDFITS,MWRFITS,READFITS,SXADDPAR: Fink merged the latest bug fixes - from Goddard. -FITS_READ: This Goddard routine seems to have had a bug introduced, - whereby MESSAGE is not always set when reading a partially-written - FITS file. I *think* this is fixed, but should see what Landsman - says about this change. -FITS_WAIT: Add _EXTRA keyword for passing options to FITS_READ, such - as /HEADER_ONLY or EXTEN=. -HOGG_OPLOT_COVAR: New ellipse plotting routine. -YANNY_READ: No need to read the data in this file (only the header - and structure definitions) if PDATA is not to be returned. - Get the ABORT statement all on one line. -YANNY_WRITE: Put quotes around empty strings. Otherwise, this is not - a valid Yanny file. Bug fix for when passing STNAMES. - -build_links: Over-write existing links, since that's what I'm usually doing. -pstogif: Perl script for converting PostScript to GIF; copied from /usr/bin - on sos.apo. -export.h: Put a copy of this file in src/slatec and src/trace, though - perhaps we should really just have an /include subdirectory? - -------------------------------------------------------------------------------- -v4_9_8 (Apr 12, 2002 idlspec2d only) running on IDL 5.3 - 5.5 -Built against idlutils v4_9_7, specflat v1_4. - -Finish development work for 2D and P-1D before big reprocessing. - -EXTRACT_OBJECT: Clean up the calls to SKYSUBTRACT, and only modify - the pixelmask in the final call to SKYSUBTRACT (when we - do the 2-D sky-subtraction). -PLOTSPEC: Bug fix to label emission lines with /ZLINE. -REDMONSTER,SKYSUBTRACT: Change threshhold for flagging both the BADSKYCHI - and REDMONSTER mask bits to a relative chi^2 of 4.0, which is - what we have traditionally used (except in the last tagged version). -SKYSUBTRACT: Do not modify the input PIXELMASK, but make a copy NEWMASK - which is modified and returned. This allows several calls to - SKYSUBTRACT from EXTRACT_OBJECT, where only the last call - actually modifies this mask. -SPREDUCE1D: Identify subclasses for galaxies and quasars based upon - emission lines. Currently set the following possible subclasses: - AGN, STARFORMING, STARBURST, BROADLINE. - -sprobot_start: Fix trivial bug in setting DISPLAY environment variable. - -------------------------------------------------------------------------------- -v4_9_7 (Apr 10, 2002 idlspec2d+idlutils) running on IDL 5.3 - 5.5 -Built against specflat v1_4. - -More development work for 2D and P-1D. -Traceset routines moved from idlspec2d -> idlutils product. - -COMBINE1FIBER: Bug fix to get the NODATA bad pixel set in the ANDMASK - where-ever it is set in the ORMASK. -ELODIE_BEST: Add FITINDX keyword. Expand default fitting range to +/-1000 km/s. -EXTRACT_OBJECT: Add "OPTCHI2" keyword to output header, recording the - mean chi^2 of the optimal extraction. - Add "SKYCHI2" keyword to output header, recording the mean chi^2 - of the sky-subtraction. -MYFLUXCALIB: Put the plate+MJD+camname in the plot titles. -PIXELMASK_BITS: Now simply call the new function SDSS_FLAGVAL(). -PCA_SOLVE: Allow OBJLOGLAM to either be a vector or an array - (now passing only a vector from PCA_STAR). -PCA_STAR: Now read the Yanny file "eigeninput_star.par", and use cz. - Clean up the code, always using the same wavelength mapping - for both input and output spectra -- this allows us to do - away with the internal procedure SPAPPEND. -PLATELIST: Make consistent with changes to spChunklist.par. - Add the elements: WSIG(MA,MIN,MAX), XSIG(MA,MIN,MAX), - XCHI2(,MIN,MAX),SKYCHI2,SCHI2(MIN,MAX). -PLATESN: For the roll call, output both the number of bad fibers for - fiber masks, and the fraction of bad pixels for pixel masks. -PLOTSPEC: Added /ORMASK, /ANDMASK keywords. Plot text for the zwarning flags. - Use /normal coordinates for plot labels, so they stay on - the plot even when zoomming in and out. -QUICKEXTRACT: Do not issue warning message for spatial widths on smear exposures - since S/N can be so low. This addresses PR #3511. -READSPEC,SPEC_APPEND: Add the /ALIGN option for aligning all the spectra - in wavelength. Also, fix a bug whereby reading objects from - multiple could get mis-aligned if the OBJHDR keyword was specified - (this bug was just introduced in the last version). -REDMONSTER,SKYSUBTRACT: Lower threshhold for flagging both the BADSKYCHI - and REDMONSTER mask bits from a relative chi^2 of 4.0 to 3.0. -SDSS_FLAGNAME: Re-write to include many more types of mask bits - from the file "spMaskbits.par". -SDSS_FLAGVAL: New function, which is the inverse of SDSS_FLAGNAME(). -SKYMASK: Remove the rejection of andmask/BRIGHTSKY, and add rejection for - ormask/BADSKYCHI. Now we reject based upon that and - ormask/REDMONSTER. For good plates, this will be rejecting fewer - points. For the recent horrible data (like plate 889/52346), - many more points are rejected since the sky chi^2 are so horrible. - This change will need some real testing. -SKYSUBTRACT: Fix stupid bug where sky-fits are rejected if the mean - sky is negative, though that only happens rarely. This did - cause all the b1 exposures on plate 283/51584 to get rejected - (one of at least several such examples). - Fix another rare indexing bug triggered on plate 318/51580-r2, - and clean up the code that re-selected the sky break points - based upon the S/N as a function of wavelength. - Do not set BADSKYCHI mask bit when NOSKY is already set. - Raise the threshhold for setting the BRIGHTSKY mask bit from - 1.25 * (median sky) to 2.0. -SPCALIB: Save "medwidth" from call to FITDISPERSION in the arcstruct. - Save "medwidth" from call to FITFLATWIDTH in the flatstruct. -SPCOADD_FRAMES: Average keywords XCHI2,SKYCHI2,XSIGMA,WSIGMA in output header. - Add output keywords XCHI2MIN,XCHI2MAX,SCHI2MIN,SCHI2MAX, - XSIGMIN,XSIGMAX,WSIGMIN,WSIGMAX. - Compute the fraction of bad pixels in total, and on each - spectrograph. Bad pixels are any with SKYMASK(INVVAR)=0, - excluding those where the NODATA bit is set in the pixel mask. - Then set the header keywords FBADPIX,FBADPIX1,FBADPIX2. -SPREDUCE: Add XSIGMA,WSIGMA to output headers. -SPREDUCE1D: Use SDSS_FLAGVAL() to set the zwarning mask bits. - Make consistent with changes to line names in "emlines.par" file - when setting the NEGATIVE_EMISSION mask bit. - Call ELODIE_BEST() for stars, and append to spZbest output structure. -TELLURIC_CORR: Center the plot title. -ZTWEAK_STAR: New proc for getting the zero-point of the stellar - templates by matching to the Elodie spectra. - -eigeninput_star.par: Version of "eigeninput_star.dat" as a Yanny file. - Add more Carbon_DQ stars. Use cz instead of z. - For stellar types O through late K, change the classifications - to agree with Elodie (hopefully the fit from 4100-6800 Ang - is good enough for this), and tweak the redshifts of those - stars also (using ZTWEAK_STAR). -emlines.par: Couple the N_II emission line ratios at 1:3. - Append the wavelength to the LINENAME structure element for clarity - in the output files. -spChunkList.par: Remove verstarg from structure, and add progname. -spEigenStar-52373.fits: New stellar eigentemplates based upon - eigeninput_star.par. -spMaskbits.par: Heavily modified file to include many more types of mask bits. -sprobot_start: Set the environment variable "DISPLAY=" so that any - IDL jobs launched from these scripts don't use any additional - IDL licenses. - -In idlutils: - -EX_MAX_PLOT: Added panel features. -HOGG_USERSYM: New symbol-making routine. -SXCOMBINEPAR: Add optional _EXTRA keywords to pass to SXADDPAR. - -goddard/Makefile: Remove tape_io from the subdir list, since this causes it to - crash when building under IRIX+6. - -The following IDL routines were moved from the "idlspec2d" product: -pro/trace/fchebyshev.pro - fchebyshev_split.pro - flegendre.pro - fpoly.pro - func_fit.pro - trace_crude.pro - trace_fweight.pro - trace_gweight.pro - traceset2pix.pro - traceset2xy.pro - xy2traceset.pro -The following C code was moved from the "idlspec2d" product: -src/trace/Makefile - evilmath.c - evilmath.h - trace_crude.c - trace_fweight.c - trace_gweight.c - -------------------------------------------------------------------------------- -v4_9_6 (Mar 29, 2002 idlspec2d+idlutils) running on IDL 5.3 - 5.5 -Built against specflat v1_4. - -Son-of-Spectro procs for spectrograph focus; changes in SOS data format. -More development work for 2D and P-1D. - -APO_CHECKLIMITS: Default to returning the strings '','yellow',or 'red', - and only turn into an HTML color if /HTML is set. -APO_LOG2HTML: Add FLAT/XSIGMA and ARC/DISPSIGMA to output tables. - Remove the line of Java-script from the HTML file that - does the auto-refresh (we'll only put that line in the - file "logfile-current.html", as created by APOREDUCE). -APOALL: Major updating of this calling script to conform with current - environment variables ASTROLOG_DIR,RAWDATA_DIR,SPECTROLOG_DIR. - Also, reduce bias,dark,smear exposures. -APOREDUCE: No longer do the test in the beginning of this procedure that - the FITS file should exist, since FITS_WAIT() will do that test - and repeatably look for the file for several minutes. - Log the versions of IDL,idlspec2d,idlutils into the log file. - When copying the HTML file to "logfile-current.html", add - the line of Java-script that does the auto-refresh. -BATCH1D,BATCH2D: Set the DISPLAY environment variable on remote machines - to always be "", so that only one IDL license on each - remote machine. Also, pipe the commands through the bash shell - on the remote machine, which means the UPS environment must - be set up in the .bashrc file on the remote machine. -COLLIMATE: New proc for collimating the spectrographs. -CREATE_LINESTRUCT: Make the LINEWAVE element double-precision. -ELODIE_BEST: New function in development for finding best-fit Elodie spectrum. -EXTRACT_IMAGE: The values in XSKY are noisy measurements, so replace them with - the predicted positions based upon the arc solution when making - the call to SKYLINE_DISPERSION() -- otherwise, it can crash - because the positions may not even be properly sorted! -EXTRACT_OBJECT: Remove call to QAPLOT_SKYDEV, since that QA plot contains - no more information than QAPLOT_SKYSHIFT. -FINDOPFILE: Add /silent option. -FITARCIMAGE: List the (0-indexed) fiber numbers that in which - any or all arc centroids are fixed. -FITDISPERSION: Re-organization of this code Add support for SIGMA being - a scalar, vector, or array. Optional MEDWIDTH output. - Output warning messages if out-of-focus. -FITFLATWIDTH: Re-organization of this code Add support for SIGMA being - a scalar, vector, or array. Optional MEDWIDTH output. - Output warning messages if out-of-focus. -FOCUSHISTORY: First stab at a proc for plotting the spectrograph focus - as a function of time. Simply use the wavelength dispersions - from the arcs, which we have in the spPlate files. -LINEBACKFIT: Force the output sigmas to be non-negative (which also - prevents a bug later in the code that assumes this is true - when measuring the chi^2 of each line, as pointed out by - John Moustakes). - Remove any lines where LINENPIX=0 by setting its AREA,EW to zero - and their errors to negative error codes, and remove from YFIT. - Evaluate YFIT even at points not included in the fit (if INVVAR=0). - Add SIGGUESS option, and allow SIGGUESS and ZGUESS to be either - scalars or vectors (one per line fit). -PLATECOMPARE: Fix 1-character typo preventing selecting the MJD - when using the TOPDIR keyword also. - Pass TOPDIR to READSPEC rather than explicitly resetting - the $SPECTRO_DATA environment variable. -PLOTSPEC: Add options /ZLINE, /NOSYN, /NOERR, TOPDIR. -QAPLOT_SKYLINE: Divide the skyline flux values by the airmass, since those - lines basically scale with airmass. -QUICKEXTRACT: Make call to FITFLATWIDTH, and trigger warning messages if - the spatial widths are too large, indicating that the spectrographs - may be out-of-focus (see PR #3462). - Add XSIGMA_QUADRANT,XSIGMA to output structure. -QUICKTRACE: Call the optimal-extraction code EXTRACT_IMAGE for the sole - purpose of measuring the spatial profile widths, then use - FITFLATWIDTH to trigger any warning messages if the spectrographs - appear out of focus (PR #3462). - Add XSIGMA_QUADRANT,XSIGMA to output structure. -QUICKWAVE: Call to FITDISPERSION to trigger warnings in the spectrographs - look out-of-focus in the wavelength dimension. - Add DISPSIGMA_QUADRANT,DISPSIGMA to output structure. -READ_ELODIE: Optionally return the Elodie header in HDR. -READSPEC: Add TOPDIR input keyword and ZLINE,LINEFLUX outputs. - Use DIMS0 keyword in primary header of spZall,spZline files - if it exists to determine the dimensions of [NPER,NOBJECT] - multi-dimensional arrays. - Option to return OBJHDR. -SDSSPROC: Add /silent option. -SKYSUBTRACT: Trigger a warning if the airmass exceeds 2.5 (note this proc - is also called by SoS). -SPADD_GUIDERINFO: Fix crash condition when very little guiding info - and indexing goes negative. Add NGUIDE to header. -SPCOADD_FRAMES: Total the NGUIDE header keywords. -SPECLINEFIT: Major re-organization of this procedure, allowing one to either - pass a plate file name, or all the input arrays (as is now done - from SPREDUCE1D). Now fit to emission lines for all object - types, not just galaxies. - Pass only those lines within the wavelength range of the spectrum - (+/- 6000 km/sec) to the line-fitting engine. This saves CPU - time that would otherwise be spent determining that these lines - cannot be fit. -SPPLAN2D,SPPLAN1D: Add more informational messages. -SPPLAN2D: If $SPECLOG_DIR is not set, then use $ASTROLOG_DIR. -SPREDUCE1D: Simply overwrite any existing spZ files, rather than - using cpbackup to save old ones. - Call SPECLINEFIT from the end of the code, since it now fits - emission lines for all types of objects (not just galaxies). - Add another ZWARNING flag for QSOs with negative emission lines. -SPREDUCE2D: Output the version of the specflat product at the start of the log. -SDSSPROC: Add "WARNING" to the warning messages about invalid or - missing BC files. -SKYLINE_DISPERSION: Re-write to get the indexing correct. The first version - was not comparing the skyline widths to the correct arcline widths. - The measurement should mean something now. -SKYSUBTRACT: Some cleaning up of the code and catch crash conditions - for when all the sky fibers are bad (as happened for - exposure #10965 of plate 644/52173). For most cases, - the code should be functionally equivalent. -TRACE320CRUDE: Fix a bug just introduced in v4_9_5 that would cause - the code to crash if ystart at any trace falls off the - left or right edge of the CCD. This caused the - plate 0179/51580 (and some others) to crash. -ZFIND: Make EIGENFILE an optional input, so that STARFLUX,STARLOGLAM0 - can be specified directly instead of passing a file name. - -emlines.dat: Add Ly_alpha,N_V,C_IV,He_II,C_III] for QSOs. - Let the widths for the Balmer series be independent from - those of other emission lines, but still constrain the Balmer - lines and other lines to have the same redshift. One such - example where the Balmer lines have a different (narrower!) - width is 406/52238-362. -idlspec2d.table: Added optional dependence upon "elodie" product. -opBC-50000.par,opBC-51809.par,opBC-51813.par: Extended two bad columns - in r1 to the top of the CCD (cols 1201+1586). -opHdrFix.par: Declare data bad on MJD 52336-52337 due to spectrograph - mechanicals problems, and other problems from 52312-52337. -opLimits.par: Add error values for FLAT/XSIGMA and ARC/DISPSIGMA. -mailhtml.sh: Comment out the rm lines (as in 1.18) and leave to killdata.sh -spChunkList.par: Add chunks 36-39. -spectroSOS.html: New warning messages if the flats or science frames - appear out-of-focus according to their spatial profile widths - (will only appear in v4_9_6 and later). - New warning messages if arcs appear out-of-focus according - to their dispersion profiles. - Add FLAT/XSIGMA and ARC/DISPSIGMA to output tables. -sprobot.sh: Add more time stamps to the robot log file. - Change the ssh command that retrieves the MJD directory list - from the remote machine -- it appears that something about the - sed command no longer works the same. -sprobot_start: Comment-out "spamdup" job in the cron file, since we don't - seem to need that hack any more to keep the NSF-automounts alive. - Set $SPECLOG_DIR=$ASTROLOG_DIR for the purposes the the Spectro - robot. - -In idlutils: -ASINH2: Better asinh (from Blanton for PROFINTERP procedure). -BSPLINE_ITERFIT: Always reset OUTMASK so that the input value does not matter. - Set YFIT to all zeros in the case that the fit fails. -EX_MAX_PLOT: Moved from /math to /plot. -FAST_FIT_SERSIC: New proc. -FIT_SERSIC,PROFINTERP,SERSIC: Bug fixes, and use ASINH2. -HOGG_INTERVAL: New proc. -MAGGIES2LUPS: New proc. -MJD2DATELIST: New proc to construct a list of MJDs and date strings - spanning a range of MJDs (useful for plot limits). -PPMTOMPEG: New proc. Wrapper for ppmtompeg, the open-source UNIX mpeg writer. -RADEC_TO_FIELDS: Configurable option allfieldfile. -SEEING_RADIAL: New proc. -SERSIC_PARAMS: New proc. -SPLOG: Add /NO_STDOUT keyword. -SXPAR: Landsman fixed to handle CRVALn of different type properly -YANNY_READ: If reading a Yanny file aborts, then log the file name, - the line number of the failure, and the line itself. - Small speed improvement made in internal proc YANNY_READSTRING. -src/spheregroup/chunks.c: Roundoff error problem fixed. -eviltarball: Comment out the reference to sdsscvs. -evilinstall: Use UPS_PROD_FLAVOR environment variable (not UPS_FLAVOR). - -In specflat: - -First set of global biases (pixbiasave-*.fits) from many nights of data. -Pixel flats now based on more nights of data. -specflat.table: Added "Flavor=Linux" - -------------------------------------------------------------------------------- -v4_9_5 (Mar 1, 2002 idlspec2d+idlutils) running on IDL 5.3 - 5.5 -Built against specflat v1_3. - -Minor bug fixes for SOS. - -APO_LOG2HTML: Generate the HTML file name even if the FITS file has a path - in the name, as happens when being called from REMOVE2REDO. -BATCH2D: Include 'smear' exposures when generating list of output files. - (Not doing this caused a crash for plates w/ only smears). -FLUXCORR_NEW: Provide a better test of valid smear correction, fixes PR #2478. -QUICKTRACE: Reverted to doing full trace for flats. Time saving by only - tracing part of the flat image introduces a risk not worth the gain. -REMOVE2REDO: Fix possible crash condition, and change calling sequence to - DJS_MODFITS in the case that an HDU should become empty. - If files have been removed, then update the HTML file so that the - files to be re-reduced are removed from the web page until then. -SPCOADD_FRAMES: set BADFLUXFACTOR only for extreme smear corrections=20x median -TRACE320CRUDE: Fix bug dependent on invvar being set. - Small change to note number of goodrows in xmask, non-critical. - -aporsync_logs.sh: Re-order the files copied so that we get the sdReport - file first. -opLimits.par: Change the good limits for bias counts from [0,20] to [0,15]. -sprobot_start: No longer reference $BATCH2DFILE,$BATCH1DFILE, which are - no longer used. -sprobot_start,sprobot.sh: Added optional $SPROBOT_RSH environment variable - for specifying the protocol for copying the raw data. -sprobotcopy.sh: Deleted. -sprobotlist.sh: Trimmed version of "sprobotcopy.sh", which only creates - the plate list and summary files once per day. - -In idlutils: -DJS_MODFITS: Added /DELETE_DATA keyword, which is functionality needed - for sos_redo (the REMOVE2REDO procedure). -------------------------------------------------------------------------------- -v4_9_4 (Feb 28, 2002 idlspec2d+idlutils) running on IDL 5.3 - 5.5 -Built against specflat v1_3. - -More development of 2D. -Minor bug fixes for SOS. -Runnable version for P-1D. - -APOREDUCE: Check how full the disks are for both the input files (sdR images), - and the outputs. Issue a warning message if either is >95% full. -ATVRAWSPEC: Display a 2D spectroscopic image with bad columns marked in red. - This is useful to see that all bad columns have been masked, - especially when looking at the 2D pixel flats. -BATCH1D,BATCH2D: Look for the parameter file batch1d.par,batch2d.par - in the TOPDIR directory rather than reading environment - variables $BATCH1DFILE,$BATCH2DFILE. -CREATE_LINESTRUCT(): Function broken out of LINEBACKFIT so that it - can be called by SPECLINEFIT. -EXTRACT_OBJECT: Added call to skyline_dispersion (for increasing the - instrumental response width to agree with the width of sky - lines), but correction is not implemented because the measurement - appears to be very noisy. Check invvar with whopping fibers. -FINDOPFILE: Find the op file corresponding to a specified MJD. - This function is broken out of SDSSPROC so that it can - be used by other procs (like ATVRAWSPEC). -FITDISPERSION: Mask regions not neighboring lines of interest. - Simplifies extraction of line widths. - Minor tweaks, npoly=10 no longer valid -> changed to npoly=5. -LINEBACKFIT: Fix possible crash condition when computing LINENPIX if - the line width is unphysically narrow and no pixels fall - within the region of an emission line. -MATCH_TRACE: Added keyword radius, and set default to 2 (rather than 3). - This seems to fix PR #2626 (Large bad regions not large enough - w/LOWFLAT) and PR #2063 (Extraction problem fibers 321-400 on - Plate 401/51788), both of which were due to some traces being - slightly off. -PLATECOMPARE: New TOPDIR optional keyword for comparing different - reductions of the same data. -PLOTSPEC: Optional /ZWARNING flag. Report error if FIBERID out of range. -PLUGHISTORY: Plot the history of unplugged fibers based upon plPlugMapM files. -QAPLOT_FCALIBVEC: Generate QA plot of apparent flux-calibration errors - by comparing spectra to synthetic (e.g., best-fit) spectra. - This proc is called at the end of SPREDUCE1D, and generates - figures and warning messages if the min/max apparent fluxing - errors are large (more than 10% anywhere). -QUICKEXTRACT: Call REJECT_SCIENCE() to decide if a science exposure is bad - due to lamps being turned on, too many saturated pixels, etc. - If number of sky fibers > 20, then select one good sky fiber - from each of the 16 bundles. Assumes 20 fibers per bundle. - This addresses PR #3238. - Fix bug whereby bad pixels can trigger whopping fibers. - Bad pixels can trigger whopping fibers. -READSPEC: Don't fail when asking for SYNFLUX, but the spZ file doesn't exist. - (This prevented PLOTSPEC from plotting anything in that case.) -REJECT_SCIENCE: Check in the header that all the flat-field screens are - open and that all the FF,NE,HGCD lamps are turned off. - With this check, exposure #11504 of plate 727/52207 would - have been automatically rejected for having the FF lamps - turned on (see PR #3169). - For now, never actually reject science frames. -SDSS_FLAGNAME: Change some of the target names to agree with what's been - changed in the target code in the past year or so. Note that - I had already updated the target names in spectro_home.html. -SDSSPROC: Change to select which op file to use based upon the MJD in the - file name rather than the MJD in the header of a Yanny param file. - Make the MJD selection of the op files a bit more clear. - Issue a warning message if there is no op file with an MJD - equal to or less than the image's MJD. - The function FINDOPFILE is broken out into its own file. -SKYLINE_DISPERSION: Function similar to fitdispersion which measures - gaussian line widths of sky lines given arc line dispersion set - as initial guess. Indices of sky fibers can be specified to - restrict width measurements to these fibers only. - The measured widths and corrections are logged to the log file - with the following info: - xpos = X position on CCD - ngood = number of good measurements (<= number of sky fibers) - flux = flux of this sky line - arc = the dispersion width measured in the arc lines - min, max, median = a sample of the distribution of widths (in - pixels) for each skyline - med_diff = the median quadrature difference: width^2 - arc^2 - stddev = the scatter in the quadrature difference for that line -SPBIASAVE: New proc to average together a set (or all!) 2D pixel biases. - This proc is modelled on SPFLATAVE. -SPCOADD_FRAMES: Remove the following keywords from the spPlate file: - PIXBIAS,FLATFILE,ARCFILE,OBJFILE. - Issue a warning message if any object fibers have OBJTYPE=NA, - which should be impossible, but the special plate 673 and - possibly others had some such fibers. - Add UNAME keyword which is the result of 'uname -n'. -SPECLINEFIT: Fixed dumb mistake on indexing that could crash. - Always create an NLINE x NOBJ output structure, with blank - entries where we do not measure lines (currently, only - measuring for GALAXY objects). -SPFLATAVE: Add comments to output header to track which files were used; - change default for input directory to current. -SPFLATTEN2: Undo the last change of using the opBC-empty.par file, since - we do want to ignore bad columns when tracing fibers, etc. -SPREDUCE: No longer issue an abort message in this proc if the science - exposure is declared bad, but issue those messages from within - the REJECT_SCIENCE() function. -SPREDUCE1D: The warning message for S/N > 200 pixels has been fixed, - and another warning added for any pixels w/ flux < -10 * noise - (e.g., unphysically low values). - Mask out points that are unphysically negative (10-sigma negatives), - and mask the neighboring 2 pixels in each direction. - Call new procedure QAPLOT_FCALIBVEC to measure flux-calib errors. -TRACE320CRUDE: Add replacement code section for traces with - bad columns near starting position. - -idlspec2d.table: Added "Flavor=Linux" -eigeninput_star.dat, spEigenStar-52333.fits: Update of stellar templates. - Use many A stars, Carbon stars, Carbon_DQ stars, WD stars - instead of just 1 for building eigentemplates. - Many more K stars, with subtypes Kearly,Kmid,Klate. - No changes to F stars yet (or even checks of their velocities). -mailhtml.sh: Grep for the "APO Spectro" from the HTML file to get the - subject line for mail; I had changed this HTML title string, - which then gave a missing subject line in the e-mail (PR #3200). -opBC-50000.par,opBC-51809.par,opBC-51813.par: Add a small trap on r2 - at 1168,1339; extend the b1 badcols 1762-1673 to the top of - the CCD by 10 more rows; extend the r2 badcol 1585 to the top - of the CCD by 908 more rows. Re-order some of the entries - for clarity. Make opBC-51809 identical to the others except - for the 4 additional entries on b1 and 1 additional on r1. -opConfig-50000.par,opECalib-50000.par: Renamed copies of opConfig.par - and opECalib.par with an MJD in their names. -opHdrFix.par: Declare bad aborted smears for plate 601 and 803. - Declare bad exposure on plate 866. - Don't include test data from MJD 52318. - Declare all r2 exposures on MJD 51779 bad due to electronics - problems (PR #2335), only effecting plate 356. - Remove bad data for plate 727 on MJD 52207. - Remove bad data for plate 724 on MJD 52240. - Remove bad smear for plate 860 on MJD 52319. -spectroSOS.html: Update to reflect new warning messages. -sprobot.sh: Bug fix where we were rsync-ing all of sos.apo:/data/spectro/$MJD - on the line where we only wanted to get the guider directory! - -In idlutils: -New procedures in pro/coord for various SDSS coordinate transformations. -Many of these were in the hoggpt product: - ATBOUND - ATBOUND2 - ETA_TO_STRIPE - ETALAMBDA_TO_STRIPE - ETALAMBDA_TO_MUNU - MUNU_TO_RADEC - RADEC_TO_FIELDS - RADEC_TO_MUNU - STRIPE_TO_ETA - STRIPE_TO_INCL -ATV: Add "R" and "C" features for overplotting rows and columns. -COMDIS: Minor bug fix, making loop variable type long. -GAUSS_KERNEL: sigma should have been sigma^2 (This proc is used - by SKYSUBTRACT,SMOOTH_HALO,RLINE_FINDEW in idlspec2d). -SPHEREMATCH,chunks.c: Bug fix for wrap. -YANNY_DIFF: New proc to compare two Yanny files, optionally copying values - from one to another. -YANNY_READ: Read lines up to 2047 characters long (instead of 1023) - without resorting to the /QUICK option. -YANNY_WRITE: Add STNAMES keyword if writing out Yanny files that were - read with YANNY_READ(/ANONYMOUS). -evilinstall: Now with all the messy confusion at Fermi, it is no longer - possible to auto-detect Linux flavors using the $SETUP_UPS - environment. It is best to first set an environment variable, ie.: - UPS_FLAVOR=Linux+2 - to over-ride the auto-detected flavor on Linux boxes, which now - always appears to come up as Linux+2.4, which no one actually uses. - -idlutils.table: Added "Flavor=Linux" - -------------------------------------------------------------------------------- -v4_9_3 (Feb 12, 2002 idlspec2d only) running on IDL 5.3 - 5.5 -Built against idlutils v4_9_2, specflat v1_3. - -Trivial fix to Makefiles for install. -Still a buggy version for P-1D. - -------------------------------------------------------------------------------- -v4_9_2 (Feb 12, 2002 idlspec2d+idlutils) running on IDL 5.3 - 5.5 -Built against specflat v1_3. - -This tagged version is for running Spectro-2D. -It also addresses some rsync bugs for Son-of-Spectro. -Re-write of velocity dispersion + line-fitting for P-1D, but a buggy version. - -APO_CHECKLIMITS: Wasn't matching on flavor. Fixed. -APO_LOG2HTML: Mistakenly was listing times as hr:sec instead of hr:min! - Remove the first word from each warning or abort message line - (which is the name of the IDL proc that generated the message, - and confuses the observers; we sill have the proc name in the - original FITS file from which the HTML is generated). -APOPLOT,PLOTSPEC: Change the http address for Kent's image server to - its new location at http://sdssmosaic.fnal.gov:8015/. -APOREDUCE: Added warning if disk space is nearly full where output - directory resides (if 'df' command reports more than 95% full). - (But not nessarily testing the directory where the raw FITS - files are copied.) -DJS_SVDFIT: This is a modified version of SVDFIT from IDL 5.3, since the version - in IDL 5.4.1 crashes when the matrix is singular!! - Presently, this is called by FIND_NMINIMA(). -FIND_NMINIMA: Add /DEBUG keyword; small changes to plotting. - Call DJS_SVDFIT instead of the IDL library function SVDFIT(), - since it looks like they broke the latter in IDL 5.4.1. -FINDCHI2MIN: Clean up this code some. - This routine used to be used by VDISPFIT, but is no longer. -FLUXCORR_NEW: Return all zeros in fluxcorr image if - "WARNING: No spectrophoto with high S/N" - No longer required checking of "Bad" values for medium S/N. -GET_TAI: A short proc to determine, beginning, mid-point, and ending - timestamps for sdR exposures. Uses TAI-BEG,TAI-END in header - if they exist; otherwise compute from TAI and EXPTIME. - This is now called from the following procs: EXTRACT_OBJECT, - SDREPORTMAKE, SPADD_GUIDERINFO, SPBIASGEN, SPCALIB. - Note that version v4_9_1 would not reduce the early data since - we had changed to using TAI-BEG, which did not exist for the - early data. -LINEBACKFIT: New generic proc for fitting emission lines + linear background - terms. -PCA_ELODIE,READ_ELODIE(): New routines for reading the Elodie eschelle - spectra and generating eigenvectors for velocity dispersion fits. -REMOVE2REDO: Never consider the compressed (sdR*.fit.gz) files for removal, - since those are our archived copies of the raw data. Only remove - the *.fit files. That way, if the data gets removed from - sdsshost.apo and we run this proc, we don't delete all copies - of the data on sos.apo! -SPADD_GUIDERINFO: New routine to compute seeing and average guider offsets - from the guiderMon param file, and add those results to a FITS - header. -SPBIASGEN,SPFLATTEN2: Use an empty bad column file (opBC-empty.par) when - generating the 2D pixel flats and bias images. -SPCOADD_FRAMES: Combine a whole bunch of header cards from the individual - input exposure for the output exposure. Specifically, this was - to address PR 2174 to average the weather data. The cards effected: - 'AZ', 'ALT', 'TAI', 'WTIME', 'AIRTEMP', 'DEWPOINT', - 'DEWDEP', 'DUSTA', 'DUSTB', 'DUSTC', 'DUSTD', 'GUSTS', 'HUMIDITY', - 'HUMIDOUT', 'PRESSURE', 'WINDD', 'WINDS', 'TEMP01', 'TEMP02', - 'TEMP03', 'TEMP04', 'HELIO_RV', 'SEEING20', 'SEEING50', 'SEEING80', - 'RMSOFF20', 'RMSOFF50', 'RMSOFF80' - We also take the min of all 'TAI-BEG' cards, and the max of - all 'TAI-END' cards. - Don't alter incoming file list, copy to internal variable list - of filenames. -SPCOMBINE: Set default value of MINSN2 to 0.2 instead of 0.0. This is a - work-around for the moment to prevent one bad exposure (#6808) - from corrupting the combine of plate 358/51818. - Also, go back to not including the smear exposures in the combine, - unless one explicitly sets the /SMEARINCLUDE keyword. - Exclude fluxcorr images with entire image = 0 -- that's now a - possible result from running FLUXCORR_NEW where the signal is - so low that we can't compute the flux-correction vectors w.r.t. - the smear exposure. -SPECLINEFIT: First calling script for line-fitting code, modelled after - SPREDUCE1D. This can be stand-alone, or called at the end - of SPREDUCE1D. For the time being, only do these fits for - objects of type GALAXY. -SPREDUCE1D: Set the ZWARNING=8 flag (synthetic spectrum is negative) only - for stars, not for QSO's any more since we use several eigen- - components to fit QSO's now, nor for CV stars for the same reason. - Use the file 'spEigenElodie.fits' for computing velocity dispersions. - Write out the best-fit velocity-dispersion spectrum as HDU#3. - Make consistent with changes to VDISPFIT. - Add CHI68P to output structure, for the 68-th percentile value - of abs(chi). - Call SPECLINEFIT at the end of this proc. -TRACE320CRUDE: Threshold to set 'BADTRACE' is now 45% bad pixels, whereas - it used to be 20%. -VDISPFIT: Option to return the best fit with YFIT keyword. - Mask out +/- 6 pix from emission lines instead of only 4 pix. - Option to pass EIGENFILE,EIGENDIR rather than use default file. - Use FIND_NMINIMA instead of FINCHI2MIN to find the minimum, - since the latter did a spline fit that could wiggle in - unconstrained ways. - In cases of bad fits, return error codes as negative VDISP_ERR. - Return a code of -4L if there is no wavelength overlap. - Insist that convolution kernel have odd number of elements, and - convolve with /CENTER keyword. - Only cache over-sampled templates if the template file name - is unchanged between calls. - Make this routine a function that returns a structure. - Also return VDISPCHI2,VDISPNPIX,VDISPDOF in output structure. -ZFIND: Add fields to the output structure needed for velocity dispersions. - -aporsync_logs.sh : - Changed the syntax to just copy these type of files in astrolog - (which we *thought* we were doing, but wasn't quite working): - Unplugged*.ps - exposureLog*.par - fiberScan*.par - guiderMon*.par - op*.par - plPlugMap*.par - sdReport*.par -sprobot.sh: Fix syntax bug when passing $upsversion to BATCH1D,BATCH2D. -killdata.sh,mailhtml.sh: Spectrolog cleanup is now done +25 MJDs afterwards, - which will allow for sos_redo, apoplot and debugging to have - a buffer. Enable the call to data_rsync.sh again for copying - the *.gz files locally from /data/spectro/$MJD to - /data/spectro/rawcopy/$MJD. -sprobot.sh: Copy the guider image directory. We mistakenly lost this copying - when we went to explicitly copying only the sdR*.fit.gz files. -sos.sh,startapo.sh: Forget to switch to UT time in a couple of instances. -opBC-5000.par,opBC-51813.par: New bad column masks, somewhat circular dependence - on pixflats. Make opBC-50000.par identical to opBC-51813.par, so - that the only different bad column masks are for the few nights from - MJD=51809->51812 as stored in opBC-51809.par. - For r2 (camcol=2), now mask all of columns 370-371 instead of parts. - For r2 (camcol=2), add mask for column 1068. - For r1 (camcol=4), mask only cols 1112-1115, not cols 1112-1117. -opBC-empty.par: New file with no bad columns for use when generating the - 2D pixel flats and bias images with SPFLATTEN2 and SPBIASGEN. -opLimits.par: Bug introduced when I reformatted the TOTAL/TOTALSN2 entries - (done to address PR #2562 to differentiate between bias and - dark current levels). This fix closes PR #3185. -spEigenElodie.fits: First 70 eigen-components of the 700-ish Elodie spectra, - generated at Sloan resolution using PCA_ELODIE. - -In idlutils: -DJS_MEDIAN: Bug fix for crash condition when WIDTH is greater than - the number of elements in the array. -DJS_REJECT : Major bug in /groupbadpix option, should now work as intended -BSPLINE_ITERFIT: Fix major bug with /GROUPBADPIX option which is used - by COMBINE1FIBER. -SPLOT: For gaussian line fitting, display the Area (not the peak height), - and try fitting absorption lines as well as emission lines. - Still not always a good fit, since I'm using the IDL build-in - CURVEFIT() rather than MPFITFUN, which would be better. -SXCOMBINEPAR: New proc to combine values of specified header cards from - many FITS headers. (This option used by SPCOADD_FRAMES.) - -------------------------------------------------------------------------------- -v4_9_1 (Jan 24, 2002 idlspec2d+idlutils) running on IDL 5.3 - 5.5 -Built against specflat v1_3. - -This tagged version is for running Son-of-Spectro; not viable for Spectro-2D. -Bug fixes to run on IDL 5.5. -Changes to how the SOS robot launches the rsync jobs and fix other SOS bugs. -Bias-subtraction now implemented (bias images in the "specflat" product). -New set of templates for P-1D; QSO's now use 4 eigentemplates. - -APO_LOG2HTML,APOREDUCE: Switch from saving the Mountain standard time (MST) - string to only storing TAI time, and computing UT on-the-fly. -APO_LOG2HTML: Compare the exposure "flavor" when looking at the operational - limits (in the "opLimits.par" file). This allows us to - differentiate between bias and dark levels (PR #2562). -APO_CHECKLIMITS,APO_COLOR2HEX: These functions broken out of APO_LOG2HTML - since they are now used by APO_PLOTSN too. -APO_PLOTBIAS: Remove ROOT_DIR keyword, but look at $RAWDATA_DIR environ - variable instead. Look for uncompressed or compressed raw files. -APO_PLOTSN,APO_LOG2HTML: Only add (S/N)^2 that is not flagged as anything - bad in the opLimits file (currently anything < 2.0 is bad). - This should be true for both the HTML table and for the S/N plots. -APOFLUXCALIB: Procedure for generating a canonical set of flux-calibration - vectors (including the superflat) for on-the-mountain plotting - of spectra using APOPLOT. -APOALL,APOREDUCE: Allow setting MINEXP to 0 sec. Change default of MINEXP - to 0 sec, so that any exposure with non-negative exposure time - is reduced by Son-of-Spectro. -APOPLOT: New proc for plotting spectra from the Son-of-Spectro outputs at APO. -APOPLOT,LOGSHEET,PLOTSPEC: Set the !QUIET option for procedures where it is - annoying to see all the IDL messages. -BATCH1D,BATCH2D: Allow NICE to be set to zero. Add UPSVERSION keyword to do - a "setup idlspec2d $UPSVERSION" on the remote machine before - executing the IDL job. This allows you to batch jobs using a - version other than that which is declared current under UPS. -BATCH2D: At the end of a batch job, spawn the following: - spawn, 'rm -f spArc*.fits* spFlat*.fits* spSky*.fits*' - in order to remove files that eat up too much disk space right now. -COMBINE1FIBER: Use the /GROUPBADPIX keyword for rejection in B-spline fits. -DANVB_CONVERT,DANVB_QSOLIST: Procs for generating a list of input - QSO's from Dan van den Berk's EDR list "plateEDR.summ" - for generating QSO eigenvectors. The output file - "eigeninput_qso.dat" is then used by PCA_QSO. -DIVIDEFLAT: Make INVVAR an optional keyword instead of an input for DIVIDEFLAT. - Change the procedures that call this. -EXTRACT_OBJECT,SPBIASGEN,SPCALIB,SPREDUCE: For TAI time, use the midpoint - of the observation as determined by TAI-BEG + 0.5 * EXPTIME - from the header. -EXTRACT_OBJECT,SPREDUCE: No longer write the sky vectors to a different file, - but write them to the spFrame files. Also, always write the - telluric-correction array, which will simply be an empty HDU - for the blue CCDs. -FIND_NMINIMA: Retain each peak only if it is at least MINSEP from all - previous peaks. -FINDSPEC: Crash if the plate list file is missing w/ an informative message. - Do not crash if spPlate files missing for some plates. -FITARCIMAGE: Bug fix to prevent IDL 5.5 from crashing. -FITSN: The logic for choosing fitting limits for all possible bands - has been moved to this function. This function will now - choose the full mag range [0,23] to fit if there are fewer - than 20 points in the default ranges and at least 3 good points. - This function is called by the following routes in Son-of-Spectro: - APO_PLOTSN -> PLOTSN -> FITSN - QUICKEXTRACT -> FITSN - This function is called by the following routes in Spectro-2D: - PLATESN -> PLOTSN -> FITSN - EXTRACT_OBJECT -> FITSN -MULTISYNTHSPEC,SYNTHSPEC: Bug fix for when wavelength spacing is different - between the template and the requested synthetic spectrum. -PCA_CVSTAR: New proc for generating eigen-spectra for CV stars. -PCA_QSO: Major re-write now that we're trying to use the eigen-spectra - from this routine for the redshift-finding code, rather - than using the single QSO template that we've used to date. - Solve for 4 eigencomponents in rest-frame [460,9300] Ang. - Try solving for one eigen-component at a time. When I tried - solving for them all simultaneously, this resulted in an - unphysical 1st eigencomponent, probably because each spectrum - only covers a small range of rest wavelength. - Also, make some plots. -PCA_SOLVE: Add NRETURN keyword for returning a different number of - eigen-components from the number used (NKEEP) for replacing - bad and noisy data points. Optionally return NEWFLUX,NEWIVAR. -PCA_STAR: Major changes to read stellar classes and subclasses, - then generate the 2-component PCA solution for all M stars - (for example) and reconstruct types M1 through M9. -PLATECOMPARE: Add /PSFILE option. -PLATELIST: Add NEXP,EXPT_B1,EXPT_B2,EXPT_R1,EXPT_R2,SMEARUSE,TAI_BEG,TAI_END, - SEEING*,RMSOFF*,AIRTEMP,CARTID to output struct. - Now show the status of 2D and Combine separately, which will make it - obvious when this problem occurs. When it happens, Status2D - might still be "Running" even though StatusCombine is "Done". - That case would mean that one of the night's of 2D data - probably crashed, and we combined from whatever subset of data - did run through from other nights. This addressed PR #2993. - Add the /KILLPARTIAL option for deleting data from partially- - combined nights, which only happens from the Spectro-Robot - operating in real-time. -PLOTSN: Changed auto-scaling of fitted magnitudes, when less than 20 objects - are in the desired range. Now the fitting range will expand to - [10.0, snmag + 1.0]. So in case of g' [10.0, 21.2], this should - work much better for the photo-z S/N estimates. -PLOTSPEC: New keyword option /NETIMAGE to plot corresponding PHOTO images - in a Netscape window using Steve Kent's web tool. -QAPLOT_SKYSUB: Don't be quite so severe with warning messages. If a single - sky fiber has bad flux levels, report as a "Warning" instead - of a "WARNING". Trigger the "WARNING" about "Max sky-residual - chi" at 5.0 instead of 4.0. -QUICKBIAS: Remove message which preempts logging. Instead, return a - set of precentiles that are all -9999, which shows up as **** - in the HTML file. -QUICKEXTRACT: Put exposure time dependence in scattered light warning. - It is the same limit as we had before for 900-sec exposures, - the same limit for shorter exposures, and a (scaled) higher - limit for longer exposures. - Include FIBERMAG in the output structure, where we use MAG[1] for - any blue frames, and MAG[3] for any red frames. -REDMONSTER: Change the warning message to "Bad sky residuals...", - and only send the message "Red Monster at..." if it - is around 6400-6600 Ang, where we think the problem may - be due to the hand-paddle LED. This change is for the - benefit of the observers (from Son-of-Spectro warnings). - Also output the peak chi. -REDMONSTERLIST: New procedure: List of number of galaxies and quasars per - plate, the number of redshifts that differ from prior reductions, - and the fraction of various 2D flags set. -REMOVE2REDO: Make more robust, and give more descriptive messages. - Another attempt to fix errors in remove2redo. It might now - work with plate argument: "sos_redo plate=665", but should - definitely work with simple invocation: "sos_redo". - --> Now a re-write of this, allowing either PLATE or EXPNUM - keywords (but not both) as of 8 Jan 2002. -RLINE_FINDPEAKS: Updated to demand minimum separation, and commented. -SDSS_FLAGS: Add "SOUTHERN_SURVEY" flags for TARGET and TTARGET. - Also, return a warning message for any bits that are set - but whose meaning is unknown. Previously, a wrong answer - would just be given in that case. -SDSSPROC: Reverted to calls to specflat_version,speclog_version that work. - New option /APPLYBIAS to bias-subtract. This fixes PR #2595, - subtracting the bias-induced flux at 3800-4600 Ang flux (b1) in - fibers 17+18. -SLATEC_EFC (in idutils): Call FILL_BSPLINE() instead of EFC2D(). -SPBIASAVE: New routine to average biases already generated with SPBIASGEN. -SPCOMBINE: Bug fix for dealing with missing spFrame files (as for plate - 204/51455, where this proc would crash). -SPPLAN2D: Change the default of MINEXP to 1 sec, so that special plates - like the M67 plate (#321) will be reduced. - Make a plan file if either science or smear exposures exist - (before it required the existence of science exposures). - Get the MJD from the directory name $RAWDATA_DIR/$MJD, not - from the header of the 1st file which may be wrong. This - happened on MJD 52247, where there was a bias taken on MJD 52246 - which messed up the plan file for plate 766 on MJD 52247. - But I will at least issue a warning in the proc if there is an - MJD that appears incorrect in one of the headers, as in this case. -SMOOTH_SUPERFLAT: Located the spurious WARNING, changed to Warning, and - upped the threshold. It was not the critical one of the two - smooth_superflat warnings. -SPBIASGEN: New routine for generating mean biases for a night. -SPCALIB,SPFLATTEN2,SPREDUCE: Use /APPLYBIAS option in call to SDSSPROC. -SPCOADD_FRAMES: Convert dispersion vectors to log-lambda units (1.0d-4, PR 1870) - When combining fibers, we used to use MAXITER=20. Now use - MAXITER=50, MAXREJ=1. - Bug fix for SMEARUSE keyword in header, which was incorrectly - always set to 'F'. -SPCOMBINE: New optional keyword MINSN2 to set the minimum S/N to include - in co-adding; default to 0.0 for the time being (see PR 1901). - Otherwise, special plates that have all bright objects will not - get combined (since the extrapolated S/N at 20th mag is so low). - Include 'smear' exposures as well as 'science' in the co-adds, - which should let us recover some spectra that are otherwise - saturated in 15 min exposures. -SPFLATGEN: Add optional keywords EXPNUM,EXPSTART,EXPEND. -SPHDRFIX: Add support for editing raw frame (sdR-*) headers by putting - OPHDRFIX entries in the sdReport-$MJD.par files, of the - same form as that used in the file "opHdrFix.par". - We've changed the structure name SPECHDRFIX to OPHDRFIX, since - we can use exactly the same thing for PT or 2.5-m data. -SPREDUCE1D, ZFIND, ZCOMPUTE, FIND_NMINIMA: Plotting and debugging - tools added with the /DOPLOT, /DEBUG keywords. -SPREDUCE1D: Change flag for small chi^2 difference to be triggered - when RCHI2DIFF < 0.01 or when RCHI2DIFF < 0.01 * RCHI2. - This means that the difference in reduced chi^2 should be at - least 0.01 if chi^2/DOF=1, or at least 0.10 if chi^2/DOF=10. - Print the versions of idlspec2d,idlutils in the log file. - If the outputs spZall,spZbest files already exist, then rename - those files rather than destructively over-writing them (using - CPBACKUP). - Added search for CV's using the eigenfile 'spEigenCVstar-*.fits'. - Search for QSO's out to z=7.0, but this will require templates - that span [525,9300] Ang. - For the moment, disable setting the 32L flag for ZWARNING, - since this seems to usually be erroneously set (when it is set) - in the 2nd pass through the ZFIND routine. I'll have to track - that down later. - Fix bug that SN_MEDIAN wasn't positive definite. - Add FRACNSIGHI,FRACNSIGLO to outputs. Trigger ZWARNING=16 from - only high outliers (FRACSIGHI) for QSO's, whereas we trigger - based upon FRACNSIGMA for all other objects. -ZCOMPUTE, FIND_NMINIMA: Return a (negative-valued) error code - from FIND_NMINIMA, which ZCOMPUTE stuffs into Z_ERR. -ZFIND: If fewer than NFIND peaks are found, then return blank structures - for the remaining (NFIND-NPEAK). Previously, some of those - fields (such as TFILE,TCOLUMN) were filled with information. - -aporsync.sh: Keep archiving guider images (gzipped). -aporsync.sh: Only copy the following select set of files: - Unplugged*.ps, fiberScan*.par, guiderMon*.par, op*.par, - plPlugMap*.par, sdReport*.par -aporsync.sh, sprobot.sh, sprobot1d.sh, sprobot2d.sh: - Log the UID and PPID (parent process ID) to the file "sos.log", - and record the timestamp both when this script is launched - and finishes. -aporsync_logs.sh, aporsync_blue.sh, aporsync_red.sh: - New scripts called from "sos.sh" instead of "aporsync.sh". - This should be more robust, and prevent the logging+guider - files from hanging up the whole system. -sos.sh, sos_status, startapo.sh: Use the new aporync scripts. -sos_start: Set defaults for RAWDATA_DIR and ASTROLOG_DIR appropriate for - the machine sos.apo. -eigeninput_cvstar.dat: New file for PCA_CVSTAR. -eigeninput_star.dat: Many more M stars added for implementing changes - to PCA_STAR. -opHdrFix.par: Correct some values for TILEID,RADEG,DECDEG for the early plates - 187, 202 and 214. -spFluxcalib-*.fits: A canonical set of flux-calibration vectors produced - by APOFLUXCALIB and used by APOPLOT. -sos_redo: Don't allow this to be run from the machine sdsshost! - --> Now a re-write of this, allowing either PLATE or EXPNUM - keywords (but not both) as of 8 Jan 2002. -sprobot_start: Copy the data over starting at 10am Princeton time, - which will be after "mailhtml.sh" is run at APO (at 7:20 am). -sprobot.sh: Run the same version of the idlspec2d product as is set up - for this script. For example, even if v4_9 is declared current, - run v4_8 if that is what's set up when this script is run. - So, you should run "sprobot_start" from the version of the code - that you want to reduce the data. The only exception is if the - current version is not a UPS version, but just NOCVS; in that - case, the version declared current on each remote host is used. - -Combine all of the install instructions into one place, -in the file "idlspec2d_install.html". The instructions -from "spectroSOS.html" have been moved there. -Add header card info to the "spectroSOS.html" document. - -In idlutils: - -All Goddard libraries updated to the versions as of 3 January 2002 from -the web site "http://idlastro.gsfc.nasa.gov/homepage.html". It looks like -the last time I did any updates was exactly two years ago, on 5 Jan 2000. -The most notable update is to MRDFITS, which Wayne Landsman modified in -Dec 2001 following suggestions from Finkbeiner & Schlegel. It explicitly -handles more data types (like UINTs, BITPIX=8, BITPIX=64) and deals with -null strings when writing structures to FITS binary tables (w/out crashing, -as it did before). -The following procedures moved within the Goddard repository: - astro/oploterr.pro (moved from /jhuapl) -The following duplicate procedure was removed from the Goddard repository: - misc/ydn2md.pro (other copy is in /plot) -All of the procedures in pro/tape_io have been removed from Goddard. -The following procedures are new to the Goddard repository: - astro/cosmo_param.pro - astro/galage.pro - astro/lumdist.pro - astro/observatory.pro - astro/planet_coords.pro - astro/gal_uvw.pro - astro/calz_unred.pro - astro/jplephinterp.pro - astro/jplephread.pro - astro/tdb2tdt.pro - astro/jplephtest.pro - astro/lsf_rotate.pro - fits/fxmove.pro - idlphot/pixwt.pro - math/meanclip.pro - math/minf_parabol_d.pro - math/value_locate.pro - math/asinh.pro - misc/fstring.pro - misc/n_bytes.pro - misc/checksum32.pro - plot/vsym.pro -Files in goddard/data and goddard/text have also been added+deleted+modified. - -BSPLINE_ITERFIT: Properly passing INMASK to allow for recouping lost data - points. Small cosmetic changes to work clearly with INMASK - and OUTMASK. -DJS_REJECT: Added GROW option to mask neighboring pixels. - Added /GROUPBADPIX option, which probably fails with multi- - dimensional data (Scott thinks). - Also some small changes to the way grouping works. -EFCMN: Small change declaring a loop index as a long int. -EXTAST: Allow GSSS coordinate systems other than RA,DEC. Specifically, - allow the use of 'GLON-GSS' and 'GLAT-GSS' which we use - in the PT image headers. This means that our version of this - code is currently branched from Wayne Landsman's Goddard version. -RMFILE: Bug fix for the case that one element of the filename array is blank. -YANNY_FREE: Don't crash if PDATA is not set. -YANNY_READ: If there are too few words on a line for a structure entry, - then set ERRCODE=-1 and return PDATA=0. - Also, increase the allowed line-length to 1023 characters - (was 255). The file can still have longer lines by using - the back-slash as a continuation. And one can still use - the /QUICK option for lines with more than 1023 characters, - but that option fails if continuation characters are used. - Add the STNAMES keyword to return the names of structures - for when /ANONYMOUS must be used. - Special-case the replacement of {{}} with "", since it seems - that some bonehead used the former to denote blank strings in - the idReport files. -vector_avsigclip_mask: Fix a bug in the C code vector_avsigclip_mask - that could potentially affect the IDL proc DJS_AVSIGCLIP - in the case when a mask is used but all points are rejected. -evilinstall: If unable to check-out or install the code, - then do not declare it current in the UPS table! - Print "ABORT" for any failures in the install. -evilmake: Turn optimization flags back on, which we commented - out some time ago because it failed on an OSF platform. - Still might not work under OSF as far as we know, but who cares? - We'll pick up 20% in speed for Spectro-2D under Linux by optimizing. - -------------------------------------------------------------------------------- -v4_9_0 (Nov 10, 2001 idlutils only) running on IDL 5.3 - 5.5 -Built against specflat v1_2. - -YANNY_PAR (in idlutils): Add INDX keyword, and fix bug in returned COUNT. - Support for KEYNAME input as an array. -evilinstall (in idlutils): If $CVSROOT is not set, then set it either - by setting up the "sdsscvs" product or explicitly setting to the - Fermi CVS location. This allows the option of installing from a - different repository (i.e., Princeton or NYU) by setting $CVSROOT - first. - -------------------------------------------------------------------------------- -v4_8_4 (Aug 29, 2001 idlspec2d+idlutils) running on IDL 5.3 - 5.4 -Built against specflat v1_2. - -Minor fix to 2D-combining code for running plates 214,340,270 which failed. - -MYFLUXCALIB: Fix bug whereby the PCA spectrum could be negative. - (The routine PCA_SOLVE() can return a negative-valued spectrum even - if all the input spectra are positive-valued.) This crashed plate 340. -PCA_SOLVE: Forgot to return optional outputs OUTMASK, USEMASK in the case - of only a single spectrum being passed. This happened in the - combining of plate 214 in MYFLUXCALIB, and crashed. -SDSSPROC: Prevent dividing-by-zero when using /APPLYPIXFLAT, - though the resulting NaN's had been caught and replaced anyway. -SPREDUCE1D: The outputs ANYANDMASK,ANYORMASK were not being constructed - properly, and were simply wrong. Should be fixed now. -sprobot,sprobot_start: Scripts now use the environment variables - SPROBOT_HOST,SPROBOT_LOCALDISKS for telling which machine to copy - the raw data from, and which local disks to write to. - These scripts are now generalized to the point where they can - be run from anywhere by just setting up environment variables. -spChunkList.par: Update chunk list for chunks 20-25 (up to plate 769). - -BSPLINE_BKPTS (in idlutils): Do not crash even in the case of very few - data points. But untested whether this actually does what we want - in that case. -BSPLINE_ITERFIT (in idlutils): Return cleanly in the case of no un-masked - data points. -eviltarball (in idlutils): New script to export a version of a code - and build a tar file for distribution. - -------------------------------------------------------------------------------- -v4_8_3 (Aug 24, 2001 idlspec2d+idlutils) running on IDL 5.3 - 5.4 -Built against specflat v1_2. - -Cut version for running P-1D, improving the redshift fits + errors. - -BATCH1D,BATCH2D: Added /CLOBBER option to reduce all specified plates, - over-writing any previous reductions. -BATCH2D: Enabled keywords PLATESTART,PLATEEND,MJD,MJSTART,MJEND. - Change the logical link to the raw data directory to be - topdir+'/rawdata' rather than simply '../rawdata'. - The latter would fail when outputs for some plates were - in other linked directories, and then '../rawdata' did not - resolve to anything. -FIND_NMINIMA: Replacement code for FIND_NPEAKS, for solving the chi^2 - minima for redshifts. This routine will fit a quadratic - if there are only 3 points, and a gaussian plus a constant - if there are more than 3 points. - If there is an error in the fit, return an error code as a - negative integer in XERR. -GET_MJD_DIR: Make leading zeros significant. This was necessary to fix - the bug that '5*' would match 500 but also 050 and 150. -MJD_MATCH: New routine split out of SPPLAN1D. -READSPEC: Minor change to prevent crashing if data for some objects - does not exist. Specifically, if structures for some objects - (e.g., ZANS or TSOBJ) did not exist, this routine could crash. - Read the synthetic spectrum from the Zbest file if ZNUM is not set. - This is much faster than regenerating the spectrum. -SPEC_APPEND: Minor change to prevent crashing if data for some objects - does not exist. (This routine is used by READSPEC). -SPPLAN1D: Now call MJD_MATCH(). -SPREDUCE1D: Change the width in velocity space for fitting the chi^2 - minima in the redshift-finding. These widths can be broader - now that we'll be fitting with gaussians rather than quadratic. - (The quadratic fits were not a good approximation when fitting - more than a few points.) Now always use at least 5 points. - Add another ZWARNING flag: Set the 32L bit if the redshift-error - warning flag is set to -1, which means that the chi^2 minimum - was at the edge of the redshift-fitting range. -VDISPFIT: Increase chi^2 computation from [0,475] to [0,600]. -ZCOMPUTE: Replace call to FIND_NPEAKS() with FIND_NMINIMA(). -ZFIND: Do not modify any errors that are less than zero, since those - can be used as just warning flags from the fit. - Increase to search lags (as specified by PMIN,PMAX) according to - the values of both PWIDTH and WIDTH. -ZFITMAX: Replace call to POLY_FIT with SVDFIT such that this routine - behaves the same under either IDL 5.3 or 5.4. It was the - case that the errors were improperly computed using the re- - write of POLY_FIT in IDL 5.4. - -New template files spEigenGal-52146.fits,spEigenStar-52146.fits -based upon v4_8_1 reductions, though using the same input list of -spectra that we've been using for ages. - -In idlutils, added MPFIT routines from Craig Markwardt. These will -be used for the redshift-finding code (specifically, FIND_NMINIMA). - -build_links (in idlutils): Do not build a link for any file if the link - name already exists. - -------------------------------------------------------------------------------- -v4_8_2 (Aug 16, 2001 idlspec2d+idlutils) running on IDL 5.3 - 5.4 -Built against specflat v1_2. - -Trivial changes for running this version at Fermi. - -SPALLREDUCE: SPECFLAT_DIR is not set now, if SPECFLAT product exists and - is specified (this proc only used at Fermi). - -PLATECOMPARE: New proc for interactive comparison of descrepant redshifts. -PLATELIST: Added /PURGE2D, /PURGE1D options. -PLOTSPEC: Added /NOERASE keyword. Minor fix to Y plotting range. - -RMFILE (in idlutils): Add support for deleting a list of files. -SPLOT (in idlutils): Minor bug fix to avoid plotting the axes twice. -evilinstall (in idlutils): Syntax changes to work with bash on spectro. - -------------------------------------------------------------------------------- -v4_8_1 (Aug 9, 2001 idlspec2d+idlutils) running on IDL 5.3 - 5.4 -Built against specflat v1_2. - -Cut version for re-running Spectro-2D on all data. - -EXTRACT_OBJECT,SMOOTH_SUPERFLAT: Better working smooth_superflat (with a QA - plot). Added "Area" to plot anything above 0.01 gives a warning. -FILTER_THRU: Fix failure mode when only a 1-d WAVEIMG is passed. -PLATELIST: Add EXPTIME to output files. -PLOTSPEC: Major re-write to allow flipping through many plots on a plate - and change plotting parameters. Or, send plots to PostScript files. - Usage of ZNUM changed -- see READSPEC for new usage. - Plot labels in green for legibility in the plot window, and - include the ZWARNING flag in the label if it is nonzero. -READSPEC: Return successfully even if redshift files not found. - Change usage of ZNUM such that ZNUM=1 is best fit, ZNUM=2 is 2nd best. -SPCOADD_FRAMES: Set keyword SMEARUSE to T or F. -SPFLATAVE, SPFLATTEN2: Handle edge effects better. In SPFLATTEN2, set - any pixels more than 2 pixels from the left/right of the first/last - fiber equal to zero. Then, in SPFLATAVE, any edge pixels that are - zero from all nights are set to unity. -VDISPFIT: Bug fix for objects with negative velocities. -idlspec2d.table: Make an optional dependence upon the "specflat" product. -sprobot.sh: Copy only the compressed ("sdR-*.fit.gz") files from - sos.apo.nmsu.edu, instead of the raw files from sdsshost.apo. - -evilinstall (in idlutils): Minor syntax changes to work with RH7.1 on spectro. -ATV (in idlutils): Handle unsupported CTYPE, CTYPE reversal gracefully; - left click on apphot (Imexam) prints coords -SPHEREMATCH (in idlutils): New code to match two lists of RA,DEC. - -------------------------------------------------------------------------------- -v4_8_0 (Jul 17, 2001 idlspec2d+idlutils) running on IDL 5.3 - 5.4 -Built against specflat v1_1. - -First cut of v4_8 with many major changes. Not yet tested for prime time. - -******MAJOR CHANGE: spFrame output is now normalized with superflat, - to correct for small scale features which cannot be removed - by standard star correction. - HDU 6 now contains the superfit in spFrame - Telluric correction has been moved to HDU 7. - The B-spline break-points in the files blue.bkpts, red.bkpts - have been changed. - -SKYSUBTRACT: Increased rejection limit for 1d fit. This should be more robust - anyway, and tighter rejection was masking out real PSF variations - which led to very big sky residuals in 2d fit. - Also decreased the number of blue side breakpoints to 1024. - -FITVACSET : keyword "airset" which return air wavelength solution with - no heliocentric correction (required to properly sample superflat - shifted to skyline frame) -FIBERFLAT : keyword superflatset to return normalized superflat fit -DIVIDEFLAT: removed all traces of fibermask -EXTRACT_OBJECT, SKYSUBTRACT, SPCALIB: Implement 2D-splining for sky- - subtraction, now set to do quadratic PSF fit. - Also, first attempt to put in airmass correction to the sky - brightness by dividing all the sky vectors by the airmass of - each fiber, then fitting for the supersky, then multiplying - back in the airmass terms. - New limits to mask in pixelflat 0.8 < flat < 1.2. - old values: 0.5 < flat < 1.5 - -EXTRACT_OBJECT: Set the "NEARWHOPPER" mask bits only for fibers within 2 - fibers of a bright fiber, but not the bright fiber itself. - For the bright fibers, add the bit "WHOPPER" (added to the - param file spMaskbits.par). - Changed the rejection thresholds yet again, and also uncommented - 2-D sky subtraction to test. -EXTRACT_OBJECT, SKYSUBTRACT: Exit gracefully in the case of no good sky - fibers. In that case, issue an "ABORT" log message, return - SKYSTRUCT=0 from SKYSUBTRACT, then do not write an spFrame file. -EXTRACT_ROW: Do not correct rejection levels in relative mode if chi^2 < 1.0. - This means that we can boost the errors to be larger than the - expected Poisson errors, but not smaller. -FIBERFLAT: Update to call SUPERFLAT() as a function instead of a proc, - though should not be functionally different yet. -FILTER_THRU: Switch to the newly-measured filtered curves, "sdss_jun2001*.dat". - Use the Goddard routine LINTERP instead of the IDL built-in - INTERPOL. Also, correct the integrations for the pixel size. -FITMEANX: Make robust to no good sky lines for tweaking wavelength solution. - In that case, the routine LOCATESKYLINES will return SKYSHIFT=0. -FPOLY, FUNC_FIT, TRACESET2PIX, TRACESET2XY, XY2TRACESET: - Added "poly" as one of the functions for trace sets. -LOCATESKYLINES: Test that the sky line file is successfully read. -MYFLUXCALIB: Fairly major changes: Do a pre-filtering of the spectra that - go into the PCA-fitting of the flux-calibration vector, and - mask out the most deviant points. Run the PCA-fitting with - more iterations to ensure convergence. Also, finally comment - this code. -PCA_SOLVE: Construct the USEMASK from the output mask (OUTMASK) instead of - from NEWIVAR. This is equivalent to TOTAL(OUTMASK,2). - This should make the flux-calibration vectors produced by - the proc MYFLUXCALIB more robust, since it rejects pixels in - the solution based upon USEMASK. -PLATELIST: Read MJDLIST keyword from the spPlate files, which will only exist - with future reductions. - Include a computation of the airmass in the FITS version of the - platelist file. -PLATESN: Pass a plot title to PLOTSN. -PLOTSN: For the 2nd page of plots with Synthesized mags. vs. PHOTO fiber mags., - make these residual plots. Also, use plot titles. -QAPLOT_SKYLINE: Plot skyline strength residuals as a function of airmass, - which in fact is exactly what they are correlated with! - Issue a warning message if the computed airmasses are out of the - range [1,3]. -REJECT_SCIENCE: New proc to reject science exposures if they are blatantly - bad (typically, if they are saturated). We have already been - doing this for arc + flat exposures. This proc is called - by SPREDUCE, and will issue an "ABORT" log message if a - science frame is rejected. -SDSSPROC: Add VERSLOG, VERSFLAT keywords to specify versions of the - products SPECLOG, SPECFLAT. - First search for files "pixflatave-*.fits", and if not found then - look for "pixflat-*.fits". -SDSSPROC, SPCALIB, SPREDUCE, SPREDUCE2D: No longer pass the name of the pixel - flat between all these files. Instead, just let SDSSPROC find the - correct one in the "specflat" product, in $SPECLOG_DIR/flats. -SMOOTH_HALO: A new version of smooth halo, which should be closer to the truth. -SORTPLUGMAP: Fill in unplugged fibers with arbitrary entries, and assign - them a FIBERID. For spectrograph #1, we assign the unplugged fibers - from the beginning of the plugmap, and for #2 from the end. - This should result in the unplugged objects being uniquely placed - on only one of the spectrographs. This addressed PR #2487. -SPCOADD_FRAMES: Include MJDLIST keyword in spPlate header, which will be - the list of all MJD's used in an SPCOMBINE. - Add BUNIT keyword for the flux, inverse variance, and - dispersion HDU's. -SPFLATAVE: New routine to average together a set (or all!) pixel flats - in the SPECFLAT directory. -SPFLATGEN: New routine. Wrapper for SPFLATTEN2 for generating pixel-to-pixel - flat-fields. -SPFLATTEN2: Major changes! Now call the 2-dimensional B-spline code when - generating the superflats. However, there seem to be problems - with this code for some combinations of NORD and NPOLY. I have - set NORD=3, NPOLY=4 which seems to be a combination that works. - (A PR has been filed against idlutils.) - Subtract the scattered light image before dividing by the super- - flat evaluations -- w/out doing this, we had bad residuals where - the scattered light contribution was large compared to the signal - (particularly in the UV). - Also, small changes to how directory names are parsed, and retain - the header from the file ALLFLATS[0] for the output file. -SUPERFLAT: Pass X2, NORD and NPOLY keywords to support fitting 2-dimensional - B-splines in the call to BSPLINE_ITERFIT(). At the moment, - this will only be implemented for the proc SPFLATTEN2, not for - FIBERFLAT. (So this shouldn't effect the spectro reductions - except for updated pixel flats.) - Also, make this a function instead of a proc. -TELLURIC_CORR: Fixed horrible bug in using fibermask to reject bad fibers - (fixes PR #2162). Also, when fitting the local continuum - around a single telluric band for a single star with a - polynomial, do the fit with weighted points and iterated - rejections using DJS_REJECT(). -THROUGHPUT_CALIB: New proc for assessing total system throughput using - the intermediate files "spFluxcalib*.fits". -TRACE320CRUDE: Added call to DJS_MASKINTERP() in the beginning to prevent - hot columns from throwing the initial trace (which does - not use the errors at all). -zplot: Quick-and-dirty script for finding and plotting a spectrum (at Princeton) - w/out setting up any path variables or SDSS products. - -f8v.abs: Increase flux-calibrating mask around H-alpha by 50%, now - masking +/- 1500 km/s (+/- 32.8 Ang). -opBC-51813.par: One extra bad column [370,1800:2047] on r2. - There are a few extra additions to match the new pixflatave-00000 files. - Instead of flagging all of the "pits", these are not included, but - can be rejected using minflat/maxflat = 0.8/1.2 - -BSPLINE_ACTION, INTRV (in idlutils): Bug fixes for 2-D spline fitting. - Include 'poly1' function in BSPLINE_ACTION(), and set the function - properly in BSPLINE_ITERFIT. -BSPLINE_FIT, BSPLINE_ITERFIT (in idlutils): - Remove MASK keyword, since it is too confusing with INMASK, OUTMASK - already used in BSPLINE_ITERFIT(). It didn't look like it was - actually used anyway! -DJS_REJECT (in idlutils): Add the GROUPDIM keyword, and allow GROUPDIM, - MAXREJ and GROUPSIZE to be vectors for several limits in the - rejection algorithm. These were some major changes, and hopefully - are completely backwards-compatable. -SPLOT (in idlutils): Changed interactive gaussfit (g) so amplitude is correct. -YANNY_READ (in idlutils): Try to make backwards-compatable with IDL 5.2 - by replacing calls to STRSPLIT() with calls to STR_SEP. -build_links (in idlutils): New script for building links between directories. -evilinstall (in idlutils): Add another option for location of sdssfue - scripts (for the installation on sos.apo), and also set the - CVS locations to the default at Fermi if not set up with - "setup sdsscvs". - -------------------------------------------------------------------------------- -v4_7_6 (Jun 9, 2001 idlspec2d only) - -data_rysnc.sh: Bug fixes in SOS copy, and adaptation to sos.apo.nmsu.edu - (i.e. ps -elf does not work, and scp1 does not exist). - -------------------------------------------------------------------------------- -v4_7_5 (Jun 8, 2001 idlspec2d only) - -data_rsync.sh: SOS should now attempt to archive data with gzip and move - offsite in the morning. -killdata.sh, mailhtml.sh, sos.sh, startapo.sh, APOREDUCE: - Adapted to work on sos.apo.nmsu.edu, and fixed file transfer - bugs. For APOREDUCE, changed spawn of "scp1" to "scp", since - the former did not exist on the new machine sos.apo. - -sprobot_start: Remove "sprobotcopy" from the cron jobs, since copying - reduced data to Fermi always fails. - Set SPECLOG_DIR=$ASTROLOG_DIR for the Spectro-Robot. - -------------------------------------------------------------------------------- -v4_7_4 (May 10, 2001 idlspec2d+idlutils) - -COPY_STRUCT, COPY_STRUCT_INX (in idlutils): - These routines didn't work properly for structures with more - than 32768 elements, because of using only 2-byte integers. - This resulted in READSPEC not working properly for large lists, - and other dependent routines such as ZPLOT (where I noticed it!). -READSPEC: Bug fix when reading one range of fiber numbers for a FITS - binary table, i.e. "READSPEC, 301, [1,2,3], ZANS=ZANS" was not - returning the correct values. -ZPLOT: Include MJD in call to READSPEC. - -spPlateList.par: Enlarge list of EDR plates to include multiple pluggings, - and other plates that should have been in the list according to van - den Berk's mail to sdss-spectro/665 on 2 May 2001. The list now - includes 136 plates, many of which are repeat observations. - -Spectro-Robot scripts: - Let the scripts "sprobot2d.sh" and "sprobot1d.sh" test whether they - are already running, and quit if they are. - Update the syntax to "ps" commands as per RHL's suggestions. - From sprobot.sh: Batch process 2D first, wait for it to complete, - then batch process 1D in the background. - -------------------------------------------------------------------------------- -v4_7_3 (Apr 30, 2001 idlspec2d+idlutils) - -PLATESN: Typo introduced on April 13th that crashes SPCOMBINE -- fixed! -SDREPORTMAKE: New proc for generating sdReport files from raw idR files. - -In idlutils: -YANNY_READ: Add optional /ANONYMOUS flag to avoid conflicts in structure - types, in the event that named structures actually change - between calls (will be used by SDREPORTMAKE). - Split lines with semi-colons, for when reading early sdReport - files with SDREPORTMAKE; however, this may introduce bugs - elsewhere because it was not done totally generally. - -In the Spectro-Robot bash scripts, change the top-level directory for -the plug-map files from $SPECLOG_DIR to $ASTROLOG_DIR. Use the $SPECLOG_DIR -only for re-running with cut versions of everything, including the "speclog" -product. - -------------------------------------------------------------------------------- -v4_7_2 (Apr 27, 2001 idlspec2d only) - -PLATELIST: Use the "spPlateList.par" file to determine which data are public. - Add a PUB column to denote if a plate is in a public data release. -PLATEMERGE: Add the /PUBLIC option to trim only to public data. - -More updates to "opHdrFix.par" file. -Makefile in /bin did not copy all executables when installing; now it does. -Update the batch files to use the wire machines as a default. -Update spectro home page to reflect v4_7 reductions. - -------------------------------------------------------------------------------- -v4_7_1 (Apr 9, 2001 idlspec2d+idlutils) - -FILTER_THRU: Add new Doi filter curves (CCD+atmosphere+telescope), and - add /TOAIR flag to convert from vacuum to air wavelengths. - (But not yet called by 2D or 1D). -PCA_SOLVE, COMPUTECHI2: Make robust to one or all of the template spectra - (e.g., the STARFLUX vectors) being identically zero. - The routine PCA_SOLVE could crash w/out special-casing this - due to an apparent bug in the built-in routine PCOMP(). -SPCOMBINE, SPCOADD_FRAMES: Optionally allow spFrame files to be g-zipped. -READSPEC, SPREDUCE1D: Allow spPlate, spZ files to be g-zipped. -READSPEC: Major changes to speed up the code when reading a sparse number - of fibers per plate. It used to read all the fibers between the - smallest and largest fiber number. Now it divides up the requested - fibers into contiguous chunks, and reads in only those chunks. - (The plate file is still only opened once, and the header is only - read once.) - -Add new mask bit: NEARWHOPPER. This effects the following routines: - EXTRACT_OBJECT, FLUXCORR_NEW, MYFLUXCALIB, TELLURIC_CORR. - These fibers are also implicitly ignored by sky-subtraction - because the sky-fiber selection implicitly assumes all fiber- - mask bits are fatal (in EXTRACT_OBJECT). - -Son-of-Spectro changes: -New executable, "sos_redo" to re-run SOS on failed reductions. -APO_APPENDLOG: Now overwrite entries in the file "logfile-$MJD.fits" if an - exposure is re-reduced (e.g., with "sos_redo"). -APO_LOG2HTML: Placed a lower limit of (S/N)^2 > 2.0 to count towards total - in each camera -QUICKEXTRACT: Change message about post-cabibs from "REQUIRED" to merely - "recommended." - Slightly loosened constraints on match_trace to +/- 0.15 pixels. -QUICKTRACE: Tweak test for putative Argon lines in flats to be a less - sensitive trigger. -REMOVE2REDO: (New proc) Checks for missing entries in logfile*fits and - removes corresponding raw data; called by script "sos_redo". -The routine FIND_WHOPPING is moved out of EXTRACT_OBJECT so that it can - be called by QUICKEXTRACT. -Changed SOS signal-to-noise warning to a yellow warning below S/N=15/pix, - and S/N=13/pix. -More entries in "opHdrFix.par" to fix all the incorrect plugmap files - known to date. However, we still need to get the correct - files copied into the /astrolog directories. - -------------------------------------------------------------------------------- -v4_7_0 (Mar 16, 2001 idlspec2d+idlutils) - -FLUXCORR_NEW: Replace deviant fluxcorr coefficients with the median - Attempt to mask large positive deviations in inverse variance - (PR #2040) -SPCOMBINE: combinedir is now a keyword for spcoadd_frames. - SPFLUX and SPCOADD_FRAMES utilize directory keywords as well. - Start by zeroing-out the 4 mask bits set by combining step. -FIBERFLAT: Simple check to exclude extrema (PR #2123) -COMBINE1FIBER: Replace the lines which errantly set NODATA with a check for the - extrema wavelengths which have no signal. (PR #1939) -SPPLANCOMB, -SPALLCOMBINE: Fixed Bug to combine a specific MJD - -BSPLINE_ITERFIT (in idlutils): Modified to avoid crashing SPREDUCE1D. - Two changes: (1) Return a valid solution even in the case where the variance - of the y-vector is zero. (2) In the case where there are too few points to - fit, still return a valid solution of all zeros. -FINDSPEC: New routine for finding SDSS spectra that match a given RA, DEC. -MULTISYNTHSPEC: New routine that is sometimes faster than SYNTHSPEC, but - I would not recommend using this yet. -MYFLUXCALIB: Make robust to degenerate case of only 1 input spectro-photo std. -PLATELIST: Lower (S/N)^2 cutoff to 13 for all G,I bands (from 15 previously). - If ZWARNING flag exists in 1D outputs, use that to determine - number of UNKNOWN or SKY identifications. -PLOTSPEC: Add ZNUM option (also new to READSPEC). -READSPEC: Add option to generate the synthetic spectra. - Add ZNUM option to get fits other than the best fit. -REDMONSTER: New routine for looking for Red Monster (large regions of - bad sky residuals, which, for example happens when the hand-paddle - light is on at the telescope). Flag these pixels with the - REDMONSTER mask bit. This routine replaces MONSTER. -SPREDUCE1D: Search for stars at velocities to +/- 1200 km/s (was 600 km/s). - No longer identify objects as UNKNOWN, but rather set bits - in a ZWARNING flag. - Add outputs ZWARNING, SN_MEDIAN, FRACNSIGMA, - COUNTS_SPECTRO, COUNTS_SYNTH, COUNTS_SKY (not used yet), - VDISP, VDISP_ERR, ANYANDMASK, ANYORMASK. - Compute velocity dispersions using real-space method (VDISPFIT). - Slightly increase width for re-fitting galaxy redshifts from - PWIDTH=5 to 7. This may fix PR #1938. - Speed up computation of FRACNSIGMA by calling FILTER_THRU() - for the whole plate at once. - Output SYNFLUX as a 2nd HDU in the spZbest*.fits file. -SYNTHSPEC: Calling sequence now allows for returning an array of spectra. -TELLURIC_CORR: If a telluric star is deemded bad, then reject all of - its FITFLUX pixels from the global fit. Previously, we were - only rejecting the pixels INDT, but still allowing bad continuum - pixels into the fit. This fixes PR #2028. -YANNY_PAR (in idlutils): If several lines have the same keyword name, - then return all of those values as an array. This makes this - routine backwards-compatable with Yanny files written previously - with SPPLAN2D. -ZFIND: Get rid of SIGMA2,SIGMA2_ERR and add VDISP,VDISP_ERR. - -Add new mask bits: BADSKYCHI, REDMONSTER, SMEARIMAGE, SMEARHIGHSN, SMEARMEDSN. - -New installation script "evilinstall" in the idlutils product. - -Add a line to the cron tables from "sos_start" and "sprobot_start" to -keep the disk auto-mount points alive at Princeton (or anywhere using an -amd daemon). - -------------------------------------------------------------------------------- -v4_6_2 (Feb 14, 2001) - -For the Fermi calling script SPALLREDUCE, add an NCOMBINE keyword -to limit the number of exposures combined. Default to a maximum of 7. - -------------------------------------------------------------------------------- -v4_6_1 (Feb 14, 2001) - -SPCOMBINE: Reverting to adderr=0.03 to prevent rejections in high S/N regions -SKYSUBTRACT: The "fix" in the previous version didn't properly check the - bad mask bits, and in fact could make things worse. Now fixed. - -------------------------------------------------------------------------------- -v4_6_0 (Feb 9, 2001) - -EXTRACT_OBJECT: Reverted to Gaussian Profile, more robust flat field - interpolation, still 3% are fluctuations when - NEARBADPIXEL is set. -FIBERFLAT: Fiber is flagged BADFLAT is more than 30% of pixels are masked. -FLATINTERP: Remove call to this code. -SPCALIB: Back to original flat field rejection levels matches object extraction. -SKYSUBTRACT: Check for NEARBADPIXEL and LOWFLAT bit as well in pixelmask. - -Add BADFLUXFACTOR mask bit when applying flux-calibration and flux-correction -vectors. - -Fixed DJS_BATCH to work properly on remote but cross-mounted CPU's. - -Plan files (spPlancomb*.par) written by SPPLAN1D now puts all the "planfile2d" -entries on the same line to be consistent with how YANNY_PAR() works now. - -Fix the spike in memory usage at the end of SPCOMBINE. - -Major changes to PLATELIST routine to return more info, and make a FITS -binary table. - -------------------------------------------------------------------------------- -v4_5_0 (Jan 31, 2001) - -Edits to raw frame headers now possible with SPHDRFIX procedure, which -reads the list of changes from opHdrFix.par file. In particular, this -allows us to edit wrong headers in some of the earliest data. - -The F-star flux-calibration vectors are generated with a PCA analysis -that now allows several rejection iterations. - -More iterations added to the arc-line centroiding in FITARCIMAGE -(addresses PR #874). - -Son-of-Spectro now tests for the Red Monster (light at 6400-6550 Ang in -science frames), and tests for argon lines in the flat-fields. The Red -Monster is presumably from the handpaddle being left plugged in. - -SPREDUCE1D: If an object is fit by a negative STAR or negative QSO, then - re-classify it as UNKNOWN. - -------------------------------------------------------------------------------- -v4_4_0 (Jan 23, 2001) - -Remove any telluric-correction beyond 8250 Ang, since we don't believe -any of it there. In fact, it looks like we were multiplying-in features -that are F-star absorption lines at 8540 and 8660 Ang. But what about -the broad feature at 8665 Ang -- is that a telluric-band, or an intrinsic -F-star feature?? - -Keyword CARTID added to headers to track cartridge ID's. - -Spectro robot scripts included in /bin directory (sprobot_start, etc.) - -For Son-of-Spectro, the WARNING and ABORT messages are now saved in HDU #5 -of the FITS log file, rather than as an element of the other HDU's. - -For Son-of-Spectro, now only copies astrolog directory of existing -/data/spectro/$MJD data directory, since from now on the astrolog directory -will never be purged. - -------------------------------------------------------------------------------- -v4_3_1 (Dec 10, 2000) - -This version of "idlspec2d" has the following enhancements for -Son-of-Spectro (SOS): - -(1) The hard-wired user/machine/version dependencies have been removed. - SOS can now be run by any user on any machine, which we may want to - do to avoid conflicts with the fiber-mapper. Though for now, this - will continue to be run from "observer@plate-mapper". -(2) Starting+stopping SOS is now done with the commands - sos_start - sos_status - sos_stop - which will start-up whichever version of the code has been either - declared or set up. Backing out of versions is now straight-forward. -(3) The code can now be managed remotely. For example, from "sdsshost" - you can type: - setup idlspec2d - sos_status observer@plate-mapper - to check the status of SOS. -(4) SOS delays execution when the fiber-mapper is being run. -(5) More logging information is kept. -(6) Identified and removed Y2022 bugs which would have stopped SOS from - working at MJD=60000. It should now work until the year 2132. -(6) Documentation is now complete, and updated to reflect all above changes. - The relevant documentation is also in the "sdssProcedures" product. - -------------------------------------------------------------------------------- -v4_3_0 (Dec 6, 2000) - -The changes are primarily to the Son-of-Spectro (SoS) code, specifically: - -(1) The FITS log files have been re-structured to significantly speed - up reading these files. This was the bottleneck that slowed SoS - down towards the end of a night, since each instance of reading these - files locked out other processes. Our hope is that now SoS will - keep up during the night. -(2) Bias and dark exposures are now analyzed, and the 98th percentile - of all values in these images are reported to SoS. For example, - if 98% of the pixels in a bias are below 8 ADU, then PERCENTILE98 = 8. -(3) The operational limits file has been edited so that SoS values - that appear in red should definitely be considered very bad. - For example, if the number of arc lines found (NLAMPS) is red, then - that arc exposure should be considered bad and another should be - taken until a good number are found. - In practice, we expect it to be rare that anything falls out of spec. - -------------------------------------------------------------------------------- -v4_2_0 (Nov 28, 2000?) - -1) Skip low S/N smear images in flux correction (previously aborted) - -2) Added bad columns to opBC.par - -3) Fixed bug in sdssproc when dealing with shifted rows between - MJD 51578 AND 51580 inclusive. - -4) Changed structure of spMerge2d to have [flux,err] as image - in HDU 0, and [andmask, ormask, dispersion] as binary table in HDU 1. - -5) Added headers cards to HDU's 1-5 in spPlate*fits to include wavelength - information. - -6) Fixed bug to work on sdR*gz files. - - -------------------------------------------------------------------------------- -v4_1_0 (Oct 20, 2000?) - -The following lists the highlights/lowlights -of the new 2d version: - -Here is a catalog of the raw 2d output: - - spFrame-cc-xxxxxxxx.fits Object Frame extraction - - HDU0 : "Flattened" counts per pixel Float 2048x320 - HDU1 : Inverse variance of HDU0 Float 2048x320 - HDU2 : Pixelmask (see below) Long 2048x320 - HDU3 : Wavelength coefficients Structure - HDU4 : Dispersion coefficients Structure - HDU5 : plPlugMapP for 320 fibers Structure - HDU6 : Telluric correction image Float 2048x320 (only for red) - - spSky-cc-xxxxxxxx.fits 2048x320 Subtracted sky image - spFlat-cc-xxxxxxxx.fits 2048x320 flat field vectors - spArc-cc-xxxxxxxx.fits 2048x320 Arc spectra extractions + - HDU1: inverse variance - spDiag2d-pppp-mmmmm.ps - spDiag2d-pppp-mmmmm.log - -Legend: - -xxxxxxxx Frame number -pppp Plate number -mmmmm MJD -cc camera -s spectrograph - -Just before combining 2 or more exposures into fully merged spectra -we perform two flux calibration steps. The outputs are: - - spFluxcalib-xxxxxxxx-s.fits Bspline structure to convert - flattened counts to 10^-17 ergs/s/cm^2/A - - spFluxcorr-xxxxxxxx-s.fits Polynomial coefficients to correct - science frames to smear frame - -The final coaddition uses spFrame, spFluxcalib, and spFluxcorr to create - - spPlate-pppp-mmmmm.fits Fully calibrated final 2d output - - HDU0 : Fluxed log-linear spectra Float npix x 640 where npix ~3900 - units of 10^(-17) ergs/s/cm^2/A - - HDU1 : Inverse variance of HDU0 - HDU2 : Pixelmask (Intersection aka AND mask) LONG npix x 640 - HDU3 : Pixelmask (Union aka OR mask) LONG npix x 640 - HDU4 : Dispersion Image (sigma of arc lines) - in units of 1.0e-4 log10 lambda - HDU5 : plPlugMapP STRUCTURE - -After the final coaddition, the data will be restructured into -individual files, one for each fiber: - - spMerge2d-pppp-mmmmm-fff.fits (fff is fiber number 1 through 640) - - HDU0 : Flux and err Float npix x 2 - HDU1 : Pixelmask AND,OR LONG npix x 2 - HDU2 : Dipsersion image Float npix x 1 - - -!!!!!Notice: Multiple MJDs with the same plugging will be combined!!!!! - into single outputs, but will be labeled with the - highest MJD of the group. - - Bad data which should not be combined - with subsequent data should be deleted before the subsequent - data is run through 2d. - -New features of v4_1_0: - - Major New Features: - - 1) Spectrophotometry: - - a) Flux calibrate with all available SPECTROPHOTO_STD and - REDDEN_STD stars. Use PCA to derive first and second - eigenspectra. Compare directly with SDSS Fundamentals to - calibrate each smear exposure. - - b) Flux correct each science frame with throughput and color - terms to match spectrophoto solution of smear image for - each fiber. Use SPECTROPHOTO_STD solution to low S/N fibers. - - 2) Correct for Near Infrared Scattering (see msg from jeg) and - possible optical scattering. The scattered light background - is estimated from a model profile of the full 2048x2048 image. - - 3) New maskbits: - - Each individual frame extracted by 2d has the following maskbits - possibly set for each pixel: (found in idlspec2d/etc/spMaskbits.par) - - # The following mask bits are for the fiber, set in FIBERMASK_BITS() -maskbits 0 NOPLUG # Fiber not listed in plugmap file -maskbits 1 BADTRACE # Bad trace from routine TRACE320CRUDE -maskbits 2 BADFLAT # Low counts in fiberflat -maskbits 3 BADARC # Bad arc solution -maskbits 4 MANYBADCOLUMNS # >10% pixels are bad columns -maskbits 5 MANYREJECTED # >10% pixels are rejected in extraction -maskbits 6 LARGESHIFT # Large spatial shift between flat and object pos'n -maskbits 7 BADSKYFIBER # Sky Fiber shows extreme residuals - -# The following mask bits are for a pixel, set in PIXELMASK_BITS() -maskbits 16 NEARBADPIXEL # Bad pixel within 3 pixels of trace -maskbits 17 LOWFLAT # Flat field less than 0.5 -maskbits 18 FULLREJECT # Pixel fully rejected in extraction (INVVAR=0) -maskbits 19 PARTIALREJECT # Some pixels rejected in extraction -maskbits 20 SCATTEREDLIGHT # Scattered light significant -maskbits 21 CROSSTALK # Cross-talk significant -maskbits 22 NOSKY # Sky level unknown at this wavelength (INVVAR=0) -maskbits 23 BRIGHTSKY # Sky level > flux + 10*(flux-error) -maskbits 24 NODATA # No data available in combine B-spline (INVVAR=0) -maskbits 25 COMBINEREJ # Rejected in combine B-spline - - Maskbits 0 through 7 apply to the full set of 2048 pixels on - blue or red spectra. - - The combined spectra found in spPlate*fits or spMerge2d*fits - have two masks: - AND mask: the bit is set if all overlapping pixels is set. - OR mask: " any overlapping pixel bit is set. - -Minor changes: - - 1) Lower limit to variance in raw image, - equivalent to S/N < 100 per pixel. - - 2) New optimal extraction profile: 0.5 * exp(-x^2) + 0.5*exp(-|x|^3) - (proftype 3) - - 3) Restricted wavelength range in telluric correction - - 4) Tighter rejection limits, more stringent cosmic rejection - limits: +- 4 sigma - - 5) Sign has been corrected in heliocentric velocity - - 6) The header has the following changes: - - CALID* has been deleted - UNSIGNED has been deleted - - FRAMESN2 S/N^2 for each frame, does not apply to merged - spectra - EXPID can accomodate more than 10 exposures - PIXFLAT: 2d pixel flat used - OPBC: bad pixel file applied - OPECALIB: opEcalib file applied - OPCONFIG: opConfig file applied - - SPEC1_G, SPEC1_R, SPEC1_I - SPEC2_G, SPEC2_R, SPEC2_I : Fiducial S/N across spectrographs - - MAG_G, MAG_R, MAG_I : Synthetic magnitude after - spectrophotometry - - - 7) Bspline package no longer requires slatec library - - 8) Sky fibers with significant extra flux are rejected - - 9) Number of bad fibers with given bits set are tabulated at end - - 10) pipeline can run and use .fit.gz gzipped raw data files, - saving about 40% of space required by raw spectro data - - 11) Spectral Resolution is output in the final merged spectra. - - 12) Use a better algorithm to tweak flat field trace to match object - trace. - - 13) Fixes in to account for electronics problem in August - -------------------------------------------------------------------------------- -v3c (May 12, 2000?) - -It still requires idlutils v3b to run. -It is tagged on the branch, v3branch, and includes the following -improvements to the 2d pipeline: - -- Auto-generation of S/N plot at conclusion of spectra merging. -- Increased number of supersky breakpoints to achieve better sky subtraction -- Added entries to opBC.par file, this will remove most of the remaining - glitches. -- Added a new routine, "spmulti", which can combine multiple MJDs - with the same plugmap file -- Heliocentric correction is now done on an exposure by exposure basis in 2d. -- Removed header cards which are no longer pertinent, and added some - extras requested by Andy. - Ones to note are NGOOD (number of good pixels in final spectrum) - SN_G, SN_R, SN_I (3 measures of S/N) -- Dispersion and spatial widths are output as a function of fiber - fiber number & pixel (in spArcInfo and spFlatInfo respectively) - diff --git a/bin/.DS_Store b/bin/.DS_Store deleted file mode 100644 index 5008ddfcf..000000000 Binary files a/bin/.DS_Store and /dev/null differ diff --git a/bin/BOSS_log b/bin/BOSS_log new file mode 100755 index 000000000..9886b1103 --- /dev/null +++ b/bin/BOSS_log @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +from boss_drp.sos.BOSS_log import build_log +from boss_drp.utils import jdate +import os +import sys +import argparse +import datetime +import astropy.time +import platform + + +if __name__ == '__main__' : + """ + Build BOSS Exposure Log + """ + parser = argparse.ArgumentParser( + prog=os.path.basename(sys.argv[0]), + description='Build BOSS Exposure Log') + + parser.add_argument('-m', '--mjd', type=str, help='MJD',required=False, default=None) + parser.add_argument('-y', '--yesterday', default=False, action='store_true', help='current mjd-1') + if 'sdss5' not in platform.node(): + parser.add_argument('-o', '--observatory', '--obs', default=None, help='Manually set observatory', type=str.lower, choices=['apo', 'lco']) + parser.add_argument('-l', '--long', default=False, action='store_true', help='Long/detailed version of log') + parser.add_argument('--new_ref', default= False, action='store_true', help='Calculate new reference values in fratio and w_shift and show in place of fratio and w_shift (edit to code to save new value is required)') + parser.add_argument('-c', '--hide_hart', '--hide_hartmann', default= False, action='store_true', help='Hide cleaned version of Hartmann Logs as a table') + parser.add_argument('-r', '--hart_raw', default = False, action='store_true', help='Print raw form (instead of table form) of Hartmann Logs') + parser.add_argument('-e', '--hide_error', default=False, action='store_true', help='Hide SOS Error and Workings') + parser.add_argument('-s', '--hide_summary', default=False, action='store_true', help='Hide data summary table') + args = parser.parse_args() + + if 'sdss5' in platform.node(): + args.observatory = None + + try: + obs = os.getenv('OBSERVATORY').lower() + if args.observatory is not None: + obs = args.observatory + except: + if args.observatory is None: + obs = input('Enter Observatory {apo,lco} ') + obs = obs.lower() + else: + obs = args.observatory.lower() + #obs = input('Enter Observatory {apo,lco}') + #obs = obs.lower() + if obs not in ['apo', 'lco']: + print('Invalid observatory') + exit() + if args.mjd is None: + args.mjd = jdate.obs(obs).astype(str) + if args.yesterday is True: args.mjd = str(int(args.mjd)-1) + if 'sdss5' in platform.node(): + datadir = '/data/spectro/' + sos_dir = '/data/boss/sos/' + else: + args.hart = False + if obs == 'apo': + datadir = os.getenv('BOSS_SPECTRO_DATA_N') + sos_dir = os.getenv('BOSS_SOS_N') + else: + datadir = os.getenv('BOSS_SPECTRO_DATA_S') + sos_dir = os.getenv('BOSS_SOS_S') + if args.hart_raw is True: + args.hide_hart = False + build_log(args.mjd, obs, Datadir=datadir, long_log = args.long, new_ref = args.new_ref, + hart=not args.hide_hart, hart_table = not args.hart_raw, hide_error=args.hide_error, + hide_summary=args.hide_summary, sos_dir = sos_dir) + diff --git a/bin/GetconfSummary.py b/bin/GetconfSummary.py deleted file mode 100644 index 2e548e767..000000000 --- a/bin/GetconfSummary.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 - -from sdss_access.path import Path -from sdss_access import Access -from os import getenv, environ -from astropy.table import Table -from pydl.pydlutils.yanny import read_table_yanny, yanny, write_table_yanny -import os.path as ptt -from glob import glob - - -def find_plPlugMapM(mjd, plate, mapname, splog=None, release='sdsswork'): - if getenv('SPECLOG_DIR') is None: - splog.info('ERROR: SPECLOG_DIR must be defined') - exit() - fibermap_file = ptt.join(getenv('SPECLOG_DIR'),str(mjd),'plPlugMapM-'+mapname+'.par') - fibermap_file_list = glob(fibermap_file) - if len(fibermap_file_list) == 0: - splog.info('plPlugMapM file '+fibermap_file+' does not exists') - return(None) - else: - return(fibermap_file_list[0]) - -# -# path = Path(release='release', preserve_envvars=True) -# print(path.lookup_names()) -# path_ops = {'mjd': mjd, 'plateid': int(plate)} -# fibermap_file = path.full('plPlugMapM', **path_ops) -# if path.exists('plPlugMapM', **path_ops): -# pass -# elif path.exists('plPlugMapM', **path_ops, remote=True): -# access = Access(release='release') -# access.remote() -# access.add('plPlugMapM', **path_ops) -# access.set_stream() -# valid = access.commit() -# if valid is False: -# splog.log('plPlugMapM file '+fibermap_file+' does not exists') -# return(None) -# return(fibermap_file) - - -def find_confSummary(confid, obs=None, no_remote=False, splog=None, release='sdsswork'): - path = Path(release=release, preserve_envvars=True) - - if obs is None: - obs = getenv('OBSERVATORY').lower() - path_ops = {'configid': confid, 'obs': obs.lower()} - fibermap_file = None - #print(path.full('confSummaryF_test', **path_ops)) - if path.exists('confSummaryF_test', **path_ops): - #local confSummaryF file exists - fibermap_file = path.full('confSummaryF_test', **path_ops) - elif (not no_remote): - access = Access(release=release) - if (path.exists('confSummaryF_test', **path_ops, remote=True)): - #Remote confSummaryF file exists - fibermap_file = path.full('confSummaryF_test', **path_ops) - access.remote() - access.add('confSummaryF_test', **path_ops) - access.set_stream() - valid = access.commit() - if valid is False: - fibermap_file = None - if fibermap_file is None: - if path.exists('confSummary_test', **path_ops): - #Local confSummary file exists - fibermap_file = path.full('confSummary_test', **path_ops) - elif (not no_remote): - access = Access(release=release) - if (path.exists('confSummary_test', **path_ops, remote=True)): - #Remote confSummary file exists - fibermap_file = path.full('confSummary_test', **path_ops) - access.remote() - access.add('confSummary_test', **path_ops) - access.set_stream() - valid = access.commit() - if valid is False: - if splog is None: - print('confSummary file '+fibermap_file+' does not exists') - else: - splog.info('confSummary file '+fibermap_file+' does not exists') - return None - if fibermap_file is None: - #No confSummary file exists - fibermap_file = path.full('confSummary', **path_ops) - if splog is None: - print('confSummary file '+fibermap_file+' does not exists') - else: - splog.info('confSummary file '+fibermap_file+' does not exists') - return None - return fibermap_file - - - -def get_confSummary(confid, obs=None, no_remote=False, splog=None, - release='sdsswork', sort=False, filter=False): - fibermap_file = find_confSummary(confid, obs=obs, no_remote=no_remote, - splog=splog, release=release) - - if fibermap_file is None: - return(Table()) - confSummary=read_table_yanny(fibermap_file,'FIBERMAP') - if filter is True: - confSummary=confSummary[np.where(confSummary['fiberType'] == 'BOSS')] - if sort is True: - confSummary.sort('fiberId') - - return(confSummary) diff --git a/bin/SOS b/bin/SOS index 4671622a2..f66344cb7 100755 --- a/bin/SOS +++ b/bin/SOS @@ -1,629 +1,13 @@ #!/usr/bin/env python3 -import sos_classes,putils, sxpar -import argparse, sys, logging, logging.handlers -import subprocess, time,os, re, glob, copy -import numpy as np -from astropy.io.fits import getheader -from multiprocessing import Process -from readfibermaps import readfibermaps +import argparse from argparse import ArgumentTypeError -import re - -#### -class fs_Config: - """file sequencer Config Info""" - - def __init__(self): - self.fitname = "" - self.fitdir = "" - self.plugname = "" - self.plugdir = "" - self.sosdir = "" - self.mjd = "" - self.fps = "T" - self.nocal = "F" - self.nodb = "F" - self.no_reject= "F" - self.sdssv_sn2= "F" - - def __str__(self): - return ("fitname: " + self.fitname + "\n" + - "fitdir: " + self.fitdir + "\n" + - "plugname: " + self.plugname + "\n" + - "plugdir: " + self.plugdir + "\n" + - "sosdir: " + self.sosdir + "\n" + - "mjd: " + self.mjd + "\n" + - "fps: " + self.fps + "\n" + - "nocal: " + self.nocal + "\n" + - "nodb: " + self.nodb + "\n" + - "no_reject:" + self.no_reject + "\n" + - "sdssv_sn2:" + self.sdssv_sn2); - -def updateMJD(workers, cfg, log): - """Check to see if a new MJD exists""" - - regex = sos_classes.Consts().MJDGlob; - try: - MJD = ls(cfg.fitsDir, regex)[-1][-5:] - if (MJD == cfg.MJD): - return - - cfg.MJD = MJD[-5:] - for worker in workers: - worker.fileCount = 0 - - log.info("Latest updated MJD found to be " + os.path.join(cfg.fitsDir, cfg.MJD)) - except: - log.critical("Could not find latest MJD in " + cfg.fitsDir) - log.critical("GOODBYE!") - sys.exit(1) - -def flavor(cfg): - """return the flavor of the fits file""" - return sxpar.sxparRetry(os.path.join(cfg.fitdir,cfg.fitname), "flavor", retries = 5)[0].lower() - -def plugging(cfg): - """return the plugging of the fits file""" - return sxpar.sxparRetry(os.path.join(cfg.fitdir,cfg.fitname), "CONFID", retries = 5)[0].lower() - - -def previousExposure(cfg, log): - """return a config for the previous exposure""" - - # Parse: sdR-b1-00114186.fit.gz - left = cfg.fitname[:7] - right = cfg.fitname[15:] - exp = (str(int(cfg.fitname[7:15]) - 1)).zfill(8) - - prevcfg = copy.copy(cfg) - prevcfg.fitname = left + exp + right - log.info("previous cfg:\n" + str(prevcfg)) - return prevcfg - - -def rule1(cfg, log): - """Handle arc/flat ordering""" - prevcfg = previousExposure(cfg, log) - log.info("Exposure Flavor: " + flavor(cfg)) - prvfitsExist = os.path.exists(os.path.join(prevcfg.fitdir, prevcfg.fitname)) - log.info("Previous exposure exists: " + str(prvfitsExist)) - if prvfitsExist: - log.info("Previous Flavor: " + flavor(prevcfg)) - log.info("Same Plugging: " + str(plugging(cfg) == plugging(prevcfg))) - - # Handle flats -- process, and arc if was previous - if flavor(cfg) == "flat": - log.info("Exposure is a flat") - processFile(cfg, log, "flat") - if prvfitsExist and flavor(prevcfg) == "arc": - log.info("Processing previous arc") - processFile(prevcfg, log, "arc") - return True - return False - - -#### -def processFile(cfg, log, flavor=""): - """call sos_apocommand on the file. Will exit with error code if the command failts.""" - - cmd = "sos_apocommand" - cmd += " -f " + cfg.fitname - cmd += " -i " + cfg.fitdir - cmd += " -p " + cfg.plugname - cmd += " -l " + cfg.plugdir - cmd += " -s " + cfg.sosdir - cmd += " -m " + cfg.mjd - if cfg.fps == "T": - cmd += " -e " - if cfg.nocal == "T": - cmd += " -a " - if cfg.nodb == "T": - cmd += " -n " - if cfg.no_reject == "T": - cmd += " -r " - if cfg.sdssv_sn2 == "T": - cmd += " -v " - prefix = "sos_apocommand(" + flavor + "): " - - i = 0 - while i < 5: - if i > 0: - sleep(2) - log.info("Trying again to get idl license") - log.info("executing: " + cmd) - rv = putils.runCommand(cmd, echo=False, prefix=prefix, logCmd=log.info) - if rv[0] != 0: - log.info("\nCommand failed with rc = " + str(rv[0]) + "\n") - sys.exit(1) - if 'Failed to acquire license.' not in rv[1]: break - - - -def sos_filesequencer(fitname, fitpath, plugname, plugpath, - sosdir, mjd, fps='T', nocal='T', log=None, - nodb='F', no_reject='F', sdssv_sn2='F' ): - """ - Checks if processing a flat if there was an arc before the flat, - then process the after the flat - Otherwise process the fits file - """ - - - config = fs_Config() - config.fitname = fitname - config.fitdir = fitpath - config.plugname = plugname - config.plugdir = plugpath - config.sosdir = sosdir - config.mjd = mjd - config.fps = fps - config.nocal = nocal - config.nodb = nodb - config.no_reject= no_reject - config.sdssv_sn2= sdssv_sn2 - # Rules return true if they processed the file and processing should stop - # process rule - - log.info("Checking Rule 1") - if not rule1(config, log): - log.info("Passed Rules; let's go!") - processFile(config, log, flavor(config)) - -#### -def processNewBOSSFiles(worker, files, cfg, log): - """ Process new fits files - - Get the plugmap name and then add the appropiate APO command to the - correctly numbered process list. - - Before the files are processed, they are sorted by name. We really want the files - sorted by time, but because of the sequence number, name is the same as time for any - given camera. - - """ - - # Sort files by name to get into the right time order - files.sort() - log.info("Sorted file list" + str(files)) - - for file in files: - log.info("processing new file: " + file) - - #- Get platetype from header (missing=BOSS) - hdr = getheader(file) - - if 'FLAVOR' not in hdr: - log.info("Skipping exposure with missing FLAVOR keyword.") - return - else: - flavor = hdr['FLAVOR'] - - if 'PLATETYP' in hdr: - platetype = hdr['PLATETYP'].upper() - else: - platetype = 'BHM&MWM' - - #- always process bias and darks, regardless of PLATETYP - #- for other exposure types, only process BOSS and EBOSS exposures - if flavor in ('bias', 'dark') or platetype in ('BHM', 'BHM&MWM'): - plugpath = getPlugMap(file, cfg, log) - - SOS_opts = {'confSummary':plugpath,'ccd':os.path.basename(file).split('-')[1], 'mjd':cfg.MJD, - 'log':True, 'log_dir': os.path.join(cfg.sosdir,str(cfg.MJD))} - readfibermaps(topdir=os.path.join(cfg.sosdir,str(cfg.MJD)), SOS=True, SOS_opts=SOS_opts, - logger=log, clobber=cfg.clobber_fibermap) - - qf = os.path.abspath(file) - fitname = os.path.basename(qf) - fitpath = os.path.dirname(qf) - - plugPath = os.path.abspath(plugpath) - plugname = os.path.basename(plugPath) - plugpath = os.path.dirname(plugPath) - - sos_filesequencer(fitname, fitpath, plugname, plugpath, - cfg.sosdir, cfg.MJD, fps=cfg.fps, - nocal=cfg.nocal, log=log, nodb=cfg.nodb, - no_reject=cfg.no_reject, sdssv_sn2=cfg.sdssv_sn2) - else: - #- Don't crash if hdr is mangled and doesn't have EXPOSURE - if 'EXPOSURE' in hdr: - log.info("Skipping %s exposure %d." % (platetype, hdr['EXPOSURE'])) - else: - log.info("Skipping %s exposure." % platetype) - return - -def writeVersionInfo(cfg, log): - """Write a version string to a file""" - - verFile = os.path.join(cfg.controlDir, sos_classes.Consts().versionFile) - rc = subprocess.getstatusoutput("idlspec2d_version") - f = open(verFile, "w") - f.write(time.ctime() + " " + rc[1] + "\n") - f.close() - log.info("Version is %s" % rc[1]) - log.info("IDLSPEC2D Module is %s" % os.getenv('IDLSPEC2D_VER')) - -#### -def getPlugMap(file, cfg, log): - """ - Returns the fully qualified name of the plugmap file - """ - - speclogDir = cfg.plugDir - - - try: - obs = os.environ['OBSERVATORY'].lower() - except: - log.critical('Not running at APO or LCO, set OBS environmental variable to run') - return "" - plugmapDir = os.path.join(speclogDir, obs, 'summary_files') - - # Get plugmap used by file - try: - plugmapFullId = sxpar.sxparRetry(file, "CONFID", retries = 5)[0] - except TypeError as t: - log.critical("\nCould not parse " + file + "\n ->" + str(t)) - return "" - except IndexError: - log.critical("\nKeyword CONFID not found in " + file) - return "" - try: - if int(plugmapFullId) == -999: - plugmapFullId = '0' - except: - plugmapFullId = '0' -# try: -# plugmapDir = os.path.join(plugmapDir, str(int(np.floor(int(plugmapFullId)/100))).zfill(4)+'XX') -# except: -# plugmapDir = os.path.join(plugmapDir, str(int(np.floor(int(0)))).zfill(4)+'XX') - try: - plugmapDir = os.path.join(plugmapDir, str(int(np.floor(int(plugmapFullId)/1000))).zfill(3)+'XXX', - str(int(np.floor(int(plugmapFullId)/100))).zfill(4)+'XX') - except: - plugmapDir = os.path.join(plugmapDir, str(int(np.floor(int(0)/1000))).zfill(3)+'XXX', - str(int(np.floor(int(0)/100))).zfill(4)+'XX') - log.info("Current confSummary directory is " + plugmapDir) - - # Parse plugmap name - plugmapName = "confSummaryF-" + plugmapFullId + ".par" - plugParse = plugmapFullId.split("-") - plugmapId = plugmapFullId #plugParse[1] - log.debug(file + " uses confSummaryF " + plugmapFullId + " with Id " + plugmapId) - log.debug(" full name of confSummaryF file is " + plugmapName) - log.debug("confId=" + plugmapId)# + ", pMJD=" + plugmapMJD) - - # Check if the file exists, if not get it and add it to svn - plugpath = os.path.join(plugmapDir, plugmapName) - if os.path.isfile(plugpath): - log.info("Found existing confSummaryF file: " + plugpath) - else: - plugmapName = "confSummary-" + plugmapFullId + ".par" - plugParse = plugmapFullId.split("-") - plugmapId = plugmapFullId #plugParse[1] - log.info("No confSummaryF file: " + plugpath) - log.debug(file + " uses confSummary " + plugmapFullId + " with Id " + plugmapId) - log.debug(" full name of confSummary file is " + plugmapName) - log.debug("confId=" + plugmapId)# + ", pMJD=" + plugmapMJD) - - plugpath = os.path.join(plugmapDir, plugmapName) - - if os.path.isfile(plugpath): - log.info("Found existing confSummary file: " + plugpath) - else: log.critical("Could not get confSummary for Id " + plugmapId) - return os.path.abspath(plugpath) - -def initializePollWorkers(workers, cfg, log): - """Initialize poll workers with latest file counts""" - - for worker in workers: - worker.fileCount = len(glob.glob(os.path.join(cfg.fitsDir, cfg.MJD, worker.glob))) - log.debug("\nInitialized PollWorker:\n" + str(worker)) - -def initializeMJD(cfg, log): - """Find the correct MJD to start looking for new files. If the user specifies an MJD just test - to see if it exists, otherwise, use the latest MJD.""" - - # First check for user specified - if cfg.MJD != "0": - path = os.path.join(cfg.fitsDir, cfg.MJD) - if not os.path.isdir(path): - log.critical("Could not find user specified MJD path: " + path) - log.critical("GOODBYE!") - log.info("Using user specified MJD " + path) - else: - regex = sos_classes.Consts().MJDGlob; - try: - log.debug("Looking for initial MJD in " + cfg.fitsDir) - cfg.MJD = ls(cfg.fitsDir, regex)[-1][-5:] - log.info("Latest initial MJD found to be " + os.path.join(cfg.fitsDir, cfg.MJD)) - except: - log.critical("Could not find the latest MJD path: " + cfg.fitsDir) - log.critical("GOODBYE!") - -def createPollWorkers(cfg, log): - """Create poll workers""" - - workers = [] - - num = 1 - for glob in cfg.globs: - p = sos_classes.PollWorker() - p.glob = glob - p.workerNumber = num - num += 1 - workers.append(p) - log.debug("\nnew PollWorker:\n" + str(p)) - - return workers - - -def initializeLogger(cfg): - """Startup logging and set the level""" - - lname = os.path.join(cfg.logDir, sos_classes.Consts().logName) - if cfg.iname != "": - lname += "-" + cfg.iname - print("Starting to log to " + lname) - - log = logging.getLogger(sos_classes.Consts().logName) - h = logging.handlers.TimedRotatingFileHandler(lname, 'midnight', 1, 5) - hc = logging.handlers.TimedRotatingFileHandler(lname + "-error", 'midnight', 1, 5) - f = logging.Formatter("%(asctime)s-%(levelname)s: %(message)s") - h.setFormatter(f) - hc.setFormatter(f) - h.setLevel(cfg.logLevel) - hc.setLevel(logging.ERROR) - log.setLevel(cfg.logLevel) - log.addHandler(h) - log.addHandler(hc) - - log.info("Hello. " + sys.argv[0] + " started.") - log.info("Startup Configuration is: \n\n" + str(cfg) + "\n\n") - - return log - -def closeLogger(log): - handlers = log.handlers[:] - for handler in handlers: - log.removeHandler(handler) - handler.close() - -def ls(dir, regex="*"): - """return a name sorted list of files in dir""" - - files = [os.path.join(dir, f) for f in glob.glob(os.path.join(dir,regex))] - files.sort() - - return files - -def lsltr(dir, regex="*"): - """return a modification-time sorted list of files in dir""" - - files = [os.path.join(dir, f) for f in glob.glob(os.path.join(dir,regex))] - files.sort(key=lambda tm: os.path.getmtime(tm)) - files1 = [f for f in files if len(f.split('.'))==3] - - return files1 - - -def redo(workers, cfg, log): - """Redo the command for files in the specified MJD""" - - # Get files - for worker in workers: - files = lsltr(os.path.join(cfg.fitsDir, cfg.MJD), worker.glob) - if cfg.exposure != None: - allfiles = files - files = [] - # should only be one, but I do it this way to be sure things are working. - # also, I do the numeric check to avoid worrying about leading zeroes. - for file in allfiles: - log.info("Checking exposure number of: " + file) - exp = re.search("sdR\-..-(\d{8})\.fit.*$", file) - if exp != None: - if int(exp.group(1)) == int(cfg.exposure): - log.info("correct exposure number") - files.append(file) - new = len(files) - log.info("Found " + str(new) + " files in " + - os.path.join(cfg.fitsDir, cfg.MJD, worker.glob)) - processNewBOSSFiles(worker, files, cfg, log) - - -def runner(pollWorkers, config, log): - """ - Run the check for fits files and processes it - """ - # Create poll workers and initialize file counts - initializePollWorkers(pollWorkers, config, logger) - # Watch for new files. Forever... Unless there are exceptions. Then - # try up to 3 times to get it working. But mostly... Forever! - crashes = 3 - while crashes > 0: - try: - watch(pollWorkers, config, logger) - except SystemExit: - raise - except: - crashes = crashes - 1 - if crashes > 0: - logger.exception("!!! Uncaught exception in watch() !!! Will Retry !!!") - else: - logger.exception("!!! TOO MANY Uncaught exceptions in watch() !!!") - raise - - -def lock(cfg, log, file): - """Only one instance of this daemon should be running a file.""" - - lockFile = os.path.join(cfg.controlDir, sos_classes.Consts().lockFileBase) - lockFile += "-" + file - lockFile += ".lock" - - lock = open(lockFile, 'w') - try: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError as xxx_todo_changeme: - (errno, errstr) = xxx_todo_changeme.args - log.info("File running with another instance") - return lock - -def unlock(cfg, file, lock): - """Unlocks a locked fits file""" - lockFile = os.path.join(cfg.controlDir, sos_classes.Consts().lockFileBase) - lockFile += "-" + file - lockFile += ".lock" - lock.close() - if os.path.exists(lockFile): - os.remove(filePath) - -def watch(workers, cfg, log): - """ Watch for new files - - When a new file comes in read the header to look for the plugmap and then check to see - if the plugmap file exists. If it doesn't, get the plugmap from the database and put it - into the proper MJD directory. Create the proper MJD directory for the plugmap if needed. - - Next, check to see if a newer MJD has been created. If there are no new files and no new - MJD then sleep for cfg.pollDelay. - - Note that only the latest MJD is ever checked, so once a new MJD is created only that MJD - will be checked. - """ - - while True: - pause = True - # First check for new files - for worker in workers: - files = lsltr(os.path.join(cfg.fitsDir, cfg.MJD), worker.glob) - if len(files) != worker.fileCount: - pause = False - new = len(files) - worker.fileCount - log.info("Found " + str(new) + " new files in " + - os.path.join(cfg.fitsDir, cfg.MJD, worker.glob)) - # File could get deleted... - if new > 0: - #locks=[] - #for file in files[-1 * new:]: - # locks.append(lock(cfg, log, file)) - processNewBOSSFiles(worker, files[-1 * new:], cfg, log) - #for i, file in enumerate(files[-1 * new:]): - # unlock(cfg, file, lock[i]) - worker.fileCount = len(files) - - # Next check for a new MJD. Don't wait if there's a new MJD - if updateMJD(workers, cfg, log): pause = False - - # Pause if asked - if pause: - log.info("Sleeping for " + str(cfg.pollDelay) + " seconds.") - time.sleep(cfg.pollDelay) - - - -def setup(CCD, cfg, exp=None, mjd=None, catchup=False, redoMode=False, systemd=False, - nodb=False, no_gz=False, no_reject=False, clobber_fibermap=False, sdssv_sn2=False): - - """ - Setup runner configuration - """ - globs = [] # fill in the command line globs - verbose = 0 - - - cfg.plugDir=os.getenv('SDSSCORE_DIR') #'/home/sdss5/software/sdsscore/main' - cfg.fitsDir='/data/spectro' - cfg.controlDir='/home/sdss5/boss/sos/control' - verbose = 6 - cfg.logDir='/home/sdss5/boss/sos/logs' - cfg.pollDelay=2 - if no_gz: - cfg.globs=["sdR-"+CCD+"-*.fit.gz","sdR-"+CCD+"-*.fit"] - else: - cfg.globs=["sdR-"+CCD+"-*.fit.gz"] - cfg.fps= 'T' - cfg.nosvn = True - cfg.bookkeep = False - cfg.nocal = 'T' - cfg.iname = CCD - cfg.clobber_fibermap = clobber_fibermap - if systemd: - redoMode=False - catchup=False - exp=None - mjd=None - if redoMode: - cfg.sosdir='/data/boss/sosredo' - cfg.iname = CCD+'_redo' - else: - cfg.sosdir='/data/boss/sos' - if catchup: - cfg.iname = CCD+'_catchup' - if catchup or redoMode: - args.nice=True - cfg.redo=True - else: - cfg.iname = CCD - #for file in glob.glob(os.path.join(cfg.controlDir,'sos_runner*.lock) - # os.remove(file) - cfg.platedb =False - if nodb: - cfg.nodb = 'T' - if no_reject: - cfg.no_reject = 'T' - if sdssv_sn2: - cfg.sdssv_sn2 = 'T' - if exp is not None: - cfg.exposure = str(exp) - cfg.iname = cfg.iname+'_'+str(exp).zfill(8) - if mjd is not None: - cfg.MJD = mjd - cfg.iname = cfg.iname+'_'+str(mjd) - # Any globs override default - if (len(globs) != 0): - cfg.globs = globs - # Don't want to apply -v on each call, so always start with a base - if (verbose > 0): - cfg.logLevel = max(1, sos_classes.Config().logLevel - verbose * 10) - return(cfg) - - -def SOS(CCD, exp=None, mjd=None, catchup=False, redoMode=False,systemd=False, nodb=False, - no_gz=False, no_reject=False, clobber_fibermap=False, sdssv_sn2=False): - """ - The SOS controller for both manual runs and systemd tasks - """ - global logger - for ex in exp: - config = sos_classes.Config(); - config = setup(CCD, config, exp=ex, mjd=mjd, catchup=catchup, - redoMode=redoMode,systemd=systemd,nodb=nodb, - no_gz=no_gz, no_reject=no_reject, - clobber_fibermap = clobber_fibermap, - sdssv_sn2=sdssv_sn2) - logger = initializeLogger(config) - writeVersionInfo(config, logger) - - # Find correct MJD to start on - initializeMJD(config, logger) - # Create poll workers and initialize file counts - pollWorkers = createPollWorkers(config, logger) - if catchup or redoMode: redo(pollWorkers, config, logger) - else: runner(pollWorkers, config, logger) - closeLogger(logger) - -def parseNumList(string): - m = re.match(r'(\d+)(?:-(\d+))?$', string) - # ^ (or use .split('-'). anyway you like.) - if not m: - raise ArgumentTypeError("'" + string + "' is not a range of number. Expected forms like '0-5' or '2'.") - start = m.group(1) - end = m.group(2) or start - return list(range(int(start,10), int(end,10)+1)) +import sys +import os +from multiprocessing import Process +from boss_drp.sos.SOS import SOS +from boss_drp.sos.SOS import parseNumList if __name__ == '__main__' : """ @@ -641,19 +25,38 @@ if __name__ == '__main__' : group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-c', '--catchup', default=False, action='store_true', help='Run Catchup on the night or (MJD)') group.add_argument('-t', '--redoMode', default=False, action='store_true', help='Save outputs of MJD or exposure to sosredo') + group.add_argument('-d', '--test', default=False,action='store_true', help='Save outputs and logs to sosredo/dev') + group.add_argument('--utah', default=False,action='store_true', help=argparse.SUPPRESS) group.add_argument('--systemd', default=False, action='store_true', help=argparse.SUPPRESS) #parser.add_argument('-e', '--exp', type=str, help='exposure id (with or without leading zeros', default=None) parser.add_argument('-e', '--exp', type=parseNumList,help='exposure id (or range of exp id 500-510) (with or without leading zeros)', default=[None]) - parser.add_argument('-m', '--mjd', type=str, help='MJD', default=None) + parser.add_argument('-m', '--mjd', type=str, nargs='*', help='MJD', default=None) + parser.add_argument('--apo', default=False, action='store_true', help=argparse.SUPPRESS) + parser.add_argument('--lco', default=False, action='store_true', help=argparse.SUPPRESS) parser.add_argument('--nodb', default=False, action='store_true', help="skip opsdb load") parser.add_argument('--no_gz', default=False, action='store_true', help="Overrides the requirement for '.gz' compressed files (experimental)") parser.add_argument('--no_reject', default=False, action='store_true', help="Overrides the Calibration rejection (use with caution)") parser.add_argument('--clobber_fibermap', '-f', default=False, action='store_true', help="Clobbers the existing spfibermap files") - parser.add_argument('--no_sdssv_sn2', default=False, action='store_true', help="Skip reporting a second set of SN2 values with updated fit parameters") + parser.add_argument('--no_sdssv_sn2', default=True, action='store_false',dest='sdssv_sn2', + help="Skip reporting a second set of SN2 values with updated fit parameters") + parser.add_argument('--sdssv_sn2', default=True, action='store_true', help=argparse.SUPPRESS) + parser.add_argument('--no_sn2_15', default=True, action='store_false', dest='sn2_15', + help="Skip reporting a set of SN2 values with a fiducial mag of 15") + parser.add_argument('--sn2_15', default=True, action='store_true', help=argparse.SUPPRESS) + + parser.add_argument('-n','--no_arc2trace', default=True, action='store_false', dest='arc2trace', help="Skip Utilizing arc2trace refinements") + parser.add_argument('-a','--arc2trace', default=True, action='store_true', help=argparse.SUPPRESS) + parser.add_argument('-o','--forcea2t', default=False, action='store_true', help="Force arc2trace for all fields (even if flat exists for field)") + parser.add_argument('-v','--verbose', default=False, action='store_true', help="prints the only (or red if joint) active SOS process to terminal") args = parser.parse_args() + if args.apo: + os.environ['OBSERVATORY'] = 'APO' + if args.lco: + os.environ['OBSERVATORY'] = 'LCO' + OBSERVATORY = os.getenv('OBSERVATORY') if OBSERVATORY.upper() == 'APO': blue = 'b1' @@ -666,15 +69,42 @@ if __name__ == '__main__' : elif args.red is True: CCDs = [red] elif args.blue is True: CCDs = [blue] - proc={} - for i, CCD in enumerate(CCDs): - proc[i]=Process(target=SOS, args=(CCD,), - kwargs={"exp":args.exp, "mjd":args.mjd, - "catchup":args.catchup,"redoMode":args.redoMode, - "systemd":args.systemd,'nodb':args.nodb, - 'no_reject':args.no_reject, - 'clobber_fibermap':args.clobber_fibermap, - 'sdssv_sn2':(not args.no_sdssv_sn2)}) - proc[i].start() - for i, CCD in enumerate(CCDs): - proc[i].join() + + if args.mjd is None: + args.mjd = [None] + for mjd in args.mjd: + proc={} + for i, CCD in enumerate(CCDs): + if args.red: + pause = False + verbose = args.verbose + elif args.blue: + pause = False + verbose = args.verbose + elif CCD == red: + pause = True + verbose = True if args.verbose else False + else: + pause = False + verbose = False + + kwrds = {"exp":args.exp, "mjd":mjd, + "catchup":args.catchup,"redoMode":args.redoMode, + "systemd":args.systemd,'nodb':args.nodb, + 'no_reject':args.no_reject, + 'clobber_fibermap':args.clobber_fibermap, + 'sdssv_sn2':(args.sdssv_sn2), + 'sn2_15':(args.sn2_15), + 'arc2trace':(args.arc2trace), + 'forcea2t':args.forcea2t, + 'pause': pause, 'test':args.test, + 'utah':args.utah, 'termverbose':verbose} + if len(CCDs) == 1: + SOS(CCD, **kwrds) + else: + proc[i]=Process(target=SOS, args=(CCD,), + kwargs=kwrds) + proc[i].start() + if len(CCDs) > 1: + for i, CCD in enumerate(CCDs): + proc[i].join() diff --git a/bin/__pycache__/putils.cpython-37.pyc b/bin/__pycache__/putils.cpython-37.pyc deleted file mode 100644 index 7519b4cb4..000000000 Binary files a/bin/__pycache__/putils.cpython-37.pyc and /dev/null differ diff --git a/bin/__pycache__/sos_classes.cpython-37.pyc b/bin/__pycache__/sos_classes.cpython-37.pyc deleted file mode 100644 index 95ccea86c..000000000 Binary files a/bin/__pycache__/sos_classes.cpython-37.pyc and /dev/null differ diff --git a/bin/__pycache__/sos_runnerd.cpython-37.pyc b/bin/__pycache__/sos_runnerd.cpython-37.pyc deleted file mode 100644 index 7e2f6971c..000000000 Binary files a/bin/__pycache__/sos_runnerd.cpython-37.pyc and /dev/null differ diff --git a/bin/__pycache__/sxpar.cpython-37.pyc b/bin/__pycache__/sxpar.cpython-37.pyc deleted file mode 100644 index bbd24afec..000000000 Binary files a/bin/__pycache__/sxpar.cpython-37.pyc and /dev/null differ diff --git a/bin/__pycache__/yanny.cpython-37.pyc b/bin/__pycache__/yanny.cpython-37.pyc deleted file mode 100644 index 1a609011b..000000000 Binary files a/bin/__pycache__/yanny.cpython-37.pyc and /dev/null differ diff --git a/bin/boss_arcs_to_traces b/bin/boss_arcs_to_traces index 7d4e7e8d5..9465abbe8 100755 --- a/bin/boss_arcs_to_traces +++ b/bin/boss_arcs_to_traces @@ -1,29 +1,84 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +from boss_drp.sos.arc_to_trace_soshtml import soshtml +from boss_drp.sos.report_err import report +from boss_drp.utils.grep import grep +from boss_drp.sos.arc2tracelogger import Logger +from boss_drp.utils.hash import create_hash + +from pyvista import boss + import argparse -import numpy as np import matplotlib import numpy as np -import copy +import traceback import os -import pdb -import matplotlib.pyplot as plt - -from pyvista import boss +import os.path as ptt +import sys +import time if __name__ == "__main__": descr = "Routine to transfer trace locations from an initial arc/flat pair given subsequent arc frames only" - parser = argparse.ArgumentParser(description=descr, - usage="boss_arcs MJD") + parser = argparse.ArgumentParser(description=descr, usage="boss_arcs_to_traces MJD") parser.add_argument("--mjd", type=int, help="MJD to process",required=True) parser.add_argument("--outdir", type=str, help="output directory") parser.add_argument("--obs", type=str, help="observatory (lco|apo)",default='lco') parser.add_argument("--vers", type=str, help="BOSS_SPECTRO_REDUX version",default='master') parser.add_argument("--threads", type=int, help="number of threads",default=8) parser.add_argument("--nskip", type=int, help="use every nskip line",default=40) - parser.add_argument("--clobber", action="store_true", - required=False, help="clobber?") + parser.add_argument("--cams", type=str, help="Supply the camera for operation with SOS files", default=None) + parser.add_argument("--fitsname", type=str, help="Supply the FitsName for SOS error reporting", default=None) + parser.add_argument("--sosdir", type=str, help="Base SOS output directory", default=None) + parser.add_argument("--clobber", action="store_true", required=False, help="clobber?") + parser.add_argument('--no_hash', default=False, action='store_true', help='Skip updating the file hash') args = parser.parse_args() - + + if args.vers.lower() == 'sos': + args.vers = '' + sos = True + if args.sosdir is not None: + os.environ["BOSS_SPECTRO_REDUX"] = ptt.join(f'{args.sosdir}',f'{args.mjd}') + else: + sos = False + matplotlib.use('Agg') - boss.arc_transform(args.mjd, obs=args.obs, clobber=args.clobber, threads=args.threads, - outdir=args.outdir, vers=args.vers) + if not sos: + try: + boss.arc_transform(args.mjd, obs=args.obs, clobber=args.clobber, threads=args.threads, + outdir=args.outdir, vers=args.vers, cams=args.cams) + except Exception as e: + print(type(e).__name__, ":", e) + exit() + print(f'Successful completion of boss_arcs_to_trace for mjd {args.mjd} obs {args.obs} at {time.ctime()}') + else: + logfile = ptt.splitext(ptt.splitext(ptt.basename(args.fitsname))[0])[0]+'.log' + logfile = logfile.replace('sdR', 'spTraceTab') + logfile = ptt.join(f'{args.sosdir}',f'{args.mjd}','trace',f'{args.mjd}',logfile) + log = Logger(logfile) + log.start(cmd = sys.argv) + cmd = sys.argv + cmd[0] = ptt.basename(cmd[0]) + try: + boss.arc_transform(args.mjd, obs=args.obs, clobber=args.clobber, threads=args.threads, + outdir=args.outdir, vers=args.vers, cams=args.cams) + except Exception as e: + print(type(e).__name__, ":", e) + + fitsname_base = ptt.splitext(ptt.splitext(ptt.basename(args.fitsname))[0])[0] + log.stop() + log.start() + try: + if grep(logfile, f'error with {fitsname_base}'): + message = f"BOSS_ARCS_TO_TRACES: {args.fitsname}: ABORT: Failure in Fitting Arc to Traces (Please retake calibrations for this field)" + print(message) + report(args.fitsname, args.cams, args.obs, args.mjd, message) + + soshtml(args.mjd, args.obs, args.sosdir) + except Exception as e: + print(type(e).__name__, ":", e) + log.stop() + if not args.no_hash: + test = create_hash(ptt.join(args.sosdir,str(args.mjd))) + if test: + print("\nsha1sum is locked") + sys.exit(0) + diff --git a/bin/build_combined_html b/bin/build_combined_html new file mode 100755 index 000000000..3fc3c4734 --- /dev/null +++ b/bin/build_combined_html @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +from boss_drp.sos.build_combined_html import build_combine_html +import argparse + + +if __name__ == '__main__': + """ + Build sos/combined/index.html + """ + parser = argparse.ArgumentParser(description="build SOS combine index page") + parser.add_argument("sosdir", type=str, help="Base SOS output directory", default='/data/boss/sos') + parser.add_argument("--force", help="Force update", action='store_true') + args = parser.parse_args() + + build_combine_html(args.sosdir, force=args.force) diff --git a/bin/build_combined_html.py b/bin/build_combined_html.py deleted file mode 100755 index ebb97dfa7..000000000 --- a/bin/build_combined_html.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -from glob import glob -import os.path as ptt -from os import rename -import time -def get_mjd(html): - filename = ptt.splitext(ptt.basename(html))[0] - int_part = filename.split('-')[1] - return(int(int_part)) - -def build_combine_html(): - directory = '/data/boss/sos/combined/' - if ptt.exists(ptt.join(directory,'index.html')): - file_time = ptt.getmtime(ptt.join(directory,'index.html')) - if time.time()-file_time < 3600*15: - print('No update required') - return - with open(ptt.join(directory,'index.html.tmp'), 'w') as f: - f.write('\n') - f.write('
\n') - f.write('-This document explains how to install the idlutils,idlspec2d,specflat products -needed for reducing SDSS spectroscopic data. The current (Nov 2003) versions -of the code are: -
- idlutils v5_3_0 - specflat v1_5 - idlspec2d v5_2_0 -
-In this document: -
- source /usr/peyton/common/licensed/idl/bin/idl_setup - setenv IDL_PATH .:+{$IDL_DIR}/lib -
- IDL_DIR=/usr/peyton/common/licensed/idl - IDL_PATH=.:+{$IDL_DIR}/lib - alias idl=$IDL_DIR/bin/idl -
- % setenv CVS_RSH ssh - % setenv CVSROOT :pserver:anonymous@sdsscvs.astro.princeton.edu:/usr/local/cvsroot -
- % CVS_RSH=ssh - % CVSROOT=:pserver:anonymous@sdsscvs.astro.princeton.edu:/usr/local/cvsroot -
- cvs login - (hit a carriage return when it asks for the "CVS password") -
- cvs export -d evilups-temp -r v0_7_14 evilups - setenv PROD_DIR_PREFIX $HOME/products - setenv PRODUCTS $PROD_DIR_PREFIX/ups_db - setenv EUPS_DIR $PROD_DIR_PREFIX/evilups - cd evilups-temp - make clean - make install - setenv EUPS_FLAVOR `/bin/uname` - source $EUPS_DIR/bin/setups.csh --
- % svn co https://projects.lbl.gov/svn/idlutils/tags/v5_3_0 $HOME/products/idlutils/$EUPS_FLAVOR/v5_3_0 - % cd $HOME/products/$EUPS_FLAVOR/idlutils/v5_3_0 - % bin/evilmake - % eups_declare -f $EUPS_FLAVOR -r $EUPS_PATH/$EUPS_FLAVOR/idlutils/v5_3_0 -c idlutils v5_3_0 -
- % evilinstall specflat v1_5 ; setup specflat - % evilinstall idlspec2d v4_10_6 ; setup idlspec2d -
-That's it! Note that "evilinstall" should work for any survey code, -not just the Spectro products. - - -
- % bash -
- svn co https://projects.lbl.gov/svn/idlutils/tags/v5_3_0 idlutils -
- % cvs -d :pserver:anonymous@sdsscvs.astro.princeton.edu:/usr/local/cvsroot login -
- % CVS_RSH=ssh - % CVSROOT=:pserver:anonymous@sdsscvs.astro.princeton.edu:/usr/local/cvsroot -
- % cvs co -r v1_5 specflat - % cvs co -r v5_2_0 idlspec2d -
- % IDLUTILS_DIR=$HOME/idlutils - % SPECFLAT_DIR=$HOME/specflat - % IDLSPEC2D_DIR=$HOME/idlspec2d - % PATH=$IDLUTILS_DIR/bin:$SPECFLAT_DIR/bin:$IDLSPEC2D_DIR/bin:$PATH - % IDL_PATH=+$IDLUTILS_DIR/goddard/pro:$IDL_PATH - % IDL_PATH=+$IDLUTILS_DIR/pro:$IDL_PATH - % IDL_PATH=+$IDLSPEC2D_DIR/pro:$IDL_PATH -
- % cd $IDLUTILS_DIR - % evilmake all - % cd $SPECFLAT_DIR - % evilmake all - % cd $IDLSPEC2D_DIR - % evilmake all -
- % tar xvf idlutils*.tar - % tar xvf specflat*.tar - % tar xvf idlspec2d*.tar -
These are the instructions for running the spectroscopic pipeline on -the raw SDSS data. You must first install -the IDL code, which are the three products: idlspec2d, idlutils, -specflat.
-The directory stucture for spectroscopic data is described by environment -variables as follows:
--$BOSS_SPECTRO_DATA/mmmmm - The raw image, "sdR-cc-eeeeeeee.fit" -$SPECLOG_DIR/mmmmm - The plug-map files, "plPlugMapM-pppp-mmmmm-rr.par" --
where mmmmm refers to an MJD (such as 51690), cc refers -to a camera name (such as b1 for blue-1), pppp refers to a plate -number (such as 0306), eeeeeeee refers to an exposure number -(such as 00003974), and rr refers to a fiber-mapper re-run number -(such as 01).
-At Princeton, these paths would be set as follows:
--BOSS_SPECTRO_DATA=/u/dss/rawdata -SPECLOG_DIR=/u/dss/astrolog --
To re-run a plate through the spectro pipeline, you need all of these files. -Both paths must be set appropriately.
- -The raw images for a particular night ($MJD) can be found at:
--sdssdata.astro.princeton.edu:/u/dss/rawdata/$MJD/sdR*.fit --
The plug-map files for a particular night ($MJD) can be found at:
--sdssdata.astro.princeton.edu:/u/dss/astrolog/$MJD/plPlugMapM*.par --
These are also available from the CVS product "speclog".
- -My convention is to put the reduced data in a directory separate from -the data where the top-level directory is $BOSS_SPECTRO_REDUX. At Princeton, -this would be set to
--BOSS_SPECTRO_REDUX=/u/dss/spectro --
Each plate is put in its own subdirectory, so the reductions of plate 306 -would be in "/u/dss/spectro/0306".
-Before running the spectro pipeline, you need to build plan files for -each plate. Create the output directory $BOSS_SPECTRO_REDUX. From there, -build the plan files...
--IDL> spplan2d -IDL> spplan1d --
This could take an hour to build plan files for all the data taken to date. -However, you can limit this to particular nights of data by setting -keywords to these procedures (see the full documentation).
-The spplan2d command builds the files "spPlan2d-pppp-mmmmm.par". -There is one such file for each night a plate is observed.
-The spplan1d command builds the files "spPlancomb-pppp-mmmmm.par". -This file merges exposures from multiple nights of observations of the same -plate if those observations were taken without re-plugging the plate. -If the plate was re-plugged between nights, then a given fiber will correspond -to different objects in each night, and those nights' data shouldn't be -combined with "spcombine".
-Note that these plan files are ASCII files (in something called a Yanny -parameter format) which can be hand-edited. That way, you can exclude -particular exposures from a reduction by commenting-out lines with hash -marks (#).
- -It takes approximately 3.5 hours to run one plate through Spectro-2D on -a 1-GHz Pentium-III, and another 8 hours to run Princeton-1D.
-In each output plate directory, you can run the following three commands -from the IDL prompt:
--IDL> spreduce2d -IDL> spcombine -IDL> spreduce1d --
The spreduce2d command reduces individual exposures to -"spFrame-cc-eeeeeeee.fits" files.
-The spcombine command combines those exposures into the reduced plate -file, "spPlate-pppp-mmmmm.fits".
-The spreduce1d command finds the redshifts, and generates the file -"spZbest-pppp-mmmmm.fits".
-A number of other supplementary files are also produced. The history -of the reductions are written to log files named "spDiag*.log", and -some PostScript plots are written to "spDiag*.ps".
- -For example, to reduce plate 306 from the command line,
--echo "spreduce2d, 'spPlan2d-0306-51690.par'" | idl >& /dev/null & -- -
We use an IDL script BATCH2D -for batch processing many plates at once, which in turn calls -DJS_BATCH:This script will run -jobs across local or remote networks using rsh -or ssh protocols. For a remote machine, the raw data files are shipped -across the network, the plate is reduced, then the reductions are shipped -back. Presumably, this would work just fine on the Fermi farms. The plan -files need to be built before running this script. Also make certain -that the remote machines have their UPS environment and the idlspec2d -product set up from the ".bashrc" file, since the remote commands are -launched from the bash shell.
-There is a Spectro-Robot that automatically fetches data, builds plan files, -and reduces it on a day-by-day basis. The command "sprobot_start" loads -the cron job. The raw data is copied to the first disk with space listed -in the SPROBOT_LOCALDISKS environment variable, then a link is built from -$BOSS_SPECTRO_DATA/$MJD to that directory. At Princeton, the disk list is -something like:
--SPROBOT_LOCALDISKS='/scr/spectro1/data/rawdata /scr/spectro2/data/rawdata' -- -
Other environment variables that need to be set for the Spectro-Robot:
-If any of the above variables are not set, then -"sprobot_start" will issue an error message and fail to load. A log file -is written to the file "$BOSS_SPECTRO_DATA/sprobot.log".
-There are two Yanny parameter files that list the computer names and protocols -to use. There are default files in "$IDLSPEC2D_DIR/examples/batch2d.par" -and "$IDLSPEC2D_DIR/examples/batch1d.par" for use with Spectro-2D and P-1D -respectively. You can over-ride these default files by putting files with -the same names in the directory $BOSS_SPECTRO_REDUX.
-The Spectro-Robot commands:
--sprobot_start -- Start the Spectro-Robot. -sprobot_status -- See if the Spectro-Robot is running. -sprobot_stop -- Stop the Spectro-Robot. --
Finally, if one wished to *not* run Princeton-1D, then the line containing -"sprobot1d.sh" would have to be removed from the file "sprobot.sh".
--In this document: -
-Related documents: -
- - -Son-of-spectro is a set of daemon jobs that runs from the observer account on -sos3.apo.nmsu.edu. The actual processes are named sos_runnerd.py and there will be four -instances running, one per camera. Although sos_runnerd.py can be started manually, users -will seldom if ever start the process using this command. Instead, there are two user -commands that know about the setup at APO. They are sos_apocontrol and sos_apostart. -sos_apocontrol is the highest level command and should be used most often. sos_apocontrol -allows for the starting, stopping and monitoring of the daemon processes using the -standard APO options. In order to start the processes, sos_apocontrol calls sos_apostart. -sos_apostart can be called manually and allows more flexibility. Each of those commands -will start four instances of sos_runnerd, one per camera, with the correct parameters for -APO.
- -At APO the log files are stored in ~observer/sos/sos-logs and there is one log file per -camera for all commands and one log file per camera that only stores errors. It is a -good idea to periodically check the error log. Each log file is rotated out daily and -saved for four days.
- -The sos daemon processes can not be automatically started at boot because they need a -password to be manually entered in order to access svn. Because of this, starting the -sos daemons should be part of the SOS3 boot procedures document.
- -There is also a redo command, sos_aporedo that allows one nights of data to be manually -rerun trough aporeduce. Again, sos_aporedo will start four processes, one per camera, -to reprocess the images.
- -For more information be sure and read the sos daemon -readme -file. Also, each command has usage information via the '-?' option. Finally, for even -more information each source file has a document header at the beginning of the file -with implementation details.
- -
- -
-$ sos_apocontrol status
-
- -There should be four instances of sos_runnerd -displayed, one per camera. The first line will be "N processes running", -the last line will be "sos_apocontrol finished running!". -Between those lines should be four instances of sos_runnerd. Each instance will be a long -line separated by a blank line.
- -If you do not see see four processes running, stop the processes and then start the -processes.
- -
-$ sos_apocontrol stopWait
-
- -If sos is processing an image, this command will allow the current idl process to finish -before stopping sos. If this command does not return after some time (~5 min), then -^C to kill the process and use the following command: - -
-$ sos_apocontrol stopKill
-
- -Then confirm the sos processes are not running.
- -
-$ sos_apocontrol status
-
- -There should be zero instances of sos_runnerd displayed. The first line will be -"0 processes running", -the last line will be "sos_apocontrol finished running!". -There should be nothing between those two lines.
- -
-$ sos_apocontrol start
-
-When sos_apocontrol is run it will check to make sure that there is an ssh-agent that -it can connect to for svn access. If there is not, it will ask for the password of -the key it uses. The password is the same as is used for the sdss3 data access web site. -
- -This password request is OK:
--Checking for agent file /home/observer/sos/control/agent.info -Found agent file -Checking agent pid 5142 -Agent has stopped running -Starting new agent. -ssh-agent -a /home/observer/sos/control/agent.socket >/home/observer/sos/control/agent.info - -About to add the private key sos uses, please enter the key password. -It is the old SDSS password, not the new SDSS3 password. - -ssh-add /home/observer/.ssh/id_dsa-sos -Enter passphrase for /home/observer/.ssh/id_dsa-sos: -- -
-One the agent is running, it should not require further passwords. -If you get the following message and a prompt for a password, exit -with control-C -
-This password request is NOT OK:
--This is a final check to avoid authentication problems which will trigger -LBL to block sos3 from accessing svn. DO NOT try to type in the password. -This is a test which must succeed without the need to type in a password. -Only type ^C to abort the script. - -svn log /home/sdss3/products/NULL/speclog/trunk >/dev/null -Enter passphrase for key '/home/observer/.ssh/id_dsa-sos': -- -
-If you get that password request, type control-c as it instructs, and -see the troubleshooting section below. -
- -When the command finishes, confirm the sos processes are running.
-
-$ sos_apocontrol status
-
-Use this command if the sos process was not started until after observing has started -for the night. This procedure will process all images already present for the current -MJD.
- -
-$ sos_apocontrol allow
-$ sos_apostart -g -e
-
- -Then confirm the sos processes are running.
- -
-This command must be run in a directory with write permissions, e.g. -/home/observer/sos/sos-logs/. -
- -To reprocess exposure EXPIDXX on MJD MJDXX -
-$ cd /home/observer/sos/sos-logs/
-$ sos_aporedo -g -m MJDXX -z EXPIDXX
-
-$ cd /home/observer/sos/sos-logs/
-$ sos_aporedo -w -g -m 00000 (Where 00000 is the MJD to reprocess)
-
- -The output files are stored in ~/sos/redo. cat ~/sos/redo/redo-b1.out for the output of -the reprocessing of the b1 camera.
- -
-$ ps ax | grep ssh-agent
-
- -This line should be present:
-
-"ssh-agent -a /home/observer/sos/control/agent.socket"
-
- -If it is not, then stop the sos processes and restart them.
- -
- -For example, to stop the current version, set up v5_4_15, then re-start: -
-$ sos_apocontrol stopWait
-$ sos_apocontrol status
- --> should say "No processes running!"
-$ setup idlspec2d v5_4_15
-$ sos_apocontrol start
-$ sos_apocontrol status
- --> should show 4 "sos_runnerd" processes running
-
-At APO, install both on the SOS machine (currently sos3.apo), and -on sdsshost.apo. This is because the documentation on the sdsshost.apo -procedures web page is linked to the installed version on that machine. - -
-Now you are ready to launch this new version of the code. - -
-Son-of-Spectro is a daemon job that runs from the "observer" account -on "sos3.apo.nmsu.edu". This daemon job monitors files written to /data/spectro/$MJD and -then launches a package of shell scripts and IDL commands to reduce -the data one frame at a time. It is part of the idlspec2d data product. - -
There are actually four daemon processes that run, one for each camera. We did -this so that both processors on the computer can be utilized. All 4 cameras should be -reduced for an exposure within a few minutes. - -
-The output directors are: -
- /data/boss/sos/combined - Copy of aporeduce output for the web site - /data/boss/sos/$MJD - Output of aporeduce for that night - $SPECLOG_DIR/$MJD - plugmap files used for running aporeduce - platedb database - (s/n)^2 for each science frame reduced -
-The data reduction is not the same as the full-up spectroscopic pipeline. -A number of shortcuts are taken to speed up the reductions and make them -very robust. For example, boxcar extraction of the spectra is used instead -of optimal extraction. - -
-Files like the following can be found in the /data/boss/sos/$MJD directory: -
- splog-b1-00006541.log - Text history of exp #6541 reduction - tset-51795-0389-00006541-b1.fits - Fiber traces (from flat exposure #6541) - wset-51795-0389-00006542-r1.fits - Wavelength solution (from arc exp #6542) - fflat-51795-0389-00006542-b1.fits - Flat-field vectors (w/ arc exp #6542) - sci-0389-b2-00006546.fits - Science spectra for plate 389 (exp #6546) - logfile-51795.fits - Summary info for night in FITS format - logfile-51795.html - Summary info for night in HTML format - snplot-51795-0389.ps - Signal-to-noise plot for nplate 389 -
-The HTML files and S/N plots for the web page are copied into the directory: -
- /data/boss/sos/combined -
-The following FITSheader cards are required to be correct for either -Son-of-Spectro or Spectro-2D to reduce the data properly. -The AIRMASS is not read from the header, but computed from RADEG,DECDEG -and the TAI-BEG,TAI-END keywords. -
-EXPOSURE= Exposure number. We always override the value of this keyword - with the exposure number in the file name. -CAMERAS = Camera name, e.g. "b1". We always override the value of this keyword - with the exposure number in the file name. -FLAVOR = The flavor of the observation, which can be "bias", "dark", - "arc", "flat", and "science" or "target". Files with other - flavors, such as "unknown", are not reduced. -MJD = MJD of observation, which must agree with the directory in which - the file resides, "/data/spectro/$MJD". -PLATEID = The plate ID, which must match that in the NAME header card. -NAME = Plug name, e.g. "0328-52277-01" for plate 328, plugged on 52277 - with a plugging ID of 01. We assume that there is a plug-map - file at APO with the name "/astrolog/$MJD/plPlugMapM-$NAME.par". -EXPTIME = The exposure time, which is used along with TAI-BEG to compute - the airmass. It is also used to determine whether the sky - brightness or scattered light is larger than tolerances. -TAI-BEG = The TAI time for the beginning of the exposure, which is used - to compute the airmass and sky-brightness gradients for sky- - subtraction. If this keyword is missing, we guess it from - EXPTIME and TAI. -TAI-END = Used with TAI-BEG. If this keyword is missing, we guess it from - EXPTIME and TAI. -TAI = This keyword is used if TAI-BEG,TAI-END are missing. -FFS = Flat-field screen positions, which should be - "0 0 0 0 0 0 0 0" for science exposures (all petals open) - "1 1 1 1 1 1 1 1" for flat or arc exposures (all petals closed) -FF = Flat-field (quartz) lamp status, which should be - "0 0 0 0" for science or arc exposures (all off) - "1 1 1 1" for flat exposures (all on) -NE = Neon lamp status, which should be - "0 0 0 0" for science or flat exposures (all off) - "1 1 1 1" for arc exposures (all on) -HGCD = HgCd lamp status, which should be - "0 0 0 0" for science or flat exposures (all off) - "1 1 1 1" for arc exposures (all on) -OBSCOMM = This keyword is used to identify the dithered flat sequence - taken in the Monthly Checkout with the "specFlats" script in SOP. - The flats in this sequence must have OBSCOMM="{dithered flats-flat}". - The arcs in this sequence must have OBSCOMM="{dithered flats-arc}". - This keyword also identifies the Hartmann focus exposures, with - the entries OBSCOMM="{focus, hartmann l}" or "{focus, hartmann r}". - Science exposures all have OBSCOMM="science ". -QUALITY = The observer-input quality for an exposure: "excellent", "test", - or "bad". Exposures are declared "test" or "bad" using the - APOFIX procedure. -- -
-The following FITS header cards are informational. We would very much -appreciate the observers to correct errors in these keywords so that -our book-keeping of survey progress is done properly: -
-TILEID = Tile ID for this plate from the plug-map file -CARTID = Cartridge ID number (1 through 9) -RA = Right ascension of telescope boresight (in degrees) -DEC = Declination of telescope boresight (in degrees) -RADEG = Right ascension of plate center for plug-map file (in degrees) -DECDEG = Declination of plate center for plug-map file (in degrees) -AIRTEMP = Temperature (deg C) -- -
-Incorrect FITS header cards for the raw spectro sdR files can be -corrected by adding entries in the sdHdrFix file -"/astrolog/$MJD/sdHdrFix-$MJD.par". The interesting header cards -from these files can be listed with the -APOHEADER -procedure run from the IDL prompt on sos.apo. The proc -APOFIX -can be used to add entries to the sdHdrFix file to denote edits. - -
-This structure should have the following definition in the sdHdrFix file: -
- typedef struct { - char fileroot; # Root of file name, without any ".fit" suffix - char keyword; # Keyword name - char value; # Keyword value (as a string) - } ophdrfix; -- -
-Following are some examples of what should appear in the sdHdrFix files -in order to correct faulty headers. (The typedef struct above must -also be in the file.) -
- ophdrfix sdR-b1-00001142 EXPTIME 60 --
- ophdrfix sdR-??-00001234 FLAVOR "unknown" --
- ophdrfix sdR-??-0000113[7-9] NAME "0198-51433-01" --
- ophdrfix sdR-??-00010850 FFS "1 1 1 1 1 1 1 1" - ophdrfix sdR-??-00010850 FF "1 1 1 1 " - ophdrfix sdR-??-00010850 NE "0 0 0 0 " - ophdrfix sdR-??-00010850 HGCD "0 0 0 0 " --
-If you have edited the sdHdrFix file and need to re-reduce an exposure, -you can do so with the sos_redo command. -When Son-of-Spectro reads in the raw FITS files, it then patches the -headers with the information in the ophdrfix entries of the sdHdrFix file. -But note that the one piece of information that cannot be patched this -way is MJD, since we need the MJD in order to find the correct sdHdrFix file!! - -
-Son-of-Spectro reduces four flavors of observations: -bias/dark, flat, arc, & science/smear. Select information is tabulated -for each of these types of observation. These values are tabulated in -yellow -if they are going out of spec, and in -red -if they are very much out of spec. - -
-The values reported are: -
-The exact values of these yellow/red limits and further explanation can -be found in the "idlspec2d" product in the file "examples/opLimits.par". - -
-There are a number of -WARNING -and -ABORT -messages that can appear if the pipeline runs into trouble when processing -a frame. These messages appear at the bottom of each table. -Each one-line message begins with the relevant file name, WARNING or ABORT, -then a brief plain-text message. - -
-Note that a single problem may cascade into a large number of warning -messages. For example, an out-of-focus spectrograph will first produce -the "Median spatial widths" message, probably followed by warnings about -bad sky-residuals. - -
-The following messages may appear: -
-If the reduction of an exposure is catastrophically bad, it may not -appear at all in the Son-of-Spectro table. However, there should still -be a log file for this exposure on the sos.apo computer: -
- /data/spectro/spectrologs/$MJD/splog-$CAMERA-$EXPOSURE.log -
-A median signal-to-noise is computed for each object in the -wavelength ranges [4000,5500] Angstroms (synthetic g-band) -and [6910,8500] Angstroms (synthetic i-band). -We plot these S/N values versus the PHOTO fiber magnitudes, which were -measured in approximately a 3-arcsec diameter aperature (the same size -as our fibers). If everything is working perfectly, then our S/N values -should correlate very well with these PHOTO magnitudes. - -
-We determine whether a plate is "done" based upon the signal-to-noise -of the fainter objects on the plate. We do this by fitting a line to -the (S/N)-vs.-magnitude plot in a specified wavelength range, then evaluating -this fit at g=20.2 mag (blue CCDs), and i=19.9 mag (red CCDs). -Approximately 10% of the main galaxy sample, 60% of the BRG's and 15% of -the QSO's are fainter than g=20.2. Very few objects (1, 1, and 5%) are -fainter than i=19.9. - -
-When the sky level is higher, we gain S/N more slowly at -the fainter magnitudes where we are sky-limited rather than photon-limited. -Without moon, we have typically found: -
- log(S/N_g) = (zeropoint) - 0.31 * g - log(S/N_i) = (zeropoint) - 0.31 * i -
-The fitting regions are denoted on the plot with vertical dotted lines. -Arrows point to the evaluation of the fit on each of the 4 cameras, with -the top panels corresponding to the blue CCD's (synthetic g-band) and -the bottom panels corresponding to the red (synthetic r-band). -The thick blue line on the figure is only a meaningless, fixed reference line. - -
-The right-hand figures plot the residuals of each object from the fit. -Symbol sizes on those right-hand plots indicate the magnitude of the -deviation from the fit line. Symbol color is the same on the left -as on the right, so -green -objects have more flux and -red -ones less. -If the scale of the telescope is wrong, then you will see a radial drop-off -in flux (red points on the edge of the plate). -If you are observing too far over in air mass, then typically you correct -to first order with a scale change, but a quadropole is left in these -residuals. -If one spectrograph has problems, then this will show up as red points -in half of one of these figures. - -
-Note that the (S/N)^2 totals listed in the table and the figure might -not exactly agree. This is because the fitting to (S/N)-vs.-magnitude -is done on individual frames for the table, but on the summed S/N over -all frames for the figure. The tabulated values are the ones we use -to declare a plate done. - -
-The historic Son-of-Spectro (SOS) reduction robot has proven to be quite robust. -However, if it appears to not be working one could check the following: -
-- % $IDL_DIR/bin/lmstat -a -
This is from version $Name$ of procedures.
- -
Maintained by -David Schlegel -and Scott Burles - - - - diff --git a/doc/www/spectro_apodisplay.html b/doc/www/spectro_apodisplay.html deleted file mode 100644 index ddd558c0b..000000000 --- a/doc/www/spectro_apodisplay.html +++ /dev/null @@ -1,206 +0,0 @@ - - - --In this document: -
-The fSAO display tool can be used to display the raw spectro images. -This can only display the raw images, without overscan-correction. -This means that there will be a baseline value of between 500 and 1250 ADU -on each of the amplifiers. You will clearly see the split in overscan -values between the left and right amplifiers. - -
-From within SOP (or IOP), type the following (substituting for $MJD -and $filename as appropriate): -
- sop> cd /data/spectro/$MJD - sop> set img [ regReadAsFits [regNew] $filename ] - sop> saoDisplay $img -
- sop> saoDisplay $img "-min 1000 -max 1300" -
-I've set up IDL on sdsshost.apo (and on sos.apo, plate-mapper.apo), -for use in displaying spectro images. Note that since all the spectro -data is automatically copied to sos.apo (by the Son-of-Spectro cron job), -you can display images on either machine. - -
-First, set up IDL and the idlspec2d product, and start up IDL: -
- % setup idlspec2d - % idl -
-Move into the directory for MJD 51886 and display the first image: -
- IDL> cd,'/data/spectro/51886' - IDL> img=rdss_fits('sdR-b1-00007538.fit') - IDL> atv,img -
-There is another tool built on top of ATV that will overplot the bad -column mask in red: -
- IDL> atvrawspec,'sdR-b1-00007538.fit' - IDL> atvrawspec,'sdR-b1-00007538.fit',min=0, max=20 ; <-- Force display range -
-To read in a raw image and median-filter before displaying, -
- IDL> img=rdss_fits('sdR-b1-00007538.fit') ; <-- Read a raw image - IDL> sdssproc,'sdR-b1-00007538.fit',img ; <-- Or read + overscan-correct - IDL> atvmedian(img,9) -
-Another useful command is -
- IDL> logsheet -
-The SOS-extracted spectra can be displayed using IDL tools on -the computer sos.apo. For the full documentation, see -http://spectro.princeton.edu/idlspec2d_doc.html#APOPLOT. - -
-You must be logged onto sos.apo (or whichever machine is -running Son-of-Spectro). Set up the idlspec2d product and start IDL with: -
-setup idlspec2d -idl -
-IDL> apoplot, 401, 100 -
-IDL> apoplot, 401, 100, nmed=9 -
-IDL> apoplot, 401, /magsort, /netimage -
-IDL> apoplot, 401, 100, mjd=52282 -
-IDL> apoplot, 401 -
-Quitting IDL: -
-IDL> exit -
-There's another useful (but slow!) command for analyzing bias and dark frames. -This is the APO_PLOTBIAS procedure. It takes as its only argument -a spectroscopic exposure number, then plots a histogram of bias values -for all 4 cameras of that exposure: -
-IDL> apo_plotbias, 7538 -
-IDL> apo_plotbias, 7538, plotfile='bias.ps' -
-After making its plot, the APO_PLOTBIAS procedure prints out a select -number of percentiles. The 2-percentile is the value at which only 2% -of the pixels are below, the 50-percentile is the value at which 50% -of the pixels are below (e.g., the median), etc. - -
-I find the typical percentiles for a bias to be: -
-Filename 02% 05% 10% 50% 90% 95% 98% ----------------- ---- ---- ----- ---- ---- ----- ---- -sdR-any-exposure -8 -5 -3 1 7 9 11 -
-This page should get anyone started using the SDSS spectra within minutes. -There are instructions for anyone to get the -publicly-available data, -or for SDSS collaborators to get the -still-proprietary data. -This page describes tools to view, query and use these data using IDL, -SuperMongo or IRAF. There are also summary ASCII files -that contain positions, magnitudes, classification information, and redshifts. - -
-
-David Schlegel, -schlegel@astro.princeton.edu - - --In this document: -
-Related documents: -
-Science web sites: -
- --Technical web sites: -
-Technical mailing lists (restricted access): -
-Plot of the cumulative number of survey-quality -spectra. - -
-Here are some nifty plots of only those objects within 2 degrees of the -equatorial plane: -
-The SDSS will observe approximately 2000 plates by the end of -its 5 year survey. Each plate is physically an aluminum disk 60 cm -in diameter that subtends 3 degrees on the sky. -640 fibers are plugged into each plate at the location of objects chosen from -the photometric survey, allowing us to spectroscopically observe 640 -objects at time. Typically, we integrate for 45 minutes on each plate. - -
-Since plates are occasionally re-observed, the plate number plus the -observation date uniquely identifies an observation. We track the -date with modified Julian date (MJD). In these re-observations, -the optical fibers may be plugged into different holes, so fiber #1 on -plate 306 won't be the same on the MJD=51637 observation as on MJD=51690. - -
-Therefore, PLATE + MJD + FIBERID uniquely determines a single observation -of a single object. The TILE number uniquely determines a position on the -sky, and there are a few instances where two PLATEs are drilled for one TILE. - -
-This list of plates is auto-updated each day based upon which reductions -have completed. It is available in the following formats: -
-Aside from PLATE, TILE and MJD, this list contains the following: -
-RA = Right ascension of plate center [J2000 degrees]. -DEC = Declination of plate center [J2000 degrees]. -MAPNAME = Mapping name for this plate; data with the same mapping name - are always combined into one plate file, even if taken on - different nights. -VERSLOG = Version of speclog product (spectroscopic log files). -VERSUTIL = Version of idlutils product. -VERSFLAT = Version of specflat product (spectroscopic pixel flats). -VERSREAD = Version of Spectro-2D code for pre-processing raw data. -VERS2D = Version of Spectro-2D code to reduce the individual frames. -VERSCOMB = Version of Spectro-2D code to combine frames into a plate file. -VERS1D = Version of Princeton-1D code to measure redshifts. -VERSTARG = Version of Target code for selecting the objects on this plate. -CHUNKNAME= The name of the Target and tiling run which produced this plate. -NEXP = Total number of frames from all cameras used in the co-adding; - since there are 4 cameras, NEXP=12 would typically mean that there - were 3 exposures in each camera. -NEXP_B1 = Number of exposures in b1 camera. -NEXP_B2 = Number of exposures in b2 camera. -NEXP_R1 = Number of exposures in r1 camera. -NEXP_R2 = Number of exposures in r2 camera. -EXPT_B1 = Total exposure time in b1 camera. -EXPT_B2 = Total exposure time in b2 camera. -EXPT_R1 = Total exposure time in r1 camera. -EXPT_R2 = Total exposure time in r2 camera. -SN2_G1 = (S/N)^2 per pixel at g=20.2 on spectrograph #1. -SN2_G2 = (S/N)^2 per pixel at g=20.2 on spectrograph #2. -SN2_R1 = (S/N)^2 per pixel at r=20.25 on spectrograph #1. -SN2_R2 = (S/N)^2 per pixel at r=20.25 on spectrograph #2. -SN2_I1 = (S/N)^2 per pixel at i=19.9 on spectrograph #1. -SN2_I2 = (S/N)^2 per pixel at i=19.9 on spectrograph #2. -N_GALAXY = The number of objects Princeton-1D called a galaxy. -N_QSO = The number of objects Princeton-1D called a QSO. -N_STAR = The number of objects Princeton-1D called a star. -N_UNKNOWN= The number of objects Princeton-1D declared spectroscopically - unknown (a ZWARNING flag is set) -N_SKY = The number of sky fibers (almost always exactly 32) -N_TARGET_MAIN = Number of objects targetted for the main galaxy survey. -N_TARGET_LRG = Number of objects targetted for the LRG galaxy survey. -N_TARGET_QSO = Number of objects targetted for the QSO survey. -SUCCESS_MAIN = Percentage success rate for objects targetted as "main" galaxies - and spectroscopically classified as either "GALAXY" or "QSO". -SUCCESS_LRG = Percentage success rate for objects targetted as LRG galaxies - and spectroscopically classified as "GALAXY". -SUCCESS_QSO = Percentage success rate for objects targetted as QSOs - and spectroscopically classified as "QSO". -STATUS2D = Status of Spectro-2D pipeline for reducing individual exposures; - "Done" if done. -STATUSCOMBINE = Status of Spectro-2D pipeline for combining individual - exposures into the spPlate files; "Done" if done. -STATUS1D = Status of Princeton-1D pipeline; "Done" if done. -QSURVEY = Status flag set to 1 for the best observation of any tile on the sky - that satisfies the following: (S/N)^2 > 13.0 for G1,G2,I1,I2, and - the target version cannot be "special" or "devel". -PUBLIC = Name of public data release; presently plates are either not public, - or part of the Early Data Release (EDR) or Data Release 1 (DR1). -- -
-The values of SN2 give an estimate of the signal-to-noise (per pixel) of a plate -at a fiducial magnitude. The fiducials are set to g=20.2, r=20.25, i=19.9. -The measurement is of the signal-to-noise squared, per pixel, for an object -at that magnitude. -All these values must be > 13 for a plate to be "marginal". -All these values must be > 15 for a plate to be "good". - -
-The best effort to avoid duplicate objects is to make use of -the SPECPRIMARY field in the spAll summary file. -That selects the best observation of any given object. - -
-Selecting only those plates with QSURVEY=1 will yield non-duplicate -observations of tiles on the sky that meet the survey requirements -for signal-to-noise. There still may be a few duplicate objects, -as some objects are put on multiple tiles for "quality assurance". -These duplicates can be identified with OBJTYPE="QA". That OBJTYPE -column exists either in HDU #5 of the spPlate files, -or in a column by that name in the spAll summary files. - -
-Note that the above statements are almost but not exactly true. -There were some target-selection errors that resulted in duplicated -targets not labelled as "QA" (notably in runs 94 and 125; see -sdss-spectro/710). -The following IDL commands can be used to select a truly unique set -of objects: -
- IDL> columns = ['PLATE','FIBER','MJD','OBJTYPE','PLUG_RA','PLUG_DEC'] - IDL> spall = hogg_mrdfits('spAll-qsurvey.fits',1,columns=columns) - IDL> spall = spall[where(strmatch(spall.objtype, 'QA*') EQ 0)] - IDL> junk = djs_angle_group(spall.plug_ra, spall.plug_dec, 1./3600, $ - gindx=gindx, gstart=gstart, gcount=gcount) - IDL> spall = spall[gindx[gstart]] -
-The version of the code used in the reductions is specified in the -header keywords VERS2D, VERSCOMB, VERS1D as described in the -data model. - -
-All these reductions were done using versions of the code tagged v4_9_8 -through v4_10_6, which represents a fairly homogeneous set of reductions. - -
-The Princeton re-reductions of the "public release" data is available -from this web site. The data are grouped into disjoint samples of when -the data became available: -the Early Data Release (EDR) with 139 plates (88,960 spectra), -Data Release 1 (DR1) adds to this 192 plates (122,880 spectra), -Data Release 2 (DR2) adds to this 279 plates (178,560 spectra), -Data Release 3 (DR3) adds to this 252 plates (161,280 spectra), -Data Release 4 (DR4) adds to this 479 plates (306,560 spectra), -Data Release 5 (DR5) adds to this 324 plates (207,360 spectra), -and Data Release 6 (DR6) adds to this 349 plates (223,360 spectra). - -
-To download the 14 Gb of EDR data -(including the spAll-EDR summary file; -if you only want the spAll file, then remove -"--accept *.fits" from the command): -
- wget --dot-style=mega --quota=0 --no-host-directories \ - --no-parent --non-verbose --recursive --retr-symlinks \ - --accept "*.fits" --accept "spAll*" \ - http://spectro.princeton.edu/spectro_EDR/ -
- wget --dot-style=mega --quota=0 --no-host-directories \ - --no-parent --non-verbose --recursive --retr-symlinks \ - --accept "*.fits" --accept "spAll*" \ - http://spectro.princeton.edu/spectro_DR1/ -
- wget --dot-style=mega --quota=0 --no-host-directories \ - --no-parent --non-verbose --recursive --retr-symlinks \ - --accept "*.fits" --accept "spAll*" \ - http://spectro.princeton.edu/spectro_DR2/ -
- wget --dot-style=mega --quota=0 --no-host-directories \ - --no-parent --non-verbose --recursive --retr-symlinks \ - --accept "*.fits" --accept "spAll*" \ - http://spectro.princeton.edu/spectro_DR3/ -
- wget --dot-style=mega --quota=0 --no-host-directories \ - --no-parent --non-verbose --recursive --retr-symlinks \ - --accept "*.fits" --accept "spAll*" \ - http://spectro.princeton.edu/spectro_DR4/ -
- wget --dot-style=mega --quota=0 --no-host-directories \ - --no-parent --non-verbose --recursive --retr-symlinks \ - --accept "*.fits" --accept "spAll*" \ - http://spectro.princeton.edu/spectro_DR5/ -
- wget --dot-style=mega --quota=0 --no-host-directories \ - --no-parent --non-verbose --recursive --retr-symlinks \ - --accept "*.fits" --accept "spAll*" \ - http://spectro.princeton.edu/spectro_DR6/ -
- wget --dot-style=mega --quota=0 --no-host-directories \ - --no-parent --non-verbose --recursive --retr-symlinks \ - --accept "*.fits" --accept "spAll*" \ - http://www.astro.princeton.edu:81/spectro_DR7/ \ - --http-user=??? --http-passwd=??? -
-Note that these reductions are not identical to the "official SDSS -data release", but have several improvements and include our redshift fits, -spectral classifications, velocity dispersions for galaxies, -and emission line fits. - -
-The spectroscopic data is reduced at Princeton, typically within 12 hours -of the data being taken. These reductions are available directly from -Princeton as described below. - -
-At Princeton, the top-level directory for the reduced data is: -
-/u/dss/spectro -
-/u/dss/spectro_v4_8 -
-To obtain the data from Princeton you will need access to the "alfred" -account. Send your SSH public key (in the file ".ssh/identity.pub") to -schlegel@astro.princeton.edu. -(See Mike Blanton's -notes on generating ssh keys.) -Once you have been added to the authorized_keys list for user "alfred", -then you can copy the data with "rsync" or "scp". - -
-Each plate has the following files associated with it: -
-$BOSS_SPECTRO_REDUX/$PLATE/spPlate-$PLATE-$MJD.fits -$BOSS_SPECTRO_REDUX/$PLATE/spZbest-$PLATE-$MJD.fits -$BOSS_SPECTRO_REDUX/$PLATE/spZline-$PLATE-$MJD.fits -$BOSS_SPECTRO_REDUX/$PLATE/spZall-$PLATE-$MJD.fits -
-Most people will only want the summary files, combined spectra and 1D outputs -(redshifts). These total about 125 Gb for 1300 plates as of Feb 2004. -The following command will copy that entire list of files (eventually). -Should the copy be interrupted, the "rsync" command can be invoked again -to successfully copy the remainder of the data. -
- rsync -arv --rsh="ssh1" --copy-unsafe-links \ - --include "*/" \ - --include "platelist*" --include "spAll*" \ - --include "*spZ*.fits" --include "spPlate*.fits" \ - --exclude "*" \ - alfred@sdssdata.astro.princeton.edu:/u/dss/spectro/ . -
-These spectroscopic reductions are completely automated with the -Spectro Robot. As data are taken at Apache Point, the data are copied -to Princeton, plan files are built that describe how to reduce the -data, and the reductions are batched on local and remote CPU's. -You can expect all spectroscopic observations to be reduced within 24 hours. - -
-An efficient way to keep your copy of these reductions current is to use -the Unix "rsync" command as described above. A cleaner implementation -of this is to use the -pullspectra -script from the "idlspec2d" product. -If you run "rsync" or "pullspectra" each day or each week, it will only -copy files that have been changed or added. This is a convenient way -to keep your data up-to-date without re-copying everything en-masse. -Note that you need "ssh" to work without asking for a -pass-phrase for "rsync" to work as a cron job (see below) -- see your system -admin if you need help with that. - -
-Finally, you could set up a "cron" job to update your copy of the data on -a daily basis. This would be done by creating a cron table file (call it -"cron.table" in this example). If you want to copy the typical set of -spectroscopic outputs every day at 3:15 am, this file would look something -like this: -
-15 3 * * * /my/directory/bin/pullspectra /my/data -
-crontab cron.table -
-crontab -r -
There are a number of "special" plates, designed for commissioning -either the hardware or target selection for the survey. These plates are: -
-We generated one monstrous file with all the spectroscopic -and photometric outputs (but not the actual images, spectra -or spectroscopic line fits). These files are FITS binary tables. -The photometric information is from the best-available -``uber-calibration'' of the imaging data as of April 2005. -The details of the photometric information can be found here: - http://photo.astro.princeton.edu/#data_model . -These files include matches to to the FIRST radio catalog, -the 2MASS catalog, and the USNO-B astrometric catalog. -The following files exist: -
-The following information is compiled for each object: -
PROGNAME | Spectroscopic program: 'main' for the SDSS main survey, - 'southern' for the SDSS Southern survey, and - 'devel', 'special', 'photoz' for other assorted programs - |
CHUNKNAME | Targetting chunk name (this is a numeric value for - all plates in the 'main' or 'southern' survey) - |
PLATEQUALITY | Quality of plate from platelist file: 'bad', 'marginal', or 'good' - |
PLATESN2 | Signal-of-noise measure of the plate (the worst of the 4 cameras) - |
SPECPRIMARY | 1 for the best spectroscopic observation of each object, 0 otherwise - |
SPECOBJ_ID | Unique ID for spectroscopic objects, starting with 1; objects at - the same coordinates have the same ID, with only one being SPECPRIMARY - |
NSPECOBS | Number of spectroscopic observations of this object - |
ZANS | ZANS structure from spZbest file - with spectroscopic classifications (CLASS), redshifts (Z), etc. - |
calibObj | calibObj structure with all imaging parameters; - fluxes are in units of nano-maggies, so MAG=22.5-2.5*log10(FLUX) - |
-In addition, we've also made ASCII files with the most relevant quantities -that one may wish to search on. These files are "spAll*.dat", and are basically -a stripped-down version of "spAll*.fits" for those FITS-challenged astronomers. -I have made some small changes to the ASCII file to make it more convenient -with Unix commands such as "awk" that want to index based upon word number: -(1) blank strings have been replaced with double-quotes, and -(2) spaces within strings have been replaced with plus signs -(i.e., replacing "BROADLINE STARBURST" with "BROADLINE+STARBURST"). - -
-These photometric outputs are from the Princeton PHOTO reductions. - -
-If you have access to the Princeton computers, -these trimmed calibObj files can be found at: -
- sdssdata.astro.princeton.edu:/u/dss/spectro/calibobj -- -
-The summary files make it convenient to select complete samples. -There are three complete samples that one can generate from the SDSS spectra: -the "main" galaxy sample, the Luminous Red galaxy sample (LRGs), and QSOs. -For complete samples, one should only use those plates that are for -the PROGNAME="main" survey (not "southern", "photoz", etc.) and the -primary observations of each object (SPECPRIMARY=1). - -
-The main galaxy survey is selected from the -target flags being set -to GALAXY, GALAXY_BIG, or GALAXY_BRIGHT_CORE. -The following bit of IDL code will select a unique list of objects targetted -by the main galaxy survey that have been spectroscopically confirmed -as galaxies or QSOs: -
-IDL> columns = ['PLATE','FIBER','MJD','OBJTYPE','PLUG_RA','PLUG_DEC', $ - 'PROGNAME','SPECPRIMARY','PRIMARGET','SECTARGET','CLASS','Z','ZWARNING'] -IDL> spall = hogg_mrdfits('spAll.fits',1,columns=columns) -IDL> imain = where(spall.specprimary EQ 1 $ - AND strmatch(spall.progname,'main*') $ - AND (spall.primtarget AND 2L^6 + 2L^7 + 2L^8) NE 0 $ - AND (strmatch(spall.class,'GALAXY*') OR strmatch(spall.class,'QSO*')) $ - AND spall.zwarning EQ 0) -
-IDL> columns = ['PLATE','FIBER','MJD','OBJTYPE','PLUG_RA','PLUG_DEC', $ - 'PROGNAME','SPECPRIMARY','PRIMARGET','SECTARGET','CLASS','Z','ZWARNING'] -IDL> spall = hogg_mrdfits('spAll.fits',1,columns=columns) -IDL> ilrg = where(spall.specprimary EQ 1 $ - AND strmatch(spall.progname,'main*') $ - AND (spall.primtarget AND 2L^5 + 2L^26) NE 0 $ - AND strmatch(spall.class,'GALAXY*') $ - AND spall.zwarning EQ 0) -
-IDL> columns = ['PLATE','FIBER','MJD','OBJTYPE','PLUG_RA','PLUG_DEC', $ - 'PROGNAME','SPECPRIMARY','PRIMARGET','SECTARGET','CLASS','Z','ZWARNING'] -IDL> spall = hogg_mrdfits('spAll.fits',1,columns=columns) -IDL> iqso = where(spall.specprimary EQ 1 $ - AND strmatch(spall.progname,'main*') $ - AND (spall.primtarget AND 2L^0 + 2L^1 + 2L^2 +2L^3 + 2L^4) NE 0 $ - AND strmatch(spall.class,'QSO*') $ - AND spall.zwarning EQ 0) -
-These same selections can be made from the ASCII version of the spAll -summary files using the Unix "awk" command. -To select the main sample of galaxies (the following must all be on one line): -
-% awk '{if (($22==1) && ($21=="main") && (and($19,2^6+2^7+2^8) != 0) - && ($4=="GALAXY" || $4=="QSO") && ($8==0)) {print $0}}' spAll.dat > galaxies.dat -
-% awk '{if (($22==1) && ($21=="main") && (and($19,2^5+2^26) != 0) - && ($4=="GALAXY") && ($8==0)) {print $0}}' spAll.dat > lrgs.dat -
-% awk '{if (($22==1) && ($21=="main") && (and($19,2^0+2^1+2^2+2^3+2^4) != 0) - && ($4=="QSO") && ($8==0)) {print $0}}' spAll.dat > qsos.dat -
-As of 31 May 2002, the above would select complete samples of -185,899 galaxies, 51,417 LRGs, and 22,245 QSOs. - -
-Since standard FITS files are used, there are many ways to access these -data. Below are descriptions of specially-written IDL, SM, and IRAF code for -looking at the data. -If you write any such tools, it would be useful to make them available -to the collaboration by posting them on this site. - -
-The absolute simplest way to look at the data is to use ATV, SAOimage or XIMTOOL -to display the plate images (spPlate*.fits), which will show you all 640 -spectra on a plate at once. - -
-You must have "idl" in your path, and the IDL_PATH -environment variable should already be set. At Princeton, you would type -the following for C-shell: -
- source /usr/peyton/common/licensed/idl/bin/idl_setup - setenv IDL_PATH .:+{$IDL_DIR}/lib -
- IDL_DIR=/usr/peyton/common/licensed/idl - IDL_PATH=.:+{$IDL_DIR}/lib - alias idl=$IDL_DIR/bin/idl -
-If you have someone maintaining survey code at your institution -(i.e., at Princeton using a Linux box), you can set up the tools -by simply typing the following for C-shell: -
- setenv EUPS_FLAVOR Linux - source /u/dss/products/evilups/bin/setups.csh # If you are using C-shell - setup idlspec2d -
- export SHELL=/bin/bash - export EUPS_FLAVOR=Linux - source /u/dss/products/evilups/bin/setups.sh # If you are using Bourne-shell - setup idlspec2d -
-If you need to install the code yourself, you need two products: -"idlutils", and "idlspec2d". If you wish to reduce the raw data -from scratch, you will also need a third product called "specflat". -See the instructions for installing this code at: -
- http://spectro.princeton.edu/idlspec2d_install.html -
-Extensive documentation exists for the commands described below, in -the file: -
-$IDLSPEC2D_DIR/doc/www/idlspec2d_doc.html -
-First, set up the "idlutils" and "idlspec2d" software, as described above. - -
-You need to set an environment variable that points to the top-level -directory of the data. At Princeton, this would be: -
-setenv BOSS_SPECTRO_REDUX /u/dss/spectro -
-Start IDL with: -
-idl -
-Then from within IDL, you can display individual spectra as following. -To display plate 401, fiber 100: -
-IDL> plotspec, 401, 100 -
-Make the same plot, but boxcar-smooth the spectrum: -
-IDL> plotspec, 401, 100, nsmooth=10 -
-Some plates are observed on multiple nights. To select one of the two -observations of plate 306: -
-IDL> plotspec, 306, 20, mjd=51690 -
-You can write the spectrum to an ASCII file as follows: -
-IDL> writespec, 401, 100 -
-IDL> writespec, 401, 100, filename='junk.dat' -
-Quitting IDL: -
-IDL> exit -
-For IDL pundits, the general-purpose routine for reading the -the coadded spectra is -READSPEC, -or SPFRAME_READ for reading -the spectra from a single exposure. - -
-The SM tools were written by Michael Strauss, Vijay Narayanan, and Robert -Lupton. -
-Problems and suggestions should be reported to Michael Strauss -(strauss@astro.princeton.edu). - --You will need a version of SM which supports FITS extensions -(version SM2_4_8 or later). If you have a valid SM license, -you or your system admin should be able to install this (see the bottom of -Michael's README file. - -
-At Princeton, you could type the following to display plate 306 fiber 100, -then overplot the best-fit template: -
-setenv BOSS_SPECTRO_REDUX /u/dss/spectro -setenv IDLSPEC2D_DIR /u/schlegel/idlspec2d -cd ~strauss/spectro_sm/release -sm -dev x11 -input_new spectro.sm -define idl 1 -plotspec 306 100 - t -
-Described below is the most basic method of plotting the spectra from IRAF. -Better tools will be posted on this page as they are available. - -
-Use the splot task in the noao.onedspec package. For example: - -
-cl> noao -no> onedspec -on> splot spPlate-0311-51665.fits[0][*,1] # plot fiber 1's flux -on> splot spPlate-0311-51665.fits[1][*,1] # plot fiber 1's inverse variance ** -on> splot spPlate-0311-51665.fits[0] # interactively plot fluxes: -Image line/aperture to plot (0:) (1): 65 # start w/fiber 65; use # to choose - # a new fiber, or ( and ) to move down and up by 1 in number; q to quit -on> phelp splot # for detailed help on splot -on> phelp onedspec # for summary of other tasks, e.g.: -on> specplot spPlate-0311-51665.fits[0][*,1],spPlate-0311-51665.fits[0][*,2] - # stack and plot fibers 1 and 2 without overlapping -- -
-Pat Hall has written an IRAF package, PHIST, for displaying SDSS spectra. -It does not have all the functionality of the IDL tools, but does have -some complementary functions. The task list is as follows: -
- dec2seg - convert decimal coordinates to sexigesimal - doigi - run existing IGI script -hexprint - convert a decimal number to hexadecimal - ionlte - calculate LTE ratios of MgI-III,AlI-IV,FeI-IV for given n_e,T ranges - lplots - overplot wavelengths of emission / absorption lines for given z - lplots2 - 2-panel lplots -mkfitsobs - make coadded FITS and text spectrum - mkobs - make text spectrum from FITS spectrum - sbqgr - sample task for converting GR's SX .tbl output to formatted ASCII - sbqsx - sample task for converting SX .tbl output to formatted ASCII - sbqsx2 - another task for converting SX .tbl output to formatted ASCII - seg2dec - convert sexigesimal coordinates to decimal - spdata - get objid,mags,ra,dec,objtype from spectrum (plate,mjd,fiber) - splat - interactively SPLOT a specific observation (plate+mjd+fiber) - splet - non-interactively SPECPLOT an observation and its error spectrum -unredden - deredden spectrum using SMC or LMC extinction curve - vcalc - calculate velocity offset in quasar rest frame - vplot - overplot wavelengths of emission / absorption lines in velocity space - vplots - multiple vplot - wcalc - calculate vacuum wavelength from air wavelength - zplot - plot spectrum in the rest frame at the specified z -- -Here's the README file on how to retrieve & install the package -for version v.2 (December 2001): - -
- PHIST -- Pat Hall's IRAF SDSS Tasks - -v.2 011207 Read "phist.hlp" ("phelp phist opt=sys" in IRAF) for an - introduction. Entire package available as a compressed tarfile - from http://astro.princeton.edu/~pathall/phist2.tar.gz - To install, enter the proper pathnames in "phist.cl" and - "phist.hd", load the IRAF "softools" package, and type - "mkhelpdb root.hd helpdb.mip", deleting the old helpdb.mip - if necessary. See "phist.hlp" for how to include - the package in your default IRAF startup. -- -
-This file contains the extracted spectra for a single CCD of -a single exposure. In general, each object has several blue exposures -and several red exposures. These spectra are still in the native -wavelength mapping, which is neither linear in wavelength nor log-wavelength. -However, the spectra are flux-calibrated, and deviant pixels are rejected -in the pixel mask (from the comparison with the other exposures). - -
-The spCFrame file contains the following:
-
-HDU #0 = Flux in units of 10^(-17) erg/s/cm^2/Ang [FLOAT] -HDU #1 = Inverse variance (1/sigma^2) for the above [FLOAT] -HDU #2 = Pixel mask [32-bit INT] -HDU #3 = Wavelength solution in log-10 Angstroms (vacuum) [FLOAT] -HDU #4 = Wavelength dispersion in units of (10^(-4) log-10 wavelength)^(-1) [FLOAT] - (these are the same units as in the combined-spectrum spPlate file) -HDU #5 = Plug-map structure from plPlugMapM file [BINARY FITS TABLE] -HDU #6 = Sky flux in units of 10^(-17) erg/s/cm^2/Ang [FLOAT] -HDU #7 = X position on the CCD [FLOAT] -HDU #8 = Superflat image [FLOAT] -- -
-The log file from running SPREDUCE2D for one night of data for one plate. - -
-The PostScript plots associated with the -spDiag2d-pppp-mmmmm.log file. - -
-This file contains the combined spectra for all exposures -(potentially spanning multiple nights) for a given mapped plate. -The first 5 HDU's contain NPIX by 640 images, where NPIX is the -number of wavelengths. The pixel scale is exactly 1e-4 in -log-wavelength (any units), which corresponds to 1e-4 * c * ln(10) -= 69.029765 km/sec. - -
-The spPlate file contains the following:
-
-HDU #0 = Flux in units of 10^(-17) erg/s/cm^2/Ang [FLOAT] -HDU #1 = Inverse variance (1/sigma^2) for the above [FLOAT] -HDU #2 = AND mask [32-bit INT] -HDU #3 = OR mask [32-fit INT] -HDU #4 = Wavelength dispersion in pixels [FLOAT] -HDU #5 = Plug-map structure from plPlugMapM file [BINARY FITS TABLE] -HDU #6 = Average sky flux in units of 10^(-17) erg/s/cm^2/Ang [FLOAT] -- -
-The following header keywords are of interest (in HDU #0):
-
-PLATEID = Plate ID -TILEID = Tile ID -MJD = Modified Julian date for first night of observation -MJDLIST = Modified Julian date for all nights of observation -MAPID = Fiber mapper ID -CARTID = Fiber cartridge used at the telescope -NAME = Name of target: Plate ID, MJD, Fiber mapper rerun -COEFF0 = Central wavelength (log10-Angstroms) of first pixel -COEFF1 = Wavelength range of each pixel (log-10 Angstroms) -CRVAL1 = (Same as COEFF0) -CD1_1 = (Same as COEFF1) -RADEG = Right ascension of plate center (J2000 degrees) -DECDEG = Declination of plate center (J2000 degrees) -AIRMASS = Airmass for mid-point of observation -RA = Right ascension of where the telescope thinks it is pointing -DEC = Declination of where the telescope thinks it is pointing -AZ = Azimuth of the telescope (degrees) -ALT = Altitude of the telescope (degrees) -NEXP = Total number of frames from all cameras used in the co-adding; - since there are 4 cameras, NEXP=12 would typically mean that there - were 3 exposures in each camera. -EXPT_B1 = b1 camera exposure time (seconds) -EXPT_B2 = b2 camera exposure time (seconds) -EXPT_R1 = r1 camera exposure time (seconds) -EXPT_R2 = r2 camera exposure time (seconds) -EXPTIME = Minimum of exposure times for all cameras -SPEC1_G = (S/N)^2 per pixel at g=20.2 on spectrograph #1 -SPEC2_G = (S/N)^2 per pixel at g=20.2 on spectrograph #2 -SPEC1_R = (S/N)^2 per pixel at r=20.25 on spectrograph #1 -SPEC2_R = (S/N)^2 per pixel at r=20.25 on spectrograph #2 -SPEC1_I = (S/N)^2 per pixel at i=19.9 on spectrograph #1 -SPEC2_I = (S/N)^2 per pixel at i=19.9 on spectrograph #2 -GOFFSTD = Median spectrophotometry minus photo mag for standard stars in g-band -ROFFSTD = Median spectrophotometry minus photo mag for standard stars in r-band -IOFFSTD = Median spectrophotometry minus photo mag for standard stars in i-band -GOFFRMS = RMS of spectrophotometry minus photo mag for standard stars in g-band -ROFFRMS = RMS of spectrophotometry minus photo mag for standard stars in r-band -IOFFRMS = RMS of spectrophotometry minus photo mag for standard stars in i-band -GOFFGAL = Median spectrophotometry minus photo mag for "main" galaxies in g-band -ROFFGAL = Median spectrophotometry minus photo mag for "main" galaxies in r-band -IOFFGAL = Median spectrophotometry minus photo mag for "main" galaxies in i-band -GOFFGAL = RMS of spectrophotometry minus photo mag for "main" galaxies in g-band -ROFFGAL = RMS of spectrophotometry minus photo mag for "main" galaxies in r-band -IOFFGAL = RMS of spectrophotometry minus photo mag for "main" galaxies in i-band -BESTEXP = Exposure number for the "best" exposure (all 4 cameras) -EXPIDxx = Identifier for the each invididual exposure+CCD used in this combine, - for ex. 'b1-00006853-00006857-00006858' refers to the b1 CCD, - exposure #6853, using flat #6857 and arc #6858 -TAI = Number of seconds since Nov 17 1858 for mid-point of observation -TAI-BEG = Number of seconds since Nov 17 1858 for beginning of observation -TAI-END = Number of seconds since Nov 17 1858 for end of observation -SEEING20= 20th-percentile of seeing during exposure (arcsec) -SEEING50= 50th-percentile of seeing during exposure (arcsec) -SEEING80= 80th-percentile of seeing during exposure (arcsec) -RMSOFF20= 20th-percentile of RMS offset of guide fibers (arcsec) -RMSOFF50= 50th-percentile of RMS offset of guide fibers (arcsec) -RMSOFF80= 80th-percentile of RMS offset of guide fibers (arcsec) -VERSIDL = Version of IDL -VERS2D = Version of idlspec2d for 2D reduction -VERSCOMB= Version of idlspec2d for combining multiple spectra -VERSFLAT= Version of specflat for spectroscopic flat-fields -VERS1D = Version of idlspec2d for 1D reduction -HELIO_RV= Barycentric velocity correction (km/sec) -- -
-There are two masks, an "AND" mask and an "OR" mask. The spectra -are constructed from 3 or more 15-minute observations, and the "AND" mask -bits are set if that bit is set for each and every input observation. -The "OR" mask bits are set if that bit is set for any of the observations. -Usually, I only look at the "AND" mask. - -
-The mask bits are set as follows:
-
- 0 NOPLUG Fiber not listed in plugmap file - 1 BADTRACE Bad trace from routine TRACE320CRUDE - 2 BADFLAT Low counts in fiberflat - 3 BADARC Bad arc solution - 4 MANYBADCOLUMNS >10% pixels are bad columns - 5 MANYREJECTED >10% pixels are rejected in extraction - 6 LARGESHIFT Large spatial shift between flat and object pos'n - 7 BADSKYFIBER Sky Fiber shows extreme residuals - 8 NEARWHOPPER Within 2 fibers of a whopping fiber (inclusive) -10 SMEARIMAGE Smear available for red and blue cameras (deprecated) -11 SMEARHIGHSN S/N sufficient for full smear fit (deprecated) -12 SMEARMEDSN S/N only sufficient for scaled median fit (deprecated) -16 NEARBADPIXEL Bad pixel within 3 pixels of trace -17 LOWFLAT Flat field less than 0.5 -18 FULLREJECT Pixel fully rejected in extraction (INVVAR=0) -19 PARTIALREJECT Some pixels rejected in extraction -20 SCATTEREDLIGHT Scattered light significant -21 CROSSTALK Cross-talk significant -22 NOSKY Sky level unknown at this wavelength (INVVAR=0) -23 BRIGHTSKY Sky level > flux + 10*(flux error) - AND sky > 2.0 * median(sky,99 pixels) -24 NODATA No data available in combine B-spline (INVVAR=0) -25 COMBINEREJ Rejected in combine B-spline -26 BADFLUXFACTOR Low flux-calibration or flux-correction factor -27 BADSKYCHI Relative chi^2 > 3 in sky residuals at this wavelength -28 REDMONSTER Contiguous region of bad chi^2 in sky residuals - (with threshhold of relative chi^2 > 3) -- -
-When low numbered bits (<16) are set, those will be set for half -of the spectra -- either the blue or red spectrograph. The higher-numbered -bits (>=16) are set for individual pixels. - -
-Which mask bits are important? The conditions that are considered -very bad are already used to set the errors to infinity for the -effected pixels (specifically, the inverse variance is set to zero). -The most useful mask bit to look at is BRIGHTSKY, which indicates -when the sky is so bright relative to the object that perhaps one -shouldn't trust any of the object flux there. Our reported errors -are meant to include sky-subtraction errors, but there are instances -(particularly around 5577) where these errors may be untrustworthy. - -
-Dispersion and sky: The dispersion per pixel and the sky flux -are computed at each pixel by re-weighting the individual spectra -at each pixel according to their formal errors. This re-weighting -is only approximate. - -
-Sky wavelengths: Note that the sky lines are slightly shifted -in the reductions because we transform the velocities to the barycenter -of the solar system. Each exposure that contributes to the co-added -spectra will have slightly different barycenter correction, so the -"average sky" contains a superposition of these slightly-offset sky lines. -These shifts keep the object spectra as-measured at the barycenter, -regardless of the time of year or the Earth's rotation relative to -the spectroscopic targets. - -
-This file contains the spectroscopic classifications and redshifts -from the Princeton-1D code. -Ths first HDU is the FITS binary table described below. -The 2nd HDU is the best-fit eigen-spectrum for each object. -These can be regenerated from the tabular information, but for -convenience I've explicitly computed them. - -
-The columns in the first HDU are as follows:
-
-PLATE Plate number -TILE Tile number -MJD Modified Julian date of observation -FIBERID Fiber ID (1 to 640) -OBJID 4-element PHOTO ID of targetted object: run, re-run, column, ID -OBJTYPE Why this object was targetted. Note that if this field says QSO, - it could be the case that this object would have been targetted - as a GALAXY or any number of other categories as well. The - PRIMTARGET and SECTARGET flags in the plug-map structure (in the - spPlate file) gives this full information. -PLUG_RA Object RA (drilled fiber position) [J2000 degrees]. -PLUG_DEC Object DEC (drilled fiber position) [J2000 degrees]. -CLASS Spectro classification: GALAXY, QSO, STAR; reductions prior to - version v4_7 will also use SKY and UNKNOWN; later versions set - the ZWARNING flags -SUBCLASS Spectro sub-classification. Galaxies and QSO's can be given - sub-classifications based upon their emission lines. - This text includes the word "AGN" if: - log10(OIII/Hbeta) > 0.7 - 1.2 * (log10(NII/Halpha) - 0.4) - This text includes the word "STARFORMING" if: - log10(OIII/Hbeta) < 0.7 - 1.2 * (log10(NII/Halpha) - 0.4) - If the H_alpha E.W. > 500 Ang, then upgrade "STARFORMING" to - "STARBURST." - In all cases for AGN, STARFORMING, and STARBURST, the 4 emission - lines used for that sub-classification must have well-measured - errors (e.g., LINEAREA_ERR > 0). - If any galaxies or quasars have lines detected at the 10-sigma - level with sigmas > 200 km/sec at the 5-sigma level, call them - "BROADLINE". - For stars, the following subclassifications are in use: - O, OB, B6, B9, A0, A0p, F2, F5, F9, G0, G2, G5, K1, K3, K5, K7, - M0V, M2V,M1, M2, M3, M4, M5, M6, M7, M8, L0, L1, L2, L3, L4, L5, - L5.5, L9, T2, Carbon, Carbon_lines, CarbonWD, CV. -Z Redshift; if the ZWARNING flag is nonzero, then assume that this - redshift is incorrect -Z_ERR Redshift error based upon fit to chi^2 minimum; negative for - invalid fit -RCHI2 Reduced chi^2 for best fit -DOF Degrees of freedom for best fit -RCHI2DIFF Difference in reduced chi^2 of best solution to 2nd best solution -TFILE Template file in $IDLSPEC2D_DIR/templates -TCOLUMN Columns to use in template file (0-indexed); unused values set to -1 -NPOLY Number of polynomial terms used in conjunction with TFILE -THETA Eigenvalue coefficients for each column in template file + each polynomial term -THETA_COVAR Covariance matrix for THETA, where the square root of the - diagonal elements gives the formal 1-sigma statistical errors -VDISP Velocity dispersion, only computed for galaxies (km/sec) - and explicitly masking emission line regions -VDISP_ERR Error in VDISP (km/sec); negative for invalid fit -VDISPZ Redshift for best-fit velocity dispersion -VDISP_ERR Error in VDISPZ -VDISPCHI2 Chi^2 for best-fit velocity dispersion -VDISPNPIX Number of pixels overlapping the templates used in the - velocity dispersion fit -VDISPDOF Degrees of freedom for best-fit velocity dispersion, equal to - VDISPNPIX minus the number of templates minus the number of - polynomial terms minus 1 (the last 1 is for the velocity dispersion) -WAVEMIN Minimum observed (vacuum) wavelength for this object (Angstroms) -WAVEMAX Maximum observed (vacuum) wavelength for this object (Angstroms) -WCOVERAGE Amount of wavelength coverage in log-10(Angstroms) -ZWARNING A flag set for bad redshift fits in place of calling CLASS=UNKNOWN. - The following bits may be set: - 0 = Sky fiber - 1 = Too little wavelength coverage (WCOVERAGE < 0.18) - 2 = Chi^2 is too close to chi^2 for the next best fit (within 0.01 - in the reduced chi^2) - 3 = Synthetic spectrum is negative (only set for stars and QSO's) - 4 = Fraction of points above 5 sigma is too large (> 5%) - 5 = Chi^2 minimum was at the edge of the redshift-fitting range, - resulting in Z_ERR being set to -1 - 6 = Negative emission in a QSO line, triggered only in QSO spectra - iff C_IV, C_III, Mg_II, H_beta, or H_alpha has - LINEAREA + 3 * LINEAREA_ERR < 0 - 7 = Unplugged fiber -SN_MEDIAN Median S/N for all good pixels -CHI68P 68-th percentile value of abs(chi) of the best-fit synthetic - spectrum to the actual spectrum, which would be 1.0 for a good fit - if the errors are properly normalized. -FRACNSIGMA Fraction of pixels deviant by more than N sigma, where N=[1,2,...10] - (igorning all points blueward of rest-frame 1216 Ang) -FRACNSIGHI Fraction of pixels high by more than N sigma, where N=[1,2,...10] - (igorning all points blueward of rest-frame 1216 Ang) -FRACNSIGLO Fraction of pixels low by more than N sigma, where N=[1,2,...10] - (igorning all points blueward of rest-frame 1216 Ang) - (incorrectly computed for versions of the code before v4_9_10) -SPECTROFLUX 5-element array of integrated flux in each of the 5 SDSS imaging - filters (ugriz); the units are nanomaggies, which is 1 at 22.5 - magnitude; convert to magnitudes with - 22.5 - 2.5 * LOG_10(SPECTROFLUX); - the u-band and z-band counts should not be used -SPECTROFLUX_IVAR: The approximate error (inverse variance) in SPECTROFLUX - as measured by the sky fibers; we assign identical errors - in linear flux units to all fibers on a plate. -SPECTROSYNFLUX: Same as SPECTROFLUX, but measured using the best-fit - synthetic eigen-spectrum rather than the actual data points -SPECTROSYNFLUX_IVAR: The approximate error (inverse variance) in SPECTROSYNFLUX - as measured by the sky fibers; we assign identical errors - in linear flux units to all fibers on a plate. -SPECTROSKYFLUX: 5-element array of the sky flux in each of the 5 SDSS imaging - filters -ANYANDMASK Mask bits which are set if any pixels for an object's ANDMASK - have that bit set (these fields incorrect in current reductions) -ANYORMASK Mask bits which are set if any pixels for an object's ORMASK - have that bit set (these fields incorrect in current reductions) -SPEC1_G (S/N)^2 at g=20.20 for spectrograph #1 (same value for 320 fibers) -SPEC1_R (S/N)^2 at r=20.25 for spectrograph #1 (same value for 320 fibers) -SPEC1_I (S/N)^2 at i=19.90 for spectrograph #1 (same value for 320 fibers) -SPEC2_G (S/N)^2 at g=20.20 for spectrograph #2 (same value for 320 fibers) -SPEC2_R (S/N)^2 at r=20.25 for spectrograph #2 (same value for 320 fibers) -SPEC2_I (S/N)^2 at i=19.90 for spectrograph #2 (same value for 320 fibers) -ELODIE_FILENAME File name for best-fit Elodie star -ELODIE_OBJECT Star name (mostly Henry Draper names) -ELODIE_SPTYPE Spectral type -ELODIE_BV (B-V) color -ELODIE_TEFF Effective temperature -ELODIE_LOGG Log10(gravity) -ELODIE_FEH [Fe/H] -ELODIE_Z Redshift -ELODIE_Z_ERR Redshift error; negative for invalid fit -ELODIE_Z_MODELERR The standard deviation in redshift amongst the 12 - best-fit stars -ELODIE_RCHI2 Reduced chi^2 -ELODIE_DOF Degrees of freedom for fit -VELOCITY_TAI Timestamp for each exposure [sec] -VELOCITY Velocity for each exposure [km/s], where the first exposure - should be zero within roundoff -VELOCITY_ERR Error in VELOCITY [km/s] -VELOCITY_COVAR Covariance matrix for VELOCITY, where the square root of the - diagonal elements gives the formal 1-sigma statistical errors -- -
-This file contains the line fits from from the Princeton-1D code. - -
-The columns in the first HDU are as follows:
-
-PLATE Plate number -MJD Modified Julian date of observation -FIBERID Fiber ID (1 to 640) -LINENAME Line name -LINEWAVE Catalog wavelength for this line in vacuum Angstroms -LINEZ Redshift -LINEZ_ERR Redshift error (negative for invalid fit) -LINESIGMA Gaussian width in km/sec -LINESIGMA_ERR Error in gaussian width (negative for invalid fit) -LINEAREA Area in gaussian fit where units are (flux-units) * Ang -LINEAREA_ERR Flux error (negative for invalid fit) -LINEEW Equivalent width (Angstroms) -LINEEW_ERR Equivalent width error (negative for invalid fit) -LINECONTLEVEL Continuum level at line center -LINECONTLEVEL_ERR Error in continuum level at line center -LINENPIX Number of good pixels within +/- 3 sigma of the line center -LINEDOF Degrees of freedom in fit, approximated as LINENPIX minus - the number of terms fit for that line, which can be - fractional if one parameter if fixed betwen several lines -LINECHI2 Chi^2 for all points within +/- 3 sigma of the line center - (negative if no such points) -- -
-The following lines are fit. Lines with the same ZINDEX are constrained to -have the same redshift. Lines with the same WINDEX are constrained to have -the same gaussian width. The flux ratios of [O_III] and [N_II] are are -constrained to be 1:3. -
LINEWAVE | LINENAME | ZINDEX | WINDEX | Flux ratio - |
---|---|---|---|---|
1215.67 | Ly_alpha | z_lya | w_ly_a | - |
1240.81 | N_V 1240 | zemission | w_n_v | - |
1549.48 | C_IV 1549 | zemission | wemission | - |
1640.42 | He_II 1640 | zemission | wemission | - |
1908.734 | C_III] 1908 | zemission | wemission | - |
2799.49 | Mg_II 2799 | zemission | wemission | - |
3726.032 | [O_II] 3725 | zemission | wemission | - |
3728.815 | [O_II] 3727 | zemission | wemission | - |
3868.75 | [Ne_III] 3868 | zemission | wemission | - |
3889.049 | H_epsilon | zemission | w_balmer | - |
3970.00 | [Ne_III] 3970 | zemission | wemission | - |
4101.734 | H_delta | zemission | w_balmer | - |
4340.464 | H_gamma | zemission | wemission | - |
4363.209 | [O_III] 4363 | zemission | wemission | - |
4685.68 | He_II 4685 | zemission | wemission | - |
4861.325 | H_beta | zemission | w_balmer | - |
4958.911 | [O_III] 4959 | zemission | wemission | - |
5006.843 | [O_III] 5007 | zemission | wemission | 3 * ([O_III] 4959) - |
5411.52 | He_II 5411 | zemission | wemission | - |
5577.339 | [O_I] 5577 | zemission | wemission | - |
6300.304 | [O_I] 6300 | zemission | wemission | - |
6363.776 | [O_I] 6363 | zemission | wemission | - |
6548.05 | [N_II] 6548 | zemission | wemission | - |
6562.801 | H_alpha | zemission | w_balmer | - |
6583.45 | [N_II] 6583 | zemission | wemission | 3 * ([N_II] 6548) - |
6716.44 | [S_II] 6716 | zemission | wemission | - |
6730.82 | [S_II] 6730 | zemission | wemission | - |
7135.790 | [Ar_III] 7135 | zemission | wemission | - |
-This file is a FITS binary table identical in form to the spZbest file, -but with all the fits ever done to each object. This includes 5 -possible galaxy redshifts, 5 possible QSO redshifts, and 1 for each -of many types of stars. They are rank-sorted for each object, -letting you go through and keep asking "but what's the next best fit?". -This is what the IDL procedure -plotspec -uses when the ZNUM keyword is specified. -For example, ZNUM=2 will over-plot the 2nd-best fit eigenspectrum. - -
-For the v4_9_8 through v5_0_0 reductions, NAXIS2=33280 in HDU#1 -for the spZall files, meaning there are fits for 33280/640 = 52 -sets of eigentemplates per object. Most of those are single-eigenspectra -stars. - -
-Not all of the parameters in the spZbest file are computed for -the spZall file. For example, the SDSS filter convolutions are not done. - -
-The log file from running the Princeton-1D reduction code (SPREDUCE1D) -on a plate. - -
-The PostScript plots associated with the -spDiag1d-pppp-mmmmm.log file. - -
-The target selection sets bits in two flags called PRIMTARGET and SECTARGET. -An object can have multiple bits set. For example, if the target selection -code determined that an object was both a QSO_SKIRT and a STAR_WHITE_DWARF, -then two bits will be set. The value of PRIMTARGET in this example will -be 2^4 + 2^19 = 524304. - -
-Let's say you want to find all objects either targetted as a carbon star, -or spectroscopically classified as one. All this information is in the -summary file "spAll.fits". From IDL, one could search for these objects -and print their PLATE, MJD, and FIBERID as follows: -
-IDL> a = mrdfits('spAll.fits',1) ; This file should be in your current directory -IDL> i = where((a.primtarget AND 2L^14) NE 0) -IDL> print, transpose([[a[i].plate],[a[i].mjd],[a[i].fiberid]]) - - [ Prints list of objects targetted as a Carbon star.] - -IDL> i = where(strmatch(a.subclass, 'Carbon*') AND a.zwarning EQ 0) -IDL> print, transpose([[a[i].plate],[a[i].mjd],[a[i].fiberid]]) - - [ Prints list of objects spectroscopically called a Carbon star.] -
-Now plot all of the objects found above: -
-IDL> i=0 & plotspec, a.plate, a.fiberid, mjd=a.mjd -
-The PRIMTARGET bit values are as follows (in hex and decimal): -
- QSO_HIZ = 0x1 = bit # 0 = 1 - QSO_CAP = 0x2 = bit # 1 = 2 - QSO_SKIRT = 0x4 = bit # 2 = 4 - QSO_FIRST_CAP = 0x8 = bit # 3 = 8 - QSO_FIRST_SKIRT = 0x10 = bit # 4 = 16 - QSO_MAG_OUTLIER = 0x2000000 = bit #25 = 33554432 - QSO_REJECT = 0x20000000 = bit #29 =536870912 - GALAXY_RED = 0x20 = bit # 5 = 32 - GALAXY_RED_II = 0x4000000 = bit #26 = 67108864 - GALAXY = 0x40 = bit # 6 = 64 - GALAXY_BIG = 0x80 = bit # 7 = 128 - GALAXY_BRIGHT_CORE= 0x100 = bit # 8 = 256 - ROSAT_A = 0x200 = bit # 9 = 512 - ROSAT_B = 0x400 = bit #10 = 1024 - ROSAT_C = 0x800 = bit #11 = 2048 - ROSAT_D = 0x1000 = bit #12 = 4096 - ROSAT_E = 0x8000000 = bit #27 =134217728 - STAR_BHB = 0x2000 = bit #13 = 8192 - STAR_CARBON = 0x4000 = bit #14 = 16384 - STAR_BROWN_DWARF = 0x8000 = bit #15 = 32768 - STAR_SUB_DWARF = 0x10000 = bit #16 = 65536 - STAR_CATY_VAR = 0x20000 = bit #17 = 131072 - STAR_RED_DWARF = 0x40000 = bit #18 = 262144 - STAR_WHITE_DWARF = 0x80000 = bit #19 = 524288 - STAR_PN = 0x10000000 = bit #28 =268435456 - SERENDIP_BLUE = 0x100000 = bit #20 = 1048576 - SERENDIP_FIRST = 0x200000 = bit #21 = 2097152 - SERENDIP_RED = 0x400000 = bit #22 = 4194304 - SERENDIP_DISTANT = 0x800000 = bit #23 = 8388608 - SERENDIP_MANUAL = 0x100000 = bit #24 = 16777216 - QSO_MAG_OUTLIER = 0x200000 = bit #25 = 33554432 - GALAXY_RED_II = 0x400000 = bit #26 = 67108864 - ROSAT_E = 0x800000 = bit #27 =134217728 - STAR_PN = 0x1000000 = bit #28 =268435456 - QSO_REJECT = 0x2000000 = bit #29 =536870912 - SOUTHERN_SURVEY = 0x8000000 = bit #31 =-2147483648 -- -
-The SECTARGET bit values are as follows (in hex and decimal): -
- LIGHT_TRAP = 0x1 = bit # 0 = 1 - REDDEN_STD = 0x2 = bit # 1 = 2 - TEST_TARGET = 0x4 = bit # 2 = 4 - QA = 0x8 = bit # 3 = 8 - SKY = 0x10 = bit # 4 = 16 - SPECTROPHOTO_STD = 0x20 = bit # 5 = 32 - GUIDE_STAR = 0x40 = bit # 6 = 64 - BUNDLE_HOLE = 0x80 = bit # 7 = 128 - QUALITY_HOLE = 0x100 = bit # 8 = 256 - HOT_STD = 0x200 = bit # 9 = 512 - SOUTHERN_SURVEY = 0x8000000 = bit #31 =-2147483648 -- -
-These flag values were copied from the "target" module files -"common/arTargetType.h" and "etc/diags.tcl". (There doesn't actually -appear to be any single file that tracks all of these flags!) - -
-Yes, the data are stored using vacuum wavelengths. This makes sense -for quasologists. But most optical astronomers know the wavelengths of -transitions as measured at S.T.P., which is how the CRC lists them for -any transitions redward of 2000 Angstroms. - -
-The IAU standard for conversion from air to vacuum wavelengths is given
-in Morton (1991, ApJS, 77, 119). For vacuum wavelengths (VAC) in Angstroms,
-convert to air wavelength (AIR) via:
-
- AIR = VAC / (1.0 + 2.735182E-4 + 131.4182 / VAC^2 + 2.76249E8 / VAC^4) -- -
-Below, I've listed the vacuum wavelength of some transitions: -
- Air Vacuum - -------- -------- - 4861.363 4862.721 H_beta - 4958.911 4960.295 [O III] - 5006.843 5008.239 [O III] - 6548.05 6549.86 [N II] - 6562.801 6564.614 H_alpha - 6583.45 6585.27 [N II] - 6716.44 6718.29 [S II] - 6730.82 6732.68 [S II] -- -
-Note that the wavelengths are also shifted such that measured velocities -will be relative to barycentric at the mid-point -of each 15-minute exposure (using TAI-BEG + 0.5 * EXPTIME from the header). -
-Here is a list of the most offensive known problems in either the -data or in the Spectro-2D reduction code as of version v4_9_11 (31 May 2002): -
-Plate 0187/51455 has no good b2 arc on the first night of data 51454. - This plate has severe scattered light problems. The r2 camera - is completely flagged as REDMONSTER in the ORMASK, and is - therefore ignored by the redshift-finding code. For that - reason, the synthetic spectra do not match the spectra - at all for the red side of fibers 321-640. -Plate 0192/51461 has severe scattered light problems. -Plate 0202/51441 has the Red Monster light at 6400-6550 Ang in the last - 2 of 4 exposures (exp #1367,1368). - This results in some wrong z=4.3 QSOs (fibers 601,639). - Apparently 2 of the b1 science exposures saturated - (exp #1366,1367). See PR #1890. -Plate 0214/51441 has the Red Monster light at 6400-6550 Ang in the first - 3 of 4 exposures (exp #1356-1368). See PR #1890. -Plate 0231/51456 has no good r1 arc on the first night of data 51455. -Plate 0260/51612 looks like funny fluxing in the blue below 4000 Ang, - probably due to very low S/N. -Plate 0354/51792 taken during moon -Plate 0360/51780 had the b1 CCD warm up during exposures 6193,6194,6199 - resulting in apparent scattered light in 6193,6194 and - rejection of frame 6199. See PR #2170. -Plate 0426/51882 has the Red Monster light at 6400-6550 Ang in exp #7366-7367. - See PR #1890. -Plate 0541/51959 has bad extraction chi^2 and sky-subtraction, especially in r1. -Plate 0644/52149 has bad extraction chi^2 and sky-subtraction. Is this the same - problem as for plate 0541/51959? --The following list of plates were rejected because of out-of-collimation -spectrographs (c.f., sdss-spectro/873): -
-Plate 0504/52316 - Many rejected pixels -Plate 0704/52205 - Many rejected pixels -Plate 0721/52228 - Many rejected pixels -Plate 0736/52210 - Many rejected pixels -Plate 0756/52237 - Badly out-of-focus -Plate 0761/52266 - Many rejected pixels -Plate 0769/52282 - Many rejected pixels -Plate 0770/52282 - Many rejected pixels -Plate 0775/52295 - Many rejected pixels -Plate 0778/52337 - Many rejected pixels -Plate 0790/52346 - Badly out-of-focus night -Plate 0791/52347 - Badly out-of-focus -Plate 0873/52347 - Badly out-of-focus -Plate 0876/52346 - Badly out-of-focus night -Plate 0889/52346 - Badly out-of-focus night; many rejected pixels -- -
-Plates 0179,0182 are FASTT plates with 12th mag stars and no sky fibers; - these data are not reducable. -Plate 0204/51455 has no good b1,b2,r2 arcs -Plate 0209/51462 has too many saturated rows in r2 for the modern reduction code -Plate 0213/51578 has no good b2 arc -Plate 0218/51461 has no good b2 arc -Plate 0235/51462 has no good b2 arc (saturated) -Plate 0260/51612 has only 2 science exposures, so will not combine -Plate 0284/51934 has no good arcs (flat-field screen was not closed for arcs); - no plan file even built -Plate 0296/51578 has no good b2 arc -Plate 0317/51589 has no flats or arcs; no plan file even built -Plate 0318 special plate needs edited plug-map files in order to run -Plate 0324/51616 only has one science exposure, so won't combine -Plate 0347/51701 has 1 saturated exposure on 51694, and 1 good one on 51701 -Plate 0560/52045 only has one science exposure, so will not combine -- -
-MJD -> 51569 (->25 Jan 2000): Shutter open during readout! - Definitely compromises data for plates 187,188,190,191, - 192,194,198,202,203,217,231. -MJD -> 51605 (->2 Mar 2000): Scattered light from internal LED's contaminate - all these data. After this date, we went through several - work-arounds that basically eliminated the problem. - Definitely compromises data for plates 187,188,190,191, - 192,194,198,202,203,214,217,231,265,266,269,283,295. -MJD 51578 - 55580 (3-5 Feb 2000): Severe "shifted row" electronics problem. - This only effects plate 295/51580. -MJD 51584,51585,51608,51614,51615,51629,51630: Raw images are the wrong size! - Unsure what this actually does to the data. - Potentially effects plates 266,267,279,281,282,283,295,303. -MJD 51688 - 51703 (22 May 2000): Electronics "dropped pixel" problem - on cameras b2,r2. Potentially effects plates 312,338,339, - 343,344,346,348,349,351,353. -MJD 51779 only (22 Aug 2000): Terrible electronics problem on r2. - This only effects plate 356/51779. -MJD 51789 - today: (2 Sep 2000): Smear implemented for most plates. -MJD 52220 - 52266, 52312 - 52356 (Nov-Dec 2001, Feb-Mar 2002) Spectrograph - slit-head latch problems. The kinematic mounts for the - slit-heads were not always engaged, resulting in out-of-focus - exposures on the instrument. The optimal extraction code - works quite poorly on these data, as does the sky-subtraction - which shows large relative chi^2 (MJD 52346 is the worst, - plates 790, 876, 889). -MJD 52899 - today: Noise source in right amplifier of r2 adding ~30 DN of noise - to ~5% of the pixels. See sdss-obs/4425 and other e-mail - traffic after 20 March 2004. -MJD 52988,52991: LED from telephone causing red spotlight on plate 1489/52991. - See sdss-spectro/1245 and sdss-spectro/1246. -MJD 53053 (18 Feb 2004): A fiber bundle on cartridge #9 spectro-1 detached - while observing plate 1372. -MJD 53670-53676 (26-28 Oct 2005): Condensation inside the b1 camera - make the data useless on plates 2264/53673, 2333/53676, - 2338/53676 (c.f. PR #6713). The data on 2265/53674 and - 2266/53675 appears surprisingly OK. - -Plates 1 - 431: Tiling bug resulted in larger color range for SPECTROPHOTO - standards, which must compromise spectro-photometry - at some level. -Plates 371 - 416: Bad QSO selection due to bug in target v2_7 (chunk 8), - resulting in a very low success rate for quasars. -- -
-Problems can be reported via the bug database at: -
- -http://spectro.princeton.edu/cgi-bin/gnatsweb.pl -
Funding for the creation and distribution of the SDSS Archive has -been provided by the Alfred P. Sloan Foundation, the Participating -Institutions, the National Aeronautics and Space Administration, the -National Science Foundation, the U.S. Department of Energy, the Japanese -Monbukagakusho, and the Max Planck Society. The SDSS Web site is -http://www.sdss.org/. -
- -The SDSS is managed by the Astrophysical Research Consortium (ARC) for the -Participating Institutions. The Participating Institutions are The University of Chicago, -Fermilab, the Institute for Advanced Study, the Japan Participation -Group, The Johns Hopkins University, Los Alamos National Laboratory, -the Max-Planck-Institute for -Astronomy (MPIA), the Max-Planck-Institute for Astrophysics (MPA), New -Mexico State University, Princeton University, the United States Naval -Observatory, and the University of Washington.
- --In this document: -
-Related documents: -
-Currently, the new reductions exist at Princeton in: -
- spectro.princeton.edu:/u/dss/spectro_test --The goal is to have at least 2 trusted people inspect each of the 388 -DR1 plates. - -
-Inspection will be done using plain text files in a central repository where -all SDSS folks have access. Template files have been auto-generated with -the redshifts and classifications from the pipelines. Alternatively, -Michael's SM scripts or my IDL code can be used to generate these simple -files on-the-fly. - -
-When someone inspects a plate, these files are edited and imported back -into this repository. - -
-There is a new product called "spinspect", which can be downloaded -(checked-out) or modified (checked-in) by a list of users that includes -the "alfred" account at Princeton. If you have access from your -Princeton account: -
- % bash - % CVSROOT=sdsscvs.astro.princeton.edu:/usr/local/cvsprivate - % CVS_RSH=ssh - % cvs co spinspect -
- % bash - % CVSROOT=alfred@sdsscvs.astro.princeton.edu:/usr/local/cvsprivate - % CVS_RSH=ssh - % cvs co spinspect -
- % cd ~/spinspect - % cvs update -
-The template files can be found in -
- spinspect/prototype/spInspect-$plate-$mjd.par -
- spinspect/data/$inspector/spInspect-$plate-$mjd-$inspector.par -
-There is also a mechanism for folks to submit a random set of objects from -multiple plates. Such a list can be put in files by the name: -
- spinspect/data/$inspector/spInspect-0000-00000-$inspector.par -
-From these files, we will be able to auto-generate files/web-pages with wrong -redshifts or interesting objects, and compare different people's ID's. -We can even auto-generate the contradiction lists and ask folks to -resolve conflicting ID's. - -
-If this is your first import and your name is "knapp", you need -to create your directory and add it to the "spinspect" product: -
- cd spinspect/data - mkdir knapp - cvs add knapp - cvs commit -m "Creating my data directory" knapp -
- cd spinspect/data/knapp - cvs add spInspect-0400-51820-knapp.par - cvs commit -m "I have inspected this" spInspect-0400-51820-knapp.par -
-The following set of problems are well-known. Problem reports already -exist for these problems, but you could reference the PR numbers in -the "comments" field of the inspection files. -
-New problems should be filed in the Bug Database -against the idlspec2d product for problems in the spectra, -or against specBS for problems in the classifications, redshifts, -or other measured quantities. - -
-For plate 400/51820, I edited the following 5 lines -in the file "spInspect-0400-51820-schlegel.par": -
-SPECINSPECT 400 51820 21 UNKNOWN BROADLINE 1.44154 schlegel "QSO" "" 1.44154 "PR 3797" -SPECINSPECT 400 51820 193 UNKNOWN BROADLINE 0.997749 schlegel "QSO" "" 0.997749 "PR 3797" -SPECINSPECT 400 51820 512 UNKNOWN BROADLINE 0.399743 schlegel "QSO" "" 0.399743 "PR 3797" -SPECINSPECT 400 51820 632 UNKNOWN BROADLINE 1.49835 schlegel "QSO" "" 1.49835 "PR 3797" -SPECINSPECT 400 51820 640 UNKNOWN "" 0.442294 schlegel "GALAXY" "" 0.442294 "Badskysub; PR 3399" -- -Below is the beginnings of a fake example file, with entries for -superpositions, etc: - -
-plate 400 -mjd 51820 -vers2d v4_9_8 -verscomb v4_9_8 -vers1d v4_9_8 -inspector knapp - -typedef struct { - int plate; - int mjd; - int fiberid; - char class[40]; - char subclass[40]; - float z; - char inspector[20]; - char manual_class[40]; - char manual_subclass[40]; - float manual_z; - char manual_comments[80]; -} SPECINSPECT; - -typedef struct { - int plate; - int mjd; - int fiberid; - char inspector[20]; - char manual_text[800]; -} SPECTEXT - -typedef struct { - int plate; - int mjd; - int fiberid; - char inspector[20]; - char manual_class[40]; - char manual_subclass[40]; - float manual_z; - char manual_comments[80]; -} SPECBLEND - -specinspect 400 51820 1 GALAXY "" 0.100000 knapp GALAXY "" 0.200 "Blend: galaxy+star" -specinspect 400 51820 2 UNKNOWN "" 0.123456 knapp "" "" 0 "" -specinspect 400 51820 3 SKY "" 0.654321 knapp "" "" 0 "" -specinspect 400 51820 4 STAR F2 0.000010 knapp "" G5 0 "" -specinspect 400 51820 5 GALAXY "" 0.100000 knapp ? "" 0 "" -specinspect 400 51820 6 GALAXY "" 0.100000 knapp UNKNOWN "" 0 "Artifact; Glitch at 5000 Ang" -... - -specblend 400 51820 1 GALAXY "" 0.100000 knapp STAR M 0.00001 "" - -specinspect 400 51820 6 knapp "This is the craziest \ - object that I have ever seen in my life. Someone \ - should really use a few nights of Keck time and see." -- -
Yesterday: ' $ - + 'MJD='+prevmjd+' | '] + + 'MJD='+prevmjd+''] textout = [textout, $ 'BOSS Spectro MJD '+mjdstr+' | '] textout = [textout, $ 'Tomorrow: ' $ - + 'MJD='+nextmjd+' |
SOS Summary Plots | ||
'$ + +'Arc Shift Plots'$ + +' | ||
'+platelist+' |
IDLSPEC2D version ' + vers2d + ' (' $ - + 'documentation).'] + '
IDLSPEC2D version ' + vers2d ];+ ' ('
+ $+ 'documentation).']
if keyword_set(run2d) then textout = [textout, '
RUN2D '+run2d ]
if (!version.release LT '5.4') then $
textout = [textout, $
@@ -380,6 +389,7 @@ pro apo_log2html, logfile, htmlfile, fps=fps, sdssv_sn2=sdssv_sn2
;---------------------------------------------------------------------------
; Loop over each plate
;---------------------------------------------------------------------------
+ disk_warnings = ''
for iplate=0, nplates-1 do begin
;----------
@@ -389,7 +399,7 @@ pro apo_log2html, logfile, htmlfile, fps=fps, sdssv_sn2=sdssv_sn2
thiscart = allcarts[iplate]
if keyword_set(fps) then thisfield = allfields[iplate] else thisfield = thisplate
textout = [textout, $
- apo_log_beginplate(thisplate, thiscart, thismjd, thisfield, camnames, outdir=outdir, fps=fps)]
+ sos_log_beginplate(thisplate, thiscart, thismjd, thisfield, camnames, outdir=outdir, fps=fps)]
;----------
; Find all biases and loop over each exposure number with any
@@ -418,7 +428,7 @@ pro apo_log2html, logfile, htmlfile, fps=fps, sdssv_sn2=sdssv_sn2
fields = ['PERCENTILE98']
formats = ['(i4)', '(f7.1)', '(f7.1)']
textout = [ textout, $
- apo_log_fields(pbias, fields, formats=formats) ]
+ sos_log_fields(pbias, fields, formats=formats) ]
endfor
endif
@@ -448,7 +458,7 @@ pro apo_log2html, logfile, htmlfile, fps=fps, sdssv_sn2=sdssv_sn2
fields = ['NGOODFIBER', 'XMID', 'XSIGMA']
formats = ['(i4)', '(f7.1)', '(f5.2)']
textout = [ textout, $
- apo_log_fields(pflats, fields, formats=formats) ]
+ sos_log_fields(pflats, fields, formats=formats) ]
endfor
endif
@@ -477,7 +487,7 @@ pro apo_log2html, logfile, htmlfile, fps=fps, sdssv_sn2=sdssv_sn2
formats = ['(f7.1)', '(f4.2)', '(i)', '(f5.2)']
fields = ['WAVEMID', 'BESTCORR', 'NLAMPS', 'WSIGMA']
textout = [ textout, $
- apo_log_fields(parcs, fields, formats=formats) ]
+ sos_log_fields(parcs, fields, formats=formats) ]
endfor
endif
@@ -510,33 +520,46 @@ pro apo_log2html, logfile, htmlfile, fps=fps, sdssv_sn2=sdssv_sn2
for iexp=0, nexp-1 do begin
textout = [ textout, $
- apo_log_fields(pscience[*,iexp], 'SKYPERSEC', $
+ sos_log_fields(pscience[*,iexp], 'SKYPERSEC', $
printnames='SKY/SEC', formats='(f8.2)') ]
endfor
;----------
; Output SN2 for science exposures
+
+ ;if keyword_set(fps) then this_sn2_15 = sn2_15 else this_sn2_15 = 0
+ if keyword_set(fps) and (long(thisfield) lt 100000) then this_sn2_15 = sn2_15 else this_sn2_15 = 0
+ this_sdssv_sn2 = 0 ; sdssv_sn2
+
for iexp=0, nexp-1 do begin
mjdstr = strtrim(string(thismjd),2)
platestr4 = config_to_string(thisplate)
expstring = string(pscience[*,iexp].expnum, format='(i8.8)')
- if not keyword_set(sdssv_sn2) then begin
- jpegfile1 = 'snplot-'+mjdstr+'-'+platestr4+'-'+expstring+'.jpeg'
- printnames = '(S/N)^2'
- textout = [ textout, $
- apo_log_fields(pscience[*,iexp], 'SN2', $
- printnames=printnames, formats='(f7.1)') ]
- endif else begin
- jpegfile1 = 'snplot-'+mjdstr+'-'+platestr4+'-'+expstring[0]+'.jpeg'
- printnames1 = '(S/N)^2'
- jpegfile_v2= 'snplot-sdssv-'+mjdstr+'-'+platestr4+'-'+expstring[0]+'.jpeg'
- printnames_v2 = 'v2 (S/N)^2'
- textout = [ textout, $
- apo_log_fields(pscience[*,iexp], ['SN2','SN2_V2'], $
- printnames=[printnames1,printnames_v2],$
- formats=['(f7.1)','(f7.1)']) ]
- endelse
+
+ jpegfile1 = 'snplot-'+mjdstr+'-'+platestr4+'-'+expstring[0]+'.jpeg'
+ jpegfile_15= 'snplot-sdssv15-'+mjdstr+'-'+platestr4+'-'+expstring[0]+'.jpeg'
+ jpegfile_v2= 'snplot-sdssv-'+mjdstr+'-'+platestr4+'-'+expstring[0]+'.jpeg'
+ sn2_lab = ['SN2']
+ sn2_pnames = ['(S/N)^2']
+ sn2_formats = ['(f7.1)']
+
+ if keyword_set(this_sdssv_sn2) then begin
+ sn2_lab = [sn2_lab,'SN2_V2']
+ sn2_pnames = [sn2_pnames, 'v2 (S/N)^2']
+ sn2_formats = [sn2_formats,'(f7.1)']
+ endif
+
+ if keyword_set(this_sn2_15) then begin
+ sn2_lab = [sn2_lab,'SN2_15']
+ sn2_pnames = [sn2_pnames, 'Mag15 (S/N)^2']
+ sn2_formats = [sn2_formats,'(f7.1)']
+ endif
+
+ textout = [ textout, $
+ sos_log_fields(pscience[*,iexp], sn2_lab, $
+ printnames=sn2_pnames,$
+ formats=sn2_formats) ]
endfor
@@ -550,15 +573,12 @@ pro apo_log2html, logfile, htmlfile, fps=fps, sdssv_sn2=sdssv_sn2
'FLAVOR', 'TOTAL', $
'CAMERA', '', $
'TOTALSN2', 0.0 )
- if keyword_set(sdssv_sn2) then begin
- rstruct = create_struct('MJD', 0L, $
- 'CONFIG', 0L, $
- 'EXPNUM', '', $
- 'TAI', '', $
- 'FLAVOR', 'TOTAL', $
- 'CAMERA', '', $
- 'TOTALSN2', 0.0, $
- 'TOTALSN2_v2', 0.0)
+ if keyword_set(this_sdssv_sn2) then begin
+ rstruct = struct_addtags(rstruct, create_struct('TOTALSN2_v2', 0.0 ))
+ endif
+
+ if keyword_set(this_sn2_15) then begin
+ rstruct = struct_addtags(rstruct, create_struct('TOTALSN2_15', 0.0 ))
endif
@@ -570,47 +590,64 @@ pro apo_log2html, logfile, htmlfile, fps=fps, sdssv_sn2=sdssv_sn2
; in the opLimits file (currently anything < 2.0 is bad).
if (pscience[icam,iexp].flavor EQ 'science' $
AND strmatch(pscience[icam,iexp].quality, 'excellent') $
- AND apo_checklimits('science', 'SN2', $
+ AND sos_checklimits('science', 'SN2', $
pscience[icam,iexp].camera, $
pscience[icam,iexp].sn2) EQ '') then begin
; AND pscience[icam,iexp].sn2 GE 2.0) then begin
ptotal[icam].totalsn2 = ptotal[icam].totalsn2 + $
pscience[icam,iexp].sn2
endif
- if keyword_set(sdssv_sn2) then begin
+ if keyword_set(this_sdssv_sn2) then begin
if (pscience[icam,iexp].flavor EQ 'science' $
AND strmatch(pscience[icam,iexp].quality, 'excellent') $
- AND apo_checklimits('science', 'SN2', $
+ AND sos_checklimits('science', 'SN2', $
pscience[icam,iexp].camera, $
pscience[icam,iexp].sn2_v2) EQ '') then begin
ptotal[icam].TOTALSN2_v2 = ptotal[icam].TOTALSN2_v2 + $
pscience[icam,iexp].sn2_v2
endif
endif
+ if keyword_set(this_sn2_15) then begin
+ if (pscience[icam,iexp].flavor EQ 'science' $
+ AND strmatch(pscience[icam,iexp].quality, 'excellent') $
+ AND sos_checklimits('science', 'SN2', $
+ pscience[icam,iexp].camera, $
+ pscience[icam,iexp].SN2_15) EQ '') then begin
+ ptotal[icam].TOTALSN2_15 = ptotal[icam].TOTALSN2_15 + $
+ pscience[icam,iexp].SN2_15
+ endif
+ endif
endfor
endfor
mjdstr = strtrim(string(thismjd),2)
platestr4 = config_to_string(thisplate)
- if not keyword_set(sdssv_sn2) then begin
- jpegfile = 'snplot-'+mjdstr+'-'+platestr4+'.jpeg'
- printnames = 'TOTAL (S/N)^2'
- textout = [ textout, $
- apo_log_fields(ptotal, 'TOTALSN2', $
- printnames=printnames, formats='(f7.1)') ]
- endif else begin
- jpegfile = 'snplot-'+mjdstr+'-'+platestr4+'.jpeg'
- printnames = 'TOTAL (S/N)^2'
- jpegfile_v2 = 'snplot-sdssv-'+mjdstr+'-'+platestr4+'.jpeg'
- printnames_v2 = 'TOTAL v2 (S/N)^2'
+ jpegfile1 = 'snplot-'+mjdstr+'-'+platestr4+'.jpeg'
+ jpegfile_15= 'snplot-sdssv15-'+mjdstr+'-'+platestr4+'.jpeg'
+ jpegfile_v2= 'snplot-sdssv-'+mjdstr+'-'+platestr4+'.jpeg'
+
+ sn2_lab = ['TOTALSN2']
+ sn2_pnames = ['TOTAL (S/N)^2']
+ sn2_formats = ['(f7.1)']
- textout = [ textout, $
- apo_log_fields(ptotal, ['TOTALSN2','TOTALSN2_V2'], $
- printnames=[printnames,printnames_v2], $
- formats=['(f7.1)','(f7.1)']) ]
- endelse
+ if keyword_set(this_sdssv_sn2) then begin
+ sn2_lab = [sn2_lab,'TOTALSN2_V2']
+ sn2_pnames = [sn2_pnames, 'Total v2 (S/N)^2']
+ sn2_formats = [sn2_formats,'(f7.1)']
+ endif
+
+ if keyword_set(this_sn2_15) then begin
+ sn2_lab = [sn2_lab,'TOTALSN2_15']
+ sn2_pnames = [sn2_pnames, 'Total Mag15 (S/N)^2']
+ sn2_formats = [sn2_formats,'(f7.1)']
+ endif
+
+ textout = [ textout, $
+ sos_log_fields(ptotal, sn2_lab, $
+ printnames=sn2_pnames,$
+ formats=sn2_formats) ]
endif
- textout = [textout, apo_log_endplate()]
+ textout = [textout, sos_log_endplate()]
;----------
; Print all WARNINGs and ABORTs for this plate
@@ -619,20 +656,36 @@ pro apo_log2html, logfile, htmlfile, fps=fps, sdssv_sn2=sdssv_sn2
else ii = -1
if (ii[0] NE -1) then begin
; Remove leading+trailing spaces
- addtext = strtrim(PPTEXT[ii].text, 2)
+ addtext_temp = strtrim(PPTEXT[ii].text, 2)
+ addtext = ''
; Remove the first word from each line (which is the name of the
; IDL proc that generated the warning or abort message)
- for jj=0, n_elements(addtext)-1 do $
- addtext[jj] = strmid( addtext[jj], strpos(addtext[jj],' ')+1 )
+ for jj=0, n_elements(addtext_temp)-1 do $
+ addtext_temp[jj] = strmid( addtext_temp[jj], strpos(addtext_temp[jj],' ')+1 )
+
+ foreach at, addtext_temp do begin
+ if strmatch(at, '*SOS disk*', /fold_case) eq 0 then begin
+ ;if keyword_set(addtext) then
+ addtext = [addtext, at]
+ endif else begin
+ at_sub = strjoin((strsplit(at,/extract))[-6:*],' ')
+ junk = where(strmatch(disk_warnings, at_sub, /fold_case), ct)
+ if ct eq 0 then begin
+ addtext = [addtext, at]
+ disk_warnings = [disk_warnings, at_sub]
+ endif
+ endelse
+ endforeach
+
addtext = repstr(addtext, 'WARNING', $
- 'WARNING')
+ 'WARNING')
addtext = repstr(addtext, 'ABORT', $
- 'ABORT')
+ 'ABORT')
textout = [textout, '
', addtext, ''] endif endfor - textout = [textout, apo_log_endfile()] + textout = [textout, sos_log_endfile()] for i=0, n_elements(textout)-1 do $ printf, html_lun, textout[i] diff --git a/pro/apo2d/apo_plotbias.pro b/pro/sos/sos_plotbias.pro similarity index 97% rename from pro/apo2d/apo_plotbias.pro rename to pro/sos/sos_plotbias.pro index b501afecb..826dfa67d 100644 --- a/pro/apo2d/apo_plotbias.pro +++ b/pro/sos/sos_plotbias.pro @@ -1,12 +1,12 @@ ;+ ; NAME: -; apo_plotbias +; sos_plotbias ; ; PURPOSE: ; Plot the histogram of bias values for all 4 cameras of a single exposure ; ; CALLING SEQUENCE: -; apo_plotbias, expnum, [ plotfile= ] +; sos_plotbias, expnum, [ plotfile= ] ; ; INPUTS: ; expnum - Exposure number @@ -51,7 +51,7 @@ ; 06-Dec-2000 Written by D. Schlegel, Princeton ;- ;------------------------------------------------------------------------------ -pro apo_plotbias, expnum, plotfile=plotfile +pro sos_plotbias, expnum, plotfile=plotfile if (n_params() LT 1) then begin print, 'Syntax - apoplotbias, expnum, [plotfile= ]' diff --git a/pro/apo2d/apo_plotsn.pro b/pro/sos/sos_plotsn.pro similarity index 76% rename from pro/apo2d/apo_plotsn.pro rename to pro/sos/sos_plotsn.pro index 58f8d21aa..1346c9b04 100644 --- a/pro/apo2d/apo_plotsn.pro +++ b/pro/sos/sos_plotsn.pro @@ -1,12 +1,12 @@ ;+ ; NAME: -; apo_plotsn +; sos_plotsn ; ; PURPOSE: ; Generate S/N plot for one plate from a FITS logfile written by APOREDUCE. ; ; CALLING SEQUENCE: -; apo_plotsn, logfile, plate, [ expnum=, plugdir=, plotfile= ] +; sos_plotsn, logfile, plate, [ expnum=, plugdir=, plotfile= ] ; ; INPUTS: ; logfile - Logfile as written by APOREDUCE. This is a FITS file @@ -33,7 +33,7 @@ ; BUGS: ; ; PROCEDURES CALLED: -; apo_checklimits() +; sos_checklimits() ; djs_lockfile() ; djs_unlockfile ; mrdfits @@ -45,8 +45,8 @@ ; 02-May-2000 Written by D. Schlegel, APO ;- ;------------------------------------------------------------------------------ -pro apo_plotsn, logfile, plate, expnum=expnum, plugdir=plugdir, $ - plotfile=plotfile, fps=fps, sdssv_sn2=sdssv_sn2 +pro sos_plotsn, logfile, plate, expnum=expnum, plugdir=plugdir, $ + plotfile=plotfile, fps=fps, sdssv_sn2=sdssv_sn2, ccd=ccd, sn2_15=sn2_15 if (NOT keyword_set(plate)) then return if (NOT keyword_set(plugdir)) then plugdir = './' @@ -58,8 +58,11 @@ pro apo_plotsn, logfile, plate, expnum=expnum, plugdir=plugdir, $ endelse platestr = plate_to_string(plate) - splog, 'Generating S/N plot for plate '+platestr - + if keyword_set(fps) then begin + splog, 'Generating S/N plot for Config '+platestr + endif else begin + splog, 'Generating S/N plot for plate '+platestr + endelse ;---------- ; Read the science frames for this plate @@ -81,14 +84,18 @@ pro apo_plotsn, logfile, plate, expnum=expnum, plugdir=plugdir, $ splog, 'debug:fullplugfile:',fullplugfile if (Not keyword_set(fps)) then begin - plugmap = readplugmap(fullplugfile,spd1,/deredden,/apotags, fibermask=fibermask,hdr=plhdr, /plates); included /deredden to match the SN2 in the html and plot-vivek + plugmap = readplugmap(fullplugfile,spd1,/deredden,/sostags, fibermask=fibermask,hdr=plhdr, /plates); included /deredden to match the SN2 in the html and plot-vivek endif else begin savdir=FILE_DIRNAME(plotfile) if strmatch(getenv('OBSERVATORY'), 'apo',/fold_case) eq 1 then begin - plugmap = readplugmap(fullplugfile, 1, /deredden, /apotags, fibermask=fibermask, hdr=plhdr, ccd='b1',savdir=savdir); included /deredden to match the SN2 in the html and plot-vivek + if not keyword_set(ccd) then ccd = 'b1' + sp = 1 endif else begin - plugmap = readplugmap(fullplugfile, 2, /deredden, /apotags, fibermask=fibermask, hdr=plhdr, ccd='b2',savdir=savdir); included /deredden to match the SN2 in the html and plot-vivek + if not keyword_set(ccd) then ccd = 'b2' + sp = 2 endelse + plugmap = readplugmap(fullplugfile, sp, /deredden, /sostags, fibermask=fibermask, hdr=plhdr, ccd=ccd,savdir=savdir); included /deredden to match the SN2 in the html and plot-vivek + endelse ;---------- ; Loop through reductions for all science frames, and add S/N @@ -106,11 +113,16 @@ pro apo_plotsn, logfile, plate, expnum=expnum, plugdir=plugdir, $ ; Test that the exposure falls within valid S/N^2 limits if keyword_set(sdssv_sn2) then begin - qkeep = apo_checklimits('science', 'SN2_v2', PPSCIENCE[ii].camera, $ + qkeep = sos_checklimits('science', 'SN2_v2', PPSCIENCE[ii].camera, $ PPSCIENCE[ii].sn2) NE 'red' endif else begin - qkeep = apo_checklimits('science', 'SN2', PPSCIENCE[ii].camera, $ - PPSCIENCE[ii].sn2) NE 'red' + if keyword_set(sn2_15) then begin + qkeep = sos_checklimits('science', 'SN2_15', PPSCIENCE[ii].camera, $ + PPSCIENCE[ii].sn2_15) NE 'red' + endif else begin + qkeep = sos_checklimits('science', 'SN2', PPSCIENCE[ii].camera, $ + PPSCIENCE[ii].sn2) NE 'red' + endelse endelse ; If EXPNUM is specified, then only use data from those exposure(s) @@ -151,9 +163,15 @@ pro apo_plotsn, logfile, plate, expnum=expnum, plugdir=plugdir, $ + ' '+var_str+'=' + strtrim(string(plate),2) sncode = 'sos2' endif else begin - plottitle = 'BOSS Spectro MJD=' + strtrim(string(mjd),2) $ - + ' '+var_str+'=' + strtrim(string(plate),2) - sncode = 'sos' + if keyword_set(sn2_15) then begin + plottitle = 'BOSS SDSS-V Mag=15 Spectro MJD=' + strtrim(string(mjd),2) $ + + ' '+var_str+'=' + strtrim(string(plate),2) + sncode = 'sos15' + endif else begin + plottitle = 'BOSS Spectro MJD=' + strtrim(string(mjd),2) $ + + ' '+var_str+'=' + strtrim(string(plate),2) + sncode = 'sos' + endelse endelse if (keyword_set(expnum)) then $ plottitle += ' exp=' + strtrim(expnum[0],2) diff --git a/pro/apo2d/aporeduce.pro b/pro/sos/sosreduce.pro similarity index 82% rename from pro/apo2d/aporeduce.pro rename to pro/sos/sosreduce.pro index be4ef3268..cf0df7e18 100644 --- a/pro/apo2d/aporeduce.pro +++ b/pro/sos/sosreduce.pro @@ -1,12 +1,12 @@ ;+ ; NAME: -; aporeduce +; sosreduce ; ; PURPOSE: ; Quick on-the-mountain reduction pipeline for 1 file at a time. ; ; CALLING SEQUENCE: -; aporeduce, filename, [ indir=, outdir=, $ +; sosreduce, filename, [ indir=, outdir=, $ ; plugfile=, plugdir=, minexp=, $ ; copydir=, /no_diskcheck, /no_lock ] ; @@ -51,12 +51,12 @@ ; scp1 does not exist on sos.apo.nmsu.edu, reverted to scp ; ; INTERNAL SUPPORT ROUTINES: -; apo_diskcheck +; sos_diskcheck ; ; PROCEDURES CALLED: -; apo_appendlog -; apo_log2html -; apo_plotsn +; sos_appendlog +; sos_log2html +; sos_plotsn ; djs_filepath() ; fits_wait() ; get_tai @@ -137,7 +137,7 @@ end ;------------------------------------------------------------------------------ ; Check disk space on the input or output disk. -pro apo_diskcheck, dirname +pro sos_diskcheck, dirname if (NOT keyword_set(dirname)) then return @@ -156,12 +156,13 @@ pro apo_diskcheck, dirname return end ;------------------------------------------------------------------------------ -pro aporeduce, filename, indir=indir, outdir=outdir, $ +pro sosreduce, filename, indir=indir, outdir=outdir, $ plugfile=plugfile, plugdir=plugdir, minexp=minexp, nocal=nocal,$ copydir=copydir, no_diskcheck=no_diskcheck, no_lock=no_lock, $ - fps=fps, noreject=noreject, sdssv_sn2=sdssv_sn2 + fps=fps, noreject=noreject, sdssv_sn2=sdssv_sn2, sn2_15=sn2_15,$ + arc2trace=arc2trace, forcea2t=forcea2t if (n_params() LT 1) then begin - doc_library, 'aporeduce' + doc_library, 'sosreduce' return endif @@ -189,7 +190,7 @@ pro aporeduce, filename, indir=indir, outdir=outdir, $ if (n_elements(filename) GT 1) then begin for ifile=0, n_elements(filename)-1 do $ - aporeduce, filename[ifile], indir=indir, outdir=outdir, $ + sosreduce, filename[ifile], indir=indir, outdir=outdir, $ plugfile=plugfile, plugdir=plugdir, minexp=minexp, $ copydir=copydir, no_diskcheck=no_diskcheck, no_lock=no_lock, $ fps=fps, noreject=noreject @@ -221,13 +222,14 @@ pro aporeduce, filename, indir=indir, outdir=outdir, $ return endif + if keyword_set(forcea2t) then arc2trace = 1 ;---------- ; Open the log file to catch WARNINGs and ABORTs. splgfile = filepath('splog-'+filec+'-'+filee+'.log', root_dir=outdir) splog, filename=splgfile, prelog=filename - splog, 'Log file ' + splgfile + ' opened' + systime() + splog, 'Log file ' + splgfile + ' opened ' + systime() t0 = systime(1) splog, 'IDL version: ' + string(!version,format='(99(a," "))') @@ -240,8 +242,8 @@ pro aporeduce, filename, indir=indir, outdir=outdir, $ ; Check disk space on both the input and the output disk. if (NOT keyword_set(no_diskcheck)) then begin - apo_diskcheck, indir - apo_diskcheck, outdir + sos_diskcheck, indir + sos_diskcheck, outdir endif ;---------- @@ -432,6 +434,10 @@ pro aporeduce, filename, indir=indir, outdir=outdir, $ if (flatexist) then begin rstruct = quickwave(fullname, tsetfile_last, wsetfile1, noreject=noreject,$ fflatfile1, lco = lco, do_lock=do_lock, nocal=nocal) + if keyword_set(arc2trace) then begin + setup_arc2trace, tsetfile_last, fflatfile1, fullname, indir, outdir, mjd, camnames[icam], fieldstr + + endif endif else begin splog, 'INFO: Arc exposure, waiting for flat before reducing' endelse @@ -444,9 +450,9 @@ pro aporeduce, filename, indir=indir, outdir=outdir, $ if (camnames[icam] eq 'r1') or (camnames[icam] eq 'r2') then splitsky = 1B else splitsky = 0B if (flatexist AND arcexist AND exptime GE minexp) then begin rstruct = quickextract(tsetfile_last, wsetfile_last, $ - fflatfile_last, fullname, outsci, fullplugfile, outdir,$ + fflatfile_last, fullname, outsci, fullplugfile, outdir, mjd,$ splitsky=splitsky, do_lock=do_lock,threshold=threshold,$ - sdssv_sn2=sdssv_sn2) + sdssv_sn2=sdssv_sn2,sn2_15=sn2_15,arc2trace=arc2trace,forcea2t=forcea2t) endif else begin if (NOT keyword_set(flatexist)) then $ splog, 'ABORT: Unable to reduce this science exposure (need flat)' @@ -548,7 +554,7 @@ pro aporeduce, filename, indir=indir, outdir=outdir, $ if (keyword_set(tstruct) OR keyword_set(rstruct)) then begin splog, 'Appending to FITS log file '+logfile - apo_appendlog, logfile, rstruct, tstruct + sos_appendlog, logfile, rstruct, tstruct splog, 'Done with append' endif @@ -568,7 +574,7 @@ pro aporeduce, filename, indir=indir, outdir=outdir, $ jpegfiletmp1 = filepath('snplot-'+mjdstr+'-'+confstr+'-'+filee+'-'+filec+'.jpeg', root_dir=outdir) jpegfile1 = filepath('snplot-'+mjdstr+'-'+confstr+'-'+filee+'.jpeg', root_dir=outdir) splog, 'Generating S/N plot '+plotfile1 - apo_plotsn, logfile, config, expnum=long(filee), plugdir=plugdir, plotfile=plotfile1, fps=fps + sos_plotsn, logfile, config, expnum=long(filee), plugdir=plugdir, plotfile=plotfile1, fps=fps, ccd=string(camnames[icam]) cmd = '/usr/bin/convert '+plotfile1+' '+jpegfiletmp1+' ; \mv '+jpegfiletmp1+' '+jpegfile1+' &' splog, 'SPAWN '+cmd, sh_out, sh_err spawn, cmd @@ -581,21 +587,50 @@ pro aporeduce, filename, indir=indir, outdir=outdir, $ jpegfile = filepath('snplot-'+mjdstr+'-'+confstr+'.jpeg', root_dir=outdir) jpegfiletmp = filepath('snplot-'+mjdstr+'-'+confstr+'-'+filec+'.jpeg', root_dir=outdir) splog, 'Generating S/N plot '+plotfile - apo_plotsn, logfile, config, plugdir=plugdir, plotfile=plotfile, fps=fps + sos_plotsn, logfile, config, plugdir=plugdir, plotfile=plotfile, fps=fps, ccd=string(camnames[icam]) cmd = '/usr/bin/convert '+plotfile+' '+jpegfiletmp+' ; \mv '+jpegfiletmp+' '+jpegfile+' &' splog, 'SPAWN '+cmd, sh_out, sh_err spawn, cmd splog, 'SPAWN out=', sh_out splog, 'SPAWN err=', sh_err splog, 'Done generating plot' + if keyword_set(sdssv_sn2) then begin ; Generate the added S/N^2 for this one exposure only - plotfile1 = filepath('snplot-sdssv-'+mjdstr+'-'+confstr+'-'+filee+'.ps', root_dir=outdir) - jpegfiletmp1 = filepath('snplot-sdssv-'+mjdstr+'-'+confstr+'-'+filee+'-'+filec+'.jpeg', root_dir=outdir) - jpegfile1 = filepath('snplot-sdssv-'+mjdstr+'-'+confstr+'-'+filee+'.jpeg', root_dir=outdir) + plotfile1_v2 = filepath('snplot-sdssv-'+mjdstr+'-'+confstr+'-'+filee+'.ps', root_dir=outdir) + jpegfiletmp1_v2 = filepath('snplot-sdssv-'+mjdstr+'-'+confstr+'-'+filee+'-'+filec+'.jpeg', root_dir=outdir) + jpegfile1_v2 = filepath('snplot-sdssv-'+mjdstr+'-'+confstr+'-'+filee+'.jpeg', root_dir=outdir) + splog, 'Generating SDSS-V S/N plot '+plotfile1_v2 + sos_plotsn, logfile, config, expnum=long(filee), plugdir=plugdir,$ + plotfile=plotfile1_v2, fps=fps,sdssv_sn2=sdssv_sn2, ccd=string(camnames[icam]) + cmd = '/usr/bin/convert '+plotfile1_v2+' '+jpegfiletmp1_v2+' ; \mv '+jpegfiletmp1_v2+' '+jpegfile1_v2+' &' + splog, 'SPAWN '+cmd, sh_out, sh_err + spawn, cmd + splog, 'SPAWN out=', sh_out + splog, 'SPAWN err=', sh_err + splog, 'Done generating plot' + + ; Generate the added S/N^2 for all exposures on this plate + plotfile_v2 = filepath('snplot-sdssv-'+mjdstr+'-'+confstr+'.ps', root_dir=outdir) + jpegfile_v2 = filepath('snplot-sdssv-'+mjdstr+'-'+confstr+'.jpeg', root_dir=outdir) + jpegfiletmp_v2 = filepath('snplot-sdssv-'+mjdstr+'-'+confstr+'-'+filec+'.jpeg', root_dir=outdir) + splog, 'Generating SDSS-V S/N plot '+plotfile_v2 + sos_plotsn, logfile, config, plugdir=plugdir, plotfile=plotfile_v2, fps=fps,sdssv_sn2=sdssv_sn2, ccd=string(camnames[icam]) + cmd = '/usr/bin/convert '+plotfile_v2+' '+jpegfiletmp_v2+' ; \mv '+jpegfiletmp_v2+' '+jpegfile_v2+' &' + splog, 'SPAWN '+cmd, sh_out, sh_err + spawn, cmd + splog, 'SPAWN out=', sh_out + splog, 'SPAWN err=', sh_err + splog, 'Done generating SDSS-V plot' + endif + if keyword_set(sn2_15) then begin + ; Generate the added S/N^2 for this one exposure only + plotfile1 = filepath('snplot-sdssv15-'+mjdstr+'-'+confstr+'-'+filee+'.ps', root_dir=outdir) + jpegfiletmp1 = filepath('snplot-sdssv15-'+mjdstr+'-'+confstr+'-'+filee+'-'+filec+'.jpeg', root_dir=outdir) + jpegfile1 = filepath('snplot-sdssv15-'+mjdstr+'-'+confstr+'-'+filee+'.jpeg', root_dir=outdir) splog, 'Generating SDSS-V S/N plot '+plotfile1 - apo_plotsn, logfile, config, expnum=long(filee), plugdir=plugdir,$ - plotfile=plotfile1, fps=fps,sdssv_sn2=sdssv_sn2 + sos_plotsn, logfile, config, expnum=long(filee), plugdir=plugdir,$ + plotfile=plotfile1, fps=fps,sn2_15=sn2_15, ccd=string(camnames[icam]) cmd = '/usr/bin/convert '+plotfile1+' '+jpegfiletmp1+' ; \mv '+jpegfiletmp1+' '+jpegfile1+' &' splog, 'SPAWN '+cmd, sh_out, sh_err spawn, cmd @@ -604,22 +639,22 @@ pro aporeduce, filename, indir=indir, outdir=outdir, $ splog, 'Done generating plot' ; Generate the added S/N^2 for all exposures on this plate - plotfile = filepath('snplot-sdssv-'+mjdstr+'-'+confstr+'.ps', root_dir=outdir) - jpegfile = filepath('snplot-sdssv-'+mjdstr+'-'+confstr+'.jpeg', root_dir=outdir) - jpegfiletmp = filepath('snplot-sdssv-'+mjdstr+'-'+confstr+'-'+filec+'.jpeg', root_dir=outdir) - splog, 'Generating SDSS-V S/N plot '+plotfile - apo_plotsn, logfile, config, plugdir=plugdir, plotfile=plotfile, fps=fps,sdssv_sn2=sdssv_sn2 + plotfile = filepath('snplot-sdssv15-'+mjdstr+'-'+confstr+'.ps', root_dir=outdir) + jpegfile = filepath('snplot-sdssv15-'+mjdstr+'-'+confstr+'.jpeg', root_dir=outdir) + jpegfiletmp = filepath('snplot-sdssv15-'+mjdstr+'-'+confstr+'-'+filec+'.jpeg', root_dir=outdir) + splog, 'Generating SDSS-V Mag 15 S/N plot '+plotfile + sos_plotsn, logfile, config, plugdir=plugdir, plotfile=plotfile, fps=fps,sn2_15=sn2_15, ccd=string(camnames[icam]) cmd = '/usr/bin/convert '+plotfile+' '+jpegfiletmp+' ; \mv '+jpegfiletmp+' '+jpegfile+' &' splog, 'SPAWN '+cmd, sh_out, sh_err spawn, cmd splog, 'SPAWN out=', sh_out splog, 'SPAWN err=', sh_err - splog, 'Done generating SDSS-V plot' + splog, 'Done generating SDSS-V Mag 15 plot' endif endif splog, 'Generating HTML file '+htmlfile - apo_log2html, logfile, htmlfile, fps=fps, sdssv_sn2=sdssv_sn2 + sos_log2html, logfile, htmlfile, fps=fps, sn2_15=sn2_15;, sdssv_sn2=sdssv_sn2 splog, 'Done generating HTML file' ; Generate a copy of the HTML file, 'logsheet-current.html', @@ -635,13 +670,28 @@ pro aporeduce, filename, indir=indir, outdir=outdir, $ spawn, 'sed ' + sedcommand + ' ' + htmlfile + ' > ' + currentfile if (keyword_set(copydir)) then begin + FILE_MKDIR, copydir splog, 'Copying files to ', copydir spawn, 'scp ' + htmlfile + ' ' + copydir spawn, 'scp ' + currentfile + ' ' + copydir - spawn, 'scp ' + logfile + ' ' + copydir - if (keyword_set(plotfile)) then $ - spawn, 'scp ' + plotfile + ' ' + plotfile1 $ - + ' ' + jpegfile + ' ' + jpegfile1 + ' ' + copydir + htmlfile_c = djs_filepath(file_basename(htmlfile), root_dir=copydir) + currentfile_c = djs_filepath(file_basename(currentfile), root_dir=copydir) + + yesterday = strtrim((long(mjd)-1),2) + sedcommand = ' -e "s/Yesterday: ' + htmlfile_c + spawn, 'sed ' + sedcommand + ' ' + currentfile + ' > ' + currentfile_c + + +; spawn, 'scp ' + logfile + ' ' + copydir +; if (keyword_set(plotfile)) then $ +; spawn, 'scp ' + plotfile + ' ' + plotfile1 $ +; + ' ' + jpegfile + ' ' + jpegfile1 + ' ' + copydir +; if (keyword_set(plotfile_v2)) then $ +; spawn, 'scp ' + plotfile_v2 + ' ' + plotfile1_v2 $ +; + ' ' + jpegfile_v2 + ' ' + jpegfile1_v2 + ' ' + copydir splog, 'Done.' endif endif diff --git a/pro/spec1d/.DS_Store b/pro/spec1d/.DS_Store deleted file mode 100644 index 5008ddfcf..000000000 Binary files a/pro/spec1d/.DS_Store and /dev/null differ diff --git a/pro/spec1d/RED b/pro/spec1d/RED deleted file mode 100644 index c376a5084..000000000 --- a/pro/spec1d/RED +++ /dev/null @@ -1,68 +0,0 @@ - -red344 = redindices(344) -red492 = redindices(492) -redall = [red344,red492] - -splot,redall.ew_mg2,redall.modelcolor[0],ps=4 -soplot,redall[i].ew_mg2,redall[i].modelcolor[0],ps=1,color='red',syms=2 - -i344=where(redall.plate EQ 344 AND redall.redsample) -i492=where(redall.plate EQ 492 AND redall.redsample) -i344=where(redall.plate EQ 344) -i492=where(redall.plate EQ 492) - -!p.multi = [0,1,2] -xrange = [0,3] -yrange = [-12,-2] -model_kcorr = redall.modelcolor[0] - 2*redall.z -model_dered = model_kcorr - redall.reddening[0] + redall.reddening[1] -djs_plot,model_kcorr[i344],redall[i344].ew_mg2,ps=1,color='red', $ - xr=xrange,yr=yrange -djs_oplot,model_kcorr[i492],redall[i492].ew_mg2,ps=4 -djs_plot,model_dered[i344],redall[i344].ew_mg2,ps=1,color='red', $ - xr=xrange,yr=yrange -djs_oplot,model_dered[i492],redall[i492].ew_mg2,ps=4 - -;------------------------------------------------------------------------------ -plate = [414,415,416,428,464,453,488,489,491,492] -redall = redindices(plate[0]) -for i=1,n_elements(plate)-1 do redall = [redall, redindices(plate[i])] -save, file='redall.ss' - -restore, 'redall.ss' -i1 = where(redall.reddening[0] LT 0.25 AND redall.redsample) -i2 = where(redall.reddening[0] GT 0.25 AND redall.redsample) -djs_plot,redall[i1].ew_mg2,redall[i1].modelcolor[0],ps=4 -djs_oplot,redall[i2].ew_mg2,redall[i2].modelcolor[0],ps=4,color='red' - -model_kcorr = redall.modelcolor[0] - 2*redall.z -model_dered = model_kcorr - redall.reddening[0] + redall.reddening[1] -!p.multi = [0,1,2] -yrange=[1.0,2.5] -djs_plot,redall[i1].ew_mg2,model_kcorr[i1],ps=1,yr=yrange,/ystyle -djs_oplot,redall[i2].ew_mg2,model_kcorr[i1],ps=1,color='red' -djs_plot,redall[i1].ew_mg2,model_dered[i1],ps=1,yr=yrange,/ystyle -djs_oplot,redall[i2].ew_mg2,model_dered[i1],ps=1,color='red' - -; Find residuals to Mg-color -acoeff = linfit(redall[i1].ew_mg2,model_dered[i1]) -cresid = model_kcorr - (acoeff[0] + acoeff[1] * redall.ew_mg2) -ugpredict = redall.reddening[0] - redall.reddening[1] -j = where(redall.redsample) -!p.multi=0 -djs_plot,ugpredict[j],cresid[j],ps=1, $ - xtitle='SFD (u-g)',ytitle='Residual (u-g)',charsize=2 -djs_oplot, [0,1], [0,1] - -;---------------------- -; Try a dispersion-color relation instead of Mg-color... -i=where(redall.reddening[0] LT 0.25 AND redall.redsample) -acoeff=linfit(redall[i].vdisp,redall[i].modelcolor[0]) -splot,alog10(redall[i].vdisp),redall[i].modelcolor[0],ps=1 -splot,alog10(redall[i].vdisp),redall[i].modelcolor[0] $ -- acoeff[0] - acoeff[1]*alog10(redall[i].vdisp),ps=1 -cresid = redall.modelcolor[0] - (acoeff[0] + acoeff[1] * alog10(redall.vdisp)) -ugpredict = redall.reddening[0] - redall.reddening[1] -j = where(redall.redsample) -splot,ugpredict[j],cresid[j],ps=4 - diff --git a/pro/spec1d/TODO b/pro/spec1d/TODO deleted file mode 100644 index b99de4d7e..000000000 --- a/pro/spec1d/TODO +++ /dev/null @@ -1,27 +0,0 @@ - -* Fix fudge bug fix in TWEAK_GAL code. Also, apodize the ends of the input - spectrum to minimize edge effects, or trim to points common to all test z's. - -* Output synthetic ugriz colors for each object measured from the actual - spectrum and from the synthesized spectrum (allowing u+z to be measured). - -* Some z errors are negative! - -* For PCA_SOLVE, trim to desired wavelength range before solving. - -* For eigenspectra, save a 2nd HDU which is the remaining inverse-variance. - -* Look at the e1/e0, e2,e0 values for the galaxies. - ---------------DONE------------------------------------------------------------- -* Write PLATELIST routine to list reduced plates. - -* For PCA_VSTND, get rid of bad endpoints made with too little data. - -* Final correlation coeff should be computed with same wavelength-range - for template, and masked + apodized the same as the object spectrum. - -* Minimize reduced chi2 instead of chi2 ? - -* Still need to compute errors in redshift, but how? - diff --git a/pro/spec1d/bandpassfilter.pro b/pro/spec1d/bandpassfilter.pro deleted file mode 100644 index d16d530b4..000000000 --- a/pro/spec1d/bandpassfilter.pro +++ /dev/null @@ -1,70 +0,0 @@ -;+ -; NAME: -; bandpassfilter -; -; PURPOSE: -; Apply a bandpass to Fourier-transformed data. -; -; CALLING SEQUENCE: -; newdata = bandpassfilter( datafft, [ klo_cut=, khi_cut= ] ) -; -; INPUTS: -; datafft - Vector of Fourier-transformed data -; -; OPTIONAL KEYWORDS: -; klo_cut - Low-frequency cutoff -; khi_cut - High-frequency cutoff -; -; OUTPUTS: -; newdata - Filtered version of DATAFFT -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; Units??? -; -; EXAMPLES: -; -; BUGS: -; -; PROCEDURES CALLED: -; -; REVISION HISTORY: -; 30-Mar-2000 Written by D. Schlegel, APO -;- -;------------------------------------------------------------------------------ -function bandpassfilter, datafft, klo_cut=klo_cut, khi_cut=khi_cut - - if (size(datafft, /n_dimen) NE 1) then $ - message, 'DATAFFT is not 1-dimensional' - - if (NOT keyword_set(klo_cut) AND NOT keyword_set(khi_cut)) then $ - return, datafft - - if (size(datafft, /tname) EQ 'DOUBLE') then PI = !dpi $ - else PI = !pi - - ndata = N_elements(datafft) - knums = fft_wavenums(ndata) - hipass = fltarr(ndata) + 1.0 - lopass = fltarr(ndata) + 1.0 - - if (keyword_set(klo_cut)) then begin - ii = where(abs(knums) LT klo_cut) - if (ii[0] NE -1) then begin - hipass[ii] = 0.5 * (1.0 - cos(PI*abs(knums[ii])/klo_cut)) - endif - endif - - if (keyword_set(khi_cut)) then begin - ii = where(abs(knums) GT khi_cut) - if (ii[0] NE -1) then begin - sep = max(knums) - khi_cut - lopass[ii] = $ - 0.5 * (1.0 - cos(PI*(max(knums) - abs(knums[ii]))/sep)) - endif - endif - - return, datafft * hipass * lopass -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/bin_spectra.pro b/pro/spec1d/bin_spectra.pro deleted file mode 100644 index 6046ddaae..000000000 --- a/pro/spec1d/bin_spectra.pro +++ /dev/null @@ -1,48 +0,0 @@ -;+ -; NAME: -; bin_spectra -; -; PURPOSE: -; Make binned spectra for the purposes of fitting for the -; bandpasses -; -; CALLING SEQUENCE: -; bin_spectra, flux, binbounds -; -; INPUTS: -; flux - flux in a set of spectra [NPIX,NSPEC] -; invvar - inverse variance in a set of spectra [NPIX,NSPEC] -; binbounds - boundaries of desired bins -; -; OPTIONAL INPUTS: -; -; OPTIONAL KEYWORDS: -; -; OUTPUTS: -; binflux - binned flux -; -; COMMENTS: -; Not currently binning the variances. Sorry. -; -; EXAMPLES: -; -; PROCEDURES CALLED: -; -; DATA FILES: -; -; REVISION HISTORY: -; 05-APr-2000 Written by M. Blanton, Fermiland -;- -;------------------------------------------------------------------------------ -pro bin_spectra, flux, invvar, binbounds, binflux=binflux - - for i=0, n_elements(binbounds)-2 do begin - denominator= total(invvar[binbounds[i]:binbounds[i+1]-1,*],1,/double) - denominator= denominator + (denominator LE 0.0) - binflux[i,*]= $ - total((flux*invvar)[binbounds[i]:binbounds[i+1]-1,*],1,/double) / $ - denominator - endfor - -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/bolton_mlpca.pro b/pro/spec1d/bolton_mlpca.pro deleted file mode 100644 index ec09fa7a7..000000000 --- a/pro/spec1d/bolton_mlpca.pro +++ /dev/null @@ -1,110 +0,0 @@ -;+ -; -; NAME: -; bolton_mlpca -; -; PURPOSE: -; Bolton's maximum-likelihood PCA implementation for astronomy -; -; USAGE: -; tempset = bolton_mlpca(objflux=objflux, objivar=objivar [, $ -; nbasis=nbasis , convtest=convtest, maxiter=maxiter, $ -; coeffs=coeffs]) -; -; ARGUMENTS: -; objflux: NPIX x NOBJ array of REST-FRAME ALIGNED object spectra -; objivar: NPIX x NOBJ inverse variance array for objflux -; -; OPTIONAL INPUTS: -; nbasis: number of basis spectra to compute (default is 5) -; convtest: convergence param. to test for completion (default 1.d-10) -; maxiter: maximum number of iterations (default is 10000) -; -; RETURNS: -; tempset: an NPIX x NBASIS array of object template basis spectra -; coeffs: the best-fit coeffs of the input pectrum sample, -; such that tempset # coeffs gives the minimum chi2 model -; describing objflux -; -; WRITTEN: -; A. Bolton, U. of Utah, 2010 May -; -; COMMENTS: -; Implements the iterative Maximum-Likelihood PCA algorithm -; (no intercepts, uncorrelated errors) described by -; Wentzell et al. 1997, Journal of Chemometrics, 11, 339 -; -;- - -function bolton_mlpca, objflux=objflux, objivar=objivar, $ - nbasis=nbasis, convtest=convtest, maxiter=maxiter, coeffs=coeffs - -; Defaults: -if (not keyword_set(nbasis)) then nbasis = 5L -if (not keyword_set(convtest)) then convtest = 1.d-10 -if (not keyword_set(maxiter)) then maxiter = 10000 - -; Transpose and double-precisionize the data. -; Transposition is done to make these look right as matrices -; to the singular-value decompositon routines. -data = double(transpose(objflux)) -ivar = double(transpose(objivar)) -; Zero out zero-ivar pixels: -data = data * (ivar gt 0.d0) - -; For convenience: -nobj = (size(data))[1] -npix = (size(data))[2] -b_ones = replicate(1.d0, nbasis) -coeffs = replicate(0.d0, nbasis, nobj) -; The following assignment seeds the iterative process: -model = data - -niter = 0L -convergence_param = 1.d12 -while ((convergence_param gt convtest) and (niter lt maxiter)) do begin - niter++ - -; Truncated SVD in the object space: - la_svd, model, w, u, v, /double - vhat = v[0:nbasis-1,*] - -; Loop over pixels to find ML model: - for i = 0L, npix-1 do begin - ivhat = vhat * (b_ones # ivar[*,i]) - ialpha = ivhat # transpose(vhat) - alpha = invert(ialpha) - beta = ivhat # data[*,i] - model[*,i] = transpose(vhat) # (alpha # beta) - endfor - - chisq1 = total((data - model)^2 * ivar) - -; Truncated SVD in the pixel space: - la_svd, transpose(model), w, u, v, /double - vhat = v[0:nbasis-1,*] - -; Loop over objects to find ML model: - for i = 0, nobj-1 do begin - ivhat = vhat * (b_ones # ivar[i,*]) - ialpha = ivhat # transpose(vhat) - alpha = invert(ialpha) - beta = ivhat # transpose(data[i,*]) - coeffs[*,i] = alpha # beta -; model[i,*] = transpose(vhat) # (alpha # beta) - endfor - model = transpose(coeffs) # vhat - - chisq2 = total((data - model)^2 * ivar) - convergence_param = (chisq1 - chisq2) / chisq2 - print, niter, chisq2, convergence_param -endwhile - -; Tidy up by making all the basis functions non-negative: -vtot = total(vhat, 2) -signflip = double(1L - 2L * long(vtot lt 0.d0)) -vhat = vhat * (signflip # replicate(1.d0, npix)) -coeffs = coeffs * (signflip # replicate(1.d0, nobj)) - -return, transpose(vhat) -end diff --git a/pro/spec1d/bolton_mlpca_demo.pro b/pro/spec1d/bolton_mlpca_demo.pro deleted file mode 100644 index 23e5320de..000000000 --- a/pro/spec1d/bolton_mlpca_demo.pro +++ /dev/null @@ -1,95 +0,0 @@ -; -; bolton_mlpca_demo -; -; Demonstrator script for Bolton's ML-PCA EigenGal implementation. -; -; bolton@utah 2010may07 -; - -; Require the following, because that's what I've vetted this on: -setenv, 'RUN2D=v5_4_9' -setenv, 'RUN1D=v5_4_9' - -; The uber-deep plate: -plate = 3851 -mjd = 55298 - -; Get all sensible galaxies: -readspec, plate, mjd=mjd, zans=zans -nobj = n_elements(zans) -useit = replicate(1B, nobj) -; Things to mask following inspection, as detailed above: -maskid = [119,149,151,152,248,315,426,431,433]-1 -useit[maskid] = 0B -idx = where(useit and (strtrim(zans.objtype,2) eq 'GALAXY') $ - and (strtrim(zans.class,2) eq 'GALAXY'), ngal) -zans = zans[idx] - -readspec, plate, zans.fiberid, mjd=mjd, flux=flux, synflux=synflux, $ - loglam=loglam, zans=zans, wave=wave, invvar=invvar, tsobj=tsobj, /align - -; Get everthing aligned in rest-frame wavelength: -npix = (size(flux))[1] -dloglam = loglam[1] - loglam[0] -pixshift = round(alog10(1. + zans.z) / dloglam) -mp = max(pixshift) -totalpix = npix + mp -rest_flux = replicate(0., totalpix, ngal) -rest_ivar = replicate(0., totalpix, ngal) -rest_synflux = replicate(0., totalpix, ngal) - -for i = 0L, ngal-1 do begin & $ - rest_flux[mp-pixshift[i]:mp-pixshift[i]+npix-1,i] = flux[*,i] & $ - rest_ivar[mp-pixshift[i]:mp-pixshift[i]+npix-1,i] = invvar[*,i] & $ - rest_synflux[mp-pixshift[i]:mp-pixshift[i]+npix-1,i] = synflux[*,i] & $ -endfor - -; Make the rest-frame loglam baseline: -loglam0_rest = loglam[0] - mp * dloglam -loglam_rest = loglam0_rest + dloglam * findgen(totalpix) - -; Impose a minimum chi^2 cut on the extreme ends -; (somewhat arbitrarily set at "one sigma" of info per galaxy): -chi2vec = total(rest_flux^2 * rest_ivar, 2) -chi2filt = median(chi2vec, 15) -chi2min = float(ngal) -irange = minmax(where(chi2filt ge chi2min)) -rest_flux = rest_flux[irange[0]:irange[1],*] -rest_ivar = rest_ivar[irange[0]:irange[1],*] -rest_synflux = rest_synflux[irange[0]:irange[1],*] -loglam_rest = loglam_rest[irange[0]:irange[1]] -totalpix = n_elements(loglam_rest) - -; Cut it down again -- not enough galaxies covering -; the highest rest wavelengths in this sample. -; Including those sparsely covered ranges makes the -; algorithm go unstable. -; -; We can fix this with inclusion of more low-z galaxies. -p_lo = 0 -p_hi = 4799 -rest_flux = rest_flux[p_lo:p_hi,*] -rest_synflux = rest_synflux[p_lo:p_hi,*] -rest_ivar = rest_ivar[p_lo:p_hi,*] -loglam_rest = loglam_rest[p_lo:p_hi] - -galtempset = bolton_mlpca(objflux=rest_flux, objivar=rest_ivar, coeffs=coeffs) - -; Check these out: -modflux = galtempset # coeffs -i = -1L - -; Up-arrow repeat: -i++ & splot, 10.^loglam_rest, rest_flux[*,i], yrange=[-1.,7.] & soplot, 10.^loglam_rest, modflux[*,i], color=3 - -; Copied from pca_gal: -get_juldate, jd -mjdstr = STRING(LONG(jd-2400000L), FORMAT='(I5)') -outfile = 'spMlpcaGal-' + mjdstr + '.fits' -sxaddpar, hdr, 'OBJECT', 'GALAXY' -sxaddpar, hdr, 'COEFF0', 10.^loglam_rest[0] -sxaddpar, hdr, 'COEFF1', dloglam -sxaddpar, hdr, 'IDLUTILS', idlutils_version(), 'Version of idlutils' -sxaddpar, hdr, 'SPEC2D', idlspec2d_version(), 'Version of idlspec2d' -mwrfits, galtempset, outfile, hdr, /create - diff --git a/pro/spec1d/catplot.pro b/pro/spec1d/catplot.pro deleted file mode 100644 index 233d919e1..000000000 --- a/pro/spec1d/catplot.pro +++ /dev/null @@ -1,1909 +0,0 @@ -;+ -; NAME: -; catplot -; -; PURPOSE: -; Modified version of SPLOT by Tremonti for inspecting spectra. -; -; CALLING SEQUENCE: -; catplot, [x], y, $ -; [color=, psym=, symsize=, thick= ] -; -; soplot, [x], y, [/autoscale], $ -; [color=, psym=, symsize=, thick= ] -; -; sxyouts, x, y, string, [alignment=, charsize=, charthick=, color=, $ -; font=, orientation= ] -; -; serase, [nerase, /norefresh] -; -; INPUTS: -; -; OUTPUT: -; -; COMMENTS: -; This code is based upon Aaron Barth's ATV procedure. -; -; SpInspect added to menu bar. "Open SpInspect File" brings up a -; dialogue box. The spInspect File must have the format -; spInspect-pppp-mmmmm-inspector.par. Upon loading this file, plotspec -; is called to loop through the fibers on the specified plate. No -; changes are written to the spInspect File until "Update SpInspect File" -; is selected from the menu bar. -; -; EXAMPLES: -; -; BUGS: -; Doesn't use the passed XRANGE, YRANGE properly yet... -; Move around widgets to be more compact above plotting window. -; Write splot_readfits. -; Make POSITION= changeable based upon CHARSIZE. -; Gaussian fitting or integrated gaussian fitting. -; Allow one to step through an image row at a time? Or link to ATV? -; Use the WCS in splot_gettrack. -; Add widget button option to fix Y range or let it float, or fix YMIN=0. -; Include options for plotting contours, etc? -; Options for XLOG, YLOG -; For FITS files, take XTITLE, YTITLE from header -; Option to pass header as param in SPLOT -; SpInspect only reads the 1st of semi-colon separated variables in -; the manual_comments field. (Writes them just fine!) -; -; PROCEDURES CALLED: -; fits_read -; -; INTERNAL SUPPORT ROUTINES: -; splot_gausspix -; splot_startup -; splot_clearkeylist -; splot_displayall -; splot_readfits -; splot_writeeps -; splot_cleartext -; splot_zoom -; splot_gettrack -; splot_event -; splot_shutdown -; splot_resize -; splot_icolor() -; splot_setheader -; splot_headinfo -; splot_headinfo_event -; splot_plot1plot -; splot_plot1text -; splot_plotwindow -; splot_plotall -; sxyouts -; splot_move_cursor -; splot_set_minmax -; splot_get_minmax -; splot_refresh -; splot_help -; splot_help_event -; splot_plotparam_refresh -; splot_plotparam -; splot_plotparam_event -; serase -; splot_autoscale -; soplot -; splot -; splot_spinspect_new -; splot_spinspect_update -; splot_spinspect_event -; splot_spinspect -; -; REVISION HISTORY: -; 28-Sep-1999 Written by David Schlegel, Princeton. -; 19-Jun-2001 Gaussfit amplitude was wrong - fixed - D. Finkbeiner -; 17-Jun-2002 Added spInspect procedures - C. Tremonti -; Modified splot_startup, splot_events -;- -;------------------------------------------------------------------------------ -; Routine to evaluate a gaussian function integrated over a pixel of width 1, -; plus any number of polynomial terms for a background level -; A[0] = center of the Gaussian -; A[1] = sigma width of the Ith Gaussian -; A[2] = normalization of the Gaussian -; A[3...] = polynomial coefficients for background terms -; 19 June 2001 factor of sqrt(2*!pi) added to make amplitude correct - DPF - -pro splot_gausspix, x, a, f, pder - - ncoeff = N_elements(a) - fac = size(a, /tname) EQ 'DOUBLE' ? sqrt(2. * !dpi) : sqrt(2. * !pi) - f = (fac * a[2] * a[1]) * $ - (gaussint((x+0.5-a[0])/a[1]) -gaussint((x-0.5-a[0])/a[1])) - - if (ncoeff GT 3) then begin - f = f + poly(x, a[3:ncoeff-1]) - endif - - return -end - -;------------------------------------------------------------------------------ - -pro splot_startup - - common splot_state, state, graphkeys - common splot_images, main_image - common splot_pdata, nplot, maxplot, plot_ptr - common splot_wcs, astr, equinox - - keylist = { key:' ', x:0.0, y:0.0 } - - state = { $ - version: '0.9' , $ ; Version number of this release - head_ptr: ptr_new() , $ ; Pointer to FITS image header - imagename: '' , $ ; FITS image file name - base_id: 0L , $ ; ID of top-level base - base_min_size: [600L, 400L] , $ ; Min size for top-level base - draw_base_id: 0L , $ ; ID of base holding draw window - draw_window_id: 0L , $ ; Window ID of draw window - draw_widget_id: 0L , $ ; Widget ID of draw widget - location_bar_id: 0L , $ ; ID of (x,y) label - wcs_bar_id: 0L , $ ; ID of WCS label - xmin_text_id: 0L , $ ; ID of XMIN= widget - xmax_text_id: 0L , $ ; ID of XMAX= widget - ymin_text_id: 0L , $ ; ID of YMIN= widget - ymax_text_id: 0L , $ ; ID of YMAX= widget - comments_text_id: 0L , $ ; ID of comments output widget - keyboard_text_id: 0L , $ ; ID of keyboard input widget - nkey: 0 , $ ; Number of elements in keylist - keylist: replicate(keylist,5) , $ ; Record of keystrokes + cursor in plot - xrange: [0.0,1.0] , $ ; X range of plot window - yrange: [0.0,1.0] , $ ; Y range of plot window - xfix: 'float' , $ ; 0=fix, 1=float - yfix: 'fix' , $ ; 0=fix, 1=float - position: [0.15,0.15,0.95,0.95], $ ; POSITION for PLOT procedure - draw_window_size: [650L, 512L] , $ ; Size of main draw window - menu_ids: lonarr(25) , $ ; List of top menu items - mouse: [0L, 0L] , $ ; Cursor position in device coords - mphys: [0.0, 0.0] , $ ; Cursor position in data coordinates - base_pad: [0L, 0L] , $ ; Padding around draw base - pad: [0L, 0L] , $ ; Padding around draw widget - headinfo_base_id: 0L , $ ; headinfo base widget id - plotparam_base_id: 0L , $ ; plotting parameter base widget id - spinspect_base_id: 0L $ ; plotting parameter base widget id - } - -; Trim this list for the time being... ??? -; How to deal with elements that are arrays? -; graphlist = ['BACKGROUND', 'CHARSIZE', 'XCHARSIZE', 'YCHARSIZE', $ -; 'CHARTHICK', 'CLIP', 'COLOR', 'FONT', 'XGRIDSTYLE', 'YGRIDSTYLE', $ -; 'LINESTYLE', 'XMARGIN', 'YMARGIN', 'XMINOR', 'YMINOR', 'NOCLIP', $ -; 'NORMAL', 'ORIENTATION', 'POSITION', 'PSYM', 'XSTYLE', 'YSTYLE', $ -; 'SUBTITLE', 'SYMSIZE', 'THICK', 'XTHICK', 'YTHICK', $ -; 'XTICKFORMAT', 'YTICKFORMAT', 'TICKLEN', 'XTICKLEN', 'YTICKLEN', $ -; 'XTICKNAME', 'YTICKNAME', 'XTICKS', 'YTICKS', 'XTICKV', 'YTICKV', $ -; 'TITLE', 'XTITLE', 'YTITLE'] - graphlist = ['CHARSIZE', 'XCHARSIZE', 'YCHARSIZE', $ - 'CHARTHICK', 'XGRIDSTYLE', 'YGRIDSTYLE', $ - 'LINESTYLE', 'XMINOR', 'YMINOR', $ - 'PSYM', 'XSTYLE', 'YSTYLE', $ - 'SYMSIZE', 'THICK', 'XTHICK', 'YTHICK', $ - 'TITLE', 'XTITLE', 'YTITLE'] - graphkeys = $ - replicate( { box_id: 0L, $ - keyword: '', value: ptr_new() }, N_elements(graphlist) ) - graphkeys.keyword = graphlist - - - nplot = 0 - maxplot = 5000 - plot_ptr = ptrarr(maxplot) - - ; Load a simple color table with the basic 8 colors - red = [0, 1, 0, 0, 0, 1, 1, 1] - green = [0, 0, 1, 0, 1, 0, 1, 1] - blue = [0, 0, 0, 1, 1, 1, 0, 1] - tvlct, 255*red, 255*green, 255*blue - - ; Define the widgets. For the widgets that need to be modified later - ; on, save their widget ID's in state variables - - base = widget_base(title = 'splot', $ - /column, /base_align_right, app_mbar = top_menu, $ - uvalue = 'splot_base', /tlb_size_events) - state.base_id = base - - tmp_struct = {cw_pdmenu_s, flags:0, name:''} - top_menu_desc = [ $ - {cw_pdmenu_s, 1, 'File'}, $ ; file menu - {cw_pdmenu_s, 0, 'ReadFits'}, $ - {cw_pdmenu_s, 0, 'WriteEPS'}, $ - {cw_pdmenu_s, 2, 'Quit'}, $ - {cw_pdmenu_s, 1, 'Erase'}, $ ; erase menu - {cw_pdmenu_s, 0, 'EraseLast'}, $ - {cw_pdmenu_s, 0, 'EraseAllButFirst'}, $ - {cw_pdmenu_s, 2, 'EraseAll'}, $ - {cw_pdmenu_s, 1, 'ImageInfo'}, $ ; info menu - {cw_pdmenu_s, 2, 'ImageHeader'}, $ - {cw_pdmenu_s, 1, 'Plot'}, $ ; plot menu - {cw_pdmenu_s, 2, 'PlotParams'}, $ - {cw_pdmenu_s, 1, 'SpInspect'}, $ ; spInspect menu - {cw_pdmenu_s, 0, 'Open SpInspect File'}, $ - {cw_pdmenu_s, 2, 'Update SpInspect File'}, $ - {cw_pdmenu_s, 1, 'Help'}, $ ; help menu - {cw_pdmenu_s, 2, 'SPLOT Help'} $ - ] - - top_menu = cw_pdmenu(top_menu, top_menu_desc, $ - ids = state.menu_ids, $ - /mbar, $ - /help, $ - /return_id, $ - uvalue = 'top_menu') - - - button_base1 = widget_base(base, /row, /base_align_right) - button_base2 = widget_base(base, /row, /base_align_right) - button_base3 = widget_base(base, /row, /base_align_right) - - - state.xmin_text_id = cw_field(button_base1, $ - uvalue = 'xmin_text', /float, $ - title = 'XMIN=', $ - value = string(state.xrange[0]), $ - /return_events, $ - xsize = 12) - - state.xmax_text_id = cw_field(button_base1, $ - uvalue = 'xmax_text', /float, $ - title = 'XMAX=', $ - value = string(state.xrange[1]), $ - /return_events, $ - xsize = 12) - - fixlist = ['Fix', 'Float'] - mode_droplist_id = widget_droplist(button_base1, $ - title = '', $ - uvalue = 'xfix', $ - value = fixlist) - - state.ymin_text_id = cw_field(button_base2, $ - uvalue = 'ymin_text', /float, $ - title = 'YMIN=', $ - value = string(state.yrange[0]), $ - /return_events, $ - xsize = 12) - - state.ymax_text_id = cw_field(button_base2, $ - uvalue = 'ymax_text', /float, $ - title = 'YMAX=', $ - value = string(state.xrange[1]), $ - /return_events, $ - xsize = 12) - - fixlist = ['Fix', 'Float'] - mode_droplist_id = widget_droplist(button_base2, $ - title = '', $ - uvalue = 'yfix', $ - value = fixlist) - - tmp_string = string('',format='(a30)') - state.location_bar_id = widget_label(button_base1, $ - value = tmp_string, $ - uvalue = 'location_bar', frame=1) - - tmp_string = string('',format='(a30)') - state.wcs_bar_id = widget_label(button_base2, $ - value = tmp_string, $ - uvalue = 'wcs_bar', frame=1) - - state.comments_text_id = widget_label(button_base3, $ -; value = '', $ - value = tmp_string, $ - uvalue='comments_text', $ - xsize=state.draw_window_size[0]-90, frame=1) - - zoomone_button = widget_button(button_base3, $ - value = 'Zoom1', $ - uvalue = 'zoom_one') - - done_button = widget_button(button_base3, $ - value = 'Done', $ - uvalue = 'done') - - state.keyboard_text_id = widget_text(button_base3, $ - /all_events, $ - scr_xsize = 1, $ - scr_ysize = 1, $ - units = 0, $ - uvalue = 'keyboard_text', $ - value = '') - - state.draw_base_id = widget_base(base, $ - /column, /base_align_left, $ - /tracking_events, $ - uvalue = 'draw_base', $ - frame = 2) - - state.draw_widget_id = widget_draw(state.draw_base_id, $ - uvalue = 'draw_window', $ - /motion_events, /button_events, $ - scr_xsize = state.draw_window_size[0], $ - scr_ysize = state.draw_window_size[1]) - - ; Create the widgets on screen - widget_control, base, /realize - - widget_control, state.draw_widget_id, get_value = tmp_value - state.draw_window_id = tmp_value - - ; Find window padding sizes needed for resizing routines. - ; Add extra padding for menu bar, since this isn't included in - ; the geometry returned by widget_info. - ; Also add extra padding for margin (frame) in draw base. - - basegeom = widget_info(state.base_id, /geometry) - drawbasegeom = widget_info(state.draw_base_id, /geometry) - state.pad[0] = basegeom.xsize - state.draw_window_size[0] - state.pad[1] = basegeom.ysize - state.draw_window_size[1] + 30 - state.base_pad[0] = basegeom.xsize - drawbasegeom.xsize $ - + (2 * basegeom.margin) - state.base_pad[1] = basegeom.ysize - drawbasegeom.ysize + 30 $ - + (2 * basegeom.margin) - - xmanager, 'splot', state.base_id, /no_block - - return -end - -;------------------------------------------------------------------------------ - -pro splot_clearkeylist - - common splot_state - - state.nkey = 0 - state.keylist.key = ' ' - state.keylist.x = 0.0 - state.keylist.y = 0.0 - - return -end - -;------------------------------------------------------------------------------ - -pro splot_displayall - splot_refresh - return -end - -;------------------------------------------------------------------------------ - -pro splot_readfits -; ??? - - return -end - -;------------------------------------------------------------------------------ - -pro splot_writeeps - - common splot_state - - widget_control, /hourglass - - filename = dialog_pickfile(filter = '*.eps', $ - file = 'splot.eps', $ - group = state.base_id, $ - /write) - tmp_result = findfile(filename, count = nfiles) - - result = '' - if (nfiles GT 0 and filename NE '') then begin - mesg = strarr(2) - mesg[0] = 'Overwrite existing file:' - tmp_string = strmid(filename, strpos(filename, '/', /reverse_search) + 1) - mesg[1] = strcompress(tmp_string + '?', /remove_all) - result = dialog_message(mesg, $ - /default_no, $ - dialog_parent = state.base_id, $ - /question) - endif - - if ((nfiles EQ 0 OR result EQ 'Yes') AND filename NE '') then begin - - screen_device = !d.name - wset, state.draw_window_id - - aspect_ratio = $ - state.draw_window_size[1] / float(state.draw_window_size[0]) - - set_plot, 'ps' - device, $ - filename = filename, $ - /color, $ - bits_per_pixel = 8, $ -; /encapsul, $ - encapsul=0, $ - /inches, $ - xsize = 6.0, $ - ysize = 6.0 * aspect_ratio - - splot_plotall - - device, /close - set_plot, screen_device - endif - - splot_cleartext - - return -end - -;------------------------------------------------------------------------------ - -pro splot_cleartext - - ; Routine to clear the widget for keyboard input when the mouse is in - ; the text window. This de-allocates the input focus from the text - ; input widget. - - common splot_state - - widget_control, state.draw_base_id, /clear_events - widget_control, state.keyboard_text_id, set_value = '' - - return -end - -;------------------------------------------------------------------------------ - -pro splot_zoom, zchange, recenter = recenter - - common splot_state - - ; Routine to do zoom in/out and recentering of image - - case zchange of - 'in': begin - state.xrange = state.mphys[0] $ - + [-0.25, 0.25] * (state.xrange[1] - state.xrange[0]) - if (state.yfix EQ 'float') then splot_autoscale_y - end - 'out': begin - state.xrange = state.mphys[0] $ - + [-1.0, 1.0] * (state.xrange[1] - state.xrange[0]) - if (state.yfix EQ 'float') then splot_autoscale_y - end - 'one': begin - splot_autoscale_x - splot_autoscale_y - end - 'none': begin ; no change to zoom level: recenter on current mouse pos'n - state.xrange = state.mphys[0] $ - + [-0.5, 0.5] * (state.xrange[1] - state.xrange[0]) - end - else: print, 'Problem in splot_zoom!' - endcase - - splot_set_minmax - splot_refresh - - return -end - -;------------------------------------------------------------------------------ - -pro splot_gettrack - - common splot_state - common splot_wcs - - ; Update location bar with x, y - - xphysize = state.xrange[1] - state.xrange[0] - xdevsize = state.draw_window_size[0] $ - * (state.position[2] - state.position[0]) - xdev0 = state.draw_window_size[0] * state.position[0] - state.mphys[0] = $ - (state.mouse[0] - xdev0) * xphysize / xdevsize + state.xrange[0] - - yphysize = state.yrange[1] - state.yrange[0] - ydevsize = state.draw_window_size[1] $ - * (state.position[3] - state.position[1]) - ydev0 = state.draw_window_size[1] * state.position[1] - state.mphys[1] = $ - (state.mouse[1] - ydev0) * yphysize / ydevsize + state.yrange[0] - - loc_string = strcompress( string(state.mphys[0], state.mphys[1]) ) - - widget_control, state.location_bar_id, set_value=loc_string - - return -end - -;------------------------------------------------------------------------------ - -pro splot_event, event - - ; Main event loop for SPLOT widgets - - common splot_state - common splot_pdata - common splot_spinspect_state, spinspect_state, pdata, yanny_structs, $ - yanny_hdr - - widget_control, event.id, get_uvalue=uvalue - - case uvalue of - 'splot_base': begin ; main window resize: preserve display range - splot_resize, event - splot_refresh - splot_cleartext - end - - 'xfix': case event.index of - 0: state.xfix = 'fix' - 1: state.xfix = 'float' - else: print, 'Unknown selection!' - endcase - - 'yfix': case event.index of - 0: state.yfix = 'fix' - 1: state.yfix = 'float' - else: print, 'Unknown selection!' - endcase - - 'top_menu': begin ; selection from menu bar - widget_control, event.value, get_value = event_name - - case event_name of - - ; File menu options: - 'ReadFits' : splot_readfits - 'WriteEPS' : splot_writeeps - 'Quit' : splot_shutdown - - ; Erase options: - 'EraseLast' : if (nplot GE 1) then serase, 1 - 'EraseAllButFirst' : if (nplot GE 1) then serase, nplot-1 - 'EraseAll' : serase - - ; Info options: - 'ImageHeader': splot_headinfo - - ; Plot options: - 'PlotParams': splot_plotparam - - ; SpInspect options: - 'Open SpInspect File': splot_spinspect - 'Update SpInspect File': begin - widget_control, spinspect_state.spinspect_file_text_id, $ - get_value = filename - print, 'Writing file ' + filename - yanny_write, filename, pdata, structs = yanny_structs, $ - hdr = yanny_hdr - end - - ; Help options: - 'SPLOT Help': splot_help - - else: print, 'Unknown event in file menu!' - endcase - - end ; end of file menu options - - ; If the mouse enters the main draw base, set the input focus to - ; the invisible text widget, for keyboard input. - ; When the mouse leaves the main draw base, de-allocate the input - ; focus by setting the text widget value. - - 'draw_base': begin - case event.enter of - 0: begin - widget_control, state.keyboard_text_id, set_value = '' - end - 1: begin - widget_control, state.keyboard_text_id, /input_focus - end - endcase - end - - 'draw_window': begin ; mouse movement or button press - - if (event.type EQ 2) then begin ; motion event - tmp_event = [event.x, event.y] - state.mouse = tmp_event - splot_gettrack - endif - - if (event.type EQ 0) then begin - case event.press of - 1: splot_zoom, 'in', /recenter - 2: splot_zoom, 'none', /recenter - 4: splot_zoom, 'out', /recenter - else: print, 'trouble in splot_event, mouse zoom' - endcase - endif - - widget_control, state.keyboard_text_id, /input_focus - - end - - 'xmin_text': begin ; text entry in 'XMIN= ' box - splot_get_minmax, uvalue, event.value - splot_displayall - end - - 'xmax_text': begin ; text entry in 'XMAX= ' box - splot_get_minmax, uvalue, event.value - splot_displayall - end - - 'ymin_text': begin ; text entry in 'YMIN= ' box - splot_get_minmax, uvalue, event.value - splot_displayall - end - - 'ymax_text': begin ; text entry in 'YMAX= ' box - splot_get_minmax, uvalue, event.value - splot_displayall - end - - 'keyboard_text': begin ; keyboard input with mouse in display window - eventchar = string(event.ch) - - if (state.nkey LT N_elements(state.keylist)) then begin - state.keylist[state.nkey].key = eventchar - state.keylist[state.nkey].x = state.mphys[0] - state.keylist[state.nkey].y = state.mphys[1] - state.nkey = state.nkey + 1 - endif else begin - splot_clearkeylist - endelse - - case eventchar of - '1': splot_move_cursor, eventchar - '2': splot_move_cursor, eventchar - '3': splot_move_cursor, eventchar - '4': splot_move_cursor, eventchar - '6': splot_move_cursor, eventchar - '7': splot_move_cursor, eventchar - '8': splot_move_cursor, eventchar - '9': splot_move_cursor, eventchar - 'g': splot_gaussfit - else: ;any other key press does nothing - endcase - widget_control, state.keyboard_text_id, /clear_events - end - - 'zoom_one': splot_zoom, 'one' - - 'done': splot_shutdown - - else: print, 'No match for uvalue....' ; bad news if this happens - - endcase - - return -end - -;------------------------------------------------------------------------------ - -pro splot_shutdown - - ; Routine to kill the splot window(s) and clear variables to conserve - ; memory when quitting splot. Since we can't delvar the splot internal - ; variables, just set them equal to zero so they don't take up a lot - ; of space. Also clear the state and the color map vectors. - - common splot_state - common splot_images - common splot_pdata - - if (xregistered ('splot')) then begin - widget_control, state.base_id, /destroy - endif - - if (nplot GT 0) then begin - serase, /norefresh - plot_ptr = 0 - endif - - if (size(state, /tname) EQ 'STRUCT') then begin - if (size(state.head_ptr, /tname) EQ 'POINTER') then $ - ptr_free, state.head_ptr - main_image = 0 - state = 0 - endif - - return -end - -;------------------------------------------------------------------------------ - -pro splot_resize, event - - ; Routine to resize the draw window when a top-level resize event occurs. - - common splot_state - - tmp_event = [event.x, event.y] - - window = (state.base_min_size > tmp_event) - - newbase = window - state.base_pad - - newsize = window - state.pad - - widget_control, state.draw_base_id, $ - xsize = newbase[0], ysize = newbase[1] - widget_control, state.draw_widget_id, $ - xsize = newsize[0], ysize = newsize[1] - - state.draw_window_size = newsize - - return -end - -;------------------------------------------------------------------------------ - -function splot_icolor, color - - if (n_elements(color) EQ 0) then color='default' - - ncolor = N_elements(color) - - ; If COLOR is a string or array of strings, then convert color names - ; to integer values - if (size(color,/tname) EQ 'STRING') then begin ; Test if COLOR is a string - - ; Detemine the default color for the current device - if (!d.name EQ 'X') then defcolor = 7 $ ; white for X-windows - else defcolor = 0 ; black otherwise - - icolor = 0 * (color EQ 'black') $ - + 1 * (color EQ 'red') $ - + 2 * (color EQ 'green') $ - + 3 * (color EQ 'blue') $ - + 4 * (color EQ 'cyan') $ - + 5 * (color EQ 'magenta') $ - + 6 * (color EQ 'yellow') $ - + 7 * (color EQ 'white') $ - + defcolor * (color EQ 'default') - - if (!d.N_colors EQ 16777216) then begin - red = [0, 1, 0, 0, 0, 1, 1, 1] - green = [0, 0, 1, 0, 1, 0, 1, 1] - blue = [0, 0, 0, 1, 1, 1, 0, 1] - colors = 255L*red + ishft(255L*green,8) + ishft(255L*blue,16) - icolor = colors[icolor] - endif - - endif else begin - icolor = long(color) - endelse - - return, icolor -end - -;------------------------------------------------------------------------------ - -pro splot_setheader, head - - ; Routine to set the image header using a pointer to a - ; heap variable. If there is no header (i.e. if SPLOT has just been - ; passed a data array rather than a filename), then make the - ; header pointer a null pointer. - ; The reason for doing it this way is that we don't know in advance - ; how many lines the header will have, so we can't just save the - ; header itself as an array in the state structure. So instead, - ; save a pointer to the header in the state structure. - - common splot_state - common splot_wcs - - ; Kill the header info window when a new image is read in - - if (xregistered('splot_headinfo')) then begin - widget_control, state.headinfo_base_id, /destroy - endif - - if (n_elements(head) GT 1) then begin - ptr_free, state.head_ptr - state.head_ptr = ptr_new(head) - - ; Get astrometry information from header, if it exists - extast, head, astr, noparams - if (noparams EQ -1) then astr = 0 - - equ = get_equinox(head, code) - if (code EQ -1) then begin - astr = 0 - equinox = 'J2000' - endif else begin - if (equ EQ 2000.0) then equinox = 'J2000' - if (equ EQ 1950.0) then equinox = 'B1950' - if (equ NE 2000.0 and equ NE 1950.0) then $ - equinox = string(equ, format = '(f6.4)') - endelse - - endif else begin - ; If there's no image header... - ptr_free, state.head_ptr - state.head_ptr = ptr_new() - astr = 0 - - widget_control, state.wcs_bar_id, set_value = $ - '---No WCS Info---' - endelse - - return -end - -;------------------------------------------------------------------------------ - -pro splot_headinfo - - common splot_state - - ; If there's no header, kill the headinfo window and exit this routine. - if (NOT ptr_valid(state.head_ptr)) then begin - if (xregistered('splot_headinfo')) then begin - widget_control, state.headinfo_base_id, /destroy - endif - - mesg = 'No header information available for this image!' - junk = dialog_message(mesg, /error, $ - dialog_parent = state.base_id) - return - endif - - ; If there is header information but not headinfo window, - ; create the headinfo window. - if (NOT xregistered('splot_headinfo')) then begin - - headinfo_base = $ - widget_base(/floating, $ - /base_align_right, $ - group_leader = state.base_id, $ - /column, $ - title = 'SPLOT image header information', $ - uvalue = 'headinfo_base') - state.headinfo_base_id = headinfo_base - - h = *(state.head_ptr) - - headinfo_text = widget_text(headinfo_base, $ - /scroll, $ - value = h, $ - xsize = 85, $ - ysize = 24) - - headinfo_done = widget_button(headinfo_base, $ - value = 'Done', $ - uvalue = 'headinfo_done') - - widget_control, headinfo_base, /realize - xmanager, 'splot_headinfo', headinfo_base, /no_block - - endif - - return -end - -;------------------------------------------------------------------------------ - -pro splot_headinfo_event, event - - common splot_state - - widget_control, event.id, get_uvalue=uvalue - - case uvalue of - 'headinfo_done': widget_control, event.top, /destroy - else: - endcase - - return -end - -;------------------------------------------------------------------------------ - -pro splot_plot1plot, iplot - - common splot_state - common splot_pdata - - widget_control, /hourglass - - ; Convert color names to index numbers - options = (*(plot_ptr[iplot])).options - c = where(tag_names(options) EQ 'COLOR', ct) - if (ct EQ 1) then options.color = splot_icolor(options.color) - -; plot, [(*(plot_ptr[iplot])).x], [(*(plot_ptr[iplot])).y], $ -; /noerase, xstyle=5, ystyle=5, xrange=!x.crange, yrange=!y.crange, $ -; _EXTRA=options -; oplot, [(*(plot_ptr[iplot])).x], [(*(plot_ptr[iplot])).y], $ -; _EXTRA=options - plot, [(*(plot_ptr[iplot])).x], [(*(plot_ptr[iplot])).y], $ - /noerase, xstyle=5, ystyle=5, xrange=state.xrange, yrange=state.yrange, $ - _EXTRA=options - - return -end - -;---------------------------------------------------------------------- - -pro splot_plot1text, iplot - - common splot_pdata - - widget_control, /hourglass - - ; Convert color names to index numbers - options = (*(plot_ptr[iplot])).options - c = where(tag_names(options) EQ 'COLOR', ct) - if (ct EQ 1) then options.color = splot_icolor(options.color) - - string_tex = (*(plot_ptr[iplot])).text -; if (keyword_set(string_tex)) then string_tex = TeXtoIDL(string_tex) - xyouts, (*(plot_ptr[iplot])).x, (*(plot_ptr[iplot])).y, $ - string_tex, _EXTRA=options - - return -end - -;---------------------------------------------------------------------- - -pro splot_plotwindow - - common splot_state - common splot_pdata - - ; Set plot window - draw box - - !p.position = state.position ; ??? - if (nplot GT 0) then begin - - ; Always plot the box with color='default' - iplot = 0 - options = (*(plot_ptr[iplot])).options - c = where(tag_names(options) EQ 'COLOR', ct) - if (ct EQ 1) then options.color = splot_icolor('default') - - plot, [0], [0], /nodata, $ - xrange=state.xrange, yrange=state.yrange, xstyle=1, ystyle=1, $ - _EXTRA=options - endif else begin - plot, [0], [0], /nodata, $ - xrange=state.xrange, yrange=state.yrange, xstyle=1, ystyle=1 - endelse - - return -end - -;---------------------------------------------------------------------- - -pro splot_plotall - common splot_state - common splot_pdata - - ; Routine to overplot line plots from SPLOT and text from SXYOUTS - splot_plotwindow - - for iplot=0, nplot-1 do begin - case (*(plot_ptr[iplot])).type of - 'points' : splot_plot1plot, iplot - 'text' : splot_plot1text, iplot - else : print, 'Problem in splot_plotall!' - endcase - endfor - - return -end - -;------------------------------------------------------------------------------ - -pro sxyouts, x, y, text, _EXTRA=options - - common splot_pdata - common splot_state - - ; Routine to overplot text - - if (NOT xregistered('splot')) then begin - print, 'You need to start SPLOT first!' - return - endif - - if (N_params() LT 3) then begin - print, 'Too few parameters for SXYOUTS' - return - endif - - if (nplot LT maxplot) then begin - - ; Set default font to 1 - if (N_elements(options) EQ 0) then begin - options = {font: 1} - endif else begin - c = where(tag_names(options) EQ 'FONT', ct) - if (ct EQ 0) then options = create_struct(options, 'font', 1) - endelse - - pstruct = { $ - type: 'text', $ ; type of plot - x: x, $ ; x coordinate - y: y, $ ; y coordinate - text: text, $ ; text to plot - options: options $ ; plot keyword options - } - - plot_ptr[nplot] = ptr_new(pstruct) - nplot = nplot + 1 - - wset, state.draw_window_id - splot_plot1text, nplot-1 - endif else begin - print, 'Too many calls to SXYOUTS' - endelse - - return -end - -;------------------------------------------------------------------------------ - -pro splot_move_cursor, direction - - ; Use keypad arrow keys to step cursor one pixel at a time. - ; Get the new track image, and update the cursor position. - - common splot_state - - i = 1L - - case direction of - '2': state.mouse[1] = (state.mouse[1] - i) > 0 - '4': state.mouse[0] = (state.mouse[0] - i) > 0 - '8': state.mouse[1] = (state.mouse[1] + i) < (state.draw_window_size[1]-1) - '6': state.mouse[0] = (state.mouse[0] + i) < (state.draw_window_size[0]-1) - '7': begin - state.mouse[1] = (state.mouse[1] + i) < (state.draw_window_size[1]-1) - state.mouse[0] = (state.mouse[0] - i) > 0 - end - '9': begin - state.mouse[1] = (state.mouse[1] + i) < (state.draw_window_size[1]-1) - state.mouse[0] = (state.mouse[0] + i) < (state.draw_window_size[0]-1) - end - '3': begin - state.mouse[1] = (state.mouse[1] - i) > 0 - state.mouse[0] = (state.mouse[0] + i) < (state.draw_window_size[0]-1) - end - '1': begin - state.mouse[1] = (state.mouse[1] - i) > 0 - state.mouse[0] = (state.mouse[0] - i) > 0 - end - - endcase - - wset, state.draw_window_id - tvcrs, state.mouse[0], state.mouse[1], /device - - splot_gettrack - - ; Prevent the cursor move from causing a mouse event in the draw window - widget_control, state.draw_widget_id, /clear_events - - return -end - -;---------------------------------------------------------------------- - -pro splot_set_minmax - - ; Updates the MIN and MAX text boxes with new values. - - common splot_state - - widget_control, state.xmin_text_id, set_value=state.xrange[0] - widget_control, state.xmax_text_id, set_value=state.xrange[1] - widget_control, state.ymin_text_id, set_value=state.yrange[0] - widget_control, state.ymax_text_id, set_value=state.yrange[1] - -end - -;---------------------------------------------------------------------- - -pro splot_get_minmax, uvalue, newvalue - - ; Change the min and max state variables when user inputs new numbers - ; in the text boxes. - - common splot_state - - case uvalue of - - 'xmin_text': begin - reads, newvalue, tmp1 - state.xrange[0] = tmp1 - end - - 'xmax_text': begin - reads, newvalue, tmp1 - state.xrange[1] = tmp1 - end - - 'ymin_text': begin - reads, newvalue, tmp1 - state.yrange[0] = tmp1 - end - - 'ymax_text': begin - reads, newvalue, tmp1 - state.yrange[1] = tmp1 - end - - endcase - - splot_set_minmax - - return -end - -;------------------------------------------------------------------------------ - -pro splot_refresh - - common splot_state - - widget_control, /hourglass - - ; Display all plots - wset, state.draw_window_id - splot_plotall - splot_plotparam_refresh ; ??? - - splot_gettrack - - ; prevent unwanted mouse clicks - widget_control, state.draw_base_id, /clear_events - - - return -end -;------------------------------------------------------------------------------ - -pro splot_help - - common splot_state - - h = 'SPLOT HELP' - h = [h, ''] - h = [h, 'MENU BAR:'] - h = [h, 'File->ReadFits: Read in a new fits image from disk'] - h = [h, 'File->WritePS: Write a PostScript file of the current display'] - h = [h, 'File->Quit: Quits SPLOT'] - h = [h, 'Erase->EraseLast: Erases the most recent plot label'] - h = [h, 'Erase->EraseAll: Erases all plot labels'] - h = [h, 'ImageInfo->ImageHeader: Display the FITS header, if there is one.'] - h = [h, 'Plot->PlotParams: Edit plot options'] - h = [h, ''] - h = [h, 'CONTROL PANEL ITEMS:'] - h = [h,'XMIN: Shows X minimum for display; click to modify'] - h = [h,'XMAX: Shows X maximum for display; click to modify'] - h = [h,'YMIN: Shows Y minimum for display; click to modify'] - h = [h,'YMAX: Shows Y maximum for display; click to modify'] - h = [h, ''] - h = [h,'MOUSE:'] - h = [h,' Button1 = Zoom in & center'] - h = [h,' Button2 = Center on current position'] - h = [h,' Button3 = Zoom out & center'] - h = [h,'BUTTONS:'] - h = [h,'Zoom1: Rescale XRANGE and YRANGE to show all data'] - h = [h,'Done: Quits SPLOT'] - h = [h, ''] - h = [h,'Keyboard commands in display window:'] - h = [h,' Numeric keypad (with NUM LOCK on) moves cursor'] - h = [h,' g: gaussian fit (gaussian integrated over each pixel)'] - h = [h, ''] - h = [h,'IDL COMMAND LINE HELP:'] - h = [h,'To plot a 1-D array:'] - h = [h,' splot, xvector, yvector [, options]'] - h = [h,'To plot a 1-D array from a FITS file:'] - h = [h,' splot, filename [, options] (enclose file name in single quotes)'] - h = [h,'To overplot text on the draw window: '] - h = [h,' sxyouts, x, y, text_string [, options] (enclose string in single quotes)'] - h = [h, ''] - h = [h,'The options for SPLOT and SXYOUTS are essentially'] - h = [h, 'the same as those for the IDL PLOT and XYOUTS commands.'] - h = [h,'The default color is red for overplots done from the IDL command line.'] - h = [h, ''] - h = [h,'Other commands:'] - h = [h,'serase [, N]: erases all (or last N) plots and text'] - h = [h,'splot_shutdown: quits SPLOT'] - h = [h,'NOTE: If SPLOT should crash, type splot_shutdown at the IDL prompt.'] - h = [h, ''] - h = [h, ''] - h = [h,strcompress('SPLOT.PRO version '+state.version+' by D. Schlegel')] - h = [h,'For full instructions, or to download the most recent version, go to:'] - h = [h,'http://www.astro.princeton.edu/~schlegel/index.html'] - h = [h, ''] - h = [h,'For the companion program ATV, go to:'] - h = [h,'http://cfa-www.harvard.edu/~abarth/atv/atv.html'] - - if (NOT xregistered('splot_help')) then begin - - helptitle = strcompress('splot v' + state.version + ' help') - - help_base = widget_base(/floating, $ - group_leader = state.base_id, $ - /column, $ - /base_align_right, $ - title = helptitle, $ - uvalue = 'help_base') - - help_text = widget_text(help_base, $ - /scroll, $ - value = h, $ - xsize = 85, $ - ysize = 24) - - help_done = widget_button(help_base, $ - value = 'Done', $ - uvalue = 'help_done') - - widget_control, help_base, /realize - xmanager, 'splot_help', help_base, /no_block - - endif - - return -end - -;------------------------------------------------------------------------------ - -pro splot_help_event, event - - widget_control, event.id, get_uvalue = uvalue - - case uvalue of - 'help_done': widget_control, event.top, /destroy - else: - endcase - - return -end - -;------------------------------------------------------------------------------ - -pro splot_plotparam_refresh - - common splot_state - common splot_pdata - - if (nplot GT 0 AND xregistered('splot_plotparam')) then begin - options = (*(plot_ptr[0])).options ; Test if exists first ??? - opnames = tag_names(options) - - for i=0, N_elements(graphkeys)-1 do begin - c = where(opnames EQ graphkeys[i].keyword, ct) - if (ct EQ 1) then $ - widget_control, graphkeys[i].box_id, set_value=options.(c[0]) - endfor - endif - - return -end - -;------------------------------------------------------------------------------ - -pro splot_plotparam_event, event - - common splot_state - common splot_pdata - - options = (*plot_ptr[0]).options ; Test if exists first ??? - - widget_control, event.id, get_uvalue=uvalue - - case uvalue of - - 'plotparam_done': widget_control, event.top, /destroy - - else: begin - uvalue = strupcase(uvalue) - if (N_elements(options) EQ 0) then begin - options = create_struct(uvalue, event.value) - endif else begin - c = where(tag_names(options) EQ uvalue, ct) - if (ct EQ 0) then $ - options = create_struct(options, uvalue, event.value) $ - else options.(c[0]) = event.value - endelse - - (*plot_ptr[0]) = { $ - type: (*plot_ptr[0]).type, $ - x: (*plot_ptr[0]).x, $ - y: (*plot_ptr[0]).y, $ - options: options $ - } - - splot_refresh - end - - endcase - - return -end - -;------------------------------------------------------------------------------ - -pro splot_plotparam - - common splot_state - - if (NOT xregistered('splot_plotparam')) then begin - - plotparam_base = $ - widget_base(/floating, $ - /base_align_left, $ - group_leader = state.base_id, $ - /column, $ - title = 'SPLOT plot params', $ - uvalue = 'plotparam_base') - - tmp_string = string('',format='(a30)') - for i=0, N_elements(graphkeys)-1 do begin - graphkeys[i].box_id = $ ; ??? - cw_field(plotparam_base, $ - /string, $ - /return_events, $ - title = graphkeys[i].keyword+':', $ - uvalue = graphkeys[i].keyword, $ - value = tmp_string) - endfor - - plotparam_done = $ - widget_button(plotparam_base, $ - value = 'Done', $ - uvalue = 'plotparam_done') - - widget_control, plotparam_base, /realize - xmanager, 'splot_plotparam', plotparam_base, /no_block - endif - - splot_plotparam_refresh - - return -end - -;------------------------------------------------------------------------------ - -pro serase, nerase, norefresh=norefresh - - common splot_pdata - - ; Routine to erase line plots from SPLOT and text from SXYOUTS. - - ; The norefresh keyword is used when a new image - ; has just been read in, and by splot_shutdown. - - if (N_params() LT 1) then nerase = nplot $ - else if (nerase GT nplot) then nerase = nplot - - for iplot=nplot-nerase, nplot-1 do begin - ptr_free, plot_ptr[iplot] - plot_ptr[iplot] = ptr_new() - endfor - - nplot = nplot - nerase - - if (NOT keyword_set(norefresh) ) then splot_refresh - - return -end - -;------------------------------------------------------------------------------ - -pro splot_gaussfit - - common splot_state - common splot_pdata - - i = where(state.keylist.key EQ 'g', ct) - if (ct EQ 1) then begin - widget_control, state.comments_text_id, $ - set_value='GAUSSFIT: Press g at other side of feature to fit' - endif else if (ct EQ 2) then begin - ; Select all data points in the first PDATA array within the - ; selected X boundaries. - xmin = min([state.keylist[i].x, state.keylist[i].x]) - xmax = max([state.keylist[i].x, state.keylist[i].x]) - j = where((*(plot_ptr[0])).x GE xmin $ - AND (*(plot_ptr[0])).x LE xmax) - if (N_elements(j) GT 3) then begin - xtemp = (*(plot_ptr[0])).x[j] - xmid = median(xtemp) - xtemp = xtemp - xmid ; Do this for numerical stability in the fit - ytemp = (*(plot_ptr[0])).y[j] - ymin = min(ytemp) - ymax = max(ytemp, imax) - - ; Set initial guess for fitting coefficients - ; Fit a gaussian + a constant sky term - a = [xtemp[imax], 0.2*(max(xtemp)-min(xtemp)), ymax-ymin, ymin] - yfit = curvefit(xtemp, ytemp, xtemp*0+1.0, a, $ - /noderivative, function_name='splot_gausspix', chisq=chisq) - - ; If an absorption line is a better fit, use that - a_abs = [xtemp[imax], 0.2*(max(xtemp)-min(xtemp)), ymin-ymax, ymax] - yfit_abs = curvefit(xtemp, ytemp, xtemp*0+1.0, a_abs, $ - /noderivative, function_name='splot_gausspix', chisq=chisq_abs) - if (chisq_abs LT chisq AND chisq_abs NE 0) then begin - yfit = yfit_abs - a = a_abs - endif - - xtemp = xtemp + xmid - a[0] = a[0] + xmid - area = a[2] * sqrt(2.*!pi) - out_string = 'GAUSSFIT: ' $ - + ' x0= ' + strtrim(string(a[0]),2) $ - + ' sig= ' + strtrim(string(a[1]),2) $ - + ' Area= ' + strtrim(string(area),2) $ - + ' sky= ' + strtrim(string(a[3]),2) - widget_control, state.comments_text_id, set_value=out_string - soplot, xtemp, yfit, color='red', psym=10 - endif else begin - widget_control, state.comments_text_id, $ - set_value='GAUSSFIT: Too few points to fit' - endelse - splot_clearkeylist - endif else begin - splot_clearkeylist - endelse - - return -end - -;------------------------------------------------------------------------------ - -pro splot_autoscale_x - - common splot_state - common splot_pdata - - ; First set default values if no data - state.xrange[0] = 0.0 - state.xrange[1] = 1.0 - - if (nplot GT 0) then begin - - ; Set plotting limits for first SPLOT - state.xrange[0] = min( (*plot_ptr[0]).x ) - state.xrange[1] = max( (*plot_ptr[0]).x ) - - ; Enlarge plotting limits if necessary for other calls to SOPLOT - for i=1, nplot-1 do begin - if ((*plot_ptr[i]).type EQ 'points') then begin - state.xrange[0] = min( [state.xrange[0], (*plot_ptr[i]).x] ) - state.xrange[1] = max( [state.xrange[1], (*plot_ptr[i]).x] ) - endif - endfor - endif - - splot_set_minmax - - return -end - -;------------------------------------------------------------------------------ - -pro splot_autoscale_y - - common splot_state - common splot_pdata - - ; When determining Y plotting limits, only look at those data points - ; within the X plotting limits. - if (state.xrange[1] GE state.xrange[0]) then xrange = state.xrange $ - else xrange = state.xrange[[1,0]] - - ; First set default values if no data - state.yrange[0] = 0.0 - state.yrange[1] = 1.0 - - if (nplot GT 0) then begin - - ; Set plotting limits for first SPLOT - igood = where( (*plot_ptr[0]).x GE xrange[0] $ - AND (*plot_ptr[0]).x LE xrange[1]) - if (igood[0] NE -1) then begin - state.yrange[0] = min( (*plot_ptr[0]).y[igood] ) - state.yrange[1] = max( (*plot_ptr[0]).y[igood] ) - endif - - ; Enlarge plotting limits if necessary for other calls to SOPLOT - for i=1, nplot-1 do begin - if ((*plot_ptr[i]).type EQ 'points') then begin - igood = where( (*plot_ptr[i]).x GE xrange[0] $ - AND (*plot_ptr[i]).x LE xrange[1]) - if (igood[0] NE -1) then begin - state.yrange[0] = $ - min( [state.yrange[0], (*plot_ptr[i]).y[igood]] ) - state.yrange[1] = $ - max( [state.yrange[1], (*plot_ptr[i]).y[igood]] ) - endif - endif - endfor - endif - - splot_set_minmax - - return -end - -;------------------------------------------------------------------------------ - -pro soplot, x, y, autoscale=autoscale, replot=replot, $ - xrange=xrange, yrange=yrange, $ - position=position, _EXTRA=options - - common splot_state - common splot_pdata - - if (N_params() LT 1) then begin - print, 'Too few parameters for SOPLOT' - return - endif - - if (NOT xregistered('splot')) then begin - print, 'Must use SPLOT before SOPLOT' - endif - - if (keyword_set(xrange)) then state.xrange = xrange - if (keyword_set(yrange)) then state.yrange = yrange - if (keyword_set(position)) then state.position = position - - if (nplot LT maxplot) then begin - - if (N_elements(options) EQ 0) then $ - options = create_struct('color', 'default') - - if (N_params() EQ 1) then begin - pstruct = { $ - type: 'points', $ ; points - x: lindgen(N_elements(x)), $ ; x coordinate - y: x, $ ; y coordinate - options: options $ ; plot keyword options - } - endif else begin - pstruct = { $ - type: 'points', $ ; points - x: x, $ ; x coordinate - y: y, $ ; y coordinate - options: options $ ; plot keyword options - } - endelse - plot_ptr[nplot] = ptr_new(pstruct) - nplot = nplot + 1 - - wset, state.draw_window_id - - if (keyword_set(autoscale) AND NOT keyword_set(xrange)) then $ - splot_autoscale_x - if (keyword_set(autoscale) AND NOT keyword_set(yrange)) then $ - splot_autoscale_y - - if (keyword_set(replot)) then begin - splot_plotall - endif else begin - splot_plot1plot, nplot-1 - endelse - endif else begin - print, 'Too many calls to SPLOT' - endelse - - return -end - -;------------------------------------------------------------------------------ - -pro catplot, x, y, _EXTRA=KeywordsForSOPLOT - - common splot_state - common splot_images - common splot_pdata - - if (N_params() LT 1) then begin - print, 'Too few parameters for SPLOT' - return - endif - - imagename = '' - head = '' - - ; If X is a filename, read in the file - if ((N_params() NE 0) AND (size(x, /tname) EQ 'STRING') ) then begin - imagename = x - fits_read, imagename, main_image, head - x = main_image[*,0] ; Set X equal to the first row of the image ??? - endif - - if (NOT xregistered('splot')) then $ - splot_startup - - state.imagename = imagename - splot_setheader, head - - if (N_params() EQ 1) then begin - xplot = lindgen(N_elements(x)) - yplot = x - endif else begin - xplot = x - yplot = y - endelse - - serase, /norefresh - soplot, xplot, yplot, /autoscale, /replot, _EXTRA=KeywordsForSOPLOT - splot_plotparam_refresh - - return -end -;------------------------------------------------------------------------------ - -pro splot_spinspect_new - - common splot_spinspect_state, spinspect_state, pdata - - i = spinspect_state.fiberid - 1 - if i gt 639 then return - - widget_control, spinspect_state.class_text_id, $ - set_value = (*pdata[0])[i].class - widget_control, spinspect_state.subclass_text_id, $ - set_value = (*pdata[0])[i].subclass - widget_control, spinspect_state.z_text_id, $ - set_value = (*pdata[0])[i].z - - widget_control, spinspect_state.manual_class_text_id, $ - set_value = (*pdata[0])[i].manual_class - widget_control, spinspect_state.manual_subclass_text_id, $ - set_value = (*pdata[0])[i].manual_subclass - widget_control, spinspect_state.manual_z_text_id, $ - set_value = (*pdata[0])[i].manual_z - widget_control, spinspect_state.manual_comments_text_id, $ - set_value = (*pdata[0])[i].manual_comments - - plotspec, spinspect_state.plateid, spinspect_state.fiberid, $ - nsmooth = spinspect_state.nsmooth - - return -end - -;------------------------------------------------------------------------------ - -pro splot_spinspect_update - - common splot_spinspect_state, spinspect_state, pdata - - i = spinspect_state.fiberid - 1 - if i gt 639 then return - - (*pdata[0])[i].inspector =spinspect_state.inspector - - widget_control, spinspect_state.manual_class_text_id, $ - get_value = manual_class - (*pdata[0])[i].manual_class = manual_class - - widget_control, spinspect_state.manual_subclass_text_id, $ - get_value = manual_subclass - (*pdata[0])[i].manual_subclass = manual_subclass - - widget_control, spinspect_state.manual_z_text_id, $ - get_value = manual_z - (*pdata[0])[i].manual_z = manual_z - - widget_control, spinspect_state.manual_comments_text_id, $ - get_value = manual_comments - (*pdata[0])[i].manual_comments = manual_comments - - return -end - -;------------------------------------------------------------------------------ - -pro splot_spinspect_event, event - - common splot_spinspect_state, spinspect_state, pdata, yanny_structs, $ - yanny_hdr - - widget_control, event.id, get_uvalue=uvalue - - case uvalue of - 'spinspect_file_text': begin - widget_control, spinspect_state.spinspect_file_text_id, $ - get_value = filename - - spinspect_state.fiberid = 1 - spinspect_state.plateid = long(strmid(filename, 10, 4)) - spinspect_state.mjd = long(strmid(filename, 15, 5)) - inspector = (strmid(filename, 21, strlen(filename) - 21 - 4))[0] - spinspect_state.inspector = inspector - help, inspector, spinspect_state.inspector_text_id - widget_control, spinspect_state.inspector_text_id, $ - set_value = 'Inspector: ' + inspector - - print, 'Reading file ' + filename - yanny_read, filename, pdata, structs = yanny_structs, hdr = yanny_hdr - splot_spinspect_new - end - - 'nsmooth_text': begin - widget_control, spinspect_state.nsmooth_text_id, get_value = nsmooth - spinspect_state.nsmooth = nsmooth - plotspec, spinspect_state.plateid, spinspect_state.fiberid, $ - nsmooth = spinspect_state.nsmooth - end - - 'inspected': begin - print, 'Inspected Fiber ' + $ - string(spinspect_state.fiberid, format = '(I3)') - splot_spinspect_update - print, (*pdata[0])[spinspect_state.fiberid - 1] - if spinspect_state.fiberid ge 640 then return - spinspect_state.fiberid = spinspect_state.fiberid + 1 - splot_spinspect_new - end - - 'skip': begin - print, 'Fiber ' + $ - string(spinspect_state.fiberid, format = '(I3)') + $ - ' NOT inspected!' - print, (*pdata[0])[spinspect_state.fiberid - 1] - if spinspect_state.fiberid ge 640 then return - spinspect_state.fiberid = spinspect_state.fiberid + 1 - splot_spinspect_new - end - - 'back': begin - if spinspect_state.fiberid le 1 then return - spinspect_state.fiberid = spinspect_state.fiberid - 1 - print, 'Moving back to fiber ' + $ - string(spinspect_state.fiberid, format = '(I3)') - splot_spinspect_new - end - - 'goto': begin - widget_control, spinspect_state.goto_text_id, $ - get_value = newfiber - if newfiber lt 1 or newfiber gt 640 then return - spinspect_state.fiberid = newfiber - print, 'Moving to Fiber ' + string(newfiber, format = '(I3)') - splot_spinspect_new - end - endcase - - return -end -;------------------------------------------------------------------------------ - -pro splot_spinspect - - common splot_state - common splot_spinspect_state, spinspect_state, pdata - - if (NOT xregistered('splot_spinspect')) then begin - - spinspect_state = { $ - plateid: 0L , $ ; Current Plate id # - fiberid: 0L , $ ; Current Fiber id # - mjd: 0L , $ ; MJD of plate - inspector: "" , $ ; Name of Inspector - nsmooth: 5 , $ ; Pixels to smooth by - spinspect_file_text_id: 0L , $ ; ID of spInspect File = widget - inspector_text_id: 0L , $ ; ID of Inspector = widget - class_text_id: 0L , $ ; ID of Class = widget - subclass_text_id: 0L , $ ; ID of Subclass = widget - z_text_id: 0L , $ ; ID of Z = widget - manual_class_text_id: 0L , $ ; ID of Manual Class = widget - manual_subclass_text_id: 0L , $ ; ID of Manual Subclass = widget - manual_z_text_id: 0L , $ ; ID of Manual Z = widget - manual_comments_text_id: 0L , $ ; ID of Manual Comments = widget - nsmooth_text_id: 0L , $ ; ID of Smooth By = widget - goto_text_id: 0L $ ; ID Goto Fiber = widget - } - - spinspect_base = widget_base(/floating, $ - /base_align_left, $ - group_leader = state.base_id, $ - /column, $ - title = 'Splot spInspect Params', $ - uvalue = 'spinspect_base') - - button_base1 = widget_base(spinspect_base, /row, /base_align_left) - button_base2 = widget_base(spinspect_base, /row, /base_align_left) - button_base3 = widget_base(spinspect_base, /row, /base_align_left) - button_base4 = widget_base(spinspect_base, /row, /base_align_left) - button_base5 = widget_base(spinspect_base, /row, /base_align_left) - button_base6 = widget_base(spinspect_base, /row, /base_align_left) - - spinspect_state.spinspect_file_text_id = cw_field(button_base1, $ - uvalue = 'spinspect_file_text', /string, $ - title = ' SpInspect File:', $ - value = '', $ - xsize = 50, /return_events) - - spinspect_state.inspector_text_id = widget_label(button_base2, $ - uvalue = 'inspector_text', $ - value = ' Inspector: ' + spinspect_state.inspector, $ - xsize = 150, /align_left ) - - spinspect_state.class_text_id = cw_field(button_base3, $ - uvalue = 'class_text', $ - title = 'Spec1d Class:', $ - xsize = 8, /string, /noedit) - - spinspect_state.subclass_text_id = cw_field(button_base3, $ - uvalue = 'subclass_text', $ - title = ' Subclass:', $ - xsize = 12, /string, /noedit) - - spinspect_state.z_text_id = cw_field(button_base3, $ - uvalue = 'z_text', $ - title = ' Z:', $ - xsize = 8, /float, /noedit) - - spinspect_state.manual_class_text_id = cw_field(button_base4, $ - uvalue = 'manual_class_text', /string, $ - title = 'Manual Class:', $ - value = '', $ - xsize = 8) - - spinspect_state.manual_subclass_text_id = cw_field(button_base4, $ - uvalue = 'manual_subclass_text', /string, $ - title = ' Subclass:', $ - value = '', $ - xsize = 12) - - spinspect_state.manual_z_text_id = cw_field(button_base4, $ - uvalue = 'manual_z_text', /float, $ - title = ' Z:', $ - value = 0.0, $ - xsize = 8) - - spinspect_state.manual_comments_text_id = cw_field(button_base5, $ - uvalue = 'manual_comments_text', /string, $ - title = ' Comments:', $ - value = ' ', $ - xsize = 37) - - spinspect_state.nsmooth_text_id = cw_field(button_base5, $ - uvalue = 'nsmooth_text', /int, $ - title = ' Smooth by:', $ - value = spinspect_state.nsmooth, $ - /return_events, $ - xsize = 4) - - inspected_button = widget_button(button_base6, $ - value = ' Fiber Inspected ', ysize = 28, $ - uvalue = 'inspected') - - skip_button = widget_button(button_base6, $ - value = ' Skip Fiber ', ysize = 28, $ - uvalue = 'skip') - - back_button = widget_button(button_base6, $ - value = ' Back 1 Fiber ', ysize = 28, $ - uvalue = 'back') - - spinspect_state.goto_text_id = cw_field(button_base6, $ - uvalue = 'goto', /int, $ - title = ' Go to Fiber:', $ - value = 1, $ - /return_events, $ - xsize = 4) - - widget_control, spinspect_base, /realize - xmanager, 'splot_spinspect', spinspect_base, /no_block - endif - - return -end diff --git a/pro/spec1d/conflist.pro b/pro/spec1d/conflist.pro deleted file mode 100644 index 9afd8661d..000000000 --- a/pro/spec1d/conflist.pro +++ /dev/null @@ -1,1338 +0,0 @@ -;+ -; NAME: -; fieldlist -; -; PURPOSE: -; Make list of reduced fields -; -; CALLING SEQUENCE: -; fieldlist, [ /create, topdir=, outdir=, run2d=, run1d=, $ -; /purge2d, /purge1d, /killpartial, skipcart=, plist= ] -; -; INPUTS -; -; OPTIONAL INPUTS: -; create - If set, then re-generate the "fieldlist.fits" file; -; if not set, then simply read this file from a previous call. -; topdir - Optional override value for the environment -; variable $BOSS_SPECTRO_REDUX. -; outdir - Optional override of both topdir and $BOSS_SPECTRO_REDUX -; for output directory location -; run2d - Optional RUN2D subdirectories to include in outputs; -; set to '' to not search subdirectories; -; set to '*' to search all subdirs; default to $RUN2D -; run1d - Optional RUN1D subdirectories to include in outputs -; set to '' to not search subdirectories; -; set to '*' to search all subdirs; default to $RUN1D -; purge2d - If set, then delete all log files for fields that are -; considered to be 'RUNNING', but not those that are 'Done', -; 'Pending' or 'FAILED'. Those fields are then listed as -; 'Pending'. Setting /PURGE2D also sets /CREATE. -; Deleting these log files will cause the next invocation -; of BATCH2D to re-reduce those fields. -; purge1d - If set, then delete all log files for fields that are -; considered to be 'RUNNING', but not those that are 'Done', -; 'Pending' or 'FAILED'. Those fields are then listed as -; 'Pending'. Setting /PURGE1D also sets /CREATE. -; Deleting these log files will cause the next invocation -; of BATCH1D to re-reduce those fields. -; killpartial - If set, then delete all files associated with a combine -; of only some nights of a multi-night field. Such files -; can be produced by the Spectro-Robot when it fully reduces -; data from one night, but then more data is obtained for -; that plugging of the same field on a later date. This -; deletes spPlate and spZ files and their logs files. -; skipcart - cart number or list of cart numbers to drop from fieldlist -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; plist - Output structure with information for each field. -; -; COMMENTS: -; RUN2D: If option isn't set, use $RUN2D env var -; RUN2D/RUN1D = '*' means seach all subdirs -; Output directory: -; If OUTDIR is set, use that. -; Otherwise, if a single RUN2D is given, use $BOSS_SPECTRO_REDUX/[RUN2D]/ -; --> Note: RUN2D option, not $RUN2D environment variable -; Otherwise (none or multiple RUN2D), use $BOSS_SPECTRO_REDUX/ -; Option TOPDIR can override the value of $BOSS_SPECTRO_REDUX -; in the above, but it won't override OUTDIR -; -; TOPDIR is used to override $BOSS_SPECTRO_REDUX for both input and -; output. OUTDIR can be used to override TOPDIR for output if you want -; to get input from one dir while writing output to a different dir. -; -; The following files are generated in the output directory -; fieldlist.fits -; fieldlist.txt -; fieldlist.html -; fieldlist-mjdsort.txt -; fieldlist-mjdsort.html -; fieldquality.txt -; fieldquality.html -; fieldquality-mjdsort.txt -; fieldqualitymjdsort.html -; -; If INFILE is a list of plan files, i.e. -; spPlancomb-0306-51690.par -; then look for the following files for the 2D reductions: -; spPlancomb-0306-51690.par -; spDiagcomb-0306-51690.log -; spPlan2d-0306-51690.par (as specified by 'planfile2d' in spPlancomb) -; spDiag2d-0306-51690.log -; spField-0306-51690.fits (as specified by 'combinefile' in spPlancomb) -; and look for the following files for the 1D reductions: -; spPlan1d-0306-51690.par -; spZbest-0306-51690.fits -; spDiag1d-0306-51690.log -; -; FIELDSN2 is set to the minimum of the 4 cameras. -; DEREDSN2 like FIELDSN2, but with dereddened SN2 values -; PLATEQUALITY defaults to 'good'. -; PLATEQUALITY is set to 'bad' if MINSN2(B) < 10.0 or MINSN2(R) < 22.0 -; --> previously if MINSN2 < 13.0 -; PLATEQUALITY is set to 'bad' if FBADPIX > 0.10 -; PLATEQUALITY is set to 'bad' if min(NEXP_*) < 3 -; -; Decide which fields constitute unique tiles with the required S/N, -; then set QSURVEY=1. Require PLATEQUALITY='good'. -; Also require PROGNAME='main'. -; -; EXAMPLES: -; -; BUGS: -; Spawns the Unix commands 'tail' and 'grep', which is very slow. -; If the spField file is missing, it doesn't try to figure out which -; cart it really is and thus ignores skipcart -; -; DATA FILES: -; $PLATELIST_DIR/platePlans.par -; $SPECLOG_DIR/opfiles/spPlateList.par -; -; PROCEDURES CALLED: -; apo_checklimits() -; chunkinfo() -; copy_struct_inx -; djs_diff_angle() -; djs_filepath() -; fileandpath() -; headfits() -; mrdfits() -; repstr() -; rmfile -; splog -; struct_print -; sxpar() -; tai2airmass() -; yanny_free -; yanny_par() -; yanny_read -; yanny_readone() -; -; INTERNAL SUPPORT ROUTINES: -; platelist_write -; -; REVISION HISTORY: -; 29-Oct-2000 Written by D. Schlegel, Princeton -; 11-Jan-2011 Stephen Bailey, LBNL -; * Updated (S/N)^2 thresholds for "bad" -; * Added get_lastline to be faster than spawn tail -1 -; 21-Aug-2012 Stephen Bailey, LBNL -; * Changed default output directory to RUN2D -; 22-Jan-2019 Hector Ibarra, adapted for the BHM HJIM -;------------------------------------------------------------------------------ - -;---------- -; get lastline of a file faster than spawning 'tail -1' -function get_lastline, filename - ;;; spawn, 'tail -1 '+filename, lastline - - openr, ilun, filename, /get_lun - lastline = '' - while (NOT eof(ilun)) do begin - readf, ilun, lastline - endwhile - close, ilun - free_lun, ilun - - return, lastline -end - -pro platelist_write, plist, trimtags=trimtags, alias=alias, $ - fileprefix=fileprefix, title=title, toptext=toptext, outdir=outdir1 - - if (keyword_set(outdir1)) then begin - outdir = outdir1 - endif else begin - outdir = getenv('BOSS_SPECTRO_REDUX') - endelse - - ascfile = djs_filepath(fileprefix+'.txt', root_dir=outdir) - htmlfile = djs_filepath(fileprefix+'.html', root_dir=outdir) - - trimdat = struct_trimtags(plist, select_tags=trimtags[0,*]) - trimstring = struct_trimtags(plist, select_tags=trimtags[0,*], $ - format=trimtags[1,*]) - struct_print, trimstring, filename=ascfile, fdigit=3, alias=alias - - field_itag = where(trimtags[0,*] eq 'field', ft) - expt_itag= where(trimtags[0,*] eq 'exptime', et) - for itag=0L, n_tags(trimdat)-1L do begin - for iarr=0L, n_elements(trimdat[0].(itag))-1L do begin - for irow=0L, n_elements(trimdat)-1L do begin - if (ft ne 0) and (trimtags[0,itag] eq 'sn2_g1' or trimtags[0,itag] eq 'sn2_i1') $ - and (double(trimdat[irow].(expt_itag)[iarr]) lt 3600) then begin - if LONG(trimdat[irow].(field_itag)[iarr]) lt 16000 then begin - markstring = apo_checklimits('SUMMARY', strupcase(trimtags[0,itag]), $ - '', trimdat[irow].(itag)[iarr], /html) - endif else begin - if (double(trimdat[irow].(itag)[iarr]) gt 0.d) then begin - scaled = double(trimdat[irow].(itag)[iarr])/(double(trimdat[irow].(expt_itag)[iarr])/3600.d) - endif else scaled = double(trimdat[irow].(itag)[iarr]) - markstring = apo_checklimits('SUMMARY', strupcase(trimtags[0,itag]), '', scaled, /html) - endelse - endif else begin - markstring = apo_checklimits('SUMMARY', $ - strupcase(trimtags[0,itag]), '', $ - trimdat[irow].(itag)[iarr], /html) - endelse - trimstring[irow].(itag)[iarr] = markstring $ - + trimstring[irow].(itag)[iarr] - if strmatch(markstring,'' - endfor - endfor - endfor - struct_print, trimstring, /html, alias=alias, tarray=tarray, css=css - openw, lun, htmlfile, /get_lun - printf, lun, '' - printf, lun, '' - printf, lun, '' - printf, lun, '' - printf, lun, '
Last Update: '+ systime()+', Last Update MJD: '+ strtrim(string(current_mjd()),2)+'
', '(S/N)^2 values are corrected for galactic dust reddening
'] - endif else begin - toptext = [toptext, '(S/N)^2 values are not corrected for galactic dust reddening
'] - endelse - - platelist_write, plist[isort1], trimtags=trimtags1, alias=alias, $ - fileprefix='fieldlist', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Fields Observed List' - platelist_write, plist[isort2], trimtags=trimtags1, alias=alias, $ - fileprefix='fieldlist-mjdsort', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Fields Observed List' - - platelist_write, plist[isort1], trimtags=trimtags2, alias=alias, $ - fileprefix='fieldquality', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Field Quality List' - platelist_write, plist[isort2], trimtags=trimtags2, alias=alias, $ - fileprefix='fieldquality-mjdsort', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Field Quality List' - - ;---------- - ; Write the FITS binary table - plist = struct_selecttags(plist, except_tags='PLOTSN') - plist = struct_selecttags(plist, except_tags='DATA') - plist = struct_selecttags(plist, except_tags='PLOTS') - plist = struct_selecttags(plist, except_tags='CHUNKHTML') - plist = struct_selecttags(plist, except_tags='TILEID') - mwrfits, plist, fitsfile, /create - - for i=0L, n_elements(zlogfile)-1L do ptr_free, zlogfile[i] - for i=0L, n_elements(zbestfile)-1L do ptr_free, zbestfile[i] - for i=0L, n_elements(zbestrun1d)-1L do ptr_free, zbestrun1d[i] - - ;---------- - ; See if the fieldlist webapp is installed; if so, update fieldlist.json - ; Only do this if there is a single run2d - if n_elements(run2d) eq 1 then begin - appdir = getenv('BOSS_SPECTRO_REDUX')+'/'+run2d+'/fieldlist' - json_exists = file_test(appdir+'/data/fieldlist.json') - - if json_exists eq 1 then begin - cmd = appdir+'/bin/platelist2json.py '+fitsfile+' > '+appdir+'/data/fieldlist.json' - spawn, cmd - endif - endif - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/conflist_old5april2021.pro b/pro/spec1d/conflist_old5april2021.pro deleted file mode 100644 index 64ac1f1c3..000000000 --- a/pro/spec1d/conflist_old5april2021.pro +++ /dev/null @@ -1,1814 +0,0 @@ -;+ -; NAME: -; fieldlist -; -; PURPOSE: -; Make list of reduced fields -; -; CALLING SEQUENCE: -; fieldlist, [ /create, topdir=, outdir=, run2d=, run1d=, $ -; /purge2d, /purge1d, /killpartial, skipcart=, /rawsn2, plist= ] -; -; INPUTS -; -; OPTIONAL INPUTS: -; create - If set, then re-generate the "platelist.fits" file; -; if not set, then simply read this file from a previous call. -; topdir - Optional override value for the environment -; variable $BOSS_SPECTRO_REDUX. -; outdir - Optional override of both topdir and $BOSS_SPECTRO_REDUX -; for output directory location -; run2d - Optional RUN2D subdirectories to include in outputs; -; set to '' to not search subdirectories; -; set to '*' to search all subdirs; default to $RUN2D -; run1d - Optional RUN1D subdirectories to include in outputs -; set to '' to not search subdirectories; -; set to '*' to search all subdirs; default to $RUN1D -; purge2d - If set, then delete all log files for plates that are -; considered to be 'RUNNING', but not those that are 'Done', -; 'Pending' or 'FAILED'. Those plates are then listed as -; 'Pending'. Setting /PURGE2D also sets /CREATE. -; Deleting these log files will cause the next invocation -; of BATCH2D to re-reduce those plates. -; purge1d - If set, then delete all log files for plates that are -; considered to be 'RUNNING', but not those that are 'Done', -; 'Pending' or 'FAILED'. Those plates are then listed as -; 'Pending'. Setting /PURGE1D also sets /CREATE. -; Deleting these log files will cause the next invocation -; of BATCH1D to re-reduce those plates. -; killpartial - If set, then delete all files associated with a combine -; of only some nights of a multi-night plate. Such files -; can be produced by the Spectro-Robot when it fully reduces -; data from one night, but then more data is obtained for -; that plugging of the same plate on a later date. This -; deletes spPlate and spZ files and their logs files. -; skipcart - cart number or list of cart numbers to drop from platelist -; rawsn2 - If set, output original raw SN2 numbers in html and text -; files; otherwise use dereddened (dust extinction corrected) -; values. Both are always written to the FITS file output. -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; plist - Output structure with information for each plate. -; -; COMMENTS: -; RUN2D: If option isn't set, use $RUN2D env var -; RUN2D/RUN1D = '*' means seach all subdirs -; Output directory: -; If OUTDIR is set, use that. -; Otherwise, if a single RUN2D is given, use $BOSS_SPECTRO_REDUX/[RUN2D]/ -; --> Note: RUN2D option, not $RUN2D environment variable -; Otherwise (none or multiple RUN2D), use $BOSS_SPECTRO_REDUX/ -; Option TOPDIR can override the value of $BOSS_SPECTRO_REDUX -; in the above, but it won't override OUTDIR -; -; TOPDIR is used to override $BOSS_SPECTRO_REDUX for both input and -; output. OUTDIR can be used to override TOPDIR for output if you want -; to get input from one dir while writing output to a different dir. -; -; The following files are generated in the output directory -; fieldlist.fits -; fieldlist.txt -; fieldlist.html -; fieldlist-mjdsort.txt -; fieldlist-mjdsort.html -; fieldquality.txt -; fieldquality.html -; fieldquality-mjdsort.txt -; fieldqualitymjdsort.html -; -; If INFILE is a list of plan files, i.e. -; spPlancomb-0306-51690.par -; then look for the following files for the 2D reductions: -; spPlancomb-0306-51690.par -; spDiagcomb-0306-51690.log -; spPlan2d-0306-51690.par (as specified by 'planfile2d' in spPlancomb) -; spDiag2d-0306-51690.log -; spField-0306-51690.fits (as specified by 'combinefile' in spPlancomb) -; and look for the following files for the 1D reductions: -; spPlan1d-0306-51690.par -; spZbest-0306-51690.fits -; spDiag1d-0306-51690.log -; -; PLATESN2 is set to the minimum of the 4 cameras. -; DEREDSN2 like PLATESN2, but with dereddened SN2 values -; PLATEQUALITY defaults to 'good'. -; PLATEQUALITY is set to 'bad' if MINSN2(B) < 10.0 or MINSN2(R) < 22.0 -; --> previously if MINSN2 < 13.0 -; PLATEQUALITY is set to 'bad' if FBADPIX > 0.10 -; PLATEQUALITY is set to 'bad' if min(NEXP_*) < 3 -; -; Decide which plates constitute unique tiles with the required S/N, -; then set QSURVEY=1. Require PLATEQUALITY='good'. -; Also require PROGNAME='main'. -; -; EXAMPLES: -; -; BUGS: -; Spawns the Unix commands 'tail' and 'grep', which is very slow. -; If the spField file is missing, it doesn't try to figure out which -; cart it really is and thus ignores skipcart -; -; DATA FILES: -; $PLATELIST_DIR/platePlans.par -; $SPECLOG_DIR/opfiles/spPlateList.par -; -; PROCEDURES CALLED: -; apo_checklimits() -; chunkinfo() -; copy_struct_inx -; djs_diff_angle() -; djs_filepath() -; fileandpath() -; headfits() -; mrdfits() -; repstr() -; rmfile -; splog -; struct_print -; sxpar() -; tai2airmass() -; yanny_free -; yanny_par() -; yanny_read -; yanny_readone() -; -; INTERNAL SUPPORT ROUTINES: -; platelist_write -; -; REVISION HISTORY: -; 29-Oct-2000 Written by D. Schlegel, Princeton -; 11-Jan-2011 Stephen Bailey, LBNL -; * Updated (S/N)^2 thresholds for "bad" -; * Added get_lastline to be faster than spawn tail -1 -; 21-Aug-2012 Stephen Bailey, LBNL -; * Changed default output directory to RUN2D -; 22-Jan-2019 Hector Ibarra, adapted for the BHM HJIM -;------------------------------------------------------------------------------ - -;---------- -; get lastline of a file faster than spawning 'tail -1' -function get_lastline, filename - ;;; spawn, 'tail -1 '+filename, lastline - - openr, ilun, filename, /get_lun - lastline = '' - while (NOT eof(ilun)) do begin - readf, ilun, lastline - endwhile - close, ilun - free_lun, ilun - - return, lastline -end - -pro platelist_write, plist, trimtags=trimtags, alias=alias, $ - fileprefix=fileprefix, title=title, toptext=toptext, outdir=outdir1 - - if (keyword_set(outdir1)) then begin - outdir = outdir1 - endif else begin - outdir = getenv('BOSS_SPECTRO_REDUX') - endelse - - ascfile = djs_filepath(fileprefix+'.txt', root_dir=outdir) - htmlfile = djs_filepath(fileprefix+'.html', root_dir=outdir) - - trimdat = struct_trimtags(plist, select_tags=trimtags[0,*]) - trimstring = struct_trimtags(plist, select_tags=trimtags[0,*], $ - format=trimtags[1,*]) - struct_print, trimstring, filename=ascfile, fdigit=3, alias=alias - - for itag=0L, n_tags(trimdat)-1L do begin - for iarr=0L, n_elements(trimdat[0].(itag))-1L do begin - for irow=0L, n_elements(trimdat)-1L do begin - markstring = apo_checklimits('SUMMARY', $ - strupcase(trimtags[0,itag]), '', $ - trimdat[irow].(itag)[iarr], /html) - trimstring[irow].(itag)[iarr] = markstring $ - + trimstring[irow].(itag)[iarr] - if strmatch(markstring,'' - endfor - endfor - endfor - struct_print, trimstring, /html, alias=alias, tarray=tarray, css=css - openw, lun, htmlfile, /get_lun - printf, lun, '' - printf, lun, '' - printf, lun, '' - printf, lun, '' - printf, lun, 'Last Update: '+ systime()+', Last Update MJD: '+ strtrim(string(current_mjd()),2)+'
', 'Last Update: '+ systime()+', Last Update MJD: '+ strtrim(string(current_mjd()),2)+'
', '(S/N)^2 values are corrected for galactic dust reddening
'] - endif else begin - toptext = [toptext, '(S/N)^2 values are not corrected for galactic dust reddening
'] - endelse - - platelist_write, plist[isort1], trimtags=trimtags1, alias=alias, $ - fileprefix='platelist', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Plates Observed List' - platelist_write, plist[isort2], trimtags=trimtags1, alias=alias, $ - fileprefix='platelist-mjdsort', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Plates Observed List' - - platelist_write, plist[isort1], trimtags=trimtags2, alias=alias, $ - fileprefix='platequality', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Plate Quality List' - platelist_write, plist[isort2], trimtags=trimtags2, alias=alias, $ - fileprefix='platequality-mjdsort', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Plate Quality List' - - if keyword_set(plates) then begin - platelist_write, plist[isort3], trimtags=trimtags1, alias=alias, $ - fileprefix='platelist-designsort', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Plates Observed List' - platelist_write, plist[isort3], trimtags=trimtags2, alias=alias, $ - fileprefix='platequality-designsort', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Plate Quality List' - endif - endif else begin - isort1 = reverse(sort(strtrim(strcompress(plist.run2d+' ' $ - +string(99999-plist.field)),2))) - toptext = [ $ - 'Last Update: '+ systime()+', Last Update MJD: '+ strtrim(string(current_mjd()),2)+'
', '(S/N)^2 values are corrected for galactic dust reddening
'] - endif else begin - toptext = [toptext, '(S/N)^2 values are not corrected for galactic dust reddening
'] - endelse - - platelist_write, plist[isort1], trimtags=trimtags1, alias=alias, $ - fileprefix='fieldlist', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Fields Observed List' - platelist_write, plist[isort2], trimtags=trimtags1, alias=alias, $ - fileprefix='fieldlist-mjdsort', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Fields Observed List' - - platelist_write, plist[isort1], trimtags=trimtags2, alias=alias, $ - fileprefix='fieldquality', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Field Quality List' - platelist_write, plist[isort2], trimtags=trimtags2, alias=alias, $ - fileprefix='fieldquality-mjdsort', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Field Quality List' - - endelse - ;---------- - ; Write the FITS binary table - plist = struct_selecttags(plist, except_tags='PLOTSN') - plist = struct_selecttags(plist, except_tags='DATA') - plist = struct_selecttags(plist, except_tags='PLOTS') - plist = struct_selecttags(plist, except_tags='CHUNKHTML') - if not keyword_set(legacy) then $ - plist = struct_selecttags(plist, except_tags='TILEID') - mwrfits, plist, fitsfile, /create - - for i=0L, n_elements(zlogfile)-1L do ptr_free, zlogfile[i] - for i=0L, n_elements(zbestfile)-1L do ptr_free, zbestfile[i] - for i=0L, n_elements(zbestrun1d)-1L do ptr_free, zbestrun1d[i] - - ;---------- - ; See if the platelist webapp is installed; if so, update platelist.json - ; Only do this if there is a single run2d - if n_elements(run2d) eq 1 then begin - appdir = getenv('BOSS_SPECTRO_REDUX')+'/'+run2d+'/platelist' - json_exists = file_test(appdir+'/data/platelist.json') - - if json_exists eq 1 then begin - cmd = appdir+'/bin/platelist2json.py '+fitsfile+' > '+appdir+'/data/platelist.json' - spawn, cmd - endif - endif - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/cosbell.pro b/pro/spec1d/cosbell.pro deleted file mode 100644 index dbf517e9b..000000000 --- a/pro/spec1d/cosbell.pro +++ /dev/null @@ -1,21 +0,0 @@ -; Return a cosbell window function for subsequent applicaiton to data -; and variance - -function cosbell, npix, fraction, double=double - - if N_params() LT 2 then return, -1 - - if (keyword_set(double)) then PI = !dpi $ - else PI = !pi - - window = fltarr(npix)+1.0 - - nfirst = lindgen(fix(npix * fraction)) - nlast = npix - lindgen(fix(npix * fraction)) - 1 - - window[nfirst] = 0.5 * (1.0 - cos(!Pi*nfirst/(n_elements(nfirst)-1))) - window[nlast] = 0.5 * (1.0 - cos(!Pi*nfirst/(n_elements(nfirst)-1))) - - return, window -end - diff --git a/pro/spec1d/danvb_convert.pro b/pro/spec1d/danvb_convert.pro deleted file mode 100644 index a582e637c..000000000 --- a/pro/spec1d/danvb_convert.pro +++ /dev/null @@ -1,68 +0,0 @@ -; Convert the file "plateEDR.summ" from Dan van den Berk from ASCII -; format to a FITS binary table, "plateEDR.summ.fits". -pro danvb_convert - - filename = 'plateEDR.summ' - outfile = 'plateEDR.summ.fits' - - readcol, filename, plate, fiber, mjd, id_manual, flag_manual, ra, dec, $ - run, camcol, field, u_fib, g_fib, r_fib, i_fib, z_fib, z_manual, $ - z_1d, zerr_1d, z_confidence, z_status, class_1d, primtarget, $ - sectarget, $ - format='(L,L,L,A,L,D,D,L,L,L, F,F,F,F,F, F,F,F,L,L,L,L)', skipline=2 - readfmt, filename, '99X,70X,A100', comments, skipline=2 - comments = strtrim(comments,2) - - danstruct = create_struct( $ - 'plate' , 0L, $ - 'fiber' , 0L, $ - 'mjd' , 0L, $ - 'id_manual' , ' ', $ - 'flag_manual' , 0L, $ - 'ra' , 0D, $ - 'dec' , 0D, $ - 'run' , 0L, $ - 'camcol' , 0L, $ - 'field' , 0L, $ - 'u_fib' , 0., $ - 'g_fib' , 0., $ - 'r_fib' , 0., $ - 'i_fib' , 0., $ - 'z_fib' , 0., $ - 'z_manual' , 0., $ - 'z_1d' , 0., $ - 'zerr_1d' , 0., $ - 'z_confidence', 0., $ - 'z_status' , 0L, $ - 'class_1d' , 0L, $ - 'primtarget' , 0L, $ - 'sectarget' , 0L, $ - 'comments' , ' ') - danstruct = replicate(danstruct, n_elements(plate)) - danstruct.plate = plate - danstruct.fiber = fiber - danstruct.mjd = mjd - danstruct.id_manual = id_manual - danstruct.flag_manual = flag_manual - danstruct.ra = ra - danstruct.dec = dec - danstruct.run = run - danstruct.camcol = camcol - danstruct.field = field - danstruct.u_fib = u_fib - danstruct.g_fib = g_fib - danstruct.r_fib = r_fib - danstruct.i_fib = i_fib - danstruct.z_fib = z_fib - danstruct.z_manual = z_manual - danstruct.z_1d = z_1d - danstruct.zerr_1d = zerr_1d - danstruct.z_confidence = z_confidence - danstruct.z_status = z_status - danstruct.class_1d = class_1d - danstruct.primtarget = primtarget - danstruct.sectarget = sectarget - danstruct.comments = comments - - mwrfits, danstruct, outfile, /create -end diff --git a/pro/spec1d/danvb_qsolist.pro b/pro/spec1d/danvb_qsolist.pro deleted file mode 100644 index 6fb03ed3b..000000000 --- a/pro/spec1d/danvb_qsolist.pro +++ /dev/null @@ -1,83 +0,0 @@ -; Select /RSAMPLE to random-sample in each redshift bin -; rather than taking the 10 brightest. -pro danvb_qsolist, rsample=rsample - - danfilename = 'plateEDR.summ.fits' - outfile = 'eigeninput_qso.dat' - - ; Make the list of plates that are unique survey-quality. - ; Ignore plates at MJD<=51605 which had electronics problems. - ; Include only plates at MJD >= 51789 which have better spectro-photometry. - platelist, plist=plist - plist = plist[where(plist.qsurvey AND plist.mjd GE 51789)] - - ; Read Van den Berk's file - danvb = mrdfits(danfilename,1) - - ; Trim Van den Berk's file to only objects on the unique survey-quality - ; plates - qgood = bytarr(n_elements(danvb)) - for i=0, n_elements(danvb)-1 do $ - if ((where(danvb[i].plate EQ plist.plate))[0] NE -1) then qgood[i] = 1 - danvb = danvb[where(qgood)] - - ; Trim to manually-classified QSO's, ignoring where he set FLAG_MANUAL=1. - ; Note that he has at least two incorrect identifications: - ; 275/51910-353, 289/51990-313, both of which are really stars. - ; Object 412/51931-263 is not a z=5.8441 QSO. - ; Object 367/51997-506 may not be a z=5.298 QSO. - indx = where(strtrim(danvb.id_manual) EQ 'QSO' $ - AND danvb.z_manual GT 0.01 AND danvb.flag_manual NE 1) - danvb = danvb[indx] - - ; Read the P-1D outputs - readspec, danvb.plate, danvb.fiber, mjd=danvb.mjd, zans=zans - - ; Do one of the following: - ; (1) Random-sample this list to 10 objects in each delta-z=0.1 bin - ; (2) Or select the 10 brightest objects (according to ZANS.SN_MEDIAN) - ; in each delta-z=0.1 bin. - qgood = bytarr(n_elements(danvb)) - deltaz = 0.1 - nperbin = 10 - iseed = 1234L - for ibin=0, long(7.0/deltaz) do begin - zlo = deltaz * ibin - zhi = deltaz * (ibin+1) - ii = where(danvb.z_manual GE zlo AND danvb.z_manual LT zhi, ni) - if (ni GE 1 and ni LE nperbin) then begin - qgood[ii] = 1 - endif else if (ni GT nperbin) then begin - if (keyword_set(rsample)) then begin - rr = randomu(iseed, ni) - qgood[ii[ (sort(rr))[0:nperbin-1] ]] = 1 - endif else begin - thissn = zans[ii].sn_median - qgood[ii[ (sort(-thissn))[0:nperbin-1] ]] = 1 - endelse - endif - endfor - - indx = where(qgood) - danvb = danvb[indx] - zans = zans[indx] - - outstruct = create_struct( $ - 'plate' , 0L, $ - 'mjd' , 0L, $ - 'fiberid' , 0L, $ - 'z' , 0D, $ ; Cast as double so more digits written out - 'comment' , ' ' ) - outstruct = replicate(outstruct, n_elements(danvb)) - outstruct.plate = danvb.plate - outstruct.mjd = danvb.mjd - outstruct.fiberid = danvb.fiber - outstruct.z = danvb.z_manual - outstruct.comment = string(zans.sn_median, format='("S/N=",f5.2)') - ii = where(danvb.flag_manual EQ 2) - if (ii[0] NE -1) then $ - outstruct[ii].comment = outstruct[ii].comment + ' BAL' - - struct_print, outstruct, filename=outfile - -end diff --git a/pro/spec1d/desi_to_sdss.pro b/pro/spec1d/desi_to_sdss.pro deleted file mode 100644 index 8ca1b92fa..000000000 --- a/pro/spec1d/desi_to_sdss.pro +++ /dev/null @@ -1,166 +0,0 @@ -;+ -; NAME: -; desi_to_sdss -; -; PURPOSE: -; Convert a DESI extracted spectra file to an SDSS spPlate format -; -; CALLING SEQUENCE: -; desi_to_sdss, filename, plate, mjd, [ outfile= ] -; -; INPUTS: -; filename - DESI spectra file -; plate - Plate label, must be valid for sdss_specobjid() -; mjd - MJD label, must be valid for sdss_specobjid() -; -; OPTIONAL INPUTS: -; outfile - Output file name; default to spPlate-$PLATE-$MJD.fits where -; PLATE is derived from the GAMA field name and tile name, -; and MJD from the data of observation. GAMA field names -; 02, 09, 12, 15, 23 are mapped to plate numbers -; 1000, 2000, 3000, 4000, 5000 with the GAMA pointing number -; added to that. -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; The DESI spectra are interpolated to the log-wavelength mapping -; of SDSS spectra using a B-spline to the flux and the variance arrays. -; -; The DESI input file assumed to be of this format: -; http://desidatamodel.readthedocs.io/en/latest/DESI_SPECTRO_REDUX/SPECPROD/spectra-NSIDE/PIXGROUP/PIXNUM/spectra-NSIDE-PIXNUM.html#hdu01 -; -; EXAMPLES: -; Convert a DESI file and run the SDSS redshifting code: -; IDL> desi_to_sdss,'weird_obj89.darksky.fits',1,56000 -; IDL> spreduce1d,'spPlate-0001-56000.fits' -; IDL> zans=mrdfits('spZbest-0001-56000.fits',1) -; IDL> print,zans.z,zans.z_err -; -; BUGS: -; -; PROCEDURES CALLED: -; -; REVISION HISTORY: -; 27-Apr-2018 Written by D. Schlegel, LBL -;- -;------------------------------------------------------------------------------ -pro desi_to_sdss, filename, plate, mjd, outfile=outfile1 - - if (size(filename,/tname) NE 'STRING') then begin - splog, 'Must specify FILENAME' - return - endif - if (NOT keyword_set(plate)) then $ - message, 'Must set PLATE' - if (NOT keyword_set(mjd)) then $ - message, 'Must set MJD' - - dradeg = 180d0 / !dpi - - plug1 = create_struct( $ - 'OBJID', lonarr(5), $ - 'RA', 0d0, $ - 'DEC', 0d0, $ - 'OBJTYPE', '', $ - 'SPECTROGRAPHID', 1L, $ - 'FIBERID', 0L ) - - fibermap = mrdfits(filename, 'FIBERMAP') - b_wave = mrdfits(filename, 'B_WAVELENGTH') - b_flux = mrdfits(filename, 'B_FLUX') - b_ivar = mrdfits(filename, 'B_IVAR') - b_mask = mrdfits(filename, 'B_MASK') - b_res = mrdfits(filename, 'B_RESOLUTION') - r_wave = mrdfits(filename, 'R_WAVELENGTH') - r_flux = mrdfits(filename, 'R_FLUX') - r_ivar = mrdfits(filename, 'R_IVAR') - r_mask = mrdfits(filename, 'R_MASK') - r_res = mrdfits(filename, 'R_RESOLUTION') - z_wave = mrdfits(filename, 'Z_WAVELENGTH') - z_flux = mrdfits(filename, 'Z_FLUX') - z_ivar = mrdfits(filename, 'Z_IVAR') - z_mask = mrdfits(filename, 'Z_MASK') - z_res = mrdfits(filename, 'Z_RESOLUTION') - if (keyword_set(fibermap) EQ 0) then $ - message, 'Unable to read file '+filename - - dims = size(b_flux,/dimens) - if (size(b_flux,/n_dimen) EQ 1) then nfiber = 1 $ - else nfiber = dims[1] - - ; Construt the output file name - if (keyword_set(outfile1)) then outfile = outfile1 $ - else outfile = 'spPlate-'+plate_to_string(plate)+'-' $ - +string(mjd,format='(i5.5)')+'.fits' - - ; Rebin to the SDSS spacing - dloglam = 1d-4 - newloglam = wavevector(alog10(min(b_wave)), alog10(max(z_wave)), binsz=dloglam) - newwave = 10^newloglam - nnew = n_elements(newloglam) - - newflux = fltarr(nnew,nfiber) - newivar = fltarr(nnew,nfiber) - - b_npix = (size(b_flux,/dimens))[0] - r_npix = (size(r_flux,/dimens))[0] - z_npix = (size(z_flux,/dimens))[0] - npix_all = (b_npix > r_npix) > z_npix - for i=0, nfiber-1 do begin - wave_all = dblarr(npix_all,3) - flux_all = fltarr(npix_all,3) - ivar_all = fltarr(npix_all,3) - wave_all[0:b_npix-1,0] = b_wave - if (b_npix LT npix_all) then $ - wave_all[b_npix:npix_all-1,0] = b_wave[b_npix-1] + dindgen(npix_all-b_npix) $ - * (b_wave[b_npix-1] - b_wave[b_npix-2]) - flux_all[0:b_npix-1,0] = b_flux[*,i] - ivar_all[0:b_npix-1,0] = b_ivar[*,i] - wave_all[0:r_npix-1,1] = r_wave - if (r_npix LT npix_all) then $ - wave_all[r_npix:npix_all-1,1] = r_wave[r_npix-1] + dindgen(npix_all-r_npix) $ - * (r_wave[r_npix-1] - r_wave[r_npix-2]) - flux_all[0:r_npix-1,1] = r_flux[*,i] - ivar_all[0:r_npix-1,1] = r_ivar[*,i] - wave_all[0:z_npix-1,2] = z_wave - if (z_npix LT npix_all) then $ - wave_all[z_npix:npix_all-1,2] = z_wave[z_npix-1] + dindgen(npix_all-z_npix) $ - * (z_wave[z_npix-1] - z_wave[z_npix-2]) - flux_all[0:z_npix-1,2] = z_flux[*,i] - ivar_all[0:z_npix-1,2] = z_ivar[*,i] - combine1fiber, alog10(wave_all), flux_all, ivar_all, $ - newloglam=newloglam, newflux=newflux1, newivar=newivar1 - newflux[*,i] = newflux1 - newivar[*,i] = newivar1 -print,i,nfiber,string(13b),format='(i4,i4,a,$)' - endfor -print - - andmask = lonarr(nnew,nfiber) - ormask = lonarr(nnew,nfiber) - dispmap = fltarr(nnew,nfiber) + 1. - skyimg = fltarr(nnew,nfiber) - plugmap = replicate(plug1, nfiber) - plugmap.ra = fibermap.ra_obs - plugmap.dec = fibermap.dec_obs - plugmap.objtype = fibermap.objtype - plugmap.fiberid = lindgen(nfiber) + 1 - - sxaddpar, hdr, 'PLATEID', plate - sxaddpar, hdr, 'MJD', mjd - sxaddpar, hdr, 'COEFF0', newloglam[0] ; wavelength solution - sxaddpar, hdr, 'COEFF1', dloglam ; wavelength solution - mwrfits, newflux, outfile, hdr, /create - mwrfits, newivar, outfile - mwrfits, andmask, outfile - mwrfits, ormask, outfile - mwrfits, dispmap, outfile - mwrfits, plugmap, outfile - mwrfits, skyimg, outfile - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/diffplate.pro b/pro/spec1d/diffplate.pro deleted file mode 100644 index 7203a1608..000000000 --- a/pro/spec1d/diffplate.pro +++ /dev/null @@ -1,65 +0,0 @@ -pro diffplate, platenum, mjd=mjd - -platenum = 306 -mjd = [51637, 51690] - -;platenum = 302 -;mjd = [51616,51688] - -platenum = 406 -mjd = [51817, 51869] - - readspec, platenum, mjd=mjd[0], flux=flux1, flerr=flerr1, wave=wave1, $ - plugmap=plug1 -;flux1=flux1[2000:3800,*] -;flerr1=flerr1[2000:3800,*] - - readspec, platenum, mjd=mjd[1], flux=flux2, flerr=flerr2, wave=wave2, $ - plugmap=plug2 -;flux2=flux2[2000:3800,*] -;flerr2=flerr2[2000:3800,*] - -nfiber=640 - for ifiber=0, nfiber-1 do begin -; for ifiber=0, 50 do begin ; ??? -print, 'FIBER', ifiber+1 - adist = djs_diff_angle(plug1[ifiber].ra, plug1[ifiber].dec, $ - plug2.ra, plug2.dec) - dmin = min(adist, imin) - if (dmin LE 1./3600 AND max(flux1[*,ifiber]) GT 0 AND $ - max(flux2[*,imin]) GT 0) then begin - ; Match fiber IFIBER in first plate with fiber IMIN in second - zoffset = alog10(wave1[0,ifiber] / wave2[0,ifiber]) / 1.d-4 - res1 = veldisp(flux1[*,ifiber], flerr1[*,ifiber], $ - flux2[*,imin], flerr2[*,imin], zoffset=zoffset) - if (NOT keyword_set(result)) then begin - result = replicate(res1, nfiber) - endif - copy_struct_inx, res1, result, index_from=0, index_to=ifiber - endif - endfor - -md=djs_median(flux1,1) -stop - -set_plot,'ps' -device,file='diff-306b.ps' -djs_plot,alog10(result.zconf),result.z*70.,ps=1,syms=0.5,yr=100*[-1,1],$ - xtitle='log_{10}(zconf)', ytitle='\Delta z [km/s]', charsize=2, $ - title='Plate 306 MJD 51637 vs 51690', /ystyle -oplot,[-5,2],[0,0]-0 -device,/close -set_plot,'x' - -i=where(result.zconf LT 0.3) -j=where(result.z*70 GT -400 AND result.z*70 LT 400) -print,mean(result[i].z*70),mean(result[j].z*70) -print,median(result[i].z*70),median(result[j].z*70) -print,stddev(result[i].z*70),stddev(result[j].z*70) - -i=where(plug1.primtarget AND 64) -djs_oplot,alog10(md[i]),result[i].z*69.,ps=2,xr=[0,3],yr=500*[-1,1],$ - color='red' - - return -end diff --git a/pro/spec1d/fft_apodize.pro b/pro/spec1d/fft_apodize.pro deleted file mode 100644 index be021b500..000000000 --- a/pro/spec1d/fft_apodize.pro +++ /dev/null @@ -1,33 +0,0 @@ -;------------------------------------------------------------------------------ -; Apodize a spectrum with cos bell before doing FFT's. -; Use FLUXERR to select the first and last good points at which to -; start the apodizations. - -pro fft_apodize, flux, fluxerr - - npix = N_elements(flux) - - if (keyword_set(fluxerr)) then begin - igood = where(fluxerr GT 0, ngood) - if (ngood LE 2) then return - i1 = igood[0] - i2 = igood[ngood-1] - if (i1 GT 0) then flux[0:i1-1] = 0 - if (i2 LT npix-1) then flux[i2+1:npix-1] = 0 - endif else begin - i1 = 0 - i2 = npix-1 - endelse - - objbell = 0.0 * flux - objbell[i1:i2] = cosbell(i2-i1+1, 0.2, $ - double=(size(flux, /tname) EQ 'DOUBLE')) - - fluxmean = total(flux*objbell)/total(objbell) - flux = (flux - fluxmean) * objbell - if (keyword_set(fluxerr)) then $ - fluxerr = fluxerr * objbell - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/fft_wavenums.pro b/pro/spec1d/fft_wavenums.pro deleted file mode 100644 index cda1a2b17..000000000 --- a/pro/spec1d/fft_wavenums.pro +++ /dev/null @@ -1,12 +0,0 @@ -;------------------------------------------------------------------------------ -; Compute the frequencies for each bin for an IDL FFT using an array -; with NPIX elements. Return them in the range (-0.5,0.5]. - -function fft_wavenums, npix - - if (npix MOD 2 EQ 0) then $ - return, [findgen(npix/2+1),-npix/2.0 + 1.0 + findgen(npix/2-1)]/npix - - return, [findgen(npix/2+1),-(npix- 1.0)/2.0 + findgen(npix/2)]/npix -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/fftpad.pro b/pro/spec1d/fftpad.pro deleted file mode 100644 index f5a9d155a..000000000 --- a/pro/spec1d/fftpad.pro +++ /dev/null @@ -1,33 +0,0 @@ - -; -; Pad input array with mean with at least npix -; -; - -function fftpad, data, direction, _EXTRA=KeywordsForFft - - if (N_elements(direction) EQ 0) then direction = -1 - - npix = n_elements(data) - nextpower = fix(alog(2*npix) / alog(2.0)) + 1 - - if (nextpower GT 31) then begin - print, 'too many elements' - return, -1 - endif - - nfinalpix = 2L^nextpower - paddata = make_array(nfinalpix,type=size(data,/type), value = mean(data)) - - first = (nfinalpix - npix)/2 - paddata[first:first+npix-1] = data - - - ; - ; Here we need to apodize the paddata to get rid of ringing... - ; Skipping for now (laziness) - - - return, fft(paddata, direction, _Extra=KeywordsForFFT) -end - diff --git a/pro/spec1d/fieldmerge.pro b/pro/spec1d/fieldmerge.pro deleted file mode 100644 index 740091c4f..000000000 --- a/pro/spec1d/fieldmerge.pro +++ /dev/null @@ -1,950 +0,0 @@ -;+ -; NAME: -; fieldmerge -; -; PURPOSE: -; Merge all Spectro-1D outputs with photoPosPlate,spInspect files -; -; CALLING SEQUENCE: -; fieldmerge, [ field=, mjd=, except_tags=, indir=, outroot=, $ -; run2d=, /include_bad, /calc_noqso, /skip_line ] -; -; INPUTS: -; -; OPTIONAL INPUTS: -; field - fields to include; default to all files -; specified by the fieldLIST routine. -; mjd - Optional MJDs corresponding to the specified fields; -; if specified, then field and MJD should have the same -; number of elements. -; except_tags - Tag names to exclude; default to '*COVAR'. -; indir - Input directory with fieldlist.fits file; passed to -; fieldlist topir option which defaults to $BOSS_SPECTRO_REDUX -; outroot - Root name for output files; default to -; $BOSS_SPECTRO_REDUX/$RUN2D/spAll; the files are then -; spAll-$RUN2D.fits, spAll-$RUN2D.dat, spAllLine-$RUN2D.dat. -; run2d - List of RUN2D subdirectories to merge, one set of output -; files per name in $RUN2D; default to all values of RUN2D -; returned by fieldLIST. -; include_bad - If set, then include bad fields -; calc_noqso - If set, then also include redshift info for best non-QSO -; redshift fits. Defaults to being set. -; skip_line - If set, skip the generation of spAllLine.fits -; mergerun2d - If set, generate a single $BOSS_SPECTRO_REDUX/spAll.fits -; file combining all RUN2D versions in -; $BOSS_SPECTRO_REDUX/fieldlist.fits. -; Ignores run2d, $RUN2D, and does *not* write a separate -; file for each RUN2D. -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; Depends upon the fieldlist.fits file written by fieldLIST. -; Trims to only 'good' fields, or those in a public data release. -; -; The SPECPRIMARY output element is used to select a unique set of -; objects in the case of duplicate observations. Any objects observed -; multiple times will have SPECPRIMARY=1 for one instance only, and =0 -; for all other instances. The criteria (in order of importance) are -; as follows: -; 1) Prefer observations with positive SN_MEDIAN in r-band -; 2) Prefer fieldQUALITY='good' over any other field quality -; 3) Prefer observations with ZWARNING=0 -; 4) Prefer objects with larger SN_MEDIAN in r-band -; -; Temporary files are created first, such as 'spAll.fits.tmp', which -; are renamed at the end of the routine to 'spAll.fits', etc. -; -; EXAMPLES: -; -; BUGS: -; -; DATA FILES: -; -; PROCEDURES CALLED: -; copy_struct -; copy_struct_inx -; djs_filepath() -; headfits() -; hogg_mrdfits() -; mrdfits() -; mwrfits_chunks -; fieldlist -; readspec -; repstr -; spheregroup -; splog -; struct_print -; sxaddpar -; sxpar() -; -; REVISION HISTORY: -; 30-Oct-2000 Written by D. Schlegel, Princeton -; 29-Jul-2010 Added EXCLUDE_CLASS and SKIP_LINE, A. Bolton, Utah -; 17-Mar-2011 Changed EXCLUDE_CLASS behavior, A. Bolton, Utah -; 30-Jun-2011 Changed EXCLUDE_CLASS to more specific and correct -; CALC_NOQSO, including proper rchi2diff, A. Bolton, Utah -; 04-Oct-2012 Added SPECBOSS tag (ASB, Utah). -;------------------------------------------------------------------------------ - - - -pro fieldmerge1, field=field, mjd=mjd, except_tags1=except_tags1, $ - indir=indir, outroot1=outroot1, run2d=run2d, include_bad=include_bad, $ - calc_noqso=calc_noqso, skip_line=skip_line, plist=plist, legacy=legacy, $ - plates=plates, photo_file=photo_file, XCSAO=XCSAO, $ - lite=lite, skip_specprimary=skip_specprimary - - dtheta = 2.0 / 3600. - - if (n_elements(except_tags1) GT 0) then except_tags = except_tags1 $ - else except_tags = '*COVAR' - if (keyword_set(outroot1)) then begin - outroot = [outroot1, outroot1+'Line'] - endif else begin - outroot = ['spAll','spAllLine'] - if (keyword_set(field) and keyword_set(mjd)) then begin - outroot='spectra/full/'+field_to_string(field)+'/'+strtrim(string(mjd),2)+'/'+outroot+'-'+field_to_string(field)+'-'+strtrim(string(mjd),2) - endif else begin - if (keyword_set(run2d)) then outroot = outroot + '-' + repstr(run2d,'/','-') - endelse - outroot = djs_filepath(outroot, root_dir=getenv('BOSS_SPECTRO_REDUX'), $ - subdir=run2d) - endelse - if (n_elements(calc_noqso) eq 0) then calc_noqso = 1B - - t1 = systime(1) - thismem = memory() - splog, outroot - - ;---------- - ; Read fieldlist if needed - - if (NOT keyword_set(plist)) then $ - conflist, plist=plist, topdir=indir, run2d=run2d - if (NOT keyword_set(plist)) then return - - ;---------- - ; Find out if this plist includes dereddened SN2 values - - plist_tags = tag_names(plist) - ii = where(plist_tags EQ 'DEREDSN2', n) - if n GT 0 then has_deredsn2 = 1 else has_deredsn2 = 0 - - ;---------- - ; Trim to good (or non-bad public) plates - - if (keyword_set(field) and keyword_set(mjd)) then begin - nplate = n_elements(plist) - - field_plate=plist.field - qkeep = bytarr(nplate) - if (keyword_set(mjd)) then begin - qkeep = qkeep OR (field_plate EQ field AND plist.mjd EQ mjd) - endif - ikeep = where(qkeep, nkeep) - if (nkeep EQ 0) then return - plist = plist[ikeep] - endif - - if (keyword_set(run2d)) then begin - qkeep = strmatch(strtrim(plist.run2d),run2d) - ikeep = where(qkeep, nkeep) - if (nkeep EQ 0) then return - plist = plist[ikeep] - endif - - qdone = strtrim(plist.status1d,2) EQ 'Done' - ;HJIM decomet this part for the final version - if (NOT keyword_set(include_bad)) then begin - qdone = qdone AND $ - (strtrim(plist.fieldquality,2) EQ 'good' $ - OR strtrim(plist.fieldquality,2) EQ 'marginal' $ - OR (strtrim(plist.public,2) NE '' AND $ - strtrim(plist.fieldquality,2) NE 'bad') ) - endif - - indx = where(qdone eq 1, ct) - ;print, indx - if (ct EQ 0) then return - plist = plist[indx] - - nfile = n_elements(plist) - nout = total(plist.n_total) - splog, 'Total number of objects = ', nout - - ;---------- - ; Find the first tsObj file that exists for use in constructing the - ; output structure. - ifile = 0 - ;;HJIM coment the next line for the final version - ;tsobj0=1 - brake_t=0 - no_photo_file=1 - ; If the photoPlate files are chosen, add the tsObj structure - if keyword_set(photo_file) then begin - while (NOT keyword_set(tsobj0)) and brake_t eq 0 do begin - ;HJIM Coment the upper line and decoment the lower line - readspec, plist[ifile].field, mjd=plist[ifile].mjd, $ - run2d=strtrim(plist[ifile].run2d), tsobj=tsobj0, $ - legacy=legacy, plates=plates, /silent - tsobj0 = tsobj0[0] - brake_t=1 - no_photo_file=0 - if (NOT keyword_set(tsobj0)) then begin - brake_t=0 - no_photo_file=1 - ifile = ifile + 1 - if (ifile EQ nfile) then $ - brake_t=1 - endif - endwhile - endif - - ;---------- - ; Create the additional tags to add to the output structure - - pstuff = create_struct( $ - 'programname' , ' ', $ - 'survey' , ' ', $ - 'fieldquality', ' ', $ - 'fieldsn2' , 0.0, $ - 'exp_disp_med', 0.D, $ - 'fiberid_list', ' ', $ - 'lambda_eff' , 0.0, $ - 'bluefiber' , 0L, $ - 'zoffset' , 0.0, $ - 'xfocal' , ' ', $ - 'yfocal' , ' ', $ - 'calibflux' , fltarr(5), $ - 'calibflux_ivar', fltarr(5),$ - 'gaia_bp', 0.0, $ - 'gaia_rp', 0.0, $ - 'gaia_g', 0.0, $ - 'cadence', ' ', $ - 'firstcarton', ' ', $ - 'carton_to_target_pk', ' ', $ - 'RACAT', 0.D,$ - 'DECCAT', 0.D,$ - 'COORD_EPOCH', 0.0,$ - 'PMRA', 0.0,$ - 'PMDEC', 0.0,$ - 'PARALLAX', 0.0,$ - 'catalogid' , long64(0), $ - 'catalogid_v0' , long64(0), $ - 'catalogid_v0p5' , long64(0), $ - ;'catalogid_v1' , long64(0), $ - 'gaia_id_dr2', long64(0), $ - 'FIBER2MAG', fltarr(5), $ - 'PSFMAG', fltarr(5), $ - 'CATDB_MAG', fltarr(5), $ - 'OPTICAL_PROV', '', $ - 'obs', '',$ - 'field', 0L, $ -; 'plate', 0L, $ - 'designs', '', $ - 'configs', '', $ - 'nexp', 0, $ - 'exptime', 0, $ - 'airmass', 0.0, $ - 'Seeing20', 0.0, $ - 'Seeing50', 0.0, $ - 'Seeing80', 0.0, $ - 'Assigned', ' ', $ - 'on_target', ' ', $ - 'valid', ' ', $ - 'healpix', 0L, $ - 'healpixgrp', 0, $ - 'healpix_path', ' ', $ - 'mjd_final', 0.D, $ - 'mjd_list', ' ', $ - 'tai_list', ' ', $ - 'fieldsnr2g_list', ' ', $ - 'fieldsnr2r_list', ' ', $ - 'fieldsnr2i_list', ' ', $ - 'RA_LIST', ' ', $ - 'DEC_LIST', ' ', $ - 'DELTA_RA_LIST', ' ',$ - 'DELTA_DEC_LIST', ' ',$ - 'moon_dist', ' ', $ - 'moon_phase', ' ', $ - 'ebv', 0.0, $ - 'ebv_type', '', $ - 'wise_mag', fltarr(4), $ - 'twomass_mag', fltarr(3), $ - 'guvcat_mag', fltarr(2), $ - ;'gaia_parallax', 0.0, $ - ;'gaia_pmra', 0.0, $ - ;'gaia_pmdec', 0.0,$ - 'fiber_offset', 0.0,$ - 'SPEC_FILE', ' ') - - if not keyword_set(skip_specprimary) then $ - pstuff = struct_addtags(pstuff, {specprimary:0B, specboss:0B, $ - boss_specobj_id:0L, nspecobs:0}) - - if keyword_set(legacy) then $ - pstuff = struct_addtags(pstuff, {chunk:'',DEREDSN2:'',primtarget:0L,sectarget:0L}) - - if keyword_set(XCSAO) then $ - pstuff = struct_addtags(pstuff, {XCSAO_rv: !values.f_nan, XCSAO_erv: !values.f_nan,$ - XCSAO_Rxc: !values.f_nan, $ - XCSAO_Teff: !values.f_nan, XCSAO_eteff: !values.f_nan,$ - XCSAO_Logg: !values.f_nan, XCSAO_elogg: !values.f_nan,$ - XCSAO_Feh: !values.f_nan, XCSAO_efeh: !values.f_nan}) - ;---------- - ; Loop through each file - - splog, 'Reading ZANS files' - for ifile=0L, nfile-1 do begin - splog, 'Reading ZANS file ',ifile+1, ' of ', nfile - field_plate=field_to_string(plist[ifile].field) - - readspec, field_plate, mjd=plist[ifile].mjd, $ - run2d=strtrim(plist[ifile].run2d), run1d=strtrim(plist[ifile].run1d), $ - zans=zans, objhdr=objhdr, $ ;; zmanual=zmanual, - plugmap=plugmap, legacy=legacy, plates=plates, /silent, unsigned=(ifile EQ 0) - - zans = struct_selecttags(zans, except_tags=['OBJID','TILE','ELODIE_FILENAME', 'ELODIE_OBJECT',$ - 'ELODIE_SPTYPE','ELODIE_BV','ELODIE_TEFF','ELODIE_LOGG',$ - 'ELODIE_FEH','ELODIE_Z','ELODIE_Z_ERR','ELODIE_Z_MODELERR',$ - 'ELODIE_RCHI2','ELODIE_DOF']) - - - - -; ASB 2011 Mar: append info on best non-galaxy and non-qso redshifts/classes: -; ASB 2011 Jun: changed to do the "no-qso" case exclusively, and more correctly. -; ASB 2011 Jul: moved this into SPREDUCE1D. Retained for now, but -; test to see if the tags are already present. To be -; removed following verification. - if (keyword_set(calc_noqso) and (tag_exist(zans, 'z_noqso') eq 0)) then begin - splog, ' Finding non-QSO redshift info.' - - ;;- JB : Change field string format PLATEPROBLEM - pstring = field_to_string(field_plate) - mstring = string(plist[ifile].mjd, format='(i5.5)') - zallfile = getenv('BOSS_SPECTRO_REDUX') + '/' + $ - strtrim(plist[ifile].run2d, 2) + '/' + $ - pstring + '/' + strtrim(plist[ifile].run1d, 2) + $ - '/spZall-' + pstring + '-' + mstring + '.fits' - zall = mrdfits(zallfile,1) - nfib = max(zall.fiberid) - min(zall.fiberid) + 1L - nzall = n_elements(zall) / nfib - zall = reform(zall, nzall, nfib) - class_all = strtrim(zall.class,2) - id_noqso = replicate(-1L, nfib) - rchi2diff_noqso = replicate(0., nfib) - for ii = 0L, nfib-1 do begin - wh_noqso = where(class_all[*,ii] ne 'QSO') - wh_noqso = (wh_noqso[sort(wh_noqso)])[0:1] - id_noqso[ii] = wh_noqso[0] - rchi2diff_noqso[ii] = total(zall[wh_noqso[0]:wh_noqso[1]-1,ii].rchi2diff) - endfor - zans_noqso = zall[id_noqso,lindgen(nfib)] - noqso_struc = replicate( $ - {z_noqso: 0., z_err_noqso: 0., zwarning_noqso: 0L, $ - class_noqso: ' ', subclass_noqso: ' ', $ - rchi2diff_noqso: 0.}, nfib) - noqso_struc.z_noqso = zans_noqso.z - noqso_struc.z_err_noqso = zans_noqso.z_err - noqso_struc.class_noqso = zans_noqso.class - noqso_struc.subclass_noqso = zans_noqso.subclass - noqso_struc.rchi2diff_noqso = rchi2diff_noqso -; Re-set the small-delta-chi2 bit: - minrchi2diff = 0.01 - small_rchi2diff = rchi2diff_noqso lt minrchi2diff - zw_new = zans_noqso.zwarning - zflagval = sdss_flagval('ZWARNING', 'SMALL_DELTA_CHI2') - zw_new = zw_new - (zw_new and zflagval) - zw_new = zw_new or (zflagval * small_rchi2diff) - noqso_struc.zwarning_noqso = zw_new - zans = struct_addtags(zans, noqso_struc) - noqso_struc = 0 - zall = 0 - zans_noqso = 0 - endif - - if keyword_set(xcsao) then begin - pstring = field_to_string(field_plate) - mstring = string(plist[ifile].mjd, format='(i5.5)') - XCSAOfile = getenv('BOSS_SPECTRO_REDUX') + '/' + $ - strtrim(plist[ifile].run2d, 2) + '/' + $ - pstring + '/' + strtrim(plist[ifile].run1d, 2) + $ - '/spXCSAO-' + pstring + '-' + mstring + '.fits' - if FILE_TEST(XCSAOfile) then begin - splog, 'Reading XCSAO file: spXCSAO-' + pstring + '-' + mstring + '.fits' - XCSAO = mrdfits(XCSAOfile,1) - endif else splog, 'No XCSAO file for spXCSAO-' + pstring + '-' + mstring + '.fits' - endif - if (ifile EQ 0) then begin - htags = ['SDSSV_BOSS_TARGET0'] - pstuff = struct_addtags(pstuff, $ - struct_selecttags(plugmap[0], select_tags=htags)) - ;pstuff = create_struct(pstuff, plutt[0]) - outdat1 = create_struct(pstuff, struct_selecttags(zans[0], except_tags=['field','fiberid_list'])) - struct_assign, {junk:0}, outdat1 ; Zero-out all elements - if keyword_set(xcsao) then begin - outdat1.xcsao_rv=!values.f_nan - outdat1.xcsao_erv=!values.f_nan - outdat1.XCSAO_Rxc=!values.f_nan - outdat1.XCSAO_Teff=!values.f_nan - outdat1.XCSAO_eteff=!values.f_nan - outdat1.XCSAO_Logg=!values.f_nan - outdat1.XCSAO_elogg=!values.f_nan - outdat1.XCSAO_Feh=!values.f_nan - outdat1.XCSAO_efeh=!values.f_nan - endif - outdat = replicate(outdat1, nout) - endif - - indx = lindgen(plist[ifile].n_total) - if (ifile GT 0) then indx += total(plist[0:ifile-1].n_total) - - tmpdat = outdat[indx] - copy_struct, zans, tmpdat - outdat[indx] = tmpdat - - ; Fill in the first columns of this output structure -; if tag_exist(plist,'programname') then begin -; outdat[indx].programname = plist[ifile].programname -; endif else outdat[indx].programname=plugmap.program - - outdat = strct_to_struct(plugmap,'*','PROGRAM',outdat,indx, outtag='programname') - outdat = strct_to_struct(plist,ifile,'CHUNK',outdat,indx) - outdat = strct_to_struct(plist,ifile,'PLATEQUALITY',outdat,indx, altTag='fieldquality') - outdat = strct_to_struct(plist,ifile,'PLATESN2',outdat,indx,outtag='fieldsn2', altTag='fieldsn2') - if has_deredsn2 then outdat = strct_to_struct(plist,ifile,'deredsn2',outdat,indx) - outdat = strct_to_struct(plugmap,'*','EXP_DISP_MED', outdat,indx) -; outdat = strct_to_struct(plist,ifile,'exptime',outdat,indx) - outdat = strct_to_struct(plist,ifile,'DESIGNID',outdat,indx) -; outdat = strct_to_struct(plist,ifile,'DESIGNS',outdat,indx) -; outdat = strct_to_struct(plist,ifile,'CONFIGS',outdat,indx) -; outdat = strct_to_struct(plist,ifile,'SURVEY',outdat,indx) - outdat = strct_to_struct(plist,ifile,'OBS',outdat,indx) - outdat = strct_to_struct(plugmap,'*','SURVEY',outdat,indx) - - outdat = strct_to_struct(plugmap,'*','AIRMASS',outdat,indx) - outdat = strct_to_struct(plugmap,'*','SEEING20',outdat,indx) - outdat = strct_to_struct(plugmap,'*','SEEING50',outdat,indx) - outdat = strct_to_struct(plugmap,'*','SEEING80',outdat,indx) - outdat = strct_to_struct(plugmap,'*','MJDLIST',outdat,indx, outTag='mjd_list') - outdat = strct_to_struct(plugmap,'*','TAILIST',outdat,indx, outTag='tai_list') - outdat = strct_to_struct(plugmap,'*','DESIGNS',outdat,indx) - outdat = strct_to_struct(plugmap,'*','CONFIGS',outdat,indx) - - - - ; Get PRIMTARGET+SECTARGET with those values from - ; the plug-map structure in spfield file. - ; HJIM decoment the next three lines for the final version - outdat = strct_to_struct(plugmap,'*','primtarget',outdat,indx) - outdat = strct_to_struct(plugmap,'*','sectarget',outdat,indx) - - outdat = strct_to_struct(plugmap,'*','lambda_eff',outdat,indx) - outdat = strct_to_struct(plugmap,'*','zoffset',outdat,indx) - outdat = strct_to_struct(plugmap,'*','XFOCAL_LIST',outdat,indx,outTag='xfocal') - outdat = strct_to_struct(plugmap,'*','YFOCAL_LIST',outdat,indx,outTag='yfocal') - outdat = strct_to_struct(plugmap,'*','bluefiber',outdat,indx) - - outdat = strct_to_struct(plugmap,'*','CALIBFLUX',outdat,indx) - outdat = strct_to_struct(plugmap,'*','CALIBFLUX_IVAR',outdat,indx) - outdat = strct_to_struct(plugmap,'*','CALIBFLUX_IVAR',outdat,indx) - outdat = strct_to_struct(plugmap,'*','GAIA_BP',outdat,indx,altTag='BP_MAG',altOutTag='GAIA_BP') - outdat = strct_to_struct(plugmap,'*','GAIA_RP',outdat,indx,altTag='RP_MAG',altOutTag='GAIA_RP') - outdat = strct_to_struct(plugmap,'*','GAIA_G',outdat,indx,altTag='GAIA_G_MAG',altOutTag='GAIA_G') - outdat = strct_to_struct(plugmap,'*','SDSSV_BOSS_TARGET0',outdat,indx) - outdat = strct_to_struct(plugmap,'*','CADENCE',outdat,indx) - outdat = strct_to_struct(plugmap,'*','FIRSTCARTON',outdat,indx) - outdat = strct_to_struct(plugmap,'*','CARTON_TO_TARGET_PK',outdat,indx) - outdat = strct_to_struct(plugmap,'*','ASSIGNED_LIST',outdat,indx, outTag='ASSIGNED') - outdat = strct_to_struct(plugmap,'*','ON_TARGET_LIST',outdat,indx, outTag='ON_TARGET') - outdat = strct_to_struct(plugmap,'*','VALID_LIST',outdat,indx, outTag='VALID') - outdat = strct_to_struct(plugmap,'*','ICATALOGID',outdat,indx, outTag='CATALOGID') - - junk = where(strmatch(tag_names(plugmap), 'catversion',/FOLD_CASE) eq 1, ct) - if ct gt 0 then begin - catversion = plugmap.catversion - v0 = where(strmatch(catversion, '0.0*') eq 1, ct) - if ct gt 0 then outdat = strct_to_struct(plugmap, v0, 'ICATALOGID', outdat, indx[v0], outTag='CATALOGID_V0') - - v0p5 = where(strmatch(catversion, '0.5*') eq 1, ct) - if ct gt 0 then outdat = strct_to_struct(plugmap, v0p5, 'ICATALOGID', outdat, indx[v0p5], outTag='CATALOGID_V0p5') - -; v1 = where(strmatch(catversion, '1.*') eq 1, ct) -; if ct gt 0 then outdat = strct_to_struct(plugmap, v1, 'ICATALOGID', outdat, indx[v1], outTag='CATALOGID_V1') - endif - - outdat = strct_to_struct(plugmap,'*','gaia_id_dr2',outdat,indx) - outdat = strct_to_struct(plugmap,'*','FIBER2MAG',outdat,indx,altTag='mag') - outdat = strct_to_struct(plugmap,'*','PSFMAG',outdat,indx) - outdat = strct_to_struct(plugmap,'*','CATDB_MAG',outdat,indx) - outdat = strct_to_struct(plugmap,'*','OPTICAL_PROV',outdat,indx) - outdat = strct_to_struct(plugmap,'*','NEXP',outdat,indx) - outdat = strct_to_struct(plugmap,'*','EXPTIME',outdat,indx) -; outdat = strct_to_struct(plugmap,'*','field',outdat,indx, altTag='plate', altOutTag='field') - outdat = strct_to_struct(plugmap,'*','mjd_final',outdat,indx) - outdat = strct_to_struct(plugmap,'*','RACAT',outdat,indx) - outdat = strct_to_struct(plugmap,'*','DECCAT',outdat,indx) - outdat = strct_to_struct(plugmap,'*','COORD_EPOCH',outdat,indx) - outdat = strct_to_struct(plugmap,'*','PMRA',outdat,indx) - outdat = strct_to_struct(plugmap,'*','PMDEC',outdat,indx) - outdat = strct_to_struct(plugmap,'*','PARALLAX',outdat,indx) - outdat = strct_to_struct(plugmap,'*','TAI_LIST',outdat,indx) - - outdat = strct_to_struct(plugmap,'*','fieldsnr2g_list',outdat,indx,altTag='platesnr2g_list', altOutTag='fieldsnr2g_list') - outdat = strct_to_struct(plugmap,'*','fieldsnr2r_list',outdat,indx,altTag='platesnr2r_list', altOutTag='fieldsnr2r_list') - outdat = strct_to_struct(plugmap,'*','fieldsnr2i_list',outdat,indx,altTag='platesnr2i_list', altOutTag='fieldsnr2i_list') - - outdat = strct_to_struct(plugmap,'*','RA_LIST',outdat,indx) - outdat = strct_to_struct(plugmap,'*','DEC_LIST',outdat,indx) - outdat = strct_to_struct(plugmap,'*','DELTA_RA_LIST',outdat,indx) - outdat = strct_to_struct(plugmap,'*','DELTA_DEC_LIST',outdat,indx) - - outdat = strct_to_struct(plugmap,'*','MOON_DIST',outdat,indx) - outdat = strct_to_struct(plugmap,'*','MOON_PHASE',outdat,indx) - outdat = strct_to_struct(plugmap,'*','EBV',outdat,indx,altTag='SFD_EBV', altOutTag='EBV') - outdat = strct_to_struct(plugmap,'*','EBV_TYPE',outdat,indx) - outdat = strct_to_struct(plugmap,'*','WISE_MAG',outdat,indx) - outdat = strct_to_struct(plugmap,'*','TWOMASS_MAG',outdat,indx) - outdat = strct_to_struct(plugmap,'*','GUVCAT_MAG',outdat,indx) - ;outdat = strct_to_struct(plugmap,'*','GAIA_PARALLAX',outdat,indx) - ;outdat = strct_to_struct(plugmap,'*','GAIA_PMRA',outdat,indx) - ;outdat = strct_to_struct(plugmap,'*','GAIA_PMDEC',outdat,indx) - outdat = strct_to_struct(plugmap,'*','fiber_offset',outdat,indx) - - - if keyword_set(xcsao) then begin - if FILE_TEST(XCSAOfile) then begin - outdat = strct_to_struct(XCSAO,'*','rv',outdat,indx,outTag='XCSAO_rv') - outdat = strct_to_struct(XCSAO,'*','erv',outdat,indx,outTag='XCSAO_erv') - outdat = strct_to_struct(XCSAO,'*','R',outdat,indx,outTag='XCSAO_Rxc') - outdat = strct_to_struct(XCSAO,'*','Teff',outdat,indx,outTag='XCSAO_Teff') - outdat = strct_to_struct(XCSAO,'*','Teff',outdat,indx,outTag='XCSAO_Teff') - outdat = strct_to_struct(XCSAO,'*','eteff',outdat,indx,outTag='XCSAO_eteff') - outdat = strct_to_struct(XCSAO,'*','Logg',outdat,indx,outTag='XCSAO_Logg') - outdat = strct_to_struct(XCSAO,'*','elogg',outdat,indx,outTag='XCSAO_elogg') - outdat = strct_to_struct(XCSAO,'*','Feh',outdat,indx,outTag='XCSAO_Feh') - outdat = strct_to_struct(XCSAO,'*','efeh',outdat,indx,outTag='XCSAO_efeh') - endif - endif - - spec_file = outdat[indx].spec_file - for fid = 0L, n_elements(spec_file)-1 do begin - spec_file[fid] = 'spec-'+ field_to_string((outdat[indx].field)[fid]) + '-' + $ - strtrim(string(plist[ifile].mjd),2) + '-' +$ - strtrim(plugmap[fid].catalogid,2)+'.fits' - endfor - outdat[indx].spec_file = spec_file - - ; Add the cas-styled specobjid to output - if not tag_exist(outdat, 'specobjid') then $ - outdat = struct_addtags(outdat, replicate({specobjid:0ULL},n_elements(outdat))) - words= STREGEX(STRTRIM(outdat[indx].run2d,2),'^v([0-9]+)_([0-9]+)_([0-9]+)', /SUB, /EXTRACT) - ; did it parse as vXX_YY_ZZ? - if words[0] ne '' then begin - rerun= (long(words[1,*])-5L)*10000L+ (long(words[2,*])*100L)+ (long(words[3,*])) - endif else begin - splog, "WARNING: Unable to parse RERUN from ", (outdat[indx])[uniq(outdat[indx].run2d)].run2d, " for CAS-style SPECOBJID; Using 0 instead" - rerun= intarr(n_elements(outdat[indx].field)) - endelse - outdat[indx].specobjid = sdss_specobjid_17(outdat[indx].field,outdat[indx].target_index,$ - outdat[indx].mjd,rerun) - - - healpix_now=1 - if keyword_set(healpix_now) then begin - - mwm_root='$MWM_HEALPIX' - healpix_path_t=outdat[indx].healpix_path - plt_t=field_to_string(outdat[indx].field) - - val_indx = where(finite(outdat[indx].RACAT) and finite(outdat[indx].DECCAT) $ - and (outdat[indx].RACAT ne -999.0) and (outdat[indx].DECCAT ne -999.0), ct) - if ct gt 0 then begin - healp=coords_to_healpix(outdat[indx[val_indx]].RACAT,outdat[indx[val_indx]].DECCAT) - outdat[indx[val_indx]].healpix=healp.healpix - outdat[indx[val_indx]].healpixgrp=healp.healpixgrp - - for fid = 0L, n_elements(zans)-1 do begin - if ((plugmap[fid].icatalogid eq 0) and $ - finite(plugmap[fid].RA) and finite(plugmap[fid].DEC)) then healpix_path_t[fid] = '' $ - else begin - healpix_path_t[fid]=mwm_root + '/'+ $ - strtrim(string(healp[fid].healpixgrp),2) + '/' + $ - strtrim(string(healp[fid].healpix),2) + '/boss/' + $ - strtrim(plist[ifile].run2d, 2)+ '/' + $ - 'spec-' + strtrim(string(plt_t[0]),2) + '-' + $ - strtrim(string(plist[ifile].mjd),2) + $ - '-' + strtrim(plugmap[fid].catalogid,2)+'.fits' - endelse - endfor - outdat[indx].healpix_path=healpix_path_t - endif - endif else begin - - outdat = strct_to_struct(plugmap,'*','HEALPIX',outdat,indx) - outdat = strct_to_struct(plugmap,'*','HEALPIXGRP',outdat,indx) - if (tag_exist(plugmap,'HEALPIX_PATH') or tag_exist(plugmap,'HEALPIX_DIR')) then begin - if tag_exist(plugmap,'HEALPIX_DIR') then healpix_path_t=plugmap.healpix_dir $ - else healpix_path_t=plugmap.healpix_path - for fid = 0L, n_elements(zans)-1 do begin - healpix_path_t[fid] = repstr(healpix_path_t[fid],'XXXX',strtrim(string(plist[ifile].mjd),2)) - endfor - outdat[indx].healpix_path=healpix_path_t - endif - endelse - endfor - - splog, 'Time to read data = ', systime(1)-t1, ' sec' - - ;---------- - ; Set the SPECPRIMARY flag to 0 or 1 - if not keyword_set(skip_specprimary) then begin - t2 = systime(1) - - ; Determine the score for each object - ; 1) Prefer observations with positive SN_MEDIAN in r-band - ; 2) Prefer fieldQUALITY='good' over any other field quality - ; 3) Prefer observations with ZWARNING=0 - ; 4) Prefer objects with larger SN_MEDIAN in r-band - ; ASBjuly2011: test against ZWARNING_NOQSO for GALAXY targets: - zw_primtest = outdat.zwarning - if tag_exist(outdat, 'ZWARNING_NOQSO') then begin - wh_galtarget = where(strmatch(outdat.objtype, 'GALAXY*'), ngaltarget) - if (ngaltarget gt 0) then zw_primtest[wh_galtarget] = outdat[wh_galtarget].zwarning_noqso - endif - if (n_elements(outdat[0].sn_median) EQ 1) then jfilt = 0 else jfilt = 2 - score = 4 * (outdat.sn_median[jfilt] GT 0) $ - + 2 * (strmatch(outdat.fieldquality,'good*') EQ 1) $ - + 1 * (zw_primtest EQ 0) $ - + (outdat.sn_median[jfilt]>0) / max(outdat.sn_median[jfilt]+1.) - - ingroup = spheregroup(outdat.fiber_ra, outdat.fiber_dec, dtheta, $ - multgroup=multgroup, firstgroup=firstgroup, $ - nextgroup=nextgroup) - - ; Set the unique object IDs - if tag_exist(outdat, 'boss_specobj_id') then outdat.boss_specobj_id = ingroup + 1L - - if tag_exist(outdat, 'specprimary') then sp = 1 else sp = 0 - if tag_exist(outdat, 'nspecobs') then ns =1 else ns = 0 - for j=0L, n_elements(firstgroup)-1L do begin - if (firstgroup[j] NE -1) then begin - if (multgroup[j] EQ 1) then begin - if sp then outdat[firstgroup[j]].specprimary = 1 - if ns then outdat[firstgroup[j]].nspecobs = 1 - endif else begin - indx = lonarr(multgroup[j]) - indx[0] = firstgroup[j] - for k=0L, multgroup[j]-2L do indx[k+1] = nextgroup[indx[k]] - foo = max(score[indx], ibest) - if sp then outdat[indx[ibest]].specprimary = 1 - if ns then outdat[indx].nspecobs = multgroup[j] - endelse - endif - endfor - - - ; ASB: Copy specprimary into specboss - ; (Thinking is that specprimary can be superseded downstream.) - if tag_exist(outdat, 'specboss') then outdat.specboss = outdat.specprimary - - splog, 'Time to assign primaries = ', systime(1)-t2, ' sec' - endif - - ;---------- - ; Pre-condition to FITS structure to have same-length strings - ; (for any given tag name) by concatenating spaces. - - ntag = n_tags(outdat) - tags = tag_names(outdat) - for itag=0L, ntag-1L do begin - if (size(outdat[0].(itag), /tname) EQ 'STRING') then begin - if (NOT keyword_set(silent)) then $ - splog, 'Padding whitespace for string array ' + tags[itag] - taglen = strlen(strtrim(outdat.(itag))) - maxlen = max(taglen) - padspace = string('', format='(a'+string(maxlen)+')') - outdat.(itag) = strmid(outdat.(itag) + padspace, 0, maxlen) - endif - endfor - - ;---------- - ; Write the output FITS file, writing one field at a time - - ; Don't allow duplicate tags between the tsObj structure and what - ; is already in the output structure. For ex, MJD is in both. - if no_photo_file eq 0 then begin - tsobj0 = struct_selecttags(tsobj0, except_tags=tag_names(outdat));;HJIM Decoment this part - platedat1 = create_struct(outdat[0], tsobj0) - endif else begin - platedat1 = outdat[0] - endelse - if (keyword_set(except_tags)) then $ - platedat1 = struct_selecttags(platedat1, except_tags=except_tags) - struct_assign, {junk:0}, platedat1 ; Zero-out all elements - - splog, 'Writing FITS file ' + outroot[0]+'.fits' - for ifile=0L, nfile-1 do begin - field_plate=plist[ifile].field - splog, 'Writing field ', ifile+1, ' of ', nfile - str_p='field' - - platedat = replicate(platedat1, plist[ifile].n_total) - indx = lindgen(plist[ifile].n_total) - if (ifile GT 0) then indx += total(plist[0:ifile-1].n_total) - if keyword_set(photo_file) then begin - readspec, field_plate, mjd=plist[ifile].mjd, $ - run2d=strtrim(plist[ifile].run2d), tsobj=tsobj, $ - legacy=legacy, plates=plates, /silent - if (keyword_set(tsobj)) then $ - copy_struct, tsobj, platedat $ - else $ - splog, 'WARNING: No tsObj file found for '+str_p+' ', outdat[indx[0]].field - endif else begin - readspec, field_plate, mjd=plist[ifile].mjd, $ - run2d=strtrim(plist[ifile].run2d), $ - legacy=legacy, plates=plates, /silent - endelse - copy_struct, outdat[indx], platedat - - ; All strings must be the same length, or appending to the FITS file - ; will result in corruption. The only string in the tsobj structure - ; is for the RERUN. - ; HJIM Decoment the next line - if no_photo_file eq 0 then $ - platedat.rerun = string(platedat.rerun+' ',format='(a3)') - - mwrfits_chunks, platedat, outroot[0]+'.fits.tmp', $ - create=(ifile EQ 0), append=(ifile GT 0) - endfor - - outdat = 0 ; Clear memory - - ;---------- - ; Create the structure for ASCII output field - - adat1 = create_struct( $ - str_p , 0L, $ - 'mjd' , 0L, $ - 'fiberid' , 0L, $ - 'class' , '', $ - 'subclass' , '', $ - 'z' , 0.0, $ - 'z_err' , 0.0, $ - 'zwarning' , 0L, $ - 'fiber_ra' , 0.0d, $ - 'fiber_dec' , 0.0d, $ - 'specprimary', 0L, $ - 'fieldsn2' , 0.0, $ - 'objtype' , '', $ - 'tileid' , 0L, $ - 'objc_type' , '', $ - 'modelflux' , fltarr(5) ) - - - if keyword_set(legacy) then $ - adat1 = struct_addtags(adat1, {chunk:'',DEREDSN2:'',$ - specprimary:0B,specboss:0B,boss_specobj_id:0L,$ - nspecobs:0}) - - tag_alias = [['SPECPRIMARY','PRIMARY'], $ - ['FIBERID','FIBER']];, $ - - ; Read the tags that we need from the FITS file - outdat = hogg_mrdfits(outroot[0]+'.fits.tmp', 1, nrowchunk=10000L, $ - columns=tag_names(adat1), /unsigned) - adat = replicate(adat1, n_elements(outdat)) - copy_struct, outdat, adat - - ; Replace any blank strings for CLASS with "". - ii = where(strtrim(adat.class,2) EQ '') - if (ii[0] NE -1) then adat[ii].class = '""' - - ; Replace any blank strings for SUBCLASS with "". - ; If SUBCLASS contains several words, then use a plus sign between - ; the words rather than a space. - adat.subclass = strtrim(adat.subclass,2) - ii = where(adat.subclass EQ '') - if (ii[0] NE -1) then adat[ii].subclass = '""' - adat.subclass = repstr(adat.subclass, ' ', '+') - - objtypes = ['UNKNOWN', 'CR', 'DEFECT', 'GALAXY', 'GHOST', 'KNOWNOBJ', $ - 'STAR', 'TRAIL', 'SKY'] - ;HJIM decomet the next line - if no_photo_file eq 0 then $ - adat.objc_type = objtypes[outdat.objc_type] - - outdat = 0 ; Clear memory - - splog, 'Writing ASCII file ' + outroot[0]+'.dat' - struct_print, adat, filename=outroot[0]+'.dat.tmp', alias=tag_alias - - adat = 0 ; Clear memory - - ;---------- - ; Create the merged line data - - if (not keyword_set(skip_line)) then begin - splog, 'Writing FITS zline file ' + outroot[1]+'.fits' - for ifile=0L, nfile-1 do begin - field_plate=plist[ifile].field - splog, 'Writing zline ', ifile+1, ' of ', nfile - readspec, field_plate, mjd=plist[ifile].mjd, $ - run2d=strtrim(plist[ifile].run2d), run1d=strtrim(plist[ifile].run1d), $ - zline=linedat, legacy=legacy, plates=plates, /silent - if not (tag_exist(linedat,'field')) then begin - adatag = create_struct('field',field_plate) - endif - if (ifile EQ 0) then begin - nobj = total(plist.n_total) - nper = n_elements(linedat) / plist[0].n_total - sxaddpar, linehdr, 'DIMS0', nper, ' Number of emission lines' - sxaddpar, linehdr, 'DIMS1', nobj, ' Number of objects' - linedat1 = linedat[0] - struct_assign, {junk:0}, linedat1 - if keyword_set(adatag) then begin - linedat1=create_struct(adatag,linedat1) - endif - endif - ; Demand that the structure has the same format as the first - ; one written. - linedat_out = replicate(linedat1, n_elements(linedat)) - ;adatag_in = replicate(adatag, n_elements(linedat)) - struct_assign, linedat, linedat_out - ;struct_assign, adatag_in, linedat_out - if keyword_set(adatag) then begin - linedat_out.field=field_plate - endif - mwrfits_chunks, linedat_out, outroot[1]+'.fits.tmp', linehdr, $ - create=(ifile EQ 0), append=(ifile GT 0) - endfor - endif - - if keyword_set(lite) then begin - exclude_spall_tags=['FIELDID_LIST','XFOCAL','YFOCAL','DESIGNS','CONFIGS',$ - 'MJD_LIST','TAI_LIST','FIELDSN2G_LIST','FIELDSN2R_LIST',$ - 'FIELDSN2I_LIST','RA_LIST', 'DEC_LIST', 'BLUEFIBER',$ - 'DELTA_RA_LIST', 'DELTA_DEC_LIST',$ - 'ZOFFSET','SDSSV_BOSS_TARGET0','ASSIGNED',$ - 'ON_TARGET','VALID','MOON_DIST','MOON_PHASE','FIBERID_LIST',$ - 'FIELDSNR2G_LIST','FIELDSNR2R_LIST','FIELDSNR2I_LIST',$ - 'TCOLUMN','NPOLY','THETA','FRACNSIGMA','FRACNSIGHI',$ - 'FRACNSIGLO','HEALPIX_PATH','carton_to_target_pk'] - - spall = mrdfits(outroot[0]+'.fits.tmp',1) - spall_lite = struct_selecttags(spall,except_tags=exclude_spall_tags) - spall_lite = struct_addtags(spall_lite, replicate(create_struct('ASSIGNED',0S,$ - 'ON_TARGET',0S,$ - 'VALID',0S,$ - 'MOON_DIST',0.0,$ - 'MOON_PHASE',0.0,$ - 'carton_to_target_pk',0LL),n_elements(spall))) - for i=0, n_elements(spall)-1 do begin - spall_lite[i].ASSIGNED = min(FIX(strsplit(spall[i].ASSIGNED,/extract))) - spall_lite[i].ON_TARGET = min(FIX(strsplit(spall[i].ON_TARGET,/extract))) - spall_lite[i].VALID = min(FIX(strsplit(spall[i].VALID,/extract))) - spall_lite[i].MOON_DIST = avg(float(strsplit(spall[i].MOON_DIST,/extract))) - spall_lite[i].MOON_PHASE = avg(float(strsplit(spall[i].MOON_PHASE,/extract))) - spall_lite[i].carton_to_target_pk = LONG64(spall[i].carton_to_target_pk) - endfor - spall_name=REPSTR(outroot[0],'spAll','spAll-lite') - MWRFITS, spall_lite, spall_name+'.fits.tmp', fits_hdr, Status=Status - spawn, ['gzip', spall_name+'.fits.tmp'], /noshell - endif - - ;---------- - ; Rename temporary files - ;print, outroot[0] - spawn, ['mv', outroot[0]+'.fits.tmp', outroot[0]+'.fits'], /noshell - spawn, ['gzip', outroot[0]+'.dat.tmp'], /noshell - spawn, ['mv', outroot[0]+'.dat.tmp.gz', outroot[0]+'.dat.gz'], /noshell - if (not keyword_set(skip_line)) then $ - spawn, ['mv', outroot[1]+'.fits.tmp', outroot[1]+'.fits'], /noshell - if keyword_set(lite) then $ - spawn, ['mv', spall_name+'.fits.tmp.gz', spall_name+'.fits.gz'], /noshell - thismem = memory() - maxmem = thismem[3] - splog, 'Maximum memory usage = ', maxmem/1.d6, ' MB' - splog, 'Total time = ', systime(1)-t1, ' sec' - - return -end - -;------------------------------------------------------------------------------ -pro fieldmerge, run2d=run2d, indir=indir, mergerun2d=mergerun2d, programs=programs, $ - legacy=legacy, plates=plates, skip_specprimary=skip_specprimary, lite=lite, _EXTRA=Extra - -RESOLVE_ALL, /QUIET, /SKIP_EXISTING, /CONTINUE_ON_ERROR -CPU, TPOOL_NTHREADS = 1 - - undefine, logfile - if TAG_EXIST(Extra,'field',/QUIET) then begin - logfile='fieldmerge' - logfile=logfile+'-'+field_to_string(Extra.field) - if TAG_EXIST(Extra,'mjd',/QUIET) then logfile=logfile+'-'+strtrim(Extra.mjd,2) - logfile = djs_filepath(logfile, root_dir='') - - logfile=logfile+'.log' - cpbackup, logfile - splog, filename=logfile - splog, 'Log file ' + logfile + ' opened ' + systime() - endif - - if keyword_set(mergerun2d) then begin - conflist, outdir=getenv('BOSS_SPECTRO_REDUX'), plist=plist - fieldmerge1, plist=plist, legacy=legacy, skip_specprimary=skip_specprimary, $ - plates=plates, lite=lite, _EXTRA=Extra - - endif else begin - - conflist, plist=plist, topdir=indir, run2d=run2d - ;print, plist;indir - if (NOT keyword_set(plist)) then return - - if keyword_set(programs) then begin - splog, 'Selecting only plates with programname:' - splog, programs - nmatch = lonarr(n_elements(plist)) - for i=0, n_elements(programs)-1 do $ - nmatch+= strmatch(strtrim(plist.PROGRAMNAME,2), strtrim(programs[i],2) ) - indx = where( nmatch GT 0, ct) - if (ct EQ 0) then begin - splog, 'No plates found with programnames : ' - splog, programs - return - endif - plist=plist[indx] - fieldmerge1, plist=plist, run2d=run2d, legacy=legacy, skip_specprimary=skip_specprimary, $ - plates=plates, lite=lite, _EXTRA=Extra - return - endif - - - - alldir = strtrim(plist.run2d) - alldir = alldir[uniq(alldir, sort(alldir))] - if (keyword_set(run2d)) then begin - nmatch = lonarr(n_elements(alldir)) - for i=0, n_elements(run2d)-1 do $ - nmatch += strmatch(alldir,run2d[i]) - indx = where(nmatch GT 0, ct) - if (ct EQ 0) then return - alldir = alldir[indx] - endif - splog, alldir - for i=0, n_elements(alldir)-1 do $ - fieldmerge1, run2d=alldir[i], indir=indir, legacy=legacy, skip_specprimary=skip_specprimary, $ - plates=plates, lite=lite, _EXTRA=Extra - - endelse - - splog, 'Successful completion of FIELDMERGE at ' + systime() - ;---------- - ; Close log files - splog, /close -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/filter_check.pro b/pro/spec1d/filter_check.pro deleted file mode 100644 index 9e719b784..000000000 --- a/pro/spec1d/filter_check.pro +++ /dev/null @@ -1,112 +0,0 @@ -;+ -; NAME: -; filter_check -; -; PURPOSE: -; Fit for the filter curves by comparing photometry and spectrophotometry -; -; CALLING SEQUENCE: -; -; filter_check,spallfile,binboundsfile -; -; INPUTS: -; spallfile - spectro info -; -; OPTIONAL INPUTS: -; -; binboundsfile - bin boundaries -; -; OPTIONAL KEYWORDS: -; -; OUTPUTS: -; binRfile - file containing output stuff -; -; COMMENTS: -; -; BUGS: -; -; EXAMPLES: -; -; PROCEDURES CALLED: -; -; DATA FILES: -; -; REVISION HISTORY: -; 05-APr-2000 Written by M. Blanton, Fermiland -;- -;------------------------------------------------------------------------------ -pro filter_check, spselect, flux,invvar,loglam, binRbase, $ -binbounds=binbounds, nosubtract=nosubtract - - ;-------- - ; Solve in each band - nselect=n_elements(spselect) - for band=1, 3 do begin - - ;-------- - ; Create photocounts - photomags=spselect[*].psfCounts[band] $ - +(spselect[*].fiberCounts[2]-spselect[*].psfCounts[2]) - photocounts=10.^(-0.4*photomags)-spselect[*].counts_spectro*0.79 - - ;-------- - ; Read in bin boundaries, bin spectra if desired - if(keyword_set(binbounds)) then begin - ;filename=binboundsbase+'.'+strtrim(string(i),2)+'.dat' - ;openr,11,filename - ;readf,nbins - ;binbounds=lonarr(nbins+1) - ;readf,binbounds - ;close,11 - nbins=n_elements(binbounds)-1 - binflux=fltarr(nbins+1-keyword_set(nosubtract),nselect) - bin_spectra,flux,invvar,binbounds,binflux=binflux - endif else begin - if(keyword_set(nosubtract)) then return - nbins=0 - binflux=fltarr(1,nselect) - endelse - - ;-------- - ; Flux convolved with bins - if(not keyword_set(nosubtract)) then $ - binflux[nbins,*]=spselect[*].counts_spectro[band] - - ;------- - ; Iterate - indx=lindgen(n_elements(spselect)) - for iter=0, 1 do begin - - ;-------- - ; Find solution - filter_solve,binflux[*,indx],photocounts[indx],binR - - ;------- - ; Sigma clip - resid=transpose(binflux##binR)-photocounts - rms=djsig(resid[indx]) - help,rms - mean=djs_mean(resid[indx]) - help,mean - indx=where((resid/rms)^2 lt 100.) - - endfor - print, binR -erase & plot, 10.0^(0.5*(loglam[binbounds[0:nbins-1]]+loglam[binbounds[1:nbins]])),binR,psym=10 -stop -erase & plot, spselect.psfcounts[1]-spselect.psfcounts[2], resid, psym=1 -djs_oplot, photocounts[indx], resid[indx], psym=1, color='red' - -stop - - ;-------- - ; Output solution - filename=binRbase+'.'+strtrim(string(band),2)+'.dat' - openw,11,filename - printf,11,binR - close,11 - - endfor - -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/filter_check_chi2.pro b/pro/spec1d/filter_check_chi2.pro deleted file mode 100644 index 295d0c826..000000000 --- a/pro/spec1d/filter_check_chi2.pro +++ /dev/null @@ -1,43 +0,0 @@ -;+ -; NAME: -; filter_check_chi2 -; -; PURPOSE: -; Calculate the chi^2 for a particular set of bandpass residuals. -; -; CALLING SEQUENCE: -; chi2 = filter_check_chi2(binflux, binR, photocounts) -; -; INPUTS: -; binflux - binned flux -; binR - parameters to use for flux -; photocounts - psf_band + (fiber_r-psf_r) (not dereddened) -; -; OPTIONAL INPUTS: -; -; OPTIONAL KEYWORDS: -; -; OUTPUTS: -; chi2 - resulting chi2 -; -; COMMENTS: -; -; BUGS: -; -; EXAMPLES: -; -; PROCEDURES CALLED: -; -; DATA FILES: -; -; REVISION HISTORY: -; 05-APr-2000 Written by M. Blanton, Fermiland -;- -;------------------------------------------------------------------------------ -pro filter_check_chi2, binflux, binR, photocounts - - fullbinR=(fltarr((size(binflux))[1])+1)##binR - chi2=total(binflux*fullbinR,/double)-total(photocounts,/double) - -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/filter_select.pro b/pro/spec1d/filter_select.pro deleted file mode 100644 index dcc2ba285..000000000 --- a/pro/spec1d/filter_select.pro +++ /dev/null @@ -1,170 +0,0 @@ -;+ -; NAME: -; filter_select -; -; PURPOSE: -; Gather spectra based on an input file of the form created by platemerge (the spAll file). -; Calculate the ugriz throughput for each object in the plates, possibly -; putting limits on target type, MJD, or signal-to-noise -; (essentially by requiring survey quality). -; -; CALLING SEQUENCE: -; filter_select, spallfile, outfile, [mjdlimits= , primtarget=, -; filter_prefix=, mingisn2=, rpsfmodel=] -; -; INPUTS: -; spallfile - spAll.fit file as created by platemerge -; filter_prefix - Use alternate prefix for filter curves to use -; (allowed are sdss or doi) -; -; OPTIONAL INPUTS: -; -; OPTIONAL KEYWORDS: -; mjdlimits - Only look in a certain range of MJDs -; primtarget - Require a certain target type -; mingisn2 - Minimum plate SN^2 in g AND i -; -; OUTPUTS: -; outfile - Fits file with all the spAll.fit info, but with -; synthetic ugriz replaced with the desired filter -; curves -; -; COMMENTS: -; -; BUGS: -; Depends on spec_append, internal routine of readspec -; -; EXAMPLES: -; -; PROCEDURES CALLED: -; filter_thru() -; spec_append -; readspec -; mrdfits() -; mwrfits -; -; REVISION HISTORY: -; 05-APr-2000 Written by M. Blanton, Fermiland -;- -;------------------------------------------------------------------------------ -pro filter_select, spallfile, outbase, filter_prefix, mjdlimits=mjdlimits, $ - primtarget=primtarget, mingisn2=mingisn2, firstn=firstn, $ - rfluxlimits=rfluxlimits, rpsfmodel=rpsfmodel, $ - flux=flux,invvar=invvar,loglam=loglam, spselect=spselect, $ - outspectra=outspectra - - ;-------- - ; Read in the file - spall=mrdfits(spallfile,1) - nall=n_elements(spall) - splog,string(nall)+' objects total' - - ;-------- - ; Select desired spectra - - ; set all initially to selected - select=intarr(nall) - select[*]=1 - - ; throw out those outside MJD limits (inclusive) - if(keyword_set(mjdlimits)) then begin - indx=where(spall.mjd lt mjdlimits[0] or spall.mjd gt mjdlimits[1],count) - if(count gt 0) then select[indx]=0 - endif - - ; throw out those outside flux limit (inclusive) - if(keyword_set(rfluxlimits)) then begin - indx=where(spall.psfCounts[2] lt rfluxlimits[0] or $ - spall.psfCounts[2] gt rfluxlimits[1],count) - if(count gt 0) then select[indx]=0 - endif - - ; throw out those outside flux limit (inclusive) - if(keyword_set(rpsfmodel)) then begin - indx=where(spall.psfcounts[2]-spall.counts_model[2] gt rpsfmodel,count) - if(count gt 0) then select[indx]=0 - endif - - if(keyword_set(primtarget)) then begin - indx=where((spall.primtarget and primtarget) eq 0,count) - if(count gt 0) then select[indx]=0 - endif - - if(keyword_set(mingisn2)) then begin - indx=where(spall.spec1_g lt mingisn2 or $ - spall.spec2_g lt mingisn2 or $ - spall.spec1_i lt mingisn2 or $ - spall.spec2_i lt mingisn2,count ) - if(count gt 0) then select[indx]=0 - endif - indx=where(select gt 0,count) - - if(keyword_set(firstn)) then begin - if (count gt firstn) then count=firstn - endif - - if(count eq 0) then begin - splog,'no objects ' - return - endif - - spselect=spall[indx[0:count-1]] - nselect=n_elements(spselect) - - ;-------- - ; Gather all of the spectra - splog,string(nselect)+' objects selected - - readspec,spselect.plate, $ - spselect.fiberid, $ - mjd=spselect.mjd, $ - flux=compflux,invvar=compinvvar, $ - loglam=comploglam,andmask=compandmask, ormask=compormask - compinvvar = skymask(compinvvar, compandmask, compormask) - compandmask = 0 ; Free memory - compormask = 0 ; Free memory - - npixobj=n_elements(comploglam)/nselect -; loglammin=max(comploglam[0,*]) -; loglammax=min(comploglam[npixobj-1,*]) - loglammin=3.5855 - loglammax=3.9610 - nloglam=round(10000l*(loglammax-loglammin)) - flux=fltarr(nloglam,nselect) - invvar=fltarr(nloglam,nselect) - loglam=fltarr(nloglam,nselect) - for i = 0, nselect-1 do begin - ilmin=10000*(loglammin-comploglam[0,i]) - flux[*,i]=compflux[ilmin:ilmin+nloglam-1,i] - invvar[*,i]=compinvvar[ilmin:ilmin+nloglam-1,i] - loglam[*,i]=comploglam[ilmin:ilmin+nloglam-1,i] - end - - ;-------- - ; Send the spectra and fluxes to the synthesizer; - ; an attempt to do here what is done in spreduce1d.pro - waveimg=10d^loglam - flambda2fnu = 1.e+6 * waveimg^2 / (3631.*2.99792e18) - flux=flux*rebin(flambda2fnu, nloglam, nselect) - fthru=filter_thru(flux, waveimg=waveimg, mask=(invvar eq 0), $ - filter_prefix=filter_prefix, /toair) - - for band=0, 4 do begin - spselect[*].counts_spectro[band]=fthru[*,band] - endfor - - if (keyword_set(outspectra)) then begin - mwrfits,spselect,outbase+'.fits',/create - openw,11,outbase+'.flux' - openw,12,outbase+'.invvar' - openw,13,outbase+'.loglam' - writeu,11,float(flux) - writeu,12,float(invvar) - writeu,13,float(loglam) - close,11 - close,12 - close,13 - endif - -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/filter_solve.pro b/pro/spec1d/filter_solve.pro deleted file mode 100644 index 0d66ae17a..000000000 --- a/pro/spec1d/filter_solve.pro +++ /dev/null @@ -1,62 +0,0 @@ -;+ -; NAME: -; filter_solve -; -; PURPOSE: -; Minimize the chi^2 by varying the filter residuals. Basically solve Ax=b. -; -; CALLING SEQUENCE: -; filter_solve,binflux,photocounts,binR -; -; INPUTS: -; binflux - binned flux -; photocounts - psf_band + (fiber_r-psf_r) (not dereddened) -; -; OPTIONAL INPUTS: -; -; OPTIONAL KEYWORDS: -; -; OUTPUTS: -; -; binR - best fit filter parameters -; -; COMMENTS: -; -; BUGS: -; -; EXAMPLES: -; -; PROCEDURES CALLED: -; -; DATA FILES: -; -; REVISION HISTORY: -; 05-APr-2000 Written by M. Blanton, Fermiland -;- -;------------------------------------------------------------------------------ -pro filter_solve, binflux, photocounts, binR - - ;------------------ - ; Make A and invert - nbins=(size(binflux))[1] - covar=fltarr(nbins,nbins) - for i = 0, nbins-1 do begin - for j = 0, nbins-1 do begin - covar[i,j]=total(binflux[i,*]*binflux[j,*],/double) - endfor - endfor - invcovar=invert(covar,/double) - - ;--------- - ; Make b - rhs=fltarr(1,nbins) - for i = 0, nbins-1 do begin - rhs[0,i]=total(binflux[i,*]*photocounts,/double) - endfor - - ;--------- - ; Multiply to get binR - binR=invcovar##rhs - -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/find_npeaks.pro b/pro/spec1d/find_npeaks.pro deleted file mode 100644 index 3a1c83b56..000000000 --- a/pro/spec1d/find_npeaks.pro +++ /dev/null @@ -1,81 +0,0 @@ -; This used to be used by ZCOMPUTE, but is no longer. -function find_npeaks, yflux, xvec, nfind=nfind, minsep=minsep, $ - width=width, ypeak=ypeak, xerr=xerr, npeak=npeak - - ndata = n_elements(yflux) - if (ndata EQ 1) then $ - return, 0 - - if (NOT keyword_set(xvec)) then xvec = lindgen(ndata) - if (NOT keyword_set(nfind)) then nfind = 1 - if (n_elements(minsep) EQ 0) then minsep = 0 - if (NOT keyword_set(width)) then width = 3 - if (xvec[1] GT xvec[0]) then isign = 1 $ ; ascending X - else isign = -1 ; descending X - - ;---------- - ; Make a copy of YFLUX for finding local maxima; this will be modified - ; each time a peak is found by filling with values of YDONE where we - ; are no longer allowed to search. - - ycopy = yflux - yderiv = [ycopy[1:ndata-1] - ycopy[0:ndata-2], 0] - ydone = min(ycopy) - - ;---------- - ; Find up to NFIND peaks - - for ifind=0, nfind-1 do begin - - ;---------- - ; Locate next maximum - - junk = max(ycopy, imax) - - ;---------- - ; Centroid on this peak - - xpeak1 = zfitmax(yflux, xvec, xguess=xvec[imax], width=width, $ - xerr=xerr1, ypeak=ypeak1) - - ;---------- - ; Save return values - - if (ifind EQ 0) then begin - xpeak = xpeak1 - xerr = xerr1 - ypeak = ypeak1 - endif else begin - xpeak = [xpeak, xpeak1] - xerr = [xerr, xerr1] - ypeak = [ypeak, ypeak1] - endelse - - ;---------- - ; Exclude from future peak-finding all points within MINSEP of this - ; peak, up until the function is increasing again. - - junk = min(abs(xvec - xvec[imax]), ixc) - ix1 = (reverse(where(isign*xvec LT (isign*xvec[imax] - minsep) $ - AND shift(yderiv,1) LT 0)))[0] - if (ix1 EQ -1) then ix1 = 0 - ix2 = (where(isign*xvec GT (isign*xvec[imax] + minsep) AND yderiv GT 0))[0] - if (ix2 EQ -1) then ix2 = ndata-1 - - ycopy[ix1:ix2] = ydone -;print,xpeak1,ix1,ixc,ix2, xvec[imax] -;splot,yflux -;soplot,ycopy,color='red' - - ;---------- - ; Test to see if we can find any more peaks - - junk = where(ycopy GT ydone, ct) - if (ct EQ 0) then ifind = nfind - - endfor - - npeak = n_elements(xpeak) - - return, xpeak -end diff --git a/pro/spec1d/findchi2min.pro b/pro/spec1d/findchi2min.pro deleted file mode 100644 index 312a90a5b..000000000 --- a/pro/spec1d/findchi2min.pro +++ /dev/null @@ -1,59 +0,0 @@ -;------------------------------------------------------------------------------ -; This routine used to be used by VDISPFIT, but is no longer as of 10-Feb-02. -;------------------------------------------------------------------------------ -pro findchi2min, x, chi2, minchi2, minsigma, errsigma, npts=npts, $ - deltachisq=deltachisq, nfine=nfine, $ - plottitle=plottitle, doplot=doplot, debug=debug - - if (NOT keyword_set(deltachisq)) then deltachisq = 1.0 - if (NOT keyword_set(nfine)) then nfine = 100 - - ; spline chi2 curve and return desired values - - nx = n_elements(x) - if (nx EQ 0) then return - - minx = min(x, max=maxx) - y2 = spl_init(x, chi2) - - x2 = findgen(nx*nfine+1)/(nx*nfine) * (maxx - minx) + minx - chifit = spl_interp(x, chi2, y2, x2) - - minchi2 = min(chifit,fitplace) - - minsigma = x2[fitplace] - - ; attempt to scale chi^2 to get errors correct ??? - - usethisdelta = deltachisq - if (keyword_set(npts)) then begin - scale = minchi2/npts - if (scale GT 1.0) then usethisdelta = deltachisq * scale - endif - - range = where(chifit - minchi2 LT usethisdelta) - - if (range[0] EQ -1) then return - - uppersigma = x2[max(range)] - minsigma - lowersigma = minsigma - x2[min(range)] - - errsigma = 0.5*(uppersigma + lowersigma) - - if (keyword_set(doplot)) then begin - plot, x, chi2, psym=1, $ - title=plottitle - djs_oplot, x2, chifit - djs_oplot, x2[range], chifit[range], color='red' - - ; Wait for a keystroke... - if (keyword_set(debug)) then begin - print, 'Press any key...' - cc = strupcase(get_kbrd(1)) - endif - - endif - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/findfiber.pro b/pro/spec1d/findfiber.pro deleted file mode 100644 index 7b36710a9..000000000 --- a/pro/spec1d/findfiber.pro +++ /dev/null @@ -1,25 +0,0 @@ -function findfiber, in, out - - nfiber = n_elements(in) - - - for i=0, nfiber - 1 do begin - - tt = where(in[i].objid[0] EQ out.run AND in[i].objid[1] EQ out.rerun AND $ - in[i].objid[2] EQ out.camcol AND in[i].objid[3] EQ out.field AND $ - in[i].objid[4] EQ out.id, n) - - if n NE 1 then begin - tt = 0 - print, i, 'object not found' - endif - - if i EQ 0 then flist = tt[0] $ - else flist = [flist, tt[0]] - - endfor - - return, flist -end - - diff --git a/pro/spec1d/findspec.pro b/pro/spec1d/findspec.pro deleted file mode 100644 index a8a09f396..000000000 --- a/pro/spec1d/findspec.pro +++ /dev/null @@ -1,258 +0,0 @@ -;+ -; NAME: -; findspec -; -; PURPOSE: -; Routine for finding SDSS spectra that match a given RA, DEC. -; -; CALLING SEQUENCE: -; findspec, [ra, dec, infile=, outfile=, searchrad=, slist=, $ -; topdir=, run2d=, run1d=, /best, /print, /sdss ] -; -; INPUTS: -; -; OPTIONAL INPUTS: -; ra - Right ascension; scalar or array in degrees. -; dec - Declination; scalar or array in degrees. -; infile - Input file with RA, DEC positions, one per line. -; If set, then this over-rides values passed in RA,DEC. -; outfile - If set, then print matches to this file. -; searchrad - Search radius in degrees; default to 3./3600 (3 arcsec). -; topdir - Optional override value for the environment -; variable $BOSS_SPECTRO_REDUX. -; run2d - Override environment variable RUN2D, searching only for -; those RUN2D reductions -; run1d - Override environment variable RUN1D, searching only for -; those RUN1D reductions -; best - If set, then return the best match for each location, where -; best is defined to be the closest object on the plate with -; the best S/N. -; This also forces the return of one structure element in SLIST -; per position, so that you get exactly a paired list between -; RA,DEC and SLIST. -; print - If set, then print matches to the terminal. -; sdss - This is a shortcut option that is exactly equivalent to -; topdir=GETENV('SPECTRO_REDUX'), run2d='26' -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; slist - Structure containing information for each match. -; -; COMMENTS: -; The search radius is set to within 1.55 degress of a plate center, -; then within 3 arcsec of an object. -; -; findspec.pro is currently setup to use BOSS data, but it still works -; with SDSS-I,II data with the following procedure: -; -; findspec, ra, dec, topdir=GETENV('SPECTRO_REDUX'), run2d='26' -; -; or -; -; findspec, ra, dec, /sdss -; -; EXAMPLES: -; Make a file "file.in" with the following two lines: -; 218.7478 -0.3745007 -; 217.7803 -0.8900855 -; -; Then run the command: -; IDL> findspec,infile='file.in',/sdss,/print -; -; This should print: -; -;PLATE MJD FIBERID RA DEC MATCHRAD -;----- ----- ------- ------------- -------------- ----------- -; 306 51637 101 218.7478 -0.3745007 0.0000 -; 306 51690 117 218.7478 -0.3745007 0.0000 -; 306 51637 201 217.7803 -0.8900855 2.9996e-05 -; 306 51690 217 217.7803 -0.8900855 2.9996e-05 -; -; BUGS: -; -; PROCEDURES CALLED: -; djs_readcol -; djs_diff_angle() -; platelist -; readspec -; struct_print -; -; REVISION HISTORY: -; 15-Feb-2001 Written by David Schlegel, Princeton. -;- -;------------------------------------------------------------------------------ -pro findspec, ra, dec, infile=infile, outfile=outfile, searchrad=searchrad, $ - slist=slist, topdir=topdir1, run2d=run2d1, run1d=run1d1, $ - best=best, print=print1, sdss=sdss - - common com_findspec, plist, nlist, lasttopdir, last2d=last2d, last1d=last1d - - if (keyword_set(sdss)) then begin - topdir1 = getenv('SPECTRO_REDUX') - run2d1 = '26' - endif - if (keyword_set(run2d1)) then run2d = strtrim(run2d1,2) $ - else run2d = getenv('RUN2D') - splog, 'Setting RUN2D=', run2d - if (keyword_set(run1d1)) then run1d = strtrim(run1d1,2) $ - else run1d = getenv('RUN1D') - splog, 'Setting RUN1D=', run2d - if (keyword_set(topdir1)) then topdir = topdir1 $ - else topdir = getenv('BOSS_SPECTRO_REDUX') - - if (keyword_set(lasttopdir) EQ 0) then lasttopdir = '' - if (keyword_set(last2d) EQ 0) then last2d = '' - if (keyword_set(last1d) EQ 0) then last1d = '' - if (keyword_set(plist) EQ 0 OR topdir NE lasttopdir $ - OR run2d NE last2d OR run1d NE last1d) then begin - lasttopdir = topdir - last2d = run2d - last1d = run1d - platelist, plist=plist, topdir=topdir, run2d=run2d, run1d=run1d -; platelist_files = file_search(topdir+'/platelist.fits') -; plates_files = file_search(topdir+'/plates-*.fits') -; if (strlen(platelist_files[0]) GT 0) then $ -; plist = mrdfits(platelist_files[0],1,/silent) -; if (strlen(plates_files[0]) GT 0) then $ -; plist = mrdfits(plates_files[0],1,/silent) -; if (NOT keyword_set(plist)) then $ -; message, 'Plate list (platelist.fits or plates-*.fits) not found in ' + topdir - nlist = n_elements(plist) - endif - - qdone = strmatch(plist.status1d,'Done*') - if (tag_exist(plist, 'RUN2D')) then begin - qdone2d = bytarr(nlist) - for i=0, n_elements(run2d)-1 do $ - qdone2d = qdone2d OR strmatch(strtrim(plist.run2d,2), run2d[i]) - qdone = qdone AND qdone2d - endif - if (tag_exist(plist, 'RUN1D')) then begin - qdone1d = bytarr(nlist) - for i=0, n_elements(run1d)-1 do $ - qdone1d = qdone1d OR strmatch(strtrim(plist.run1d,2), run1d[i]) - qdone = qdone AND qdone1d - endif - idone = where(qdone, ndone) - if (ndone EQ 0) then begin - splog, 'No reduced plates!' - slist = 0 - return - endif - - ;---------- - ; Read an input file if specified - - if (keyword_set(infile)) then begin - djs_readcol, infile, ra, dec, format='(D,D)' - endif - nobj = n_elements(ra) - - if (NOT keyword_set(searchrad)) then searchrad = 3./3600. - - ;---------- - ; Create output structure - - blanklist = create_struct(name='slist', $ - 'plate' , 0L, $ - 'mjd' , 0L, $ - 'fiberid' , 0L, $ - 'ra' , 0.d, $ - 'dec' , 0.d, $ - 'matchrad', 0.0 ) - - ; Set default return values - if (keyword_set(best)) then slist = replicate(blanklist, nobj) $ - else slist = 0 - - ;---------- - ; Match all plates with objects - - nplate = n_elements(plist) - spherematch, ra, dec, plist[idone].racen, plist[idone].deccen, $ - searchrad+1.55, imatch1, itmp, dist12, maxmatch=0 - if (imatch1[0] EQ -1) then return - imatch2 = idone[itmp] - - ;---------- - ; Read all relevant plates - - ; Read the number of fibers per plate, or assume 640 if old SDSS-I platelist - if (tag_exist(plist,'N_TOTAL')) then n_total = plist.n_total $ - else n_total = replicate(640L, n_elements(plist)) - - iplate = imatch2[uniq(imatch2,sort(imatch2))] - i0 = 0L - for i=0L, n_elements(iplate)-1L do begin - readspec, plist[iplate[i]].plate, mjd=plist[iplate[i]].mjd, $ - plugmap=plugmap1, topdir=topdir, run2d=run2d, /silent - if (n_elements(plugmap1) EQ 1) then $ - message, 'Failed reading plate ='+strtrim(plist[iplate[i]].plate,2) $ - + ' MJD='+strtrim(plist[iplate[i]].mjd,2) - if (i EQ 0) then plugmap = replicate(create_struct('PLATE',0L,'MJD',0L, $ - 'FIBERID',0L,'RA',0d,'DEC',0d), total(n_total[iplate])) - index_to=i0+lindgen(n_total[iplate[i]]) - plugmap[index_to].plate = plist[iplate[i]].plate - plugmap[index_to].mjd = plist[iplate[i]].mjd - plugmap[index_to].fiberid = plugmap1.fiberid - plugmap[index_to].ra = plugmap1.ra - plugmap[index_to].dec = plugmap1.dec -; copy_struct_inx, plugmap1, plugmap, index_to=index_to - - i0 += n_total[iplate[i]] - endfor - spherematch, ra, dec, plugmap.ra, plugmap.dec, searchrad, $ - i1, i2, d12, maxmatch=0 - if (i1[0] EQ -1) then return - - if (NOT keyword_set(best)) then begin - - ;------------------------------------------------------------------------ - ; RETURN ALL MATCHES - ;------------------------------------------------------------------------ - - slist = replicate(blanklist, n_elements(i1)) - slist.plate = plugmap[i2].plate - slist.mjd = plugmap[i2].mjd - slist.fiberid = plugmap[i2].fiberid - slist.ra = plugmap[i2].ra - slist.dec = plugmap[i2].dec - slist.matchrad = d12 - - endif else begin - ;------------------------------------------------------------------------ - ; RETURN ONLY BEST MATCH PER OBJECT - ;------------------------------------------------------------------------ - - ; Read the median S/N for each spectrum - readspec, plugmap[i2].plate, mjd=plugmap[i2].mjd, $ - plugmap[i2].fiberid, zans=zans, topdir=topdir, $ - run2d=run2d, run1d=run1d, /silent - - ; We have all the possible matches. Now sort those in the order - ; of each input object, but with the last entry being the best - ; according to the SN_MEDIAN value. - isort = sort(i1 + (zans.sn_median>0)/max(zans.sn_median+1.)) - i1 = i1[isort] - i2 = i2[isort] - d12 = d12[isort] - - iuniq = uniq(i1) - slist[i1[iuniq]].plate = plugmap[i2[iuniq]].plate - slist[i1[iuniq]].mjd = plugmap[i2[iuniq]].mjd - slist[i1[iuniq]].fiberid = plugmap[i2[iuniq]].fiberid - slist[i1[iuniq]].ra = plugmap[i2[iuniq]].ra - slist[i1[iuniq]].dec = plugmap[i2[iuniq]].dec - slist[i1[iuniq]].matchrad = d12[iuniq] - endelse - - ;---------- - ; Print to terminal and/or output file - - if (keyword_set(print1)) then struct_print, slist - if (keyword_set(outfile)) then struct_print, slist, filename=outfile - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/fitredshift.pro b/pro/spec1d/fitredshift.pro deleted file mode 100644 index e4eecca98..000000000 --- a/pro/spec1d/fitredshift.pro +++ /dev/null @@ -1,275 +0,0 @@ -;+ -; NAME: -; fitredshift -; -; PURPOSE: -; Find the most significant correlation peak (soon to be peaks) -; return optional keywords in units of pixels -; -; CALLING SEQUENCE: -; fitredshift, fluxfft, starfft, $ -; [ nsearch=, zfit=, z_err=, veldispfit=, veldisp_err=, /doplot ] -; -; INPUTS: -; fluxfft - complex fft of prepared galaxy spectrum -; fluxerr - error vector for flux (only test if it is zero) -; starfft - complex fft of stellar template -; starerr - error vector for flux (only test if it is zero) -; -; OPTIONAL KEYWORDS: -; nsearch - number of peaks to search, almost always only 1 is searched -; zmin - Minimum z (in pixels) to allow (should be < 0) -; doplot - plot the correlation peak and fits in an xwindow -; -; OPTIONAL OUTPUTS: -; zfit - best fit z, this should evolve into an array of z's -; with accompanying z_errs and z_confidences -; z_err - centroid errors from gaussfit of correlation peak -; veldisp - sigma of cross-correlation peak -; veldisp_err- error on sigma -; -; COMMENTS: -; -; Use doplot keyword to see how well peak is being fit -; Still need to work on exact selection criteria for MOST significant peak -; or even better: measure all peaks with probability > 1% -; -; EXAMPLES: -; -; BUGS: -; -; Hardwired exclusion of blueshifts greater than 100 pixels -; this helps the noisiest cases -; -; PROCEDURES CALLED: -; findmaxarea -; curvefit -; gauss_funct -; -; REVISION HISTORY: -; 25-Mar-2000 Written by S. Burles, FNAL -; 26-Jun-2000 D. Finkbeiner - modified to properly weight corr -; vector for variable overlap -; 26-Jun-2000 (v 1.5) altered algorithm to set peak search boundary -; at the half height of the peak, rather than zero. This fixes -; cases where there is a close double peak. -;- -;------------------------------------------------------------------------------ -; This routine locates the 20 highest peaks, and measures -; the peak, center, and area of each. xcen and peak -; contain the best velues. - -pro findmaxarea, look, xcen, peak, maxarea, cen=cen, area=area, pks=pks - - ruin = look - nx = n_elements(look) - x = findgen(nx) - area = fltarr(20) - dev = fltarr(20) - cen = fltarr(20) - pks = fltarr(20) - - for i=0, 19 do begin - - pks[i] = max(ruin,velcen) - - if pks[i] GT 0.0 then begin - - ; find bounds of positive deviation with cap at +/- 3 pixels - lowerbound = max([where(ruin LT 0 AND x LT velcen),velcen-4]) + 1 - upperbound = min([where(ruin LT 0 AND x GT velcen),velcen+4]) - 1 - area[i] = total(ruin[lowerbound:upperbound]) - - dev[i] = stddev([look[velcen-100:lowerbound-1],$ - look[upperbound+1:velcen+100]], /double) - cen[i] = velcen - ruin[lowerbound:upperbound] = 0.0 - endif else i=20 - endfor - - meandev = mean(dev) - dev = dev/meandev - - maxarea = max(area,areaplace) - maxpks = max(pks,pksplace) - - place = pksplace - if (areaplace NE pksplace) then begin - print, 'Max area position does not equal Max peak position', $ - cen[areaplace], cen[pksplace] - print, 'Using Area + 1.5*peak to decide' - maxcomb = max(area + 1.5*pks,place) - endif - - xcen = cen[place] - peak = pks[place] - return -end - -;------------------------------------------------------------------------------ -pro fitredshift, fluxfft, fluxerr, starfft, starerr, $ - nsearch=nsearch, zmin=zmin, zfit=zfit, z_err=z_err, $ - veldispfit=veldisp, veldisp_err=veldisp_err, zconf=zconf, doplot=doplot - - if (NOT keyword_set(nsearch)) then nsearch = 5 - -; returned value defaults - zfit = 0. - z_err = 999. - zconf = 0. - veldisp = 0. - veldisp_err = 999. - -; check dimensions - ASSUME everything is already padded to 2^N - IF stddev([n_elements(fluxfft), n_elements(fluxerr), $ - n_elements(starfft), n_elements(starerr)]) NE 0 THEN BEGIN - help, fluxfft, fluxerr, starfft, starerr - message, 'dimensions do not match!' - ENDIF - - IF (total(abs(fluxfft)) EQ 0) OR (total(abs(starfft)) EQ 0) THEN BEGIN - print, 'FITREDSHIFT: FAILED - array full of zeros' - return - ENDIF - - -; The following compiles GAUSS_FUNCT, which is in the routine GAUSSFIT - junk = gaussfit(findgen(7),findgen(7)) - -; compute correlation of object flux to stellar template - corr = float(fft(fluxfft * conj(starfft),/inverse)) - nx = n_elements(corr) - pad = nx / 2 - -; shift corr vector so there is no discontinuity at zero lag. - corr = shift(corr, pad) - -; Need to fill an array of length fluxfilt which records the number -; of good pixels cross-correlated between star and galaxy as a function -; of shift. We construct this from the error vectors. - - denom = shift(double( $ - fft(fft(fluxerr NE 0)*conj(fft(starerr NE 0)), /inverse)), pad) *nx - - reweight = pad / (denom > 10.0) ; never use less than 10 pix - - -; This loop finds the redshift by searching the nsearch highest peaks - - good = 0 - x = lindgen(nx) - - for i=0, nsearch-1 do begin - - newcorr = corr * sqrt(reweight) ; A hack!!!??? - - if (keyword_set(zmin)) then begin -; ??? - newcorr[0:pad+zmin-1] = 0.0 - endif - - findmaxarea, newcorr, velcen, peak, cen=cen, area=area, pks=pks - - ; Let xtemp be centered about velcen for all newcorr values above 0.0 - - lowerbound = max(where(newcorr LT peak/2 AND x LT velcen)) - upperbound = min(where(newcorr LT peak/2 AND x GT velcen)) - xtemp = x[lowerbound:upperbound] - velcen - if (!version.release LT '5.4') then begin - parabola = poly_fit(xtemp, newcorr[xtemp+velcen], 2, yfit) - endif else begin - parabola = poly_fit(double(xtemp), double(newcorr[xtemp+velcen]), 2, $ - yfit=yfit) - endelse - - if (parabola[2] GE 0.0) then begin - print, 'peak is not well fit at ', velcen - newcorr[xtemp+velcen] = 0.0 - endif else if (total(newcorr[xtemp+velcen]) LT 0.0) then begin - print, 'total newcorr is less than zero at ', velcen - newcorr[xtemp+velcen] = 0.0 - endif else if (total(yfit) LT 0.0) then begin - print, 'total fit is less than zero at ', velcen - newcorr[xtemp+velcen] = 0.0 - endif else begin - i = nsearch - good = 1 -; let's attempt to fit a gaussian with no background terms - endelse - - endfor - - if (NOT good) then begin - print, 'No good peaks found' - return - endif - - xcen = (-0.5 * parabola[1]/parabola[2]) - height = (poly([xcen], parabola))[0] - - ytemp = yfit > 0.0 - guesssig = sqrt(total((xtemp-xcen)^2 * ytemp) / total(ytemp)) - - left = long(velcen + xcen - 1) - lindgen(100) - right = long(velcen + xcen + 1) + lindgen(100) - asig = stddev(newcorr[left]-newcorr[right], /double) / sqrt(2.) - - ; Here's my attempt to fit a gaussian to the correlation peak - ; The main problem here is to decide where the baseline of the - ; gaussian falls. FWHM and sigma depend on where the gaussian fit - ; goes to zero. Very troubling. - -; new xtemp - lowerbound = max(where(newcorr LT height/2 AND x LT velcen))-2 - upperbound = min(where(newcorr LT height/2 AND x GT velcen))+2 - xtemp2 = x[lowerbound:upperbound] - velcen - - - - if (asig GT 0) then weights = xtemp2*0.0 + 1.0/asig^2 $ - else weights = xtemp2*0.0 + 1.0 - - base = -height/3. ; empirical guess of gaussian baseline - - a = double([height, xcen, guesssig, base]) -; gaussf = curvefit(xtemp,newcorr[xtemp+velcen] + height/3.0, weights, $ -; a+0D, gausserrors, function_name="GAUSS_FUNCT") -; gaussf = gaussf - height/3.0 - - gaussf = curvefit(xtemp2,newcorr[xtemp2+velcen], weights, $ - a, gausserrors, function_name="GAUSS_FUNCT", iter=iter) - - IF abs(a[1]) GT 3 THEN BEGIN - message, 'curvefit failed to converge!!!', /info - print, 'Reverting to parabola solution' - - ENDIF - - -; if (keyword_set(doplot)) then begin -; wset,0 - djs_plot, x-velcen, newcorr, ps =10, xr=[-40,40], $ - title='Best correlation peak w/fits (Green:gauss, Red: Parabola)' - djs_oplot, xtemp, poly(xtemp, parabola), color='red' - djs_oplot, xtemp2, gaussf, color='green' -; endif - - zfit = velcen + a[1] - pad - z_err = gausserrors[1] - veldisp = a[2] - veldisp_err = gausserrors[2] - -; zconf = a[0] ; height of gaussian - - twopiei = 2.0 * !dpi * complex(0.0,1.0) - knums = fft_wavenums(n_elements(starfft)) - phase = exp( - twopiei * knums * zfit) - model = double(fft(starfft*phase,/inv)) - gal = double(fft(fluxfft, /inv)) - - starmask = where(starerr NE 0) - chisq = total((model-gal)^2*fluxerr*starmask)/total(starmask) - zconf = chisq - - return -end diff --git a/pro/spec1d/fixfits1d.pro b/pro/spec1d/fixfits1d.pro deleted file mode 100644 index 50d8499c9..000000000 --- a/pro/spec1d/fixfits1d.pro +++ /dev/null @@ -1,78 +0,0 @@ -;------------------------------------------------------------------------------ -pro fixfits1d - - vers = 'v4_1_2' - - filename = findfile('0*/spZ*.fits', count=nfile) - - for ifile=0, nfile-1 do begin - print, 'Modifying ' + filename[ifile] - - hdr = headfits(filename[ifile]) - - sxaddpar, hdr, 'NAXIS', 0 - sxdelpar, hdr, 'NAXIS1' - sxdelpar, hdr, 'NAXIS2' - sxaddpar, hdr, 'EXTEND', 'T', after='NAXIS' - sxdelpar, hdr, 'WCSDIM' - - sxaddpar, hdr, 'VERSUTIL', vers - sxaddpar, hdr, 'VERSREAD', vers - sxaddpar, hdr, 'VERS2D', vers - sxaddpar, hdr, 'VERSCOMB', vers - sxaddpar, hdr, 'VERS1D', vers, $ - 'Version of idlspec2d for 1D reduction', after='VERSCOMB' - - modfits, filename[ifile], 0, hdr - endfor -end - -;------------------------------------------------------------------------------ -pro fixfits2d - - vers = 'v4_1_2' - - filename = findfile('0*/spPlate*.fits', count=nfile) - - for ifile=0, nfile-1 do begin - ;---------- - ; Fix the first HDU - - print, 'Modifying ' + filename[ifile] - - hdr = headfits(filename[ifile]) - nhead = n_elements(hdr) - - sxdelpar, hdr, 'WCSDIM' - sxdelpar, hdr, 'FOCUS' - - sxaddpar, hdr, 'VERSUTIL', vers - sxaddpar, hdr, 'VERSREAD', vers - sxaddpar, hdr, 'VERS2D', vers - sxaddpar, hdr, 'VERSCOMB', vers - - ; Make certain we have exactly the same number of header cards - for iadd=0, nhead-n_elements(hdr)-1 do $ - sxaddpar, hdr, 'HISTORY', ' ' - - modfits, filename[ifile], 0, hdr - - ;---------- - ; Fix the other HDU's - - for ihdu=1, 4 do begin - nexthdr = headfits(filename[ifile], exten=ihdu) - sxaddpar, nexthdr, 'WAT0_001', sxpar(hdr, 'WAT0_001') - sxaddpar, nexthdr, 'WAT1_001', sxpar(hdr, 'WAT1_001') - sxaddpar, nexthdr, 'CRVAL1', sxpar(hdr, 'CRVAL1') - sxaddpar, nexthdr, 'CD1_1', sxpar(hdr, 'CD1_1') - sxaddpar, nexthdr, 'CRPIX1', sxpar(hdr, 'CRPIX1') - sxaddpar, nexthdr, 'CTYPE1', sxpar(hdr, 'CTYPE1') - sxaddpar, nexthdr, 'DC-FLAG', sxpar(hdr, 'DC-FLAG') - - modfits, filename[ifile], 0, nexthdr, exten_no=ihdu - endfor - - endfor -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/fixmjd.pro b/pro/spec1d/fixmjd.pro deleted file mode 100644 index a954f44c1..000000000 --- a/pro/spec1d/fixmjd.pro +++ /dev/null @@ -1,60 +0,0 @@ -; Fix the MJD in the output files spPlate, spZbest, spZall to be consistent -; with the file name. These are the header keywords MJD in HDU#0 of all -; files, and the MJD element of the structures in the spZbest,spZall files. -;------------------------------------------------------------------------------ -pro fixmjd - - ;--------------------------------------------------------------------------- - ; Fix spPlate files - - filename = findfile('0*/spPlate*.fits', count=nfile) - - for ifile=0, nfile-1 do begin - print, 'Reading ' + filename[ifile] - - mjdname = long( strmid( fileandpath(filename[ifile]), 13, 5) ) - - hdr = headfits(filename[ifile]) - mjdhdr = sxpar(hdr, 'MJD') - - if (mjdname NE mjdhdr) then begin - print, 'MODIFY!' - sxaddpar, hdr, 'MJD', mjdname - djs_modfits, filename[ifile], 0, hdr - endif - endfor - - ;--------------------------------------------------------------------------- - ; Fix spZbest,spZall files - - filename = findfile('0*/spZbest*.fits', count=nfile) - - for ifile=0, nfile-1 do begin - print, 'Reading ' + filename[ifile] - - mjdname = long( strmid( fileandpath(filename[ifile]), 13, 5) ) - - hdr = headfits(filename[ifile]) - mjdhdr = sxpar(hdr, 'MJD') - - if (mjdname NE mjdhdr) then begin - print, 'MODIFY!' - ; Modify spZbest file - sxaddpar, hdr, 'MJD', mjdname - djs_modfits, filename[ifile], 0, hdr - zans = mrdfits(filename[ifile], 1, hdr1) - zans.mjd = mjdname - djs_modfits, filename[ifile], zans, exten_no=1 - - ; Modify spZall file - zallfile = repstr(filename[ifile], 'spZbest', 'spZall') - hdr = headfits(zallfile) - sxaddpar, hdr, 'MJD', mjdname - djs_modfits, zallfile, 0, hdr - zans = mrdfits(zallfile, 1, hdr1) - zans.mjd = mjdname - djs_modfits, zallfile, zans, exten_no=1 - endif - endfor -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/fixvdisp.pro b/pro/spec1d/fixvdisp.pro deleted file mode 100644 index 05cc416f9..000000000 --- a/pro/spec1d/fixvdisp.pro +++ /dev/null @@ -1,43 +0,0 @@ -; * Fix the MJD in the output files spZbest, spZall to be consistent -; with the file name. -; * Compute VDISP,VDISP_ERR for galaxies and add to structures. -;------------------------------------------------------------------------------ -pro fixvdisp - - ;--------------------------------------------------------------------------- - ; Fix spZbest files - - filename = findfile('0*/spZbest*.fits', count=nfile) - - for ifile=0, nfile-1 do begin - print, 'Reading ' + filename[ifile] - - zans = mrdfits(filename[ifile], 1, zhdr) - spawn, 'mv -f ' + filename[ifile] + ' ' + filename[ifile]+'.OLD' - - mjdname = long( strmid( fileandpath(filename[ifile]), 13, 5) ) - - ; Compute the velocity dispersions only if there are galaxies - ; in this file, and VDISP doesn't yet exist in the structure. - igal = where(strtrim(zans.class,2) EQ 'GALAXY') - qdo = (where(tag_names(zans) EQ 'VDISP'))[0] EQ -1 - if (qdo EQ 1 AND igal[0] NE -1) then begin - thisplatefile = repstr(filename[ifile], 'spZbest', 'spPlate') - objflux = mrdfits(thisplatefile, 0, objhdr) - objivar = mrdfits(thisplatefile, 1) - - vdispfit, objflux[*,igal], objivar[*,igal], hdr=objhdr, $ - zobj=zans[igal].z, sigma=sigma, sigerr=sigerr - - newans = struct_addtags(zans, $ - replicate(create_struct('VDISP', 0.0, 'VDISP_ERR', 0.0), 640)) - newans[igal].vdisp = sigma - newans[igal].vdisp_err = sigerr - - sxaddpar, zhdr, 'MJD', mjdname ; Force the MJD to be correct - mwrfits, 0, filename[ifile], zhdr, /create - mwrfits, newans, filename[ifile] - endif - endfor -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/fourier_difference.pro b/pro/spec1d/fourier_difference.pro deleted file mode 100644 index c25368098..000000000 --- a/pro/spec1d/fourier_difference.pro +++ /dev/null @@ -1,122 +0,0 @@ - -;+ -; NAME: -; fourier_difference -; -; PURPOSE: -; Perform a chi2 fit to the fourier difference between a single -; galaxy and a broadened stellar template to calculate velocity dispersion -; and uncertainty on velocity dispersion -; -; CALLING SEQUENCE: -; answers = fourier_difference(galfft, starfft, galvar0, starvar0, $ -; testsigma=, lowlimit=, highlimit=, $ -; deltachisq=deltachisq, /doplot) -; -; INPUTS: -; galfft - Fourier transform of galaxy -; starfft - Fourier transform of stellar template -; galvar0 - error in galaxy fft (0th element of galaxy error FFT) -; starvar0 - error in stellar fft (0th element of stellar error FFT) -; -; OPTIONAL KEYWORDS: -; testsigma - Array of sigma values to calculate chi2 -; lowlimit - lower boundary of chi2 sum (in knums units) -; highlimit - upper boundary of chi2 sum (in knums units) -; deltachisq - chi2 difference from minimum to set error on velocity dispersion -; doplot - Output plots to xwindow -; -; OUTPUTS: -; answers - Four element array with: -; [minchi2, minsigma, errsigma, bestalpha] -; bestalpha is the normalization constant between galaxy and star -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; -; EXAMPLES: -; -; BUGS: -; Currently, this is very slow, as we have to check 11 normalizations -; for each element in testsigma array -; -; Need to ensure that confidence level returned as errsigma is proper -; -; PROCEDURES CALLED: -; -; REVISION HISTORY: -; 25-Mar-2000 Written by S. Burles, FNAL -;- -;------------------------------------------------------------------------------ -function alpha_chisq, alpha, gal, star, galvar, starvar, br, minchi2 - - ntry = n_elements(alpha) - if (ntry EQ 0) then return, -1 - - chisq = fltarr(ntry) - for i=0,ntry -1 do begin - diff = gal - star*br*alpha[i] - rediff = float(diff*conj(diff)) - chisq[i] = total(rediff/(galvar + starvar*alpha[i]^2*br^2)) - endfor - - findchi2min, alpha, chisq, minchi2, bestalpha - - return, bestalpha -end - -;------------------------------------------------------------------------------ -function fourier_difference, galfft, starfft, galvar0, starvar0, $ - testsigma=testsigma, lowlimit = lowlimit, highlimit=highlimit, $ - deltachisq=deltachisq, doplot=doplot - - if (NOT keyword_set(lowlimit)) then lowlimit = 1.0/80.0 - if (NOT keyword_set(highlimit)) then highlimit = 1.0/2.2 - - if (size(galfft, /tname) EQ 'DOUBLE') then PI = !dpi $ - else PI = !pi - - knums = fft_wavenums(N_elements(galfft)) - - inside = where(abs(knums) GT lowlimit AND $ - abs(knums) LT highlimit, ninside) - - if (inside[0] EQ -1) then begin - print, 'No pixels in correct frequency range' - return, -1 - endif - - if (n_elements(testsigma) EQ 0) then testsigma = findgen(30)*0.2 - - ; This is a slow minimizer, stepping through 30 sigmas to - ; find best one. This method is the fft difference method - - nloop = n_elements(testsigma) - chi2diff = fltarr(nloop) - sigma = fltarr(nloop) - alpha = fltarr(nloop) - - alphatry = findgen(11)*0.2 - - for i=0,nloop-1 do begin - broad = exp(-(knums*testsigma[i] * 2.0 * PI)^2/2.0) - - alpha[i] = alpha_chisq(alphatry, galfft[inside], starfft[inside], $ - galvar0, starvar0, broad[inside], bestchisq) - chi2diff[i] = bestchisq - -; answer = amoeba(1.0e-2, function_name='alpha_chisq', $ -; p0 = [1.1], scale=[0.1], nmax = 20, function_value=areturn) - - endfor - - findchi2min, testsigma, chi2diff, minchi2, minsigma, errsigma, $ - deltachisq = deltachisq, doplot=doplot, npts=ninside - -; oplot, testsigma, alpha, ps=2 - minc = min(chi2diff, alphaplace) - bestalpha =alpha[alphaplace] - return, [minchi2, minsigma, errsigma, bestalpha] -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/fourier_quotient.pro b/pro/spec1d/fourier_quotient.pro deleted file mode 100644 index db7a85732..000000000 --- a/pro/spec1d/fourier_quotient.pro +++ /dev/null @@ -1,157 +0,0 @@ -;+ -; NAME: -; fourier_quotient -; -; PURPOSE: -; Perform a chi2 fit to the fourier quotient of a single -; galaxy and a broadened stellar template to calculate velocity dispersion -; and uncertainty on velocity dispersion -; -; CALLING SEQUENCE: -; answers = fourier_quotient(galfft, starfft, galvar0, starvar0, $ -; testsigma=, lowlimit=, highlimit=, $ -; deltachisq=, /doplot) -; -; INPUTS: -; galfft - Fourier transform of galaxy -; starfft - Fourier transform of stellar template -; galvar0 - error in galaxy fft (0th element of galaxy error FFT) -; starvar0 - error in stellar fft (0th element of stellar error FFT) -; -; OPTIONAL KEYWORDS: -; testsigma - Array of sigma values to calculate chi2 -; lowlimit - lower boundary of chi2 sum (in knums units) -; highlimit - upper boundary of chi2 sum (in knums units) -; deltachisq - chi2 difference from minimum to set error on velocity dispersion -; doplot - Output plots to xwindow -; -; OUTPUTS: -; answers - Four element array with: -; [minchi2, minsigma, errsigma, bestalpha] -; bestalpha is the normalization constant between galaxy and star -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; -; Same inputs and outputs as fourier_difference -; -; EXAMPLES: -; -; BUGS: -; -; Need to ensure that confidence level returned as errsigma is proper -; -; PROCEDURES CALLED: -; -; -; REVISION HISTORY: -; 25-Mar-2000 Written by S. Burles, FNAL -; 27-Jun-2000 Completely rewritten - Finkbeiner & SWAT team -;- -;------------------------------------------------------------------------------ -function fourier_quotient, galfft, starfft, galvar0, starvar0, $ - testsigma2=testsigma2, lowlimit = lowlimit, highlimit=highlimit, $ - deltachisq=deltachisq, doplot=doplot, broadarr=broadarr - -; testsigma2 is trial velocity dispersion^2 - - if (NOT keyword_set(lowlimit)) then lowlimit = 1.0/80.0 - if (NOT keyword_set(highlimit)) then highlimit = 1.0/5. - - - knums = fft_wavenums(n_elements(galfft)) - inside = where(abs(knums) GT lowlimit AND $ - abs(knums) LT highlimit, ninside) - - if (inside[0] EQ -1) then begin - print, 'No pixels in correct frequency range' - return, -1 - endif - - if (n_elements(testsigma2) EQ 0) then testsigma2 = findgen(30)*0.2 - - nloop = n_elements(testsigma2) - chi2 = fltarr(nloop) - sigma = fltarr(nloop) - alpha = fltarr(nloop) - - alphatry = findgen(21)*0.1 - - galnorm = djsig(float(galfft[inside]), sigrej=5) - starnorm = djsig(float(starfft[inside]), sigrej=5) - - q = galfft[inside]/starfft[inside]*starnorm/galnorm - -; Reject outliers on q (they can be quite large and drive the fit) - qs = exp(smooth(alog(q), 25, /edge)) - dif = float(alog((q/qs))) - djs_iterstat, dif, sigma=qsig, sigrej=5 - wbad = where(abs(dif) GT qsig*5, ct) - IF ct GT 0 THEN q[wbad] = qs[wbad] - -; Now reevaluate smoothed q - qs = exp(smooth(alog(q), 9, /edge)) -qs = median(q, 75) - - var = (galvar0/ float(galfft[inside]*conj(galfft[inside])) + $ - starvar0 / float(starfft[inside]*conj(starfft[inside])) * $ - float(q)^2) - -; Define broadarr (array of broadening functions) if not already defined - ones = 1.+fltarr(n_elements(q)) - IF NOT keyword_set(broadarr) THEN BEGIN - broadarr = dblarr(n_elements(inside), nloop) - for i=0,nloop-1 do begin - IF testsigma2[i] EQ 0 THEN broad = ones ELSE BEGIN - fsig = 1.d/(2.*!dpi)/sqrt(abs(testsigma2[i])) - broad = gauss_periodic(knums[inside], [1., 0., fsig], shft=1.) - IF testsigma2[i] LT 0 THEN broad = 1./broad - ENDELSE - broadarr[*, i] = broad - ENDFOR - - - ENDIF - - for i=0,nloop-1 do begin - broad = broadarr[*, i] - broad = broad/max(broad) - - alpha[i] = total(float(qs) * broad)/total(broad^2) -; nul residual (for just noise) - nul = sqrt(alpha[i]^2*broad^2+1) - res = broad*alpha[i]*starfft[inside]/starnorm-galfft[inside]/galnorm - -;IF i EQ 50 THEN stop - chi2[i] = total(abs(res)^2)/n_elements(inside)/mean(nul) - -; alpha[i] = total(float(q) * broad / var)/total(broad^2/var) - - endfor - -; Now do loop of negatives - -; now plot - - deltachisq = 1./n_elements(inside) - findchi2min, testsigma2, chi2, minchi2, minsigma, errsigma, $ - deltachisq = deltachisq, npts= ninside - - bestalpha = (interpol(alpha, testsigma2, minsigma))[0] - fsig= 1.d/(2.*!dpi)/sqrt(minsigma) - broad = gauss_periodic(knums[inside], [1., 0., fsig], shft=1.) - res = broad*starfft[inside]-galfft[inside] - -if NOT keyword_set(errsigma) then return, 0 ; ??? - errsigma = errsigma*sqrt(minchi2) - - IF keyword_set(doplot) THEN BEGIN - plot, knums[inside], qs, ps=3, yr=[-1, 2] - oplot, knums[inside], broad*bestalpha, ps=3 - plot, testsigma2, chi2, ps=-7 - oplot,[0,100],[1,1]*minchi2+deltachisq - ENDIF - - return, [minchi2, minsigma, errsigma, bestalpha] -end diff --git a/pro/spec1d/fundamental_line.pro b/pro/spec1d/fundamental_line.pro deleted file mode 100644 index bef4a8ae3..000000000 --- a/pro/spec1d/fundamental_line.pro +++ /dev/null @@ -1,76 +0,0 @@ -;+ -; NAME: -; fundamental_line -; PURPOSE: -; Crush the fundamental plane to a measly line. -; CALLING SEQUENCE: -; INPUTS: -; OPTIONAL INPUTS: -; KEYWORDS -; OUTPUTS: -; COMMENTS: -; BUGS: -; Using RA and Dec for matches rather than OBJID because tsObj files -; sometimes busted; search code for "HACK". -; EXAMPLES: -; PROCEDURES CALLED: -; REVISION HISTORY: -; 25-Jun-2000 Written by Hogg (IAS) -;- -pro fundamental_line, spname=spname,tsname=tsname,omegam=omegam,omegal=omegal -;; set cosmographic parameters - if NOT keyword_set(omegam) then omegam= 0.2 - if NOT keyword_set(omegal) then omegal= 0.0 -;; set aperture radii for photo radial profile outputs, in arcsec - photorad= [0.23,0.68,1.03,1.76,3.00,4.63,7.43,11.42,18.20,28.20,44.21, $ - 69.00,107.81,168.20,263.00] - maxnrad= 15 -;; set aperture radii for measurements, in arcsec at one Hubble length! - physrad= [1.0,2.0,3.0] -;; if no filenames set, use debugging defaults - if NOT keyword_set(spname) then $ - spname= '/data/spectro/2d_3c/0306/2dnew/spPlate-0306-51690.fits' - if NOT keyword_set(tsname) then $ - tsname= '/data/spectro/plates/tsObj-51637-306.fit' -;; read object IDs and the photo outputs (tsObj) -; HACK: USING RA AND DEC BECAUSE TSOBJ ARE SOMETIMES BUSTED - plugmap= mrdfits(spname,4) - ra= plugmap.ra - dec= plugmap.dec - tsobj= mrdfits(tsname,1) -;; load redshifts -;; begin loop over spectra - for spectrum=0,n_elements(plugmap)-1 do begin - z= 0.20 - help, z -;; compute aperture radii in arcsec, from redshifts and cosmography - rad= physrad/angdidis(z,omegam,omegal) -;; identify relevant entry in the tsObj file - obj= where((abs(tsobj.dec-dec[spectrum]) LT (1.0/3600.0)) AND $ - (abs(tsobj.ra-ra[spectrum]) LT $ - (1.0/(3600.0*cos(dec[spectrum]*!DPI/180.0)))),nobj) - obj= obj[0] - help, nobj -;; check that the object exists, the object is unique, and that -;; there are enough entries in the i and r-band radial profiles - if (nobj EQ 1) AND $ - (photorad[(tsobj.nprof)[3,obj]] GE max(rad)) AND $ - (photorad[(tsobj.nprof)[4,obj]] GE max(rad)) then begin -;; compute aperture luptitudes with interpolation - lupr= interpol((tsobj.profmean)[*,3,obj],photorad,rad) - lupi= interpol((tsobj.profmean)[*,4,obj],photorad,rad) -;; compute 2 (fairly) independent surface brightnesses - sb= 10.0^(0.4*(24.0-lupr)) - print, sb - area= !DPI*rad^2 - pseudosky= (sb[2]*area[2]-sb[1]*area[1])/(area[2]-area[1]) - help, pseudosky - sbinner= sb[0]-pseudosky - sbouter= (sb[1]*area[1]-sb[0]*area[0])/(area[1]-area[0])-pseudosky -help, sbinner,sbouter - endif -stop - endfor -;; done - return -end diff --git a/pro/spec1d/gama_to_sdss.pro b/pro/spec1d/gama_to_sdss.pro deleted file mode 100644 index eeb89b088..000000000 --- a/pro/spec1d/gama_to_sdss.pro +++ /dev/null @@ -1,160 +0,0 @@ -;+ -; NAME: -; gama_to_sdss -; -; PURPOSE: -; Convert a 2dF GAMA extracted spectra file to an SDSS spPlate format -; -; CALLING SEQUENCE: -; gama_to_sdss, filename, [ outfile= ] -; -; INPUTS: -; filename - 2dF GAMA spectra file, where HDU #0 is the flux [NPIX,NFIBER], -; HDU #1 is the variance, and HDU #2 contains catalog info -; -; OPTIONAL INPUTS: -; outfile - Output file name; default to spPlate-$PLATE-$MJD.fits where -; PLATE is derived from the GAMA field name and tile name, -; and MJD from the data of observation. GAMA field names -; 02, 09, 12, 15, 23 are mapped to plate numbers -; 1000, 2000, 3000, 4000, 5000 with the GAMA pointing number -; added to that. -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; The 2dF spectra are interpolated to the log-wavelength mapping -; of SDSS spectra using a B-spline to the flux and the variance arrays. -; -; EXAMPLES: -; Convert a 2dF GAMA file and run the SDSS redshifting code: -; IDL> gama_to_sdss,'SG09_Y3_012.fits',outfile='spPlate-2012-55246.fits' -; IDL> spreduce1d,'spPlate-2012-55246.fits' -; -; BUGS: -; The 2dF wavelength scale is assumed to be vacuum barycentric. -; -; PROCEDURES CALLED: -; -; REVISION HISTORY: -; 23-Feb-2011 Written by D. Schlegel, LBL -;- -;------------------------------------------------------------------------------ -pro gama_to_sdss, filename, outfile=outfile1 - - if (size(filename,/tname) NE 'STRING') then begin - splog, 'Must specify FILENAME' - return - endif - - plate = long(strmid(fileandpath(filename),8,3)) - fieldname = strmid(fileandpath(filename),2,2) - case fieldname of - '02' : plate += 1000L - '09' : plate += 2000L - '12' : plate += 3000L - '15' : plate += 4000L - '23' : plate += 5000L - else : plate += 9000L - endcase - - dradeg = 180d0 / !dpi - - plug1 = create_struct( $ - 'OBJID', lonarr(5), $ - 'RA', 0d0, $ - 'DEC', 0d0, $ - 'OBJTYPE', '', $ - 'SPECTROGRAPHID', 1L, $ - 'FIBERID', 0L ) - - flux = mrdfits(filename, 0, hdr) - if (keyword_set(flux) EQ 0) then $ - message, 'Unable to read file '+filename - var = mrdfits(filename, 1) - cat = mrdfits(filename, 2) - dims = size(flux,/dimens) - npix = dims[0] - nfiber = dims[1] - dwave = sxpar(hdr,'CDELT1') - wave = (dindgen(npix) - sxpar(hdr,'CRPIX1') - 1) * dwave $ - + sxpar(hdr,'CRVAL1') - - ; Construt the output file name - mjd = sxpar(hdr,'UTMJD') - if (keyword_set(outfile1)) then outfile = outfile1 $ - else outfile = 'spPlate-'+plate_to_string(plate)+'-' $ - +string(mjd,format='(i5.5)')+'.fits' - - ; Rebin to the SDSS spacing - dloglam = 1d-4 - newloglam = wavevector(alog10(min(wave)), alog10(max(wave)), binsz=dloglam) - newwave = 10^newloglam - nnew = n_elements(newloglam) - gmask = finite(flux) AND finite(var) AND var GT 0 - ibad = where(gmask EQ 0, nbad) - if (nbad GT 0) then flux[ibad] = 0 - if (nbad GT 0) then var[ibad] = 0 - - -; Simple linear interpolation - newbadmask = fltarr(nnew,nfiber) - for i=0, nfiber-1 do begin -; newflux[*,i] = rebin_spectrum(flux[*,i], wave, newwave) -; newvar[*,i] = rebin_spectrum(var[*,i], wave, newwave) - newbadmask[*,i] = rebin_spectrum(float(gmask[*,i] EQ 0), wave, newwave) - endfor - newmask = newbadmask EQ 0 -; newivar = newmask / (newvar + (newmask EQ 0)) - - newflux = fltarr(nnew,nfiber) - newivar = fltarr(nnew,nfiber) - loglam = alog10(wave) - for i=0, nfiber-1 do begin - combine1fiber, loglam, flux[*,i], 1./(var[*,i] + (var[*,i] EQ 0)), $ - newloglam=newloglam, newflux=newflux1, newivar=newivar1 - newflux[*,i] = newflux1 - newivar[*,i] = newivar1 - endfor - newflux = float(newflux) * newmask - newivar = float(newivar) * newmask - - ; Get the units right, making them per Ang although the spacing - ; is no longer uniform in wavelength - wratio = rebin((10d0^(dloglam) - 1d0) * newwave / dwave, [nnew, nfiber]) - newflux /= wratio - newivar *= wratio^2 - - andmask = lonarr(npix,nfiber) - ormask = lonarr(npix,nfiber) - dispmap = fltarr(npix,nfiber) + 1. - skyimg = fltarr(npix,nfiber) - plugmap = replicate(plug1, nfiber) - plugmap.ra = cat.ra * dradeg - plugmap.dec = cat.dec * dradeg - objtype = strmid(cat.name,0,1) - plugmap.objtype = (objtype EQ 'G' ? 'GALAXY' : '') $ - + (objtype EQ 'X' ? 'SKY' : '') $ - + (objtype EQ 'S' ? 'SPECTROPHOTO_STD' : '') $ - + (objtype EQ 'P' ? 'NA' : '') $ - + (objtype EQ 'F' ? 'NA' : '') ; fiducial/guide - plugmap.objtype = 'GALAXY' ; default value - plugmap.fiberid = lindgen(nfiber) + 1 - - sxaddpar, hdr, 'PLATEID', plate - sxaddpar, hdr, 'MJD', mjd - sxaddpar, hdr, 'COEFF0', newloglam[0] ; wavelength solution - sxaddpar, hdr, 'COEFF1', dloglam ; wavelength solution - mwrfits, newflux, outfile, hdr, /create - mwrfits, newivar, outfile - mwrfits, andmask, outfile - mwrfits, ormask, outfile - mwrfits, dispmap, outfile - mwrfits, plugmap, outfile - mwrfits, skyimg, outfile - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/gauss_periodic.pro b/pro/spec1d/gauss_periodic.pro deleted file mode 100644 index ecd3c69e2..000000000 --- a/pro/spec1d/gauss_periodic.pro +++ /dev/null @@ -1,21 +0,0 @@ -; DPF 29 Mar 2000 -; Compute gaussian with periodic boundary conditions -FUNCTION gauss_periodic, x, a, shft=shft - - amp = a[0] - cen = a[1] - sig = a[2] - - n = n_elements(x) - IF (NOT keyword_set(shft)) THEN $ ; assume linear - shft = ((max(x)-min(x))*n)/(n-1) - - g = 0 - FOR i=-3, 3 DO BEGIN - z = (x-cen-i*shft)/sig - ez = exp(-z^2/2.d) - g = g+amp*ez - ENDFOR - - return, g -END diff --git a/pro/spec1d/hogg_extinction.pro b/pro/spec1d/hogg_extinction.pro deleted file mode 100644 index 062735ea3..000000000 --- a/pro/spec1d/hogg_extinction.pro +++ /dev/null @@ -1,104 +0,0 @@ -;+ -; NAME: -; hogg_extinction -; PURPOSE: -; Plot some things having to do with extinction. -; COMMENTS: -; - Check out Burki et al, A&ASS, 112, 383 (1995). -; BUGS: -; - Many things hard-coded. -; - No comment header. -;- -pro hogg_extinction -prefix= 'hogg_extinction' -camname= ['b1','b2','r1','r2'] -ncam= 4 - -outfilename= prefix+'.fits' -if (NOT file_test(outfilename)) then begin - if (NOT keyword_set(efficiency)) then begin - savefile= 'plot_thru.ss' - if (NOT file_test(savefile)) then plot_thru - splog, 'restoring '+savefile - restore, savefile - endif - k0= dblarr(n_elements(loglam)) - k0_invvar= k0 - camamp= dblarr(n_elements(loglam),ncam) - camamp_invvar= camamp - - for ii=0L,n_elements(loglam)-1L do begin - if ((ii MOD 100) EQ 0) then splog, 1D1^(loglam[ii]) - vgood= where((efficiency[ii,*] GT 0.0) AND $ - (airmass GT 1.0) AND $ - (airmass LT 1.5),nvgood) - if (nvgood GT 100) then begin - lne= reform(alog(efficiency[ii,vgood]),nvgood) - aa= [[airmass[vgood]]] - thiscamlist= [-1] - for cc=0,ncam-1 do begin - thiscam= double(strmid(explist[vgood],0,2) EQ camname[cc]) - if (total(thiscam) GT 0.0) then begin - thiscamlist= [thiscamlist,cc] - aa= [[aa],[thiscam]] - endif - endfor - thiscamlist= thiscamlist[1:n_elements(thiscamlist)-1] - ww= replicate(1d0,nvgood) - hogg_iter_linfit, transpose(aa),lne,ww,xx,covar=covar -; plot, airmass[good[vgood]],lne,psym=1 -; oplot, !X.CRANGE,transpose([[1,1],[!X.CRANGE]]##xx),psym=0 - k0[ii]= xx[0] - k0_invvar[ii]= 1.0/covar[0,0] - for cc=0,n_elements(thiscamlist)-1 do begin - camamp[ii,thiscamlist[cc]]= xx[1+cc] - camamp_invvar[ii,thiscamlist[cc]]= 1.0/covar[1+cc,1+cc] - endfor - endif - endfor - splog, 'writing file '+outfilename - mwrfits, [[loglam],[k0],[k0_invvar],[camamp],[camamp_invvar]], $ - outfilename,/create -endif - -set_plot, 'ps' -device, filename= prefix+'.ps',/color -hogg_plot_defaults -readcol, '~/Longslit/calib/extinction/atm_trans_am1.0.dat', $ - longwave,longthru -longwave= alog10(longwave*1D4) -longthru= -2.5*alog10(longthru) -readcol, '~/primus/data/atmosphere.dat', $ - primwave,primthru -primwave= alog10(primwave) -filename= prefix+'.fits' -foo= mrdfits(filename) -good= where((foo[*,2] GT 0.0)) -hoggwave= foo[good,0] -hoggthru= foo[good,1] -hoggthru= -1.086*hoggthru -burkwave= hoggwave -burkthru= 0.17*(1D1^burkwave/4400.0)^(-4.05)+0.08*(1D1^burkwave/4400.0)^(-1.39) -xrange= alog10([3500,9500]) -plot, hoggwave,hoggthru,psym=10,/nodata, $ - xrange=xrange,xtitle= 'log!d10!n wavelength (A)', $ - yrange=[0.01,1.5],ytitle= 'extinction (mag/airmass)', $ - title='all cameras',/ylog -oplot, longwave,longthru,color=djs_icolor('grey'),psym=10 -oplot, primwave,primthru,color=djs_icolor('grey'),psym=10 -oplot, burkwave,burkthru,color=djs_icolor('grey') -oplot, hoggwave,hoggthru,psym=10 -; oplot, 1D1^foo[good,0],foo[good,1]+2.0/sqrt(foo[good,2]),psym=10 -; oplot, 1D1^foo[good,0],foo[good,1]-2.0/sqrt(foo[good,2]),psym=10 -for cc=0,ncam-1 do begin - plot, 1D1^foo[good,0],foo[good,cc+3],psym=10, $ - xrange=xrange,xtitle= 'wavelength (A)', $ - yrange=[-5,0],ytitle= 'ln(throughput)', $ - title=camname[cc] -; oplot, 1D1^foo[good,0],foo[good,3]+2.0/sqrt(foo[good,4]),psym=10 -; oplot, 1D1^foo[good,0],foo[good,3]-2.0/sqrt(foo[good,4]),psym=10 -endfor -device,/close - -return -end diff --git a/pro/spec1d/kurucz_fitsfile.pro b/pro/spec1d/kurucz_fitsfile.pro deleted file mode 100644 index c97a36c4a..000000000 --- a/pro/spec1d/kurucz_fitsfile.pro +++ /dev/null @@ -1,88 +0,0 @@ -;+ -; NAME: -; kurucz_fitsfile -; -; PURPOSE: -; Generate a single FITS file from a list of ASCII-formatted Kurucz models -; -; CALLING SEQUENCE: -; kurucz_fitsfile, [ fileprefix, outfile ] -; -; INPUTS: -; -; OPTIONAL INPUTS: -; fileprefix - Use all files in the current directory matching this string; -; default to 'a*.spc' -; outfile - Name of output FIST file; default 'kurucz_stds_raw_v5.fits' -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; The input ASCII files were generated by Christy Tremonti using -; Kurucz' code. The file name is assumed to encode the stellar -; parameters of metallicity (FEH), effective temperature (TEFF) -; and gravity (G). For example, the file 'am05k2_5000_4.0.spc' -; is interpreted to have FEH=-0.5, TEFF=5000, G=4.0. I don't know -; what the "k2" in the filename means. -; -; HDU#0 of the output file has the fluxes. -; HDU#1 of the output file is a FITS binary table with the stellar parameters. -; -; EXAMPLES: -; -; PROCEDURES CALLED: -; mwrfits -; sxpaddpar -; -; REVISION HISTORY: -; 18-Jan-2003 Written by D. Schlegel, Princeton -;- -;------------------------------------------------------------------------------ -pro kurucz_fitsfile, fileprefix, outfile - - if (NOT keyword_set(fileprefix)) then fileprefix = 'a*.spc' - if (NOT keyword_set(outfile)) then $ - outfile = 'kurucz_stds_raw_v5.fits' - - files = findfile(fileprefix, count=nfile) - if (nfile EQ 0) then begin - print, 'No input files found' - return - endif - - kindx = replicate( create_struct( $ - 'MODEL', '', $ - 'FEH' , 0., $ - 'TEFF' , 0., $ - 'G' , 0., $ - 'MAG' , fltarr(5)), nfile) - - for ifile=0L, nfile-1 do begin - print, 'Reading file ', ifile+1, ' of ', nfile - readcol, files[ifile], wave, flux, format='(D,F)' - if (ifile EQ 0) then begin - npix = n_elements(flux) - allflux = fltarr(npix, nfile) - endif - allflux[*,ifile] = flux - kindx[ifile].model = files[ifile] - kindx[ifile].feh = 0.1 * float(strmid(files[ifile],2,2)) $ - * (strmid(files[ifile],1,1) EQ 'm' ? -1 : 1) - kindx[ifile].teff = float(strmid(files[ifile],7,4)) - kindx[ifile].g = float(strmid(files[ifile],12,3)) - endfor - - hdr = [''] - sxaddpar, hdr, 'CRVAL1', double(wave[0]) - sxaddpar, hdr, 'CD1_1', (wave[npix-1] - wave[0]) / (npix - 1) - sxaddpar, hdr, 'CRPIX1', 1L - sxaddpar, hdr, 'CTYPE1', 'LINEAR', ' Air wavelegnths [Ang]' - - mwrfits, allflux, outfile, hdr, /create - mwrfits, kindx, outfile - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/lambda_match.pro b/pro/spec1d/lambda_match.pro deleted file mode 100644 index 077aeff35..000000000 --- a/pro/spec1d/lambda_match.pro +++ /dev/null @@ -1,42 +0,0 @@ -; 27-Jun-2000 D. Finkbeiner -; Force spectra on to same wavelength system -; 28-Jun-2000 DF vectorized to handle arr being a 2-d array - -PRO lambda_match, refwave, objwave, arr - -; right now assume objwave is just 1d array - - nref = n_elements(refwave) ; number of samples in reference array - nwave = (size(objwave))[1] - sarr = (size(arr))[1] ; size of spectra - narr = n_elements(arr)/sarr ; number of spectra - - IF sarr NE nwave THEN BEGIN - print, 'LAMBDA_MATCH: Objwave and arr must have same dimensions' - help, objwave, arr - return - ENDIF - - rat = mean(objwave[*, 0]/refwave) - shf = round(alog10(rat)*10000) - - IF shf GT 0 THEN BEGIN - print, 'LAMBDA_MATCH ', shf, ' pixel shift' - arr = [fltarr(shf, narr), arr] - ENDIF ELSE BEGIN - IF shf LT 0 THEN BEGIN - arr = arr[(-shf):sarr-1, *] - ENDIF - ENDELSE - - npad = nref-nwave-shf ; how many more to add at end - - IF npad GT 0 THEN BEGIN - arr = [arr, fltarr(npad, narr)] - ENDIF - IF npad LT 0 THEN BEGIN ; remove if too many - arr = arr[0:nref-1, *] - ENDIF - - return -END diff --git a/pro/spec1d/make_regress1d.pro b/pro/spec1d/make_regress1d.pro deleted file mode 100644 index 0b35eb928..000000000 --- a/pro/spec1d/make_regress1d.pro +++ /dev/null @@ -1,143 +0,0 @@ -;+ -; NAME: -; make_regress1d -; -; PURPOSE: -; Generate a regression table from spectro 1-D from Chicago-1D outputs files. -; -; CALLING SEQUENCE: -; make_regress1d, [listfile, indir=, outfile= ] -; -; INPUTS: -; -; OPTIONAL INPUTS: -; listfile - List with plate number and MJD (one such pair per line) -; for plates to use. Default to the file -; $IDLSPEC2D_DIR/etc/regress1d_all.plates, which lists plates -; 300 to 309. -; indir - Input directory for the Chicago-1D output files, of the -; format 'spDiag1d-'+mjd+'-'+plate+'.par, i.e. -; 'spDiag1d-51666-0300.par'; default to '.' -; outfile - Name of output file; default to -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; An ASCII file with minimal object information and redshifts is tabulated -; from the Chicago-1D outputs. -; -; The Chicago outputs are changed in some special cases that are assumed -; to be missing or wrong. -; -; EXAMPLES: -; -; BUGS: -; -; PROCEDURES CALLED: -; djs_readcol -; -; INTERNAL SUPPORT ROUTINES: -; create_zregress() -; -; DATA FILES: -; $IDLSPEC2D_DIR/etc/regress1d_all.plates -; -; REVISION HISTORY: -; 27-Jun-2000 Written by D. Schlegel, Princeton -;------------------------------------------------------------------------------ -function create_zregress, num - - ftemp = create_struct( name='ZREGRESS', $ - 'PLATE', 0L, $ - 'MJD', 0L, $ - 'FIBERID', 0L, $ - 'Z', 0.D, $ - 'CLASS', '', $ - 'PRIMTARGET', 0L, $ - 'SECTARGET', 0L, $ - 'COMMENTS', '' ) - - zregress = replicate(ftemp, num) - - return, zregress -end - -;------------------------------------------------------------------------------ -pro make_regress1d, listfile, indir=indir, outfile=outfile - - if (NOT keyword_set(listfile)) then $ - listfile = filepath('regress1d_all.plates', $ - root_dir=getenv('IDLSPEC2D_DIR'), subdirectory='etc') - if (NOT keyword_set(indir)) then indir = '.' - if (NOT keyword_set(outfile)) then outfile = 'regress1d_all.dat' - - djs_readcol, listfile, platevec, mjdvec - - nplate = n_elements(platevec) - - zregress = create_zregress(nplate * 640) - - ii = 0 - for iplate=0, nplate-1 do begin - ; Read the Yanny param file for Chicago-1D outputs for this plate - mjdstr = string(mjdvec[iplate],format='(i5.5)') - platestr = plate_to_string(platevec[iplate]) - filename = filepath('spDiag1d-' + mjdstr + '-' + platestr + '.par', $ - root_dir=indir) - yanny_read, filename, tt, /quick - cdiag = *tt[0] - yanny_free, tt - - ; Change the CLASS for 'SPEC_HIZ_QSO' to 'SPEC_QSO' - kk = where(cdiag.class EQ 'SPEC_HIZ_QSO') - if (kk[0] NE -1) then cdiag[kk].class = 'SPEC_QSO' - - for ifiber=0, 639 do begin - zregress[ii].plate = platevec[iplate] - zregress[ii].mjd = mjdvec[iplate] - zregress[ii].fiberid = ifiber + 1 - jj = (where(cdiag.fiberid EQ ifiber+1))[0] - if (jj NE -1) then begin - zregress[ii].primtarget = cdiag[jj].primtarget - zregress[ii].sectarget = cdiag[jj].sectarget - zregress[ii].z = cdiag[jj].zfinal - zregress[ii].class = ' ' + strmid(cdiag[jj].class,5) + ' ' - endif else begin - zregress[ii].class = ' MISSING ' - endelse - - ; Find out if this is a sky fiber, and change CLASS to 'SKY' - if (zregress[ii].sectarget AND 2^4) then begin - zregress[ii].z = 0 - zregress[ii].class = ' SKY' - endif - - ; Declare as 'UNKNOWN' the following cases: - ; Any object with redshift less than -1000 - ; A QSO with a redshift less than 1000 - if (zregress[ii].z LT -1000./3e5 OR $ - (zregress[ii].z LT 1000./3d5 AND $ - strtrim(zregress[ii].class,2) EQ 'QSO')) then begin - zregress[ii].z = 0 - zregress[ii].class = ' UNKNOWN' - endif - - ii = ii + 1 - endfor - endfor - - get_lun, olun - openw, olun, outfile - printf, olun, $ - '#PLATE MJD FIBER Z CLASS PRIMTARGET SECTARGET COMMENTS' - for jj=0, n_elements(zregress)-1 do begin - printf, olun, zregress[jj], format='(i6,i5,i4,f10.5,a10,i12,i12,a)' - endfor - close, olun - free_lun, olun - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/matchplugmap.pro b/pro/spec1d/matchplugmap.pro deleted file mode 100644 index 2ec0027d2..000000000 --- a/pro/spec1d/matchplugmap.pro +++ /dev/null @@ -1,29 +0,0 @@ -;------------------------------------------------------------------------------ -pro matchplugmap, plug1, plug2, indx1, indx2, mindist=mindist - - if (NOT keyword_set(mindist)) then mindist = 1.0 / 3600 - - nfiber1 = n_elements(plug1) - nfiber2 = n_elements(plug2) - indx1 = lindgen(nfiber1) - indx2 = lonarr(nfiber1) - 1L - - for ifiber=0, nfiber1-1 do begin - adist = djs_diff_angle(plug1[ifiber].ra, plug1[ifiber].dec, $ - plug2.ra, plug2.dec) - dmin = min(adist, imin) - if (dmin LE mindist) then indx2[ifiber] = imin - endfor - - igood = where(indx2 NE -1L) - if (igood[0] NE -1) then begin - indx1 = indx1[igood] - indx2 = indx2[igood] - endif else begin - indx1 = -1L - indx2 = -1L - endelse - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/multisynthspec.pro b/pro/spec1d/multisynthspec.pro deleted file mode 100644 index 69f2477ad..000000000 --- a/pro/spec1d/multisynthspec.pro +++ /dev/null @@ -1,165 +0,0 @@ -;+ -; NAME: -; multisynthspec -; -; PURPOSE: -; Construct synthetic spectrum from eigen-templates. -; -; CALLING SEQUENCE: -; synflux = multisynthspec(zans, [ loglam=, hdr=, eigendir= ]) -; -; INPUTS: -; zans - Structure(s) with redshift-fit information (from SPREDUCE1D). -; -; OPTIONAL KEYWORDS: -; loglam - Log-10 wavelengths at which to synthesize the spectrum; -; this can either be a single vector if all spectra have -; the same wavelength mapping, or an array of [NPIX,NOBJ]. -; hdr - If specified, then use this header to construct LOGLAM. -; Either LOGLAM or HDR must be specified. -; eigendir - Directory for EIGENFILE; default to $IDLSPEC2D/templates. -; -; OUTPUTS: -; synflux - Synthetic spectra -; -; COMMENTS: -; This routine is meant to be faster than SYNTHSPEC when generating -; many spectra that share the same templates. This works by generating -; the B-spline coefficients only once per template, then evaluating -; these B-splines at the exact wavelengths of each output spectrum. -; This will NOT be exactly the same as the results from SYNTHSPEC, -; because the individual eigen-spectra are spline-shifted before adding -; them up, rather than adding them up then spline-shifting (as SYNTHSPEC -; does). -; -; EXAMPLES: -; -; BUGS: -; -; DATA FILES: -; $IDLSPEC2D_DIR/etc/TEMPLATEFILES -; -; PROCEDURES CALLED: -; combine1fiber -; concat_dir() -; poly_array() -; readfits() -; sxpar() -; -; REVISION HISTORY: -; 20-Aug-2000 Written by D. Schlegel, Princeton -;------------------------------------------------------------------------------ -function multisynthspec, zans, loglam=objloglam, hdr=hdr, eigendir=eigendir - - nobj = n_elements(zans) - if (n_elements(eigendir) EQ 0) then $ - eigendir = concat_dir(getenv('IDLSPEC2D_DIR'), 'templates') - - ;---------- - ; Determine the wavelength mapping for the object spectra, - ; which are the same for all of them. - - if (keyword_set(objloglam)) then begin - dims = size(objloglam, /dimens) - naxis1 = dims[0] - objdloglam = objloglam[1] - objloglam[0] - endif else if (keyword_set(hdr)) then begin - naxis1 = sxpar(hdr, 'NAXIS1') - objloglam0 = sxpar(hdr, 'COEFF0') - objdloglam = sxpar(hdr, 'COEFF1') - objloglam = objloglam0 + dindgen(naxis1) * objdloglam - endif else begin - print, 'Either LOGLAM or HDR must be specified' - return, -1 - endelse - - ;---------- - ; Construct the output spectra - - newflux = fltarr(naxis1, nobj) - - ;---------- - ; Get name(s) of template file, ignoring any that are blank strings - - tfiles = strtrim(zans.tfile,2) - alltfile = tfiles[ uniq(tfiles, sort(tfiles)) ] - if (N_elements(alltfile) EQ 1 AND alltfile[0] EQ '') then return, newflux - alltfile = alltfile[where(alltfile NE '')] - - ;--------------------------------------------------------------------------- - ; Loop through each template file - - for ifile=0, n_elements(alltfile)-1 do begin - - ; List of objects that use this template file - iobj = where(tfiles EQ alltfile[ifile], niobj) - itheta = lonarr(niobj) - - ;---------- - ; Read the template file. - ; (Assume that the wavelength binning is the same as for the objects - ; in log-wavelength.) - - starflux = readfits(djs_filepath(alltfile[ifile], root_dir=eigendir), $ - shdr) - starloglam0 = sxpar(shdr, 'COEFF0') - stardloglam = sxpar(shdr, 'COEFF1') - - ndim = size(starflux, /n_dimen) - dims = size(starflux, /dimens) - npixstar = dims[0] - starloglam = starloglam0 + dindgen(npixstar) * stardloglam - if (ndim EQ 1) then nstar = 1 $ - else nstar = dims[1] - - ;---------- - ; Add as many polynomial terms as we might need for this template file - - maxnpoly = max([ zans[iobj].npoly ]) - if (keyword_set(maxnpoly)) then $ - starflux = [ [starflux], [poly_array(npixstar,maxnpoly)] ] - - ;---------- - ; Loop through each column in this template file - - for istar=0, nstar+maxnpoly-1 do begin - - ; Identify the coefficients for each object using this template/column - if (istar LT nstar) then begin - for i=0, niobj-1 do $ - itheta[i] = (where(zans[iobj[i]].tcolumn EQ istar))[0] - endif else begin - for i=0, niobj-1 do $ - if (istar-nstar LT zans[iobj[i]].npoly) then $ - itheta[i] = (where(zans[iobj[i]].tcolumn EQ -1))[0] + istar-nstar $ - else $ - itheta[i] = -1 - endelse - - indx = where(itheta NE -1, nthis) - if (indx[0] NE -1) then begin - thisobj = iobj[indx] - thistheta = fltarr(nthis) - for i=0, nthis-1 do $ - thistheta[i] = zans[thisobj[i]].theta[itheta[indx[i]]] - if (size(objloglam, /n_dimen) EQ 1) then $ - thisloglam = objloglam # replicate(1,nthis) $ - else $ - thisloglam = objloglam[*,thisobj] - for i=0, nthis-1 do $ - thisloglam[*,i] = thisloglam[*,i] - alog10(1+zans[thisobj[i]].z) - - combine1fiber, starloglam, starflux[*,istar], $ - newloglam=thisloglam, newflux=addflux - addflux = reform(addflux, naxis1, nthis) - - for i=0, nthis-1 do $ - newflux[*,thisobj[i]] = newflux[*,thisobj[i]] $ - + addflux[*,i] * thistheta[i] - endif - endfor - endfor - - return, newflux -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/pca_cvstar.pro b/pro/spec1d/pca_cvstar.pro deleted file mode 100644 index 9a27ac9ee..000000000 --- a/pro/spec1d/pca_cvstar.pro +++ /dev/null @@ -1,137 +0,0 @@ -;------------------------------------------------------------------------------ -pro pca_cvstar - - wavemin = 0. - wavemax = 0. - snmax = 100 - niter = 10 - nkeep = 3 - minuse = 3 - - get_juldate, jd - mjdstr = string(long(jd-2400000L), format='(i5)') - outfile = 'spEigenCVstar-' + mjdstr + '.fits' - plotfile = 'spEigenCVstar-' + mjdstr + '.ps' - - dfpsplot, plotfile, /color, /landscape - colorvec = ['default', 'red', 'green', 'blue'] - - ;---------- - ; Read the input spectra - - eigenfile = filepath('eigeninput_cvstar.dat', $ - root_dir=getenv('IDLSPEC2D_DIR'), subdirectory='templates') - djs_readcol, eigenfile, skip=2, plate, mjd, fiber, zfit, subclass, $ - format='(L,L,L,D,A)' -; indx = where(strmid(subclass,0,1) EQ 'CV') -; plate = plate[indx] -; mjd = mjd[indx] -; fiber = fiber[indx] -; zfit = zfit[indx] -; subclass = subclass[indx] - - readspec, plate, fiber, mjd=mjd, flux=objflux, invvar=objivar, $ - andmask=andmask, ormask=ormask, loglam=objloglam, plugmap=plugmap - - ;---------- - ; Insist that all of the requested spectra exist - - imissing = where(plugmap.fiberid EQ 0, nmissing) - if (nmissing GT 0) then begin - for i=0, nmissing-1 do $ - print, 'Missing plate=', slist[imissing[i]].plate, $ - ' mjd=', slist[imissing[i]].mjd, $ - ' fiber=', slist[imissing[i]].fiberid - message, string(nmissing) + ' missing object(s)' - endif - - ;---------- - ; Do not fit where the spectrum may be dominated by sky-sub residuals. - - objivar = skymask(objivar, andmask, ormask) -andmask = 0 ; Free memory -ormask = 0 ; Free memory - - nobj = (size(objflux, /dimens))[1] - objdloglam = objloglam[1] - objloglam[0] - - if (keyword_set(snmax)) then begin - ifix = where(objflux^2 * objivar GT snmax^2) - if (ifix[0] NE -1) then objivar[ifix] = (snmax/objflux[ifix])^2 - endif - - ;---------- - ; Do PCA solution - - pcaflux = pca_solve(objflux, objivar, objloglam, zfit, $ - wavemin=wavemin, wavemax=wavemax, $ - niter=niter, nkeep=nkeep, newloglam=newloglam, eigenval=eigenval, $ - usemask=usemask, acoeff=acoeff) - pcaflux = float(pcaflux) - -;readspec, plate, fiber, mjd=mjd, plugmap=plugmap -;readspec, plate, fiber, mjd=mjd, zans=zans -;aratio = acoeff[1,*]/acoeff[0,*] -;ug = plugmap.mag[0] - plugmap.mag[1] -;gr = plugmap.mag[1] - plugmap.mag[2] -;ri = plugmap.mag[2] - plugmap.mag[3] -;iz = plugmap.mag[3] - plugmap.mag[4] -;plot, gr, aratio, ps=4 -;stop - - ;---------- - ; Fill in bad data with a running median of good data - - qgood = usemask GE minuse - igood = where(qgood, ngood) - ibad = where(qgood EQ 0, nbad) - medflux = 0 * pcaflux - if (nbad GT 0) then begin - for i=0, nkeep-1 do begin - medflux[igood,i] = $ - djs_median(pcaflux[igood,i], width=51, boundary='nearest') - medflux[*,i] = djs_maskinterp(medflux[*,i], qgood EQ 0, /const) - endfor - pcaflux[ibad,*] = medflux[ibad,*] - endif - - ;---------- - ; Make plots - - djs_plot, 10^newloglam, pcaflux[*,0], $ - xrange=minmax(10^newloglam), yrange=minmax(pcaflux), /xstyle, $ - color=colorvec[0], $ - xtitle='Wavelength [Ang]', ytitle='Flux [arbitrary units]', $ - title='CV Stars: Eigenspectra' - for i=1, nkeep-1 do begin - djs_oplot, 10^newloglam, pcaflux[*,i], $ - color=colorvec[i MOD n_elements(colorvec)] - endfor - - aratio10 = acoeff[1,*] / acoeff[0,*] - aratio20 = acoeff[2,*] / acoeff[0,*] - djs_plot, aratio10, aratio20, /nodata, $ - xtitle='Eigenvalue Ratio (a_1/a_0)', $ - ytitle='Eigenvalue Ratio (a_2/a_0)', $ - title='CV Stars: Eigenvalue Ratios' - for j=0, n_elements(aratio10)-1 do $ - djs_xyouts, aratio10[j], aratio20[j], align=0.5, $ - string(plate_to_string(plate[j]), fiber[j], format='(a,"-",i3)'), $ - color=colorvec[j MOD n_elements(colorvec)] - - ;---------- - ; Write output file - - sxaddpar, hdr, 'OBJECT', 'STAR_CV' - sxaddpar, hdr, 'COEFF0', newloglam[0] - sxaddpar, hdr, 'COEFF1', objdloglam - for i=0, n_elements(eigenval)-1 do $ - sxaddpar, hdr, 'EIGEN'+strtrim(string(i),1), eigenval[i] - - mwrfits, pcaflux, outfile, hdr, /create - - dfpsclose - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/pca_cvstar_boss.pro b/pro/spec1d/pca_cvstar_boss.pro deleted file mode 100644 index b3a029e99..000000000 --- a/pro/spec1d/pca_cvstar_boss.pro +++ /dev/null @@ -1,143 +0,0 @@ -; Written by Schlegel (????) -; Tweaked by Bolton (utah2011jun) for BOSS CV sample. - -;------------------------------------------------------------------------------ -pro pca_cvstar_boss - - wavemin = 0. - wavemax = 0. - snmax = 100 - niter = 10 - nkeep = 3 - minuse = 3 - - get_juldate, jd - mjdstr = string(long(jd-2400000L), format='(i5)') - outfile = 'spEigenCVstar-' + mjdstr + '.fits' - plotfile = 'spEigenCVstar-' + mjdstr + '.ps' - - dfpsplot, plotfile, /color, /landscape - colorvec = ['default', 'red', 'green', 'blue'] - - ;---------- - ; Read the input spectra - -; if (not keyword_set(eigenfile)) then $ - eigenfile = filepath('eigeninput_cvstar_boss.dat', $ - root_dir=getenv('IDLSPEC2D_DIR'), subdirectory='templates') -; djs_readcol, eigenfile, skip=2, plate, mjd, fiber, zfit, subclass, $ -; format='(L,L,L,D,A)' - readcol, eigenfile, plate, mjd, fiber, zfit, subclass, $ - format='L,L,L,F,A', comment='#' -; indx = where(strmid(subclass,0,1) EQ 'CV') -; plate = plate[indx] -; mjd = mjd[indx] -; fiber = fiber[indx] -; zfit = zfit[indx] -; subclass = subclass[indx] - - readspec, plate, fiber, mjd=mjd, flux=objflux, invvar=objivar, $ - andmask=andmask, ormask=ormask, loglam=objloglam, plugmap=plugmap - - ;---------- - ; Insist that all of the requested spectra exist - - imissing = where(plugmap.fiberid EQ 0, nmissing) - if (nmissing GT 0) then begin - for i=0, nmissing-1 do $ - print, 'Missing plate=', slist[imissing[i]].plate, $ - ' mjd=', slist[imissing[i]].mjd, $ - ' fiber=', slist[imissing[i]].fiberid - message, string(nmissing) + ' missing object(s)' - endif - - ;---------- - ; Do not fit where the spectrum may be dominated by sky-sub residuals. - - objivar = skymask(objivar, andmask, ormask) -andmask = 0 ; Free memory -ormask = 0 ; Free memory - - nobj = (size(objflux, /dimens))[1] - objdloglam = objloglam[1] - objloglam[0] - - if (keyword_set(snmax)) then begin - ifix = where(objflux^2 * objivar GT snmax^2) - if (ifix[0] NE -1) then objivar[ifix] = (snmax/objflux[ifix])^2 - endif - - ;---------- - ; Do PCA solution - - pcaflux = pca_solve(objflux, objivar, objloglam, zfit, $ - wavemin=wavemin, wavemax=wavemax, $ - niter=niter, nkeep=nkeep, newloglam=newloglam, eigenval=eigenval, $ - usemask=usemask, acoeff=acoeff) - pcaflux = float(pcaflux) - -;readspec, plate, fiber, mjd=mjd, plugmap=plugmap -;readspec, plate, fiber, mjd=mjd, zans=zans -;aratio = acoeff[1,*]/acoeff[0,*] -;ug = plugmap.mag[0] - plugmap.mag[1] -;gr = plugmap.mag[1] - plugmap.mag[2] -;ri = plugmap.mag[2] - plugmap.mag[3] -;iz = plugmap.mag[3] - plugmap.mag[4] -;plot, gr, aratio, ps=4 -;stop - - ;---------- - ; Fill in bad data with a running median of good data - - qgood = usemask GE minuse - igood = where(qgood, ngood) - ibad = where(qgood EQ 0, nbad) - medflux = 0 * pcaflux - if (nbad GT 0) then begin - for i=0, nkeep-1 do begin - medflux[igood,i] = $ - djs_median(pcaflux[igood,i], width=51, boundary='nearest') - medflux[*,i] = djs_maskinterp(medflux[*,i], qgood EQ 0, /const) - endfor - pcaflux[ibad,*] = medflux[ibad,*] - endif - - ;---------- - ; Make plots - - djs_plot, 10^newloglam, pcaflux[*,0], $ - xrange=minmax(10^newloglam), yrange=minmax(pcaflux), /xstyle, $ - color=colorvec[0], $ - xtitle='Wavelength [Ang]', ytitle='Flux [arbitrary units]', $ - title='CV Stars: Eigenspectra' - for i=1, nkeep-1 do begin - djs_oplot, 10^newloglam, pcaflux[*,i], $ - color=colorvec[i MOD n_elements(colorvec)] - endfor - - aratio10 = acoeff[1,*] / acoeff[0,*] - aratio20 = acoeff[2,*] / acoeff[0,*] - djs_plot, aratio10, aratio20, /nodata, $ - xtitle='Eigenvalue Ratio (a_1/a_0)', $ - ytitle='Eigenvalue Ratio (a_2/a_0)', $ - title='CV Stars: Eigenvalue Ratios' - for j=0, n_elements(aratio10)-1 do $ - djs_xyouts, aratio10[j], aratio20[j], align=0.5, $ - string(plate_to_string(plate[j]), fiber[j], format='(a,"-",i3)'), $ - color=colorvec[j MOD n_elements(colorvec)] - - ;---------- - ; Write output file - - sxaddpar, hdr, 'OBJECT', 'STAR_CV' - sxaddpar, hdr, 'COEFF0', newloglam[0] - sxaddpar, hdr, 'COEFF1', objdloglam - for i=0, n_elements(eigenval)-1 do $ - sxaddpar, hdr, 'EIGEN'+strtrim(string(i),1), eigenval[i] - - mwrfits, pcaflux, outfile, hdr, /create - - dfpsclose - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/pca_elodie.pro b/pro/spec1d/pca_elodie.pro deleted file mode 100644 index a9e07cc24..000000000 --- a/pro/spec1d/pca_elodie.pro +++ /dev/null @@ -1,88 +0,0 @@ -;------------------------------------------------------------------------------ -; Generate the eigen-vectors from the Elodie eschelle data, -; for use in fitting velocity dispersions. -;------------------------------------------------------------------------------ -pro pca_elodie - - outfile = 'spEigenElodie.fits' - - ;---------- - ; Read all the Elodie spectra - - allfiles = elodie_filelist(minwave=minwave) - nfile = n_elements(allfiles) - - t0 = systime(1) - for ifile=0, nfile-1 do begin - splog, 'Reading file ', ifile+1, ' of ', nfile - thisflux = read_elodie(allfiles[ifile], loglam=loglam, minloglamclip=alog10(minwave)) - if (NOT keyword_set(allflux)) then allflux = thisflux $ - else allflux = [[allflux],[thisflux]] - endfor - npix = n_elements(loglam) - splog, 'Time to read all files = ', systime(1) - t0 - - ;---------- - ; Trim wavelengths to those covered by the majority of the objects - - fracgpix = total(allflux NE 0, 2) / nfile - igood = where(fracgpix GT 0.95) - i1 = igood[0] - i2 = (reverse(igood))[0] - loglam = loglam[i1:i2] - allflux = allflux[i1:i2,*] - - ;---------- - ; Interpolate over bad data, of which there is very little - - allflux = djs_maskinterp(allflux, allflux EQ 0, /const, iaxis=0) - - ;---------- - ; Compute something like the equiv. width for H-alpha, so that we - ; can reject active stars - - lwav = 6564.6 - i1 = where(loglam GT alog10(lwav-5) AND loglam LT alog10(lwav+5), num1) - i2 = where(loglam GT alog10(lwav-10) AND loglam LT alog10(lwav+10), num2) - - sum1 = total(allflux[i1,*],1) / num1 - sum2 = total(allflux[i2,*],1) / num2 - fline = sum1 / (sum2 + (sum2 LE 0)) - - ;---------- - ; Compute the wavelength coverage for each object - - fraccov = total(allflux NE 0,1) / npix - - ;---------- - ; Select objects for PCA - - istar = where(fline LT 1.1 AND fraccov GT 0.95, nstar) - splog, 'Selecting ', nstar, ' objects of ', nfile - t0 = systime(1) - pres = pcomp(transpose(allflux[*,istar]), eigenval=eigenval, /double) - splog, 'Time to compute PCA = ', systime(1) - t0 - pres = transpose(pres) - - ;---------- - ; Keep only the most significant eigenvectors - - ikeep = where(eigenval GT 0.01) - eigenval = eigenval[ikeep] - pres = float( pres[*,ikeep] ) - - ;---------- - ; Write output file - - dloglam = loglam[1] - loglam[0] - sxaddpar, hdr, 'OBJECT', 'GALAXY' - sxaddpar, hdr, 'COEFF0', loglam[0] - sxaddpar, hdr, 'COEFF1', dloglam - for i=0, n_elements(eigenval)-1 do $ - sxaddpar, hdr, 'EIGEN'+strtrim(string(i),1), eigenval[i] - - mwrfits, pres, outfile, hdr, /create - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/pca_gal.pro b/pro/spec1d/pca_gal.pro deleted file mode 100644 index cc4dce290..000000000 --- a/pro/spec1d/pca_gal.pro +++ /dev/null @@ -1,228 +0,0 @@ -;+ -; NAME: -; pca_gal -; -; PURPOSE: -; Wrapper on pca_solve.pro -; -; CALLING SEQUENCE: -; pca_gal, [inputfile=inputfile], [wavemin=wavemin], [wavemax=wavemax], $ -; [binsz=binsz], [niter=niter], [savefile=savefile], [/flux] -; -; INPUTS: -; None -; -; OPTIONAL INPUTS: -; inputfile - File containing plate, mjd, fiber, redshift. If not -; specified, it will read eigeninput_gal.dat from -; $IDLSPEC2D_DIR/templates -; wavemin - minimum wavelength, default 1850 A. -; wavemax - maximum wavelength, default 9300 A. -; binsz - Override the bin size of the final wavelength mapping. -; flux - Plot redshift-shifted spectra. -; niter - Number of iterations to pass to pca_solve, default 10. -; savefile - Save the input spectra to a named file for debugging. -; -; OUTPUTS: -; None, but creates the files spEigenGal-MJD.fits & spEigenGal-MJD.ps -; -; DATA FILES: -; $IDLSPEC2D_DIR/templates/eigeninput_gal.dat -; -; PROCEDURES CALLED: -; get_juldate -; dfpsplot -; djs_readcol -; readspec -; skymask() -; wavevector() -; pca_solve() -; djs_median() -; djs_plot -; djs_oplot -; djs_xyouts -; sxaddpar -; mwrfits -; dfpsclose -; -; REVISION HISTORY: -; Written a long time ago. -; Updated for BOSS inputs by B. A. Weaver, NYU -; -; VERSION: -; $Id: pca_gal.pro 50005 2011-05-26 18:56:10Z weaver $ -; -;------------------------------------------------------------------------------ -PRO pca_gal, inputfile=inputfile, wavemin=wavemin, wavemax=wavemax, $ - binsz=binsz, flux=flux, niter=niter, savefile=savefile - ; - ; Define initial parameters - ; - IF ~KEYWORD_SET(inputfile) THEN BEGIN - inputfile = FILEPATH('eigeninput_gal.dat', $ - ROOT_DIR=GETENV('IDLSPEC2D_DIR'), SUBDIRECTORY='templates') - ENDIF - IF ~KEYWORD_SET(wavemin) THEN wavemin = 1850. ; Good to z=1 - IF ~KEYWORD_SET(wavemax) THEN wavemax = 10000. - snmax = 100 - IF ~KEYWORD_SET(niter) THEN niter = 10 - nkeep = 4 - minuse = 10 - ; - ; Name the output files - ; - get_juldate, jd - mjdstr = STRING(LONG(jd-2400000.5), FORMAT='(I5)') - outfile = 'spEigenGal-' + mjdstr + '.fits' - plotfile = 'spEigenGal-' + mjdstr + '.ps' - dfpsplot, plotfile, /color, /landscape - colorvec = ['default', 'red', 'green', 'blue', 'magenta', 'cyan'] - ; - ; Read the input spectra - ; - djs_readcol, inputfile, plate, mjd, fiber, zfit, format='(L,L,L,D)' - readspec, plate, fiber, mjd=mjd, flux=objflux, invvar=objivar, $ - andmask=andmask, ormask=ormask, plugmap=plugmap, loglam=objloglam - ; - ; Insist that all of the requested spectra exist - ; - imissing = WHERE(plugmap.fiberid EQ 0, nmissing) - IF (nmissing GT 0) THEN BEGIN - FOR i=0, nmissing-1 DO $ - PRINT, 'Missing plate=', plate[imissing[i]], $ - ' mjd=', mjd[imissing[i]], $ - ' fiber=', fiber[imissing[i]] - MESSAGE, STRING(nmissing) + ' missing object(s)' - ENDIF - ; - ; Do not fit where the spectrum may be dominated by sky-sub residuals. - ; - objivar = skymask(objivar, andmask, ormask) - andmask = 0 ; Free memory - ormask = 0 ; Free memory - nobj = (SIZE(objflux, /dimens))[1] - IF (KEYWORD_SET(snmax)) THEN BEGIN - ifix = WHERE(objflux^2 * objivar GT snmax^2) - IF (ifix[0] NE -1) THEN objivar[ifix] = (snmax/objflux[ifix])^2 - ENDIF - ; - ; Set the new wavelength mapping here... - ; If the binsz keyword is not set, then bin size is determined from the - ; first spectrum returned by readspec. This is fine in the case where - ; all the spectra have the same bin size (though their starting - ; wavelengths may differ). However, this may not be a safe - ; assumption in the future. - ; - IF ~KEYWORD_SET(binsz) THEN binsz = objloglam[1,0] - objloglam[0,0] - newloglam = wavevector(ALOG10(wavemin), ALOG10(wavemax), binsz=binsz) - ; - ; Do PCA solution - ; - pcaflux = pca_solve(objflux, objivar, objloglam, zfit, $ - wavemin=wavemin, wavemax=wavemax, $ - niter=niter, nkeep=nkeep, newloglam=newloglam, eigenval=eigenval, $ - acoeff=acoeff, usemask=usemask, newflux=newflux, newivar=newivar) - pcaflux = FLOAT(pcaflux) - ; - ; Fill in bad data with a running median of good data - ; - qgood = usemask GE minuse - igood = WHERE(qgood, ngood) - ibad = WHERE(qgood EQ 0, nbad) - medflux = 0 * pcaflux - IF (nbad GT 0) THEN BEGIN - FOR i=0, nkeep-1 DO BEGIN - medflux[igood,i] = $ - djs_median(pcaflux[igood,i], width=51, boundary='nearest') - medflux[*,i] = djs_maskinterp(medflux[*,i], qgood EQ 0, /const) - ENDFOR - pcaflux[ibad,*] = medflux[ibad,*] - ENDIF - ; - ; Dump input fluxes to a file for debugging purposes. - ; - IF KEYWORD_SET(savefile) THEN $ - SAVE, newloglam, newflux, newivar, FILENAME=savefile - ; - ; Make plots - ; - nspectra = (SIZE(newflux,/DIMENSIONS))[1] - IF KEYWORD_SET(flux) THEN BEGIN - nfluxes = 30L - separation = 5.0 - nplots = nspectra/nfluxes - IF nspectra MOD nfluxes GT 0 THEN nplots = nplots + 1L - FOR k = 0L, nplots-1L DO BEGIN - istart = k*nfluxes - iend = ((istart + nfluxes) < nspectra) - 1L - djs_plot, 10^newloglam, newflux[*,istart], $ - xrange=minmax(10^newloglam), $ - yrange=[MIN(newflux[*,istart]),MAX(newflux[*,iend])+separation*(nfluxes-1)], /xstyle, $ - color=colorvec[0], $ - xtitle='Wavelength [\AA]', $ - ytitle='Flux [10^{-17} erg cm^{-2} s^{-1} \AA^{-1}] + Constant', $ - title=STRING(istart+1L,iend+1L,FORMAT='("Galaxies: Input Spectra ",I4,"-",I4)') ;, /xlog - FOR i=istart+1L, iend DO $ - djs_oplot, 10^newloglam, newflux[*,i]+separation*(i MOD nfluxes), $ - color=colorvec[i MOD N_ELEMENTS(colorvec)] - ENDFOR - ENDIF - djs_plot, 10^newloglam, TOTAL(newivar EQ 0,2)/nspectra, $ - color=colorvec[0], xtitle='Wavelength [\AA]', $ - ytitle='Fraction of spectra with missing data', $ - title='Missing Data' - djs_plot, 10^newloglam, pcaflux[*,0], $ - xrange=minmax(10^newloglam), yrange=minmax(pcaflux), /xstyle, $ - color=colorvec[0], $ - xtitle='Wavelength [\AA]', ytitle='Flux [arbitrary units]', $ - title='Galaxies: Eigenspectra' ;, /xlog - FOR i=1, nkeep-1 DO $ - djs_oplot, 10^newloglam, pcaflux[*,i], $ - color=colorvec[i MOD N_ELEMENTS(colorvec)] - aratio10 = acoeff[1,*] / acoeff[0,*] - aratio20 = acoeff[2,*] / acoeff[0,*] - aratio30 = acoeff[3,*] / acoeff[0,*] - djs_plot, aratio10, aratio20, /nodata, $ - xtitle='Eigenvalue Ratio (a_1/a_0)', $ - ytitle='Eigenvalue Ratio (a_2/a_0)', $ - title='Galaxies: Eigenvalue Ratios' - FOR j=0, N_ELEMENTS(aratio10)-1 DO $ - djs_xyouts, aratio10[j], aratio20[j], align=0.5, $ - STRING(plate[j], fiber[j], FORMAT='(I4.4,"-",I4.4)'), $ - color=colorvec[j MOD N_ELEMENTS(colorvec)],charsize=0.5 - djs_plot, aratio20, aratio30, /nodata, $ - xtitle='Eigenvalue Ratio (a_2/a_0)', $ - ytitle='Eigenvalue Ratio (a_3/a_0)', $ - title='Galaxies: Eigenvalue Ratios' - FOR j=0, N_ELEMENTS(aratio20)-1 DO $ - djs_xyouts, aratio20[j], aratio30[j], align=0.5, $ - STRING(plate[j], fiber[j], FORMAT='(I4.4,"-",I4.4)'), $ - color=colorvec[j MOD N_ELEMENTS(colorvec)],charsize=0.5 - ; - ; Write output file - ; - sxaddpar, hdr, 'OBJECT', 'GALAXY' - sxaddpar, hdr, 'COEFF0', newloglam[0] - sxaddpar, hdr, 'COEFF1', binsz - sxaddpar, hdr, 'IDLUTILS', idlutils_version(), 'Version of idlutils' - sxaddpar, hdr, 'SPEC2D', idlspec2d_version(), 'Version of idlspec2d' - sxaddpar, hdr, 'RUN2D', GETENV('RUN2D'), 'Version of 2d reduction' - sxaddpar, hdr, 'RUN1D', GETENV('RUN1D'), 'Version of 1d reduction' - FOR i=0, N_ELEMENTS(eigenval)-1 DO $ - sxaddpar, hdr, 'EIGEN'+STRTRIM(STRING(i),1), eigenval[i] - mwrfits, pcaflux, outfile, hdr, /create - ; - ; Create a table of inputs - ; - sxaddpar, hdr2, 'FILENAME', inputfile - inputs0 = {inputs, plate:0L, mjd:0L, fiber:0L, redshift:0.0D} - inputs = REPLICATE(inputs0,N_ELEMENTS(plate)) - inputs.plate = plate - inputs.mjd = mjd - inputs.fiber = fiber - inputs.redshift = zfit - mwrfits, inputs, outfile, hdr2 - dfpsclose - RETURN -END -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/pca_lrgtest.pro b/pro/spec1d/pca_lrgtest.pro deleted file mode 100644 index 1d8d7c942..000000000 --- a/pro/spec1d/pca_lrgtest.pro +++ /dev/null @@ -1,168 +0,0 @@ -;+ -; NAME: -; pca_lrgtest -; -; PURPOSE: -; Build PCA templates for LRGs within a specified redshift range. -; -; CALLING SEQUENCE: -; pca_lrgtest, [ platenums, nkeep=, zrange= ] -; -; INPUTS: -; -; OPTIONAL INPUTS: -; platenums - Plate number(s) from which to select the LRGs; if not set, -; then select all 'main' survey plates with QSURVEY=1 -; in the plate list file (which are required to be unique -; tiles with good quality observations) -; nkeep - Number of PCA templates to keep (and to use for -; noisy data replacement); default to 2. -; zrange - 2-element array with redshift range for fitting; if not set, -; then call this routine iteratively with redshift ranges -; starting at [0,0.05] and extending to [0.45,0.50], spaced -; every 0.05 in redshift. -; -; OUTPUT: -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; The output are written to FITS files named "spLRG_xxx_yyy.fits" where -; "xxx" is the starting redshift times 100, and "yyy" is the ending -; redshift times ten. -; -; EXAMPLES: -; Create a set of PCA templates for LRGs from plates 400 through 409: -; IDL> pca_lrgtest, 400+lindgen(10) -; -; BUGS: -; -; PROCEDURES CALLED: -; djs_maskinterp() -; djs_median -; mwrfits -; pca_solve() -; platelist -; readspec -; splog -; skymask() -; sxaddhist -; sxaddpar -; wavevector() -; -; REVISION HISTORY: -; 23-Mar-2001 Written by David Schlegel, Princeton. -; 17-Sep-2003 Modified for N. Padmanabhan -;- -;------------------------------------------------------------------------------ -pro pca_lrgtest, platenums, nkeep=nkeep, zrange=zrange - - if (NOT keyword_set(zrange)) then begin - for z1=0.0, 0.46, 0.05 do begin - pca_lrgtest, platenums, nkeep=nkeep, zrange=[z1,z1+0.05] - endfor - return - endif - - ;---------- - ; Set defaults - - wavemin = 2500. - wavemax = 9200. - snmax = 100 - niter = 10 - if (NOT keyword_set(nkeep)) then nkeep = 2 - minuse = 5 - outfile = string(long(zrange[0]*100), long(zrange[1]*100), $ - format='("spLRG_", i3.3, "_", i3.3, ".fits")') - - ;---------- - ; Get the list of plates if not specified - - if (NOT keyword_set(platenums)) then begin - platelist, plist=plist - platenums = plist[ where(plist.qsurvey) ].plate - endif - - ;---------- - ; Read the input spectra. - ; Select good spectra of galaxies that were actually targetted - ; as LRGs in the main survey. - - readspec, platenums, zans=zans, plug=plug - indx = where(((plug.primtarget AND 2LL^5) NE 0 $ - OR (plug.primtarget AND 2LL^26) NE 0) $ - AND strtrim(zans.class) EQ 'GALAXY' $ - AND zans.zwarning EQ 0 $ - AND zans.z GE zrange[0] AND zans.z LE zrange[1]) - zans = zans[indx] - splog, 'Number of objects = ', n_elements(zans) - - readspec, zans.plate, zans.fiberid, mjd=zans.mjd, $ - flux=objflux, invvar=objivar, $ - andmask=andmask, ormask=ormask, plugmap=plugmap, loglam=objloglam - - ;---------- - ; Do not fit where the spectrum may be dominated by sky-sub residuals. - - objivar = skymask(objivar, andmask, ormask) -andmask = 0 ; Free memory -ormask = 0 ; Free memory - - nobj = (size(objflux, /dimens))[1] - objdloglam = objloglam[1] - objloglam[0] - - if (keyword_set(snmax)) then begin - ifix = where(objflux^2 * objivar GT snmax^2) - if (ifix[0] NE -1) then objivar[ifix] = (snmax/objflux[ifix])^2 - endif - - ;---------- - ; Set the new wavelength mapping here... - - newloglam = wavevector(alog10(wavemin), alog10(wavemax), binsz=objdloglam) - - ;---------- - ; Do PCA solution - - pcaflux = pca_solve(objflux, objivar, objloglam, zans.z, $ - niter=niter, nkeep=nkeep, newloglam=newloglam, eigenval=eigenval, $ - usemask=usemask) - pcaflux = float(pcaflux) - - ;---------- - ; Fill in bad data with a running median of good data - - qgood = usemask GE minuse - igood = where(qgood, ngood) - ibad = where(qgood EQ 0, nbad) - medflux = 0 * pcaflux - if (nbad GT 0) then begin - for i=0, nkeep-1 do begin - medflux[igood,i] = $ - djs_median(pcaflux[igood,i], width=51, boundary='nearest') - medflux[*,i] = djs_maskinterp(medflux[*,i], qgood EQ 0, /const) - endfor - pcaflux[ibad,*] = medflux[ibad,*] - endif - - ;---------- - ; Write output file - - sxaddpar, hdr, 'OBJECT', $ - 'LRG z=' + string(zrange[0]) + ' ' + string(zrange[1]) - sxaddpar, hdr, 'COEFF0', newloglam[0] - sxaddpar, hdr, 'COEFF1', objdloglam - sxaddpar, hdr, 'NPLATE', n_elements(platenums), ' Number of plates used' - sxaddpar, hdr, 'NGALAXY', n_elements(zans), ' Number of galaxies in fit' - for i=0, n_elements(eigenval)-1 do $ - sxaddpar, hdr, 'EIGEN'+strtrim(string(i),1), eigenval[i] - for i=0, n_elements(platenums)-1 do $ - sxaddhist, 'Plate ' + strtrim(platenums[i],2) + ' with ' $ - + strtrim(long(total(zans.plate EQ platenums[i])),2) + ' galaxies', hdr - - mwrfits, pcaflux, outfile, hdr, /create - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/pca_qso.pro b/pro/spec1d/pca_qso.pro deleted file mode 100644 index e170a63c2..000000000 --- a/pro/spec1d/pca_qso.pro +++ /dev/null @@ -1,277 +0,0 @@ -;+ -; NAME: -; pca_qso -; -; PURPOSE: -; Wrapper on pca_solve.pro -; -; CALLING SEQUENCE: -; pca_qso, [inputfile=inputfile], [wavemin=wavemin], [wavemax=wavemax], $ -; [binsz=binsz], [niter=niter], [savefile=savefile], [/flux], $ -; [/allatonce] -; -; INPUTS: -; None -; -; OPTIONAL INPUTS: -; inputfile - File containing plate, mjd, fiber, redshift. If not -; specified, it will read eigeninput_qso.dat from -; $IDLSPEC2D_DIR/templates -; wavemin - minimum wavelength, default 460 A. -; wavemax - maximum wavelength, default 10000 A. -; binsz - Override the bin size of the final wavelength mapping. -; flux - Plot redshift-shifted spectra. -; niter - Number of iterations to pass to pca_solve, default 200. -; savefile - Save the input spectra to a named file for debugging. -; allatonce - Solve for components all at once instead of one at a time. -; -; OUTPUTS: -; None, but creates the files spEigenQSO-MJD.fits & spEigenQSO-MJD.ps -; -; DATA FILES: -; $IDLSPEC2D_DIR/templates/eigeninput_qso.dat -; -; PROCEDURES CALLED: -; get_juldate -; dfpsplot -; djs_readcol -; readspec -; skymask() -; wavevector() -; pca_solve() -; computechi2() -; djs_median() -; djs_plot -; djs_oplot -; djs_xyouts -; sxaddpar -; mwrfits -; dfpsclose -; -; REVISION HISTORY: -; Written a long time ago. -; Updated for BOSS inputs by B. A. Weaver, NYU -; -; VERSION: -; $Id: pca_qso.pro 50208 2012-01-24 15:47:52Z weaver $ -; -;------------------------------------------------------------------------------ -PRO pca_qso, inputfile=inputfile, wavemin=wavemin, wavemax=wavemax, $ - binsz=binsz, flux=flux, niter=niter, savefile=savefile - ; - ; Define initial parameters - ; - IF ~KEYWORD_SET(inputfile) THEN BEGIN - inputfile = FILEPATH('eigeninput_qso.dat', $ - ROOT_DIR=GETENV('IDLSPEC2D_DIR'), SUBDIRECTORY='templates') - ENDIF - IF ~KEYWORD_SET(wavemin) THEN wavemin = 460. ; Good to z=1 - IF ~KEYWORD_SET(wavemax) THEN wavemax = 10000. - snmax = 100 - IF ~KEYWORD_SET(niter) THEN niter = 200 - nkeep = 4 - minuse = 3 - ; - ; Name the output files - ; - get_juldate, jd - mjdstr = STRING(LONG(jd-2400000.5), FORMAT='(I5)') - outfile = 'spEigenQSO-' + mjdstr + '.fits' - plotfile = 'spEigenQSO-' + mjdstr + '.ps' - dfpsplot, plotfile, /color, /landscape - colorvec = ['default', 'red', 'green', 'blue', 'magenta', 'cyan'] - ; - ; Read the input spectra - ; - djs_readcol, inputfile, plate, mjd, fiber, zfit, format='(L,L,L,D)' - readspec, plate, fiber, mjd=mjd, flux=objflux, invvar=objivar, $ - andmask=andmask, ormask=ormask, plugmap=plugmap, loglam=objloglam - ; - ; Insist that all of the requested spectra exist - ; - imissing = WHERE(plugmap.fiberid EQ 0, nmissing) - IF (nmissing GT 0) THEN BEGIN - FOR i=0, nmissing-1 DO $ - PRINT, 'Missing plate=', plate[imissing[i]], $ - ' mjd=', mjd[imissing[i]], $ - ' fiber=', fiber[imissing[i]] - MESSAGE, STRING(nmissing) + ' missing object(s)' - ENDIF - ; - ; Do not fit where the spectrum may be dominated by sky-sub residuals. - ; - objivar = skymask(objivar, andmask, ormask) - andmask = 0 ; Free memory - ormask = 0 ; Free memory - nobj = (SIZE(objflux, /dimens))[1] - IF (KEYWORD_SET(snmax)) THEN BEGIN - ifix = WHERE(objflux^2 * objivar GT snmax^2) - IF (ifix[0] NE -1) THEN objivar[ifix] = (snmax/objflux[ifix])^2 - ENDIF - ; - ; Set the new wavelength mapping here... - ; If the binsz keyword is not set, then bin size is determined from the - ; first spectrum returned by readspec. This is fine in the case where - ; all the spectra have the same bin size (though their starting - ; wavelengths may differ). However, this may not be a safe - ; assumption in the future. - ; - IF ~KEYWORD_SET(binsz) THEN binsz = objloglam[1,0] - objloglam[0,0] - newloglam = wavevector(ALOG10(wavemin), ALOG10(wavemax), binsz=binsz) - IF KEYWORD_SET(allatonce) THEN BEGIN - ; - ; Do PCA solution -- all components at once. - ; - ; The following would solve for all the eigen-vectors at once. - ; This can result in an unphysical 1st eigencomponent, probably - ; because each spectrum only covers a small range of rest wavelength. - ; - pcaflux = pca_solve(objflux, objivar, objloglam, zfit, $ - wavemin=wavemin, wavemax=wavemax, $ - niter=niter, nkeep=nkeep, newloglam=newloglam, eigenval=eigenval) - ENDIF ELSE BEGIN - ; - ; Do PCA solution -- but one component at a time. - ; - FOR ikeep=0, nkeep-1 DO BEGIN - splog, 'Solving for eigencomponent #', ikeep+1, ' of ', nkeep - pcaflux1 = pca_solve(objflux, objivar, objloglam, zfit, $ - wavemin=wavemin, wavemax=wavemax, $ - niter=niter, nkeep=1, newloglam=newloglam, $ - newflux=newflux, newivar=newivar, eigenval=eigenval1, $ - acoeff=acoeff1, usemask=usemask) - IF (ikeep EQ 0) THEN BEGIN - saveflux = newflux - pcaflux = pcaflux1 - eigenval = eigenval1 - ; acoeff = acoeff1 - ENDIF ELSE BEGIN - pcaflux = [[pcaflux], [pcaflux1]] - ; - ; These eigenvalues are not normalized properly!!!??? - ; - eigenval = [eigenval, eigenval1] - ; acoeff = [acoeff, acoeff1] - ENDELSE - ; - ; Re-solve for the coefficients using all PCA components so far - ; - acoeff = FLTARR(ikeep+1,nobj) - FOR iobj=0, nobj-1 DO BEGIN - junk = computechi2(saveflux[*,iobj], SQRT(newivar[*,iobj]), $ - pcaflux, acoeff=theta) - acoeff[*,iobj] = theta - ENDFOR - ; - ; Prevent re-binning of spectra on subsequent calls to PCA_SOLVE(). - ; - objloglam = 0 - ; objflux = newflux - acoeff1 ## pcaflux1 - objflux = saveflux - acoeff ## pcaflux - objivar = newivar - ENDFOR - ENDELSE - pcaflux = FLOAT(pcaflux) - ; - ; Fill in bad data with a running median of good data - ; - qgood = usemask GE minuse - igood = WHERE(qgood, ngood) - ibad = WHERE(qgood EQ 0, nbad) - medflux = 0 * pcaflux - IF (nbad GT 0) THEN BEGIN - FOR i=0, nkeep-1 DO BEGIN - medflux[igood,i] = $ - djs_median(pcaflux[igood,i], width=51, boundary='nearest') - medflux[*,i] = djs_maskinterp(medflux[*,i], qgood EQ 0, /const) - ENDFOR - pcaflux[ibad,*] = medflux[ibad,*] - ENDIF - ; - ; Dump input fluxes to a file for debugging purposes. - ; - IF KEYWORD_SET(savefile) THEN $ - SAVE, newloglam, newflux, newivar, FILENAME=savefile - ; - ; Make plots - ; - nspectra = (SIZE(newflux,/DIMENSIONS))[1] - IF KEYWORD_SET(flux) THEN BEGIN - nfluxes = 30L - separation = 5.0 - nplots = nspectra/nfluxes - IF nspectra MOD nfluxes GT 0 THEN nplots = nplots + 1L - FOR k = 0L, nplots-1L DO BEGIN - istart = k*nfluxes - iend = ((istart + nfluxes) < nspectra) - 1L - djs_plot, 10^newloglam, newflux[*,istart], $ - xrange=minmax(10^newloglam), $ - yrange=[MIN(newflux[*,istart]),MAX(newflux[*,iend])+separation*(nfluxes-1)], /xstyle, $ - color=colorvec[0], $ - xtitle='Wavelength [\AA]', $ - ytitle='Flux [10^{-17} erg cm^{-2} s^{-1} \AA^{-1}] + Constant', $ - title=STRING(istart+1L,iend+1L,FORMAT='("QSOs: Input Spectra ",I4,"-",I4)') ;, /xlog - FOR i=istart+1L, iend DO $ - djs_oplot, 10^newloglam, newflux[*,i]+separation*(i MOD nfluxes), $ - color=colorvec[i MOD N_ELEMENTS(colorvec)] - ENDFOR - ENDIF - djs_plot, 10^newloglam, TOTAL(newivar EQ 0,2)/nspectra, $ - color=colorvec[0], xtitle='Wavelength [\AA]', $ - ytitle='Fraction of spectra with missing data', $ - title='Missing Data' - djs_plot, 10^newloglam, pcaflux[*,0], $ - xrange=minmax(10^newloglam), yrange=minmax(pcaflux), /xstyle, $ - color=colorvec[0], $ - xtitle='Wavelength [\AA]', ytitle='Flux [arbitrary units]', $ - title='QSOs: Eigenspectra' ;, /xlog - FOR i=1, nkeep-1 DO $ - djs_oplot, 10^newloglam, pcaflux[*,i], $ - color=colorvec[i MOD N_ELEMENTS(colorvec)] - aratio10 = acoeff[1,*] / acoeff[0,*] - aratio20 = acoeff[2,*] / acoeff[0,*] - aratio30 = acoeff[3,*] / acoeff[0,*] - djs_plot, aratio10, aratio20, /nodata, $ - xtitle='Eigenvalue Ratio (a_1/a_0)', $ - ytitle='Eigenvalue Ratio (a_2/a_0)', $ - title='QSOs: Eigenvalue Ratios' - FOR j=0, N_ELEMENTS(aratio10)-1 DO $ - djs_xyouts, aratio10[j], aratio20[j], align=0.5, $ - STRING(plate[j], fiber[j], FORMAT='(I4.4,"-",I4.4)'), $ - color=colorvec[j MOD N_ELEMENTS(colorvec)],charsize=0.5 - djs_plot, aratio20, aratio30, /nodata, $ - xtitle='Eigenvalue Ratio (a_2/a_0)', $ - ytitle='Eigenvalue Ratio (a_3/a_0)', $ - title='QSOs: Eigenvalue Ratios' - FOR j=0, N_ELEMENTS(aratio20)-1 DO $ - djs_xyouts, aratio20[j], aratio30[j], align=0.5, $ - STRING(plate[j], fiber[j], FORMAT='(I4.4,"-",I4.4)'), $ - color=colorvec[j MOD N_ELEMENTS(colorvec)],charsize=0.5 - ; - ; Write output file - ; - sxaddpar, hdr, 'OBJECT', 'QSO' - sxaddpar, hdr, 'COEFF0', newloglam[0] - sxaddpar, hdr, 'COEFF1', binsz - sxaddpar, hdr, 'IDLUTILS', idlutils_version(), 'Version of idlutils' - sxaddpar, hdr, 'SPEC2D', idlspec2d_version(), 'Version of idlspec2d' - sxaddpar, hdr, 'RUN2D', GETENV('RUN2D'), 'Version of 2d reduction' - sxaddpar, hdr, 'RUN1D', GETENV('RUN1D'), 'Version of 1d reduction' - FOR i=0, N_ELEMENTS(eigenval)-1 DO $ - sxaddpar, hdr, 'EIGEN'+STRTRIM(STRING(i),1), eigenval[i] - mwrfits, pcaflux, outfile, hdr, /create - ; - ; Create a table of inputs - ; - sxaddpar, hdr2, 'FILENAME', inputfile - inputs0 = {inputs, plate:0L, mjd:0L, fiber:0L, redshift:0.0D} - inputs = REPLICATE(inputs0,N_ELEMENTS(plate)) - inputs.plate = plate - inputs.mjd = mjd - inputs.fiber = fiber - inputs.redshift = zfit - mwrfits, inputs, outfile, hdr2 - dfpsclose - RETURN -END -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/pca_solve.pro b/pro/spec1d/pca_solve.pro deleted file mode 100644 index 01ca06ef5..000000000 --- a/pro/spec1d/pca_solve.pro +++ /dev/null @@ -1,266 +0,0 @@ -;+ -; NAME: -; pca_solve -; -; PURPOSE: -; Iteratively find PCA solution for noisy or gappy spectra. -; -; CALLING SEQUENCE: -; res = pca_solve( objflux, objivar, objloglam, [ zfit, $ -; wavemin=, wavemax=, newloglam=, $ -; maxiter=, niter=, nkeep=, nreturn=, eigenval=, acoeff=, outmask=, $ -; usemask=, /quiet, _EXTRA= ] ) -; -; INPUTS: -; objflux - Object fluxes [NPIX,NSPEC] -; objivar - Object inverse variances [NPIX,NSPEC] -; -; OPTIONAL INPUTS: -; objloglam - Object wavelengths in log10(Angstroms) -; [NPIX] if the same wavelength mapping for all spectra, -; or [NPIX,NSPEC] if the wavelength mappings are different. -; zfit - Redshifts of each input spectrum [NSPEC]; if set, then -; each input spectrum is de-redshifted to z=0. -; wavemin - Minimum wavelength to use in PCA solution, in Angstroms; -; default to the minimum (de-redshifted) input wavelength. -; wavemax - Maximum wavelength to use in PCA solution, in Angstroms -; default to the minimum (de-redshifted) input wavelength. -; newloglam - PCA wavelength sampling in log-10(Angstroms) [NNEWPIX] -; maxiter - Number of rejection iterations; default to 0 (no rejection) -; niter - Number of PCA iterations; default to 10. -; nkeep - Number of PCA components to keep in each iteration -; and use in replacing noisy or missing data; default to 3. -; nreturn - Number of PCA components to return; default to the same as -; NKEEP. -; quiet - Minimal output to splog -; _EXTRA - Keywords for DJS_REJECT(). -; -; OUTPUTS: -; res - PCA spectra in rest-frame [NNEWPIX,NKEEP] -; -; OPTIONAL OUTPUTS: -; newloglam - PCA wavelength sampling in log-10(Angstroms) [NNEWPIX] -; newflux - Rebinned OBJFLUX on the wavelength-mapping NEWLOGLAM. -; newivar - Rebinned OBJIVAR on the wavelength-mapping NEWLOGLAM. -; eigenval - Eigenvalue for each output eigenspectra [NRETURN] -; acoeff - PCA coefficients [NRETURN,NOBJ] -; outmask - Output mask from DJS_REJECT() [NNEWPIX,NOBJ] -; usemask - Number of unmasked spectra used for each pixel, so these -; are integers in the range 0 to NSPEC [NNEWPIX]; this is -; equivalent to TOTAL(OUTMASK,2). -; -; COMMENTS: -; The best-fit eigenspectra for each of the input spectra can be determined -; for object number IOBJ by ACOEFF[*,IOBJ] # RES. -; -; EXAMPLES: -; -; BUGS: -; -; PROCEDURES CALLED: -; combine1fiber -; computechi2() -; djs_mean() -; djs_reject() -; splog -; wavevector() -; -; REVISION HISTORY: -; 10-Oct-2000 Written by D. Schlegel, Princeton -; 07-Jul-2003 added quiet keyword - C. Tremonti -; -; VERSION: -; $Id: pca_solve.pro 49529 2010-06-03 17:46:50Z kushner $ -; -;- -FUNCTION pca_solve, objflux, objivar, objloglam, zfit, $ - wavemin=wavemin, wavemax=wavemax, newloglam=newloglam, $ - newflux=newflux, newivar=newivar, maxiter=maxiter, $ - niter=niter, nkeep=nkeep, nreturn=nreturn, eigenval=eigenval, $ - acoeff=acoeff, outmask=outmask, usemask=usemask, quiet=quiet, $ - _EXTRA=KeywordsForReject - ; - ; - ; - IF (N_ELEMENTS(maxiter) EQ 0) THEN maxiter = 0 - IF ~KEYWORD_SET(niter) THEN niter = 10 - IF ~KEYWORD_SET(nkeep) THEN nkeep = 3 - IF ~KEYWORD_SET(nreturn) THEN nreturn = nkeep - ndim = SIZE(objflux, /N_DIMEN) - dims = SIZE(objflux, /DIMENS) - npix = dims[0] - IF (ndim EQ 1) THEN nobj = 1 ELSE nobj = dims[1] - splog, 'Building PCA from ', nobj, ' object spectra' - ; - ; The redshift of each object in pixels would be LOGSHIFT/OBJDLOGLAM - ; - IF KEYWORD_SET(zfit) THEN logshift = ALOG10(1.0D + zfit) $ - ELSE logshift = DBLARR(nobj) - ; - ; Determine the new wavelength mapping - ; - IF KEYWORD_SET(objloglam) THEN BEGIN ; ??? - IF KEYWORD_SET(newloglam) THEN $ - objdloglam = ABS(newloglam[1] - newloglam[0]) $ - ELSE BEGIN - igood = WHERE(objloglam NE 0) - objdloglam = ABS(objloglam[1] - objloglam[0]) - logmin = MIN(objloglam[igood]) - MAX(logshift) - logmax = MAX(objloglam[igood]) - MIN(logshift) - IF KEYWORD_SET(wavemin) THEN logmin = logmin > ALOG10(wavemin) - IF KEYWORD_SET(wavemax) THEN logmax = logmax < ALOG10(wavemax) - newloglam = wavevector(logmin, logmax, binsz=objdloglam) - ENDELSE - nnew = N_ELEMENTS(newloglam) - newflux = DBLARR(nnew,nobj) - newivar = DBLARR(nnew,nobj) - ndim = SIZE(objloglam, /N_DIMEN) - IF (ndim EQ 1) THEN qwavevec = 0B $ - ELSE BEGIN - qwavevec = 1B - IF ((SIZE(objloglam, /DIMENS))[1] NE nobj) THEN $ - MESSAGE, 'Wrong number of dimensions for OBJLOGLAM' - ENDELSE - ; - ; Shift each spectra to z=0 and sample at the output wavelengths - ; - FOR iobj=0L, nobj-1L DO BEGIN - indx = WHERE(objloglam[*,iobj*qwavevec] GT 0) - splog,'OBJECT ',iobj - combine1fiber, objloglam[indx,iobj*qwavevec]-logshift[iobj], $ - objflux[indx,iobj], objivar[indx,iobj], $ - newloglam=newloglam, binsz=objdloglam, newflux=flux1, $ - newivar=ivar1 ;, /verbose - newflux[*,iobj] = flux1 - newivar[*,iobj] = ivar1 - ENDFOR - ENDIF ELSE BEGIN - newflux = objflux - newivar = objivar - nnew = (SIZE(objflux,/DIMENS))[0] - ENDELSE - ; - ; Construct the synthetic weight vector, to be used when replacing - ; the low-S/N object pixels with the reconstructions. - ; - synwvec = DBLARR(nnew) + 1.0D ; Set to 1 if no data for this wavelength - FOR ipix=0L, nnew-1L DO BEGIN - indx = WHERE(newivar[ipix,*] NE 0) - IF (indx[0] NE -1) THEN $ - synwvec[ipix] = djs_mean(newivar[ipix,indx]) - ENDFOR - ; - ; Compute a mean spectrum, and use this to replace masked pixels. - ; Use only the NUSE spectra with flux levels at least 5% of the median - ; flux level. For wavelengths with no unmasked data in any spectrum, - ; just average all the spectra for lack of anything better to do. - ; - ; normflux = TOTAL(newflux,1) / nnew - ; iuse = WHERE(normflux GT 0.05 * MEDIAN(normflux), nuse) - ; synflux = DBLARR(nnew) - ; usemask = LONARR(nnew) - ; FOR ipix=0L, nnew-1L DO BEGIN - ; ibad = WHERE(newivar[ipix,iuse] EQ 0, nbad) - ; usemask[ipix] = nuse - nbad - ; IF (nbad LT nuse) THEN BEGIN - ; synflux[ipix] = TOTAL( newflux[ipix,iuse] * newivar[ipix,iuse]) $ - ; / TOTAL(newivar[ipix,iuse] * normflux[iuse]) - ; ENDIF ELSE BEGIN - ; synflux[ipix] = TOTAL( newflux[ipix,iuse] / normflux[iuse]) / nuse - ; ENDELSE - ; ENDFOR - ; FOR iobj=0L, nobj-1L DO BEGIN - ; ibad = WHERE(newivar[*,iobj] EQ 0) - ; IF (ibad[0] NE -1) THEN $ - ; newflux[ibad,iobj] = synflux[ibad] * normflux[iobj] - ; ENDFOR - ; - ; Construct the USEMASK from the output mask (OUTMASK) instead of - ; from NEWIVAR. - ; - ; IF (nobj EQ 1) THEN usemask = newivar NE 0 $ - ; ELSE usemask = TOTAL(newivar NE 0, 2) - ; - ; If there is only 1 object spectrum, then all we can do is return it - ; (after it has been re-binned). - ; - IF (nobj EQ 1) THEN BEGIN - IF ARG_PRESENT(eigenval) THEN eigenval = 1.0D - IF ARG_PRESENT(acoeff) THEN acoeff = 1.0D - IF ARG_PRESENT(outmask) THEN outmask = LONARR(nnew) + 1L - IF ARG_PRESENT(usemask) THEN usemask = LONARR(nnew) + 1L - RETURN, newflux - ENDIF - ; - ; Rejection iteration loop - ; - qdone = 0 - iiter = 0 - ; - ; Begin with all points good (unless the inverse variance is zero). - ; - ; outmask = MAKE_ARRAY(DIMENSION=SIZE(newflux,/DIMENS), /BYTE) + 1B - outmask = 0 - inmask = newivar NE 0 - WHILE ((qdone EQ 0) AND (iiter LE maxiter)) DO BEGIN - qdone = djs_reject(newflux, ymodel, inmask=inmask, outmask=outmask, $ - invvar=newivar, _EXTRA=KeywordsForReject) - ; - ; Iteratively do the PCA solution - ; - filtflux = newflux - acoeff = DBLARR(nkeep,nobj) - t0=SYSTIME(1) - FOR ipiter=0L, niter-1L DO BEGIN - eigenval = 1 ; Set so that the PCOMP() routine returns this. - coeff = 1 ; Set so that the PCOMP() routine returns this. - totflux = DBLARR(nobj) - FOR iobj=0L, nobj-1L DO $ - totflux[iobj] = TOTAL(ABS(filtflux[*,iobj] - filtflux[0,iobj])) - igoodobj = WHERE(totflux GT 0, ngoodobj) - IF (ngoodobj EQ nobj) THEN BEGIN - pres = PCOMP(TRANSPOSE(filtflux), EIGENVAL=eigenval, /DOUBLE) - ENDIF ELSE BEGIN - tmp_pres = PCOMP(TRANSPOSE(filtflux[*,igoodobj]), $ - EIGENVAL=tmp_eigenval, /DOUBLE) - pres = DBLARR(nobj,nnew) - pres[igoodobj,*] = tmp_pres - eigenval = DBLARR(1,nobj) - eigenval[0,igoodobj] = tmp_eigenval - ENDELSE - maskivar = newivar * outmask - sqivar = SQRT(maskivar) - ; bvec = filtflux * sqivar - ; mmatrix = pres[0:nkeep-1,*] - ; FOR i=0L, nkeep-1L DO $ - ; mmatrix[i,*] = mmatrix[i,*] * sqivar - ; mmatrixt = TRANSPOSE(mmatrix) - FOR iobj=0L, nobj-1L DO BEGIN - chi2 = computechi2(newflux[*,iobj], sqivar[*,iobj], $ - TRANSPOSE(pres[0:nkeep-1,*]), acoeff=theta) - synflux = theta # pres[0:nkeep-1,*] - filtflux[*,iobj] = (maskivar[*,iobj] * newflux[*,iobj] + $ - synwvec * synflux) / (maskivar[*,iobj] + synwvec) - acoeff[*,iobj] = theta - ; splot,filtflux[*,iobj] - ; soplot,synflux,color='red' - ENDFOR - ; writefits, 'test-'+STRTRIM(STRING(ipiter),1)+'.fits', $ - ; FLOAT(TRANSPOSE(pres[0:nkeep-1,*])) - IF ~KEYWORD_SET(quiet) THEN $ - splog, 'Elapsed time for iteration #', ipiter, ' = ', $ - SYSTIME(1)-t0 - ENDFOR ; End PCA iterations - ; - ; Now set YMODEL for rejecting points - ; - ymodel = acoeff ## TRANSPOSE(pres[0:nkeep-1,*]) - iiter = iiter + 1 - ENDWHILE ; End rejection iterations - IF ARG_PRESENT(usemask) THEN BEGIN - IF (nobj EQ 1) THEN usemask = outmask $ - ELSE usemask = TOTAL(outmask, 2) - ENDIF - eigenval = eigenval[0:nreturn-1] - RETURN, TRANSPOSE(pres[0:nreturn-1,*]) -END diff --git a/pro/spec1d/pca_star.pro b/pro/spec1d/pca_star.pro deleted file mode 100644 index 5878ded44..000000000 --- a/pro/spec1d/pca_star.pro +++ /dev/null @@ -1,255 +0,0 @@ -;+ -; NAME: -; pca_star -; -; PURPOSE: -; Wrapper on pca_solve.pro -; -; CALLING SEQUENCE: -; pca_star, [inputfile=inputfile], $ -; [binsz=binsz], [niter=niter], [savefile=savefile], [/flux] -; -; INPUTS: -; None -; -; OPTIONAL INPUTS: -; inputfile - File containing plate, mjd, fiber, redshift. If not -; specified, it will read eigeninput_star.par from -; $IDLSPEC2D_DIR/templates -; flux - Plot input spectra. -; niter - Number of iterations to pass to pca_solve, default 10. -; savefile - Save the input spectra to a named file for debugging. -; -; OUTPUTS: -; None, but creates the files spEigenStar-MJD.fits & spEigenStar-MJD.ps -; -; DATA FILES: -; $IDLSPEC2D_DIR/templates/eigeninput_star.dat -; -; PROCEDURES CALLED: -; get_juldate -; dfpsplot -; yanny_read -; yanny_free -; readspec -; skymask() -; wavevector() -; pca_solve() -; computechi2() -; djs_median() -; djs_plot -; djs_oplot -; djs_xyouts -; sxaddpar -; mwrfits -; dfpsclose -; -; REVISION HISTORY: -; Written a long time ago. -; Updated for BOSS inputs by B. A. Weaver, NYU -; -; VERSION: -; $Id: pca_star.pro 50208 2012-01-24 15:47:52Z weaver $ -; -;------------------------------------------------------------------------------ -PRO pca_star, inputfile=inputfile, flux=flux, niter=niter, savefile=savefile - ; - ; Define initial parameters - ; - IF ~KEYWORD_SET(inputfile) THEN BEGIN - inputfile = FILEPATH('eigeninput_star.par', $ - ROOT_DIR=GETENV('IDLSPEC2D_DIR'), SUBDIRECTORY='templates') - ENDIF - wavemin = 0 - wavemax = 0 - snmax = 100 - IF ~KEYWORD_SET(niter) THEN niter = 10 - cspeed = 2.99792458d5 - ; - ; Name the output files - ; - - get_juldate, jd - mjdstr = STRING(LONG(jd-2400000.5), FORMAT='(I5)') - outfile = 'spEigenStar-' + mjdstr + '.fits' - plotfile = 'spEigenStar-' + mjdstr + '.ps' - dfpsplot, plotfile, /color, /landscape - colorvec = ['default', 'red', 'green', 'blue', 'magenta', 'cyan'] - ; - ; Read the input spectra - ; - yanny_read, inputfile, pdat - slist = *pdat[0] - yanny_free, pdat - readspec, slist.plate, slist.fiberid, mjd=slist.mjd, $ - flux=objflux, invvar=objivar, $ - andmask=andmask, ormask=ormask, plugmap=plugmap, loglam=objloglam, /align - ; - ; Insist that all of the requested spectra exist - ; - imissing = WHERE(plugmap.fiberid EQ 0, nmissing) - IF (nmissing GT 0) THEN BEGIN - FOR i=0, nmissing-1 DO $ - PRINT, 'Missing plate=', plate[imissing[i]], $ - ' mjd=', mjd[imissing[i]], $ - ' fiber=', fiber[imissing[i]] - MESSAGE, STRING(nmissing) + ' missing object(s)' - ENDIF - ; - ; Do not fit where the spectrum may be dominated by sky-sub residuals. - ; - objivar = skymask(objivar, andmask, ormask) - andmask = 0 ; Free memory - ormask = 0 ; Free memory - nobj = (SIZE(objflux, /dimens))[1] - objdloglam = objloglam[1] - objloglam[0] - IF (KEYWORD_SET(snmax)) THEN BEGIN - ifix = WHERE(objflux^2 * objivar GT snmax^2) - IF (ifix[0] NE -1) THEN objivar[ifix] = (snmax/objflux[ifix])^2 - ENDIF - ; - ; Find the list of unique star types - ; - isort = SORT(slist.class) - classlist = slist[isort[UNIQ(slist[isort].class)]].class - ; - ; LOOP OVER EACH STAR TYPE - ; - FOR iclass=0, N_ELEMENTS(classlist)-1 DO BEGIN - ; - ; Find the subclasses for this stellar type - ; - indx = WHERE(slist.class EQ classlist[iclass], nindx) - thesesubclass = slist[indx].subclass - isort = SORT(thesesubclass) - subclasslist = thesesubclass[isort[UNIQ(thesesubclass[isort])]] - nsubclass = N_ELEMENTS(subclasslist) - ; - ; Solve for 2 eigencomponents if we have specified subclasses - ; for this stellar type. - ; - IF (nsubclass EQ 1) THEN nkeep = 1 ELSE nkeep = 2 - newloglam = objloglam - pcaflux = pca_solve(objflux[*,indx], objivar[*,indx], objloglam, $ - slist[indx].cz/cspeed, wavemin=wavemin, wavemax=wavemax, $ - niter=niter, nkeep=nkeep, newloglam=newloglam, $ - eigenval=eigenval, acoeff=acoeff, usemask=usemask) - ; - ; Interpolate over bad flux values in the middle of a spectrum, - ; and set fluxes to zero at the blue+red ends of the spectrum. - ; - ; minuse = 1 ; ? - minuse = FLOOR((nindx+1) / 3.) - qbad = usemask LT minuse - ; - ; Interpolate over all bad pixels - ; - FOR j=0, nkeep-1 DO $ - pcaflux[*,j] = djs_maskinterp(pcaflux[*,j], qbad EQ 1, /const) - ; - ; Set bad pixels at the very start or end of the spectrum to zero instead - ; - npix = N_ELEMENTS(qbad) - IF (qbad[0]) THEN $ - pcaflux[0:(where(qbad EQ 0))[0]-1,*] = 0 - IF (qbad[npix-1]) THEN $ - pcaflux[(REVERSE(WHERE(qbad EQ 0)))[0]+1:npix-1,*] = 0 - ; - ; The following would plot the 0th object and overplot the best-fit PCA - ; - ; ii=0 - ; splot,10^newloglam,objflux[*,indx[ii]] - ; junk=pcaflux[*,0] * (acoeff[0,ii])[0] + pcaflux[*,1] * (acoeff[1,ii])[0] - ; soplot,10^newloglam,junk,color='red' - ; - ; Re-normalize the first eigenspectrum to a mean of 1 - ; - norm = MEAN(pcaflux[*,0]) - pcaflux = pcaflux / norm - acoeff = acoeff * norm - ; - ; Now loop through each stellar subclass and reconstruct - ; an eigenspectrum for that subclass - ; - thesesubclassnum = LONARR(N_ELEMENTS(thesesubclass)) - FOR isub=0, nsubclass-1 DO BEGIN - ii = WHERE(thesesubclass EQ subclasslist[isub]) - thesesubclassnum[ii] = isub - IF (nkeep EQ 1) THEN BEGIN - thisflux = pcaflux - ENDIF ELSE BEGIN - aratio = acoeff[1,ii] / acoeff[0,ii] - thisratio = MEDIAN(aratio, /EVEN) - thisflux = pcaflux[*,0] + thisratio * pcaflux[*,1] - ENDELSE - ; - ; The output wavelength mapping is the same for everything, - ; so we can simply stack the PCA spectra. - ; - IF KEYWORD_SET(fullflux) THEN fullflux = [[fullflux], [thisflux]] $ - ELSE fullflux = thisflux - IF KEYWORD_SET(namearr) THEN namearr = [namearr, subclasslist[isub]] $ - ELSE namearr = subclasslist[isub] - plotflux = thisflux / MAX(thisflux) ; Re-scale for plotting - IF (isub EQ 0) THEN $ - djs_plot, 10^newloglam, plotflux, color=colorvec[0], $ - xtitle='Wavelength [\AA]', ytitle='Flux [arbitrary units]', $ - title='STAR '+classlist[iclass]+': Eigenspectra Reconstructions' $ - ELSE $ - djs_oplot, 10^newloglam, plotflux, $ - color=colorvec[isub MOD N_ELEMENTS(colorvec)] - nnew = N_ELEMENTS(newloglam) - XYOUTS, 10^newloglam[nnew-1], plotflux[nnew-1], $ - subclasslist[isub], align=-0.5, $ - color=djs_icolor(colorvec[isub MOD N_ELEMENTS(colorvec)]) - ENDFOR - IF (nkeep GT 1) THEN BEGIN - allratio = TRANSPOSE(acoeff[1,*] / acoeff[0,*]) - isort = SORT(thesesubclassnum) - djs_plot, thesesubclassnum[isort], allratio[isort], ps=-4, $ - xrange=[-1,nsubclass], xstyle=1, xtickname=subclasslist, $ - xtickv=LINDGEN(nsubclass), xticks=nsubclass-1, $ - xtitle='Subclass', ytitle='Eigenvalue Ratio (a_1/a_0)', $ - title='STAR '+classlist[iclass]+': Eigenvalue Ratios' - FOR j=0, N_ELEMENTS(indx)-1 DO $ - XYOUTS, thesesubclassnum[isort[j]], allratio[isort[j]], $ - align=0.0, orient=45, $ - STRING(slist[indx[isort[j]]].plate, slist[indx[isort[j]]].fiberid, $ - FORMAT='(I4,"-",I3)') - ENDIF - ENDFOR - ; - ; Construct header for output file - ; - sxaddpar, hdr, 'OBJECT', 'STAR' - sxaddpar, hdr, 'COEFF0', newloglam[0] - sxaddpar, hdr, 'COEFF1', objdloglam - sxaddpar, hdr, 'IDLUTILS', idlutils_version(), 'Version of idlutils' - sxaddpar, hdr, 'SPEC2D', idlspec2d_version(), 'Version of idlspec2d' - sxaddpar, hdr, 'RUN2D', GETENV('RUN2D'), 'Version of 2d reduction' - sxaddpar, hdr, 'RUN1D', GETENV('RUN1D'), 'Version of 1d reduction' - ; - ; Add a space to the name below, so that 'F' appears as a string and - ; not as a logical. - ; - FOR i=0, N_ELEMENTS(namearr)-1 do $ - sxaddpar, hdr, 'NAME'+STRTRIM(STRING(i),2), namearr[i]+' ' - ; - ; Write output file - ; - mwrfits, FLOAT(fullflux), outfile, hdr, /create - ; - ; Create a table of inputs - ; - sxaddpar, hdr2, 'FILENAME', inputfile - inputs0 = {inputs, plate:0L, mjd:0L, fiber:0L, redshift:0.0D} - inputs = REPLICATE(inputs0,N_ELEMENTS(slist)) - inputs.plate = slist.plate - inputs.mjd = slist.mjd - inputs.fiber = slist.fiberid - inputs.redshift = slist.cz/cspeed - mwrfits, inputs, outfile, hdr2 - dfpsclose - RETURN -END -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/pca_star_boss.pro b/pro/spec1d/pca_star_boss.pro deleted file mode 100644 index beb5b7b8b..000000000 --- a/pro/spec1d/pca_star_boss.pro +++ /dev/null @@ -1,141 +0,0 @@ -; Generate both an output FITS file and a PostScript plot. -; -; Loosely based on pca_star by Schlegel (????), -; Substantially rewritten by Bolton (utah2011june) to inherit Indo-US archetype spectra, -; to do PCA of BOSS spectra for weird types not represented in Indo-US, -; and to simplify some of the pleasantries. -;------------------------------------------------------------------------------ -pro pca_star_boss, filename - -; Read in the BOSS-sampled indo-US archetype spectra -; and define the target baseline from them: -ius_fname = filepath('spArchIndoUS.fits', $ - root_dir=getenv('IDLSPEC2D_DIR'), subdirectory='templates') - -influx = mrdfits(ius_fname,0,ius_hdr) -coeff0 = sxpar(ius_hdr, 'COEFF0') -coeff1 = sxpar(ius_hdr, 'COEFF1') -npix = sxpar(ius_hdr, 'NAXIS1') -newloglam = findgen(npix) * coeff1 + coeff0 -n_ius = sxpar(ius_hdr, 'NAXIS2') - - snmax = 100 - niter = 10 - cspeed = 2.99792458d5 - - minfrac = 0.75 ; minimum fraction of spectra contributing to a pixel - pbuff = 25 ; pixel buffer for averaging for padding value - - get_juldate, jd - mjdstr = string(long(jd-2400000L), format='(i5)') - outfile = 'spEigenStar-' + mjdstr + '.fits' - plotfile = 'spEigenStar-' + mjdstr + '.ps' - - ;---------- - ; Read the input spectra - - if (NOT keyword_set(filename)) then $ - filename = filepath('eigeninput_star_boss.dat', $ - root_dir=getenv('IDLSPEC2D_DIR'), subdirectory='templates') - thisfile = (findfile(filename))[0] - if (NOT keyword_set(thisfile)) then begin - print, 'File not found: ' + filename - return - endif - readcol, thisfile, plate, mjd, fiberid, zstar, subclass, $ - format='L,L,L,F,A', comment='#' - - readspec, plate, fiberid, mjd=mjd, plugmap=plugmap, $ - flux=flux, invvar=invvar, loglam=loglam, andmask=andmask, ormask=ormask, /align - - ;---------- - ; Insist that all of the requested spectra exist - - imissing = where(plugmap.fiberid EQ 0, nmissing) - if (nmissing GT 0) then begin - for i=0, nmissing-1 do $ - print, 'Missing plate=', slist[imissing[i]].plate, $ - ' mjd=', slist[imissing[i]].mjd, $ - ' fiber=', slist[imissing[i]].fiberid - message, string(nmissing) + ' missing object(s)' - endif - - ;---------- - ; Do not fit where the spectrum may be dominated by sky-sub residuals. - - invvar = skymask(invvar, andmask, ormask) - andmask = 0 ; Free memory - ormask = 0 ; Free memory - - nobj = (size(flux, /dimens))[1] - objdloglam = loglam[1] - loglam[0] - - if (keyword_set(snmax)) then begin - ifix = where(flux^2 * invvar GT snmax^2) - if (ifix[0] NE -1) then invvar[ifix] = (snmax/flux[ifix])^2 - endif - - ;---------- - ; Find the list of unique star types - - classlist = subclass[sort(subclass)] - classlist = classlist[uniq(classlist)] - -; How many classes are there? -nclass = n_elements(classlist) - -; Array to hold the new templates: -tflux = fltarr(npix, nclass) - - ;---------- - ; LOOP OVER EACH STAR TYPE - - for iclass=0, n_elements(classlist)-1 do begin - - ;---------- - ; Find the examples of this stellar type - - indx = where(subclass EQ classlist[iclass], nindx) - -; Do the PCA: - pcaflux = pca_solve(flux[*,indx], invvar[*,indx], loglam, $ - zstar[indx], niter=niter, usemask=usemask, $ - newloglam=newloglam, nkeep=1, nreturn=1) - -; Pad the cruddy bits by hand: - minuse = round(minfrac * nindx) - wh_use = where(usemask ge minuse) - umax = max(wh_use) - umin = min(wh_use) - hi_mean = mean(pcaflux[umax-pbuff+1:umax]) - lo_mean = mean(pcaflux[umin:umin+pbuff-1]) - pcaflux[0:umin-1] = lo_mean - pcaflux[umax+1:npix-1] = hi_mean - -; Assign to output array: - - tflux[*,iclass] = pcaflux - - endfor - - ;---------- - ; Construct header for output file - -hdr = ius_hdr -sxdelpar, hdr, 'NAXIS1' -sxdelpar, hdr, 'NAXIS2' -for i = 0L, nclass-1 do sxaddpar, hdr, 'NAME'+strtrim(string(i)+n_ius,2), classlist[i], $ - 'BOSS PCA template' - -; Concatenate output arrays: -outflux = [[influx], [tflux]] -n_out = (size(outflux))[2] - -; Normalize to have average flux of unity in each spectrum: -for i = 0L, n_out-1 do outflux[*,i] = outflux[*,i] / mean(outflux[*,i]) - - mwrfits, float(outflux), outfile, hdr, /create - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/pca_vstnd.pro b/pro/spec1d/pca_vstnd.pro deleted file mode 100644 index 289184746..000000000 --- a/pro/spec1d/pca_vstnd.pro +++ /dev/null @@ -1,56 +0,0 @@ -;------------------------------------------------------------------------------ -pro pca_vstnd - -; wavemin = 3300 -; wavemax = 8800 - snmax = 100 - niter = 10 - nkeep = 2 - - get_juldate, jd - mjdstr = string(long(jd-2400000L), format='(i5)') - outfile = 'spEigenVstnd-' + mjdstr + '.fits' - - ;---------- - ; Read the input spectra - - eigenfile = filepath('eigeninput_vstnd.dat', $ - root_dir=getenv('IDLSPEC2D_DIR'), subdirectory='templates') - djs_readcol, eigenfile, skip=2, plate, mjd, fiber, zfit, format='(L,L,L,D)' - - readspec, plate, fiber, mjd=mjd, flux=objflux, invvar=objivar, $ - andmask=andmask, ormask=ormask, plugmap=plugmap, loglam=objloglam - - ;---------- - ; Do not fit where the spectrum may be dominated by sky-sub residuals. - - objivar = skymask(objivar, andmask, ormask) -andmask = 0 ; Free memory -ormask = 0 ; Free memory - - nobj = (size(objflux, /dimens))[1] - objdloglam = objloglam[1] - objloglam[0] - - if (keyword_set(snmax)) then begin - ifix = where(objflux^2 * objivar GT snmax^2) - if (ifix[0] NE -1) then objivar[ifix] = (snmax/objflux[ifix])^2 - endif - - ;---------- - - pcaflux = pca_solve(objflux, objivar, objloglam, zfit, $ - wavemin=wavemin, wavemax=wavemax, $ - niter=niter, nkeep=nkeep, newloglam=newloglam, eigenval=eigenval) - pcaflux = float(pcaflux) - - sxaddpar, hdr, 'OBJECT', 'VELOCITY_STANDARD' - sxaddpar, hdr, 'COEFF0', newloglam[0] - sxaddpar, hdr, 'COEFF1', objdloglam - for i=0, n_elements(eigenval)-1 do $ - sxaddpar, hdr, 'EIGEN'+strtrim(string(i),1), eigenval[i] - - mwrfits, pcaflux, outfile, hdr, /create - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/plate_spec_image.pro b/pro/spec1d/plate_spec_image.pro deleted file mode 100644 index 24eaf8816..000000000 --- a/pro/spec1d/plate_spec_image.pro +++ /dev/null @@ -1,121 +0,0 @@ -;+ -; NAME: -; plate_spec_image -; PURPOSE: -; Create spectroscopic images for a plate -; CALLING SEQUENCE: -; plate_spec_image, plate, mjd=, run2d=, run1d=, topdir= -; INPUTS -; plate - plate # -; mjd - MJD of observation -; OPTIONAL INPUTS: -; run1d - 1d rerun number ('' by default) -; run2d - 2d rerun number ('' by default) -; topdir - directory to use instead of $BOSS_SPECTRO_REDUX -; xra - [2] wavelength range to plot (Ang) -; OPTIONAL KEYWORDS: -; /noclobber - do not clobber existing image PNG files -; (will clobber HTML) -; COMMENTS: -; Creates files in: -; $BOSS_SPECTRO_REDUX/images/[run2d]/[run1d]/[plate4]-[mjd] -; with the naming convention: -; spec-image-[plate4]-[mjd]-[fiber4].png -; spec-image-[plate4]-[mjd]-[fiber4].thumb.png -; Also creates an index.html file which is just a table of -; the thumbnails. -; REVISION HISTORY: -; 15-May-2009 by M. Blanton, NYU -; VERSION: -; $Id: plate_spec_image.pro 64936 2015-09-21 18:50:32Z julianbautista $ -;------------------------------------------------------------------------------ -pro plate_spec_image, plate, mjd=mjd, run2d=run2d, run1d=run1d, $ - topdir=topdir, xra=xra, silent=silent, $ - noclobber=noclobber, legacy=legacy, plates=plates - - RESOLVE_ALL, /QUIET, /SKIP_EXISTING, /CONTINUE_ON_ERROR - CPU, TPOOL_NTHREADS = 1 - -if(NOT keyword_set(run2d)) then run2d='' -if(NOT keyword_set(run1d)) then run1d='' -if(NOT keyword_set(topdir)) then topdir= getenv('BOSS_SPECTRO_REDUX') - - -get_field_type, fieldid=plate, mjd=mjd, legacy=legacy, plates=plates, fps=fps - -pmjd=field_to_string(plate)+'-'+string(mjd, f='(i5.5)') -outdir=topdir+'/images/'+run2d+'/'+run1d+'/'+pmjd -file_mkdir, outdir -readspec, plate, mjd=mjd, zans=zans, run2d=run2d, run1d=run1d, $ - topdir=topdir,legacy=legacy,plates=plates, plug=plug - -if(n_tags(zans) eq 0) then begin - splog, 'No 1d reductions for this plate.' - return -endif - -if file_test(outdir+'/index.html') then rmfile, outdir+'/index.html' -openw, unit, outdir+'/tmp-index.html', /get_lun -printf, unit, '' -printf, unit, '' -; printf, unit, '' -printf, unit, '' -printf, unit, '' -printf, unit, '' -printf, unit, ''+pmjdf+ $
- ' ' - printf, unit, '' - printf, unit, ' | '
- if(((i mod nper) eq nper-1L) OR (i eq n_elements(zans)-1L)) then $
- printf, unit, '
Last Update: '+ systime()+'
', '(S/N)^2 values are corrected for galactic dust reddening
'] - endif else begin - toptext = [toptext, '(S/N)^2 values are not corrected for galactic dust reddening
'] - endelse - - platelist_write, plist[isort1], trimtags=trimtags1, alias=alias, $ - fileprefix='platelist', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Plates Observed List' - platelist_write, plist[isort2], trimtags=trimtags1, alias=alias, $ - fileprefix='platelist-mjdsort', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Plates Observed List' - - platelist_write, plist[isort1], trimtags=trimtags2, alias=alias, $ - fileprefix='platequality', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Plate Quality List' - platelist_write, plist[isort2], trimtags=trimtags2, alias=alias, $ - fileprefix='platequality-mjdsort', toptext=toptext, outdir=outdir, $ - title='SDSS Spectroscopy Plate Quality List' - - ;---------- - ; Write the FITS binary table - - mwrfits, plist, fitsfile, /create - - for i=0L, n_elements(zlogfile)-1L do ptr_free, zlogfile[i] - for i=0L, n_elements(zbestfile)-1L do ptr_free, zbestfile[i] - for i=0L, n_elements(zbestrun1d)-1L do ptr_free, zbestrun1d[i] - - ;---------- - ; See if the platelist webapp is installed; if so, update platelist.json - ; Only do this if there is a single run2d - if n_elements(run2d) eq 1 then begin - appdir = getenv('BOSS_SPECTRO_REDUX')+'/'+run2d+'/platelist' - json_exists = file_test(appdir+'/data/platelist.json') - - if json_exists eq 1 then begin - cmd = appdir+'/bin/platelist2json.py '+fitsfile+' > '+appdir+'/data/platelist.json' - spawn, cmd - endif - endif - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/platemerge.pro b/pro/spec1d/platemerge.pro deleted file mode 100644 index 412a05cbf..000000000 --- a/pro/spec1d/platemerge.pro +++ /dev/null @@ -1,647 +0,0 @@ -;+ -; NAME: -; platemerge -; -; PURPOSE: -; Merge all Spectro-1D outputs with photoPosPlate,spInspect files -; -; CALLING SEQUENCE: -; platemerge, [ plate=, mjd=, except_tags=, indir=, outroot=, $ -; run2d=, /include_bad, /calc_noqso, /skip_line ] -; -; INPUTS: -; -; OPTIONAL INPUTS: -; plate - Plates to include; default to all files -; specified by the PLATELIST routine. -; mjd - Optional MJDs corresponding to the specified PLATEs; -; if specified, then PLATE and MJD should have the same -; number of elements. -; except_tags - Tag names to exclude; default to '*COVAR'. -; indir - Input directory with platelist.fits file; passed to -; platelist topir option which defaults to $BOSS_SPECTRO_REDUX -; outroot - Root name for output files; default to -; $BOSS_SPECTRO_REDUX/$RUN2D/spAll; the files are then -; spAll-$RUN2D.fits, spAll-$RUN2D.dat, spAllLine-$RUN2D.dat. -; run2d - List of RUN2D subdirectories to merge, one set of output -; files per name in $RUN2D; default to all values of RUN2D -; returned by PLATELIST. -; include_bad - If set, then include bad plates -; calc_noqso - If set, then also include redshift info for best non-QSO -; redshift fits. Defaults to being set. -; skip_line - If set, skip the generation of spAllLine.fits -; mergerun2d - If set, generate a single $BOSS_SPECTRO_REDUX/spAll.fits -; file combining all RUN2D versions in -; $BOSS_SPECTRO_REDUX/platelist.fits. -; Ignores run2d, $RUN2D, and does *not* write a separate -; file for each RUN2D. -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; Depends upon the platelist.fits file written by PLATELIST. -; Trims to only 'good' plates, or those in a public data release. -; -; The SPECPRIMARY output element is used to select a unique set of -; objects in the case of duplicate observations. Any objects observed -; multiple times will have SPECPRIMARY=1 for one instance only, and =0 -; for all other instances. The criteria (in order of importance) are -; as follows: -; 1) Prefer observations with positive SN_MEDIAN in r-band -; 2) Prefer PLATEQUALITY='good' over any other plate quality -; 3) Prefer observations with ZWARNING=0 -; 4) Prefer objects with larger SN_MEDIAN in r-band -; -; Temporary files are created first, such as 'spAll.fits.tmp', which -; are renamed at the end of the routine to 'spAll.fits', etc. -; -; EXAMPLES: -; -; BUGS: -; -; DATA FILES: -; -; PROCEDURES CALLED: -; copy_struct -; copy_struct_inx -; djs_filepath() -; headfits() -; hogg_mrdfits() -; mrdfits() -; mwrfits_chunks -; platelist -; readspec -; repstr -; spheregroup -; splog -; struct_print -; sxaddpar -; sxpar() -; -; REVISION HISTORY: -; 30-Oct-2000 Written by D. Schlegel, Princeton -; 29-Jul-2010 Added EXCLUDE_CLASS and SKIP_LINE, A. Bolton, Utah -; 17-Mar-2011 Changed EXCLUDE_CLASS behavior, A. Bolton, Utah -; 30-Jun-2011 Changed EXCLUDE_CLASS to more specific and correct -; CALC_NOQSO, including proper rchi2diff, A. Bolton, Utah -; 04-Oct-2012 Added SPECBOSS tag (ASB, Utah). -;------------------------------------------------------------------------------ - -pro platemerge1, plate=plate, mjd=mjd, except_tags=except_tags1, $ - indir=indir, outroot=outroot1, run2d=run2d, include_bad=include_bad, $ - calc_noqso=calc_noqso, skip_line=skip_line, plist=plist - - dtheta = 2.0 / 3600. - - if (n_elements(except_tags1) GT 0) then except_tags = except_tags1 $ - else except_tags = '*COVAR' - if (keyword_set(outroot1)) then begin - outroot = [outroot1, outroot1+'Line'] - endif else begin - outroot = ['spAll','spAllLine'] - if (keyword_set(run2d)) then outroot = outroot + '-' + run2d - outroot = djs_filepath(outroot, root_dir=getenv('BOSS_SPECTRO_REDUX'), $ - subdir=run2d) - endelse - if (n_elements(calc_noqso) eq 0) then calc_noqso = 1B - - t1 = systime(1) - thismem = memory() - - ;---------- - ; Read platelist if needed - - if (NOT keyword_set(plist)) then $ - platelist, plist=plist, topdir=indir, run2d=run2d - - if (NOT keyword_set(plist)) then return - - ;---------- - ; Find out if this plist includes dereddened SN2 values - - plist_tags = tag_names(plist) - ii = where(plist_tags EQ 'DEREDSN2', n) - if n GT 0 then has_deredsn2 = 1 else has_deredsn2 = 0 - - ;---------- - ; Trim to good (or non-bad public) plates - - if (keyword_set(plate)) then begin - nplate = n_elements(plist) - if (keyword_set(mjd) AND n_elements(mjd) NE nplate) then $ - message, 'Number of elements in PLATE and MJD must agree' - - qkeep = bytarr(nplate) - if (keyword_set(mjd)) then begin - for i=0L, n_elements(plate)-1 do begin - for j=0L, n_elements(mjd)-1 do begin - qkeep = qkeep OR (plist.plate EQ plate[i] AND plist.mjd EQ mjd[j]) - endfor - endfor - endif else begin - for i=0L, n_elements(plate)-1 do $ - qkeep = qkeep OR plist.plate EQ plate[i] - endelse - ikeep = where(qkeep, nkeep) - if (nkeep EQ 0) then return - plist = plist[ikeep] - endif - - if (keyword_set(run2d)) then begin - qkeep = strmatch(strtrim(plist.run2d),run2d) - ikeep = where(qkeep, nkeep) - if (nkeep EQ 0) then return - plist = plist[ikeep] - endif - - qdone = strtrim(plist.status1d,2) EQ 'Done' - if (NOT keyword_set(include_bad)) then begin - qdone = qdone AND $ - (strtrim(plist.platequality,2) EQ 'good' $ - OR strtrim(plist.platequality,2) EQ 'marginal' $ - OR (strtrim(plist.public,2) NE '' AND $ - strtrim(plist.platequality,2) NE 'bad') ) - endif - indx = where(qdone, ct) - if (ct EQ 0) then return - plist = plist[indx] - - nfile = n_elements(plist) - nout = total(plist.n_total) - splog, 'Total number of objects = ', nout - - ;---------- - ; Find the first tsObj file that exists for use in constructing the - ; output structure. - - ifile = 0 - while (NOT keyword_set(tsobj0)) do begin - readspec, plist[ifile].plate, mjd=plist[ifile].mjd, $ - run2d=strtrim(plist[ifile].run2d), tsobj=tsobj0, /silent - if (keyword_set(tsobj0)) then begin - tsobj0 = tsobj0[0] - endif else begin - ifile = ifile + 1 - if (ifile EQ nfile) then $ - message, 'No photoPosPlate files found!' - endelse - endwhile - - ;---------- - ; Create the additional tags to add to the output structure - - pstuff = create_struct( $ - 'programname' , ' ', $ - 'chunk' , ' ', $ - 'platequality', ' ', $ - 'platesn2' , 0.0, $ - 'deredsn2' , 0.0, $ - 'primtarget' , 0L, $ - 'sectarget' , 0L, $ - 'lambda_eff' , 0.0, $ - 'bluefiber' , 0L, $ - 'zoffset' , 0.0, $ - 'xfocal' , 0.0, $ - 'yfocal' , 0.0, $ - 'boss_target1', 0LL, $ - 'boss_target2', 0LL, $ - 'ancillary_target1', 0LL, $ - 'ancillary_target2', 0LL, $ - 'eboss_target0', 0LL, $ - ;;- JB adding 4 new bits - 'eboss_target1', 0LL, $ - 'eboss_target2', 0LL, $ - 'eboss_target_id', 0LL, $ - 'thing_id_targeting', 0LL, $ - 'specprimary' , 0B, $ - 'specboss' , 0B, $ - 'boss_specobj_id' , 0L, $ - 'nspecobs' , 0, $ -;;- SB Oct 2012: remove QSO VAC inputs for DR10 -;; 'z_person' , 0.0, $ -;; 'class_person', 0L, $ -;; 'z_conf_person', 0L, $ -;; 'comments_person', '', $ - 'calibflux' , fltarr(5), $ - 'calibflux_ivar', fltarr(5) ) - - ;---------- - ; Loop through each file - - splog, 'Reading ZANS files' - for ifile=0L, nfile-1 do begin - print, 'Reading ZANS file ',ifile+1, ' of ', nfile - - readspec, plist[ifile].plate, mjd=plist[ifile].mjd, $ - run2d=strtrim(plist[ifile].run2d), run1d=strtrim(plist[ifile].run1d), $ - zans=zans, objhdr=objhdr, $ ;; zmanual=zmanual, - plugmap=plugmap, /silent, unsigned=(ifile EQ 0) - - zans = struct_selecttags(zans, except_tags='OBJID') - -; ASB 2011 Mar: append info on best non-galaxy and non-qso redshifts/classes: -; ASB 2011 Jun: changed to do the "no-qso" case exclusively, and more correctly. -; ASB 2011 Jul: moved this into SPREDUCE1D. Retained for now, but -; test to see if the tags are already present. To be -; removed following verification. -;; if keyword_set(calc_noqso) then begin - if (keyword_set(calc_noqso) and (tag_exist(zans, 'z_noqso') eq 0)) then begin - print, ' Finding non-QSO redshift info.' - - ;;- JB : Change plate string format PLATEPROBLEM - pstring = plate_to_string(plist[ifile].plate) - mstring = string(plist[ifile].mjd, format='(i5.5)') - zallfile = getenv('BOSS_SPECTRO_REDUX') + '/' + $ - strtrim(plist[ifile].run2d, 2) + '/' + $ - pstring + '/' + strtrim(plist[ifile].run1d, 2) + $ - '/spZall-' + pstring + '-' + mstring + '.fits' - zall = mrdfits(zallfile,1) - nfib = max(zall.fiberid) - min(zall.fiberid) + 1L - nzall = n_elements(zall) / nfib - zall = reform(zall, nzall, nfib) - class_all = strtrim(zall.class,2) - id_noqso = replicate(-1L, nfib) - rchi2diff_noqso = replicate(0., nfib) - for ii = 0L, nfib-1 do begin - wh_noqso = where(class_all[*,ii] ne 'QSO') - wh_noqso = (wh_noqso[sort(wh_noqso)])[0:1] - id_noqso[ii] = wh_noqso[0] - rchi2diff_noqso[ii] = total(zall[wh_noqso[0]:wh_noqso[1]-1,ii].rchi2diff) - endfor - zans_noqso = zall[id_noqso,lindgen(nfib)] - noqso_struc = replicate( $ - {z_noqso: 0., z_err_noqso: 0., zwarning_noqso: 0L, $ - class_noqso: ' ', subclass_noqso: ' ', $ - rchi2diff_noqso: 0.}, nfib) - noqso_struc.z_noqso = zans_noqso.z - noqso_struc.z_err_noqso = zans_noqso.z_err - noqso_struc.class_noqso = zans_noqso.class - noqso_struc.subclass_noqso = zans_noqso.subclass - noqso_struc.rchi2diff_noqso = rchi2diff_noqso -; Re-set the small-delta-chi2 bit: - minrchi2diff = 0.01 - small_rchi2diff = rchi2diff_noqso lt minrchi2diff - zw_new = zans_noqso.zwarning - zflagval = sdss_flagval('ZWARNING', 'SMALL_DELTA_CHI2') - zw_new = zw_new - (zw_new and zflagval) - zw_new = zw_new or (zflagval * small_rchi2diff) - noqso_struc.zwarning_noqso = zw_new - zans = struct_addtags(zans, noqso_struc) - noqso_struc = 0 - zall = 0 - zans_noqso = 0 - endif - - if (ifile EQ 0) then begin - outdat1 = create_struct(pstuff, zans[0]) - struct_assign, {junk:0}, outdat1 ; Zero-out all elements - outdat = replicate(outdat1, nout) - endif - - indx = lindgen(plist[ifile].n_total) - if (ifile GT 0) then indx += total(plist[0:ifile-1].n_total) - - ; The following is very slow, so we do this differently... -; copy_struct_inx, zans, outdat, index_to=indx - tmpdat = outdat[indx] - copy_struct, zans, tmpdat - outdat[indx] = tmpdat - - ; Fill in the first columns of this output structure - outdat[indx].programname = plist[ifile].programname - outdat[indx].chunk = plist[ifile].chunk - outdat[indx].platequality = plist[ifile].platequality - outdat[indx].platesn2 = plist[ifile].platesn2 - if has_deredsn2 then $ - outdat[indx].deredsn2 = plist[ifile].deredsn2 - - ; Read the following from the manual inspection - ;- SB Oct 2012: removed for DR10 - ;; if (keyword_set(zmanual[0])) then begin - ;; outdat[indx].z_person = zmanual.z_person - ;; outdat[indx].class_person = zmanual.class_person - ;; outdat[indx].z_conf_person = zmanual.z_conf_person - ;; outdat[indx].comments_person = zmanual.comments - ;; endif - - ; Get PRIMTARGET+SECTARGET with those values from - ; the plug-map structure in spPlate file. - outdat[indx].primtarget = plugmap.primtarget - outdat[indx].sectarget = plugmap.sectarget - outdat[indx].lambda_eff = plugmap.lambda_eff - if (tag_exist(plugmap,'zoffset')) then $ - outdat[indx].zoffset = plugmap.zoffset - if (tag_exist(plugmap,'xfocal')) then $ - outdat[indx].xfocal = plugmap.xfocal - if (tag_exist(plugmap,'yfocal')) then $ - outdat[indx].yfocal = plugmap.yfocal - if (tag_exist(plugmap,'bluefiber')) then $ - outdat[indx].bluefiber = plugmap.bluefiber - if (tag_exist(plugmap,'boss_target1')) then $ - outdat[indx].boss_target1 = plugmap.boss_target1 - if (tag_exist(plugmap,'boss_target2')) then $ - outdat[indx].boss_target2 = plugmap.boss_target2 - if (tag_exist(plugmap,'ancillary_target1')) then $ - outdat[indx].ancillary_target1 = plugmap.ancillary_target1 - if (tag_exist(plugmap,'ancillary_target2')) then $ - outdat[indx].ancillary_target2 = plugmap.ancillary_target2 - if (tag_exist(plugmap,'eboss_target0')) then $ - outdat[indx].eboss_target0 = plugmap.eboss_target0 - ;;- JB adding 4 new bits - if (tag_exist(plugmap, 'eboss_target1')) then $ - outdat[indx].eboss_target1 = plugmap.eboss_target1 - if (tag_exist(plugmap, 'eboss_target2')) then $ - outdat[indx].eboss_target2 = plugmap.eboss_target2 - if (tag_exist(plugmap, 'eboss_target_id')) then $ - outdat[indx].eboss_target_id = plugmap.eboss_target_id - if (tag_exist(plugmap, 'thing_id_targeting')) then $ - outdat[indx].thing_id_targeting = plugmap.thing_id_targeting - - ; Read the following from the plug-map if those tags exist - if (tag_exist(plugmap,'CALIBFLUX')) then $ - outdat[indx].calibflux = plugmap.calibflux - if (tag_exist(plugmap,'CALIBFLUX_IVAR')) then $ - outdat[indx].calibflux_ivar = plugmap.calibflux_ivar - endfor - - splog, 'Time to read data = ', systime(1)-t1, ' sec' - - ;---------- - ; Set the SPECPRIMARY flag to 0 or 1 - - t2 = systime(1) - - ; Determine the score for each object - ; 1) Prefer observations with positive SN_MEDIAN in r-band - ; 2) Prefer PLATEQUALITY='good' over any other plate quality - ; 3) Prefer observations with ZWARNING=0 - ; 4) Prefer objects with larger SN_MEDIAN in r-band -; ASBjuly2011: test against ZWARNING_NOQSO for GALAXY targets: - zw_primtest = outdat.zwarning - if tag_exist(outdat, 'ZWARNING_NOQSO') then begin - wh_galtarget = where(strmatch(outdat.objtype, 'GALAXY*'), ngaltarget) - if (ngaltarget gt 0) then zw_primtest[wh_galtarget] = outdat[wh_galtarget].zwarning_noqso - endif - if (n_elements(outdat[0].sn_median) EQ 1) then jfilt = 0 $ - else jfilt = 2 - score = 4 * (outdat.sn_median[jfilt] GT 0) $ - + 2 * (strmatch(outdat.platequality,'good*') EQ 1) $ -;;; + 1 * (outdat.zwarning EQ 0) $ ; replaced with line below ASBjuly2011 - + 1 * (zw_primtest EQ 0) $ - + (outdat.sn_median[jfilt]>0) / max(outdat.sn_median[jfilt]+1.) - - ingroup = spheregroup(outdat.plug_ra, outdat.plug_dec, dtheta, $ - multgroup=multgroup, firstgroup=firstgroup, nextgroup=nextgroup) - - ; Set the unique object IDs - outdat.boss_specobj_id = ingroup + 1L - - for j=0L, n_elements(firstgroup)-1L do begin - if (firstgroup[j] NE -1) then begin - if (multgroup[j] EQ 1) then begin - outdat[firstgroup[j]].specprimary = 1 - outdat[firstgroup[j]].nspecobs = 1 - endif else begin - indx = lonarr(multgroup[j]) - indx[0] = firstgroup[j] - for k=0L, multgroup[j]-2L do indx[k+1] = nextgroup[indx[k]] - foo = max(score[indx], ibest) - outdat[indx[ibest]].specprimary = 1 - outdat[indx].nspecobs = multgroup[j] - endelse - endif - endfor - - ; ASB: Copy specprimary into specboss - ; (Thinking is that specprimary can be superseded downstream.) - outdat.specboss = outdat.specprimary - - splog, 'Time to assign primaries = ', systime(1)-t2, ' sec' - - ;---------- - ; Pre-condition to FITS structure to have same-length strings - ; (for any given tag name) by concatenating spaces. - - ntag = n_tags(outdat) - tags = tag_names(outdat) - for itag=0L, ntag-1L do begin - if (size(outdat[0].(itag), /tname) EQ 'STRING') then begin - if (NOT keyword_set(silent)) then $ - print, 'Padding whitespace for string array ' + tags[itag] - taglen = strlen(strtrim(outdat.(itag))) - maxlen = max(taglen) - padspace = string('', format='(a'+string(maxlen)+')') - outdat.(itag) = strmid(outdat.(itag) + padspace, 0, maxlen) - endif - endfor - - ;---------- - ; Write the output FITS file, writing one plate at a time - - ; Don't allow duplicate tags between the tsObj structure and what - ; is already in the output structure. For ex, MJD is in both. - tsobj0 = struct_selecttags(tsobj0, except_tags=tag_names(outdat)) - platedat1 = create_struct(outdat[0], tsobj0) - if (keyword_set(except_tags)) then $ - platedat1 = struct_selecttags(platedat1, except_tags=except_tags) - struct_assign, {junk:0}, platedat1 ; Zero-out all elements - - splog, 'Writing FITS file ' + outroot[0]+'.fits' - for ifile=0L, nfile-1 do begin - print, 'Writing plate ', ifile+1, ' of ', nfile - - platedat = replicate(platedat1, plist[ifile].n_total) - indx = lindgen(plist[ifile].n_total) - if (ifile GT 0) then indx += total(plist[0:ifile-1].n_total) - readspec, plist[ifile].plate, mjd=plist[ifile].mjd, $ - run2d=strtrim(plist[ifile].run2d), tsobj=tsobj, /silent - if (keyword_set(tsobj)) then $ - copy_struct, tsobj, platedat $ - else $ - splog, 'WARNING: No tsObj file found for plate ', outdat[indx[0]].plate - copy_struct, outdat[indx], platedat - - ; All strings must be the same length, or appending to the FITS file - ; will result in corruption. The only string in the tsobj structure - ; is for the RERUN. - platedat.rerun = string(platedat.rerun+' ',format='(a3)') - - mwrfits_chunks, platedat, outroot[0]+'.fits.tmp', $ - create=(ifile EQ 0), append=(ifile GT 0) - endfor - - outdat = 0 ; Clear memory - - ;---------- - ; Create the structure for ASCII output - - adat1 = create_struct( $ - 'plate' , 0L, $ - 'mjd' , 0L, $ - 'fiberid' , 0L, $ - 'class' , '', $ - 'subclass' , '', $ - 'z' , 0.0, $ - 'z_err' , 0.0, $ - 'zwarning' , 0L, $ - 'plug_ra' , 0.0d, $ - 'plug_dec' , 0.0d, $ - 'specprimary', 0L, $ - 'chunk' , '', $ - 'platesn2' , 0.0, $ - 'deredsn2' , 0.0, $ - 'objtype' , '', $ - 'boss_target1', 0LL, $ - 'ancillary_target1', 0LL, $ - 'eboss_target0', 0LL, $ - ;;- JB adding 4 new bits - 'eboss_target1', 0LL, $ - 'eboss_target2', 0LL, $ - 'eboss_target_id', 0LL, $ - 'thing_id_targeting', 0LL, $ - 'tileid' , 0L, $ - 'objc_type' , '', $ - 'modelflux' , fltarr(5) ) - ;; 'z_person' , 0.0, $ - ;; 'class_person', 0L, $ - ;; 'z_conf_person', 0L ) - - tag_alias = [['SPECPRIMARY','PRIMARY'], $ - ['FIBERID','FIBER'], $ - ['BOSS_TARGET1','BOSS1'], $ - ['EBOSS_TARGET0','EBOSS0'], $ - ;;- JB adding 4 new aliases - ['EBOSS_TARGET1','EBOSS1'], $ - ['EBOSS_TARGET2','EBOSS2'], $ - ['EBOSS_TARGET_ID','EBOSSID'], $ - ['THING_ID_TARGETING','THIDTARG'],$ - ['ANCILLARY_TARGET1','ANCILLARY1']] - - ; Read the tags that we need from the FITS file - outdat = hogg_mrdfits(outroot[0]+'.fits.tmp', 1, nrowchunk=10000L, $ - columns=tag_names(adat1), /unsigned) - adat = replicate(adat1, n_elements(outdat)) - copy_struct, outdat, adat - - ; Replace any blank strings for CLASS with "". - ii = where(strtrim(adat.class,2) EQ '') - if (ii[0] NE -1) then adat[ii].class = '""' - - ; Replace any blank strings for SUBCLASS with "". - ; If SUBCLASS contains several words, then use a plus sign between - ; the words rather than a space. - adat.subclass = strtrim(adat.subclass,2) - ii = where(adat.subclass EQ '') - if (ii[0] NE -1) then adat[ii].subclass = '""' - adat.subclass = repstr(adat.subclass, ' ', '+') - - objtypes = ['UNKNOWN', 'CR', 'DEFECT', 'GALAXY', 'GHOST', 'KNOWNOBJ', $ - 'STAR', 'TRAIL', 'SKY'] - adat.objc_type = objtypes[outdat.objc_type] - - outdat = 0 ; Clear memory - - splog, 'Writing ASCII file ' + outroot[0]+'.dat' - struct_print, adat, filename=outroot[0]+'.dat.tmp', alias=tag_alias - - adat = 0 ; Clear memory - - ;---------- - ; Create the merged line data - - if (not keyword_set(skip_line)) then begin - splog, 'Writing FITS zline file ' + outroot[1]+'.fits' - for ifile=0L, nfile-1 do begin - splog, 'Writing zline ', ifile+1, ' of ', nfile - readspec, plist[ifile].plate, mjd=plist[ifile].mjd, $ - run2d=strtrim(plist[ifile].run2d), run1d=strtrim(plist[ifile].run1d), $ - zline=linedat, /silent - - if (ifile EQ 0) then begin - nobj = total(plist.n_total) - nper = n_elements(linedat) / plist[0].n_total - sxaddpar, linehdr, 'DIMS0', nper, ' Number of emission lines' - sxaddpar, linehdr, 'DIMS1', nobj, ' Number of objects' - linedat1 = linedat[0] - struct_assign, {junk:0}, linedat1 - endif - - ; Demand that the structure has the same format as the first - ; one written. - linedat_out = replicate(linedat1, n_elements(linedat)) - struct_assign, linedat, linedat_out - - mwrfits_chunks, linedat_out, outroot[1]+'.fits.tmp', linehdr, $ - create=(ifile EQ 0), append=(ifile GT 0) - endfor - endif - - ;---------- - ; Rename temporary files - - spawn, ['mv', outroot[0]+'.fits.tmp', outroot[0]+'.fits'], /noshell - spawn, ['gzip', outroot[0]+'.dat.tmp'], /noshell - spawn, ['mv', outroot[0]+'.dat.tmp.gz', outroot[0]+'.dat.gz'], /noshell - if (not keyword_set(skip_line)) then $ - spawn, ['mv', outroot[1]+'.fits.tmp', outroot[1]+'.fits'], /noshell - - thismem = memory() - maxmem = thismem[3] - splog, 'Maximum memory usage = ', maxmem/1.d6, ' MB' - splog, 'Total time = ', systime(1)-t1, ' sec' - - return -end - -;------------------------------------------------------------------------------ -pro platemerge, run2d=run2d, indir=indir, mergerun2d=mergerun2d, programs=programs, $ - _EXTRA=Extra - - if keyword_set(mergerun2d) then begin - platelist, outdir=getenv('BOSS_SPECTRO_REDUX'), plist=plist - platemerge1, plist=plist, _EXTRA=Extra - - endif else begin - - platelist, plist=plist, topdir=indir, run2d=run2d - - if (NOT keyword_set(plist)) then return - - if keyword_set(programs) then begin - print, 'Selecting only plates with programname:' - print, programs - nmatch = lonarr(n_elements(plist)) - for i=0, n_elements(programs)-1 do $ - nmatch+= strmatch(strtrim(plist.PROGRAMNAME,2), strtrim(programs[i],2) ) - indx = where( nmatch GT 0, ct) - if (ct EQ 0) then begin - print, 'No plates found with programnames : ' - print, programs - return - endif - plist=plist[indx] - platemerge1, plist=plist, run2d=run2d, _EXTRA=Extra - return - endif - - - - alldir = strtrim(plist.run2d) - alldir = alldir[uniq(alldir, sort(alldir))] - if (keyword_set(run2d)) then begin - nmatch = lonarr(n_elements(alldir)) - for i=0, n_elements(run2d)-1 do $ - nmatch += strmatch(alldir,run2d[i]) - indx = where(nmatch GT 0, ct) - if (ct EQ 0) then return - alldir = alldir[indx] - endif - - for i=0, n_elements(alldir)-1 do $ - platemerge1, run2d=alldir[i], indir=indir, _EXTRA=Extra - - endelse -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/plotm67.pro b/pro/spec1d/plotm67.pro deleted file mode 100644 index 1aaa048bb..000000000 --- a/pro/spec1d/plotm67.pro +++ /dev/null @@ -1,34 +0,0 @@ -; Plot velocity errors from M67 plate (plate 321) -pro plotm67 - - dfpsplot, 'm67vels.ps' - - cspeed = 3.e5 - csize = 2.0 - - readspec, 321, zans=zans, plug=plug - indx = where(plug.expl GT -900 and zans.zwarning EQ 0 $ - AND strmatch(zans.class,'STAR*')) - zans = zans[indx] - plug = plug[indx] - - cz_dave = zans.z * cspeed - cz_err = zans.z_err * cspeed - cz_cat = plug.expl - vdiff = cz_dave - cz_cat - - plot, cz_cat, cz_dave, psym=4, charsize=csize, $ - xtitle='Catalog cz [km/s]', ytitle='SDSS cz [km/s]', $ - title='Velocities in M67 (Plate 321)' - djs_oploterr, cz_cat, cz_dave, yerr=cz_err - djs_oplot, !x.crange, !x.crange - - ibad = where(abs(vdiff) GT 20) - djs_xyouts, cz_cat[ibad], cz_dave[ibad], $ - ' '+strtrim(zans.subclass,2), charsize=csize - - dfpsclose - - return -end - diff --git a/pro/spec1d/plotspec.pro b/pro/spec1d/plotspec.pro deleted file mode 100644 index 737ab5de5..000000000 --- a/pro/spec1d/plotspec.pro +++ /dev/null @@ -1,719 +0,0 @@ -;+ -; NAME: -; plotspec -; -; PURPOSE: -; Routine for plotting spectra from Princeton-1D spectro outputs. -; -; CALLING SEQUENCE: -; plotspec, plate, [ fiberid, mjd=, znum=, zmanual=, $ -; nsmooth=, /zline, /nosyn, /noerr, $ -; /sky, /ormask, /andmask, psfile=, /restframe, /netimage, $ -; /zwarning, /allexp, _EXTRA= ] -; -; INPUTS: -; plate - Plate number(s) -; -; OPTIONAL INPUTS: -; fiberid - Fiber number(s); if not set, then plot all fibers for -; each plate specified. -; mjd - MJD number(s); if not set, then select the most recent -; data for each plate (largest MJD). -; znum - If set, then return not the best-fit redshift, but the -; ZUM-th best-fit; e.g., set ZNUM=2 for second-best fit. -; zmanual - If set, then do an on-the-fly fit to either a galaxy -; template (if ZMANUAL[0]<1) or QSO (if ZMANUAL[0]>=1), -; overriding ZNUM. If this is a 2-element array, then fit -; between all redshifts in the range [ZMANUAL[0],ZMANUAL[1]]. -; Do not include any polynomial terms with the PCA templates. -; nsmooth - If set, then boxcar smooth both the object and synthetic -; spectra with a width equal to NSMOOTH. -; zline - If set, then overplot the emission line fits. -; nosyn - If set, then do not overplot the synthetic fit spectrum (blue). -; noerr - If set, then do not overplot the error vector (red). -; sky - If set, then overplot the sky spectrum (green). -; ormask - If set, then plot the OR-mask bits in yellow crosses. -; andmask - If set, then plot the AND-mask bits in red squares. -; psfile - If set, then send plot to a PostScript file instead of -; to the SPLOT interactive widget. The PostScript file name -; can be set explicitly, e.g. with PSFILE='test.ps'. Or if -; you simply set this as a flag, e.g. with /PSFILE, then the -; default file name is "spec-pppp-mmmmm-fff.ps", -; where pppp=plate number, mmmmm=MJD, fff=fiber ID. -; If FIBERID is specified, then put all plots in a single file -; named "spec-pppp-mmmmm.ps". -; restframe - If set, then plot the wavelengths in the rest frame, -; e.g. divide the wavelengths by (1+z). -; netimage - If set, then launch a Netscape browser with the object -; image from Steve Kent's web site. This only works if -; Netscape is running and has permissions at the site -; "http://sdssmosaic.fnal.gov:8015". ??? -; This is disabled if PSFILE is set. -; zwarning - If set, then only select those non-sky fibers where the -; ZWARNING flag has been set; can be used with or without -; specifying fiber numbers with FIBERID. -; allexp - If set, then plot all the individual exposure spectra, -; rather than the co-added spectrum. -; _EXTRA - Keywords for SPLOT and XYOUTS, such as XRANGE, YRANGE, THICK, -; or keywords for READSPEC and READONESPEC such as TOPDIR, -; RUN2D, RUN1D. -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; The data are read with READSPEC. See the documentation for that -; routine to see how to set environment variables that describe where -; the data files are. -; -; EXAMPLES: -; Plot the spectrum of plate 401, fiber #100 using the SPLOT plotting tool: -; IDL> plotspec, 401, 100 -; -; The spectrum is shown in white, the errors in red (except masked points -; are set to zero), and the best-fit eigenspectrum in blue. The mouse -; buttons will zoom in (left), recenter (center), or zoom out (right). -; The frame can be saved as a PostScript file by selecting File->WriteEPS -; from the left-hand corner. -; -; Make the same plot, but boxcar-smooth the spectrum and limit the -; wavelength range to [4000,5000] Angstroms: -; IDL> plotspec, 401, 100, nsmooth=10, xrange=[5000,6000] -; -; Some plates are observed on multiple nights. To select one of the two -; observations of plate 306: -; IDL> plotspec, 306, 20, mjd=51690 -; -; Loop through all the spectra for plate 401, interactively: -; IDL> plotspec, 401 -; -; Plot all the spectra from plate 401 to a single PostScript file: -; IDL> plotspec, 401, /psfile -; -; Plot all the spectra from plate 401 to 640 individual PostScript files: -; IDL> plotspec, 401, lindgen(640)+1, /psfile -; -; Plot a list of 3 objects, each with its own plate, MJD, and fiberid: -; IDL> plate = [400,400,401] -; IDL> mjd = [51820,51820,51788] -; IDL> fiberid = [10,11,20] -; IDL> plotspec, plate, mjd=mjd, fiberid -; -; BUGS: -; If the user interactively rescales in Y, then the labels for ORMASK -; and ANDMASK are no longer lined up vertically with the bit mask plot. -; -; PROCEDURES CALLED: -; dfpsclose -; dfpsplot -; djs_icolor() -; djs_oplot -; djs_plot -; djs_xyouts -; plotspec_image -; readspec -; soplot -; splot -; sdss_flagname() -; sxyouts -; synthspec() -; textoidl() -; -; INTERNAL SUPPORT ROUTINES: -; plotspec_mask -; plotspec1 -; -; REVISION HISTORY: -; 01-Sep-2000 Written by D. Schlegel, Princeton -;- -;------------------------------------------------------------------------------ -pro plotspec_mask, wave, thismask, psfile=psfile, nolabel=nolabel, $ - _EXTRA=Extra - - bitlabel = sdss_flagname('SPPIXMASK', 2UL^32-1, /silent) - bitnum = where(bitlabel NE '', nlabel) - bitlabel = bitlabel[bitnum] - - for ilabel=0, nlabel-1 do begin - ypos = ( (ilabel+1) * !y.crange[1] + (nlabel-ilabel) * !y.crange[0] ) $ - / (nlabel+1) - ynorm = ( (ilabel+0.8) * !y.window[1] + (nlabel-ilabel) * !y.window[0] ) $ - / float(nlabel+1) - ipix = where((thismask AND 2L^bitnum[ilabel]) NE 0, npix) - if (npix GT 0) then begin - if (keyword_set(psfile)) then begin - djs_oplot, [wave[ipix]], [replicate(ypos,npix)], $ - _EXTRA=Extra - endif else begin - soplot, [wave[ipix]], [replicate(ypos,npix)], $ - _EXTRA=Extra - endelse - endif - if (NOT keyword_set(nolabel)) then begin - if (keyword_set(psfile)) then begin - djs_xyouts, !x.window[1], ynorm, bitlabel[ilabel]+' ', $ - charsize=1.4, align=1.0, _EXTRA=Extra, /normal - endif else begin - sxyouts, !x.window[1], ynorm, bitlabel[ilabel]+' ', $ - charsize=1.4, align=1.0, _EXTRA=Extra, /normal - endelse - endif - endfor - - return -end -;------------------------------------------------------------------------------ -pro plotspec1, plate, fiberid, mjd=mjd, znum=znum, zmanual=zmanual, $ - nsmooth=nsmooth1, zline=q_zline, nosyn=nosyn, noerr=noerr, sky=sky, $ - ormask=ormask, andmask=andmask, $ - psfile=psfile, xrange=passxr, yrange=passyr, noerase=noerase, $ - restframe=restframe, netimage=netimage, allexp=allexp, _EXTRA=Extra, $ - legacy=legacy, plates=plates - - cspeed = 2.99792458e5 - textcolor = 'green' - linecolor = 'magenta' - orcolor = 'yellow' - andcolor = 'red' - - readspec, plate, fiberid, mjd=mjd, znum=znum, flux=objflux, $ - wave=wave, plug=plug, zans=zans, _EXTRA=Extra, /silent, $ - legacy=legacy, plates=plates - if (NOT keyword_set(objflux)) then begin - print, plate, mjd, fiberid, $ - format='("Spectrum not found for plate=", i6, " MJD=", i5, " fiber=", i3)' - return - endif - if (keyword_set(allexp)) then begin - readonespec, plate, fiberid, mjd=mjd, wave=allwave, flux=allflux, $ - _EXTRA=Extra, /silent - ndim = size(allflux,/n_dimen) - if (ndim EQ 1) then nexp = 1 $ - else nexp = (size(allflux,/dimens))[1] - endif - if (keyword_set(restframe)) then begin - wave = wave / (1. + zans.z) - if (keyword_set(allwave)) then allwave = allwave / (1. + zans.z) - endif - if (NOT keyword_set(noerr)) then $ - readspec, plate, fiberid, mjd=mjd, flerr=objerr, _EXTRA=Extra, /silent, $ - legacy=legacy, plates=plates - if (keyword_set(zmanual)) then begin - readspec, plate, fiberid, mjd=mjd, invvar=objivar, loglam=loglam, $ - objhdr=hdr, _EXTRA=Extra, /silent, legacy=legacy, plates=plates - if (zmanual[0] LT 1.) then eigenfile = 'spEigenGal-*.fits' $ - else eigenfile = 'spEigenQSO-*.fits' - npoly = 0 - if (n_elements(zmanual) EQ 1) then zrange=[zmanual,zmanual] $ - else zrange=zmanual[0:1] - res_manual = zfind(objflux, objivar, hdr=hdr, $ - eigenfile=eigenfile, npoly=npoly, zmin=zrange[0], zmax=zrange[1], $ - pspace=1, nfind=1, width=1) - synflux = synthspec(res_manual, loglam=loglam) - endif else begin - if (NOT keyword_set(nosyn)) then $ - readspec, plate, fiberid, mjd=mjd, znum=znum, synflux=synflux, $ - _EXTRA=Extra, /silent, legacy=legacy, plates=plates - endelse - if (keyword_set(sky)) then $ - readspec, plate, fiberid, mjd=mjd, sky=sky, _EXTRA=Extra, /silent, $ - legacy=legacy, plates=plates - if (keyword_set(ormask)) then $ - readspec, plate, fiberid, mjd=mjd, ormask=ormask, _EXTRA=Extra, /silent, $ - legacy=legacy, plates=plates - if (keyword_set(andmask)) then $ - readspec, plate, fiberid, mjd=mjd, andmask=andmask, _EXTRA=Extra, /silent, $ - legacy=legacy, plates=plates - if (keyword_set(zans) AND keyword_set(q_zline)) then $ - readspec, plate, fiberid, mjd=mjd, zline=zline, lineflux=lineflux, $ - _EXTRA=Extra, /silent, legacy=legacy, plates=plates - - if (keyword_set(nsmooth1)) then nsmooth = nsmooth1 $ - else nsmooth = 1 - - if (nsmooth GT 1) then begin - objflux = smooth(objflux, nsmooth) - if (keyword_set(allflux)) then begin - for iexp=0, nexp-1 do $ - allflux[*,iexp] = smooth(allflux[*,iexp], nsmooth) - endif - if (keyword_set(synflux)) then $ - synflux = smooth(synflux, nsmooth) - if (keyword_set(lineflux)) then $ - lineflux = smooth(lineflux, nsmooth) - endif - - ;plug.mag - targstring = strmatch(plug.objtype,'SKY*') ? 'SKY ' : '' - if (tag_exist(plug,'PRIMTARGET')) then $ - targstring += sdss_flagname('TARGET', plug.primtarget, /concat)+' ' - if (tag_exist(plug,'SECTARGET')) then $ - targstring += sdss_flagname('TTARGET', plug.sectarget, /concat)+' ' - if (tag_exist(plug,'BOSS_TARGET1')) then $ - targstring += sdss_flagname('BOSS_TARGET1', plug.boss_target1, /concat)+' ' - if (tag_exist(plug,'BOSS_TARGET2')) then $ - targstring += sdss_flagname('BOSS_TARGET2', plug.boss_target2, /concat)+' ' - ;- JEB adding eboss target bits - if (tag_exist(plug,'EBOSS_TARGET0')) then $ - targstring += sdss_flagname('EBOSS_TARGET0', plug.eboss_target0, /concat)+' ' - if (tag_exist(plug,'EBOSS_TARGET1')) then $ - targstring += sdss_flagname('EBOSS_TARGET1', plug.eboss_target1, /concat)+' ' - if (tag_exist(plug,'EBOSS_TARGET2')) then $ - targstring += sdss_flagname('EBOSS_TARGET2', plug.eboss_target2, /concat)+' ' - if (tag_exist(plug,'ANCILLARY_TARGET1')) then targstring += $ - sdss_flagname('ANCILLARY_TARGET1', plug.ancillary_target1, /concat)+' ' - if (tag_exist(plug,'ANCILLARY_TARGET2')) then targstring += $ - sdss_flagname('ANCILLARY_TARGET2', plug.ancillary_target2, /concat)+' ' - targstring = strtrim(targstring) ; get rid of trailing spaces - - csize = 1.75 - if (keyword_set(passyr)) then begin - yrange = passyr - ymin = yrange[0] - ymax = yrange[1] - endif else begin - if (keyword_set(synflux)) then $ - yrange = minmax(synflux) $ - else $ - yrange = minmax(objflux) - - if (yrange[0] EQ yrange[1]) then yrange = minmax(objflux) - ymin = (1.3 * yrange[0] - 0.3 * yrange[1]) < 0 - ymax = -0.3 * yrange[0] + 1.3 * yrange[1] - if (ymax EQ ymin) then ymax = ymin + 1 - yrange = [ymin, ymax] - endelse - if (keyword_set(passxr)) then xrange = passxr $ - else xrange = minmax(wave) - if (keyword_set(ormask) OR keyword_set(andmask)) then $ - xrange[1] = 1.15 * xrange[1] - 0.15 * xrange[0] - - title = 'Plate ' + strtrim(string(plate),2) $ - + ' Fiber ' + strtrim(string(fiberid),2) $ - + ' MJD=' + strtrim(string(mjd),2) - if (keyword_set(restframe)) then xtitle = 'Rest-Frame Wavelength [Ang]' $ - else xtitle = 'Observed Wavelength [Ang]' - if (keyword_set(psfile)) then begin - djs_plot, xrange, yrange, /nodata, xrange=xrange, yrange=yrange, $ - xtitle=xtitle, ytitle=TeXtoIDL('Flux [10^{-17} erg/s/cm^2/Ang]'), $ - title=title, charsize=csize, _EXTRA=KeywordsForSplot, /xstyle, /ystyle - if (keyword_set(allexp)) then begin - for iexp=0, nexp-1 do $ - djs_oplot, allwave[*,iexp], allflux[*,iexp], $ - psym=(nsmooth GT 1) ? 0 : 3, _EXTRA=KeywordsForSplot - endif else begin - djs_oplot, wave, objflux, _EXTRA=KeywordsForSplot - endelse - if (NOT keyword_set(noerr)) then $ - djs_oplot, wave, objerr, color='red', _EXTRA=KeywordsForSplot - if (keyword_set(sky)) then $ - djs_oplot, wave, sky, color='green', lw=2, _EXTRA=KeywordsForSplot - if (keyword_set(synflux)) then $ - djs_oplot, wave, synflux, color='blue', lw=2, _EXTRA=KeywordsForSplot - endif else begin - if (NOT keyword_set(noerase)) then $ - splot, xrange, yrange, /nodata, xrange=xrange, yrange=yrange, $ - xtitle=xtitle, ytitle=TeXtoIDL('Flux [10^{-17} erg/s/cm^2/Ang]'), $ - title=title, charsize=csize, _EXTRA=KeywordsForSplot - if (keyword_set(allexp)) then begin - for iexp=0, nexp-1 do $ - soplot, allwave[*,iexp], allflux[*,iexp], $ - _EXTRA=KeywordsForSplot - endif else begin - soplot, wave, objflux, _EXTRA=KeywordsForSplot - endelse - if (NOT keyword_set(noerr)) then $ - soplot, wave, objerr, color='red', _EXTRA=KeywordsForSplot - if (keyword_set(sky)) then $ - soplot, wave, sky, color='green', lw=2, _EXTRA=KeywordsForSplot - if (keyword_set(synflux)) then $ - soplot, wave, synflux, color='blue', lw=2, _EXTRA=KeywordsForSplot - endelse - - xpos = 0.9 * !x.window[0] + 0.1 * !x.window[1] - dypos = 0.05 * (!y.window[0] - !y.window[1]) - ypos = !y.window[1] + 1.5 * dypos - - if (keyword_set(zmanual) OR keyword_set(zans)) then begin - if (keyword_set(zmanual)) then begin - zstring = res_manual.tfile $ - + ' z=' + string(res_manual.z,format='(f8.5)') - endif else begin - zstring = zans.class + ' ' + zans.subclass - cz = zans.z * cspeed - if (abs(cz) LT 3000) then $ - zstring += ' cz=' + string(cz,format='(f6.0)') + ' km/s' $ - else $ - zstring += ' z=' + string(zans.z,format='(f8.5)') - if (zans.zwarning NE 0) then $ - zstring += ' (' $ - + sdss_flagname('ZWARNING', zans.zwarning, /concat) + ')' - if (keyword_set(znum)) then $ - zstring += ' (fit #' + strtrim(string(znum),2) + ')' - endelse - - if (keyword_set(psfile)) then $ - xyouts, xpos, ypos, zstring, $ - charsize=csize, color=djs_icolor(textcolor), /normal, $ - _EXTRA=KeywordsForSplot $ - else $ - sxyouts, xpos, ypos, zstring, $ - charsize=csize, color=textcolor, /normal, $ - _EXTRA=KeywordsForSplot - - ypos = ypos + dypos - - if (keyword_set(res_manual)) then thisrchi2 = res_manual.rchi2 $ - else thisrchi2 = zans.rchi2 - if (keyword_set(psfile)) then $ - xyouts, xpos, ypos, $ - TeXtoIDL('X^2_r =' + strtrim(string(thisrchi2, format='(f7.3)'),2)), $ - charsize=csize, color=djs_icolor(textcolor), /normal, $ - _EXTRA=KeywordsForSplot $ - else $ - sxyouts, xpos, ypos, $ - TeXtoIDL('X^2_r =' + strtrim(string(thisrchi2, format='(f7.3)'),2)), $ - charsize=csize, color=textcolor, /normal, $ - _EXTRA=KeywordsForSplot - endif - - if (keyword_set(lineflux)) then begin - if (keyword_set(psfile)) then $ - djs_oplot, wave, lineflux, color=linecolor, lw=2, _EXTRA=KeywordsForSplot $ - else $ - soplot, wave, lineflux, color=linecolor, lw=2, _EXTRA=KeywordsForSplot - - linewave = zline.linewave $ - * (1 + zline.linez * (keyword_set(restframe) EQ 0)) - ; Convert line sigma from km/sec to Angstroms - linesigma = linewave * zline.linesigma / cspeed - linepeak = zline.linecontlevel + zline.linearea / (sqrt(2*!pi) * linesigma) - for iline=0, n_elements(zline)-1 do begin - if (zline[iline].linearea_err GT 0) then begin - if (keyword_set(psfile)) then $ - xyouts, linewave[iline], linepeak[iline], $ - ' '+zline[iline].linename, orient=90, $ - charsize=0.75*csize, color=djs_icolor(linecolor), $ - _EXTRA=KeywordsForSplot $ - else $ - sxyouts, linewave[iline], linepeak[iline], $ - ' '+zline[iline].linename, orient=90, $ - charsize=0.75*csize, color=linecolor, $ - _EXTRA=KeywordsForSplot - endif - endfor - endif - - if (keyword_set(ormask)) then begin - plotspec_mask, wave, ormask, psfile=psfile, $ - psym=1, symsize=0.6, color=orcolor, $ - nolabel=keyword_set(andmask), _EXTRA=KeywordsForSplot - endif - - if (keyword_set(andmask)) then begin - plotspec_mask, wave, andmask, psfile=psfile, $ - psym=6, symsize=0.6, color=andcolor, _EXTRA=KeywordsForSplot - endif - - if (keyword_set(targstring)) then begin - ypos = ypos + dypos - if (keyword_set(psfile)) then $ - xyouts, xpos, ypos, 'Target = '+targstring, $ - charsize=csize, color=djs_icolor(textcolor), /normal, $ - _EXTRA=KeywordsForSplot $ - else $ - sxyouts, xpos, ypos, 'Target = '+targstring, $ - charsize=csize, color=textcolor, /normal, $ - _EXTRA=KeywordsForSplot - endif - - if (keyword_set(netimage) AND NOT keyword_set(psfile)) then begin - netstring = 'http://sdssmosaic.fnal.gov:8015/template/tsSingle.tml?run=' $ - + strtrim(string(plug.objid[0]),2) $ - + '&camcol=' + strtrim(string(plug.objid[2]),2) $ - + '&field=' + strtrim(string(plug.objid[3]),2) $ - + '&ra=' + strtrim(string(plug.ra),2) $ - + '&dec=' + strtrim(string(plug.dec),2) - spawn, '\netscape -remote "openURL(' + netstring + ')"' -print,netstring - endif - - return -end -;------------------------------------------------------------------------------ -pro plotspec, plate, fiberid, mjd=mjd, znum=znum, nsmooth=nsmooth, $ - zline=zline, nosyn=nosyn, noerr=noerr, sky=sky, $ - ormask=ormask, andmask=andmask, $ - psfile=psfile, xrange=xrange, yrange=yrange, noerase=noerase, $ - restframe=restframe, netimage=netimage, zwarning=zwarning, allspec=allspec, $ - _EXTRA=Extra, legacy=legacy, plates=plates, confid=confid - - if (n_params() LT 1) then begin - doc_library, 'plotspec' - return - endif - - quiet = !quiet - !quiet = 1 - - ;---------- - ; If MJD is not set, then find the MJD for each plate - - nplate = n_elements(plate) - if (NOT keyword_set(mjd)) then begin - mjd = lonarr(nplate) - for iplate=0, nplate-1 do begin - mjd1 = 0 - readspec, plate[iplate], mjd=mjd1, _EXTRA=Extra, /silent - if (NOT keyword_set(mjd1)) then begin - print, 'No MJD found for plate ', plate[iplate] - !quiet = quiet - return - endif - mjd[iplate] = mjd1 - endfor - endif else begin - if (n_elements(mjd) NE nplate) then begin - print, 'Number of elements in PLATE and MJD do not agree' - !quiet = quiet - return - endif - endelse - - ;---------- - ; If /ZWARNING is set, then find the flagged fibers to plot. - - if (keyword_set(zwarning)) then begin - if (keyword_set(fiberid)) then begin - print, 'FIBERID and /ZWARNING cannot both be set.' - !quiet = quiet - return - endif - - for iplate=0L, nplate-1L do begin - readspec, plate[iplate], mjd=mjd[iplate], $ - _EXTRA=Extra, /silent, zans=zans - if (NOT keyword_set(zans)) then begin - print, 'No spZ file found for selecting ZWARNING flags' - !quiet = quiet - return - endif - indx = where((zans.zwarning AND 1) EQ 0 AND zans.zwarning NE 0, nthis) - if (nthis GT 0) then begin - if (NOT keyword_set(fiberid)) then begin - platelist = replicate(plate[iplate], nthis) - mjdlist = replicate(mjd[iplate], nthis) - fiberid = zans[indx].fiberid - endif else begin - platelist = [platelist, replicate(plate[iplate], nthis)] - mjdlist = [mjdlist, replicate(mjd[iplate], nthis)] - fiberid = [fiberid, zans[indx].fiberid] - endelse - endif - endfor - if (NOT keyword_set(fiberid)) then begin - print, 'No non-sky fibers with ZWARNING flag set' - !quiet = quiet - return - endif - nfiber = n_elements(fiberid) - print, 'Selecting ', nfiber, ' non-sky fibers with ZWARNING flag set' - endif - - ;---------- - ; Set FIBERID to [1,...,NFIBER] (for each plate) if not set. - ; - ; If writing to a PostScript file, then all plots are in the same file - ; either if PSFILE is that file name, or if FIBERID is not specified - ; (and then all spectra are being plotted). - - if (NOT keyword_set(fiberid)) then begin - readspec, plate, mjd=mjd, 0*plate+1, nfiber=nfiber_tmp, $ - _EXTRA=Extra, /silent - nfiber_tot = long(total(nfiber_tmp)) - if (nfiber_tot EQ 0) then begin - print, 'No fibers found' - !quiet = quiet - return - endif - platelist = lonarr(nfiber_tot) - mjdlist = lonarr(nfiber_tot) - fiberid = lonarr(nfiber_tot) - j = 0L - for iplate=0L, nplate-1L do begin - platelist[j:j+nfiber_tmp[iplate]-1] = plate[iplate] - mjdlist[j:j+nfiber_tmp[iplate]-1] = mjd[iplate] - fiberid[j:j+nfiber_tmp[iplate]-1] = lindgen(nfiber_tmp[iplate]) + 1 - j += nfiber_tmp[iplate] - endfor - if (keyword_set(psfile)) then begin - q_onefile = 1 - psfilename = string(plate_to_string(plate[0]), mjd[0], $ - format='("spec-",a,"-",i5.5,".ps")') - endif - endif - -; if (min(fiberid) LT 1 OR max(fiberid) GT 640) then begin -; print, 'Invalid FIBERID (must be between 1 and 640)' -; return -; endif - - ;---------- - ; If FIBERID is specified, and writing to a PostScript file, - ; then open only one PS file for all plots. - - nfiber = n_elements(fiberid) - if (size(psfile,/tname) EQ 'STRING' AND nfiber GT 1) then begin - psfilename = psfile - q_onefile = 1 - endif - - ;---------- - ; If /ZWARNING is not set, then construct the PLATELIST,MJDLIST. - - if (n_elements(platelist) EQ 0) then begin - if (nplate EQ 1) then begin - platelist = replicate(plate, nfiber) - mjdlist = replicate(mjd, nfiber) - endif else begin - platelist = plate - mjdlist = mjd - endelse - endif else if (n_elements(platelist) NE n_elements(fiberid)) then begin - print, 'Number of elements in PLATE and FIBERID do not agree.' - !quiet = quiet - return - endif - - ;---------- - ; Loop over each plot - - ifiber = 0L - while (ifiber LT nfiber) do begin - - ;---------- - ; Open the PostScript file if appropriate - - if (keyword_set(psfile)) then begin - if (NOT keyword_set(q_onefile)) then begin - if (size(psfile,/tname) EQ 'STRING') then $ - psfilename = psfile $ - else $ - psfilename = string(plate_to_string(platelist[ifiber]), mjdlist[ifiber], $ - fiberid[ifiber], format='("spec-", a,"-",i5.5,"-",i3.3,".ps")') - endif - - if (NOT keyword_set(q_onefile) OR ifiber EQ 0) then begin - dfpsplot, psfilename, /color, /square - endif - endif - - plotspec1, platelist[ifiber], fiberid[ifiber], mjd=mjdlist[ifiber], $ - znum=znum, zmanual=zmanual, nsmooth=nsmooth, zline=zline, $ - nosyn=nosyn, noerr=noerr, $ - sky=sky, ormask=ormask, andmask=andmask, psfile=psfile, $ - xrange=xrange, yrange=yrange, noerase=noerase, netimage=netimage, $ - restframe=restframe, allexp=allexp, _EXTRA=Extra, $ - legacy=legacy, plates=plates - - if (keyword_set(psfile)) then begin - if (NOT keyword_set(q_onefile) OR ifiber EQ nfiber-1) then dfpsclose - ifiber = ifiber + 1 - endif else begin - if (ifiber LT nfiber-1) then begin - if (keyword_set(nsmooth)) then $ - sstring = ' (currently=' + strtrim(string(nsmooth),2) + ')' $ - else $ - sstring = '' - - print, 'Press b=back one fiber' - print, ' p=select new plate' - print, ' f=select new fiber number' - print, ' n=change which PCA-fit to plot' - print, ' q=quit (and enter interactive mode for this plot)' - print, ' s=change boxcar smoothing' + sstring - print, ' x=change X plotting range' - print, ' y=change Y plotting range' - print, ' z=manual z' - print, ' v=view reconstructed frame' - print, ' any other key=forward' - - cc = strupcase(get_kbrd(1)) - print, cc - case cc of - 'V': begin - if (keyword_set(getenv('PHOTOOP_DIR'))) then begin - readspec, platelist[ifiber], fiberid[ifiber], $ - mjd=mjdlist[ifiber], zans=zans, _EXTRA=Extra, /silent, $ - legacy=legacy, plates=plates - plotspec_image, ra=zans.plug_ra, dec=zans.plug_dec, $ - cutout=300, /calibrate, /register, /allid - endif else begin - print, 'Need to set up photoop product to display images' - endelse - end - 'B': begin - ifiber = (ifiber - 1) > 0 - zmanual = 0. - end - 'P': begin - read, plate, mjd, prompt='Enter new plate and MJD (enter 0 for unknown MJD): ' - if (NOT keyword_set(mjd)) then $ - readspec, plate, mjd=mjd, _EXTRA=Extra, /silent, $ - legacy=legacy, plates=plates - if (NOT keyword_set(mjd)) then begin - print, 'MJD not found for plate ', plate - !quiet = quiet - return - endif - readspec, plate, mjd=mjd, nfiber=nfiber, $ - _EXTRA=Extra, /silent, $ - legacy=legacy, plates=plates - platelist = replicate(plate,nfiber) - mjdlist = replicate(mjd,nfiber) - zmanual = 0. - end - 'N': begin - read, znum, prompt='Enter 1=best redshift, 2=2nd best, ...: ' - znum = long(znum) > 0 - zmanual = 0. - end - 'F': begin - read, newfiber, prompt='Enter new fiber number: ' - ifiber = ((long(newfiber)-1) > 0) < (nfiber-1) - zmanual = 0. - end - 'Q': ifiber = nfiber - 'S': begin - read, nsmooth, prompt='Enter boxcar smoothing width (0=none): ' - nsmooth = long(nsmooth) > 0 - end - 'X': begin - read, xmin, xmax, prompt='Enter new X range values (0 0=full range): ' - if (xmin EQ 0 AND xmax EQ 0) then xrange = 0 $ - else xrange = [xmin, xmax] - end - 'Y': begin - read, ymin, ymax, prompt='Enter new Y range values (0 0=full range): ' - if (ymin EQ 0 AND ymax EQ 0) then yrange = 0 $ - else yrange = [ymin, ymax] - end - 'Z': begin - read, zmanual, prompt='Enter redshift guess (zmin zmax)...: ' - end - else: begin - ifiber = ifiber + 1 - zmanual = 0. - end - endcase - endif else begin - ifiber = nfiber - endelse - endelse - endwhile - - !quiet = quiet - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/plotspec_image.pro b/pro/spec1d/plotspec_image.pro deleted file mode 100644 index 9502d4800..000000000 --- a/pro/spec1d/plotspec_image.pro +++ /dev/null @@ -1,44 +0,0 @@ -;+ -; NAME: -; plotspec_image -; -; PURPOSE: -; Procedure to display an image, launched from PLOTSPEC -; -; CALLING SEQUENCE: -; plotspec_image, ra, dec, [ _EXTRA= ] -; -; INPUTS: -; ra - Right ascension (degrees) -; dec - Declination (degrees) -; -; OPTIONAL INPUTS: -; _EXTRA - Keywords to pass to FPBIN_TO_FRAME -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; -; EXAMPLES: -; -; BUGS: -; -; PROCEDURES CALLED: -; atv -; fpbin_to_frame() -; -; REVISION HISTORY: -; 30-Sep-2005 Written by D. Schlegel, Princeton -;- -;------------------------------------------------------------------------------ -pro plotspec_image, ra, dec, _EXTRA=EXTRA - - fpframe = fpbin_to_frame(ra=ra, dec=dec, rerun='*', $ - _EXTRA=EXTRA, hdr=hdr) - atv, fpframe, head=hdr - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/plotspecqa.pro b/pro/spec1d/plotspecqa.pro deleted file mode 100644 index 0418aa66e..000000000 --- a/pro/spec1d/plotspecqa.pro +++ /dev/null @@ -1,74 +0,0 @@ -;+ -; NAME: -; plotspecqa -; -; PURPOSE: -; Wrapper for PLOTSPEC to plot standard stars and/or sky spectra. -; -; CALLING SEQUENCE: -; plotspecqa, [ /standards, /skies, _EXTRA= ] -; -; INPUTS: -; plate - Plate number(s) -; -; OPTIONAL INPUTS: -; standards - Select standard stars for plotting -; skies - Select sky spectra for plotting -; _EXTRA - Kewords for PLOTSPEC -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; -; EXAMPLES: -; -; BUGS: -; -; PROCEDURES CALLED: -; plotspec -; readspec -; splog -; -; REVISION HISTORY: -; 15-Mar-2006 Written by D. Schlegel, Princeton -;- -;------------------------------------------------------------------------------ -pro plotspecqa, plate, fiberid, mjd=mjd, standards=standards, skies=skies, $ - _EXTRA=EXTRA - - if (keyword_set(standards) OR keyword_set(skies)) then begin - for iplate=0L, n_elements(plate)-1L do begin - if (keyword_set(mjd)) then thismjd = mjd[iplate] $ - else thismjd = 0 - readspec, plate[iplate], fiberid, mjd=thismjd, plugmap=plug - if (NOT keyword_set(plug)) then begin - splot, 'No plugmap structure found for the selected plate' - return - endif - qstandards = keyword_set(standards) $ - AND (strmatch(plug.objtype,'SPECTROPHOTO_STD*') $ - OR strmatch(plug.objtype,'REDDEN_STD*')) - qskies = keyword_set(skies) AND strmatch(plug.objtype,'SKY*') - iplot = where(qstandards OR qskies, nplot) - if (nplot EQ 0) then begin - splog, 'No standard stars or skies selected' - return - endif - if (keyword_set(standards)) then $ - splog, 'Selected ', fix(total(qstandards)), $ - ' standards on plate ', plate[iplate] - if (keyword_set(skies)) then $ - splog, 'Selected ', fix(total(qskies)), $ - ' skies on plate ', plate[iplate] - plotspec, plate[iplate], plug[iplot].fiberid, mjd=thismjd, $ - _EXTRA=EXTRA - endfor - endif else begin - plotspec, plate, fiberid, mjd=mjd, _EXTRA=EXTRA - endelse - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/plotzrepeats.pro b/pro/spec1d/plotzrepeats.pro deleted file mode 100644 index d00f1e818..000000000 --- a/pro/spec1d/plotzrepeats.pro +++ /dev/null @@ -1,99 +0,0 @@ -; Routine to compare redshift errors between multiple observations -; of the same plate. -pro plotzrepeats, plate, mjd - - if (NOT keyword_set(plate)) then plate = 406 - if (NOT keyword_set(mjd)) then mjd = [51817,51869,51876,51900] - - cspeed = 3.e5 - dtheta = 1.0/3600. - - plotfile = string(plate_to_string(plate), format='("zrepeat-", a,".ps")') - - ;---------- - ; Read in all MJDs for this plate - - nmjd = n_elements(mjd) - if (nmjd LE 1) then begin - print, 'No duplicate MJDs for this plate' - return - endif - readspec, replicate(plate,nmjd), mjd=mjd, zans=zans, plug=plug - - ;---------- - ; Find all pairs of the same object - - nmatch = djs_angle_match(zans.plug_ra, zans.plug_dec, dtheta=dtheta, $ - mcount=mcount, mindx=mindx, mdist=mdist, mmax=nmjd) - - ii = where(mindx NE -1, npair) - indx1 = long(ii / nmjd) - indx2 = mindx[ii] - - ;---------- - ; Loop over each class of object - - !p.multi = [0,1,2] - csize = 2.0 - dfpsplot, plotfile - - classlist = ['GALAXY', 'STAR', 'QSO'] - - for iclass=0, n_elements(classlist)-1 do begin - jj = where(strmatch(zans[indx1].class, classlist[iclass]+'*') $ - AND strmatch(zans[indx2].class, classlist[iclass]+'*') $ - AND zans[indx1].z_err GT 0 $ - AND zans[indx2].z_err GT 0 $ - AND zans[indx1].zwarning EQ 0 $ - AND zans[indx2].zwarning EQ 0, nj) - - if (classlist[iclass] EQ 'QSO') then yrange=[-2000,2000] $ - else if (classlist[iclass] EQ 'STAR') then yrange = [-200,200] $ - else yrange = [-200,200] - - if (nj GT 1) then begin - - if (nj LT 1000) then psym = 4 $ - else psym = 3 - - vdiff = (zans[indx1[jj]].z - zans[indx2[jj]].z) * cspeed - verr = sqrt(zans[indx1[jj]].z_err^2 + zans[indx2[jj]].z_err^2) $ - * cspeed - chi = vdiff / verr - cmed = median(abs(chi)) - - print, classlist[iclass] - abschi = abs(chi) - chisort = abschi[sort(abschi)] - chi67 = chisort[fix(0.67*n_elements(chisort))] - print, 'Median(|chi|) = ', cmed - print, 'Median(verr) = ', median(verr) - print, '|Chi| at 67% = ', chi67 - - plothist, chi, bin=0.20, xrange=[-8,8], /xstyle, $ - xtitle=textoidl('\chi = (z_1-z_2) / z_{err}'), $ - ytitle='Number', charsize=csize, $ - title=classlist[iclass]+' Redshift Errors from Repeats' - - djs_xyouts, -7, 0.9*!y.crange[1], charsize=0.75*csize, $ - 'Median(cz_{err})=' + string(median(verr), format='(f6.1)') + ' km/s' - djs_xyouts, -7, 0.8*!y.crange[1], charsize=0.75*csize, $ - '\chi at 67% =' + string(chi67, format='(f5.2)') - -; xplot = plug[indx1[jj]].sn_median - xplot = plug[indx1[jj]].mag[2] - plot, xplot, vdiff, $ - psym=psym, symsize=0.5, yrange=yrange, charsize=csize, $ -; xtitle='Median S/N per pixel', $ - xtitle='r-mag', xrange=[15,22], /xstyle, $ - ytitle=textoidl('(z_1-z_2) [km/s]'), $ - title=classlist[iclass]+' Redshift Errors from Repeats' - djs_oplot, !x.crange, [0,0] - djs_oploterr, xplot, vdiff, $ - yerr=verr - endif - endfor - - dfpsclose - -end diff --git a/pro/spec1d/poly_fill.pro b/pro/spec1d/poly_fill.pro deleted file mode 100644 index 3dc7ab79f..000000000 --- a/pro/spec1d/poly_fill.pro +++ /dev/null @@ -1,35 +0,0 @@ -;+ -; NAME: -; poly_fill -; -; PURPOSE: -; Compare various interpolation methods. -; -; VERSION: -; $Id: poly_fill.pro 50641 2014-11-17 20:39:17Z weaver $ -;- -PRO poly_fill, lambda, flux, invvar, spectrum - l = 250 - goodpts = WHERE( invvar[*,spectrum] GT 0, ngood ) - nflux = N_ELEMENTS(flux[*,spectrum]) - mingood = MIN(goodpts) - maxgood = MAX(goodpts) - smoothflux = SMOOTH(flux[*,spectrum],l,/EDGE_TRUNCATE) - interpflux = djs_maskinterp(flux[*,spectrum],invvar[*,spectrum] EQ 0,/const) - pixels = LINDGEN(nflux) - newflux = interpflux - IF mingood GT 0 THEN BEGIN - damp1 = FLOAT(mingood < l) - newflux *= 0.5*(1.0+ERF(FLOAT(pixels-mingood)/damp1)) - ENDIF - IF maxgood LT nflux-1 THEN BEGIN - damp2 = FLOAT(maxgood < l) - newflux *= 0.5*(1.0+ERF(FLOAT(maxgood-pixels)/damp2)) - ENDIF - djs_plot, lambda, flux[*,spectrum], color='white', xtitle='Wavelength [\AA]', $ - ytitle='Flux [10^{-17} erg cm^{-2} s^{-1} \AA^{-1}]' - djs_oplot, lambda, interpflux, color='red' - djs_oplot, lambda, newflux, color='green' - djs_oplot, lambda, smoothflux, color='blue' - RETURN -END diff --git a/pro/spec1d/primfac.pro b/pro/spec1d/primfac.pro deleted file mode 100644 index 5b9c9b9e8..000000000 --- a/pro/spec1d/primfac.pro +++ /dev/null @@ -1,17 +0,0 @@ -; D. Finkbeiner -; Prime factor routine for calculating FFT efficiency. -; totally bonehead. -PRO primfac, n_in - - n = float(n_in) - FOR i=2L, n+1 DO BEGIN - WHILE n/i-long(n/i) EQ 0.0 DO BEGIN - n = n/i - print, i, format='(I6,$)' - ENDWHILE - - ENDFOR - print - return -END - diff --git a/pro/spec1d/qsorebin.pro b/pro/spec1d/qsorebin.pro deleted file mode 100644 index 713e00648..000000000 --- a/pro/spec1d/qsorebin.pro +++ /dev/null @@ -1,59 +0,0 @@ -;------------------------------------------------------------------------------ -pro qsorebin - - objdloglam = 1.d-4 - wavemin = 525. - wavemax = 9300. - minuse = 40 - - ;---------- - ; Read the template files - ; Assume that the wavelength binning is the same as for the objects - ; in log-wavelength. - - tfile = filepath('qso.template', $ - root_dir=getenv('IDLSPEC2D_DIR'), subdirectory='templates') - djs_readcol, tfile, twave, tflux, terr, lq, uq, usemask, $ - format='(D,F,F,F,F,L)' - tloglam0 = alog10(twave[0]) - tdloglam = alog10(twave[1] / twave[0]) - igood = where(terr GT 0) - tivar = 0 * terr - tivar[igood] = 1. / (terr[igood])^2 - - ;---------- - ; Set the new wavelength mapping here... - - newloglam = wavevector(alog10(wavemin), alog10(wavemax), binsz=objdloglam) - - ;---------- - ; Re-bin to the same pixel bin size. - - newflux = interpol(tflux, alog10(twave), newloglam) - newivar = interpol(tivar, alog10(twave), newloglam) - newuse = interpol(float(usemask), alog10(twave), newloglam) - - ;---------- - ; Fill in bad data with a running median of good data - - qgood = newuse GE minuse AND newivar GT 0 - igood = where(qgood, ngood) - ibad = where(qgood EQ 0, nbad) - medflux = 0 * newflux - if (nbad GT 0) then begin - medflux[igood] = djs_median(newflux[igood], width=51, boundary='nearest') - medflux = djs_maskinterp(medflux, qgood EQ 0, /const) - newflux[ibad] = medflux[ibad] - endif - - ;---------- - ; Write output file - - sxaddpar, hdr, 'COEFF0', newloglam[0] - sxaddpar, hdr, 'COEFF1', objdloglam - mwrfits, float(newflux), 'spEigenQSO.fits', hdr, /create - - return -end -;------------------------------------------------------------------------------ - diff --git a/pro/spec1d/readspec_footprint.pro b/pro/spec1d/readspec_footprint.pro deleted file mode 100644 index 2e18fcb0a..000000000 --- a/pro/spec1d/readspec_footprint.pro +++ /dev/null @@ -1,131 +0,0 @@ -;+ -; NAME: -; readspec_footprint -; -; PURPOSE: -; Routine for reading 1D spectro outputs on a given plate footprint -; -; CALLING SEQUENCE: -; readspec, plate, [mjd=, topdir=, /best, /silent, $ -; zans=, plugmap=, tsobj= ] -; -; INPUTS: -; plate - Plate number -; -; OPTIONAL INPUTS: -; mjd - MJD number(s); if not set, then select the most recent -; data for this plate (largest MJD). -; best - If set, then select the best observation of each object, -; using the same algorithm to choose the primary observation -; in the PLATEMERGE routine (for spAll files). -; silent - If set, then call MRDFITS with /SILENT. -; topdir - Top-level directory for data; default to the environment -; variable $BOSS_SPECTRO_REDUX. -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; mjd - If not specified, then this returns the MJD of PLATE -; zans - Redshift output structure [NFIBER] -; plugmap - Plug-map entries [NFIBER] -; tsobj - tsObj-structure output [NFIBER] -; -; COMMENTS: -; Read all of the SDSS spectroscopic observations within 1.49 degrees -; of the center of the specified plate. If /BEST is set, then trim -; to the best observation of each object. -; -; EXAMPLES: -; Read all the SDSS spectroscopic observations on the SDSS-3 test -; plate 2634, trimming to the best observations of each object: -; IDL> readspec_footprint, 2634, zans=zans, /best -; -; BUGS: -; -; PROCEDURES CALLED: -; djs_diff_angle() -; platelist -; readspec -; spheregroup() -; sxpar() -; -; REVISION HISTORY: -; 24-Apr-2007 Written by David Schlegel, LBL -;- -;------------------------------------------------------------------------------ -pro readspec_footprint, platenum, mjd=mjd, best=best, silent=silent, $ - zans=zans, plugmap=plug, tsobj=tsobj - - if (n_elements(platenum) EQ 0) then $ - message, 'PLATE is not specified or is not a scalar' - - ; Find all plates that may overlap this first plate - readspec, platenum, mjd=mjd, objhdr=hdr, silent=silent - if (NOT keyword_set(hdr)) then $ - message, 'Plate does not exist ' + string(platenum) - ra = sxpar(hdr, 'RA') - dec = sxpar(hdr, 'DEC') - platelist, plist=plist - plist = plist[where(strmatch(plist.status1d,'Done*'))] -; plist = plist[where(strmatch(plist.status1d,'Done*') $ -; AND (strmatch(plist.platequality,'good*') $ -; OR strmatch(plist.platequality,'marginal*')))] - adist = djs_diff_angle(plist.ra, plist.dec, ra, dec) - plist = plist[where(adist LE 3.)] - - ; Read all these plates - readspec, plist.plate, mjd=plist.mjd, zans=zans, silent=silent - if (arg_present(plug)) then $ - readspec, plist.plate, mjd=plist.mjd, plug=plug, silent=silent - if (arg_present(tsobj)) then $ - readspec, plist.plate, mjd=plist.mjd, tsobj=tsobj, silent=silent - - ; Store the plate quality for each object - pquality = strarr(n_elements(zans)) - for iplate=0L, n_elements(plist)-1L do begin - indx = where(zans.plate EQ plist[iplate].plate $ - AND zans.mjd EQ plist[iplate].mjd) - pquality[indx] = plist[iplate].platequality - endfor - - ; Trim to only those within the footprint of the requested plate - adist = djs_diff_angle(zans.plug_ra, zans.plug_dec, ra, dec) - indx = where(adist LT 1.49) - zans = zans[indx] - if (arg_present(plug)) then plug = plug[indx] - if (arg_present(tsobj)) then tsobj = tsobj[indx] - pquality = pquality[indx] - - ; Trim to the best observation of each object, using the same - ; scoring algorithm as in PLATEMERGE - if (keyword_set(best)) then begin - specprimary = bytarr(n_elements(zans)) - score = 4 * (zans.sn_median GT 0) $ - + 2 * (strmatch(pquality,'good*') EQ 1) $ - + 1 * (zans.zwarning EQ 0) $ - + (zans.sn_median>0) / max(zans.sn_median+1.) - dtheta = 2.0 / 3600. - ingroup = spheregroup(zans.plug_ra, zans.plug_dec, dtheta, $ - multgroup=multgroup, firstgroup=firstgroup, nextgroup=nextgroup) - for j=0L, n_elements(firstgroup)-1L do begin - if (firstgroup[j] NE -1) then begin - if (multgroup[j] EQ 1) then begin - specprimary[firstgroup[j]] = 1 - endif else begin - indx = lonarr(multgroup[j]) - indx[0] = firstgroup[j] - for k=0L, multgroup[j]-2L do indx[k+1] = nextgroup[indx[k]] - foo = max(score[indx], ibest) - specprimary[indx[ibest]] = 1 - endelse - endif - endfor - indx = where(specprimary) - zans = zans[indx] - if (arg_present(plug)) then plug = plug[indx] - if (arg_present(tsobj)) then tsobj = tsobj[indx] - endif - - return -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/redindices.pro b/pro/spec1d/redindices.pro deleted file mode 100644 index de5677a2a..000000000 --- a/pro/spec1d/redindices.pro +++ /dev/null @@ -1,167 +0,0 @@ -;------------------------------------------------------------------------------ -; De-redshift and rebin spectra to specified wavelengths [WAVE1,WAVE2] -pro redindices_rebin, objloglam, objflux, objivar, zfit, $ - wave1, wave2, newloglam, newflux, newivar - - if (size(objflux,/n_dimen) EQ 1) then nobj = 1 $ - else nobj = (size(objflux,/dimens))[1] - - dloglam = 1.d-4 ; fix the wavelength spacing - loglam1 = alog10(wave1) - 5 * dloglam ; pad with 5 pixels on left - loglam2 = alog10(wave2) + 5 * dloglam ; pad with 5 pixels on right - newloglam = loglam1 + dindgen((loglam2-loglam1) / dloglam) * dloglam - nnew = n_elements(newloglam) - newflux = fltarr(nnew,nobj) - newivar = fltarr(nnew,nobj) - - ;---------- - ; Shift each spectra to z=0 and sample at the output wavelengths - - logshift = alog10(1.d + zfit) - for iobj=0, nobj-1 do begin - indx = where(objloglam[*,iobj] GT 0) -print, format='("Shift object ",i5," of ",i5,a1,$)', $ - iobj, nobj, string(13b) - combine1fiber, objloglam[indx,iobj]-logshift[iobj], $ - objflux[indx,iobj], objivar[indx,iobj], $ - newloglam=newloglam, binsz=dloglam, newflux=flux1, newivar=ivar1 - newflux[*,iobj] = flux1 - newivar[*,iobj] = ivar1 - endfor -print,'' - - return -end - -;------------------------------------------------------------------------------ -function redindices_mean, flux, ivar, mnerr=mnerr - - if (size(flux,/n_dimen) EQ 1) then nobj = 1 $ - else nobj = (size(flux,/dimens))[1] - - mn = fltarr(nobj) - mnerr = fltarr(nobj) - - for iobj=0, nobj-1 do begin - wtot = total(ivar[*,iobj]) - mn[iobj] = total(flux[*,iobj] * ivar[*,iobj]) / (wtot + (wtot EQ 0)) - mnerr[iobj] = 1. / sqrt(wtot + (wtot EQ 0)) - endfor - - return, mn -end - -;------------------------------------------------------------------------------ -function redindices_ew, objloglam, objflux, objivar, zfit, $ - lowave, midwave, hiwave - - redindices_rebin, objloglam, objflux, objivar, zfit, $ - lowave[0], hiwave[1], newloglam, newflux, newivar - ii = where(newloglam GE alog10(lowave[0]) AND newloglam LE alog10(lowave[1])) - mean1 = redindices_mean(newflux[ii,*], newivar[ii,*], mnerr=mnerr1) - ii = where(newloglam GE alog10(midwave[0]) $ - AND newloglam LE alog10(midwave[1])) - mean2 = redindices_mean(newflux[ii,*], newivar[ii,*], mnerr=mnerr2) - ii = where(newloglam GE alog10(hiwave[0]) AND newloglam LE alog10(hiwave[1])) - mean3 = redindices_mean(newflux[ii,*], newivar[ii,*], mnerr=mnerr3) - continuum = (mean1 + mean3) / 2. - ew = (mean2 - continuum) * (midwave[1] - midwave[0]) / continuum - - return, ew -end - -;------------------------------------------------------------------------------ -function redindices, plate, mjd=mjd - -print, '--> WORKING ON PLATE ', plate - maglimit = 18.0 ; de-reddened r-band magnitude limit - - ;---------- - ; Pre-select galaxies - - readspec, plate, mjd=mjd, zans=zans, plug=plug, tsobj=tsobj - indx = where(strtrim(zans.class) EQ 'GALAXY' AND zans.zwarning EQ 0 $ - AND plug.mag[2] - tsobj.reddening[2,*] LT maglimit) - - ;---------- - ; Read the 2D output file - - readspec, plate, plug[indx].fiberid, mjd=mjd, $ - flux=objflux, invvar=objivar, loglam=objloglam, $ - andmask=andmask, ormask=ormask - zans = zans[indx] - tsobj = tsobj[indx] - - ;---------- - ; Mask around bright sky lines - - objivar = skymask(objivar, andmask, ormask) -andmask = 0 ; Free memory -ormask = 0 ; Free memory - - ;---------- - ; Measure the 4000-Ang break. - -print, 'Measuring 4000-Ang break' - lowave = [3800.,3900.] - hiwave = [4025.,4200.] - redindices_rebin, objloglam, objflux, objivar, zans.z, $ - lowave[0], hiwave[1], newloglam, newflux, newivar - ii = where(newloglam GE alog10(lowave[0]) AND newloglam LE alog10(lowave[1])) - mean1 = redindices_mean(newflux[ii,*], newivar[ii,*], mnerr=mnerr1) - ii = where(newloglam GE alog10(hiwave[0]) AND newloglam LE alog10(hiwave[1])) - mean2 = redindices_mean(newflux[ii,*], newivar[ii,*], mnerr=mnerr2) - dinv4000 = mean1 / mean2 - - ;---------- - ; Measure the Halpha+[NII] equiv. width - -print, 'Measuring H-alpha EW' - thiswave = 6564. - lowave = thiswave + [-75,-35] - midwave = thiswave + [-35,35] - hiwave = thiswave + [35,75] - ew_halpha = redindices_ew(objloglam, objflux, objivar, zans.z, $ - lowave, midwave, hiwave) - - ;---------- - ; Measure the Mg2 equiv. width - -print, 'Measuring Mg2 EW' - lowave = [4897,4958] - midwave = [5156,5197] - hiwave = [5303,5367] - airtovac, lowave - airtovac, midwave - airtovac, hiwave - ew_mg2 = redindices_ew(objloglam, objflux, objivar, zans.z, $ - lowave, midwave, hiwave) - - ;---------- - ; Now select our sample - - iselect = where(dinv4000 LT 0.6 AND ew_halpha LT 3.0) - - modelcolor = tsobj.counts_model[0:3,*] - tsobj.counts_model[1:4,*] - - redstruct = create_struct( $ - 'modelcolor', fltarr(4), $ - 'redsample', 0L, $ - 'dinv4000', 0.0, $ - 'ew_halpha', 0.0, $ - 'ew_mg2', 0.0, $ - zans[0], $ - tsobj[0], $ - name='REDSTRUCT' ) - redstruct = replicate(redstruct, n_elements(tsobj)) - redstruct.modelcolor = modelcolor - redstruct.dinv4000 = dinv4000 - redstruct.ew_halpha = ew_halpha - redstruct.ew_mg2 = ew_mg2 - redstruct[iselect].redsample = 1 - struct_assign, zans, redstruct, /nozero - struct_assign, tsobj, redstruct, /nozero - - return, redstruct -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/redmonsterlist.pro b/pro/spec1d/redmonsterlist.pro deleted file mode 100644 index 3e1cf453d..000000000 --- a/pro/spec1d/redmonsterlist.pro +++ /dev/null @@ -1,93 +0,0 @@ -; List of number of galaxies and quasars per plate, the number of redshifts -; that differ from prior reductions, and the fraction of various 2D flags set. - -;------------------------------------------------------------------------------ -pro redmonsterlist, olddir=olddir - -; if (NOT keyword_set(olddir)) then olddir = '/u/dss/spectro_v4_7' - - splog, filename='monsterlist.log' - - platelist, plist=plist - idone = where(strtrim(plist.statuscombine,2) EQ 'Done') - plist = plist[idone] - - splog, 'PLATE MJD %Gal %QSO Whoppr Reject ' $ - + 'Scattr X-Talk BrtSky BadFlx BadChi Monstr' $ - + (keyword_set(olddir) ? ' Ndiff ':''), /noname - - splog, '----- ----- ------ ------ ------ ------ ' $ - + '------ ------ ------ ------ ------ ------' $ - + (keyword_set(olddir) ? ' ------':''), /noname - - for i=0, n_elements(plist)-1 do begin - readspec, plist[i].plate, mjd=plist[i].mjd, $ - plug=plug, ormask=ormask, zans=zans, /silent - - if (keyword_set(olddir)) then $ - readspec, plist[i].plate, mjd=plist[i].mjd, $ - zans=oldz, topdir=olddir, /silent - - if (keyword_set(zans) AND keyword_set(oldz)) then begin - zdiff = 3.e5 * abs(zans.z - oldz.z) - junk = where(zdiff GT 500. AND zans.zwarning EQ 0 $ - and oldz.zwarning EQ 0, ndiff) - endif else begin - ndiff = 0 - endelse - - igal = where((plug.primtarget AND $ - (2L^6 + 2L^7 + 2L^8 + 2L^5 + 2L^26)) NE 0, ngal) - iqso = where((plug.primtarget AND $ - (2L^0 + 2L^1 + 2L^2 + 2L^3 + 2L^4 + 2L^25)) NE 0, nqso) - - if (ngal GT 0 AND keyword_set(zans)) then begin - junk = where(zans[igal].zwarning EQ 0 $ - AND (strtrim(zans[igal].class) EQ 'GALAXY' $ - OR strtrim(zans[igal].class) EQ 'QSO'), nggal) - fgal = 100 * float(nggal) / ngal - endif else begin - fgal = 0. - endelse - if (nqso GT 0 AND keyword_set(zans)) then begin - junk = where(zans[iqso].zwarning EQ 0 $ - AND strtrim(zans[iqso].class) EQ 'QSO', ngqso) - fqso = 100 * float(ngqso) / nqso - endif else begin - fqso = 0. - endelse - - junk = where((ormask AND (2L^8+2L^9)) NE 0, nwhopper) - junk = where((ormask AND (2L^18+2L^19)) NE 0, nreject) - junk = where((ormask AND 2L^20) NE 0, nscatter) - junk = where((ormask AND 2L^21) NE 0, nxtalk) - junk = where((ormask AND 2L^23) NE 0, nbsky) - junk = where((ormask AND 2L^26) NE 0, nbadflux) - junk = where((ormask AND 2L^27) NE 0, nbadchi) - junk = where((ormask AND 2L^28) NE 0, nmonster) - - npix = n_elements(ormask) - fwhopper = 100 * float(nwhopper) / npix - freject = 100 * float(nreject) / npix - fscatter = 100 * float(nscatter) / npix - fxtalk = 100 * float(nxtalk) / npix - fbsky = 100 * float(nbsky) / npix - fbadflux = 100 * float(nbadflux) / npix - fbadchi = 100 * float(nbadchi) / npix - fmonster = 100 * float(nmonster) / npix - - if (keyword_set(olddir)) then $ - splog, string(plist[i].plate, plist[i].mjd, fgal, fqso, $ - fwhopper, freject, fscatter, fxtalk, fbsky, fbadflux, $ - fbadchi, fmonster, ndiff, $ - format='(i5,i6,2f7.1," ",8f7.2,i7)'), /noname $ - else $ - splog, string(plist[i].plate, plist[i].mjd, fgal, fqso, $ - fwhopper, freject, fscatter, fxtalk, fbsky, fbadflux, $ - fbadchi, fmonster, $ - format='(i5,i6,2f7.1," ",8f7.2)'), /noname - endfor - - splog, /close -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/redplate.pro b/pro/spec1d/redplate.pro deleted file mode 100644 index b9ecab3ee..000000000 --- a/pro/spec1d/redplate.pro +++ /dev/null @@ -1,19 +0,0 @@ -; Find the reddening for a bunch of plates. -pro redplate, legacy=legacy, plates=plates - - ;platelist, plist=plist - conflist, plist=plist, legacy=legacy, plates=plates - ;print, plist - plist = plist[where(plist.qsurvey, nplate)] - ;print, plist - ebv = fltarr(nplate) - for iplate=0, nplate-1 do begin - readspec, plist[iplate].field, mjd=plist[iplate].mjd, plug=plug, $ - legacy=legacy, plates=plates - euler, plug.ra, plug.dec, ll, bb, 1 - ebv[iplate] = mean(dust_getval(ll,bb,/interp, /noloop)) - print, 'Field ', iplate, ' of ', nplate,' ebv ',ebv[iplate] - endfor - -stop -end diff --git a/pro/spec1d/reformat_spec.pro b/pro/spec1d/reformat_spec.pro deleted file mode 100644 index 996ab4bcf..000000000 --- a/pro/spec1d/reformat_spec.pro +++ /dev/null @@ -1,484 +0,0 @@ -;+ -; NOTE: reprocess the single spectra spSepc files -; NAME: -; reformat_spec -; -; PURPOSE: -; Reformat spectra into a single fits file per object, combining all necessary -; pieces from spField, spZall, spZline and spZbest -; -; For each object, there is one file per target, -; containing the finall coadded spectrum, and each blue-red merged single spectrum per exposure -; frames. The file contains the next headers: -; -; HDU 0 : Header info -; HDU 1 : Coadded spectrum -; HDU 2 : Summary metadata copied from spZbest -; HDU 3 : Summary metadata copied from spZall -; HDU 4 : Line fitting metadata from spZline -; HDU 5 : Individual frame spectra per exposure -; -; The format of each spec file is: -; -; HDU 0 : -; Header : from input spField with additional keywords from specObj: -; -; Keyword specObjColumn Comment -; ------- ----------- ------- -; [*] PLUG_RA PLUG_RA RA of object [deg] -; [*] PLUG_DEC PLUG_DEC dec of object [deg] -; TARGET_ID TARGET_ID Unique object identifier -; -; [*] Note that RA and DEC already exist in the spPlate headers -; but they are the telescope boresite RA and DEC, not the -; RA and DEC of the object. -; -; Data : None -; HDU 1 : Coadded Spectrum -; Header : Minimum to define table -; Data : Binary table with columns taken from the original spSpec and spZbest: -; flux : flux in 10^-17 ergs/s/cm^2/A -; loglam : log10(lambda[A]) -; ivar : inverse variance -; and_mask : mask bits which affect every spectrum in coadd -; or_mask : mask bits which affect at least one spectrum in coadd -; wdisp : wavelength dispersion in dloglam units -; sky : subtracted sky flux in 10^-17 ergs/s/cm^2/A -; wresl : spectral resolution in A units -; model : best fit model for classification & redshift (from spZbest) -; -; HDU 2 : Copy of row for this object from spZbest table + plugmap info of spFrame -; SPECTRO_REDUX/RUN2D/FIELD/RUN1D/spZbest*.fits -; -; HDU 3 : Copy of rows for this object from spZall table -; SPECTRO_REDUX/RUN2D/FIELD/RUN1D/spZline*.fits -; -; HDU 4 : Copy of rows for this object from spZline table -; SPECTRO_REDUX/RUN2D/FIELD/RUN1D/spZline*.fits -; -; HDU 5 .. +n_exp : Individual frames. -; For each exposure there is one HDU. -; These are in the order of the FPS confiuration_id, and -; -; Header : Minimum to define table -; Data: Binary table with columns taken from the original spSpec: -; flux : flux in 10^-17 ergs/s/cm^2/A -; loglam : log10(lambda[A]) -; ivar : inverse variance -; and_mask : mask bits which affect every spectrum -; or_mask : mask bits which affect at least one spectrum -; wdisp : wavelength dispersion in dloglam units -; sky : subtracted sky flux in 10^-17 ergs/s/cm^2/A -; wresl : spectral resolution in A units -; -; CALLING SEQUENCE: -; reformat_spec, [ platefile, fiberid=, run1d=, /doplot, /debug, chop_data= ] -; -; INPUTS: -; -; OPTIONAL INPUTS: -; fieldfile - Field file(s) from spectro-2D; default to all files -; matching 'spField*.fits' -; run1d - Optional override value for the environment variable $RUN1D -; doplot - If set, then generate plots. Send plots to a PostScript -; file spDiagDebug1d-$PLATE-$MJD.ps unless /DEBUG is set. -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; Input files are read from the current directory. -; Output files are written to the subdirectory $RUN2D/SPECTRA/$MJD. -; -; EXAMPLES: -; -; BUGS: -; -; DATA FILES: -; The procedure requires these files: -; ZALLFILE = 'spZall-Field-MJD.fits' -; ZBESTFILE = 'spZbest-Field-MJD.fits' -; ZLINEFILE = 'spZline-Field-MJD.fits' -; -; PROCEDURES CALLED: -; -; REVISION HISTORY: -; 27-Jan-2020 Written by Hector Ibarra at UIUC -;------------------------------------------------------------------------------ -; -pro struct_delete_field, struct, tag - ;Delete an existing field from a structure. - ; - ;Inputs: - ; tag (string) Case insensitive tag name describing structure field to - ; delete. Leading and trailing spaces will be ignored. If the requested - ; field does not exist, the structure is returned unchanged and without - ; error. - ; - ;Input/Output: - ; struct (structure) structure to be modified. - ; - ;Examples: - ; - ; Delete sme.wave from structure: - ; - ; IDL> struct_delete_field, sme, 'wave' - ; - ;History: - ; 2003-Jul-26 Valenti Adapted from struct_replace_field.pro. - ; 2020-Jan-27 HJIM Adapted for bhm. - if n_params() lt 2 then begin - print, 'syntax: struct_delete_field, struct, tag' - return - endif - - ;Check that input is a structure. - if size(struct, /tname) ne 'STRUCT' then begin - message, 'first argument is not a structure' - endif - - ;Get list of structure tags. - tags = tag_names(struct) - ntags = n_elements(tags) - - ;Check whether the requested field exists in input structure. - ctag = strupcase(strtrim(tag, 2)) ;canoncial form of tag - itag = where(tags eq ctag, nmatch) - if nmatch eq 0 then return - itag = itag[0] ;convert to scalar - ;print,itag - ;Copy any fields that precede target field. - if itag gt 0 then begin ;target field occurs first - ;new = create_struct(tags[0], struct.(0)) ;initialize structure - valt=struct.(0);initialize structure - nd=SIZE(valt, /N_DIMENSIONS) - if nd eq 1 then begin - if n_elements(valt) eq 5 or n_elements(valt) eq 10 then begin - valf=valt - siz_str=1 - endif else begin - valf=valt[0] - siz_str=n_elements(valt) - endelse - endif else begin - if nd eq 2 then begin - if n_elements(valt) eq 100 then begin - valf=valt - siz_str=1 - endif else begin - valf=valt[*,0] - siz_str=n_elements(valt[0,*]) - endelse - endif else begin - valf=valt[*,0] - siz_str=n_elements(valt[0,*]) - endelse - endelse - if tags[0] eq 'NEXP' then begin - valf=0 - endif - if tags[0] eq 'TARGET_INDEX' then begin - valf=0 - endif - new=replicate(create_struct(tags[0],valf),siz_str) - new.(0)=valt - for i=1, itag-1 do begin ;insert leading unchange - valt=struct.(i) - nd=SIZE(valt, /N_DIMENSIONS) - if nd eq 1 then begin - if n_elements(valt) eq 5 or n_elements(valt) eq 10 then begin - valf=valt - siz_str=1 - endif else begin - valf=valt[0] - siz_str=n_elements(valt) - endelse - endif else begin - if nd eq 2 then begin - if n_elements(valt) eq 100 then begin - valf=valt - siz_str=1 - endif else begin - valf=valt[*,0] - siz_str=n_elements(valt[0,*]) - endelse - endif else begin - valf=valt[*,0] - siz_str=n_elements(valt[0,*]) - endelse - endelse - if tags[i] eq 'NEXP' then begin - valf=0 - endif - if tags[i] eq 'TARGET_INDEX' then begin - valf=0 - endif - values_t=replicate(create_struct(tags[i],valf),siz_str) - new=struct_addtags(new,values_t) - new.(i)=valt - endfor - endif - - ;Replicate remainder of structure after desired tag. - for i=itag+1, ntags-1 do begin - valt=struct.(i) - nd=SIZE(valt, /N_DIMENSIONS) - if nd eq 1 then begin - if n_elements(valt) eq 5 or n_elements(valt) eq 10 or n_elements(valt) eq 35 or n_elements(valt) eq 4 or n_elements(valt) eq 3 or n_elements(valt) eq 2 then begin; the magnitude vector has 5 elements - valf=valt - siz_str=1 - endif else begin - valf=valt[0] - siz_str=n_elements(valt) - endelse - endif else begin - if nd eq 2 then begin - if n_elements(valt) eq 100 then begin - valf=valt - siz_str=1 - endif else begin - valf=valt[*,0] - siz_str=n_elements(valt[0,*]) - endelse - endif else begin - if nd eq 3 then begin - if n_elements(valt[*,*,0]) eq 100 then begin - valf=valt[*,*,0] - siz_str=n_elements(valt[0,0,*]) - endif ;else begin - ; valf=valt[*,0] - ; siz_str=n_elements(valt[0,*]) - ;endelse - endif else begin - valf=valt[*,0] - siz_str=n_elements(valt[0,*]) - endelse - ;print,valt[*,*,0] - ;valf=valt[*,0] - ;siz_str=n_elements(valt[0,*]) - endelse - endelse - if tags[i] eq 'NEXP' then begin - valf=0 - endif - if tags[i] eq 'TARGET_INDEX' then begin - valf=0 - endif - values_t=replicate(create_struct(tags[i],valf),siz_str) - new=struct_addtags(new,values_t) - new.(i-1)=valt - endfor - - ;Replace input structure with new structure. - struct = new - -end - -;------------------------------------------------------------------------------ - -pro reformat_spec, platefile, run1d=run1d1, doplot=doplot, spectradir=spectradir, $ - run2d=run2d, plates=plates, legacy=legacy, sky=sky, lite=lite,XCSAO=XCSAO - -RESOLVE_ALL, /QUIET, /SKIP_EXISTING, /CONTINUE_ON_ERROR -CPU, TPOOL_NTHREADS = 1 - - spectro_redux = getenv('BOSS_SPECTRO_REDUX') - if (NOT keyword_set(platefile)) then begin - platefile = findfile('spField*.fits*', count=nplate) - endif else begin - if (size(platefile,/tname) NE 'STRING') then $ - message, 'FieldFILE must be a file name' - if (keyword_set(platefile)) then nplate = n_elements(platefile) $ - else nplate = 0 - endelse - if (keyword_set(run1d1)) then run1d = strtrim(run1d1,2) $ - else run1d = getenv('RUN1D') - - if (keyword_set(run2d)) then run2d = strtrim(run2d,2) $ - else run2d = getenv('RUN2D') - comb=spectro_redux+'/'+run2d+'/' - if (NOT keyword_set(spectradir)) then spectradir=comb - rsky=1 - if (keyword_set(sky)) then rsky = sky - ;---------- - ; If multiple plate files exist, then call this script recursively - ; for each such plate file. - - if (nplate EQ 0) then begin - splog, 'No fields files specified or found' - return - endif else begin - if (nplate EQ 1) then begin - platefile = platefile[0] - endif else begin - for i=0, nplate-1 do begin - reformat_spec, platefile[i], run1d=run1d, doplot=doplot, spectradir=spectradir, $ - run2d=run2d, plates=plates, legacy=legacy, lite=lite - endfor - return - endelse - endelse - - platemjd=(strsplit(repstr(platefile,".fits",""), '-',/extract))[1:2] - fieldid=platemjd[0] - thismjd=platemjd[1] - platemjd=strjoin(platemjd, '-') - zallfile = djs_filepath('spZall-' + platemjd + '.fits', root_dir=run1d) - zbestfile = djs_filepath('spZbest-' + platemjd + '.fits', root_dir=run1d) - zlinefile = djs_filepath('spZline-' + platemjd + '.fits', root_dir=run1d) - if keyword_set(XCSAO) then XCSAOFILE = djs_filepath('spXCSAO-' + platemjd + '.fits', root_dir=run1d) - if not FILE_TEST(XCSAOfile) then XCSAO = 0 - logfile = djs_filepath('spec-' + platemjd + '.log', root_dir='') - - if (keyword_set(logfile)) then begin - cpbackup, logfile - splog, filename=logfile - splog, 'Log file ' + logfile + ' opened ' + systime() - endif - splog,'Writing the final output files' - plugmap = mrdfits(platefile,5,/silent) - zall = mrdfits(zallfile,1,/silent) - zbest = mrdfits(zbestfile,1,/silent) - zline = mrdfits(zlinefile,1,/silent) - zmodel = mrdfits(zbestfile,2,/silent) - if keyword_set(XCSAO) then begin - XCSAO = mrdfits(XCSAOFILE,1,/silent) - XCSAO_str={XCSAO_rv:0.D,XCSAO_erv:0.D,$ - XCSAO_Rxc:0.D, $ - XCSAO_Teff:0.D,XCSAO_eteff:0.D,$ - XCSAO_Logg:0.D,XCSAO_elogg:0.D,$ - XCSAO_Feh:0.D,XCSAO_efeh:0.D} - endif - - target_ind=plugmap.target_index - single_basefile='coadd/'+thismjd+'/spSpec-'+platemjd+'-' - single_out_basefile='spec-'+platemjd+'-' - - if keyword_set(lite) then begin - lit_p='/lite' - endif else begin - lit_p='/full' - endelse - - spawn,'mkdir -p '+spectradir+'spectra' - spawn,'mkdir -p '+spectradir+'spectra'+lit_p - spawn,'mkdir -p '+spectradir+'spectra'+lit_p+'/'+fieldid - spawn,'mkdir -p '+spectradir+'spectra'+lit_p+'/'+fieldid+'/'+thismjd - dir_finalsp=spectradir+'spectra'+lit_p+'/'+fieldid+'/'+thismjd - - get_field_type, fieldid=fieldid, mjd=thismjd, legacy=legacy, plates=plates, fps=fps - - - foreach target_i, target_ind, itarget do begin -; for itarget=0, n_elements(target_ind)-1 do begin - plug_target=plugmap[itarget] - if fieldid lt 16000 then begin - otype=strtrim(plug_target.objtype,2) - spec=1 - if rsky eq 0 and otype eq 'SKY' then spec=0 - endif else begin - fibt=strtrim(plug_target.fibertype,2) - otype=strtrim(plug_target.objtype,2) - spec=1 - if rsky eq 0 and otype eq 'SKY' then spec=0 - endelse - if spec eq 1 then begin - if fieldid lt 15000 then begin - indx0 = (where((zbest.target_index EQ target_i))) - zbest_target=zbest[indx0] - zbest_model = zmodel[*,indx0] - endif else begin - zbest_target=zbest[itarget] - zbest_model = zmodel[*,itarget] - endelse - indx = (where((zall.target_index EQ target_i))) - tags = tag_names(zall) - ntags = n_elements(tags) - zall_targ=zall[indx] - indx2 = (where((zline.target_index EQ target_i))) - tags = tag_names(zline) - ntags = n_elements(tags) - zline_targ=zline[indx2] - if keyword_set(XCSAO) then begin - indx3 = (where((XCSAO.TARGET_INDEX EQ target_i))) - tags_rv = tag_names(XCSAO) - ntags_rv = n_elements(tags_rv) - XCSAO_targ=replicate(XCSAO_str,1) - if (tag_exist(XCSAO,'RV')) then XCSAO_targ.XCSAO_rv=XCSAO[indx3].RV - if (tag_exist(XCSAO,'ERV')) then XCSAO_targ.XCSAO_erv=XCSAO[indx3].ERV - if (tag_exist(XCSAO,'R')) then XCSAO_targ.XCSAO_Rxc = XCSAO[indx3].R - if (tag_exist(XCSAO,'Teff')) then XCSAO_targ.XCSAO_Teff = XCSAO[indx3].Teff - if (tag_exist(XCSAO,'eteff')) then XCSAO_targ.XCSAO_eteff = XCSAO[indx3].eteff - if (tag_exist(XCSAO,'Logg')) then XCSAO_targ.XCSAO_Logg = XCSAO[indx3].Logg - if (tag_exist(XCSAO,'elogg')) then XCSAO_targ.XCSAO_elogg = XCSAO[indx3].elogg - if (tag_exist(XCSAO,'Feh')) then XCSAO_targ.XCSAO_Feh = XCSAO[indx3].Feh - if (tag_exist(XCSAO,'efeh')) then XCSAO_targ.XCSAO_efeh = XCSAO[indx3].efeh - endif - - - - if keyword_set(legacy) then begin - single_file=single_basefile+string(plug_target.target_index,format='(i4.4)')+'.fits' - endif else begin - single_file=single_basefile+strtrim(plug_target.catalogid,2)+'.fits' - endelse - junk = mrdfits(single_file,0,hdr0,/silent) - coadd = mrdfits(single_file,1,/silent) - values_t=replicate(create_struct('model',0.0),n_elements(coadd.flux)) - coadd=struct_addtags(coadd,values_t) - - coadd.model=zbest_model - struct_delete_field,plug_target,'objtype' - struct_delete_field,zbest_target,'fiberid' - struct_delete_field,zall_targ,'fiberid' - struct_delete_field,zline_targ,'fiberid' - if keyword_set(legacy) then struct_delete_field,zbest_target,'field' - fin_plug=struct_addtags(plug_target,struct_selecttags(zbest_target, except_tags=['field','fiberid_list','target_index','fiber_ra','fiber_dec'])) - if keyword_set(XCSAO) then fin_plug=struct_addtags(fin_plug,XCSAO_targ) - nexp=plug_target.nexp - if keyword_set(legacy) then begin - file_name=single_out_basefile+string(plug_target.target_index,format='(i4.4)')+'.fits' - endif else begin - file_name=single_out_basefile+strtrim(plug_target.catalogid,2)+'.fits' - endelse - fulloutname_spec = djs_filepath(file_name, root_dir=dir_finalsp) - ;splog,'File '+file_name+' was created' - ; HDU # 0 header - mwrfits, junk, fulloutname_spec, hdr0, /create, /silent - ; HDU # 1 header - sxaddpar, coadd_val, 'EXTNAME', 'COADD', ' Coadded spectrum' - mwrfits, coadd, fulloutname_spec, coadd_val, /silent - sxdelpar, coadd_val, 'COMMENT' - ;delvar,coadd_val - ; HDU # 2 Summary metadata copied from spZbest - sxaddpar, hdrplug, 'EXTNAME', 'SPALL', ' Spall structure' - mwrfits, fin_plug, fulloutname_spec, hdrplug, /silent - sxdelpar, hdrplug, 'COMMENT' - ;delvar,hdrplug - ; HDU # 3 Summary metadata copied from spZpall - sxaddpar, hdrzall, 'EXTNAME', 'ZALL', ' Zall structure' - mwrfits, zall_targ, fulloutname_spec, hdrzall, /silent - sxdelpar, hdrplug, 'COMMENT' - ;delvar,hdrplug - ; HDU # 4 Summary metadata copied from spZlines - sxaddpar, hdrzline, 'EXTNAME', 'ZLINE', ' Zlines structure' - mwrfits, zline_targ, fulloutname_spec, hdrzline, /silent - sxdelpar, hdrplug, 'COMMENT' - ;delvar,hdrplug - ;print,nexp - ;print,single_file - if not keyword_set(lite) then begin - for i=0, nexp-1 do begin - single = mrdfits(single_file,3+i,hdri, /silent) - sxdelpar, hdri, 'COMMENT' - mwrfits, single, fulloutname_spec, hdri, /silent - endfor - endif - splog,'File '+file_name+' was created, target '+strtrim(string(itarget+1),2)+' of '+strtrim(string(n_elements(target_ind)),2)+' targets' - endif - ;stop - endforeach - splog, 'Successful completion of REFORMAT_SPEC at ' + systime() - if (keyword_set(logfile)) then splog, /close - end diff --git a/pro/spec1d/rline_findew.pro b/pro/spec1d/rline_findew.pro deleted file mode 100644 index 8b37a105f..000000000 --- a/pro/spec1d/rline_findew.pro +++ /dev/null @@ -1,54 +0,0 @@ -pro rline_findew, spec, sigma=sigma, ew, ewinv, fopt, finv - - if NOT keyword_set(sigma) then sigma=1.0 - - boxarea = 5*sigma - hpix = fix(boxarea-1)/2 - intarea = 2*hpix+1 - boxcar = replicate(1,intarea) - if boxarea-intarea GT 0 then $ - boxcar = [0.5*(boxarea-intarea),boxcar, 0.5*(boxarea-intarea)] - - ;---------------------------------------------------------------------------- - ; - ; Do boxcar convolution first - ; - ;---------------------------------------------------------------------------- - - mask = float(spec.finv LE 0) - nspec = n_elements(spec) - - smask = convol(mask, boxcar, /edge_truncate) - - var = 1.0/(spec.finv + mask) - - sub = spec.flux - spec.model - zero = where(mask) - if zero[0] NE -1 then sub[zero] = 0 - - ew = convol(sub, boxcar, /edge_truncate) - ewvar = convol(var, boxcar, /edge_truncate) - - ewinv = ewvar * 0.0 - good = where(mask EQ 0 AND ewvar GT 0) - if good[0] NE -1 then ewinv[good] = 1.0/ewvar[good] - - ;---------------------------------------------------------------------------- - ; - ; Now do optimized gaussian detection - ; - ;---------------------------------------------------------------------------- - - gkernel = gauss_kernel(sigma) - - finv = convol(spec.finv, gkernel^2, /edge_truncate) - fnumerator = convol(sub*spec.finv, gkernel, /edge_truncate) - - smask = convol(mask, gkernel, /edge_truncate) - good = where(smask LT 0.4 AND finv GT 0) - fopt = 0.0*finv - - if good[0] NE -1 then fopt[good] = fnumerator[good] / finv[good] - - return -end diff --git a/pro/spec1d/rline_findpeaks.pro b/pro/spec1d/rline_findpeaks.pro deleted file mode 100644 index 2a60334a8..000000000 --- a/pro/spec1d/rline_findpeaks.pro +++ /dev/null @@ -1,100 +0,0 @@ -;+ -; NAME: -; rline_findpeaks -; -; PURPOSE: -; Given a two dimensional array for Equivalent Width (EW) and -; weights (EW inverse variance), this function returns a structure -; containing peak positions, heights, and significance level (SN). -; -; Only postive peaks are detected, so transform the array is negative -; peaks are sought. -; -; CALLING SEQUENCE: -; peak_list = rline_findpeaks(ew, ewinv, npeak=, threshold=, minsep=) -; -; INPUTS: -; ew - Positive equivalent width array -; ewinv - Associated inverse variance -; -; OPTIONAL INPUTS: -; npeak - Maximum number of peaks to locate (default 20) -; threshold - Minimum significance level (sigma) to return (default 5.0) -; minsep - Minimum pixel separation to between peaks (default 2.5) -; -; OPTIONAL KEYWORDS: -; -; OUTPUTS: -; peak_list - Array of structures with peak information -; -; COMMENTS: -; -; EXAMPLES: -; -; PROCEDURES CALLED: -; find_npeaks -; -; DATA FILES: -; -; REVISION HISTORY: -; 05-Jan-2001 Written by S. Burles, Fermiland -; 17-Dec-2001 Commented, updated to work with new find_npeaks.pro -;- -;------------------------------------------------------------------------------ -function rline_findpeaks, ew, ewinv, npeak = npeak, threshold=threshold, $ - minsep=minsep - - npix = (size(ew,/dim))[0] - nspec = (size(ew,/dim))[1] - xtab = lindgen(npix) - - if NOT keyword_set(npeak) then npeak = 20 - if NOT keyword_set(threshold) then threshold=5.0 - if NOT keyword_set(minsep) then minsep=2.5 - - sn = ew * sqrt(ewinv) - - ttemp = { fiber : -1L, x : -1.0, y: -1.0, sn : 0.0, xerr : 0.0 , $ - class : 'UNK'} - - full_list = 0 - - - for i=0, nspec -1 do begin - - xpeak = find_npeaks(sn[*,i], nfind=npeak, ypeak=ypeak, xerr=xerr, $ - minsep=minsep) - - high = where(ypeak GE threshold, nhigh) - if nhigh GT 1 then begin - mask = lonarr(nhigh) + 1 - for j=nhigh-1,1,-1 do begin - identical = where(abs(xpeak[high[j]] - xpeak[high[0:j-1]]) LT minsep) - if identical[0] NE -1 then mask[j] = 0 - endfor - goodmask = where(mask, nhigh) - if nhigh GT 0 then high=high[goodmask] - endif - - if nhigh GT 0 then begin - linterp, xtab, ew[*,i], xpeak[high], y - tt = replicate(ttemp, nhigh) - tt.fiber = i - tt.x = xpeak[high] - tt.y = y - tt.sn = ypeak[high] - tt.xerr = xerr[high] - - if keyword_set(full_list) then full_list = [full_list, tt] $ - else full_list = tt - - endif - endfor - - return, full_list -end - - - - - diff --git a/pro/spec1d/rline_getplate.pro b/pro/spec1d/rline_getplate.pro deleted file mode 100644 index 6d89cea21..000000000 --- a/pro/spec1d/rline_getplate.pro +++ /dev/null @@ -1,68 +0,0 @@ - -; PRIMTARGET - If set, then select only objects where at least one bit -; in its PRIMTARGET flag matches one bit in this parameter. -; CLASS - If set, then only select objects that match this classification, -; i.e. 'GALAXY' -; MAXRCHI2 - If set, then only select objects whose reduced chi^2 is -; less than or equal to this value. -; QUICK - If set, then only return ZANS and PLUG in the return structure. -;------------------------------------------------------------------------------ -function rline_getplate, plate, mjd=mjd, $ - primtarget=primtarget, class=class, maxrchi2=maxrchi2, quick=quick - - ; First read the plug-map - - readspec, plate, mjd=mjd, plug=plug, zans=zans - if (NOT keyword_set(plug)) then return, 0 - - ; Set MASK=1 for objects to return - - mask = lonarr(n_elements(plug)) + 1 - - if (keyword_set(primtarget)) then $ - mask = mask * (plug.primtarget NE 0) $ - * ((plug.primtarget AND primtarget) NE 0) - - if (keyword_set(class)) then $ - mask = mask * (strtrim(zans.class,2) EQ class) - - if (keyword_set(maxrchi2)) then $ - mask = mask * (zans.rchi2 LE maxrchi2) - - igood = where(mask) - if (igood[0] EQ -1) then return, 0 - - if (keyword_set(quick)) then begin - tt = { plug : plug[0], $ - zans : zans[0] } - tt = replicate(tt, n_elements(igood)) - tt.zans = zans[igood] - tt.plug = plug[igood] - return, tt - endif - - readspec, zans[igood].plate, zans[igood].fiberid, mjd=zans[igood].mjd, $ - flux=flux, invvar=finv, loglam=loglam, plug=plug, zans=zans - - model = loglam * 0 - for i=0,n_elements(zans)-1 do $ - model[*,i] = synthspec(zans[i], loglam=loglam[*,i]) - - tt = { flux : flux[*,0], $ - finv : finv[*,0], $ - loglam : float(loglam[*,0]) , $ - model : float(model[*,0]), $ - plug : plug[0], $ - zans : zans[0] } - - tt = replicate(tt, n_elements(zans)) - tt.flux = temporary(flux) - tt.finv = temporary(finv) - tt.loglam = temporary(loglam) - tt.model = temporary(model) - tt.plug = temporary(plug) - tt.zans = temporary(zans) - - return, tt -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/rline_loop.pro b/pro/spec1d/rline_loop.pro deleted file mode 100644 index 2aa45581a..000000000 --- a/pro/spec1d/rline_loop.pro +++ /dev/null @@ -1,95 +0,0 @@ -pro rline_loop, plate=plate, mjd=mjd - - primtarget = 32 ; BRG's - - ;---------- - ; Get a list of plates - - if (keyword_set(plate)) then begin - nplate = n_elements(plate) - if (NOT keyword_set(mjd)) then mjd = lonarr(nplate) ; zeros - plist = replicate(create_struct('plate', 0L, 'mjd', 0L), nplate) - plist.plate = plate - plist.mjd = mjd - endif else begin - ; If PLATE is not specified, then determine a list of good plates... - platelist, plist=plist - ii = where(plist.nums[0] GT 0 $ -; AND plist.plate GE 300 AND plist.plate LE 349 $ ; ???? - AND plist.snvec[0] GT 15 AND plist.snvec[1] GT 15 $ - AND plist.snvec[2] GT 15 AND plist.snvec[3] GT 15) - plist = plist[ii] - endelse - - ;---------- - ; Loop through each plate - - nplate = n_elements(plist) - for iplate=0, nplate-1 do begin - - splog, 'WORKING ON PLATE ', plist[iplate].plate, $ - ' MJD ', plist[iplate].mjd - spec = rline_getplate(plist[iplate].plate, mjd=plist[iplate].mjd, $ - primtarget=primtarget, class='GALAXY', maxrchi2=2.0) - - if (keyword_set(spec)) then begin - - ; Accumulate ZANS and PLUG for the full sample - sampzans = struct_append(sampzans, spec.zans) - sampplug = struct_append(sampplug, spec.plug) - - rline_findew, spec, ew, ewinv, fopt, finv - pks = rline_findpeaks(ew, ewinv) - - ; Convert the peak position from pixel to log-wavelength - npix = n_elements(spec[0].loglam) - xvec = findgen(npix) - linterp, xvec, spec[0].loglam, pks.x, xloglam - - if (keyword_set(pks)) then begin - lines = rline_matchpeaks(spec, pks) - for ipk=0, n_elements(lines)-1 do $ - allpks = struct_append( allpks, $ - create_struct('zans', spec[pks[ipk].fiber].zans, $ - 'plug', spec[pks[ipk].fiber].plug, $ - pks[ipk], $ - 'xloglam', xloglam[ipk], $ - lines[ipk]) ) - - endif - endif - endfor - -save,file='rline.ss' -stop - -j = where(allpks.lambda EQ 0 AND allpks.xloglam LT alog10(7500)) -goodpks = allpks[j] -readspec, goodpks.zans.plate, goodpks.zans.fiberid, mjd=goodpks.zans.mjd, $ - flux=flux, flerr=flerr, invvar=invvar, andmask=andmask, ormask=ormask, $ - loglam=loglam, wave=wave, zans=zans -synflux = 0 * flux -for i=0, n_elements(goodpks)-1 do $ - synflux[*,i] = synthspec(goodpks[i].zans, loglam=loglam[*,i]) -save,file='goodpks.ss' - -restore,'goodpks.ss' -k=1 -thiswave = 10^goodpks[k].xloglam -ii=where(wave[*,k] NE 0) -splot,wave[ii,k],flux[ii,k], xrange=thiswave+[-200,200] -soplot,wave[ii,k],synflux[ii,k],color='blue' -soplot,wave[ii,k],flerr[ii,k],color='red' -soplot,[thiswave,thiswave],!y.crange,color='green - - -j = where(allpks.lambda EQ 0 AND allpks.xloglam LT alog10(7500)) -splot,10^allpks.xloglam,allpks.sn,ps=3 -soplot,10^allpks[j].xloglam,allpks[j].sn,ps=3,color='red' -k = j[where(allpks[j].sn GT 20)] - -jj=j[0] -plotspec,allpks[jj].zans.plate,allpks[jj].zans.fiberid -soplot,10^allpks[jj].xloglam,1,ps=4,charsize=4,color='green' - -end diff --git a/pro/spec1d/rline_matchpeaks.pro b/pro/spec1d/rline_matchpeaks.pro deleted file mode 100644 index bd9901f22..000000000 --- a/pro/spec1d/rline_matchpeaks.pro +++ /dev/null @@ -1,61 +0,0 @@ -;------------------------------------------------------------------------------ -function linematch1, loglam, dloglam, zshift - - common com_linelist, linelist - - ;---------- - ; Read line lists and convert to vacuum - - if (NOT keyword_set(linelist)) then begin - linefile = filepath('linematch.par', $ - root_dir=getenv('IDLSPEC2D_DIR'), subdirectory='etc') - yanny_read, linefile, pdata - linelist = *pdata[0] - yanny_free, pdata - vaclambda = linelist.lambda - airtovac, vaclambda - linelist.lambda = vaclambda - endif - - ;---------- - ; Make a line list where the emission line wavelengths are redshifted - ; to that of the galaxy. - - thislist = linelist.lambda * (1 + zshift * (linelist.type EQ 'EMISSION')) -;for i=0,n_elements(thislist)-1 do $ -; print,linelist[i].type,linelist[i].lambda,thislist[i] - - ;---------- - ; Identify this feature with the nearest line. - - logdist = min(abs(loglam - alog10(thislist)), imin) - if (logdist LE dloglam) then return, linelist[imin] - - ; Return no match - retval = linelist[0] - struct_assign, {junk:0}, retval ; Zero-out all elements - return, retval -end -;------------------------------------------------------------------------------ -function rline_matchpeaks, specstruct, pkstruct - - ;---------- - ; Loop through each possible line - - npix = n_elements(specstruct[0].loglam) - xvec = findgen(npix) - - dloglam = 4.0e-4 ; Sky+object line exclusion width of 4 pixels - - npeak = n_elements(pkstruct) - for ipeak=0, npeak-1 do begin - thisspec = specstruct[ pkstruct[ipeak].fiber ] - linterp, xvec, thisspec.loglam, pkstruct[ipeak].x, thisloglam - thisline = linematch1(thisloglam, dloglam, thisspec.zans.z) - if (ipeak EQ 0) then allline = thisline $ - else allline = [allline, thisline] - endfor - - return, allline -end -;------------------------------------------------------------------------------ diff --git a/pro/spec1d/rline_plotsamp.pro b/pro/spec1d/rline_plotsamp.pro deleted file mode 100644 index 92472380e..000000000 --- a/pro/spec1d/rline_plotsamp.pro +++ /dev/null @@ -1,47 +0,0 @@ -pro rline_plotsamp - - restore, 'rline_sample.ss' - - dfpsplot, 'rline_samp.ps', /color - - psize = 2.0 - pthick = 2 - - xrange = [0.0, 0.6] - yrange = [15.5,21.5] - plot, [0], [0], /nodata, $ - position=[0.15,0.50,0.95,0.92], $ - ytitle='r-band mag', $ - xrange=xrange, yrange=yrange, xstyle=1, ystyle=1, $ - charsize=psize, charthick=pthick, $ - xtickname=replicate(' ',30), $ - title='SDSS Luminous Red Galaxy Sample' - - oplot, sampzans.z, sampplug.mag[2], ps=3 - - binsz = 0.01 - nbin = long( (xrange[1]-xrange[0]) / binsz ) + 1 - xhist = findgen(nbin) * binsz - yhist = lonarr(nbin) - for i=0, n_elements(sampzans.z)-1 do begin - ibin = long(sampzans[i].z / binsz + 0.5) - if (ibin GE 0 AND ibin LT nbin) then yhist[ibin] = yhist[ibin] + 1 - endfor - - yrange = [0,1.05*max(yhist)] - plot, [0], [0], /nodata, /noerase, $ - position=[0.15,0.15,0.95,0.50], $ - xtitle='Redshift', ytitle='Number', $ - xrange=xrange, yrange=yrange, xstyle=1, ystyle=1, $ - charsize=psize, charthick=pthick - -; oplot, xhist, yhist, psym=10, thick=2 - plothist, sampzans.z, bin=binsz, thick=2, /fill, /overplot - - djs_xyouts, 0.4, 0.8*!y.crange[1], $ - 'N_{gal} = '+strtrim(string(n_elements(sampzans)),2), charsize=psize - - dfpsclose - - return -end diff --git a/pro/spec1d/rline_plotsome.pro b/pro/spec1d/rline_plotsome.pro deleted file mode 100644 index 4799a04c5..000000000 --- a/pro/spec1d/rline_plotsome.pro +++ /dev/null @@ -1,132 +0,0 @@ -; This reads the save save written by RLINE_SAMPLE. -; Set PLOTNUM to 0, 1, or 2 for plot number -pro rline_plotsome, plotnum - - if (NOT keyword_set(plotnum)) then plotnum = 0 -; nsmooth = 3 - - restore, 'goodpks.ss' - - junk = {pinfo, iselect: 0L, string1: '', string2: ''} - pinfo = replicate( {pinfo}, 4 ) - - if (plotnum EQ 0) then begin - pinfo.iselect= [ 15 , 213 , 226 , 140 ] - pinfo.string1= [ 'H\alpha' , 'H\alpha' , 'H\alpha' , 'H\alpha' ] - pinfo.string2= [ 'z=0.072' , 'z=0.136' , 'z=0.039' , 'z=0.018' ] - endif else if (plotnum EQ 1) then begin - pinfo.iselect= [ 106 , 37 , 35 , 276 ] - pinfo.string1= [ 'O II' , 'O II' , 'O II' , 'O II' ] - pinfo.string2= [ 'z=0.272' , 'z=0.441' , 'z=0.806' , 'z=0.156' ] - endif else if (plotnum EQ 2) then begin - pinfo.iselect= [ 179 , 93 , 143 , 66 ] - pinfo.string1= [ 'Ly\alpha ??', 'Ly\alpha ??', 'Ly\alpha ??', 'Ly\alpha ??' ] - pinfo.string2= [ 'z=4.22' , 'z=2.17' , 'z=2.24' , 'z=5.13' ] - endif - - dfpsplot, 'rline-panel-'+strtrim(string(plotnum),2)+'.ps', /color - - iselect = pinfo.iselect - psize = 1.5 - pthick = 2 -pthick=1 - - ; Interpolate over bad sky regions - invvar = skymask(invvar, andmask) - fluxinterp = djs_maskinterp(flux, invvar EQ 0, /const, iaxis=0) - - nplot = n_elements(iselect) - yoffset = 0.90 / nplot -; ypos = [0.10, 0.10+yoffset] ; Do this to not separate each spectrum in Y - ypos = [0.10, 0.06+yoffset] ; Do this to separate each spectrum in Y - for iplot=0, nplot-1 do begin - if (iplot EQ 0) then begin - xtitle='Wavelength [Ang]' - noerase = 0 - xtickname = '' - endif else begin - xtitle='' - noerase = 1 - xtickname = replicate(' ',30) - endelse - if (iplot EQ nplot-1) then begin - title1 = 'FULL SDSS SPECTRUM' - title2 = 'ROGUE LINES' - endif - ytitle = 'Flux' - - ; Info on the rogue line - thispeak = goodpks[iselect[iplot]] - thiswave = 10.^thispeak.xloglam - - ; Info on the foreground/background galaxy - thisz = thispeak.zans.z - - iw = where(wave[*,iselect[iplot]] NE 0) - xplot = wave[iw,iselect[iplot]] - yplot1 = fluxinterp[iw,iselect[iplot]] - yplot2 = flux[iw,iselect[iplot]] - synplot = synflux[iw,iselect[iplot]] - errplot = flerr[iw,iselect[iplot]] - - ; Smooth... - if (keyword_set(nsmooth)) then begin - yplot1 = smooth(yplot1,nsmooth) - synplot = smooth(synplot,nsmooth) - errplot = smooth(errplot,nsmooth) - endif - - ; Plot the whole spectrum - - xrange = [3700,9200] - yrange = 1.3 * minmax(smooth(yplot1,5)) - yrange[0] = yrange[0] < 0 - yrange[0] = yrange[0] > (-0.1 * yrange[1]) - - plot, xplot, yplot1, /nodata, noerase=noerase, $ - position=[0.10,ypos[0],0.65,ypos[1]], $ - xtitle=xtitle, ytitle=ytitle, $ - xrange=xrange, yrange=yrange, xstyle=1, ystyle=1, $ - charsize=psize, charthick=pthick, $ - xtickname=xtickname, title=title1 - djs_oplot, xplot, yplot1, thick=pthick - djs_oplot, xplot, synplot, color='blue', thick=pthick - djs_oplot, xplot, errplot, color='red', thick=pthick - djs_xyouts, 0.9*xrange[0]+0.1*xrange[1], 0.2*yrange[0]+0.8*yrange[1], $ - 'Galaxy @ z='+string(thisz,format='(f5.2)'), $ - charsize=psize, charthick=pthick - - ; Indicate the rogue em. line - - yarr = [yrange[0], 0.75*yrange[0]+0.25*yrange[1]] - djs_arrow, thiswave, yarr[0], thiswave, yarr[1], $ - color='green', thick=pthick, /data - - ; Plot the spectrum around the rogue em. line - - xrange = thiswave+[-100,100] - plot, xplot, yplot2, /nodata, /noerase, $ - position=[0.65,ypos[0],0.97,ypos[1]], $ - xrange=xrange, yrange=yrange, xstyle=1, ystyle=1, $ - charsize=psize, charthick=pthick, $ - ytickname=replicate(' ',30), title=title2 - djs_oplot, xplot, yplot2, thick=pthick, ps=10 - djs_oplot, xplot, synplot, color='blue', thick=pthick - - djs_xyouts, 0.35*xrange[0]+0.65*xrange[1], 0.2*yrange[0]+0.8*yrange[1], $ - pinfo[iplot].string1, charsize=psize, charthick=pthick, color='green' - djs_xyouts, 0.35*xrange[0]+0.65*xrange[1], 0.35*yrange[0]+0.65*yrange[1], $ - pinfo[iplot].string2, charsize=psize, charthick=pthick, color='green' - - ; Indicate the rogue em. line - - djs_arrow, thiswave, yarr[0], thiswave, yarr[1], $ - color='green', thick=pthick, /data - - ypos = ypos + yoffset - endfor - - dfpsclose - - return -end diff --git a/pro/spec1d/rline_sample.pro b/pro/spec1d/rline_sample.pro deleted file mode 100644 index bd83bc916..000000000 --- a/pro/spec1d/rline_sample.pro +++ /dev/null @@ -1,47 +0,0 @@ -; This is a hacked version of RLINE_LOOP, for just getting the full -; sample selection of BRG's. -pro rline_sample, plate=plate, mjd=mjd - - primtarget = 32 ; BRG's - - ;---------- - ; Get a list of plates - - if (keyword_set(plate)) then begin - nplate = n_elements(plate) - if (NOT keyword_set(mjd)) then mjd = lonarr(nplate) ; zeros - plist = replicate(create_struct('plate', 0L, 'mjd', 0L), nplate) - plist.plate = plate - plist.mjd = mjd - endif else begin - ; If PLATE is not specified, then determine a list of good plates... - platelist, plist=plist - ii = where(plist.nums[0] GT 0 $ -; AND plist.plate GE 300 AND plist.plate LE 349 $ ; ???? - AND plist.snvec[0] GT 15 AND plist.snvec[1] GT 15 $ - AND plist.snvec[2] GT 15 AND plist.snvec[3] GT 15) - plist = plist[ii] - endelse - - ;---------- - ; Loop through each plate - - nplate = n_elements(plist) - for iplate=0, nplate-1 do begin - - splog, 'WORKING ON PLATE ', plist[iplate].plate, $ - ' MJD ', plist[iplate].mjd - spec = rline_getplate(plist[iplate].plate, mjd=plist[iplate].mjd, $ - primtarget=primtarget, class='GALAXY', maxrchi2=2.0, /quick) - - if (keyword_set(spec)) then $ - sampzans = struct_append(sampzans, spec.zans) - if (keyword_set(spec)) then $ - sampplug = struct_append(sampplug, spec.plug) - - endfor - -save,file='rline_sample.ss' -stop - -end diff --git a/pro/spec1d/sdss_spec_image.pro b/pro/spec1d/sdss_spec_image.pro deleted file mode 100644 index a8f6464cb..000000000 --- a/pro/spec1d/sdss_spec_image.pro +++ /dev/null @@ -1,350 +0,0 @@ -;+ -; NAME: -; sdss_spec_image -; PURPOSE: -; Create an image of a spectrum -; CALLING SEQUENCE: -; sdss_spec_image, outbase, plate, fiber, mjd=, xra= -; INPUTS: -; outbase - output file base -; plate - plate # -; fiber - fiber # -; mjd - MJD of observation -; OPTIONAL INPUTS: -; xra - [2] wavelength limits to plot (Ang) -; OPTIONAL KEYWORDS: -; /noclobber - do not clobber existing image files -; COMMENTS: -; Creates the files: -; [outbase].jpg -; [outbase].thumb.jpg -; which have a picture of the specified spectrum, -; with some nice annotation. -; REVISION HISTORY: -; Written by MRB, NYU 14-05-2010 -;------------------------------------------------------------------------------ -function sdss_spec_smooth, loglam,flux,vdisp - -if vdisp GT 1.0 then begin - nlambda= n_elements(loglam) - pixsize= $ - abs(alog(10.)*2.99792e+5*(loglam[nlambda-1]-loglam[0])/double(nlambda)) - smoothing= vdisp/pixsize ; pixels - npix= long(4.0*ceil(smoothing))*2L+3 - klam= findgen(npix)-float(npix-1.)/2. - kernel= exp(-0.5*(klam/smoothing)^2)/sqrt(2.*!DPI)/smoothing - kernel= kernel/total(kernel) - if(n_elements(kernel) ge n_elements(flux)) then $ - smoothed_spec= flux $ - else $ - smoothed_spec= convol(flux,kernel,/edge_truncate) -endif else begin - smoothed_spec= flux -endelse - -return, smoothed_spec - -end -;; -pro sdss_spec_image, outbase, plate, fiber, mjd=mjd, run2d=run2d, $ - run1d=run1d, topdir=topdir, xra=xra, silent=silent, $ - noclobber=noclobber, legacy=legacy, plates=plates - -common com_sdss_spec_image, plans - -if(n_tags(plans) eq 0) then $ - plans= yanny_readone(getenv('PLATELIST_DIR')+'/platePlans.par') -sscale=1.5 -if(NOT keyword_set(xsize)) then xsize= 10.5*sscale -if(NOT keyword_set(ysize)) then ysize= 7.5*sscale -if(NOT keyword_set(xra)) then xra= [3501., 10499.] - -if(n_elements(outbase) eq 0 OR $ - n_elements(plate) eq 0 OR $ - n_elements(fiber) eq 0) then $ - message, 'Usage: sdss_spec_image, outbase, plate, fiber, mjd=' - -if(n_elements(outbase) gt 1 OR $ - n_elements(plate) gt 1 OR $ - n_elements(fiber) gt 1) then $ - message, 'OUTBASE, PLATE, FIBER must be scalar' - -if(keyword_set(noclobber) gt 0 and $ - file_test(outbase+'.png') gt 0 and $ - file_test(outbase+'.thumb.png') gt 0) then $ - return - -readspec, plate, fiber, mjd=mjd, zans=zans, flux=flux, wave=wave, $ - invvar=invvar, run2d=run2d, run1d=run1d, topdir=topdir, plug=plug, $ - silent=silent, legacy=legacy, plates=plates - -ipl= where(plate eq plans.plateid, npl) -igd= where(invvar gt 0, ngd) -if(ngd gt 0) then begin - ist= min(igd) - ind= max(igd) - flux= flux[ist:ind] - invvar= invvar[ist:ind] - wave= wave[ist:ind] -endif - -if(NOT keyword_set(axis_char_scale)) then axis_char_scale= 1.75*sscale -if(NOT keyword_set(tiny)) then tiny=1.d-4 - -bangp=!P -bangx=!X -bangy=!Y -!P.FONT= -1 -set_plot, "PS" -!P.BACKGROUND= djs_icolor('white') -!P.COLOR= djs_icolor('black') - -device, file=outbase+'.ps',/inches,xsize=xsize,ysize=ysize, $ - xoffset=(8.5-xsize)/2.0,yoffset=(11.0-ysize)/2.0,/color -!P.THICK= 2.0*sscale -!P.CHARTHICK= !P.THICK & !X.THICK= !P.THICK & !Y.THICK= !P.THICK -!P.CHARSIZE= 1.0 -!P.PSYM= 0 -!P.LINESTYLE= 0 -!P.TITLE= '' -!X.STYLE= 1 -!X.CHARSIZE= axis_char_scale -!X.MARGIN= [1,1]*0.5 -!X.OMARGIN= [7,7]*axis_char_scale -!X.RANGE= 0 -!X.TICKS= 0 -!Y.STYLE= 1 -!Y.CHARSIZE= !X.CHARSIZE -!Y.MARGIN= 0.6*!X.MARGIN -!Y.OMARGIN= 0.6*!X.OMARGIN -!Y.RANGE= 0 -!Y.TICKS= !X.TICKS -!P.MULTI= [1,1,1] -xyouts, 0,0,'!6' -colorname= ['red','green','blue','magenta','cyan','dark yellow', $ - 'purple','light green','orange','navy','light magenta', $ - 'yellow green'] -ncolor= n_elements(colorname) -loadct,0,/silent - -sflux= sdss_spec_smooth(alog10(wave), flux, 100.) - -igd= where(invvar gt 0 and abs(wave-5577.) gt 4. and wave lt 10000. and wave gt 3700., ngd) - -if(ngd gt 0) then $ - yra= minmax(sflux[igd]) $ -else $ - yra= minmax(sflux) -size= 0.07*(yra[1]-yra[0]) -yra=yra+[-1.2,1.7]*size*1.7 -if(yra[0] lt -2.) then yra[0]=-1.999 - -djs_plot, [0], [0], xra=xra, yra=yra, /nodata, $ - xtitle='!6Wavelength (Angstroms)!6', $ - ytitle='!8f_\lambda !6(10^{-17} erg/s/cm^2/Ang)!6' - -if(npl gt 0) then begin - title0= 'Survey: !8'+strtrim(plans[ipl].survey,2)+ $ - '!6 Program: !8'+strtrim(plans[ipl].programname,2) - targets='' - targ_title='Target' - if(plans[ipl].survey eq 'sdss') then $ - targets= strtrim(strjoin(sdss_flagname('TARGET', plug.primtarget),' '),2) - if(plans[ipl].survey eq 'segue1') then $ - targets= strtrim(strjoin(sdss_flagname('SEGUE1_TARGET', plug.primtarget),' '),2) - if(plans[ipl].survey eq 'boss') then $ - targets= strtrim(strjoin(sdss_flagname('BOSS_TARGET1', plug.boss_target1),' '),2)+ $ - ' '+strtrim(strjoin(sdss_flagname('ANCILLARY_TARGET1', plug.ancillary_target1),' '),2) - if(plans[ipl].survey eq 'bhm-mwm') then begin - targ_title='Firstcarton' - targets= plug.firstcarton - endif - if(plans[ipl].survey eq 'bhm') then begin - targ_title='Firstcarton' - targets= plug.firstcarton - endif - if(plans[ipl].survey eq 'mwm') then begin - targ_title='Firstcarton' - targets= plug.firstcarton - endif - if(plans[ipl].survey eq 'mwm-bhm') then begin - targ_title='Firstcarton' - targets= plug.firstcarton - endif - if (strtrim(targets,2) eq 'NA') then begin - targ_title='Firstcarton' - targets=strtrim(plug.objtype,2) - endif - title0= title0+' !6'+targ_title+': !8'+targets+'!6' -endif else begin - prog=strtrim(plug.program,2) - if strlen(prog) eq 0 then prog='NA' - fcart=strtrim(plug.firstcarton,2) - if strlen(fcart) eq 0 then fcart='NA' - title0= 'Program: !8'+prog+'!6,'+$ - ' Firstcarton: !8'+fcart+'!6' -endelse - - -lab_temp='Field=' - -if tag_exist(zans,'fiber_ra') then begin - title1= 'RA='+strtrim(string(f='(f40.5)', zans.fiber_ra),2)+', '+ $ - 'Dec='+strtrim(string(f='(f40.5)', zans.fiber_dec),2)+', '+ $ - lab_temp+strtrim(string(zans.field),2)+', '+ $ - 'TargetIndex='+strtrim(string(zans.target_index),2)+', '+ $ - 'MJD='+strtrim(string(zans.mjd),2) -endif else begin - title1= 'RA='+strtrim(string(f='(f40.5)', zans.plug_ra),2)+', '+ $ - 'Dec='+strtrim(string(f='(f40.5)', zans.plug_dec),2)+', '+ $ - lab_temp+strtrim(string(zans.field),2)+', '+ $ - 'Fiber='+strtrim(string(zans.fiberid),2)+', '+ $ - 'MJD='+strtrim(string(zans.mjd),2) -endelse - -if (not keyword_set(legacy)) then begin - title0=title0+', '+'CatID='+strtrim(string(plug.catalogid),2) -endif - - -if(zans.z lt 1000./299792.) then $ - zstr= '!8cz='+strtrim(string(long(zans.z*299792.)),2)+'+/-'+ $ - strtrim(string(long(zans.z_err*299792.)),2)+'!6 km/s' $ -else $ - zstr= '!8z='+strtrim(string(f='(f40.5)',zans.z),2)+'\pm!8'+ $ - strtrim(string(f='(f40.5)',zans.z_err),2) +'!6' -title2= zstr+', Class='+strtrim(zans.class) -if tag_exist(zans,'subclass') then title2+=' '+strtrim(zans.subclass) -mag_vec=plug.mag -m_i=mag_vec[3] -title2=title2+', mag!8_{i,fib2}='+strtrim(string(f='(f40.2)',m_i),2)+'!6' -warnings= strtrim(strjoin(sdss_flagname('ZWARNING', zans.zwarning),' '),2) -;;; print, zans.zwarning -if(keyword_set(warnings) gt 0) then $ - title3= 'Warnings: '+warnings $ -else $ - title3='No warnings' -title3=title3+', idlspec2d='+run2d - - - -xst= !X.CRANGE[0]+0.01*(!X.CRANGE[1]-!X.CRANGE[0]) -if(keyword_set(title0)) then begin - yst= !Y.CRANGE[0]+1.14*(!Y.CRANGE[1]-!Y.CRANGE[0]) - xyouts, xst, yst, title0, charsize=1.2*sscale -endif -yst= !Y.CRANGE[0]+1.10*(!Y.CRANGE[1]-!Y.CRANGE[0]) -djs_xyouts, xst, yst, title1, charsize=1.2*sscale -yst= !Y.CRANGE[0]+1.06*(!Y.CRANGE[1]-!Y.CRANGE[0]) -djs_xyouts, xst, yst, title2, charsize=1.2*sscale -yst= !Y.CRANGE[0]+1.02*(!Y.CRANGE[1]-!Y.CRANGE[0]) -xyouts, xst, yst, title3, charsize=1.2*sscale - -err= fltarr(n_elements(sflux)) -igd= where(invvar gt 0, ngd) -if(ngd gt 0) then $ - err[igd]= sqrt(1./invvar[igd]) -ibd= where(invvar le 0, nbd) -if(nbd gt 0) then $ - err[ibd]= (yra[1]-yra[0]) -fillx= [wave, reverse(wave)] -filly= [sflux+err, reverse(sflux-err)] -fillx= [fillx, fillx[0]] -filly= [filly, filly[0]] - -oploterror, wave, sflux, err, /nohat, errcolor=djs_icolor('light gray'), errthick=4 -;polyfill, fillx, filly, noclip=0, color=djs_icolor('light gray') - -djs_oplot, wave, sflux, th=6 - - -awave= [ 4300., 5895., 5175., 8498., 8542., 8662., 3968., 3938.] -aname= [ "G", "Na D", "Mg", "CaII", "", "", "H", "K"] - -ewave= [3727., 3869.7867, 4105.8884, 4341.6803, 4364.3782, $ - 4862.6778, 4960.2140, 5008.1666, 5876., 6301.9425, $ - 6549.7689, 6564.6127, 6585.1583, 6718.1642, 6732.5382, $ - 7137.6370, 2800., 1216., 1549., 1640., 1909., 2326., 1400. ] - -ename= textoidl(['OII', 'NeIII', 'H\delta', 'H\gamma', 'OIII', $ - 'H\beta', '', 'OIII', 'HeI', 'OI', 'NII', $ - 'H\alpha', 'NII', '', 'SII', 'ArIII', 'MgII', 'Ly\alpha', $ - 'CIV', 'HeII', 'CIII', 'CII', 'SiIV+OIV']) - -xsize= 0.07*(xra[1]-xra[0]) - -yoff= replicate(0., n_elements(ewave)) -xoff= replicate(0., n_elements(ewave)) -ioff= where(abs(ewave-6585.) lt 3., noff) -xoff[ioff]=0.3*xsize -yoff[ioff]=-0.5*size -ioff= where(abs(ewave-6549.) lt 3., noff) -xoff[ioff]=-0.3*xsize -yoff[ioff]=-0.5*size -ioff= where(abs(ewave-4862.) lt 3., noff) -xoff[ioff]=-0.1*xsize -ioff= where(abs(ewave-5008.) lt 3., noff) -xoff[ioff]=0.05*xsize -ioff= where(abs(ewave-4364.) lt 3., noff) -xoff[ioff]=0.20*xsize -ioff= where(abs(ewave-4341.) lt 3., noff) -xoff[ioff]=-0.05*xsize -yoff[ioff]=size - -ewave=ewave*(1.+zans.z) -iwave= where(ewave gt xra[0] and ewave lt xra[1], nwave) -for j=0L, nwave-1L do begin - i= iwave[j] - inear= where(wave gt ewave[i]-100. and wave lt ewave[i]+100 and $ - invvar gt 0 and abs(wave-5577.) gt 4., nnear) - if(nnear gt 0) then begin - eval= max(sflux[inear]) - djs_oplot, [ewave[i], ewave[i]], $ - eval[0]*1.05+[0.,size]+yoff[i], color='blue', th=3 - djs_xyouts, ewave[i]+xoff[i], eval[0]*1.05+yoff[i]+size*1.2, align=0.5, $ - ename[i], noclip=0, charsize=sscale, charthick=2. - endif -endfor - -yoff= replicate(-size*0.2, n_elements(awave)) -xoff= replicate(0., n_elements(awave)) - -ioff= where(abs(awave-3938.) lt 3., noff) -xoff[ioff]=-0.07*xsize -ioff= where(abs(awave-3968.) lt 3., noff) -xoff[ioff]=0.07*xsize - -awave=awave*(1.+zans.z) -iwave= where(awave gt xra[0]+100. and awave lt xra[1]-100., nwave) -for j=0L, nwave-1L do begin - i= iwave[j] - inear= where(wave gt awave[i]-100. and wave lt awave[i]+100 and $ - invvar gt 0 and abs(wave-5577.) gt 4., nnear) - if(nnear gt 0) then begin - aval= min(sflux[inear]) - djs_oplot, [awave[i], awave[i]], $ - aval[0]*0.95-[0.,size]+yoff[i], color='red', th=3 - djs_xyouts, awave[i]+xoff[i], aval[0]*0.95+yoff[i]-size*1.5, align=0.5, $ - aname[i], noclip=0, charsize=sscale, charthick=2. - endif -endfor - -device,/close -if(keyword_set(bangp)) then !P=bangp -if(keyword_set(bangx)) then !X=bangx -if(keyword_set(bangy)) then !Y=bangy -set_plot,'x' - -if(not keyword_set(silent)) then splog, 'Converting to JPG image.' -spawn, ['convert',outbase+'.ps', '-quality', '100', outbase+'.jpg'], /nosh -spawn, ['convert',outbase+'.jpg', outbase+'.gif'], /nosh -spawn, ['convert',outbase+'.gif', outbase+'.png'], /nosh -if(not keyword_set(silent)) then splog, 'Making thumbnail image.' -spawn, ['convert',outbase+'.png', '-scale', '180x130', outbase+'.thumb.png'], $ - /nosh -rmfile, outbase+'.ps' -rmfile, outbase+'.jpg' -rmfile, outbase+'.gif' -if(not keyword_set(silent)) then splog, 'Done.' - -end diff --git a/pro/spec1d/solvefilter.pro b/pro/spec1d/solvefilter.pro deleted file mode 100644 index 8a8c441a1..000000000 --- a/pro/spec1d/solvefilter.pro +++ /dev/null @@ -1,823 +0,0 @@ -;+ -; NAME: -; solvefilter -; -; PURPOSE: -; Solve for the 2.5-m imaging filter curves by using the spectra. -; -; CALLING SEQUENCE: -; solvefilter, [ filttype=, filternum=, plate=, mjd=, $ -; starerr=, qsoerr=, wavemin=, wavemax=, magrej=, sncut=, maxiter=, $ -; fluxpath=, value=, fixed= ] -; -; INPUTS: -; -; OPTIONAL INPUTS: -; filttype - Type of functional form for the filter curve. Options are: -; 'sdss': Modified SDSS filter curve (default). 3 params. -; 'tanh': Function with tanh() shape at edges. 5 params. -; filternum - Filter number, 1=g, 2=r, 3=i; default to 3. -; plate - Plate number(s); if not specified, then select all DR1 plates -; with number > 431. -; mjd - MJD for each PLATE. -; starerr - Fractional error to add in quadrature to photometric errors -; for stars; default to 0.05; if <=0, then do not use stars. -; qsoerr - Fractional error to add in quadrature to photometric errors -; for QSOs; default to 0.15; if <=0, then do not use QSOs. -; wavemin - Minimum wavelength for spectra during computation; -; default to 3800 Ang. -; wavemax - Maximum wavelength for spectra during computation; -; default to 9300 Ang. -; magrej - Reject any objects where the raw photo vs. spectro magnitude -; difference is more than MAGREJ from the median difference -; for that plate. This will reject wild outliers, which are -; often objects where there is a bright blend but the PHOTO -; flux is only for a fainter child. Default value is 0.5 mag. -; Or, QSOs that have varied. -; sncut - Minimum SN_MEDIAN (median S/N per pixel) for spectroscopic -; objects used in sample; default to 2.0 -; maxiter - Maximum number of iterations in call to MPFIT(); default to 200 -; fluxpath - Path name for spPlate files used for reading the spectra. -; The spZ files are still read from $BOSS_SPECTRO_REDUX/$PLATE -; regardless of this keyword. -; value - Initial guess values for the fit parameters. The default -; values are chosen to closely match the Gunn Jun-2001 curves. -; fixed - A vector of elements set to 0 for each parameter to be fit, -; and 1 for each parameter value to fix. Default to fitting -; all parameters. -; -; OUTPUTS: -; -; OPTIONAL OUTPUTS: -; -; COMMENTS: -; For FILTTYPE='tanh', the following function response(loglam) is fit: -; m = (a[4] - 1) / (a[1] - a[0]) -; b = 1 - m * a[0] -; Filter = tahn((loglam - a[0])*a[2]) + 1) -; * tahn((a[1] - loglam)*a[3]) + 1) -; * (m * loglam + b) -; There are a total of five a[] parameters above. -; The default initial-guess values for g-band are: -; a = [alog10(3950), alog10(5325), 100, 140, 2.0] -; The default initial-guess values for r-band are: -; a = [alog10(5580), alog10(6750), 130, 160, 1.2] -; The default initial-guess values for i-band are: -; a = [alog10(6915), alog10(8210), 150, 220, 0.55] -; -; For FILTTYPE='sdss', we fit a modified version of the Gunn Jun-2001 -; filter curves. The wavelength scale is remapped with a shift and -; rescaling, which has the effect of moving the filter edges. The -; filter shape is also multiplied by a function that is linear in -; log-wavelength, which has the effect of changing the broad-band -; slope of the filter. Given a filter curve Gunnfilt(loglam), it is -; re-mapped as follows: -; slopeterm = (loglam - 3.5)^theta[2] -; Filter(loglam) = Gunnfilt((loglam + a[1]) * a[0]) * slopeterm -; The default initial-guess values for the three a[] parameters are always: -; a = [0, 1.0, 0.01] -; The effect of a positive a[2] makes the filter slope more upwards -; with wavelength, and a negative a[2] makes it slope more downwards. -; -; Iteratively solve for the SDSS 2.5-m filter curves, using one of -; the several possible parameterizations as specified by FILTTYPE. -; For each possible filter curve, we regress the spectroscopic magnitude -; vs. the photometric magnitude. In order to not be sensitive to -; photometric calibration errors, each group of objects (same RUN, RERUN, -; CAMCOL, PLATE, SPECTROGRAPHID) is allowed to have a floating zero-point -; offset. These offsets are plotted as MAGOFFSET in one of the final plots. -; -; We only use spectroscopically-confirmed stars and QSOs, not any galaxies. -; Anything targetted as a galaxy is rejected. Any blended objects are -; rejected. Anything with the following bits set is rejected: -; OBJECT2_SATUR_CENTER, OBJECT2_INTERP_CENTER, OBJECT2_PSF_FLUX_INTERP. -; These last cuts remove stars with CRs in the core on the i-band images -; that were targetted as QSOs -- this was actually due to a bug in PHOTO -; that called stars CRs when the seeing was too good, then the incorrect -; CR-removal made the i-band 2 mags fainter, and these things were targetted -; as QSOs. -; -; All calculations are done in vacuum wavelengths, but then converted -; to air wavelengths at the end. -; -; EXAMPLES: -; Solve for the i-band filter using Tremonti's re-reductions of the spectra: -; IDL> solvefilter, filternum=3, fluxpath='/scr/wire50/cat/recalib/kurucz' -; -; BUGS: -; I should average together more telluric spectra for better S/N. -; I should use the extinction coeff for each imaging night. -; Do Christy's flux-calibrations improve things? -; Do any objects argue for light leaks? -; Should we more heavily weight the QSOs? -; -; DATA FILES: -; $IDLUTILS/data/filters/sdss_jun2001_$FILTER_atm.dat -; $BOSS_SPECTRO_REDUX/0432/spFrame-r2-00007466.fits* (for telluric-correction) -; $BOSS_SPECTRO_REDUX/$PLATE/spPlate-$PLATE-$MJD.fits -; $BOSS_SPECTRO_REDUX/$PLATE/spZbest-$PLATE-$MJD.fits -; $BOSS_SPECTRO_REDUX/plates/tsObj*-$PLATE.fit -; -; PROCEDURES CALLED: -; dfpsclose -; dfpsplot -; djs_int2bin() -; djs_maskinterp() -; djs_oplot -; djs_plot -; djs_xyouts -; headfits() -; idlspec2d_version() -; mpfit() -; mrdfits() -; readcol -; readspec -; sdss_run2mu() (in photoop product) -; skymask() -; splog -; tai2airmass() -; traceset2xy -; wavevector() -; -; INTERNAL SUPPORT ROUTINES: -; solvefiltshape() -; solvefiltfn() -; -; REVISION HISTORY: -; 05-Nov-2002 Written by David Schlegel, Princeton. -;- -;------------------------------------------------------------------------------ -forward_function mpfit, solvefiltfn - -;------------------------------------------------------------------------------ -; Construct the filter curve corresponding to this set of parameters -; Return the Gunn filter curve if THETA is not set. - -function solvefiltshape, theta, loglam - - common com_solvefilt, groupnum, bigloglam, taugunn, $ - bigflux, photoflux, photoinvsig, tauextinct, ntot, nbigpix, $ - airmass, gunnfilt, filternum, filttype, groupratio, spectroflux - - if (NOT keyword_set(theta)) then $ - return, gunnfilt[*,filternum] - - case filttype of - 'tanh': begin - mm = (theta[4] - 1.d0) / (theta[1] - theta[0]) - bb = 1.d0 - mm * theta[0] - fcurve = (tanh((loglam - theta[0])*theta[2]) + 1.d0) $ - * (tanh((theta[1] - loglam)*theta[3]) + 1.d0) $ - * (mm * loglam + bb) - end - 'sdss': begin - slopeterm = (bigloglam - 3.5d0)^theta[2] - linterp, bigloglam, gunnfilt[*,filternum] * slopeterm, $ - (bigloglam - theta[0]) * theta[1], fcurve - end - endcase - - if (total(finite(fcurve)) NE n_elements(fcurve)) then $ - message, 'NaN in filter shape' - - fcurve = fcurve > 0 - - return, fcurve -end -;------------------------------------------------------------------------------ -function solvefiltfn, theta - - common com_solvefilt, groupnum, bigloglam, taugunn, $ - bigflux, photoflux, photoinvsig, tauextinct, ntot, nbigpix, $ - airmass, gunnfilt, filternum, filttype, groupratio, spectroflux - - ngroup = max(groupnum) + 1 - groupratio = dblarr(ngroup) - - ; Construct the filter curve corresponding to this set of parameters - fcurve = solvefiltshape(theta, bigloglam) - sumfilt = total(fcurve) - - ; Change from f_lambda to f_nu - flambda2fnu = 10^(2*bigloglam) / 2.99792d18 * 10^((48.6 - 2.5*17.)/2.5) - - ; Loop through each group of spectra, integrate over the filter curve, - ; and minimize the spectro/photo flux normalization for that group. - spectroflux = dblarr(ntot) - leftall = dblarr(ntot) - rightall = dblarr(ntot) - for igroup=0L, ngroup-1 do begin - indx = where(groupnum EQ igroup, nthis) - - ; Get the extinction curve for these objects. If THETA is undefined, - ; then use the Gunn extinction curve. - meanair = mean(airmass[indx]) -; if (NOT keyword_set(theta)) then fextinct = exp(-meanair * taugunn) $ - if (NOT keyword_set(theta)) then fextinct = 1. $ ; ??? - else fextinct = exp(-meanair * tauextinct) - fmult = fcurve * flambda2fnu * fextinct - - spectroflux[indx] = $ - total(bigflux[*,indx] * rebin(fmult,nbigpix,nthis), 1) $ - / (sumfilt + (sumfilt LE 0)) - leftval = spectroflux[indx] * photoinvsig[indx] - rightval = photoflux[indx] * photoinvsig[indx] - denom = total(leftval^2) - if (denom GT 0) then groupratio[igroup] = total(leftval * rightval) / denom $ - else groupratio[igroup] = 1 - leftval = leftval * groupratio[igroup] - leftall[indx] = leftval - rightall[indx] = rightval - endfor - - ; Return a vector of all the chi's. - return, leftall - rightall -end -;------------------------------------------------------------------------------ -pro solvefilter, filttype=filttype1, filternum=filternum1, $ - plate=plate, mjd=mjd, starerr=starerr, qsoerr=qsoerr, $ - wavemin=wavemin, wavemax=wavemax, $ - magrej=magrej, sncut=sncut, maxiter=maxiter, fluxpath=fluxpath, $ - value=value, fixed=fixed - - ;---------- - ; Set common block - - common com_solvefilt, groupnum, bigloglam, taugunn, $ - bigflux, photoflux, photoinvsig, tauextinct, ntot, nbigpix, $ - airmass, gunnfilt, filternum, filttype, groupratio, spectroflux - - if (keyword_set(filttype1)) then filttype = filttype1 $ - else filttype = 'sdss' - if (n_elements(filternum1) NE 0) then filternum = filternum1 $ - else filternum = 3 ; Default to i-band - if (n_elements(starerr) EQ 0) then starerr = 0.05 - if (n_elements(qsoerr) EQ 0) then qsoerr = 0.15 - if (NOT keyword_set(wavemin)) then wavemin = 3800.d0 - if (NOT keyword_set(wavemax)) then wavemax = 9300.d0 - if (NOT keyword_set(magrej)) then magrej = 0.5 - if (NOT keyword_set(sncut)) then sncut = 2.0 - if (NOT keyword_set(maxiter)) then maxiter = 200 - wcovcut = 0.32 - filtname = ['u','g','r','i','z'] - if (starerr LE 0) then usestars = 0 $ - else usestars = 1 - if (qsoerr LE 0) then useqsos = 0 $ - else useqsos = 1 - - if (filternum LT 1 OR filternum GT 3) then $ - message, 'I only can cope with FILTERNUM=1,2, or 3' - - t0 = systime(1) - - bigloglam = wavevector(alog10(wavemin), alog10(wavemax)) - nbigpix = n_elements(bigloglam) - - ;---------- - ; Read Gunn's measurement of the filter -- including the telluric bands. - - gunnfilt = dblarr(nbigpix,n_elements(filtname)) - for ifilt=0, n_elements(filtname)-1 do begin - filename = filepath('sdss_jun2001_'+filtname[ifilt]+'_atm.dat', $ - root_dir=getenv('IDLUTILS_DIR'), subdirectory=['data','filters']) - readcol, filename, fwave1, fthru1, fthru2, fthru3, fext1, /silent - - ; Convert wavelengths to vacuum. - airtovac, fwave1 - - fthru = 0.5 * (fthru1 + fthru2) / fext1 ; Average these two columns - linterp, alog10(fwave1), fthru, bigloglam, gunnfilt1 - gunnfilt[*,ifilt] = gunnfilt1 - - ; Assemble the data to get the atmospheric extinction curve - if (ifilt EQ 0) then begin - fwave = fwave1 - fext = fext1 - endif else begin - fwave = [fwave, fwave1] - fext = [fext, fext1] - endelse - endfor - - isort = uniq(fwave, sort(fwave)) - fwave = fwave[isort] - fext = fext[isort] - linterp, alog10(fwave), fext, bigloglam, gunnextinct - taugunn = -alog(gunnextinct) / 1.3 ; Scale from 1.3 to 1.0 airmasses - - ;---------- - ; Set the structure to pass to MPFIT - - blankpar = {value:0.D, fixed:0, limited:[0b,0b], $ - limits:[0.D,0], mpmaxstep: 0.D} - - case filttype of - 'tanh': begin - parinfo = replicate(blankpar, 5) - if (filternum EQ 1) then $ - parinfo.value = [alog10(3950), alog10(5325), 100, 140, 2.0] - if (filternum EQ 2) then $ - parinfo.value = [alog10(5580), alog10(6750), 130, 160, 1.2] - if (filternum EQ 3) then $ - parinfo.value = [alog10(6915), alog10(8210), 150, 220, 0.55] - parinfo.limited = [[0,1], [1,0], [1,0], [1,0], [1,0]] - logmid = 0.5 * (parinfo[0].value + parinfo[1].value) - parinfo.limits = [[0,logmid-5.d-4], [logmid+5.d-4,0], $ - [0.0,0], [0.0,0], [0,0]] - parinfo.mpmaxstep = [5.d-4, 5.d-4, 10., 10., 0.05] - end - 'sdss': begin - parinfo = replicate(blankpar, 3) - parinfo.value = [0.d-5, 1.000, 0.01] - parinfo.mpmaxstep = [5.d-4, 0.01, 0.02] - end - else: message, 'Unknown FILTTYPE' - endcase - - if (keyword_set(value)) then parinfo.value = value - if (keyword_set(fixed)) then parinfo.fixed = fixed - - ;---------- - ; Construct the atmospheric extinction curve - - ; Start with a simple expression for the extinction -; tausimple = 10^(12.4 - 3.6 * bigloglam) - - ; Start with Gunn's extinction curve, but interpolating - ; over the telluric bands which will be replaced with - ; high-resolution spectra of those features. - bigwave = 10^bigloglam - vactoair, bigwave - tmask = (bigwave GT 6800 AND bigwave LT 7000) $ - OR (bigwave GT 7100 AND bigwave LT 7400) $ - OR (bigwave GT 7550 AND bigwave LT 7750) $ - OR (bigwave GT 8050 AND bigwave LT 8350) - tausimple = djs_maskinterp(taugunn, tmask, /const) - - framefile = filepath('spFrame-r2-00007466.fits*', $ - root_dir=getenv('BOSS_SPECTRO_REDUX'), subdir='0432') - fcalibfile = filepath('spFluxcalib-r2-00007466.fits*', $ - root_dir=getenv('BOSS_SPECTRO_REDUX'), subdir='0432') - framefile = (findfile(framefile))[0] - fcalibfile = (findfile(fcalibfile))[0] - hdr = headfits(framefile) - thisair = sxpar(hdr, 'AIRMASS') - wset = mrdfits(framefile, 3) - traceset2xy, wset, xx, tloglam - telluric = mrdfits(fcalibfile, 0) - linterp, tloglam[*,0], telluric[*,0], bigloglam, tellcorr - tautelluric = -alog(tellcorr)/thisair ; Scale back to one airmass - tauextinct = (tausimple + tautelluric) > 0 - - ;---------- - ; Find the list of good plates if not provided. - ; Default to using all DR1 plates with PLATE>431. - - if (keyword_set(plate)) then begin - splog, 'Using user-supplied list of plates' - plist = replicate(create_struct('plate', 0L, 'mjd', 0L), $ - n_elements(plate)) - plist.plate = plate - if (keyword_set(mjd)) then begin - plist.mjd = mjd - endif else begin - ; Determine the MJD for each plate number - readspec, plate, replicate(1,n_elements(plate)), mjd=mjd1 - plist.mjd = mjd1 - endelse - endif else begin - splog, 'Find list of good plates' - platelist, plist=plist - iuse = where(strmatch(plist.public,'*DR1*') AND plist.plate GT 431 $ - AND strmatch(plist.status1d,'Done*')) - plist = plist[iuse] - endelse - - ;---------- - ; Make certain that the data exists for all these plates, - ; especially if PATH is set to read the flux vectors from - ; some non-standard directory. - - splog, 'Test existence of flux data for all plates' - nplate = n_elements(plist) - qkeep = bytarr(nplate) - for iplate=0, nplate-1 do begin - readspec, plist[iplate].plate, mjd=plist[iplate].mjd, path=fluxpath, $ - objhdr=objhdr - if (keyword_set(objhdr)) then qkeep[iplate] = 1B - endfor - ikeep = where(qkeep, nplate) - if (nplate EQ 0) then $ - message, 'No plate data found in specified path' - plist = plist[ikeep] - splog, 'Number of usable plates = ', nplate - - ;---------- - ; Loop over each plate, and accumulate groups of objects - - nplate = n_elements(plist) - gcounter = 0L - for iplate=0, nplate-1 do begin - splog, 'Reading redshift data for plate #', plist[iplate].plate, $ - ' (', iplate+1, ' of ', nplate, ')' - readspec, plist[iplate].plate, mjd=plist[iplate].mjd, $ - zans=zans, tsobj=tsobj, plug=plug, /silent - - ; Identify objects targetted as galaxies (we don't want these) - qgalaxy = (plug.primtarget AND 2L^6+2L^7+2L^8+2L^5+2L^26) NE 0 - - ; Identify objects that are not blends - bflag = djs_int2bin(ulong(tsobj.objc_flags), ndigit=32) - bflag2 = djs_int2bin(ulong(tsobj.objc_flags2), ndigit=32) - qblend = transpose(bflag[3,*] EQ 1 AND bflag[6,*] EQ 0) - qbright = transpose(bflag[1,*]) - qchild = transpose(bflag[4,*]) - qsingle = (qblend EQ 0) AND (qbright EQ 0) AND (qchild EQ 0) - - ; Find which objects have interpolated pixels near the center - ; (we'll throw them out) - qinterp = transpose(bflag2[11,*] OR bflag2[12,*] OR bflag2[15,*]) - - ; Compute the airmass for each object -; junk = sdss_run2mu(tsobj.run, tsobj.field, tai=tai) -; airmass1 = tai2airmass(zans.plug_ra, zans.plug_dec, tai=tai) -; if (min(airmass1) LT 0.99 OR max(airmass1) GT 3.5) then $ -; message, 'Invalid AIRMASS' - airmass1 = tsobj.airmass[2] - - ; Group objects with the same run+rerun+camcol+plate+spectrographid - idstring = string(tsobj.run) + string(tsobj.rerun) $ - + string(tsobj.camcol) + string(zans.plate) + string(plug.spectrographid) - idlist = idstring[ uniq(idstring, sort(idstring)) ] - ngroup = n_elements(idlist) - - ; Reject wild mag outliers, which are often objects where there - ; is a bright blend but the PHOTO flux is only for a fainter child - magdiff = - 2.5 * alog10(zans.spectroflux[filternum]) $ - - tsobj.psfflux[filternum] - meddiff = median(magdiff) - - qstar = strmatch(zans.class,'STAR*') - qqso = strmatch(zans.class,'QSO*') - - for igroup=0, ngroup-1 do begin - indx = where(idstring EQ idlist[igroup]) - - igood = where(zans[indx].zwarning EQ 0 $ - AND qsingle[indx] EQ 1 $ - AND qgalaxy[indx] EQ 0 $ - AND qinterp[indx] EQ 0 $ - AND (qstar[indx]*usestars OR qqso[indx]*useqsos) $ - AND magdiff[indx] GT meddiff-magrej $ - AND magdiff[indx] LT meddiff+magrej $ - AND zans[indx].wcoverage GT wcovcut $ - AND zans[indx].sn_median GT sncut, ngood) - - if (ngood GE 2) then begin - if (gcounter EQ 0) then begin - groupnum = replicate(gcounter, ngood) - zall = zans[indx[igood]] - tsall = tsobj[indx[igood]] - airmass = airmass1[indx[igood]] - endif else begin - groupnum = [groupnum, replicate(gcounter, ngood)] - zall = [zall, zans[indx[igood]]] - tsall = [tsall, tsobj[indx[igood]]] - airmass = [airmass, airmass1[indx[igood]]] - endelse - gcounter = gcounter + 1 - endif - endfor - endfor - ntot = n_elements(zall) - splog, 'Number of spectra = ', ntot - splog, 'Number of groups = ', max(groupnum)+1 - - ;---------- - ; Construct the big matrices - - bigflux = fltarr(nbigpix, ntot) - - ;---------- - ; Read in the actual spectra - - splog, 'Reading spectra' - for iplate=0, nplate-1 do begin - splog, 'Reading spectra for plate #', plist[iplate].plate, $ - ' (', iplate+1, ' of ', nplate, ')' - indx = where(zall.plate EQ plist[iplate].plate $ - AND zall.mjd EQ plist[iplate].mjd, nthis) - if (nthis GT 0) then begin - ; Read in the spectra, and interpolate over bad points - fiberid = zall[indx].fiberid - readspec, plist[iplate].plate, mjd=plist[iplate].mjd, fiberid, $ - flux=objflux, loglam=loglam, invvar=objivar, $ - andmask=andmask, ormask=ormask, path=fluxpath, /align - npix = n_elements(loglam) - objivar = skymask(objivar, andmask, ormask) - objflux = djs_maskinterp(objflux, objivar LE 0, iaxis=0, /const) - - if (bigloglam[0] LT loglam[0]) then begin - i1 = (where(bigloglam GE loglam[0]))[0] - j1 = 0L - endif else begin - i1 = 0L - j1 = (where(loglam GE bigloglam[0]))[0] - endelse - ncopy = (nbigpix - i1) < (npix - j1) - bigflux[i1:i1+ncopy-1,indx] = objflux[j1:j1+ncopy-1,*] - endif - endfor -objflux = 0 -andmask = 0 -ormask = 0 -objivar = 0 - - ;---------- - ; Decide upon the object counts and errors from PHOTO. - -; photoflux = 10.d0^(-tsall.psfcounts[filternum]/2.5) -; photoflerr = tsall.psfcountserr[filternum] * abs(photoflux) - photoflux = tsall.psfflux[filternum] - photoflivar = tsall.psfflux_ivar[filternum] - qgood = photoflivar GT 0 - - ; Add an additional error term - qstar = strmatch(zall.class,'STAR*') - qqso = strmatch(zall.class,'QSO*') -; photoinvsig = 1. / sqrt( photoflerr^2 $ -; + (qstar * starerr * abs(photoflux))^2 $ -; + (qqso * qsoerr * abs(photoflux))^2 ) - photoinvsig = qgood / sqrt( 1./(photoflivar+qgood-1) $ - + (qstar * starerr * abs(photoflux))^2 $ - + (qqso * qsoerr * abs(photoflux))^2 ) - - ; Now convert these to AB flux, according to the numbers derived - ; by Hogg on 13 Aug 2002. - aboffsets = [-0.042, 0.036, 0.015, 0.013, -0.002] - photoflux = photoflux * 10.d0^(-aboffsets[filternum]/2.5) - - ;---------- - ; Compute the chi^2 for the initial guess parameters - - origchi = solvefiltfn(parinfo.value) - dof = ntot - n_elements(parinfo) - ngroup - origrchi2 = total(origchi^2) / dof - splog, 'Start Chi2/DOF = ', origrchi2 - - ;---------- - ; Do the actual fit to the filter curve - - t1 = systime(1) - theta = mpfit('solvefiltfn', $ - parinfo=parinfo, perror=perror, maxiter=maxiter, $ - nfev=nfev, niter=niter, status=status) - t2 = systime(1) - chivec = solvefiltfn(theta) - chi2pdof = total(chivec^2) / dof - - splog, 'Time for non-linear fitting = ', t2-t1, ' sec' - splog, 'Number of iterations = ', niter - splog, 'Number of function evaluations = ', nfev - splog, 'Fit values = ', theta - splog, 'Fit errors = ', perror - splog - splog, 'Median |chi| = ', median(abs(chivec)) - splog, 'Final Chi2/DOF = ', chi2pdof - - ;---------- - ; Identify the 10 most deviant points - - nworst = 10 - iworst = (reverse(sort(abs(chivec))))[0:nworst-1] - - ;---------- - ; Reconstruct the filters at 1.3 airmasses - - ; Gunn filter curve - finitial = solvefiltshape(parinfo.value, bigloglam) * exp(-1.3 * tauextinct) - finitial = finitial * mean(gunnfilt[*,filternum]) / mean(finitial) - junk1 = solvefiltfn() ; Force evaluation of spectroflux - initdiff1 = -2.5 * alog10(spectroflux / photoflux) - initdiff2 = -2.5 * alog10(spectroflux * groupratio[groupnum] / photoflux) - - ; 1st guess filter curve - fguess = solvefiltshape(parinfo.value, bigloglam) * exp(-1.3 * tauextinct) - fguess = fguess * mean(gunnfilt[*,filternum]) / mean(fguess) - junk1 = solvefiltfn(parinfo.value) ; Force evaluation of spectroflux - guessdiff1 = -2.5 * alog10(spectroflux / photoflux) - guessdiff2 = -2.5 * alog10(spectroflux * groupratio[groupnum] / photoflux) - - ; Best-fit filter curve - fbest = solvefiltshape(theta, bigloglam) * exp(-1.3 * tauextinct) - fbest = fbest * mean(gunnfilt[*,filternum]) / mean(fbest) - fbest = fbest * (fbest GT 0) + 0.0 * (fbest LE 0) ; Get rid of values -0.00 - junk2 = solvefiltfn(theta) ; Force evaluation of spectroflux - magdiff1 = -2.5 * alog10(spectroflux / photoflux) - magdiff2 = -2.5 * alog10(spectroflux * groupratio[groupnum] / photoflux) - - ;---------- - ; Derive the reduced chi^2 for each plate - - rchi2plate = fltarr(nplate) - for iplate=0, nplate-1 do begin - indx = where(zall.plate EQ plist[iplate].plate $ - AND zall.mjd EQ plist[iplate].mjd, nthis) - if (nthis GT 0) then begin - ngroup1 = n_elements(uniq(groupnum[indx])) - thisdof = nthis - ngroup1 - rchi2plate[iplate] = total(chivec[indx]^2) / thisdof - endif - endfor - - ;---------- - ; Compute the magnitude shift that we needed to apply to each - ; group of spectroscopic mags to agree with the photo mags - - magoffset = -2.5 * alog10(groupratio) - - ;---------- - ; Make plots - - datestring = strlowcase(string((strsplit(systime(),/extract))[[2,1,4]], $ - format='(i2.2,a,a)')) - plottitle = 'Best-Fit ' + filtname[filternum]+'-band Filter ' + datestring - plotfile = 'sdss_djs_' + datestring + '_' + filtname[filternum] + '.ps' - - csize = 1.1 - dfpsplot, plotfile, /square, /color - - xrange = [ bigwave[(where(fbest GT 0.01))[0]] - 300, $ - bigwave[(reverse(where(fbest GT 0.01)))[0]] + 300 ] - plot, bigwave, gunnfilt[*,filternum]>fguess>fbest, /nodata, $ - xtitle='Air Wavelength [Ang]', ytitle='Filter Response at 1.3 Airmass', $ - charsize=csize, xrange=xrange, /xstyle, title=plottitle - djs_oplot, bigwave, gunnfilt[*,filternum], color='cyan' - djs_oplot, bigwave, fguess, color='red' - djs_oplot, bigwave, fbest, color='green' - xplot = total(!x.crange * [0.95,0.05]) - yplot = !y.crange[1] - djs_xyouts, xplot, 0.32*yplot, 'Mamoru/Gunn Jun-2001', $ - charsize=csize, color='cyan' - djs_xyouts, xplot, 0.26*yplot, 'Initial guess for fit' $ - + ' \chi^2_r=' + string(origrchi2,format='(f6.3)'), $ - charsize=csize, color='red' - djs_xyouts, xplot, 0.20*yplot, 'Schlegel Best-Fit '+datestring $ - + ' \chi^2_r=' + string(chi2pdof,format='(f6.3)'), $ - charsize=csize, color='green' - djs_xyouts, xplot, 0.14*yplot, 'DOF =' + string(dof) $ - + ' \Delta \chi^2 =' + string((origrchi2-chi2pdof)*dof), $ - charsize=csize - - !p.multi = [0,1,2] - iplot = where(rchi2plate GT 0) - djs_plot, [plist[iplot].plate], [rchi2plate[iplot]], psym=4, $ - xtitle='Plate Number', ytitle='\chi^2 / DOF', $ - charsize=csize, title=plottitle - - ; These are the values that we would *subtract* from the spectro mags - iuniq = uniq(groupnum) - plot, [zall[iuniq].plate], [magoffset], psym=4, $ - xtitle='Plate Number', ytitle='(SPECTRO - PHOTO) Mag Offset per group', $ - charsize=csize, title=plottitle - !p.multi = 0 - - !p.multi = [0,1,2] - plot, zall.plate + zall.fiberid/1000., guessdiff1, psym=3, $ - xtitle='Plate Number', ytitle='(SPECTRO - PHOTO) w/out mag offsets', $ - charsize=csize, title='Initial Guess Filter' - oplot, !x.crange, [0,0] - plot, zall.plate + zall.fiberid/1000., magdiff1, psym=3, $ - yrange=!y.crange, /ystyle, $ ; Use same Y plotting limits as above - xtitle='Plate Number', ytitle='(SPECTRO - PHOTO) w/out mag offsets', $ - charsize=csize, title=plottitle - oplot, !x.crange, [0,0] - - plot, zall.plate + zall.fiberid/1000., guessdiff2, psym=3, $ - xtitle='Plate Number', ytitle='(SPECTRO - PHOTO) w/ mag offsets', $ - charsize=csize, title='Initial Guess Filter' - oplot, !x.crange, [0,0] - plot, zall.plate + zall.fiberid/1000., magdiff2, psym=3, $ - yrange=!y.crange, /ystyle, $ ; Use same Y plotting limits as above - xtitle='Plate Number', ytitle='(SPECTRO - PHOTO) w/ mag offsets', $ - charsize=csize, title=plottitle - oplot, !x.crange, [0,0] - ; Label the NWORST worst points, according to their chi-deviation - djs_xyouts, total(!x.crange*[0.95,0.05]), total(!y.crange*[0.08,0.92]), $ - 'Worst outliers by \chi^2 in red', color='red' - djs_oplot, zall[iworst].plate+zall[iworst].fiberid/1000., $ - magdiff[iworst], psym=4, color='red' - for i=0, nworst-1 do $ - djs_xyouts, zall[iworst[i]].plate+zall[iworst[i]].fiberid/1000., $ - magdiff2[iworst[i]], string(plate_to_string(zall[iworst[i]].plate), $ - zall[iworst[i]].mjd, zall[iworst[i]].fiberid, $ - format='(" ",a,"/",i5,"-",i3," ")'), orient=90, $ - align=(magdiff2[iworst[i]] LT 0), color='red' - - istar = where(strmatch(zall.class,'STAR*'), nstar) - iqso = where(strmatch(zall.class,'QSO*'), nqso) -; photocolor = tsall.psfcounts[2] - tsall.psfcounts[3] - photocolor = -2.5 * alog10(tsall.psfflux[2] / tsall.psfflux[3]) - - if (nstar GT 1) then begin - plot, photocolor[istar], guessdiff2[istar], psym=3, charsize=csize, $ - xtitle='(r-i) for stars', ytitle='(SPECTRO - PHOTO) w/ mag offsets', $ - title='Initial Guess Filter' - oplot, !x.crange, [0,0] - isort = sort(photocolor[istar]) - djs_oplot, photocolor[istar[isort]], $ - djs_median(guessdiff2[istar[isort]],width=51{now}
"]) + mask_b = np.full_like(meds_b, np.NaN) + mask_b[np.where((beta_b >= beta) & (beta_b <=beta+dbeta))] = 1 + mask_b[:,np.where(np.nansum(mask_b,axis=0)<=1)[0]] = np.NaN + + mask_r = np.full_like(meds_r, np.NaN) + mask_r[np.where((beta_r >= beta) & (beta_r <=beta+dbeta))] = 1 + mask_r[:,np.where(np.nansum(mask_r,axis=0)<=1)[0]] = np.NaN + + if not ((np.nansum(mask_b) > 0) or (np.nansum(mask_r) > 0)): + if ptt.exists(outfile): + remove(outfile) + return + makedirs(output_dir,exist_ok=True) + + med_plot(Allflat,directory,outfile, version, obsf, meds_b*mask_b, meds_r*mask_r, + qbad_b, qbad_r, fiberids, title=title, name=outtitle, + mask_b=mask_b, mask_r=mask_r) + + +def make_pos_plots(Allflat,directory, version, obs, meds_b, meds_r, qbad_b, qbad_r, fiberids, drot, dalt, alt, rot, subf): + now =today.strftime("Last Updated: %a %b %d %H:%M:%S %Y (MJD: "+'{:.3f}'.format(Time(today).mjd)+")") + if drot is None: + output_dir=ptt.join(directory, obs, 'alt',subf) + indx = np.where((Allflat['ALT'].value >= alt) & (Allflat['ALT'].value <= alt+dalt))[0] + outfile=ptt.join(output_dir,f'alt{str(alt).zfill(3)}-{str(alt+dalt).zfill(3)}.html') + outtitle=f'alt{alt}-{alt+dalt}' + title = '\n'.join([f"{now}
"]) + elif dalt is None: + output_dir=ptt.join(directory, obs, 'rot',subf) + indx = np.where((Allflat['ROT'].value >= rot) & (Allflat['ROT'].value <= rot+drot))[0] + outfile=ptt.join(output_dir,f'rot{str(rot).zfill(3)}-{str(rot+drot).zfill(3)}.html') + outtitle=f'rot{rot}-{rot+drot}' + title = '\n'.join([f"{now}
"]) + else: + output_dir=ptt.join(directory, obs, 'tele_pos',subf) + indx = np.where((Allflat['ALT'].value >= alt) & (Allflat['ALT'].value <= alt+dalt) & + (Allflat['ROT'].value >= rot) & (Allflat['ROT'].value <= rot+drot))[0] + outfile=ptt.join(output_dir,f'alt{str(alt).zfill(3)}-{str(alt+dalt).zfill(3)}_rot{str(rot).zfill(3)}-{str(rot+drot).zfill(3)}.html') + outtitle=f'alt{alt}-{alt+dalt}_rot{rot}-{rot+drot}' + title = '\n'.join([f"{now}
"]) + if len(indx) == 0: + if ptt.exists(outfile): + remove(outfile) + return + makedirs(output_dir,exist_ok=True) + med_plot(Allflat[indx],directory,outfile, version, obs, meds_b[indx], meds_r[indx], + qbad_b[indx], qbad_r[indx], fiberids[indx], title=title, name=outtitle) + +def position_plots(Allflat,directory, version, obs, meds_b, meds_r, qbad_b, qbad_r,beta_b, beta_r, fiberids, plates=False): + makedirs(ptt.join(directory, obs, 'tele_pos'),exist_ok=True) + makedirs(ptt.join(directory, obs, 'Rot'), exist_ok=True) + makedirs(ptt.join(directory, obs, 'alt'), exist_ok=True) + if plates is False: makedirs(ptt.join(directory, obs, 'beta_ang'),exist_ok=True) + fibers=fiberids#np.tile(np.arange(1,501),(len(qbad_b),1)) + + tqdm.write('Making drot=dalt plots') + for step in [5,10,15]: + for alt in np.arange(25,90,step): + for rot in np.arange(0, 360, step): + make_pos_plots(Allflat,directory, version, obs, meds_b, meds_r, qbad_b, qbad_r, fibers, step, step, alt, rot, str(step)) + + tqdm.write('Making drot or dalt plots') + for step in [5,10,15]: + for deg in np.arange(25,90,step): + make_pos_plots(Allflat, directory, version, obs, meds_b, meds_r, qbad_b, qbad_r, fibers, None, step, deg, None, str(step)) + make_pos_plots(Allflat, directory, version, obs, meds_b, meds_r, qbad_b, qbad_r, fibers, step, None, None, rot, str(step)) + tqdm.write('Making dBeta plots') + if plates is False: + for step in [5,10,15]: + for beta in np.arange(0,180,step): + make_beta_plots(Allflat,directory, version, obs, meds_b, meds_r, qbad_b, qbad_r, beta_b, beta_r, fibers, beta, step, str(step)) + +def hex_to_rgb_tuple(hex_color): + # Convert hex to RGB tuple + rgb_tuple = tuple(int(hex_color.lstrip('#')[i:i+2], 16) for i in (0, 2, 4)) + return f'rgb({rgb_tuple[0]},{rgb_tuple[1]},{rgb_tuple[2]})' + +def generate_discrete_sequence(base_scale, num_colors): + # Convert all base scale colors to RGB strings + base_scale_rgb = [hex_to_rgb_tuple(color) for color in base_scale] + + # Create a linear space for interpolating colors across the entire base scale + t_values = np.linspace(0, 1, num_colors) + + # Generate the discrete color sequence + discrete_sequence = [] + for t in t_values: + # Find the appropriate position in the base scale + scaled_t = t * (len(base_scale_rgb) - 1) + idx = int(scaled_t) + fractional_t = scaled_t - idx + + # Handle edge case where idx is at the last color + if idx >= len(base_scale_rgb) - 1: + discrete_sequence.append(base_scale_rgb[-1]) + else: + color = pc.find_intermediate_color(base_scale_rgb[idx], base_scale_rgb[idx + 1], fractional_t, colortype='rgb') + discrete_sequence.append(color) + + return discrete_sequence + +def col2html(row, raw=False, ccd = 'Blue', directory=''): + bsd = 'BOSS_SPECTRO_DATA_N' if row['Obs'] == 'apo' else 'BOSS_SPECTRO_DATA_S' + bsd = chpc2html(getenv(bsd)) + if raw: + template = ('PNG '+ + 'FITs') + else: + template = ('PNG '+ + 'FITs '+ + 'Trans') + col = f'{ccd} Raw' if raw else f'{ccd} Reduced' + return(template.format(value=row[col], directory = chpc2html(directory), + bsd=bsd, obs= row['Obs'])) + +def med_plot(Allflat,directory,filename, ver, obsf, meds_b, meds_r, qbad_b, qbad_r, + fiberids, title=None, name=None, mask_b=None, mask_r=None): + ccds = ['b1','r1'] if obsf == 'apo' else ['b2','r2'] + umjs = np.unique(Allflat['MJD']) + discrete_palette = generate_discrete_sequence(pc.sequential.Viridis, len(umjs)) + random.shuffle(discrete_palette) + + obs = Allflat['OBS'].value.tolist() + mjd =Allflat['MJD'].value.astype(str).tolist() + expsids = np.char.zfill(Allflat['EXP'].value.astype(str),8) + red_reduced=create_att_path(obs,mjd,expsids,ccd=ccds[1],ftype='spFlat') + red_raw=create_att_path(obs,mjd,expsids,ccd=ccds[1],ftype='sdR') + blue_reduced=create_att_path(obs,mjd,expsids,ccd=ccds[0],ftype='spFlat') + blue_raw=create_att_path(obs,mjd,expsids,ccd=ccds[0],ftype='sdR') + red_trans=create_att_path(obs,mjd,expsids,ccd=ccds[1], ftype='spFlat', subdir='transmission/') + blue_trans=create_att_path(obs,mjd,expsids,ccd=ccds[0], ftype='spFlat', subdir='transmission/') + + meta = pd.DataFrame({'exp':np.char.zfill(Allflat['EXP'].value.astype(str),8), + 'MJD':Allflat['MJD'].value.astype(str).tolist(), + 'TAI':Allflat['TAI'].value.tolist(), + 'qbad_b':qbad_b, + 'qbad_r':qbad_r, + 'airmass':Allflat['AIRMASS'].value.tolist(), + 'Alt':Allflat['ALT'].value.tolist(), + 'rot':Allflat['ROT'].value.tolist(), + 'Obs':Allflat['OBS'].value.tolist(), + 'Blue Raw':blue_raw, + 'Blue Reduced':blue_reduced, + 'Red Raw':red_raw, + 'Red Reduced':red_reduced}) + + meta['Blue Raw'] = meta.apply(lambda row: col2html(row, raw = True, ccd = 'Blue', directory=directory), axis=1) + meta['Blue Reduced'] = meta.apply(lambda row: col2html(row, ccd = 'Blue', directory=directory), axis=1) + meta['Red Raw'] = meta.apply(lambda row: col2html(row, raw = True, ccd = 'Red', directory=directory), axis=1) + meta['Red Reduced'] = meta.apply(lambda row: col2html(row, raw = False, ccd = 'Red', directory=directory), axis=1) + + hr = '{now}
", + f"N Flats Included: {str(np.nansum(mask_b,axis=0)[0])}
"]) + + if name is None: + name = f"{obsf.upper()} BOSS Flats: {ver}" + + fignames = {'all':f'BOSSFlats_{obsf.upper()}_{ver}', + 'select':f'BOSSFlats_{obsf.upper()}_{ver}_select', + 'div':f'BOSSFlats_{obsf.upper()}_{ver}_PerMaxDev', + 'std':f'BOSSFlats_{obsf.upper()}_{ver}_STD', + 'mad':f'BOSSFlats_{obsf.upper()}_{ver}_MAD', + 'count':f'BOSSFlats_{obsf.upper()}_{ver}_Count'} + for fig in figs.keys(): + config = {'toImageButtonOptions': {'format': 'png','filename': fignames[fig], + 'scale': 6 }, # Multiply title/legend/axis/canvas sizes by this factor + 'responsive': True} # Ensure the figure is responsive + + fig_params = dict(full_html=False, default_height='900px', default_width='100%', + include_plotlyjs='cdn', config=config) + figs[fig] = figs[fig].to_html(**fig_params) + plotly_jinja_data = {"fig": figs['all'], + "fig_selct": figs['select'], + "fig_meta": meta.to_html(classes='scrollable-table',escape=False, index=False), + "fig_dev": figs['div'], + "fig_std": figs['std'], + "fig_mad": figs['mad'], + "fig_count": figs['count'], + "title":title, "name":name, "favicon":favicon} + + with open(filename, "w", encoding="utf-8") as output_file: + with open(ptt.join(idlspec2d_dir,'templates','html','FieldLib_template.html')) as template_file: + j2_template = Template(template_file.read()) + output_file.write(j2_template.render(plotly_jinja_data)) + + + +def csv_dump(Allflat, directory, version, fobs, meds_dic, qbad_dic, beta_dic, confs_dic, confFiberids_dic): + makedirs(ptt.join(directory,fobs,'csv'), exist_ok=True) + fiberid = np.arange(1,501) + exps=np.char.zfill(Allflat['EXP'].value.astype(str),8) + rot=Allflat['ROT'].value + alt=Allflat['ALT'].value + obs=Allflat['OBS'].value + mjd=Allflat['MJD'].value + try: tai=Allflat['TAI'].value + except: tai=np.full_like(Allflat['MJD'].value.tolist(), np.NaN) + ccds = ['b1','r1'] if fobs == 'apo' else ['b2','r2'] + + for i,row in enumerate(tqdm(meds_dic[ccds[0]], desc='CSV EXP', leave=False, position = 1)): + flat_meta=pd.DataFrame() + for ccd in ccds: + flat_meta=pd.concat([flat_meta,pd.DataFrame({'FIBERID':fiberid, 'CONFFIBERID': confFiberids_dic[ccd][i], + 'EXPID':[exps[i]]*500, + 'SPECTROGRAPH':[ccd]*500,'CONFIGID':confs_dic[ccd][i], + 'MED_FLAT_VALUE':meds_dic[ccd][i], 'ROT':rot[i]*500, + 'ALT':[alt[i]]*500, 'OBS':np.array([obs[i]]*500), + 'MJD':[mjd[i]]*500, 'TAI':[tai[i]]*500, + 'BETA':beta_dic[ccd][i]})], ignore_index=False) + flat_meta.to_csv(ptt.join(directory,fobs,'csv','summary_'+exps[i]+'.csv'),index=False) + +def analysis(directory, version, mjd=None, noplot=False, obs='apo', + lowFiber=.8, run='all', TraceIDs=False): + Allflat=Table(fits.getdata(ptt.join(directory,'calibs','allflats.fits'),1)) + Allflat = Allflat[Allflat['OBS'] == obs] + Allflat= tabunique(Allflat, keys='EXP') + Allflat.sort('EXP') + #Allflat = Allflat[:10] + if run in ['all','med']: + if mjd is None: + outfile=ptt.join(directory,"FlatAnalysis-"+version+"_"+obs+".html") + else: + Allflat=Allflat[np.where(np.array(Allflat['MJD'].value.tolist()).astype(int) == int(mjd))] + outfile=ptt.join(directory,"FlatAnalysis-"+version+"-mjd"+str(mjd)+"_"+obs+".html") + if max(np.asarray(Allflat['MJD'].value).astype(int)) < 59550: + plates = True + else: plates = False + meds_dic={} + qbad_dic={} + beta_dic={} + assigns_dic={} + confs_dic={} + confFiberids_dic={} + ccds = ['b1','r1'] if obs == 'apo' else ['b2','r2'] + if run in ['all','med','pos','csv','lowfiber']: + for ccd in tqdm(ccds, desc='CCDs', position=0): + meds=None + qbad=[] + confFiberids = None + betas=None + confs=None + for i, exp in enumerate(tqdm(Allflat['EXP'].value.tolist(),desc='Exposure', position=1, leave=False)): + + if obs not in assigns_dic.keys(): + assigns_dic[obs]=read_fiberAssignments(ptt.join(directory,'fiberAssignments',obs,'fiberAssignments.csv')) + ff = ptt.join(directory,'calibs', obs, str(Allflat['MJD'].value.tolist()[i]),'spFlat-'+ccd+'-'+str(exp).zfill(8)+'.fits.gz') + try: + data=fits.getdata(ff,0) + mid=data.shape[1]//2 + data=data[:,mid-1000:mid+1000] + med=np.median(data,axis=1) + except: + data=None + med = np.full(500,np.NaN) + if data is not None: + hdr=fits.getheader(ff,0) + try: + confid = hdr['CONFID'] + except: + confid = None + if confid is not None: + confSummary = find_confSummary(confid, obs) + else: + confSummary = None + else: + confSummary = None + if confSummary is not None: + beta = confSummary['beta'].value + confFiberid = confSummary['fiberId'].value + else: + beta = np.full(500,np.NaN) + confFiberid = np.full(500,np.NaN) + + if betas is None: + meds = med + betas = beta + confFiberids = confFiberid + confs = np.array([confid]*len(beta)) + else: + meds=np.vstack([meds,med]) + betas=np.vstack([betas,beta]) + confFiberids=np.vstack([confFiberids,confFiberid]) + confs=np.vstack([confs,np.array([confid]*len(beta))]) + + if run in ['all','med','pos','lowfiber']: + ID_lowFiber(ff, ptt.join(directory, 'calibs', obs, 'lowFlagged', str(Allflat['MJD'].value.tolist()[i])) ,med, confFiberids, threshold=lowFiber) + if run in ['lowfiber','csv']: continue + if ptt.exists(ff) : + qbad.append(0) + if noplot is False: + mjdstr=str(Allflat['MJD'].value.tolist()[i]) + makedirs(ptt.join(directory, 'calibs', obs, mjdstr),exist_ok=True) + makedirs(ptt.join(directory, 'calibs', obs, 'transmission', mjdstr),exist_ok=True) + if not ptt.exists(build_savename(ff,ptt.join(directory, 'calibs', obs, mjdstr))): + plot_flat(ff,ptt.join(directory, 'calibs', obs, mjdstr)) + raw_flat=get_raw(ff,obs, mjdstr) + plot_raw(raw_flat,ptt.join(directory, 'calibs', obs, mjdstr)) + plot_thruput_v_sextant(ff,str(Allflat['MJD'].value.tolist()[i]), ptt.join(directory, 'calibs', obs, 'transmission', mjdstr), assigns_dic[obs]) + plot_thruput_v_sextant(ff,str(Allflat['MJD'].value.tolist()[i]), ptt.join(directory, 'calibs', obs, mjdstr), assigns_dic[obs]) + else: qbad.append(1) + if run not in ['csv']: + ID_lowFiber('merged_'+ccd, ptt.join(directory, 'calibs',obs, 'lowFlagged') ,meds, confFiberids, threshold=lowFiber, clobber=True) + meds_dic[ccd]=meds + qbad_dic[ccd]=np.asarray(qbad) + beta_dic[ccd]=betas + confs_dic[ccd]=confs + confFiberids_dic[ccd]=confFiberids + if run in ['lowfiber','csv']: continue + if run == 'lowfiber': return + + if run in ['all', 'med']: + tqdm.write('Producing med_plot') + + tabs=med_plot(Allflat,ptt.join(directory,'calibs'), + ptt.join(directory,f"FlatAnalysis-{version}_{obs}.html"), + version, obs, meds_dic[ccds[0]], + meds_dic[ccds[1]], qbad_dic[ccds[0]], + qbad_dic[ccds[1]], confFiberids_dic[ccds[0]]) + if run in ['all', 'pos']: + position_plots(Allflat, ptt.join(directory,'calibs'), version, obs, + meds_dic[ccds[0]], meds_dic[ccds[1]], + qbad_dic[ccds[0]], qbad_dic[ccds[1]], + beta_dic[ccds[0]], beta_dic[ccds[1]], + confFiberids_dic[ccds[0]], plates=plates) + if run in ['all','csv']: + csv_dump(Allflat, ptt.join(directory,'calibs'), version, obs, meds_dic, qbad_dic, beta_dic, confs_dic,confFiberids_dic) + if run in ['timeSeries', 'all']: + plot_thruput_timeseries(ptt.join(directory,'calibs', obs,'csv'), Traceid=TraceIDs) diff --git a/python/boss_drp/Flatlib/build.py b/python/boss_drp/Flatlib/build.py new file mode 100644 index 000000000..929cc5c8c --- /dev/null +++ b/python/boss_drp/Flatlib/build.py @@ -0,0 +1,54 @@ + +import glob +import os.path as ptt +from os import getenv +from astropy.io import fits +from astropy.coordinates import AltAz, EarthLocation +from astropy.time import Time +import astropy.units as u +from astropy.table import Table +from datetime import datetime, timedelta +from tqdm import tqdm +from collections import OrderedDict +def build(dir_, obs): + cols = {'MJD':'MJD','ROT':'ROTPOS','ALT':'ALT','AZ':'AZ','EXP':'exposure','TAI':'TAI','CARTID':'CARTID'} + catfile = ptt.join(dir_,'calibs','allflats.fits') + if ptt.exists(catfile): + catalog = Table.read(catfile, format='fits') + else: + catalog = None + for obs in obs: + sp= 1 if obs =='apo' else 2 + if obs == 'apo': + sp = 1 + location = EarthLocation(lat = 32.780361*u.deg, lon=254.179532*u.deg, height=2788*u.m) + tf = '%Y-%m-%dT%H:%M:%S' + else: + location = EarthLocation(lat = -29.01597*u.deg, lon=289.307920*u.deg, height=2380*u.m) + sp = 2 + tf = '%Y-%m-%dT%H:%M:%S.%f' + flatlib = ptt.join(dir_,'calibs',obs,'?????',f'spFlat-b{sp}*.fits*') + for ff in tqdm(glob.glob(flatlib)): + if catalog is not None: + if ptt.basename(ff).split('.')[0].split('-')[-1] in catalog['EXP']: + continue + hdr = fits.getheader(ff) + meta = OrderedDict({}) + for col in cols.keys(): + if col== 'TAI': + meta['TAI'] = hdr['TAI-BEG'] + (hdr['EXPTIME']/2.0) + else: + meta[col] = hdr[cols[col]] + meta['OBS'] = obs + meta['qbad_b'] = 1 + meta['qbad_r'] = 1 + obstime = Time(datetime.strptime(hdr['DATE-OBS'], tf) + timedelta(seconds=hdr['EXPTIME']/2.0)) + altaz = AltAz(alt= meta['ALT']*u.deg,az=meta['AZ']*u.deg,obstime=obstime,location=location) + meta['AIRMASS'] = altaz.secz.value + if catalog is None: + for key in meta.keys(): + meta[key] = [meta[key]] + catalog = Table(meta) + else: + catalog.add_row(meta) + catalog.write(catfile, overwrite=True) diff --git a/python/boss_drp/Flatlib/plot.py b/python/boss_drp/Flatlib/plot.py new file mode 100644 index 000000000..eadc98f3e --- /dev/null +++ b/python/boss_drp/Flatlib/plot.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +import os.path as ptt +import matplotlib.pyplot as plt +from mpl_toolkits.axes_grid1 import make_axes_locatable +from os import getenv, makedirs, sep, rename +from astropy.io import fits +import argparse +import glob +import numpy as np +import pandas as pd +import requests +from tqdm import tqdm + +def plot_flat(filename, save_dir): + loc=get_mdj_obs(filename) + outname, shortname = build_savename(filename,save_dir,return_short=True) + try: + flat_arr=fits.getdata(filename,0) + except: + print('error with '+filename) + flat_arr = [[np.NaN,np.NaN],[np.NaN,np.NaN]] + plt.figure(figsize=(12, 2), dpi=100) + plt.imshow(flat_arr,cmap='gray') + plt.xlabel('pixel') + plt.ylabel('Fiber') + plt.title('MJD:'+str(loc['mjd'])+' '+shortname) + plt.savefig(outname,bbox_inches='tight') + plt.close('all') + return + +def build_savename(filename, save_dir,return_short=False, mod=''): + makedirs(save_dir, exist_ok=True) + filename=ptt.basename(filename) + filename=ptt.splitext(ptt.splitext(filename)[0])[0] + if return_short is True: + return(ptt.join(save_dir, filename+mod+'.png'),filename) + return(ptt.join(save_dir, filename+mod+'.png')) + + +def read_fiberAssignments(filename): + if not ptt.exists(filename): + makedirs(ptt.dirname(filename), exist_ok=True) + if 'lco' in filename: + url = 'https://raw.githubusercontent.com/sdss/fps_calibrations/main/lco/wok_calibs/duPontFlatCMM/fiberAssignments.csv' + else: + url = 'https://raw.githubusercontent.com/sdss/fps_calibrations/main/apo/wok_calibs/sloanFlatCMM/fiberAssignments.csv' + response = requests.get(url) + if response.status_code == 200: + # Write the content to a local file + with open(filename, 'wb') as f: + f.write(response.content) + else: + raise Exception(f'Failed to download file: {response.status_code}') + + fiberAssign=pd.read_csv(filename,index_col=0) + fiberAssign=fiberAssign[np.isfinite(fiberAssign.BOSSFiber)] + fiberAssign=fiberAssign.sort_values(by=['BOSSFiber']) + return(fiberAssign) + +def plot_thruput_v_sextant(filename,mjd, save_dir, fiberAssignments): + outname, shortname = build_savename(filename, save_dir, return_short=True, mod='_trans') + def reject_outliers(data, m=2): + return data[abs(data - np.mean(data)) < m * np.std(data)] + + try: + data=fits.getdata(filename,0) + except: + print('error with '+filename) + data = [[np.NaN,np.NaN],[np.NaN,np.NaN]] + mean=np.mean(data,axis=1) + arr0=mean + arr=(reject_outliers(mean,m=3)) + filtdata=data + while np.abs(np.mean(arr)-np.mean(arr0)) > .0001: + arr0=arr + arr=reject_outliers(arr,m=3) + trans=mean/np.mean(arr) + + ax=plt.subplot() + for i in range(1, 7): + filt=np.where(fiberAssignments.Sextant.values == i)[0] + im=ax.scatter(fiberAssignments.Sextant.values[filt], trans[filt], marker='.', + lw=.1,ec='k',c=np.arange(len(filt)), cmap='CMRmap') + ax.set_xlabel('Sextant') + ax.set_ylabel('Relative Transmission (to mean)') + plt.title('MJD:'+str(mjd)+' '+shortname) + divider = make_axes_locatable(ax) + cax = divider.append_axes("right", size="2%", pad=0.05) + cbar=plt.colorbar(im, cax=cax) + cbar.set_label('Fiber within Sextant') + plt.savefig(outname, bbox_inches='tight') + plt.close('all') + return + +def plot_thruput_timeseries(csv_dir, Traceid=False): + files = glob.glob(ptt.join(csv_dir,'summary_*.csv')) + ts_r = None + ts_b = None + ids_b = None + ids_r = None + for file in tqdm(files, desc='Reading CSVs', leave=False, position=1): + df = pd.read_csv(file) + df_b = df[df['SPECTROGRAPH'].isin(['b1','b2'])] + df_r = df[df['SPECTROGRAPH'].isin(['r1','r2'])] + + if ts_r is None: + if not Traceid: + ids_b = df_b['CONFFIBERID'] + ids_r = df_r['CONFFIBERID'] + else: + ids_b = df_b['FIBERID'] + ids_r = df_r['FIBERID'] + ts_b = pd.DataFrame([df_b['MED_FLAT_VALUE'].tolist()], index=[df.iloc[0]['MJD']], columns = ids_b) + ts_r = pd.DataFrame([df_r['MED_FLAT_VALUE'].tolist()], index=[df.iloc[0]['MJD']], columns = ids_r) + + + else: + if not Traceid: + ids_b = df_b['CONFFIBERID'] + ids_r = df_r['CONFFIBERID'] + else: + ids_b = df_b['FIBERID'] + ids_r = df_r['FIBERID'] + mjd_u = df.iloc[0]['MJD'] + if mjd_u not in ts_b.index: + try: + ts_b.loc[mjd_u] = dict(zip(ids_b.tolist(),df_b['MED_FLAT_VALUE'].tolist())) + except: + tqdm.write('Error with '+file) + else: + temp = ts_b.loc[mjd_u].copy() + new_data = dict(zip(ids_b.tolist(), df_b['MED_FLAT_VALUE'].tolist())) + for key, value in new_data.items(): + temp[key] = value # Replace the value for duplicate keys + ts_b.loc[mjd_u] = temp + if mjd_u not in ts_r.index: + try: + ts_r.loc[mjd_u] = dict(zip(ids_r.tolist(),df_r['MED_FLAT_VALUE'].tolist())) + except: + tqdm.write('Error with '+file) + else: + temp = ts_r.loc[mjd_u].copy() + new_data = dict(zip(ids_r.tolist(), df_r['MED_FLAT_VALUE'].tolist())) + for key, value in new_data.items(): + temp[key] = value # Replace the value for duplicate keys + ts_r.loc[mjd_u] = temp + + ts_b = ts_b.sort_index() + ts_r = ts_r.sort_index() + makedirs(ptt.join(ptt.dirname(csv_dir),'timeseries'), exist_ok=True) + if Traceid: + flag = 'Trace ' #fiberid on chip + else: + flag = 'Fiber ' #Fiberid on slit + cols = ts_b.columns.tolist()+ts_r.columns.tolist() + cols = np.sort(np.unique(np.asarray(cols))).astype(int).tolist() + for i in range(len(cols)): + if((i % 10) == 0): + fig, axs= plt.subplots(2,1,figsize=(10,5), dpi =100) + s =i + axs[0].plot(ts_b.index.tolist(), ts_b[cols[i]], marker='.', label=f'{flag}{cols[i]}') + axs[1].plot(ts_r.index.tolist(), ts_r[cols[i]], marker='.', label=f'{flag}{cols[i]}') + axs[0].set_ylabel('Blue Relative \nTransmission (to mean)') + axs[1].set_ylabel('Red Relative \nTransmission (to mean)') + axs[1].set_xlabel('MJD') + axs[0].set_ylim([-.05,1.2]) + axs[1].set_ylim([-.05,1.2]) + axs[0].legend(ncols=10, fontsize=7, framealpha=.2,bbox_to_anchor=(0.5, 1.15),loc='upper center')#,bbox_transform=axs[0].transAxes,) + axs[1].legend(ncols=10, fontsize=7, framealpha=.2,bbox_to_anchor=(0.5, 1.15),loc='upper center')#,bbox_transform=axs[0].transAxes,) +# axs[0].legend(ncols=10, fontsize=8, framealpha=.2,bbox_to_anchor=(0.5, 1.05),loc='center',bbox_transform=axs[0].transAxes,) +# axs[1].legend(ncols=10, fontsize=8, framealpha=.2,bbox_to_anchor=(0.5, 1.05),loc='center',bbox_transform=axs[0].transAxes,) + if((i % 10) == 9) or (i==499): + fig.tight_layout() + plt.subplots_adjust()#top=.93) # Adjust top to fit the legends + if Traceid: + plt.savefig(ptt.join(ptt.dirname(csv_dir),'timeseries', + f'timeseries_Tracefiber{str(s+1).zfill(3)}-{str(i+1).zfill(3)}.png'),bbox_inches='tight') + else: + plt.savefig(ptt.join(ptt.dirname(csv_dir),'timeseries', + f'timeseries_fiber{str(s+1).zfill(3)}-{str(i+1).zfill(3)}.png'),bbox_inches='tight') + plt.close('all') + + if Traceid: + build_preview(ptt.dirname(csv_dir),'timeseries_Tracefiber', deep=1, nper=2,pattern='timeseries_Tracefiber*png') + with open(ptt.join(ptt.dirname(csv_dir),'timeseries_Tracefiber_data_blue.html'),'w') as f: f.write(ts_b.to_html()) + with open(ptt.join(ptt.dirname(csv_dir),'timeseries_Tracefiber_data_red.html'),'w') as f: f.write(ts_r.to_html()) + + else: + build_preview(ptt.dirname(csv_dir),'timeseries', deep=1, nper=2,pattern='timeseries_fiber*png') + with open(ptt.join(ptt.dirname(csv_dir),'timeseries_data_red.html'),'w') as f: f.write(ts_r.to_html()) + with open(ptt.join(ptt.dirname(csv_dir),'timeseries_data_blue.html'),'w') as f: f.write(ts_b.to_html()) + +def plot_raw(filename, save_dir): + loc=get_mdj_obs(filename) + img_arr=fits.getdata(filename,0) + filename=ptt.basename(filename) + filename=ptt.splitext(ptt.splitext(filename)[0])[0] + plt.figure(figsize=(6, 6), dpi=100) + plt.imshow(img_arr,cmap='gray') + plt.xlabel('pixel') + plt.ylabel('pixel') + plt.title('MJD:'+str(loc['mjd'])+' '+filename) + plt.savefig(ptt.join(save_dir, filename+'.png'),bbox_inches='tight') + plt.close('all') + return + +def get_mdj_obs(filename): + mjd=ptt.basename(ptt.dirname(ptt.normpath(filename))) + obs=ptt.basename(ptt.dirname(ptt.dirname(ptt.normpath(filename)))) + return({'mjd':mjd,'obs':obs}) + +def get_raw(filename, obs, mjd): + filename=ptt.basename(filename) + filename=filename.replace('spFlat-','sdR-') + filename=filename.replace('.fits.gz','.fit.gz') + if obs.lower() == 'apo': env='BOSS_SPECTRO_DATA_N' + else: env='BOSS_SPECTRO_DATA_S' + filename=ptt.join(getenv(env),str(mjd),filename) + return(filename) + +def build_preview(directory, name, deep=1, nper=2,pattern='spFlat-??-????????.png'): + sub = sep.join(['*']*deep) + figs = glob.glob(ptt.join(directory,sub,pattern)) + with open(ptt.join(directory, 'tmp-index.html'), 'w') as f: + f.write(''+'\n') + f.write(''+'\n') + f.write(''+'\n') + f.write(''+'\n') + f.write(''+'\n') + f.write('{ff} | \n')
+ if(((i % nper) == nper-1) or (i == len(figs)-1)): f.write(''+'\n')
+ f.write('
Last Update: """+time.ctime()+""", Last Update MJD: """+mjd+"""
-(S/N)^2 values are not corrected for galactic dust reddening
""" - else: - head3="""(S/N)^2 values are corrected for galactic dust reddening
""" + red = 'not ' if not legacy else '' foot = """ """ - splog.log(ptt.join(path,name)) - - with open(ptt.join(path,name), "w") as fhtml: - fhtml.write(head) - fhtml.write(head2) - fhtml.write(head3) - fhtml.write(html) - fhtml.write(foot) - fhtml.close() - -if __name__ == '__main__' : - """ - Build/load Fieldlist - """ - parser = argparse.ArgumentParser( - prog=ptt.basename(sys.argv[0]), - description='Build/load BOSS Fieldlist') - - parser.add_argument('--create', '-c', action='store_true', help='Create Fieldlist') - parser.add_argument('--topdir', type=str, help='Optional override value for the environment variable $BOSS_SPECTRO_REDUX', default = getenv('BOSS_SPECTRO_REDUX')) - parser.add_argument('--run1d', type=str, help='Optional override value for the enviro variable $RUN1D', nargs='*', default=[getenv('RUN1D')]) - parser.add_argument('--run2d', type=str, help='Optional override value for the enviro variable $RUN2D', nargs='*', default=[getenv('RUN2D')]) - parser.add_argument('--outdir', type=str, help='Optional output directory (defaults to topdir/$RUN2D)', default=None) - parser.add_argument('--skipcart', type=str, help='Option list of cartridges to skip', nargs='*', default=None) - parser.add_argument('--epoch', action='store_true', help='Produce FieldList for epoch coadds') - parser.add_argument('--basehtml', type=str, help='html path for figure (defaults to relative from topdir)') - parser.add_argument('--logfile', type=str, help='Manually Set logfile (including path)', default=None) - parser.add_argument('--debug', action='store_true', help='Overrides the logger of the simplified error messages and prints standard python errors') - parser.add_argument('--noplot', action='store_true', help='Skips updating the sky plots') - args = parser.parse_args() - - splog.no_exception = args.debug - Field_list = fieldlist(**vars(args)) + + fl_pd_full = fl_pd.copy() + fl_pd = None + for sort in sorts: + if sort.lower() == 'mjd': + fl_pd_full = fl_pd_full.sort_values(by=['MJD','FIELD'], ascending = False, key=lambda col: col.astype(int)) + else: + fl_pd_full = fl_pd_full.sort_values(by=['FIELD','MJD'], key=lambda col: col.astype(int)) + for obs in obss: + fl_pd = fl_pd_full.copy() + tname = name + + if obs is not None: + fl_pd = fl_pd.loc[fl_pd['OBS'] == obs] + tname = tname+'_'+obs.upper() + obsstr = obs.upper() + else: + obsstr = '' + + if sort.lower() == 'mjd': + tname = tname+'-mjdsort' + fl_pd.columns = fl_pd.columns.str.replace('_',' ', regex=False) + html = fl_pd.to_html(escape=False, render_links=True, index=False) + + + tname = tname+'.html' + splog.log(ptt.join(path,tname)) + template = ptt.join(idlspec2d_dir,'templates','html','fieldlist_template.html') + + jinja_data = dict(RUN2D=run2d,date=time.ctime(),MJD=mjd,obs=obsstr, + favicon=favicon, basehtml=basehtml,red=red, + FIELDLIST_TABLE=html, title=title.format(obs=obsstr)) + with open(ptt.join(path,tname), "w", encoding="utf-8") as output_file: + with open(template) as template_file: + j2_template = Template(template_file.read()) + output_file.write(j2_template.render(jinja_data)) + + + + + fl_pd = None + fl_pd_full = None + head = head2 = head3 = html = foot = thead2 = None + + diff --git a/bin/fieldmerge.py b/python/boss_drp/post/fieldmerge.py similarity index 54% rename from bin/fieldmerge.py rename to python/boss_drp/post/fieldmerge.py index b0f7a9ac4..4284cd1ac 100755 --- a/bin/fieldmerge.py +++ b/python/boss_drp/post/fieldmerge.py @@ -1,27 +1,34 @@ #!/usr/bin/env python3 +from boss_drp import idlspec2d_dir +from boss_drp.post.fieldlist import fieldlist +from boss_drp.field import field_to_string, field_spec_dir, fieldgroup +from boss_drp.field import field_dir as create_field_dir +from boss_drp.utils import (merge_dm, Splog, get_lastline) +from boss_drp.utils import match as wwhere +from boss_drp.post import plot_sky_targets, plot_sky_locations +from boss_drp.utils import specobjid, retry + +from sdss_semaphore.targeting import TargetingFlags import argparse import sys import os.path as ptt -from os import getenv, makedirs, remove +from os import getenv, makedirs, remove, rename import numpy as np from astropy.io import fits -from astropy.table import Table, Column, vstack, join, unique, setdiff -from fieldlist import wwhere, plot_sky, fieldlist, get_lastline +from astropy.table import (Table, Column, vstack, join, + unique, setdiff, hstack,MaskedColumn) from healpy import ang2pix from pydl.pydlutils.spheregroup import spheregroup import time -from datetime import timedelta +from datetime import timedelta, datetime import warnings -from field import field_to_string -from merge_dm import merge_dm from glob import glob -from splog import Splog import gc -splog = Splog() -run2d_warn = True +import shutil -from sdss_semaphore.targeting import TargetingFlags +run2d_warn = True +splog = Splog() def read_zans(sp1d_dir, field, mjd): @@ -29,9 +36,10 @@ def read_zans(sp1d_dir, field, mjd): if ptt.exists(zansfile): splog.log('Reading Zbest file: '+ptt.basename(zansfile)) try: - zans = Table(fits.getdata(zansfile)) + zans = Table.read(zansfile) for key in zans.colnames: zans.rename_column(key,key.upper()) + zans.meta = {} except: splog.log('Error reading '+zansfile) zans = None @@ -40,14 +48,16 @@ def read_zans(sp1d_dir, field, mjd): zans = None return(zans) -def read_zline(sp1d_dir, field, mjd): +def read_zline(sp1d_dir, field, mjd, obs): zlinefile = ptt.join(sp1d_dir,'spZline-' + field + '-' + mjd + '.fits') if ptt.exists(zlinefile): splog.log('Reading Zline file: '+ptt.basename(zlinefile)) try: - zline = Table(fits.getdata(zlinefile)) + zline = Table.read(zlinefile) for key in zline.colnames: zline.rename_column(key,key.upper()) + zline.add_column(obs, name = 'OBS') + zline.meta = {} except: splog.log('Error reading '+zlinefile) zline = None @@ -61,9 +71,10 @@ def read_xcsao(sp1d_dir, field, mjd, spAll): if ptt.exists(XCSAOfile): splog.log('Reading XCSAO file: '+ptt.basename(XCSAOfile)) try: - XCSAO_tab = Table(fits.getdata(XCSAOfile,1)) + XCSAO_tab = Table.read(XCSAOfile,1) for key in XCSAO_tab.colnames: XCSAO_tab.rename_column(key,key.upper()) + XCSAO_tab.meta = {} XCSAO_tab.remove_columns(['EBV','FIBERID_LIST','OBJID','FIELDID']) renames = {'RV':'XCSAO_RV','ERV':'XCSAO_ERV','R':'XCSAO_RXC','TEFF':'XCSAO_TEFF', @@ -98,12 +109,18 @@ def read_fibermap(field_dir, field, mjd, epoch=False, allsky=False): if ptt.exists(spfieldfile): splog.log('Reading Fibermap: '+ptt.basename(spfieldfile)) try: - fibermap = Table(fits.getdata(spfieldfile, 5)) + fibermap = Table.read(spfieldfile, 5) + fibermap.meta = {} hdr = fits.getheader(spfieldfile,0) - fibermap['SPEC_FILE'] = np.char.add(np.char.add('spec-'+field+'-'+mjd+'-', np.char.strip(np.asarray(fibermap['CATALOGID']).astype(str))),'.fits') - renames = {'XFOCAL_LIST':'XFOCAL','YFOCAL_LIST':'YFOCAL','CARTON_TO_TARGET_PK_LIST':'CARTON_TO_TARGET_PK','ASSIGNED_LIST':'ASSIGNED', - 'ON_TARGET_LIST':'ON_TARGET','VALID_LIST':'VALID','DECOLLIDED_LIST':'DECOLLIDED','ICATALOGID':'CATALOGID', - 'GAIA_ID_DR2': 'GAIA_ID', 'MJDLIST': 'MJD_LIST','PROGRAM':'PROGRAMNAME'} + fibermap['SPEC_FILE'] = np.char.add(np.char.add('spec-'+field+'-'+mjd+'-', + np.char.strip(np.asarray(fibermap['CATALOGID']).astype(str))),'.fits') + renames = {'XFOCAL_LIST':'XFOCAL','YFOCAL_LIST':'YFOCAL', + 'CARTON_TO_TARGET_PK_LIST':'CARTON_TO_TARGET_PK', + 'ASSIGNED_LIST':'ASSIGNED', 'ON_TARGET_LIST':'ON_TARGET', + 'VALID_LIST':'VALID','DECOLLIDED_LIST':'DECOLLIDED', + 'TOO_LIST':'TOO','ICATALOGID':'CATALOGID', + 'GAIA_ID_DR2': 'GAIA_ID', 'MJDLIST': 'MJD_LIST', + 'PROGRAM':'PROGRAMNAME'} for key in fibermap.colnames: fibermap.rename_column(key,key.upper()) @@ -122,23 +139,27 @@ def read_fibermap(field_dir, field, mjd, epoch=False, allsky=False): fibermap.add_column(hp, name = 'HEALPIX') fibermap.add_column(np.floor(fibermap['HEALPIX']/1000).astype(int), name = 'HEALPIXGRP') - - fibermap.add_column(-999, name = 'CATALOGID_V0') - fibermap.add_column(-999, name = 'CATALOGID_V0P5') - catid = fibermap['CATALOGID'] - cv0 = fibermap['CATALOGID_V0'] - cv0p5 = fibermap['CATALOGID_V0P5'] - iv0 = np.where(wwhere(fibermap['CATVERSION'].data,'0.0*'))[0] - iv0p1 = np.where(wwhere(fibermap['CATVERSION'].data,'0.1*'))[0] - iv0p5 = np.where(wwhere(fibermap['CATVERSION'].data,'0.5*'))[0] - cv0[iv0] = catid[iv0] - cv0[iv0p1] = catid[iv0p1] - cv0p5[iv0p5] = catid[iv0p5] + if 'CATALOGID_V0' not in fibermap.colnames: + fibermap.add_column(-999, name = 'CATALOGID_V0') + fibermap.add_column(-999, name = 'CATALOGID_V0P5') + fibermap.add_column(-999, name = 'CATALOGID_V1') + catid = fibermap['CATALOGID'] + cv0 = fibermap['CATALOGID_V0'] + cv0p5 = fibermap['CATALOGID_V0P5'] + cv1 = fibermap['CATALOGID_V1'] + iv0 = np.where(wwhere(fibermap['CATVERSION'].data.astype(str),'0.0*'))[0] + iv0p1 = np.where(wwhere(fibermap['CATVERSION'].data.astype(str),'0.1*'))[0] + iv0p5 = np.where(wwhere(fibermap['CATVERSION'].data.astype(str),'0.5*'))[0] + iv1 = np.where(wwhere(fibermap['CATVERSION'].data.astype(str),'1.*'))[0] + cv0[iv0] = catid[iv0] + cv0[iv0p1] = catid[iv0p1] + cv0p5[iv0p5] = catid[iv0p5] + cv1[iv1] = cv1[iv1] HEALPIX_PATH = np.char.add(np.char.add('$MWM_HEALPIX/', np.asarray(fibermap['HEALPIXGRP']).astype(str)),'/') HEALPIX_PATH = np.char.add(np.char.add(HEALPIX_PATH, np.asarray(fibermap['HEALPIX']).astype(str)),'/boss/') HEALPIX_PATH = np.char.add(HEALPIX_PATH, fibermap['SPEC_FILE']) - idx = np.where(catid == 0)[0] + idx = np.where(fibermap['CATALOGID'] == 0)[0] HEALPIX_PATH[idx] = '' fibermap.add_column(HEALPIX_PATH, name = 'HEALPIX_PATH') except: @@ -163,10 +184,11 @@ def read_spline(splinefile, skip_line=False, clobber=False): if exists and not skip_line: try: splog.log('Reading spline file: '+ptt.basename(splinefile)) - spline = Table(fits.getdata(splinefile)) + spline = Table.read(splinefile) if len(spline) == 0: remove(splinefile) spline = None + spline.meta = {} except: splog.log('Failure opening '+ splinefile) spline = None @@ -188,10 +210,11 @@ def read_spall(spAllfile, clobber=False): if exists: try: splog.log('Reading spAll file: '+ptt.basename(spAllfile)) - spAll = Table(fits.getdata(spAllfile)) + spAll = Table.read(spAllfile) if len(spAll) == 0: remove(spAllfile) spAll = None + spAll.meta = {} except: splog.log('Failure opening '+ spAllfile) spAll = None @@ -201,7 +224,6 @@ def read_spall(spAllfile, clobber=False): - def oneField(row, field, mjd, skip_line=False, include_bad=False, legacy=False, dev=False, skip_specprimary=False, XCSAO=False, indir='.', clobber=False, epoch=False, merge_only=False, custom = None, allsky=None): @@ -212,19 +234,18 @@ def oneField(row, field, mjd, skip_line=False, include_bad=False, legacy=False, row['RUN1D'] = row['RUN2D'] if custom is None: if epoch is True: - sp1d_dir = ptt.join(indir, row['RUN2D'], field, 'epoch', row['RUN1D']) - field_dir = ptt.join(indir,row['RUN2D'], field) + field_dir = create_field_dir(ptt.join(indir,row['RUN2D']), field) + sp1d_dir = ptt.join(field_dir, 'epoch', row['RUN1D']) else: - sp1d_dir = ptt.join(indir, row['RUN2D'], field, row['RUN1D']) - field_dir = ptt.join(indir, row['RUN2D'],field) + field_dir = create_field_dir(ptt.join(indir,row['RUN2D']), field) + sp1d_dir = ptt.join(field_dir, row['RUN1D']) else: if epoch is True: - sp1d_dir = ptt.join(indir, row['RUN2D'], custom, 'epoch', row['RUN1D']) - field_dir = ptt.join(indir,row['RUN2D'], custom) + field_dir = create_field_dir(ptt.join(indir,row['RUN2D']), field, custom=True) + sp1d_dir = ptt.join(field_dir, 'epoch', row['RUN1D']) else: - sp1d_dir = ptt.join(indir, row['RUN2D'], custom, row['RUN1D']) - field_dir = ptt.join(indir, row['RUN2D'],custom) - field = custom + field_dir = create_field_dir(ptt.join(indir,row['RUN2D']), field, custom=True) + sp1d_dir = ptt.join(field_dir, row['RUN1D']) dev = False spAllfile, spAlllitefile, splinefile, spAlldatfile = build_fname(indir, row['RUN2D'], field=field, mjd=mjd, @@ -283,10 +304,10 @@ def oneField(row, field, mjd, skip_line=False, include_bad=False, legacy=False, spAll.rename_column(key,key.upper()) spAll['FIBER_RA'] = spAll['FIBER_RA'].data.astype(float) spAll['FIBER_DEC'] = spAll['FIBER_DEC'].data.astype(float) - spAll = build_specobjid(spAll) + spAll = build_specobjid(spAll, epoch = epoch, custom = custom) if spline is None and skip_line is False: - spline = read_zline(sp1d_dir, field, mjd) + spline = read_zline(sp1d_dir, field, mjd, row['OBSERVATORY']) if spline is not None: if len(spline) > 0: @@ -302,52 +323,24 @@ def oneField(row, field, mjd, skip_line=False, include_bad=False, legacy=False, -def build_specobjid(spAll): - global run2d_warn - - lsh = lambda x,s: np.uint64(x)*np.uint64(2**s) - - - run2ds = spAll['RUN2D'].data - - run2ds = np.char.strip(run2ds) - rerun = np.zeros(len(run2ds)) - - if 'SPECOBJID' not in spAll.colnames: - spAll.add_column(np.uint64(0),name='SPECOBJID') +def build_specobjid(spAll,custom=None, epoch=False): + splog.info('Building SPECOBJIDs') + if epoch: + coadd = 'epoch' + elif custom is not None: + coadd = custom else: - spAll.remove_column('SPECOBJID') - spAll.add_column(np.uint64(0),name='SPECOBJID') - - for row in spAll: - try: - if isinstance(row['RUN2D'].strip(),str): - n,m,p = row['RUN2D'].strip().split('_') - n = int(n[1:]) - m = int(m) - p = int(p) - run2d = (n-5)*10000 + m*100 + p - elif row['RUN2D'].strip().isnumeric(): - run2d = int(run2d) - else: - if run2d_warn: - splog.log(f"WARNING: Unable to parse RERUN from {np.unique(run2ds)} for CAS-style SPECOBJID; Using 0 instead") - run2d_warn = False - run2d = 0 - except: - if run2d_warn: - splog.log(f"WARNING: Unable to parse RERUN from {np.unique(run2ds)} for CAS-style SPECOBJID; Using 0 instead") - run2d_warn = False - run2d = 0 - specobjid = np.uint64(0) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - specobjid |= lsh(row['FIELD'],50) | lsh(row['TARGET_INDEX'],38) | lsh(row['MJD']-50000,24) | lsh(run2d,10) - row['SPECOBJID'] = specobjid + coadd = 'daily' + spAll.add_column(Column('',name='SPECOBJID', dtype=object)) + spAll['SPECOBJID'] = specobjid.encode(spAll['SDSS_ID'].data, + spAll['FIELD'].data.astype(str), + spAll['MJD'].data, coadd, + spAll['RUN2D'].data.astype(str), + fiberid=spAll['TARGET_INDEX'].data, + allnew = False) return(spAll) - def specPrimary(spAll): t2 = time.time() # Determine the score for each object @@ -409,8 +402,7 @@ def specPrimary(spAll): splog.log('Time to assign primaries = '+str(timedelta(seconds = time.time() - t2))) return(spAll) - -def specPrimary_sdssid(spAll): +def specPrimary_sdssid(spAll, update = False): t2 = time.time() # Determine the score for each object # 1) Prefer observations with positive SN_MEDIAN in r-band @@ -432,28 +424,65 @@ def specPrimary_sdssid(spAll): score = (4 * (spAll['SN_MEDIAN'][:,jfilt] > 0) + 2*(wwhere(spAll['FIELDQUALITY'],'good*')) + 1 * (zw_primtest == 0) + (spAll['SN_MEDIAN'][:,jfilt] > 0)) / max(spAll['SN_MEDIAN'][:,jfilt]+1.) - spAll.add_column(score,'SCORE') - sdssids = np.unique(spAll['SDSS_ID'].values) - - spAll.add_column(0,name='SPECPRIMARY') - spAll.add_column(0,name='SPECBOSS') - spAll.add_column(0,name='NSPECOBS') - for id in sdssids: - idx = np.where(SpAll['SDSS_ID'].values == id)[0] - SpAll[idx]['NSPECOBS'] = len(idx) - primary = np.argmax(SpAll[idx]['SCORE'].values) - spAll[idx[primary]]['SPECPRIMARY'] = 1 - spAll[idx[primary]]['SPECBOSS'] = 1 + + specprim = spAll['SPECPRIMARY'] + specboss = spAll['SPECBOSS'] + nspecobs = spAll['NSPECOBS'] + if isinstance(spAll['SDSS_ID'], MaskedColumn): + sdssids = spAll['SDSS_ID'].filled(-999).data + else: + sdssids = spAll['SDSS_ID'].data + sidx = np.where((sdssids < 0))[0] + specprim[sidx] = -999 + specboss[sidx] = -999 + nspecobs[sidx] = -999 + + if update: + splog.log(f'Keeping current and updating only') + idx_new = np.where(specprim == -1)[0] + sdssids = np.unique(sdssids[idx_new]) + else: + spAll['SPECPRIMARY'] = 0 + spAll['SPECBOSS'] = 0 + spAll['NSPECOBS'] = 0 + sdssids = np.unique(sdssids) + sdssids = sdssids[sdssids >= 0] + counter_step = round(len(sdssids)/100) + for i, id in enumerate(sdssids): + if (i % counter_step) == 0: + if i + counter_step < len(sdssids): + splog.info(f'Assigning SpecPrimaries: {i+1} - {i+100000} (of {len(sdssids)})') + else: + splog.info(f'Assigning SpecPrimaries: {i+1} - {len(sdssids)} (of {len(sdssids)})') + idx = np.where(spAll['SDSS_ID'].data == id)[0] + nspecobs[idx] = len(idx) + if update: + specprim[idx] = 0 + specboss[idx] = 0 + primary = np.argmax(score[idx]) + specprim[idx[primary]] = 1 + specboss[idx[primary]] = 1 splog.log('Time to assign primaries = '+str(timedelta(seconds = time.time() - t2))) return(spAll) def build_custom_fieldlist(indir, custom, run2d, run1d): flist = Table() - for spfield in glob(ptt.join(indir, run2d,custom,f'spFullsky-{custom}-*.fits')): + topdir2d = ptt.join(indir, run2d) + spfields = [] + for cc in [custom,custom+'_lco',custom+'_apo']: + spfields.extend(glob(ptt.join(create_field_dir(topdir2d,cc, custom=True), + f'spFullsky-{cc}-*.fits'))) + for spfield in spfields: hdr = fits.getheader(spfield,0) - spdiagcomblog = ptt.join(indir, run2d,custom,f"spDiagcomb-{custom}-{str(hdr['RUNMJD'])}.log") + cc = ptt.basename(spfield).split('-')[1] + if cc.split('_')[-1] in ['lco','apo']: + obs = cc.split('_')[-1].upper() + else: + obs = '' + spdiagcomblog = ptt.join(create_field_dir(topdir2d,cc, custom=True), + f"spDiagcomb-{cc}-{str(hdr['RUNMJD'])}.log") if ptt.exists(spdiagcomblog): lastline = get_lastline(spdiagcomblog) if 'Successful completion' in lastline: @@ -464,7 +493,8 @@ def build_custom_fieldlist(indir, custom, run2d, run1d): STATUSCOMBINE = 'RUNNING' else: STATUSCOMBINE = 'Pending' - spDiag1dlog = ptt.join(indir, run2d, custom, run1d, f"spDiag1d-{custom}-{str(hdr['MJD'])}.log") + spDiag1dlog = ptt.join(create_field_dir(topdir2d,cc, custom=True), + run1d, f"spDiag1d-{cc}-{str(hdr['MJD'])}.log") if ptt.exists(spDiag1dlog): lastline = get_lastline(spDiag1dlog) if 'Successful completion' in lastline: @@ -494,23 +524,24 @@ def build_custom_fieldlist(indir, custom, run2d, run1d): except: sn2_i2 = np.NaN flist_row = Table({'RUN2D':[run2d], 'RUN1D':[run1d], - 'PROGRAM':[custom], 'FIELD':[0], 'MJD': [hdr['MJD']], + 'PROGRAMNAME':[custom], 'FIELD':[0], 'MJD': [hdr['MJD']], 'STATUS2D':['Done'], 'STATUSCOMBINE':[STATUSCOMBINE], 'STATUS1D':[STATUS1D],'FIELDQUALITY':['good'], 'FIELDSN2':[np.nanmin( np.array([sn2_g1,sn2_i1,sn2_g2,sn2_i2]))], - 'OBS':['']}) + 'OBSERVATORY':[obs]}) flist = vstack([flist, flist_row]) - print(flist) + flist.pprint_all() return(flist) def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), skip_line=False, include_bad=False, legacy=False, skip_specprimary=False, - lite=False, XCSAO=False, field=None, mjd=None, programs=None, clobber=False, dev=False, + update_specprimary=True, lite=False, XCSAO=False, field=None, + mjd=None, programs=None, clobber=False, dev=False, datamodel=None, line_datamodel=None, verbose=False, epoch =False, outroot=None, - logfile=None, remerge_fmjd=None, merge_only=False, limit=None, - custom=None, allsky=False, run1d=None): + logfile=None, remerge_fmjd=None, remerge_mjd=None, merge_only=False, limit=None, + custom=None, allsky=False, run1d=None, bkup=False, mjdstart=None): try: SDSSC2BV = TargetingFlags.meta['SDSSC2BV'] @@ -522,22 +553,31 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), mjd = str(mjd) if datamodel is None: - datamodel = ptt.join(getenv('IDLSPEC2D_DIR'), 'datamodel', 'spall_dm.par') + datamodel = ptt.join(idlspec2d_dir, 'datamodel', 'spall_dm.par') if line_datamodel is None: - line_datamodel = ptt.join(getenv('IDLSPEC2D_DIR'), 'datamodel', 'spzline_dm.par') + line_datamodel = ptt.join(idlspec2d_dir, 'datamodel', 'spzline_dm.par') - if epoch is True: - fieldlist_file = ptt.join(indir, run2d, 'epoch', 'fieldlist-'+run2d+'.fits') + dflags = [indir, run2d,'summary'] + if custom is not None: + dflags.append(custom) + fflag = '-'+custom + elif epoch is True: + dflags.append('epoch') + fflag = '-epoch' else: - fieldlist_file = ptt.join(indir, run2d, 'fieldlist-'+run2d+'.fits') - - if field is not None and mjd is not None: + dflags.append('daily') + fflag ='' + fieldlist_file = ptt.join(*dflags, 'fieldlist-'+run2d+fflag+'.fits') + + if (custom is not None) and (mjd is not None): +# if field is not None: +# custom = field +# field = None + field_dir = create_field_dir(ptt.join(indir, run2d), field, custom=True) + elif field is not None and mjd is not None: + field_dir = create_field_dir(ptt.join(indir, run2d),field) if epoch is True: - field_dir = ptt.join(indir, run2d, field, 'epoch') - else: - field_dir = ptt.join(indir, run2d, field) - elif (custom is not None) and (mjd is not None): - field_dir = ptt.join(indir, run2d, custom) + field_dir = ptt.join(field_dir, 'epoch') if logfile is None: @@ -549,11 +589,11 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), logfile = ptt.join(field_dir,'spAll-'+custom+'-'+mjd+'.log') elif field is None and mjd is None: if epoch is True: - logfile = ptt.join(indir, run2d,'spAll_epoch-'+run2d+'.log') + logfile = ptt.join(indir, run2d,'summary','epoch','spAll_epoch-'+run2d+'.log') elif custom is None: - logfile = ptt.join(indir, run2d,'spAll-'+run2d+'.log') + logfile = ptt.join(indir, run2d,'summary','daily','spAll-'+run2d+'.log') else: - logfile = ptt.join(indir, f'spAll_{custom}-{run2d}.log') + logfile = ptt.join(indir, run2d,'summary',fieldgroup(custom, custom=True),f'spAll_{custom}-{run2d}.log') else: if epoch is True: logfile = ptt.join(indir, 'spAll_epoch.log') @@ -563,7 +603,8 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), logfile = ptt.join(indir, f'spAll_{custom}.log') if dev: logfile = logfile.replace('spAll','spAll_dev') - + if len(ptt.dirname(logfile)) > 0: + makedirs(ptt.dirname(logfile),exist_ok=True) splog.open(logfile=logfile, backup=False) splog.log('Log file '+logfile+' opened '+ time.ctime()) @@ -573,11 +614,11 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), if ptt.exists(fieldlist_file): splog.log(f'Reading fieldlist file: {fieldlist_file}') try: - flist = Table(fits.getdata(fieldlist_file)) + flist = Table.read(fieldlist_file) except: time.sleep(90) try: - flist = Table(fits.getdata(fieldlist_file)) + flist = Table(fieldlist_file) except: pass else: @@ -585,11 +626,11 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), if ptt.exists(fieldlist_file): splog.log(f'Reading fieldlist file: {fieldlist_file}') try: - flist = Table(fits.getdata(fieldlist_file)) + flist = Table.read(fieldlist_file) except: time.sleep(90) try: - flist = Table(fits.getdata(fieldlist_file)) + flist = Table.read(fieldlist_file) except: pass if (flist is None) and (dev is False): @@ -601,20 +642,19 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), splog.log('ERROR: '+ptt.basename(fieldlist_file)+' is required (Running Now)') flist = fieldlist(create=True, topdir=indir, run2d=[run2d], run1d=[getenv('RUN1D')], outdir=None, legacy=legacy, custom=custom, basehtml=None, epoch=epoch, - logfile=fmlog, noplot=True) + logfile=fmlog, noplot=True, return_tab = True) else: flist = build_custom_fieldlist(indir, custom, run2d, run1d) - full_flist = flist - #if run2d is not None: - # flist = flist[np.where(flist['RUN2D'] == run2d)[0]] + flist = flist['RUN2D','RUN1D','PROGRAMNAME','FIELD','MJD','STATUS2D', + 'STATUSCOMBINE','STATUS1D','FIELDQUALITY','FIELDSN2','OBSERVATORY'] if programs is not None: ftemp = None for prog in programs: if ftemp is None: - ftemp = flist[np.where(flist['PROGRAM'] == prog)[0]] + ftemp = flist[np.where(flist['PROGRAMNAME'] == prog)[0]] else: - ftemp = vstack([ftemp, flist[np.where(flist['PROGRAM'] == prog)[0]]]) + ftemp = vstack([ftemp, flist[np.where(flist['PROGRAMNAME'] == prog)[0]]]) flist = ftemp if allsky is False: @@ -628,7 +668,8 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), spAll_fmjds = None spline_fmjds = None - if (field is not None) and (mjd is not None): + + if (field is not None) and (mjd is not None) and (custom is None): idx = np.where((flist['FIELD'] == field) & (flist['MJD'] == int(mjd)))[0] if len(idx) == 0: flist.add_row({'FIELD': field, 'MJD':mjd, 'STATUS2D': 'unknown'}) @@ -638,95 +679,176 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), epoch=epoch, allsky=allsky, custom=custom) if not clobber: + spAll = None + spline = None if ptt.exists(spallfile): splog.log(f'Reading Existing spAll file: {spallfile}') - spAll = Table(fits.getdata(spallfile)) - try: - spAll_fmjds = spAll['FIELD','MJD'] - spAll_fmjds = unique(spAll_fmjds,keys=['FIELD','MJD']) - except: - spAll_fmjds = Table(names = ['FIELD','MJD']) + spAll = Table.read(spallfile) elif ptt.exists(spallfile.replace('.gz','')): splog.log(f"Reading Existing spAll file: {spallfile.replace('.gz','')}") - spAll = Table(fits.getdata(spallfile.replace('.gz',''))) try: - spAll_fmjds = spAll['FIELD','MJD'] - spAll_fmjds = unique(spAll_fmjds,keys=['FIELD','MJD']) + spAll = Table.read(spallfile.replace('.gz','')) except: - spAll_fmjds = Table(names = ['FIELD','MJD']) + time.sleep(60) + spAll = Table.read(spallfile.replace('.gz','')) else: - spAll_fmjds = Table(names = ['FIELD','MJD']) + spAll_fmjds = Table(names = ['FIELD','MJD','OBS']) + if spAll is not None: + try: + spAll_fmjds = spAll['FIELD','MJD','OBS'] + if isinstance(spAll_fmjds['FIELD'], MaskedColumn): + spAll_fmjds['FIELD'] = spAll_fmjds['FIELD'].filled(0) + spAll_fmjds = unique(spAll_fmjds,keys=['FIELD','MJD','OBS']) + except Exception as e: + splog.warning(f'{type(e).__name__}: {e}') + spAll_fmjds = Table(names = ['FIELD','MJD','OBS']) + try: + spAll.meta = {} + spAll_fmjds.meta = {} + except: + pass if ptt.exists(splinefile): splog.log(f'Reading Existing spLine file: {splinefile}') - spline = Table(fits.getdata(splinefile)) try: - spline_fmjds = spline['FIELD','MJD'] - spline_fmjds = unique(spline_fmjds,keys=['FIELD','MJD']) + spline = Table.read(splinefile) except: - spAll_fmjds = Table(names = ['FIELD','MJD']) + time.sleep(60) + spline = Table.read(splinefile) elif ptt.exists(splinefile.replace('.gz','')): splog.log(f"Reading Existing spLine file: {splinefile.replace('.gz','')}") - spline = Table(fits.getdata(splinefile.replace('.gz',''))) try: - spline_fmjds = spline['FIELD','MJD'] - spline_fmjds = unique(spline_fmjds,keys=['FIELD','MJD']) + spline = Table.read(splinefile.replace('.gz','')) except: - spAll_fmjds = Table(names = ['FIELD','MJD']) + time.sleep(60) + spline = Table.read(splinefile.replace('.gz','')) else: - spline_fmjds = Table(names = ['FIELD','MJD']) - + spline_fmjds = Table(names = ['FIELD','MJD','OBS']) + if spline is not None: + try: + try: + spline_fmjds = spline['FIELD','MJD','OBS'] + except: + spline_fmjds = spline['FIELD','MJD'] + spline_fmjds.add_column('unknown', name='OBS') + if isinstance(spline_fmjds['FIELD'], MaskedColumn): + spline_fmjds['FIELD'] = spline_fmjds['FIELD'].filled(0) + if isinstance(spAll_fmjds['OBS'], MaskedColumn): + spline_fmjds['OBS'] = spline_fmjds['OBS'].astype(object) + spline_fmjds['OBS'].filled('unknown') + spline_fmjds['OBS'] = spline_fmjds['OBS'].astype(str) + if isinstance(spAll_fmjds['FIELD'], MaskedColumn): + spline_fmjds['FIELD'] = spAll_fmjds['FIELD'].filled(0) + spline_fmjds = unique(spline_fmjds,keys=['FIELD','MJD','OBS']) + + except Exception as e: + splog.warning(f'{type(e).__name__}: {e}') + spline_fmjds = Table(names = ['FIELD','MJD','OBS']) + try: + spline.meta = {} + spline_fmjds.meta = {} + except: + pass if remerge_fmjd is not None: - idx = np.where((spAll['FIELD'] == int(remerge_fmjd.split('-')[0])) & (spAll['MJD'] == int(remerge_fmjd.split('-')[1])))[0] - idxl = np.where((spline['FIELD'] == int(remerge_fmjd.split('-')[0])) & (spline['MJD'] == int(remerge_fmjd.split('-')[1])))[0] + idx = np.where((spAll_fmjds['FIELD'] == int(remerge_fmjd.split('-')[0])) & + (spAll_fmjds['MJD'] == int(remerge_fmjd.split('-')[1])))[0] + idxl = np.where((spline_fmjds['FIELD'] == int(remerge_fmjd.split('-')[0])) & + (spline_fmjds['MJD'] == int(remerge_fmjd.split('-')[1])))[0] if len(idx) > 0: - spAll.remove_rows(idx) - spAll_fmjds = spAll['FIELD','MJD'] - spAll_fmjds = unique(spAll_fmjds,keys=['FIELD','MJD']) + spAll_fmjds.remove_rows(idx) + spAll_fmjds = unique(spAll_fmjds,keys=['FIELD','MJD','OBS']) if len(idxl) > 0: - spline.remove_rows(idxl) - spline_fmjds = spline['FIELD','MJD'] - spline_fmjds = unique(spline_fmjds,keys=['FIELD','MJD']) + spline_fmjds.remove_rows(idxl) + spline_fmjds = unique(spline_fmjds,keys=['FIELD','MJD','OBS']) + if remerge_mjd is not None: + idx = np.where((spAll_fmjds['MJD'] == int(remerge_mjd)))[0] + idxl = np.where((spline_fmjds['MJD'] == int(remerge_mjd)))[0] + if len(idx) > 0: + spAll_fmjds.remove_rows(idx) + spAll_fmjds = unique(spAll_fmjds,keys=['FIELD','MJD','OBS']) + if len(idxl) > 0: + spline_fmjds.remove_rows(idxl) + spline_fmjds = unique(spline_fmjds,keys=['FIELD','MJD','OBS']) flist.sort(['MJD','FIELD']) + if mjdstart is not None: + splog.info(f"Only Checking Field-MJDs with MJD >= {mjdstart}") + flist = flist[flist['MJD'] >= mjdstart] j = 0 for i, row in enumerate(flist): + tfield_str = f"Field:{row['FIELD']} MJD:{row['MJD']} OBS:{row['OBSERVATORY']} ({i+1}/{len(flist)})" if spAll_fmjds is not None: - idx = spAll_fmjds[(spAll_fmjds['FIELD'] == int(row['FIELD'])) & (spAll_fmjds['MJD'] == int(row['MJD']))] - idxl = spline_fmjds[(spline_fmjds['FIELD'] == int(row['FIELD'])) & (spline_fmjds['MJD'] == int(row['MJD']))] + idx = spAll_fmjds[(spAll_fmjds['FIELD'] == int(row['FIELD'])) & + (spAll_fmjds['MJD'] == int(row['MJD'])) & + (spAll_fmjds['OBS'] == row['OBSERVATORY'])] + + idxl = spline_fmjds[(spline_fmjds['FIELD'] == int(row['FIELD'])) & + (spline_fmjds['MJD'] == int(row['MJD'])) & + ((spline_fmjds['OBS'] == row['OBSERVATORY']) | + (spline_fmjds['OBS'] == 'unknown'))] if len(idx)*len(idxl) > 0: - splog.log(f"Skipping (Complete) Field:{row['FIELD']} MJD:{row['MJD']} ({i+1}/{len(flist)})") + splog.log(f"Skipping (Complete) {tfield_str}") continue - if (field is not None) and (mjd is not None): + if (field is not None) and (mjd is not None) and (allsky is False): if (row['STATUS2D'].lower().strip() != 'done') or (row['STATUSCOMBINE'].lower().strip() != 'done') or (row['STATUS1D'].lower().strip() != 'done'): - splog.log(f"Checking incomplete status ({row['STATUS2D'].strip()} RUN2D) Field:{row['FIELD']} MJD:{row['MJD']} ({i+1}/{len(flist)})") + splog.log(f"Checking incomplete status ({row['STATUS2D'].strip()} RUN2D) {tfield_str}") fmlog = f'fieldlist-{field}-{mjd}.log' - row = fieldlist(create=True, topdir=indir, run2d=[run2d], run1d=[getenv('RUN1D')], - outdir=None, legacy=legacy, custom=custom, basehtml=None, epoch=epoch, - logfile=fmlog, field=field, mjd=mjd, noplot=True, fmsplog=splog) - flist[i] = row + row = retry(fieldlist, retries=3, delay = 5, logger=splog.log, + create=True, topdir=indir, run2d=[run2d], run1d=[getenv('RUN1D')], + outdir=None, legacy=legacy, custom=custom, basehtml=None, epoch=epoch, + logfile=fmlog, field=field, mjd=mjd, noplot=True, fmsplog=splog) + + try: + flist[i] = row + except: + try: + flist[i] = row[flist.colnames] + except: + + print(flist.colnames) + print('-----------------') + print(row) + print('------------------') + print(row[flist.colnames]) + + print('------------------') + print(len(row)) + print(len(flist.columns)) + raise if row['STATUS2D'].lower().strip() != 'done': - splog.log(f"Skipping ({row['STATUS2D'].strip()} RUN2D) Field:{row['FIELD']} MJD:{row['MJD']} ({i+1}/{len(flist)})") + splog.log(f"Skipping ({row['STATUS2D'].strip()} RUN2D) {tfield_str}") continue elif row['STATUSCOMBINE'].lower().strip() != 'done': - splog.log(f"Skipping ({row['STATUSCOMBINE'].strip()} Combine) Field:{row['FIELD']} MJD:{row['MJD']} ({i+1}/{len(flist)})") + splog.log(f"Skipping ({row['STATUSCOMBINE'].strip()} Combine) {tfield_str}") continue elif row['STATUS1D'].lower().strip() != 'done': - splog.log(f"Skipping ({row['STATUS1D'].strip()} RUN1D) Field:{row['FIELD']} MJD:{row['MJD']} ({i+1}/{len(flist)})") + splog.log(f"Skipping ({row['STATUS1D'].strip()} RUN1D) {tfield_str}") continue - splog.log(f"Reading/Building Field:{row['FIELD']} MJD:{row['MJD']} ({i+1}/{len(flist)})") - onefield = oneField(row, row['FIELD'], row['MJD'], + splog.log(f"Reading/Building {tfield_str}") + if dev: + dev1 = True if merge_only else False + else: + dev1 = False + field_clobber = clobber if not merge_only else False + if custom is not None: + rfield = '_'.join([row['PROGRAMNAME'],row['OBSERVATORY'].lower()]) + else: + rfield = row['FIELD'] + onefield = oneField(row, rfield, row['MJD'], skip_line=skip_line, include_bad=include_bad, - legacy=legacy, skip_specprimary=skip_specprimary, dev=dev, - XCSAO=XCSAO, indir=indir, clobber=clobber, epoch = epoch, + legacy=legacy, skip_specprimary=skip_specprimary, dev=dev1, + XCSAO=XCSAO, indir=indir, clobber=field_clobber, epoch = epoch, merge_only=merge_only, custom = custom, allsky = allsky) + if onefield['spall'] is None: + continue if not merge_only: - write_spAll(onefield['spall'], onefield['spline'], None, indir, run2d, datamodel, - line_datamodel, outroot=None, field=row['FIELD'], mjd = row['MJD'], + write_spAll(onefield['spall'].copy(), onefield['spline'].copy(), None, + indir, run2d, datamodel, + line_datamodel, outroot=None, field=rfield, mjd = row['MJD'], verbose=verbose, dev=dev, clobber=clobber, epoch=epoch, silent=True, custom = custom, allsky = allsky, SDSSC2BV = SDSSC2BV) - if onefield['spall'] is None: - continue + + onefield['spall']['SPECPRIMARY'] = -1 if not skip_line: if onefield['spline'] is None: continue @@ -742,11 +864,29 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), if spAll[col].shape[1] > onefield['spall'][col].shape[1]: coldat = onefield['spall'][col].data pad = spAll[col].shape[1] - onefield['spall'][col].shape[1] - onefield['spall'][col] = np.pad(coldat, [(0,0),(pad,0)], mode = 'constant', constant_values= 0) + if col.upper() == 'SDSS5_TARGET_FLAGS': + onefield['spall'][col] = np.pad(coldat, [(0,0),(0,pad)], + mode = 'constant', + constant_values= 0) + else: + onefield['spall'][col] = np.pad(coldat, [(0,0),(pad,0)], + mode = 'constant', + constant_values= 0) + coldat = None + pad = None elif spAll[col].shape[1] < onefield['spall'][col].shape[1]: coldat = spAll[col].data pad = onefield['spall'][col].shape[1] - spAll[col].shape[1] - spAll[col] = np.pad(coldat, [(0,0),(pad,0)], mode = 'constant', constant_values= 0) + if col.upper() == 'SDSS5_TARGET_FLAGS': + spAll[col] = np.pad(coldat, [(0,0),(0,pad)], + mode = 'constant', + constant_values= 0) + else: + spAll[col] = np.pad(coldat, [(0,0),(pad,0)], + mode = 'constant', + constant_values= 0) + coldat = None + pad = None spAll = vstack([spAll,onefield['spall']]) if onefield['spline'] is not None: @@ -754,14 +894,16 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), spline = onefield['spline'] else: spline = vstack([spline,onefield['spline']]) - if merge_only: - if limit is not None: - j = j+1 - del onefield - gc.collect() - splog.log(f'{j}:{limit} ({time.ctime()})') - if j >= limit: - break + if (merge_only) and (limit is not None): + j = j+1 + del onefield + gc.collect() + splog.log(f'{j}:{limit} ({time.ctime()})') + if j >= limit: + break + else: + del onefield + gc.collect() if custom is not None and mjd is not None: field = custom if not(field is not None and mjd is not None): if spline is not None: @@ -773,15 +915,14 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), spAll = unique(spAll, keys='CATALOGID', keep='last') spAll.sort('CATALOGID') if spline is not None: - dropped = setdiff(spall_raw, spAll, keys=['TARGET_INDEX','MJD','FIELD']) - dropped = dropped['TARGET_INDEX','MJD','FIELD','CATALOGID'] + dropped = setdiff(spall_raw, spAll, keys=['TARGET_INDEX','MJD','OBS']) + dropped = dropped['TARGET_INDEX','MJD','OBS','CATALOGID'] if len(dropped) > 0: dropped = unique(dropped) for row in dropped: idx = np.where((spline['TARGET_INDEX'].data == row['TARGET_INDEX']) & - (spline['MJD'].data == row['MJD']) & - (spline['FIELD'].data == row['FIELD']))[0] + (spline['MJD'].data == row['MJD']))[0] spline.remove_rows(idx) if spAll is None: @@ -789,68 +930,85 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), splog.info('EXITING!!') exit() if not skip_specprimary: - spAll = specPrimary(spAll) + #spAll = specPrimary(spAll) + spAll = specPrimary_sdssid(spAll, update=update_specprimary) if lite is True: - spAll_lite = spAll.copy() - spAll_lite.rename_column('DELTA_RA_LIST', 'DELTA_RA') - spAll_lite.rename_column('DELTA_DEC_LIST', 'DELTA_DEC') - for i in range(len(spAll)): + splog.info('Creating spAll-lite Table') + mr = len(spAll) + + + + for col in ['ASSIGNED','ON_TARGET','VALID','DECOLLIDED', 'TOO','CARTON_TO_TARGET_PK']: + if col not in spAll.columns: + splog.info(f'{col} missing from spAll') + spAll[col] = '-999' + for col in ['MOON_DIST','MOON_PHASE','DELTA_RA_LIST','DELTA_DEC_LIST']: + if col not in spAll.columns: + splog.info(f'{col} missing from spAll') + spall[col] = 'nan' + + spAll_lite = spAll['ASSIGNED','ON_TARGET','VALID','DECOLLIDED', 'TOO', + 'MOON_DIST','MOON_PHASE','CARTON_TO_TARGET_PK', + 'DELTA_RA_LIST','DELTA_DEC_LIST'].copy() + for i in range(mr): + if (i % 100000) == 0: + if i + 100000 < mr: + splog.info(f'Re-Formatting arrays in spAll-lite rows: {i+1} - {i+100000} (of {mr})') + else: + splog.info(f'Re-Formatting arrays in spAll-lite rows: {i+1} - {mr} (of {mr})') + + for col in ['ASSIGNED','ON_TARGET','VALID','DECOLLIDED','TOO','CARTON_TO_TARGET_PK']: + try: + if col in ['CARTON_TO_TARGET_PK']: + spAll_lite[col][i] = str(int(np.asarray(spAll[col][i].split())[0])) + else: + spAll_lite[col][i] = str(min(np.asarray(spAll[col][i].split()).astype(int))) + except Exception as e: + splog.warning(f'{col}: {type(e).__name__}: {e}') + print(f'{col}: {type(e).__name__}: {e}') + spAll_lite[col][i] = '0' with warnings.catch_warnings(): warnings.filterwarnings(action='ignore', message='Mean of empty slice') - try: - spAll_lite[i]['ASSIGNED'] = str(min(np.array(spAll_lite[i]['ASSIGNED'].split()).astype(int))) - except: - spAll_lite[i]['ASSIGNED'] = '0' - try: - spAll_lite[i]['ON_TARGET'] = str(min(np.array(spAll_lite[i]['ON_TARGET'].split()).astype(int))) - except: - spAll_lite[i]['ON_TARGET'] = '0' - try: - spAll_lite[i]['VALID'] = str(min(np.array(spAll_lite[i]['VALID'].split()).astype(int))) - except: - spAll_lite[i]['VALID'] = '0' - try: - spAll_lite[i]['DECOLLIDED'] = str(min(np.array(spAll_lite[i]['DECOLLIDED'].split()).astype(int))) - except: - spAll_lite[i]['DECOLLIDED'] = '0' - try: - spAll_lite[i]['MOON_DIST'] = str(np.nanmean(np.array(spAll_lite[i]['MOON_DIST'].split()).astype(float))) - except: - spAll_lite[i]['MOON_DIST'] = 'nan' - try: - spAll_lite[i]['MOON_PHASE'] = str(np.nanmean(np.array(spAll_lite[i]['MOON_PHASE'].split()).astype(float))) - except: - spAll_lite[i]['MOON_PHASE'] = 'nan' - try: - spAll_lite[i]['DELTA_RA'] = str(np.nanmean(np.array(spAll_lite[i]['DELTA_RA'].split()).astype(float))) - except: - spAll_lite[i]['DELTA_RA'] = 'nan' - try: - spAll_lite[i]['DELTA_DEC'] = str(np.nanmean(np.array(spAll_lite[i]['DELTA_DEC'].split()).astype(float))) - except: - spAll_lite[i]['DELTA_DEC'] = 'nan' - try: - spAll_lite[i]['CARTON_TO_TARGET_PK'] = str(int(np.array(spAll_lite[i]['CARTON_TO_TARGET_PK'].split())[0])) - except: - spAll_lite[i]['CARTON_TO_TARGET_PK'] = '0' - - for col in ['ASSIGNED','ON_TARGET','VALID','DECOLLIDED']: - spAll_lite[col].fill_value = False - spAll_lite[col] = spAll_lite[col].astype(bool) + for col in ['MOON_DIST','MOON_PHASE','DELTA_RA_LIST','DELTA_DEC_LIST']: + try: + spAll_lite[col][i] = str(np.nanmean(np.asarray(spAll[col][i].split()).astype(float))) + except Exception as e: + print(f'{col}: {type(e).__name__}: {e}') + splog.warning(f'{col}: {type(e).__name__}: {e}') + spAll_lite[col][i] = 'nan' + + for col in ['ASSIGNED','ON_TARGET','VALID','DECOLLIDED','TOO','CARTON_TO_TARGET_PK']: + spAll_lite[col].fill_value = -999 + try: + spAll_lite[col] = spAll_lite[col].astype(int) + except Exception as e: + splog.warning(f'{type(e).__name__}: {e}') + column_data = np.array(spAll_lite[col]) + column_data = np.array([x.decode('utf-8') if x is not None else None for x in column_data]) + column_data = np.asarray([x == 'True' if x is not None else None for x in column_data]).astype(int) + spAll_lite.remove_column(col) + spAll_lite[col] = MaskedColumn(column_data, mask=[x is None for x in column_data]) + column_data = None + del column_data + + spAll_lite.rename_column('DELTA_RA_LIST', 'DELTA_RA') + spAll_lite.rename_column('DELTA_DEC_LIST', 'DELTA_DEC') for col in ['CARTON_TO_TARGET_PK']: spAll_lite[col] = spAll_lite[col].astype(int) for col in ['MOON_DIST','MOON_PHASE','DELTA_RA','DELTA_DEC']: spAll_lite[col] = spAll_lite[col].astype(float) + + else: spAll_lite = None - write_spAll(spAll, spline, spAll_lite, indir, run2d, datamodel, line_datamodel, + spAll_file = write_spAll(spAll, spline, spAll_lite, indir, run2d, datamodel, line_datamodel, epoch=epoch, dev=dev, outroot=outroot, field=field, mjd=mjd, verbose=verbose, clobber=True, custom = custom, allsky = allsky, - SDSSC2BV = SDSSC2BV) + SDSSC2BV = SDSSC2BV, bkup=bkup) if spAll is None: splog.info('No valid spAll entries') @@ -858,8 +1016,19 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), exit() if allsky is False: - #flist = Table(fits.getdata(fieldlist_file)) - plot_sky(ptt.dirname(fieldlist_file), full_flist, fieldlist_file) + spallfile, spalllitefile, splinefile, spAlldatfile = build_fname(indir, run2d, outroot=outroot, dev=dev, + epoch=epoch, custom=custom, allsky=allsky) + + + + spAll = None + spline = None + spAll_lite = None + if not(field is not None and mjd is not None): + outdir = ptt.dirname(fieldlist_file) + fieldlist_file = ptt.basename(fieldlist_file) + plot_sky_locations(outdir, fieldlist_file, splog) + plot_sky_targets(outdir, spallfile, splog, nobs=True) if field is not None and mjd is not None: splog.log(f'Successful completion of fieldmerge for {field}-{mjd} at '+ time.ctime()) @@ -878,49 +1047,51 @@ def build_fname(indir, run2d, outroot=None, field=None, mjd=None, dev=False, splinefile = ptt.join(outroot+'Line'+'.fits.gz') spAlldatfile = ptt.join(outroot+'.dat.gz') else: + cc = False if custom is not None: - if allsky is True: - field = custom - elif field is not None: - field = field_to_string(field) + cc= True elif field is not None: field = field_to_string(field) if field is not None and mjd is not None: - field = field_to_string(field) + if not cc: + field = field_to_string(field) mjd = str(mjd) - if epoch is True: - specfull_dir = ptt.join(indir, run2d, 'epoch', 'spectra','full', field, mjd) - else: - specfull_dir = ptt.join(indir, run2d, 'spectra','full', field, mjd) + specfull_dir = field_spec_dir(indir, run2d,field, mjd, epoch=epoch, + custom = cc, custom_name=custom) spallfile = ptt.join(specfull_dir, 'spAll-'+field+'-'+mjd+'.fits.gz') spalllitefile = ptt.join(specfull_dir, 'spAll-lite-'+field+'-'+mjd+'.fits.gz') splinefile = ptt.join(specfull_dir, 'spAllLine-'+field+'-'+mjd+'.fits.gz') spAlldatfile = ptt.join(specfull_dir, 'spAll-'+field+'-'+mjd+'.dat.gz') - elif run2d is not None: - if epoch is True: - spAll_dir = ptt.join(indir, run2d, 'epoch') - else: - spAll_dir = ptt.join(indir, run2d) - - if custom is None: - spallfile = ptt.join(spAll_dir, 'spAll-'+run2d+'.fits.gz') - spalllitefile = ptt.join(spAll_dir, 'spAll-lite-'+run2d+'.fits.gz') - splinefile = ptt.join(spAll_dir, 'spAllLine-'+run2d+'.fits.gz') - spAlldatfile = ptt.join(spAll_dir, 'spAll-'+run2d+'.dat.gz') - else: - spallfile = ptt.join(spAll_dir, 'spAll-'+run2d+'-'+custom+'.fits.gz') - spalllitefile = ptt.join(spAll_dir, 'spAll-lite-'+run2d+'-'+custom+'.fits.gz') - splinefile = ptt.join(spAll_dir, 'spAllLine-'+run2d+'-'+custom+'.fits.gz') - spAlldatfile = ptt.join(spAll_dir, 'spAll-'+run2d+'-'+custom+'.dat.gz') else: - if epoch is True: - spAll_dir = ptt.join(indir, 'epoch') + fflags = [] + dflags = [indir] + + if run2d is not None: + fflags.append(run2d) + dflags.extend([run2d, 'summary']) else: - spAll_dir = ptt.join(indir) - spallfile = ptt.join(spAll_dir, 'spAll.fits.gz') - spalllitefile = ptt.join(spAll_dir, 'spAll-lite.fits.gz') - splinefile = ptt.join(spAll_dir, 'spAllLine.fits.gz') - spAlldatfile = ptt.join(spAll_dir, 'spAll.dat.gz') + dflags.append('summary') + if custom is not None: + dflags.append(custom) + #spall_dir = ptt.join(indir, run2d, 'summary', fieldgroup(custom, custom=True)) + fflags.append(custom) + elif epoch is True: + dflags.append('epoch') + fflags.append('epoch') + #spAll_dir = ptt.join(indir, run2d, 'summary','epoch') + else: + #spAll_dir = ptt.join(indir, run2d) + dflags.append('daily') + + spall_dir = ptt.join(*dflags) + fflags = '-'.join(fflags) + if len(fflags) > 0: + fflags = '-'+fflags + spallfile = ptt.join(spall_dir, 'spAll'+fflags+'.fits.gz') + spalllitefile = ptt.join(spall_dir, 'spAll-lite'+fflags+'.fits.gz') + splinefile = ptt.join(spall_dir, 'spAllLine'+fflags+'.fits.gz') + spAlldatfile = ptt.join(spall_dir, 'spAll'+fflags+'.dat.gz') + if dev: spallfile = spallfile.replace('spAll','spAll_dev') spalllitefile = spalllitefile.replace('spAll','spAll_dev') @@ -932,7 +1103,7 @@ def build_fname(indir, run2d, outroot=None, field=None, mjd=None, dev=False, def write_spAll(spAll, spline, spAll_lite, indir, run2d, datamodel, line_datamodel, epoch=False, dev=False, outroot=None, field=None, mjd = None, verbose=False, clobber=False, silent=False, custom = None, - allsky = False, SDSSC2BV = ''): + allsky = False, SDSSC2BV = '', tmpext = '.tmp', bkup = False): spallfile, spalllitefile, splinefile, spAlldatfile = build_fname(indir, run2d, outroot=outroot, field=field, mjd=mjd, dev=dev, @@ -940,25 +1111,45 @@ def write_spAll(spAll, spline, spAll_lite, indir, run2d, datamodel, line_datamod drop_cols = None date = time.ctime() - if spline is not None: - spline = merge_dm(table=spline, ext = 'spZline', name = 'SPLINE', dm = line_datamodel, - splog=splog, drop_cols=drop_cols, verbose=verbose) - if spAll_lite is not None: - #spall_lite = spall_lite[np.where(spall_lite['CATALOGID'] != -999)[0]] - spAll_lite = merge_dm(table=spAll_lite, ext = 'SPALL_lite', name = 'SPALL', dm = datamodel, - splog=splog,drop_cols=drop_cols, verbose=verbose) exists = ptt.exists(spallfile) if not clobber else False - if spAll is None: return + if spAll is None: return(spallfile) + if bkup: + if ptt.exists(spallfile): + try: + bkup_str = datetime.strptime(fits.getheader(spallfile,0)['DATE'],'%c').isoformat() + except: + bkup_str = (datetime.now() - timedelta(days=1)).isoformat() + + if not ptt.exists(f"{spallfile}.bkup-{bkup_str}"): + shutil.copy2(spallfile,f"{spallfile}.bkup-{bkup_str}") + shutil.copy2(spalllitefile,f"{spalllitefile}.bkup-{bkup_str}") + shutil.copy2(splinefile,f"{splinefile}.bkup-{bkup_str}") + + if not exists: hdul = merge_dm(ext = 'Primary', hdr = {'RUN2D':run2d, 'Date':time.ctime(), 'SDSSC2BV': SDSSC2BV}, dm = datamodel, splog=splog, verbose=verbose) - spAll = merge_dm(table=spAll, ext = 'SPALL', name = 'SPALL', dm = datamodel, splog=splog,drop_cols=drop_cols, verbose=verbose) + splog.info('Formatting spAll table') + spAll = merge_dm(table=spAll, ext = 'SPALL', name = 'SPALL', dm = datamodel, + splog=splog, drop_cols=drop_cols, verbose=verbose) makedirs(ptt.dirname(spallfile), exist_ok=True) splog.log('Writing '+spallfile) - fits.HDUList([hdul,spAll]).writeto(spallfile, overwrite=True, checksum=True) - del hdul, spAll + fits.HDUList([hdul,spAll]).writeto(spallfile.replace('.gz',tmpext+'.gz'), + overwrite=True, checksum=True) + hdul = None + if spAll_lite is not None: + spAll = Table(spAll.data) + for col in spAll_lite.colnames: + try: + spAll.remove_column(col) + except Exception as e: + splog.warning(f'{col} {type(e).__name__}: {e}') + pass + spAll = hstack([spAll, spAll_lite], join_type = 'exact') + else: + spAll = None gc.collect() elif not silent: splog.log('Skipping '+spallfile+' (exists)') @@ -970,8 +1161,12 @@ def write_spAll(spAll, spline, spAll_lite, indir, run2d, datamodel, line_datamod 'Date':time.ctime(), 'SDSSC2BV': SDSSC2BV}, dm = datamodel, splog=splog, verbose=verbose) - fits.HDUList([hdul_lite, spAll_lite]).writeto(spalllitefile, overwrite=True, checksum=True) + splog.info('Formatting spAll-lite table') + spAll = merge_dm(table=spAll, ext = 'SPALL_lite', name = 'SPALL', dm = datamodel, + splog=splog, drop_cols=drop_cols, verbose=verbose) splog.log('Writing '+spalllitefile) + fits.HDUList([hdul_lite, spAll]).writeto(spalllitefile.replace('.gz',tmpext+'.gz'), + overwrite=True, checksum=True) del hdul_lite, spAll_lite gc.collect() elif not silent: @@ -982,55 +1177,31 @@ def write_spAll(spAll, spline, spAll_lite, indir, run2d, datamodel, line_datamod makedirs(ptt.dirname(splinefile), exist_ok=True) hdul_line = merge_dm(ext = 'Primary', hdr = {'RUN2D':run2d,'Date':time.ctime()}, dm = line_datamodel, splog=splog, verbose=verbose) - fits.HDUList([hdul_line,spline]).writeto(splinefile, overwrite=True, checksum=True) + splog.info('Formatting spAllLine table') + spline = merge_dm(table=spline, ext = 'spZline', name = 'SPLINE', dm = line_datamodel, + splog=splog, drop_cols=drop_cols, verbose=verbose) splog.log('Writing '+splinefile) + fits.HDUList([hdul_line,spline]).writeto(splinefile.replace('.gz',tmpext+'.gz'), + overwrite=True, checksum=True) del hdul_line, spline gc.collect() elif not silent: splog.log('Skipping '+splinefile+' (exists)') + try: + rename(spallfile.replace('.gz',tmpext+'.gz'),spallfile) + except: + pass + try: + rename(spalllitefile.replace('.gz',tmpext+'.gz'),spalllitefile) + except: + pass + try: + rename(splinefile.replace('.gz',tmpext+'.gz'),splinefile) + except: + pass + #spAll.write(spAlldatfile, format='ascii.fixed_width_two_line') - return - -if __name__ == '__main__' : - """ - build spAll - """ - parser = argparse.ArgumentParser( - prog=ptt.basename(sys.argv[0]), - description='Build BOSS spAll Summary File') - - parser.add_argument('--run2d', type=str, help='Optional override value for the enviro variable $RUN2D', default=getenv('RUN2D')) - parser.add_argument('--indir', type=str, help='Optional override value for the environment variable $BOSS_SPECTRO_REDUX', default = getenv('BOSS_SPECTRO_REDUX')) - - - parser.add_argument('--skip_line', action='store_true', help='skip the generation of spAllLine.fits') - parser.add_argument('--include_bad', action='store_true', help='include bad fields') - parser.add_argument('--legacy', action='store_true', help='Include columns used by SDSS-IV and depreciated in SDSS-V') - parser.add_argument('--skip_specprimary', action='store_true', help='Skip creation of specprimary and associated columns') - parser.add_argument('--lite', action='store_true', help='Produce lite version of spAll file') - parser.add_argument('--XCSAO', action='store_true', help='Include XCSAO columns') - parser.add_argument('--field', '-f', type=str, help='Run for a single Field', default=None) - parser.add_argument('--mjd', '-m', type=str, help='Run for a single MJD', default=None) - parser.add_argument('--clobber', action='store_true', help='Clobber all spAll-field-mjd files') - parser.add_argument('--verbose', action='store_true', help='Log columns not saved') - parser.add_argument('--logfile', type=str, help='Manually set logfile') - parser.add_argument('--epoch', action='store_true', help='Produce spAll for epoch coadds') - parser.add_argument('--dev', action='store_true', help=argparse.SUPPRESS) - parser.add_argument('--programs', nargs='*', help='List of programs to include') - parser.add_argument('--datamodel', type=str, help='Supply a spAll datamodel file (defaults to $IDLSPEC2D/datamodel/spall_dm.par') - parser.add_argument('--line_datamodel', type=str, help='Supply a spline datamodel file (defaults to $IDLSPEC2D/datamodel/spzline_dm.par') - parser.add_argument('--outroot', type=str, help='Path and root of filename for output (defaults to $BOSS_SPECTRO_REDUX/$RUN2D/{field}/{mjd}/spAll)') - parser.add_argument('--remerge_fmjd', '-r',type=str, help='Field-MJD to replace in spAll') - parser.add_argument('--merge_only', '-o', action='store_true', help='Skip Building new spAll-Field-MJD files and just merge existing') - parser.add_argument('--allsky', action='store_true', help='Build spAll for Allsky Custom Coadd') - parser.add_argument('--custom', type=str, help='Name of Custom Coadd') - parser.add_argument('--run1d', type=str, help='Optional override value for the enviro variable $RUN1D (only for custom allsky coadds)', default=getenv('RUN1D')) - parser.add_argument('--limit', type=int, help='Limit number of Field-MJD spAll files to read before save', default = None) - - args = parser.parse_args() + return(spallfile) - if args.merge_only is True: - args.clobber = False - fieldmerge(**vars(args)) diff --git a/python/boss_drp/post/plot_QA.py b/python/boss_drp/post/plot_QA.py new file mode 100644 index 000000000..b931faa24 --- /dev/null +++ b/python/boss_drp/post/plot_QA.py @@ -0,0 +1,925 @@ +#!/usr/bin/env python3 +from boss_drp.field import field_spec_dir +from boss_drp.utils import load_env +from boss_drp import daily_dir, favicon, idlspec2d_dir +from boss_drp.utils import match as wwhere + + +try: + from sdssdb.peewee.sdss5db.targetdb import database + test = database.set_profile(load_env('DATABASE_PROFILE', default='pipelines')) + from sdssdb.peewee.sdss5db.targetdb import Field, Cadence, DesignMode, Design, DesignToField + from sdssdb.peewee.sdss5db import opsdb + from sdssdb.peewee.sdss5db.opsdb import Exposure, CameraFrame, Camera +except: + if load_env('DATABASE_PROFILE', default='pipelines').lower() in ['pipelines','operations']: + print('ERROR: No SDSSDB access') + exit() + else: + print('WARNING: No SDSSDB access') + +from pydl.pydlutils.yanny import read_table_yanny + +from astropy.io import fits +from astropy.table import Table, unique, vstack +from astropy.time import Time + +import pandas as pd +from os import getenv, environ, makedirs +import numpy as np +import argparse +import os.path as ptt +import time +import logging +import warnings +import datetime +import re +from tqdm import tqdm +from glob import glob + + + +import matplotlib.pyplot as plt +import matplotlib +matplotlib.use('agg') + +try: + import plotly.graph_objects as go + from plotly.subplots import make_subplots + import plotly.colors as pc + colors={'g':'green','y':'goldenrod','r':'red','m':'magenta','b':'blue', 'k':'black'} + axopts = dict( gridcolor='lightgrey', linecolor='darkgrey', showline=True, + mirror=True, tickcolor='darkgrey', ticks='outside',tickformat='.0f', + tickprefix='',ticksuffix='',tickmode='auto', + tickformatstops=[dict(dtickrange=[0,10], value='.2f'), + dict(dtickrange=[10,None],value='.0f')]) + ayopts = dict( gridcolor='lightgrey', linecolor='darkgrey', showline=True, + mirror=True, tickcolor='darkgrey', ticks='outside',tickmode='auto') + import plotly + from jinja2 import Template +except: + plotly = None + +filters = ['G','R','I'] +class Formatter(logging.Formatter): + def __init__(self): + super().__init__(fmt="%(levelno)d: %(msg)s", datefmt=None, style='%') + def format(self, record): + # Save the original format configured by the user + # when the logger formatter was instantiated + format_orig = self._style._fmt + if record.levelno == logging.INFO: + self._style._fmt = "%(message)s" + elif record.levelno == logging.DEBUG: + self._style._fmt = '%(funcName)s: %(message)s' + else: + self._style._fmt = "%(levelname)s: %(message)s" + # Call the original formatter class to do the grunt work + result = logging.Formatter.format(self, record) + # Restore the original format configured by the user + self._style._fmt = format_orig + return result + + +def load_fields(clobber_lists=False): + if not ptt.exists(ptt.join(daily_dir,'etc', 'RM_fields')) or clobber_lists is True: + try: + rm_fields = [] + field = DesignToField.select().join(Design).join(DesignMode).where(DesignMode.label == 'dark_rm').switch(DesignToField).join(Field) + for t in field: rm_fields.append(t.field.field_id) + field = DesignToField.select().join(Design).join(DesignMode).where(DesignMode.label == 'dark_rm_eng').switch(DesignToField).join(Field) + for t in field: rm_fields.append(t.field.field_id) + rm_fields=list(set(rm_fields)) + trm_fields=np.char.add(np.asarray(rm_fields).astype(str),'\n').tolist() + + if ptt.exists(ptt.join(daily_dir,'etc')): + with open(ptt.join(daily_dir,'etc', 'RM_fields'), 'w') as f: + f.writelines(trm_fields) + except: + rm_fields = [] + if not cron: warnings.warn('RM_fields file is missing and DB query failed', UserWarning) + else: splog.warnings('Warning: RM_fields file is missing and DB query failed') + else: + with open(ptt.join(daily_dir,'etc', 'RM_fields'), 'r') as f: + rm_fields = np.array(f.readlines()) + rm_fields= np.char.replace(rm_fields, '\n','').astype(int).tolist() + + if ptt.exists(ptt.join(daily_dir,'etc', 'RM_plates')): + with open(ptt.join(daily_dir,'etc', 'RM_plates')) as f: + rm_plates = np.array(f.readlines()) + rm_plates= np.char.replace(rm_plates, '\n','').astype(int).tolist() + rm_fields.extend(rm_plates) + + if not ptt.exists(ptt.join(daily_dir,'etc', 'DarkMonitor_fields')) or clobber_lists is True: + try: + monit_fields = [] + field = DesignToField.select().join(Design).join(DesignMode).where(DesignMode.label == 'dark_monit').switch(DesignToField).join(Field) + for t in field: monit_fields.append(t.field.field_id) + field = DesignToField.select().join(Design).join(DesignMode).where(DesignMode.label == 'dark_monit_eng').switch(DesignToField).join(Field) + for t in field: monit_fields.append(t.field.field_id) + monit_fields=list(set(monit_fields)) + tmonit_fields=np.char.add(np.asarray(monit_fields).astype(str),'\n').tolist() + if ptt.exists(ptt.join(daily_dir,'etc')): + with open(ptt.join(daily_dir,'etc', 'DarkMonitor_fields'), 'w') as f: + f.writelines(tmonit_fields) + except: + monit_fields = [] + if not cron: warnings.warn('DarkMonitor_fields file is missing and DB query failed', UserWarning) + else: splog.warnings('Warning: DarkMonitor_fields file is missing and DB query failed') + else: + with open(ptt.join(daily_dir,'etc', 'DarkMonitor_fields'), 'r') as f: + monit_fields = np.array(f.readlines()) + monit_fields= np.char.replace(monit_fields, '\n','').astype(int).tolist() + return (rm_fields, monit_fields) + +def plot_milestone(obs, axs, max_mjd, im=3, jm=2, html=False): + milestones = read_table_yanny(ptt.join(daily_dir,'etc','fiber_milestones.par'), 'MILESTONE') + milestones = milestones[np.where(milestones['obs'] == obs)] + labels = [] + for row in milestones: + for i in range(0,im): + for j in range(0,jm): + + if row['mjd'] > max_mjd + 1: + continue + label = row['label'] if row['label'] != '' else None + + if not html: + if row['linestyle'] in ['-', '--', '-.', ':', 'None', ' ', + '', 'solid', 'dashed', 'dashdot', 'dotted']: + ls = row['linestyle'] + else: + ls = ( 0, tuple(np.array(tuple(row['linestyle'].replace(',','').replace(' ',''))).astype(int))) + if jm == 1: + axs[i].axvline(row['mjd'], color=row['color'], lw = .5, ls = ls, label=label) + else: + axs[i,j].axvline(row['mjd'], color=row['color'], lw = .5, ls = ls, label=label) + else: + if row['bklinestyle'] is None: continue + #'solid', 'dot', 'dash', 'longdash', 'dashdot', 'longdashdot' + bk2p = {'dotted':'dot','dashed':'dash','3 3 1 3 1 3':'longdashdot'} + if row['bklinestyle'] in bk2p.keys(): + row['bklinestyle'] = bk2p[row['bklinestyle']] + + taxs = dict(row = i+1, col=1) if jm == 1 else dict(row=i+1,col=j+1) + line = dict(color=colors[row['color']], width=1, dash=row['bklinestyle']) + axs.add_shape(type="line", x0=row['mjd'],x1=row['mjd'],y0=0,y1=1, opacity=.8, + xref=f'x1', yref=f'y domain', name=label, showlegend=False, + line=line,**taxs) + # Add an invisible scatter trace to serve as a legend entry for the line + if (label is not None and i == 0 and j == 0): + axs.add_trace(go.Scatter(x=[None],y=[None],mode='lines', opacity=.8, + line=line,showlegend=True, name=label)) + return(axs, milestones) + +def split_by_nan(x, y): + isnan = np.isnan(y) + segments = [] + start_idx = 0 + for idx in range(1, len(isnan)): + if isnan[idx] != isnan[idx - 1]: + if not isnan[start_idx:idx].all(): + segments.append((x[start_idx:idx], y[start_idx:idx])) + start_idx = idx + if not isnan[start_idx:].all(): + segments.append((x[start_idx:], y[start_idx:])) + return segments + +def plot_QA(run2ds, test, mjds={}, obs='APO', testp='/test/sean/', clobber_lists=False, + publish = False, epoch=False, cron = False, fast_opsdb=False, html=False, + html_name = None): + + if cron: + makedirs(ptt.join(daily_dir,'logs','QA'),exist_ok=True) + logging.getLogger('matplotlib.font_manager').setLevel(logging.WARNING) + logging.getLogger('peewee').setLevel(logging.WARNING) + splog = logging.getLogger('root') + filelog = logging.FileHandler(ptt.join(daily_dir,'logs','QA', + datetime.datetime.today().strftime("%m%d%Y")+f'-{obs}.log')) + filelog.setLevel(logging.DEBUG) + filelog.setFormatter(Formatter()) + console = logging.StreamHandler() + console.setLevel(logging.DEBUG) + console.setFormatter(Formatter()) + splog.addHandler(filelog) + splog.addHandler(console) + splog.setLevel(logging.DEBUG) + if plotly is None and html is True: + if cron: + splog.warnings('plotly not installed... defaulting to no html') + else: + warnings.warn('plotly not installed... defaulting to no html', UserWarning) + html = False + + rm_fields, monit_fields = load_fields(clobber_lists=clobber_lists) + all_data = None + all_mdate = '' + all_mdate_f = 0 + + for rd,run2d in enumerate(run2ds): + if test[rd] is True: test_path = testp + else: test_path = '' + old_paths = False + + es = 'epoch' if epoch else 'daily' + try: + datafile = ptt.join(getenv("BOSS_SPECTRO_REDUX"),test_path, run2d,'summary',es,'spCalib_QA-'+run2d+'.fits') + data = Table.read(datafile, format='fits') + for col in data.colnames: + if data[col].dtype.kind == 'S': # 'S' indicates byte strings + data[col] = data[col].astype(str) + data = data.to_pandas() + except: + print('Checking old path') + datafile = ptt.join(getenv("BOSS_SPECTRO_REDUX"),test_path, run2d,'spCalib_QA-'+run2d+'.csv') + data=pd.read_csv(datafile) + old_paths = True + mdate_f = ptt.getctime(datafile) + mdate = time.ctime(ptt.getctime(datafile)) + if all_mdate_f < mdate_f: + all_mdate_f = mdate_f + all_mdate = mdate + if mjds is not None: + if run2d in mjds.keys(): + if mjds[run2d][0] is not None: data=data[data['MJD']>=mjds[run2d][0]] + if mjds[run2d][1] is not None: data=data[data['MJD']<=mjds[run2d][1]] + if obs is not None: + data=data[data['OBS'] == obs] + max_mjd = np.max(data['MJD'].values) + RM=data[data['FIELD'].isin(rm_fields)] + Monit=data[data['FIELD'].isin(monit_fields)] + if all_data is None: + all_data = data + else: + all_data = pd.concat([all_data,data], ignore_index=True) + if not html: + rm_style = {'ls':'', 'marker':'.', 'color':'C1', 'alpha':.5, 'label':'RM Fields'} + main_style = {'ls':'', 'marker':'.', 'color':'C0', 'alpha':.2, 'label':'All Fields'} + monit_style = {'ls':'', 'marker':'.', 'color':'C3', 'alpha':.5, 'label':'DarkMonitoring Fields'} + ylim = [-0.10,0.10] + ylim_r = [-0.001,0.15] + if rd == 0: + fig, axs = plt.subplots(3,2, figsize=[12,6]) + axs, milestones = plot_milestone(obs, axs, max_mjd) + else: + rm_style['label'] = None + main_style['label'] = None + monit_style['label'] = None + for i in range(0,3): + axs[i,0].axhline(0, alpha= .2, color='k') + axs[i,1].axhline(0.025, alpha= .2, color='k') + axs[i,0].plot(data['MJD'], data[filters[i]+'_MEAN'], **main_style) + axs[i,0].plot(Monit['MJD'], Monit[filters[i]+'_MEAN'], **monit_style) + axs[i,0].plot(RM['MJD'], RM[filters[i]+'_MEAN'], **rm_style) + axs[i,1].plot(data['MJD'], data[filters[i]+'_SIG'], **main_style) + axs[i,1].plot(Monit['MJD'], Monit[filters[i]+'_SIG'], **monit_style) + axs[i,1].plot(RM['MJD'], RM[filters[i]+'_SIG'], **rm_style) + axs[i,0].set_ylim(ylim) + axs[i,1].set_ylim(ylim_r) + if i != 2: + axs[i,0].set_xticklabels([]) + axs[i,1].set_xticklabels([]) + else: + axs[i,0].set_xlabel('MJD') + axs[i,1].set_xlabel('MJD') + axs[i,0].set_ylabel('['+filters[i].lower()+']') + axs[i,1].set_ylabel('['+filters[i].lower()+']') + if i == 0: + axs[0,0].set_title('mean log(synflux/calibflux)') + axs[0,1].set_title('sigma log(synflux/calibflux)') + else: + if rd == 0: + test_path = testp if test[0] is True else '' + save_dir = getenv('BOSS_QA_DIR', default=getenv('BOSS_SPECTRO_REDUX')) + if html_name is None: + html_name = f'BOSS_QA-{obs}.html' + savename = ptt.join(save_dir,html_name) + #output_file(filename=savename, title=f'{obs} BOSS QA') + axs = make_subplots(rows=4, cols=2,shared_xaxes=True, shared_yaxes=False, + vertical_spacing=0.02,horizontal_spacing=.05, + subplot_titles = ('mean log(synflux/calibflux)', + 'sigma log(synflux/calibflux)', + '','','','','','')) + axs.update_xaxes(title='MJD',row=4) + for r in [1,2,3]: + axs.update_yaxes(title=f'[{filters[r-1].lower()}]',row=r) + axs.update_yaxes(title='SEEING50',row=4) + axs.update_layout( paper_bgcolor='white', plot_bgcolor='white') + for fr in [1,2,3]: + axs.update_yaxes(range=[-0.10,0.10], row=fr, col=1) + axs.update_yaxes(range=[-0.001,0.15], row=fr, col=2) + axs.update_xaxes(showgrid=False, row=fr, **axopts) + axs.update_yaxes(showgrid=False, row=fr, **ayopts) + + axs.add_shape(type='line', x0=0, x1=1, y0=0.025, y1=0.025, + xref='x domain', yref=f'y1',opacity=.2, + line=dict(color='black', dash='solid',), + row=fr, col=2) + axs.add_shape(type='line', x0=0, x1=1, y0=0, y1=0, + xref='x domain', yref=f'y1',opacity=.2, + line=dict(color='black', dash='solid',), + row=fr, col=1) + + axs.update_xaxes(showgrid=True, row=4, **axopts) + axs.update_yaxes(showgrid=True, row=4, **ayopts) + axs, milestones = plot_milestone(obs, axs, max_mjd, im=4, jm=2, html=True) + rm_style = dict(marker=dict(size=5, color='orange'),name='RM Fields', + mode='markers', opacity=.5) + + main_style = dict(marker=dict(size=5, color='blue'),name='All Fields', + mode='markers', opacity=.1) + + monit_style = dict(marker=dict(size=5, color='red'), mode='markers', + name='DarkMonitoring Fields', opacity=.5) + + for fr in [1,2,3]: + for c in [1,2]: + sl = True if (fr == 1 and c==1 and rd == 0) else False + + type = '_MEAN' if c ==1 else '_SIG' + ht='MJD,Mean: %{x:.2f},%{y:.2f}' if c ==1 else 'MJD,Sigma: %{x:.2f},%{y:.2f}' + if len(data) > 0: + axs.add_trace(go.Scatter(x=data['MJD'], y=data[filters[fr-1]+type], + showlegend=sl, **main_style,hovertemplate=ht), row=fr,col=c) + if len(Monit) > 0: + axs.add_trace(go.Scatter(x=Monit['MJD'], y=Monit[filters[fr-1]+type], + showlegend=sl, **monit_style,hovertemplate=ht), row=fr,col=c) + if len(RM) > 0: + axs.add_trace(go.Scatter(x=RM['MJD'], y=RM[filters[fr-1]+type], + showlegend=sl, **rm_style,hovertemplate=ht), row=fr,col=c) + + if not html: + if publish: + plt.tight_layout(rect=(0,.07,1,1)) + nmiles = len(milestones[milestones['label'] != '']) + axs[2,0].legend(frameon=True,fontsize=8,loc=2,ncol=((nmiles+3)//2), bbox_to_anchor=(0, -.35)) + else: + plt.tight_layout() + axs[0,0].legend(frameon=True,fontsize=3,loc=2,ncol=2) + plt.gcf().text(0.02,0.02,'Updated: '+mdate+ ' (Latest MJD:'+str(max(data['MJD']))+')', fontsize=6, ha='left') + plt.gcf().text(0.98,0.02,'Run2d='+','.join(run2ds), fontsize=6, ha='right') + plt.gcf().text(0.50,0.02,'Observatory='+obs, fontsize=6, ha='center') + + ctype = 'epoch' if epoch else 'daily' + if len(run2ds) == 1: + outdir = ptt.join(getenv("BOSS_SPECTRO_REDUX"), test_path, run2d, 'summary') + outdir = ptt.join(outdir,ctype,'spCalib_QA-'+run2ds[0]+'-'+obs+'.png') + else: + outdir = ptt.join(getenv("BOSS_SPECTRO_REDUX"), test_path) + outdir = ptt.join(outdir,f'spCalib_QA-{ctype}-'+'+'.join(run2ds)+'-'+obs+'.png') + + plt.savefig(outdir, dpi=200) + plt.show() + if len(run2ds) == 1: + plotsn2=True + else: + plotsn2 = False + else: + see = None + dark_style = dict(marker=dict(size=3, color='orange'),name='Dark', + mode='markers', opacity=.5) + bright_style = dict(marker=dict(size=3, color='green'),name='Bright', + mode='markers', opacity=.5) + plate_style = dict(marker=dict(size=3, color='blue'),name='Plate', + mode='markers', opacity=.5) + bidx = [] + didx = [] + data = all_data + if (len(run2ds) == 1) or (len(mjds) == len(run2ds)): + plotsn2 = True + else: + plotsn2 = False + + if plotsn2: + mjd_limits = mjds + spall = None + mjds = [] + mj = [] + fsn2g = [] + fsn2r = [] + fsn2i = [] + exptime = [] + fcad = [] + see = [] + idx = 0 + + ep = '' if not epoch else 'epoch' + if cron: splog.info(f"Reading Field-MJD spAll for run2d={' '.join(run2ds)}") + + for i, row in tqdm(data.iterrows(), total=data.shape[0]): + idx = idx + 1 + if mjd_limits is not None: + r2d = None + for ir2d, run2d in enumerate(run2ds): + if mjd_limits[run2d][0] is not None: + if int(row['MJD']) < mjd_limits[run2d][0]: + continue + if mjd_limits[run2d][1] is not None: + if int(row['MJD']) > mjd_limits[run2d][1]: + continue + r2d = run2d + break + else: + ir2d = 0 + r2d = run2ds[0] + if r2d is None: + if cron: + splog.info('skipping: spAll-'+str(row['FIELD']).zfill(6)+'-'+str(row['MJD'])+'.fits') + continue + + test_path = testp if test[ir2d] is True else '' + if not old_paths: + spallfile = glob(ptt.join(field_spec_dir(ptt.join(getenv("BOSS_SPECTRO_REDUX"),test_path), + run2d, row['FIELD'], row['MJD'], + epoch = epoch, full= True), + f"spAll-{str(row['FIELD']).zfill(6)}-{row['MJD']}.fits*")) + else: + if epoch: + spallfile = glob(ptt.join(ptt.join(getenv("BOSS_SPECTRO_REDUX"),test_path), + run2d, 'epoch','spectra','full', + str(row['FIELD']).zfill(6), str(row['MJD']), + f"spAll-{str(row['FIELD']).zfill(6)}-{row['MJD']}.fits*")) + + else: + spallfile = glob(ptt.join(ptt.join(getenv("BOSS_SPECTRO_REDUX"),test_path), + run2d, 'spectra','full', + str(row['FIELD']).zfill(6), str(row['MJD']), + f"spAll-{str(row['FIELD']).zfill(6)}-{row['MJD']}.fits*")) + + if cron: + splog.info(f'({idx}/{data.shape[0]}) '+'spAll-'+str(row['FIELD']).zfill(6)+'-'+str(row['MJD'])+'.fits'+ f'(run2d:{r2d})') + if len(spallfile) == 0: + continue + else: + spallfile = spallfile[0] + try: + spall = Table(fits.getdata(spallfile,1)) + except Exception as e: + print(e) + continue + for row in (spall): + tobs = None + if np.isnan(row['MJD_FINAL']): + continue + if row['MJD_FINAL'] <= 59847: + tobs = 'APO' + elif (int(row['FIELD']) > 100000) & (row['MJD_FINAL'] < 60191): + tobs = 'APO' + else: + if 'OBS' in row.colnames: + tobs = row['OBS'] + else: + try: + tobs = row['OBSERVATORY'] + except: + continue + if tobs not in ['APO','LCO']: + idx = np.where(data['FIELD'] == int(row['FIELD']))[0] + if len(idx) == 0: continue + else: tobs = obs.upper() + if tobs != obs.upper(): + continue + nexp = len(row['FIELDSNR2R_LIST'].split()) + + if row['MJD_FINAL'] == 0: + mjds.extend([row['MJD']]*nexp) + else: + mjds.extend([row['MJD_FINAL']]*nexp) + +# mj.extend([row['MJD']]*nexp) + mj.extend((np.atleast_1d(np.asarray(row['TAI_LIST'].split())).astype(float)/(24*3600)).tolist()) + fsn2g.extend(row['FIELDSNR2G_LIST'].split()) + fsn2r.extend(row['FIELDSNR2R_LIST'].split()) + fsn2i.extend(row['FIELDSNR2I_LIST'].split()) + exptime.extend([row['EXPTIME']/row['NEXP']]*nexp) + fcad.extend([row['CADENCE']]*nexp) + see.extend([row['SEEING50']]*nexp) + mjds = np.asarray(mjds) + mj = np.asarray(mj) + exptime = np.asarray(exptime) + fcad = np.asarray(fcad) + fsn2g = np.asarray(fsn2g).astype(float) + fsn2r = np.asarray(fsn2r).astype(float) + fsn2i = np.asarray(fsn2i).astype(float) + see = np.asarray(see).astype(float) + test = Table({'MJD': mjds,'MJ':mj, 'exptime':exptime, 'fcad':fcad,'fsn2g':fsn2g,'fsn2r':fsn2r,'fsn2i':fsn2i,'see':see}) + if len(test) == 0: + return + test = unique(test,keys=['MJ','exptime','fcad','fsn2g','fsn2r','fsn2i']) + mjds = test['MJ'].data + exptime = test['exptime'].data + fsn2g = test['fsn2g'].data + fsn2r = test['fsn2r'].data + fsn2i = test['fsn2i'].data + fcad = test['fcad'].data + see = test['see'].data + + didx = np.where(wwhere(fcad, 'dark*'))[0] + bidx = np.where(wwhere(fcad, 'bright*'))[0] + pidx = np.where(wwhere(fcad, 'plate*'))[0] + + if not html: + fig, axs = plt.subplots(4,1, figsize=[12,8]) + + axs, milestones = plot_milestone(obs, axs, max_mjd, im=4, jm=1) + + axs[0] = plot_sn2_filt(axs[0], mjds, fsn2g, exptime, pidx, bidx, didx, 'FieldSN2_G/(900s)') + axs[1] = plot_sn2_filt(axs[1], mjds, fsn2r, exptime, pidx, bidx, didx, 'FieldSN2_R/(900s)') + axs[1].set_ylim(-.5,22) + axs[2] = plot_sn2_filt(axs[2], mjds, fsn2i, exptime, pidx, bidx, didx, 'FieldSN2_I/(900s)') + axs[2].set_ylim(-.5,22) + + + axs[3] = plot_sn2_filt(axs[3], mjds, see, np.full_like(see, 900.0), pidx, bidx, didx, 'SEEING50', labelbottom=True) + ylim = list(axs[3].get_ylim()) + if ylim[1] > 5.2: + ylim[1] = 5.2 + if ylim[0] > 0: + ylim[0] = -.2 + axs[3].set_ylim(ylim) + + axs[0].set_title('Field SNR^2 Per Exposure (900s)') + fig.tight_layout() + plt.subplots_adjust(hspace=0,bottom=.08) + plt.gcf().text(0.02,0.02,'Updated: '+mdate+ ' (Latest MJD:'+str(max(mj))+')', fontsize=6, ha='left') + plt.gcf().text(0.98,0.02,'Run2d='+','.join(run2ds), fontsize=6, ha='right') + plt.gcf().text(0.50,0.02,'Observatory='+obs, fontsize=6, ha='center') + outdir = ptt.join(getenv("BOSS_SPECTRO_REDUX"), test_path, run2d) + outdir = ptt.join(outdir,'summary') + if not epoch: + plt.savefig(ptt.join(outdir,'daily','SN2-'+run2ds[0]+'-'+obs+'.png'), dpi=200) + else: + plt.savefig(ptt.join(outdir,'epoch','SN2-'+run2ds[0]+'-'+obs+'.png'), dpi=200) + plt.show() + else: + axs2 = make_subplots(rows=4, cols=1, shared_xaxes=True, shared_yaxes=False,vertical_spacing=0.02, + subplot_titles=('Field SNR^2 Per Exposure (900s)','','','')) + axs2.update_xaxes(title='MJD',row=4) + for r in [1,2,3]: + axs2.update_yaxes(title=f'FieldSN2_{filters[r-1].upper()}/(900s)',row=r) + axs2.update_xaxes(showgrid=True, row=r, **axopts) + axs2.update_yaxes(showgrid=True, row=r, **ayopts) + axs2.update_xaxes(showgrid=True, row=4, **axopts) + axs2.update_yaxes(showgrid=True, row=4, **ayopts) + axs2.update_yaxes(title='SEEING50',row=4) + axs2.update_layout( paper_bgcolor='white', plot_bgcolor='white') + for r in [1,2,3]: + axs2.add_shape(go.layout.Shape(type="line", x0=0,x1=1,y0=0,y1=0, + xref=f'x domain', yref=f'y', opacity=.2, + line=dict(color="black", dash="solid")), + row=r,col=1) + + axs2, milestones = plot_milestone(obs, axs2, max_mjd, im=4, jm=1, html=True) + fsn2 = {'G':fsn2g,'R':fsn2r,'I':fsn2i} + didx = np.where(wwhere(fcad, 'dark*'))[0] + bidx = np.where(wwhere(fcad, 'bright*'))[0] + pidx = np.where(wwhere(fcad, 'plate*'))[0] + for i, filt in enumerate(filters): + fi = i+1 + sl = True if fi == 1 else False + ht='MJD,FieldSN2: %{x:.2f},%{y:.2f}' + if len(pidx) > 0: + axs2.add_trace(go.Scatter(x=mjds[pidx], y=fsn2[filt][pidx], + showlegend=sl, **plate_style,hovertemplate=ht), row=fi, col=1) + if len(bidx) > 0: + axs2.add_trace(go.Scatter(x=mjds[bidx], y=fsn2[filt][bidx], + showlegend=sl, **bright_style,hovertemplate=ht), row=fi, col=1) + if len(didx) > 0: + axs2.add_trace(go.Scatter(x=mjds[didx], y=fsn2[filt][didx], + showlegend=sl, **dark_style,hovertemplate=ht), row=fi, col=1) + if len(didx) > 0: + moving_mjd, moving_avg, moving_16, moving_84 = runAvg(mjds[didx],fsn2[filt][didx]*900.0/exptime[didx]) + color=dark_style['marker']['color'] + for (x_seg, upper_seg), (_, lower_seg) in zip(split_by_nan(moving_mjd, moving_84), split_by_nan(moving_mjd, moving_16)): + axs2.add_trace(go.Scatter(x=x_seg, y=upper_seg,fill=None, mode='lines',opacity=.1, + line=dict(color=color, width=.5), showlegend=False),row=fi, col=1) + axs2.add_trace(go.Scatter(x=x_seg, y=lower_seg, fill='tonexty',opacity=.1, + mode='lines', line=dict(color=color, width=.5), showlegend=False), row=fi, col=1) + axs2.add_trace(go.Scatter(x=moving_mjd,y=moving_avg, mode='lines',opacity=1, + line=dict(color=color, width=1),showlegend=False), row=fi, col=1) + if len(pidx) > 0: + moving_mjd, moving_avg, moving_16, moving_84 = runAvg(mjds[pidx],fsn2[filt][pidx]*900.0/exptime[pidx]) + color=plate_style['marker']['color'] + for (x_seg, upper_seg), (_, lower_seg) in zip(split_by_nan(moving_mjd, moving_84), split_by_nan(moving_mjd, moving_16)): + axs2.add_trace(go.Scatter(x=x_seg, y=upper_seg,fill=None, mode='lines',opacity=.1, + line=dict(color=color, width=.5), showlegend=False),row=fi, col=1) + axs2.add_trace(go.Scatter(x=x_seg, y=lower_seg, fill='tonexty',opacity=.1, + mode='lines', line=dict(color=color, width=.5), showlegend=False), row=fi, col=1) + axs2.add_trace(go.Scatter(x=moving_mjd,y=moving_avg, mode='lines',opacity=1, + line=dict(color=color, width=1),showlegend=False), row=fi, col=1) + + i = 3 + fr = i+1 + ht='MJD,SEEING50: %{x:.2f},%{y:.2f}' + if len(pidx) > 0: + axs2.add_trace(go.Scatter(x=mjds[pidx], y=see[pidx], + showlegend=False, **plate_style,hovertemplate=ht), row=fr, col=1) + for c in [1,2]: + axs.add_trace(go.Scatter(x=mjds[pidx], y=see[pidx],showlegend=False, + **plate_style,hovertemplate=ht), row=4, col=c) + if len(bidx) > 0: + axs2.add_trace(go.Scatter(x=mjds[bidx], y=see[bidx], + showlegend=False, **bright_style,hovertemplate=ht), row=fr, col=1) + for c in [1,2]: + axs.add_trace(go.Scatter(x=mjds[bidx], y=see[bidx],showlegend=False, + **bright_style,hovertemplate=ht), row=4, col=c) + if len(didx) > 0: + axs2.add_trace(go.Scatter(x=mjds[didx], y=see[didx], + showlegend=False, **dark_style,hovertemplate=ht), row=fr, col=1) + for c in [1,2]: + axs.add_trace(go.Scatter(x=mjds[didx], y=see[didx],showlegend=False, + **dark_style,hovertemplate=ht), row=4, col=c) + + if len(didx) > 0: + moving_mjd, moving_avg, moving_16, moving_84 = runAvg(mjds[didx],see[didx]) + color=dark_style['marker']['color'] + for (x_seg, upper_seg), (_, lower_seg) in zip(split_by_nan(moving_mjd, moving_84), split_by_nan(moving_mjd, moving_16)): + axs2.add_trace(go.Scatter(x=x_seg, y=upper_seg,fill=None, mode='lines',opacity=.1, + line=dict(color=color, width=.5), showlegend=False),row=4, col=1) + axs2.add_trace(go.Scatter(x=x_seg, y=lower_seg, fill='tonexty',opacity=.1, + mode='lines', line=dict(color=color, width=.5), showlegend=False), row=4, col=1) + axs2.add_trace(go.Scatter(x=moving_mjd,y=moving_avg, mode='lines',opacity=1, + line=dict(color=color, width=1),showlegend=False), row=4, col=1) + + + for c in [1,2]: + for (x_seg, upper_seg), (_, lower_seg) in zip(split_by_nan(moving_mjd, moving_84), split_by_nan(moving_mjd, moving_16)): + axs.add_trace(go.Scatter(x=x_seg, y=upper_seg,fill=None, mode='lines',opacity=.1, + line=dict(color=color, width=.5), showlegend=False),row=4, col=c) + axs.add_trace(go.Scatter(x=x_seg, y=lower_seg, fill='tonexty',opacity=.1, + mode='lines', line=dict(color=color, width=.5), showlegend=False), row=4, col=c) + axs.add_trace(go.Scatter(x=moving_mjd,y=moving_avg, mode='lines',opacity=1, + line=dict(color=color, width=1),showlegend=False), row=4, col=c) + + if len(pidx) > 0: + moving_mjd, moving_avg, moving_16, moving_84 = runAvg(mjds[pidx],see[pidx]) + color=plate_style['marker']['color'] + for (x_seg, upper_seg), (_, lower_seg) in zip(split_by_nan(moving_mjd, moving_84), split_by_nan(moving_mjd, moving_16)): + axs2.add_trace(go.Scatter(x=x_seg, y=upper_seg,fill=None, mode='lines',opacity=.1, + line=dict(color=color, width=.5), showlegend=False),row=4, col=1) + axs2.add_trace(go.Scatter(x=x_seg, y=lower_seg, fill='tonexty',opacity=.1, + mode='lines', line=dict(color=color, width=.5), showlegend=False), row=4, col=1) + axs2.add_trace(go.Scatter(x=moving_mjd,y=moving_avg, mode='lines',opacity=1, + line=dict(color=color, width=1),showlegend=False), row=4, col=1) + + for c in [1,2]: + for (x_seg, upper_seg), (_, lower_seg) in zip(split_by_nan(moving_mjd, moving_84), split_by_nan(moving_mjd, moving_16)): + axs.add_trace(go.Scatter(x=x_seg, y=upper_seg,fill=None, mode='lines',opacity=.1, + line=dict(color=color, width=.5), showlegend=False),row=4, col=c) + axs.add_trace(go.Scatter(x=x_seg, y=lower_seg, fill='tonexty',opacity=.1, + mode='lines', line=dict(color=color, width=.5), showlegend=False), row=4, col=c) + axs.add_trace(go.Scatter(x=moving_mjd,y=moving_avg, mode='lines',opacity=1, + line=dict(color=color, width=1),showlegend=False), row=4, col=c) + + axs.update_layout(legend=dict(orientation='h', x=0.5,y=-0.1, xanchor='center', + yanchor='top', font=dict(size=8),traceorder='normal', + itemwidth=30, tracegroupgap=1,borderwidth=.5, + bgcolor='rgba(255, 255, 255, 0.5)', bordercolor='black')) + axs2.update_layout(legend=dict(orientation='h', x=0.5,y=-0.12, xanchor='center', + yanchor='top', font=dict(size=8),traceorder='normal', + itemwidth=30, tracegroupgap=1,borderwidth=.5, + bgcolor='rgba(255, 255, 255, 0.5)', bordercolor='black')) + +# axs.update_layout(legend=dict(x=0,y=1,traceorder='normal',orientation='v', +# font=dict(size=8), bgcolor='rgba(255, 255, 255, 0.5)', +# bordercolor='black', borderwidth=.5, tracegroupgap=1)) +# axs2.update_layout(legend=dict(x=0,y=1,traceorder='normal',orientation='v', +# font=dict(size=8), bgcolor='rgba(255, 255, 255, 0.5)', +# bordercolor='black', borderwidth=.5, tracegroupgap=1)) + if see is not None: + ymin = np.nanmin(see) + ymax = np.nanmax(see) + if ymin < 0: + ymin = -0.2 + if ymax > 5.2: + ymax = 5.2 + axs.update_yaxes(range=[ymin, ymax], row=4) + axs2.update_yaxes(range=[ymin, ymax], row=4) + + + if cron: splog.info(f'Reading OPSDB Exposures') + try: + opsdb.database.set_profile('operations') + environ['OBSERVATORY'] = obs.upper() + opsdb.database.connect() # This will recreate the base model class and reload all the model classes + CCDs = ['b2','r2'] if obs.upper() == 'LCO' else ['b1','r1'] + from sdssdb.peewee.sdss5db.opsdb import Exposure, CameraFrame, Camera, Configuration + except Exception as e: + print(e) + if not cron: warnings.warn('No Connection to the SDSS-V OpsDB (internal only)', UserWarning) + else: splog.warnings('WARNING: No Connection to the SDSS-V OpsDB (internal only)') + sos_data = None + Configuration = None + + if Configuration is not None: + if not fast_opsdb: + blue = CameraFrame.select(CameraFrame.exposure.exposure_no.alias('expid'),CameraFrame.exposure.exposure_time.alias('exptime'), CameraFrame.exposure.start_time, CameraFrame.sn2.alias(CCDs[0]), Design.design_mode.alias('design_mode')).join(Exposure).join(Configuration).join(Design, on=(Configuration.design_id == Design.design_id)).join(DesignMode).switch(CameraFrame).join(Camera).where(Camera.label == CCDs[0]).dicts() + red = CameraFrame.select(CameraFrame.exposure.exposure_no.alias('expid'), CameraFrame.sn2.alias(CCDs[1])).join(Exposure).switch(CameraFrame).join(Camera).where(Camera.label == CCDs[1]).dicts() + sos_data=pd.DataFrame(blue) + sos_data_r = pd.DataFrame(red) + sos_data = pd.merge(sos_data, sos_data_r, on='expid', how='outer', indicator=True) + sos_data = sos_data.loc[sos_data['_merge'] == 'both'] + sos_data = sos_data.assign(mjd=lambda x: Time(x.start_time).mjd) + sos_data = sos_data.sort_values('mjd') + _dir = ptt.dirname(savename) + _name = ptt.basename(savename).replace('.html','_sos.csv') + sos_data.to_csv(ptt.join(_dir,f'.{_name}')) + else: + _dir = ptt.dirname(savename) + _name = ptt.basename(savename).replace('.html','_sos.csv') + if ptt.exists(ptt.join(_dir,f'.{_name}')): + sos_data = pd.read_csv(ptt.join(_dir,f'.{_name}')) + else: + sos_data = None + if cron:splog.info(f'No OPSDB SOS data') + else: warnings.warn('No OPSDB SOS data') + if cron: splog.info(f'Done Reading OPSDB Exposures') + + if sos_data is not None: + for ccd in CCDs: + sos_data[f'{ccd}_900'] = sos_data[ccd]/sos_data['exptime']*900 + axs3 = make_subplots(rows=3, cols=1, shared_xaxes=True, shared_yaxes=False,vertical_spacing=0.02, + subplot_titles=('SOS SNR^2 Per Exposure (900s)','','','')) + axs3.update_xaxes(title='MJD',row=3) + axs3.update_yaxes(title=f'SOS SN2_{CCDs[0]}/(900s)',row=1) + axs3.update_yaxes(title=f'SOS SN2_{CCDs[1]}/(900s)',row=2) + axs3.update_yaxes(title='SEEING50',row=3) + axs3.update_layout( paper_bgcolor='white', plot_bgcolor='white') + for r in [1,2,3]: + axs3.update_xaxes(showgrid=True, row=r, **axopts) + axs3.update_yaxes(showgrid=True, row=r, **ayopts) + axs3, milestones = plot_milestone(obs, axs3, max_mjd, im=3, jm=1, html=True) + dsidx = np.where(wwhere(sos_data['design_mode'].values, 'dark*'))[0] + bsidx = np.where(wwhere(sos_data['design_mode'].values, 'bright*'))[0] + + for i, ccd in enumerate(CCDs): + r = i + 1 + sl = True if r == 1 else False + ht = 'MJD,SOS_SN2: %{x:.2f},%{y:.2f}' + axs3.add_shape(go.layout.Shape(type="line", x0=0,x1=1,y0=0,y1=0, + xref=f'x domain', yref=f'y', opacity=.2, + line=dict(color="black", dash="solid")), + row=r,col=1) + + if len(bsidx) > 0: + axs3.add_trace(go.Scatter(x=sos_data['mjd'].values[bsidx], + y=sos_data[f'{ccd}_900'].values[bsidx], + showlegend=sl, **bright_style,hovertemplate=ht), row=r, col=1) + + if len(dsidx) > 0: + axs3.add_trace(go.Scatter(x=sos_data['mjd'].values[dsidx], + y=sos_data[f'{ccd}_900'].values[dsidx], + showlegend=sl, **dark_style,hovertemplate=ht), row=r, col=1) + color=dark_style['marker']['color'] + moving_mjd, moving_avg, moving_16, moving_84 = runAvg(sos_data['mjd'].values[dsidx], + sos_data[f'{ccd}_900'].values[dsidx]) + + for (x_seg, upper_seg), (_, lower_seg) in zip(split_by_nan(moving_mjd, moving_84), split_by_nan(moving_mjd, moving_16)): + axs3.add_trace(go.Scatter(x=x_seg, y=upper_seg,fill=None, mode='lines',opacity=.1, + line=dict(color=color, width=.5), showlegend=False),row=r, col=1) + axs3.add_trace(go.Scatter(x=x_seg, y=lower_seg, fill='tonexty',opacity=.1, + mode='lines', line=dict(color=color, width=.5), showlegend=False), row=r, col=1) + axs3.add_trace(go.Scatter(x=moving_mjd,y=moving_avg, mode='lines',opacity=1, + line=dict(color=color, width=1),showlegend=False), row=r, col=1) + + i = 2 + fr = i+1 + ht = 'MJD,Seeing: %{x:.2f},%{y:.2f}' + if len(bidx) > 0: + axs3.add_trace(go.Scatter(x=mjds[bidx], y=see[bidx], + showlegend=False, **bright_style,hovertemplate=ht), row=fr, col=1) + if len(didx) > 0: + axs3.add_trace(go.Scatter(x=mjds[didx], y=see[didx], + showlegend=False, **dark_style,hovertemplate=ht), row=fr, col=1) + moving_mjd, moving_avg, moving_16, moving_84 = runAvg(mjds[didx],see[didx]) + color=dark_style['marker']['color'] + for (x_seg, upper_seg), (_, lower_seg) in zip(split_by_nan(moving_mjd, moving_84), split_by_nan(moving_mjd, moving_16)): + axs3.add_trace(go.Scatter(x=x_seg, y=upper_seg,fill=None, mode='lines',opacity=.1, + line=dict(color=color, width=.5), showlegend=False),row=fr, col=1) + axs3.add_trace(go.Scatter(x=x_seg, y=lower_seg, fill='tonexty',opacity=.1, + mode='lines', line=dict(color=color, width=.5), showlegend=False), row=fr, col=1) + axs3.add_trace(go.Scatter(x=moving_mjd,y=moving_avg, mode='lines',opacity=1, + line=dict(color=color, width=1),showlegend=False), row=fr, col=1) + + # axs3.update_layout(legend=dict(x=0,y=1,traceorder='normal',orientation='v', + # font=dict(size=8), bgcolor='rgba(255, 255, 255, 0.5)', + # bordercolor='black', borderwidth=.5, tracegroupgap=1)) + axs3.update_layout(legend=dict(orientation='h', x=0.5,y=-0.12, xanchor='center', + yanchor='top', font=dict(size=8),traceorder='normal', + itemwidth=30, tracegroupgap=1,borderwidth=.5, + bgcolor='rgba(255, 255, 255, 0.5)', bordercolor='black')) + + if see is not None: + ymin = np.nanmin(see) + ymax = np.nanmax(see) + if ymin < 0: + ymin = -0.2 + if ymax > 5.2: + ymax = 5.2 + axs3.update_yaxes(range=[ymin, ymax], row=4) + sosmjd = max(sos_data['mjd']) + else: + axs3= [] + sosmjd = '' + + title = '\n'.join([f"Spectro-photometric Plot Updated: {all_mdate}
",
+ f" Latest Full Pipeline MJD:{max(data['MJD'])}
",
+ f" Latest SOS Pipeline MJD:{sosmjd}
",
+ f" Last Updated: {time.ctime()} (MJD: {int(Time.now().mjd)})
') - - f.write('
',addstring, + re.sub('BOSS Spectro','BOSS Spectro (Current)', line))))) + diff --git a/python/boss_drp/sos/sdR_hdrfix.py b/python/boss_drp/sos/sdR_hdrfix.py new file mode 100755 index 000000000..8aa1efb61 --- /dev/null +++ b/python/boss_drp/sos/sdR_hdrfix.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +from boss_drp.utils import putils +from boss_drp.utils.lock import lock, unlock +from boss_drp.utils.hash import create_hash +from boss_drp.sos.run_log2html import run_soslog2html + +from pydl.pydlutils import yanny +from astropy.table import Table, unique +from astropy.io import fits +from os import getenv, remove, sep +from os import path as ptt +from glob import glob +import shutil +from collections import OrderedDict +import platform +from time import sleep +import re +import numpy as np + +def getLastMJD(silent=True): + if 'sdss5' not in platform.node(): + print('mjd is required when not running at observatories') + exit() + else: + path = ptt.join('/','data','spectro', '?????') + def get_key(fp): + if not ptt.isdir(fp): return(0) + filename = ptt.basename(fp) + int_part = filename.split()[0] + return(int(int_part)) + files = sorted(glob(path),key=get_key) + mjd = ptt.basename(files[-1]) + if silent is not True: + print('Latest MJD %s' % mjd) + return mjd + +def read_sdHdrFix(sdHdrFix_file): + try: + return(yanny.read_table_yanny(sdHdrFix_file, 'OPHDRFIX')) + except: + return(None) + + +def fixhdr(expid, hdrcards, mjd=None, obs=getenv('OBSERVATORY'), clobber=False, cameras='??'): + if mjd is None: mjd = getLastMJD() + + sdHdrFix_file = ptt.join(getenv('SDHDRFIX_DIR'), obs.lower(), 'sdHdrfix', 'sdHdrFix-'+str(mjd)+'.par') + + updates= None + + if clobber is not True: + updates = read_sdHdrFix(sdHdrFix_file) + + if updates is None: + updates = Table(names=('fileroot', 'keyword', 'value'), + descriptions = ('Root of file name, without any ".fit" suffix', 'Keyword name', 'Keyword value (as a string)'), + dtype = ('S20', 'S9', 'S80')) + + updates.meta = OrderedDict({'MJD': str(mjd) +" # Modified Julian Date for sdHdrFix file", + 'OBS': str(obs) +" # Observatory" }) + + for key in hdrcards.keys(): + frame = 'sdR-'+cameras+'-'+str(expid).zfill(8) + updates.add_row((frame, key.upper(), hdrcards[key])) + if key.lower() == 'quality': + fixSOSlog(frame,mjd,hdrcards[key],obs) + updates = unique(updates, keys=['fileroot','keyword'], keep='last') + + print('Writing to: ',sdHdrFix_file) + print(updates) + + if ptt.exists(sdHdrFix_file): + remove(sdHdrFix_file) + yanny.write_ndarray_to_yanny(sdHdrFix_file, updates, structnames='OPHDRFIX',hdr=updates.meta, comments=None) + +def fixSOSlog(frame,mjd,quality,obs): + logfiles = [] + logfiles.append(ptt.abspath(ptt.join(sep,'data','boss','sos',f'{mjd}',f'logfile-{mjd}.fits'))) + logfiles.append(ptt.abspath(ptt.join(sep,'data','boss','sosredo',f'{mjd}',f'logfile-{mjd}.fits'))) + logfiles.append(ptt.abspath(ptt.join(sep,'data','boss','sosredo','dev',f'{mjd}',f'logfile-{mjd}.fits'))) + for lf in logfiles: + if ptt.exists(lf): + if lock(f'{lf}', pause = 5): + try: + with fits.open(lf, mode='update') as hdul: + print(f'Updating {lf}') + for ext in [1,2,3,4]: + try: + hdul[ext] + except: + continue + if '??' in frame: + ccds = ['b1','r1'] if obs.lower() == 'apo' else ['b2','r2'] + else: + ccds = [None] + + for ccd in ccds: + tframe = frame.replace('??',ccd) if ccd is not None else frame + if hdul[ext].data is None: continue + idx = np.where(hdul[ext].data['FILENAME'] == f'{tframe}.fit.gz')[0] + if len(idx) == 0: + continue + else: + hdul[ext].data['QUALITY'][idx[0]] = quality + hdul.flush() + finally: + unlock(f'{lf}') + run_soslog2html(lf, mjd, obs) + + test = create_hash(ptt.dirname(lf)) + if test: + print("\nsha1sum is locked") + else: + continue + else: + continue +class Range(object): + def __init__(self, start, end): + self.start = start + self.end = end + def __eq__(self, other): + return self.start <= other <= self.end + def __repr__(self): + return '{{{0} - {1}}}'.format(self.start, self.end) + diff --git a/python/boss_drp/sos/sos_classes.py b/python/boss_drp/sos/sos_classes.py new file mode 100644 index 000000000..08d600fef --- /dev/null +++ b/python/boss_drp/sos/sos_classes.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python + +import os, sys +import os.path as ptt + +""" Miscellaneous classes for sos programs """ + +#### +class Consts: + """Holds various constants use by the sos programs""" + + def __init__(self): + self.lockFileBase = "sos_runner" + self.dieFileName = "sos_die.die.die" + self.logName = "sos_log" + self.configName = "sos_config.conf" + self.MJDGlob = "[0-9]"*5 + self.versionFile = "sos_version" + self.licensePause = "5" + + def __repr__(self): + return self.__str__() + + def __str__(self): + return ("Consts:\n" + + "lockFile: " + self.lockFileBase + "\n" + + "dieFile: " + self.dieFileName + "\n" + + "logName: " + self.logName + "\n" + + "configName: " + self.configName + "\n" + + "versionFile: " + self.versionFile + "\n" + + "MJDGlob: " + self.MJDGlob); + + + +#### +class Config: + """Holds the configuration information read from the command line or ini""" + + def __init__(self): + self.MJD = "0" + self.fps = True + self.exposure = None + self.fitsDir = '/data/spectro' + self.plugDir = os.getenv('SDSSCORE_DIR') + self.controlDir = '/home/sdss5/boss/sos/control' + self.logDir = '/home/sdss5/boss/sos/logs' + self.sosdir = '/data/boss/sos' + self.dlogLevel = 40 + self.logLevel = 40 + self.iname = "" + self.globs = ["*"] + self.command = "" + self.pollDelay = 2 + self.nosvn = True + self.nocal = True + self.bookkeep = False + self.nice = False + self.platedb = False + self.redo = False + self.catchup = False + self.utah = False + self.test = False + self.nodb = False + self.no_reject = False + self.clobber_fibermap = False + self.sdssv_sn2 = False + self.sn2_15 = False + self.arc2trace = False + self.forcea2t = False + self.pause = False + self.verbose = 6 + self.termverbose=False + self.set_logLevel() + + def set_logLevel(self): + # Don't want to apply -v on each call, so always start with a base + if (self.verbose > 0): + self.logLevel = max(1, self.dlogLevel - self.verbose * 10) + + def setup(self, CCD='b1', mjd = None, exp = None, + redo=False, catchup=False, test=False, systemd=False, + no_gz=False, nodb=False, no_reject = False, sdssv_sn2 = False, + pause = False, arc2trace=False, forcea2t=False, sn2_15 = False, + clobber_fibermap=False, utah=False, termverbose=False): + self.nodb = nodb + self.no_reject = no_reject + self.sdssv_sn2 = sdssv_sn2 + self.sn2_15 = sn2_15 + self.arc2trace = arc2trace + self.forcea2t = forcea2t + self.pause = pause + self.clobber_fibermap = clobber_fibermap + self.set_logLevel() + self.iname = CCD + self.termverbose = termverbose + + if no_gz: + self.globs=["sdR-"+CCD+"-*.fit.gz","sdR-"+CCD+"-*.fit"] + else: + self.globs=["sdR-"+CCD+"-*.fit.gz"] + + if systemd: + self.redo = False + self.catchup = False + exp = None + mjd = None + self.termverbose = False + + if redo: + self.sosdir='/data/boss/sosredo' + self.nice=True + self.redo=True + self.iname = self.iname+'_redo' + elif test: + self.sosdir='/data/boss/sosredo/dev' + self.nice=True + self.redo=True + self.iname = self.iname+'_dev' + self.logDir = '/data/boss/sosredo/dev/logs/' + self.controlDir = '/data/boss/sosredo/dev/control/' + self.nodb = True + self.test = True + elif catchup: + self.nice=True + self.redo=True + self.catchup = True + self.iname = self.iname+'_catchup' + elif utah: + self.redo=True + self.nodb=True + self.utah=True + self.iname = self.iname+'_utah' + self.nice=True + self.pause=False + self.sosdir = ptt.join(os.getenv('BOSS_SPECTRO_REDUX'), 'sos', os.getenv('RUN2D', default=''), '{obs}') + self.sosdir = self.sosdir.format(obs=os.getenv('OBSERVATORY').lower()) + self.logDir = ptt.join(self.sosdir, 'logs') + self.controlDir = ptt.join(self.sosdir, 'control') + if os.getenv('OBSERVATORY').lower() == 'apo': + self.fitsDir = os.getenv('BOSS_SPECTRO_DATA_N') + else: + self.fitsDir = os.getenv('BOSS_SPECTRO_DATA_S') + else: + pass + + if exp is not None: + self.exposure = str(exp) + self.iname = self.iname+'_'+str(exp).zfill(8) + if mjd is not None: + self.MJD = mjd + self.iname = self.iname+'_'+str(mjd) + + if not ptt.exists(self.logDir): + os.makedirs(self.logDir, exist_ok = True) + if not ptt.exists(self.controlDir): + os.makedirs(self.controlDir, exist_ok = True) + if not ptt.exists(self.sosdir): + os.makedirs(self.sosdir, exist_ok = True) + + + def __repr__(self): + return self.__str__() + + def __str__(self): + return ("Config:\n" + + "MJD: " + self.MJD + "\n" + + "Exposure " + str(self.exposure) + "\n" + + "sosDir " + self.sosdir + "\n" + + "fitsDir: " + self.fitsDir + "\n" + + "plugDir: " + self.plugDir + "\n" + + "controlDir: " + self.controlDir + "\n" + + "logDir: " + self.logDir + "\n" + + "logLevel: " + str(self.logLevel) + "\n" + + "FPSMode: " + str(self.fps) + "\n" + + "iname: " + self.iname + "\n" + + "glob: " + str(self.globs) + "\n" + + "command: " + self.command + "\n" + + "pollDelay: " + str(self.pollDelay) + "\n" + + "nice: " + str(self.nice) + "\n" + + "plateDb: " + str(self.platedb) + "\n" + + "redo: " + str(self.redo) + "\n" + + "test: " + str(self.test) + "\n" + + "catchup: " + str(self.catchup) + "\n" + + "bookkeep: " + str(self.bookkeep) + "\n" + + "NoSvn: " + str(self.nosvn) + "\n" + + "noDB: " + str(self.nodb) + "\n" + + "no_reject: " + str(self.no_reject) + "\n" + + "clobber_fibermap: " + str(self.clobber_fibermap)+ "\n" + + "sdssv_sn2: " + str(self.sdssv_sn2) + "\n" + + "sn2_15: " + str(self.sn2_15) + "\n" + + "arc2trace: " + str(self.arc2trace) + "\n" + + "forcea2t: " + str(self.forcea2t) + "\n"+ + "pause: " + str(self.pause)); + + +#### +class PollWorker: + """Holds information for each poller""" + + def __init__(self): + self.workerNumber = 0 + self.glob = "*" + self.fileCount = 0 + + + def __repr__(self): + return self.__str__() + + def __str__(self): + return ("PollWorker:\n" + + "workerNumber: " + str(self.workerNumber) + "\n" + + "glob: " + self.glob + "\n" + + "fileCount: " + str(self.fileCount)); + diff --git a/python/boss_drp/spec1d/__init__.py b/python/boss_drp/spec1d/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bin/fluxcorr_prior.py b/python/boss_drp/spec1d/fluxcorr_prior.py similarity index 60% rename from bin/fluxcorr_prior.py rename to python/boss_drp/spec1d/fluxcorr_prior.py index 6c70b6714..324d61b3c 100755 --- a/bin/fluxcorr_prior.py +++ b/python/boss_drp/spec1d/fluxcorr_prior.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Try solving with a prior that fluxcorr = 1 @@ -7,27 +7,19 @@ import sys import os import os.path -import numpy as N -import pylab as P -#import fitsio +import numpy as np +import matplotlib.pyplot as plt from astropy.io import fits as pyf from numpy.polynomial import chebyshev from scipy.sparse.construct import spdiags -#import yanny import os import types -import importlib.machinery -#import importlib as imp - -location = os.getenv('IDLSPEC2D_DIR') -#yanny = imp.load_source('yanny', location+'/bin/yanny.py') -loader = importlib.machinery.SourceFileLoader('yanny', location+'/bin/yanny.py') -yanny = types.ModuleType(loader.name) -loader.exec_module(yanny) +#import importlib.machinery +from pydl.pydlutils import yanny #------------------------------------------------------------------------- def read_plan(planfile): - plan = yanny.read_yanny(planfile) + plan = yanny.read_table_yanny(planfile) plandir = os.path.dirname(planfile) framefiles = dict(b1=list(), b2=list(), r1=list(), r2=list()) @@ -96,8 +88,8 @@ def read_data(indir, framefiles, xythrucorr=False): fx.close() - flux = N.array(flux) - ivar = N.array(ivar) + flux = np.array(flux) + ivar = np.array(ivar) flux[ivar==0] = 0.0 #- cosmetics return flux, ivar, goodframes @@ -108,10 +100,10 @@ def calc_fluxcorr(flux, ivar, prior=0.5): print("Calculating flux corrections") #- The array to fill - fluxcorr = N.ones(flux.shape) + fluxcorr = np.ones(flux.shape) #- Determine range to actually fill - ii = N.where( N.sum(N.sum(ivar, axis=0), axis=0) > 0 )[0] + ii = np.where( np.sum(np.sum(ivar, axis=0), axis=0) > 0 )[0] imin, imax = ii[0], ii[-1]+1 flux = flux[:, :, imin:imax] ivar = ivar[:, :, imin:imax] @@ -119,9 +111,9 @@ def calc_fluxcorr(flux, ivar, prior=0.5): nexp, nspec, npix = flux.shape #- Make coadd; handle cases where sum(weights) = 0 - weighted_flux = N.sum(flux*ivar, axis=0) - sum_weights = N.sum(ivar, axis=0) - coadd = N.zeros(weighted_flux.shape) + weighted_flux = np.sum(flux*ivar, axis=0) + sum_weights = np.sum(ivar, axis=0) + coadd = np.zeros(weighted_flux.shape) ii = (sum_weights > 0) coadd[ii] = weighted_flux[ii] / sum_weights[ii] @@ -129,11 +121,11 @@ def calc_fluxcorr(flux, ivar, prior=0.5): #- FL = diag(coadd).dot(ChebyPolys) #- flux[i] = FL.dot(c[i]) npoly = 4 - xx = N.linspace(-1, 1, npix) + xx = np.linspace(-1, 1, npix) - L = N.zeros( (npix, npoly) ) + L = np.zeros( (npix, npoly) ) for i in range(npoly): - c = N.zeros(npoly) + c = np.zeros(npoly) c[i] = 1.0 L[:,i] = chebyshev.chebval(xx, c) @@ -143,13 +135,13 @@ def calc_fluxcorr(flux, ivar, prior=0.5): for iexp in range(nexp): #- Create diagonal weights matrix, throwing out worst 5% for robustness weights = ivar[iexp, ispec].copy() - if N.sum(weights) == 0: - corr = N.zeros(npoly) + if np.sum(weights) == 0: + corr = np.zeros(npoly) corr[0] = 1.0 c.append(corr) continue - wcut = N.percentile(weights[weights>0], 5) + wcut = np.percentile(weights[weights>0], 5) weights[weights{self.name}