From f52bd83b2b0ec6f8f2b7767d4e2197e710c94682 Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 27 Mar 2024 09:53:27 -0500 Subject: [PATCH 1/6] Fix plate fibermask bug --- pro/spec2d/readplugmap.pro | 29 ++--------------------------- pro/spec2d/spreduce.pro | 3 ++- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/pro/spec2d/readplugmap.pro b/pro/spec2d/readplugmap.pro index 07328448..1526df1a 100644 --- a/pro/spec2d/readplugmap.pro +++ b/pro/spec2d/readplugmap.pro @@ -216,33 +216,6 @@ function readplugmap, plugfile, spectrographid, plugdir=plugdir, savdir=savdir, if not keyword_set(FILE_TEST(spFibermap)) then $ message, 'Missing spFibermap file: '+ spFibermap -; if keyword_set(apotags) then begin -; splog, 'Missing spFibermap file: ', spFibermap -; flags = ' --clobber --SOS --log' -; flags = flags + ' --topdir /data/boss/sos/'+strtrim(mjd,2) -; flags = flags + ' --confSummary '+plugfile -; flags = flags + ' --ccd '+ccd -; flags = flags + ' --mjd '+mjd -; obs = (yanny_par_fc(filehdr, 'observatory'))[0] -; if strmatch(obs, '*LCO*',/fold_case) then $ -; flags = flags + ' --lco' -; -; cmd = "readfibermaps.py "+ flags -; splog,cmd -; spawn, cmd, dat -; if not keyword_set(FILE_TEST(spFibermap)) then $ -; message, 'Missing spFibermap file: '+ spFibermap -; endif else begin -; splog, 'Missing spFibermap file: ', spFibermap -; flags = ' --spplan2d '+spplan -; flags = flags + ' --clobber' -; cmd = "readfibermaps.py "+ flags -; splog,cmd -; spawn, cmd, dat -; splog, dat -; if not keyword_set(FILE_TEST(spFibermap)) then $ -; message, 'Missing spFibermap file: '+ spFibermap -; endelse endif hdr_struct=MRDFITS(spFibermap, 'SUMMARY', sumhdr,/silent, status=st) @@ -419,6 +392,8 @@ function readplugmap, plugfile, spectrographid, plugdir=plugdir, savdir=savdir, nfiber = n_elements(plugmap) endelse if (keyword_set(spectrographid)) then begin + nt=where(fibermask ne -100,ct) + if ct ne 0 then fibermask = fibermask[nt] indx = (spectrographid-1)*nfiber/2 + lindgen(nfiber/2) plugmap = plugmap[indx] fibermask = fibermask[indx] diff --git a/pro/spec2d/spreduce.pro b/pro/spec2d/spreduce.pro index c0409044..5cee110f 100644 --- a/pro/spec2d/spreduce.pro +++ b/pro/spec2d/spreduce.pro @@ -131,6 +131,7 @@ pro spreduce, flatname, arcname, objname, run2d=run2d, plugfile=plugfile, $ + djs_filepath(plugfile, root_dir=plugdir) return endif + fibermaskobj = fibermask endif else begin fps=1 calobssum = readplugmap(plugfile, spectrographid, plugdir=plugdir,$ @@ -393,7 +394,7 @@ airmass = tai2airmass(sxpar(objhdr,'RADEG'),sxpar(objhdr,'DECDEG'), tai=tai, sit OR (*(bestflat.fibermask) AND fibermask_bits('BADFLAT')) $ OR (*(bestarc.fibermask) AND fibermask_bits('BADARC')) endif else begin - fibermask = fibermask $ + fibermask = fibermaskobj $ OR (*(bestflat.fibermask) AND fibermask_bits('BADTRACE')) $ OR (*(bestflat.fibermask) AND fibermask_bits('BADFLAT')) $ OR (*(bestarc.fibermask) AND fibermask_bits('BADARC')) From dcd1d584890fb18c0ed4a5cef0cf5fc9a379113f Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 27 Mar 2024 09:53:47 -0500 Subject: [PATCH 2/6] fix bug with large spcalib_qa files --- pro/spec1d/spcalib_qa.pro | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pro/spec1d/spcalib_qa.pro b/pro/spec1d/spcalib_qa.pro index cc039504..f6a1206d 100644 --- a/pro/spec1d/spcalib_qa.pro +++ b/pro/spec1d/spcalib_qa.pro @@ -192,8 +192,10 @@ pro SpCalib_QA, run2d=run2d, fieldid=fieldid, mjd=mjd, rerun=rerun, nobkup=nobku splog, 'retrying ',out_csv goto, retry endif - ins = Replicate(outs, nrows) + ;ins = Replicate(outs, nrows) temp_in = read_csv(out_csv) + nrows = n_elements(temp_in.FIELD01) + ins = Replicate(outs, nrows) foreach tag, tag_names(ins), i do ins.(i) = temp_in.(i) match = where(ins.field eq fieldid and ins.mjd eq mjd, ct) From 6a392d6d3aa6323cd3ac897ce9e8f0b2f2ba131b Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 27 Mar 2024 09:54:12 -0500 Subject: [PATCH 3/6] fix null values in fits tables --- datamodel/fieldList_dm.par | 38 ++++++++++++++++++------------------- datamodel/spall_dm.par | 32 +++++++++++++++---------------- datamodel/spfibermap_dm.par | 16 ++++++++-------- 3 files changed, 43 insertions(+), 43 deletions(-) diff --git a/datamodel/fieldList_dm.par b/datamodel/fieldList_dm.par index 1ff907f6..73efa977 100644 --- a/datamodel/fieldList_dm.par +++ b/datamodel/fieldList_dm.par @@ -58,13 +58,13 @@ EXT1 PROGRAMNAME A "" "" "Program name within a give EXT1 FIELDQUALITY A "" "" "" EXT1 FIELDSN2 E "" "" "Overall (S/N)^2 for field; min of cameras" EXT1 DEREDSN2 E "" "" "Overall Dereddened (S/N)^2 for field" -EXT1 QSURVEY K "" 0 "1 for an survey quality plate, 0 otherwise" +EXT1 QSURVEY K "" -1 "1 for an survey quality plate, 0 otherwise" EXT1 MJDLIST A "" "" "List of MJD of each included exposures" -EXT1 NEXP K "" 0 "Number of Included Exposures" -EXT1 NEXP_B1 K "" 0 "Number of Included Exposures from b1" -EXT1 NEXP_R1 K "" 0 "Number of Included Exposures from r1" -EXT1 NEXP_B2 K "" 0 "Number of Included Exposures from b2" -EXT1 NEXP_R2 K "" 0 "Number of Included Exposures from r2" +EXT1 NEXP K "" -1 "Number of Included Exposures" +EXT1 NEXP_B1 K "" -1 "Number of Included Exposures from b1" +EXT1 NEXP_R1 K "" -1 "Number of Included Exposures from r1" +EXT1 NEXP_B2 K "" -1 "Number of Included Exposures from b2" +EXT1 NEXP_R2 K "" -1 "Number of Included Exposures from r2" EXT1 EXPT_B1 E "" "" "Total Exposure Time of b1" EXT1 EXPT_R1 E "" "" "Total Exposure Time of r1" EXT1 EXPT_B2 E "" "" "Total Exposure Time of b2" @@ -103,7 +103,7 @@ EXT1 GROFFGAL E "" "" "Spectrophoto offset for G- EXT1 GRRMSGAL E "" "" "Spectrophoto RMS for G-R in galaxies" EXT1 RIOFFGAL E "" "" "Spectrophoto offset for R-I in galaxies" EXT1 RIRMSGAL E "" "" "Spectrophoto RMS for R-I in galaxies" -EXT1 NGUIDE K "" 0 "Number of guider frames during the exposures" +EXT1 NGUIDE K "" -999 "Number of guider frames during the exposures" EXT1 SEEING20 E "arcsec" "" "Mean 20% seeing during exposures" EXT1 SEEING50 E "arcsec" "" "Mean 50% seeing during exposures" EXT1 SEEING80 E "" "" "Mean 80% seeing during exposures" @@ -126,18 +126,18 @@ EXT1 SCHI2MAX E "" "" "Maximum skyChi2 over all e EXT1 FBADPIX E "" "" "Fraction of bad pixels" EXT1 FBADPIX1 E "" "" "Fraction of bad pixels from spectrograph #1" EXT1 FBADPIX2 E "" "" "Fraction of bad pixels from spectrograph #2" -EXT1 N_TOTAL K "" 0 "Number of Sources" -EXT1 N_GALAXY K "" 0 "Number of Galaxies" -EXT1 N_QSO K "" 0 "Number of QSOs" -EXT1 N_STAR K "" 0 "Number of Stars" -EXT1 N_UNKNOWN K "" 0 "Number of Unknown Sources" -EXT1 N_SKY K "" 0 "Number of Skys" -EXT1 N_STD K "" 0 "Number of Standards" -EXT1 N_TARGET_QSO K "" 0 "Number of QSO Targeted" -EXT1 N_TARGET_MAIN K "" 0 "Number of Main Galaxy targeted" -EXT1 N_TARGET_LRG1 K "" 0 "Number of LRG1 targeted" -EXT1 N_TARGET_LRG2 K "" 0 "Number of LRG2 targeted" -EXT1 N_TARGET_ELG K "" 0 "Number of ELG targted" +EXT1 N_TOTAL K "" -999 "Number of Sources" +EXT1 N_GALAXY K "" -999 "Number of Galaxies" +EXT1 N_QSO K "" -999 "Number of QSOs" +EXT1 N_STAR K "" -999 "Number of Stars" +EXT1 N_UNKNOWN K "" -999 "Number of Unknown Sources" +EXT1 N_SKY K "" -999 "Number of Skys" +EXT1 N_STD K "" -999 "Number of Standards" +EXT1 N_TARGET_QSO K "" -999 "Number of QSO Targeted" +EXT1 N_TARGET_MAIN K "" -999 "Number of Main Galaxy targeted" +EXT1 N_TARGET_LRG1 K "" -999 "Number of LRG1 targeted" +EXT1 N_TARGET_LRG2 K "" -999 "Number of LRG2 targeted" +EXT1 N_TARGET_ELG K "" -999 "Number of ELG targted" EXT1 SUCCESS_MAIN E "" "" "Success rate of Main Galaxy targets" EXT1 SUCCESS_LRG1 E "" "" "Success rate of LRG1s" EXT1 SUCCESS_LRG2 E "" "" "Success rate of LRG2s" diff --git a/datamodel/spall_dm.par b/datamodel/spall_dm.par index 9ab90d64..91a5928c 100644 --- a/datamodel/spall_dm.par +++ b/datamodel/spall_dm.par @@ -100,12 +100,12 @@ EXT1 RA_LIST A "deg" "" "List of measured RA per EXT1 DEC_LIST A "deg" "" "List of measured DEC per exposure" EXT1 DELTA_RA_LIST A "deg" "" "List of designed RA offsets per exposure" EXT1 DELTA_DEC_LIST A "deg" "" "List of designed DEC offsets per exposure" -EXT1 FIBER_OFFSET K "" 0 "Flag identifying the fiber was offset by design" +EXT1 FIBER_OFFSET K "" -999 "Flag identifying the fiber was offset by design" EXT1 XFOCAL A "mm" "" "Hole/robot x-axis position in focal plane" EXT1 YFOCAL A "mm" "" "Hole/robot y-axis position in focal plane" EXT1 ZOFFSET E "micron" "" "Backstopping offset distance" EXT1 LAMBDA_EFF E "AA" "" "Wavelength to optimize hole location for" -EXT1 BLUEFIBER K "" 0 "1 if assigned target a blue fiber; 0 otherwise" +EXT1 BLUEFIBER K "" -999 "1 if assigned target a blue fiber; 0 otherwise" EXT1 HEALPIX K "" -999 "healpix pixel number nside=128" EXT1 HEALPIXGRP K "" -999 "Rounded-down integer value of healpix / 1000" EXT1 HEALPIX_PATH A "" "" "Path to spec fits file in SAS healpix structure" @@ -154,7 +154,7 @@ EXT1 CLASS A "" "" "Spectro classification: EXT1 SUBCLASS A "" "" "Spectro sub-classification" EXT1 Z E "" "" "Redshift; incorrect for nonzero ZWARNING flag" EXT1 Z_ERR E "" "" "z error from chi^2 min; negative is invalid fit" -EXT1 ZWARNING K "" 0 "A flag for bad z fits in place of CLASS=UNKNOWN" +EXT1 ZWARNING K "" -999 "A flag for bad z fits in place of CLASS=UNKNOWN" EXT1 RCHI2 E "" "" "Reduced chi^2 for best fit" EXT1 DOF K "" -999 "Degrees of freedom for best fit" EXT1 RCHI2DIFF E "" "" "Diff in reduced chi^2 of 2 best solutions" @@ -236,7 +236,7 @@ EXTLITE PSFMAG 5E "" "" "[u, g, r, i, z] opti EXTLITE FIBER2MAG 5E "" "" "[u, g, r, i, z] optical Fiber2 magnitudes" EXTLITE CATDB_MAG 5E "" "" "[u, g, r, i, z] Raw TargetDB magnitudes" EXTLITE GAIA_G_MAG E "" "" "Gaia G magnitude" -EXTLITE GRI_GAIA_TRANSFORM K "" 0 "provenance of photometry in SDSS-V plate design" +EXTLITE GRI_GAIA_TRANSFORM K "" -999 "provenance of photometry in SDSS-V plate design" EXTLITE BP_MAG E "" "" "Gaia BP magnitude" EXTLITE RP_MAG E "" "" "Gaia RP magnitude" EXTLITE GAIA_ID K "" 0 "Gaia DR3 SourceID" @@ -258,7 +258,7 @@ EXTLITE PMDEC E "mas" "" "Proper motion in Dec EXTLITE PARALLAX E "mas" "" "Parallax" EXTLITE DELTA_RA E "" "" "The maximum fiber offset in any exposure" EXTLITE DELTA_DEC E "" "" "The maximum fiber offset in any exposure" -EXTLITE FIBER_OFFSET K "" 0 "Flag identifying the fiber was offset by design" +EXTLITE FIBER_OFFSET K "" -999 "Flag identifying the fiber was offset by design" EXTLITE LAMBDA_EFF E "AA" "" "Wavelength to optimize hole location for" EXTLITE HEALPIX K "" -999 "healpix pixel number nside=128" EXTLITE HEALPIXGRP K "" -999 "Rounded-down integer value of healpix / 1000" @@ -282,15 +282,15 @@ EXTLITE SEEING80 E "arcsec" "" "Mean 80% seeing duri EXTLITE MOON_DIST E "" "" "Mean Moon-target separation of Coadded Spectra" EXTLITE MOON_PHASE E "" "" "Mean Moon phase of the Coadded Spectra" -EXTLITE ASSIGNED K "" 0 "Whether this fibre was assigned to a target" -EXTLITE ON_TARGET K "" 0 "Whether this fibre is on target" -EXTLITE VALID K "" 0 "alpha and beta angles are valid" -EXTLITE DECOLLIDED K "" 0 "this positioner had to be moved to decollide it" -EXTLITE ANYANDMASK K "" 0 "For each bit, any pixel has bit set in ANDMASK" -EXTLITE ANYORMASK K "" 0 "For each bit, any pixel has bit set in ORMASK" -EXTLITE SPECPRIMARY K "" 0 "Best version of spectrum at this location" -EXTLITE SPECBOSS K "" 0 "Best version of spectrum at this location" -EXTLITE BOSS_SPECOBJ_ID K "" 0 "Same as SPECPRIMARY" +EXTLITE ASSIGNED K "" -999 "Whether this fibre was assigned to a target" +EXTLITE ON_TARGET K "" -999 "Whether this fibre is on target" +EXTLITE VALID K "" -999 "alpha and beta angles are valid" +EXTLITE DECOLLIDED K "" -999 "this positioner had to be moved to decollide it" +EXTLITE ANYANDMASK K "" -999 "For each bit, any pixel has bit set in ANDMASK" +EXTLITE ANYORMASK K "" -999 "For each bit, any pixel has bit set in ORMASK" +EXTLITE SPECPRIMARY K "" -999 "Best version of spectrum at this location" +EXTLITE SPECBOSS K "" -999 "Best version of spectrum at this location" +EXTLITE BOSS_SPECOBJ_ID K "" -999 "Same as SPECPRIMARY" EXTLITE NSPECOBS K "" 0 "Number of spectral observations" EXTLITE SPECTROFLUX 5E "" "" "Spectrum projected onto SDSS-{ugriz} filter" EXTLITE SPECTROFLUX_IVAR 5E "" "" "Inverse variance of SPECTROFLUX" @@ -304,7 +304,7 @@ EXTLITE CLASS A "" "" "Spectro classificati EXTLITE SUBCLASS A "" "" "Spectro sub-classification" EXTLITE Z E "" "" "Redshift; incorrect for nonzero ZWARNING flag" EXTLITE Z_ERR E "" "" "z error from chi^2 min; negative is invalid fit" -EXTLITE ZWARNING K "" 0 "A flag for bad z fits in place of CLASS=UNKNOWN" +EXTLITE ZWARNING K "" -999 "A flag for bad z fits in place of CLASS=UNKNOWN" EXTLITE RCHI2 E "" "" "Reduced chi^2 for best fit" EXTLITE DOF K "" -999 "Degrees of freedom for best fit" EXTLITE RCHI2DIFF E "" "" "Diff in reduced chi^2 of 2 best solutions" @@ -320,7 +320,7 @@ EXTLITE CHI68P E "" "" "68% of abs(chi) of s EXTLITE Z_NOQSO E "" "" "Redshift of the best-fit non-QSO model" EXTLITE Z_ERR_NOQSO E "" "" "Formal one-sigma error on Z_NOQSO" EXTLITE ZNUM_NOQSO K "" 0 "Best fit z/class index excluding QSO; 1-indexed" -EXTLITE ZWARNING_NOQSO K "" 0 "Redshift warning flag for Z_NOQSO" +EXTLITE ZWARNING_NOQSO K "" -999 "Redshift warning flag for Z_NOQSO" EXTLITE CLASS_NOQSO A "" "" "Spectro class of best-fit non-QSO model" EXTLITE SUBCLASS_NOQSO A "" "" "Spectro sub-class of best-fit non-QSO model" EXTLITE RCHI2DIFF_NOQSO E "" "" "Reduced chi^2 diff to next-best non-QSO model" diff --git a/datamodel/spfibermap_dm.par b/datamodel/spfibermap_dm.par index 4e2cdbd0..b2cc12ff 100644 --- a/datamodel/spfibermap_dm.par +++ b/datamodel/spfibermap_dm.par @@ -160,15 +160,15 @@ EXTX FIBERID K "" -999 "Fiber ID for the spectrograph" EXTX CONFFIBERID K "" -999 "Raw Slithead FiberIDs" EXTX FIBERTYPE A "" "" "Type of fibre (APOGEE, BOSS, or Metrology)" EXTX HOLETYPE A "" "" "Plate Hole type" -EXTX ASSIGNED K "" 0 "Whether this fibre was assigned to a target" -EXTX ON_TARGET K "" 0 "Whether this fibre is on target" -EXTX VALID K "" 0 "alpha and beta angles are valid" -EXTX DECOLLIDED K "" 0 "this positioner had to be moved to decollide it" +EXTX ASSIGNED K "" -999 "Whether this fibre was assigned to a target" +EXTX ON_TARGET K "" -999 "Whether this fibre is on target" +EXTX VALID K "" -999 "alpha and beta angles are valid" +EXTX DECOLLIDED K "" -999 "this positioner had to be moved to decollide it" EXTX FIBERMASK K "" -999 "Fiber level bitmasks (maps to SPPIXMASK)" EXTX BADSTDMASK K "" -999 "Standard rejected by magnitude range of design" EXTX THROUGHPUT K "" -999 "Plate Fiber throughput (0-65535, 0=no light)" EXTX SCI_EXPTIME D "" "" "Rescaled Science Exposure Time for offset plate" -EXTX BLUEFIBER K "" 0 "BOSS to assign this target a 'blue' fiber" +EXTX BLUEFIBER K "" -999 "BOSS to assign this target a 'blue' fiber" EXTX ALPHA D "" "" "The alpha angles of the positioner arms." EXTX BETA D "" "" "The beta angles of the positioner arms." EXTX XWOK D "" "" "The X value in the FPS WOK coordinates" @@ -187,7 +187,7 @@ EXTX RA D "" "" "Calculated on sky fiber RA including delta_RA" EXTX DEC D "" "" "Calculated on sky fiber Dec including delta_DEC" EXTX DELTA_RA D "" "" "The amount in RA this fiber has been offset" EXTX DELTA_DEC D "" "" "The amount in Dec this fiber has been offset" -EXTX FIBER_OFFSET K "" 0 "Flag identifying the fiber was offset by design" +EXTX FIBER_OFFSET K "" -999 "Flag identifying the fiber was offset by design" EXTX LAMBDA_EFF D "" "" "Wavelength used for coordinate transformations" EXTX SPECTROGRAPHID K "" "-1" "Spectrograph to which the fibre is assigned." # (SDSS-V: 1 = BOSS, 2 = APOGEE) or (SDSS-IV: 1=BOSS SP1, 2=BOSS SP2)" EXTX OPTICAL_PROV A "" "" "The source of the CATDB_MAG optical magnitudes" @@ -195,7 +195,7 @@ EXTX MAG 5D "" "" "[u, g, r, i, z] optical magnitudes" EXTX PSFMAG 5D "" "" "[u, g, r, i, z] optical PSF magnitudes" EXTX FIBER2MAG 5D "" "" "[u, g, r, i, z] optical Fiber2 magnitudes" EXTX CATDB_MAG 5D "" "" "[u, g, r, i, z] Raw TargetDB magnitudes" -EXTX V05_REV_MAG K "" 0 "1: revised with RevisedMagnitude Table, 0: raw" +EXTX V05_REV_MAG K "" -999 "1: revised with RevisedMagnitude Table, 0: raw" EXTX GAIA_G_MAG D "" "" "Gaia G magnitude" EXTX BP_MAG D "" "" "Gaia BP magnitude" EXTX RP_MAG D "" "" "Gaia RP magnitude" @@ -212,7 +212,7 @@ EXTX EBV_RJCE D "" "" "RJCE dust extinction" EXTX EBV_3D D "" "" "Merged 3D dust extinction" EXTX EBV_3DSRC A "" "" "Source of Merged 3D dust extinction" -EXTX GRI_GAIA_TRANSFORM K "" 0 "provenance of photometry in SDSS-V plate design" +EXTX GRI_GAIA_TRANSFORM K "" -999 "provenance of photometry in SDSS-V plate design" EXTX CALIBFLUX 5D "" "" "SDSS band [u,g,r,i,z] target calibration flux" EXTX CALIBFLUX_IVAR 5D "" "" "target calibration flux inverse variance" EXTX CALIB_STATUS 5K "" -999 "target calibration status flag" From 3c7621e34105f7f42d2dd0068139e0fc317b7ea5 Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 27 Mar 2024 09:54:59 -0500 Subject: [PATCH 4/6] fix bug for unset fiber_offset --- bin/readfibermaps.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/bin/readfibermaps.py b/bin/readfibermaps.py index fda391b5..47711c01 100755 --- a/bin/readfibermaps.py +++ b/bin/readfibermaps.py @@ -174,6 +174,13 @@ def readfibermaps(spplan2d=None, topdir=None, clobber=False, SOS=False, no_db=Fa splog.open(logfile = ptt.join(SOS_opts['log_dir'], spFibermap.replace('.fits','.log')), append= (not clobber)) splog.info('readfibermaps started at '+ time.ctime()) + try: + with fits.open(ptt.join(topdir, spFibermap), mode='update', checksum=True, output_verify="fix") as hdul: + hdul.verify('fix') + hdul.flush() + clobber = False + except: + clobber = True if not clobber: if (ptt.exists(ptt.join(topdir, spFibermap))): try: @@ -325,7 +332,7 @@ def buildfibermap(fibermap_file, run2d, obs, field, mjd, exptime=None, indir=Non fibermap = readPlateplugMap(fibermap_file, fibermap, mjd, SOS=SOS, fast=fast, exptime=exptime, plates=plates, legacy=legacy, no_db=no_db, indir=indir, release=release, no_remote=no_remote, dr19=dr19) - + fibermap['fiber_offset'] = 0 elif fps: fibermap = read_table_yanny(fibermap_file, 'FIBERMAP') fibermap.convert_bytestring_to_unicode() @@ -339,6 +346,7 @@ def buildfibermap(fibermap_file, run2d, obs, field, mjd, exptime=None, indir=Non dcol[dcol.data == -999] = np.NaN fibermap = readFPSconfSummary(fibermap, mjd, sos=SOS, no_db = no_db, fast=fast, release=release, no_remote=no_remote, dr19=dr19) + fibermap = flag_offset_fibers(fibermap) fibermap['SCI_EXPTIME'] = np.NaN hdr['CARTRIDGEID'] = 'FPS-S' if hdr['observatory'] == 'LCO' else 'FPS-N' @@ -351,11 +359,11 @@ def buildfibermap(fibermap_file, run2d, obs, field, mjd, exptime=None, indir=Non def flag_offset_fibers(fibermap): splog.info('Flagging offset Fibers') if not bool(int(fibermap.meta['is_dithered'])): - offsets = np.zeros(len(fibermap), dtype=bool) - fibermap.add_column(offsets,name='fiber_offset') + #offsets = np.zeros(len(fibermap), dtype=bool) + #fibermap.add_column(offsets,name='fiber_offset') foff = fibermap['fiber_offset'] - indx = np.where(np.logical_or((fibermap['DELTA_RA'].data != 0), (fibermap['DELTA_DEC'].data != 0)))[0] + indx = np.where(np.logical_or((fibermap['delta_ra'].data != 0), (fibermap['delta_dec'].data != 0)))[0] foff[indx] = 1 return(fibermap) @@ -1128,6 +1136,7 @@ def readPlateplugMap(plugfile, fibermap, mjd, SOS=False, for i, row in enumerate(fibermap): if 'bhm_spiders_clusters-efeds' in row['FIRSTCARTON']: mag.data[i,1:] = mag.data[i,1:] - psffibercor + fibermap.add_column(Column('Plates', name = 'cadence')) else: programname = None From fba9a7382ce2e812ac60334b846f952e659cb987 Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 27 Mar 2024 09:55:33 -0500 Subject: [PATCH 5/6] fix bug with spAll_lite Delta_RA, Delta_DEC --- bin/fieldmerge.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/bin/fieldmerge.py b/bin/fieldmerge.py index f0220971..b0f7a9ac 100755 --- a/bin/fieldmerge.py +++ b/bin/fieldmerge.py @@ -793,35 +793,45 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), if lite is True: spAll_lite = spAll.copy() + spAll_lite.rename_column('DELTA_RA_LIST', 'DELTA_RA') + spAll_lite.rename_column('DELTA_DEC_LIST', 'DELTA_DEC') for i in range(len(spAll)): with warnings.catch_warnings(): warnings.filterwarnings(action='ignore', message='Mean of empty slice') try: - spAll_lite[i]['ASSIGNED'] = str(min(np.array(spAll[i]['ASSIGNED'].split()).astype(int))) + spAll_lite[i]['ASSIGNED'] = str(min(np.array(spAll_lite[i]['ASSIGNED'].split()).astype(int))) except: spAll_lite[i]['ASSIGNED'] = '0' try: - spAll_lite[i]['ON_TARGET'] = str(min(np.array(spAll[i]['ON_TARGET'].split()).astype(int))) + spAll_lite[i]['ON_TARGET'] = str(min(np.array(spAll_lite[i]['ON_TARGET'].split()).astype(int))) except: spAll_lite[i]['ON_TARGET'] = '0' try: - spAll_lite[i]['VALID'] = str(min(np.array(spAll[i]['VALID'].split()).astype(int))) + spAll_lite[i]['VALID'] = str(min(np.array(spAll_lite[i]['VALID'].split()).astype(int))) except: spAll_lite[i]['VALID'] = '0' try: - spAll_lite[i]['DECOLLIDED'] = str(min(np.array(spAll[i]['DECOLLIDED'].split()).astype(int))) + spAll_lite[i]['DECOLLIDED'] = str(min(np.array(spAll_lite[i]['DECOLLIDED'].split()).astype(int))) except: spAll_lite[i]['DECOLLIDED'] = '0' try: - spAll_lite[i]['MOON_DIST'] = str(np.nanmean(np.array(spAll[i]['MOON_DIST'].split()).astype(float))) + spAll_lite[i]['MOON_DIST'] = str(np.nanmean(np.array(spAll_lite[i]['MOON_DIST'].split()).astype(float))) except: spAll_lite[i]['MOON_DIST'] = 'nan' try: - spAll_lite[i]['MOON_PHASE'] = str(np.nanmean(np.array(spAll[i]['MOON_PHASE'].split()).astype(float))) + spAll_lite[i]['MOON_PHASE'] = str(np.nanmean(np.array(spAll_lite[i]['MOON_PHASE'].split()).astype(float))) except: spAll_lite[i]['MOON_PHASE'] = 'nan' try: - spAll_lite[i]['CARTON_TO_TARGET_PK'] = str(int(np.array(spAll[i]['CARTON_TO_TARGET_PK'].split())[0])) + spAll_lite[i]['DELTA_RA'] = str(np.nanmean(np.array(spAll_lite[i]['DELTA_RA'].split()).astype(float))) + except: + spAll_lite[i]['DELTA_RA'] = 'nan' + try: + spAll_lite[i]['DELTA_DEC'] = str(np.nanmean(np.array(spAll_lite[i]['DELTA_DEC'].split()).astype(float))) + except: + spAll_lite[i]['DELTA_DEC'] = 'nan' + try: + spAll_lite[i]['CARTON_TO_TARGET_PK'] = str(int(np.array(spAll_lite[i]['CARTON_TO_TARGET_PK'].split())[0])) except: spAll_lite[i]['CARTON_TO_TARGET_PK'] = '0' @@ -832,7 +842,7 @@ def fieldmerge(run2d=getenv('RUN2D'), indir= getenv('BOSS_SPECTRO_REDUX'), for col in ['CARTON_TO_TARGET_PK']: spAll_lite[col] = spAll_lite[col].astype(int) - for col in ['MOON_DIST','MOON_PHASE']: + for col in ['MOON_DIST','MOON_PHASE','DELTA_RA','DELTA_DEC']: spAll_lite[col] = spAll_lite[col].astype(float) else: spAll_lite = None From 68d5e0ff591139398a47108bf947962ea5d8d685 Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 27 Mar 2024 09:56:10 -0500 Subject: [PATCH 6/6] bug fixes --- bin/dailylogger.py | 6 +++--- bin/spSpec_reformat.py | 4 +++- bin/spplan_target.py | 14 ++++++++++---- bin/uubatchpbs.py | 2 ++ bin/uurundaily.py | 13 +++++++++---- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/bin/dailylogger.py b/bin/dailylogger.py index b9bc8c6a..0c506afb 100644 --- a/bin/dailylogger.py +++ b/bin/dailylogger.py @@ -83,10 +83,10 @@ def send_email(subject, email_file, attachment, logger, content=None, from_domai attachment = np.atleast_1d(attachment) msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' for fa in attachment: - if ptt.exists(attachment): - with open(attachment, 'rb') as fp: + if ptt.exists(fa): + with open(fa, 'rb') as fp: logdata = fp.read() - msg.add_attachment(logdata, maintype='text', subtype='plain', filename=ptt.basename(attachment)) + msg.add_attachment(logdata, maintype='text', subtype='plain', filename=ptt.basename(fa)) s = smtplib.SMTP('localhost') s.send_message(msg) s.quit() diff --git a/bin/spSpec_reformat.py b/bin/spSpec_reformat.py index 9c101c97..1fee519d 100755 --- a/bin/spSpec_reformat.py +++ b/bin/spSpec_reformat.py @@ -18,6 +18,8 @@ import warnings from PIL import Image, PngImagePlugin +mpl.use('Agg') + import time import logging from splog import Splog @@ -698,7 +700,7 @@ def SDSS_specplot(basedir, Coadd_Table, spAll, catalogID, files = Table(), xra=[ parser.add_argument('--plot', '-p', action='store_true', help='Create spec plots') parser.add_argument('--epoch', '-e', action='store_true', help='Run for epoch Coadds') parser.add_argument('--lsdr10', action='store_true', help='Include Legacy Survey DR10 links on HTML') - parser.add_argument('--allsky', action='store_true', help='Include Legacy Survey DR10 links on HTML') + parser.add_argument('--allsky', action='store_true', help='Reformat for Allsky Custom Coadd') args = parser.parse_args() spSpec_reformat(args.topdir, args.run2d, args.run1d, args.field, args.mjd, diff --git a/bin/spplan_target.py b/bin/spplan_target.py index e15d5ee3..a890b233 100644 --- a/bin/spplan_target.py +++ b/bin/spplan_target.py @@ -189,6 +189,12 @@ def build_plan(spAll, use_catid=False, coadd_mjdstart = None): plan = vstack([plan,new]) #if len(plan)> 400: break + if len(plan) == 0: + splog.info('No Valid TARGETS') + plan = None + else: + plan['FMJD_LIST']=plan['FMJD_LIST'].astype(str) + plan = mod_epoch(plan) plan['FMJD_LIST']=plan['FMJD_LIST'].astype(str) plan = mod_epoch(plan) @@ -279,10 +285,10 @@ def CustomCoadd(name, topdir, run2d, run1d, cartons=None, catalogids=None, obs=N if mjd is not None: mjd_start = mjd_end = mjd plan = build_plan(spAll, use_catid=use_catid, coadd_mjdstart = coadd_mjdstart) - write_plan(name, plan, topdir, run2d, run1d, mjdstart, mjdend,clobber=clobber, - rerun1d=rerun1d, use_catid=use_catid, obs=obs, - coadd_mjdstart=coadd_mjdstart) - + if plan is not None: + write_plan(name, plan, topdir, run2d, run1d, mjdstart, mjdend,clobber=clobber, + rerun1d=rerun1d, use_catid=use_catid, obs=obs, + coadd_mjdstart=coadd_mjdstart) splog.close() return diff --git a/bin/uubatchpbs.py b/bin/uubatchpbs.py index ccd1e3b2..7e41c4ac 100755 --- a/bin/uubatchpbs.py +++ b/bin/uubatchpbs.py @@ -126,6 +126,8 @@ def build_cmd(topdir=None,run2d=None,run1d=None,idlutils_1d=None, spSpecRef_key = "" # " --lsdr10" flist_key = "" fmerge_key = " --include_bad" + if run2d.lower() == 'master': + spSpecRef_key =" --lsdr10" if epoch: spSpecRef_key = spSpecRef_key + " --epoch" diff --git a/bin/uurundaily.py b/bin/uurundaily.py index bf94ae3e..4bb9a16d 100755 --- a/bin/uurundaily.py +++ b/bin/uurundaily.py @@ -133,8 +133,9 @@ def get_key(fp): if ptt.isdir(path.replace('?????', str(lastmjd))): mjd.append(lastmjd) else: + logger.info('skipping '+str(lastmjd)+' for '+mod+' obs='+obs) send_email('skipping '+str(lastmjd)+' for '+mod+' obs='+obs, - ptt.join(getenv('HOME'), 'daily', 'etc','emails'), None, from_domain=from_domain) + ptt.join(getenv('HOME'), 'daily', 'etc','emails'), None, logger, from_domain=from_domain) #email(subj = 'skipping '+str(lastmjd)+' for '+mod+' obs='+obs) lastmjd = lastmjd - 1 if len(mjd) == 0: @@ -191,8 +192,8 @@ def dailysummary(queue1, obs, run2d, run1d, module, logger, epoch = False, build cores = 2 fmerge_cmd = ptt.join(getenv('HOME'),'daily','cmd','run_pyfieldmerge_epoch_{run2d}') - queue2.create(label = f"BOSS_Summary_{'-'.join(obs)}_{run2d}", nodes = 1, ppn = cores, walltime = "24:00:00", - alloc='sdss-np', qos = 'sdss', partition = 'sdss-np', mem_per_cpu = 64000, shared = True) + queue2.create(label = f"BOSS_Summary_{'-'.join(obs)}_{run2d}", nodes = 1, ppn = 64, walltime = "24:00:00", + alloc='sdss-np', qos = 'sdss', partition = 'sdss-np', mem_per_cpu = 7500, shared = False) if not ptt.exists(fmerge_cmd): makedirs(ptt.join(getenv('HOME'),'daily','cmd'),exist_ok=True) @@ -234,7 +235,11 @@ def dailysummary(queue1, obs, run2d, run1d, module, logger, epoch = False, build if percomp1 == 100 and percomppost == 100: running=False logger.info('exiting code') - return('Complete '+run2d +' MJD='+str(jdate) +' OBS='+','.join(obs),[fmerge_log+".o.log", fmerge_log+".e.log"] ) + if build: + attachments = [fmerge_log+".o.log", fmerge_log+".e.log"] + else: + attachments = None + return('Complete '+run2d +' MJD='+str(jdate) +' OBS='+','.join(obs), attachments)#[fmerge_log+".o.log", fmerge_log+".e.log"] ) time.sleep(pause) return (None, None)