diff --git a/python/sdss_access/path/path.py b/python/sdss_access/path/path.py index 9b951d8..3a60cc7 100644 --- a/python/sdss_access/path/path.py +++ b/python/sdss_access/path/path.py @@ -325,20 +325,32 @@ def extract(self, name, example): # handle special functions; perform a drop in replacement if re.match('@spectrodir[|]', template): template = re.sub('@spectrodir[|]', os.environ['BOSS_SPECTRO_REDUX'], template) - elif re.search('@platedir[|]', template): + if re.search('@platedir[|]', template): template = re.sub('@platedir[|]', r'(.*)/{plateid:0>6}', template) - elif re.search('@definitiondir[|]', template): + if re.search('@definitiondir[|]', template): template = re.sub('@definitiondir[|]', '{designid:0>6}', template) - elif re.search('@apgprefix[|]', template): + if re.search('@apgprefix[|]', template): template = re.sub('@apgprefix[|]', '{prefix}', template) - elif re.search('@healpixgrp[|]', template): + if re.search('@healpixgrp[|]', template): template = re.sub('@healpixgrp[|]', '{healpixgrp}', template) - elif re.search('@configgrp[|]', template): + if re.search('@configgrp[|]', template): template = re.sub('@configgrp[|]', '{configgrp}', template) - elif re.search('@isplate[|]', template): + if re.search('@isplate[|]', template): template = re.sub('@isplate[|]', '{isplate}', template) - elif re.search('@pad_fieldid[|]', template): + if re.search('@pad_fieldid[|]', template): template = re.sub('@pad_fieldid[|]', '{fieldid}', template) + if re.search('@spcoaddfolder[|]',template): + template = re.sub('@spcoaddfolder[|]', '{spcoaddfolder}', template) + if re.search('@sptypefolder[|]',template): + template = re.sub('@sptypefolder[|]', '{sptypefolder}', template) + if re.search('@epochflag[|]', template): + template = re.sub('@epochflag[|]', '{epochflag}', template) + if re.search('@spcoaddobs[|]',template): + template = re.sub('@spcoaddobs[|]', '{obs}', template) + if re.search('@spcoaddgrp[|]',template): + template = re.sub('@spcoaddgrp[|]', '{spcoaddgrp}', template) + if re.search('@fieldgrp[|]',template): + template = re.sub('@fieldgrp[|]', '{fieldgrp}', template) if re.search('@plateid6[|]', template): template = re.sub('@plateid6[|]', '{plateid:0>6}', template) if re.search('@component_default[|]', template): @@ -347,6 +359,8 @@ def extract(self, name, example): template = re.sub('@cat_id_groups[|]', '{cat_id_groups}', template) if re.search('@sdss_id_groups[|]', template): template = re.sub('@sdss_id_groups[|]', '{sdss_id_groups}', template) + if re.search('@tilegrp[|]', template): + template = re.sub('@tilegrp[|]', '{tilegrp}', template) # check if template has any brackets haskwargs = re.search('[{}]', template) @@ -386,12 +400,28 @@ def extract(self, name, example): drval = re.match('^DR[1-9][0-9]', value).group(0) otherval = value.split(drval)[-1] pdict = {keys[0]: drval, keys[1]: otherval} + elif keys == ['fieldid','isplate']: + # for {fieldid}{isplate} as isplate is calculated automatically + if value.endswith('p') and value[:-1].isdigit(): + value = value[:-1] + pdict = {keys[0]:value} + elif keys == ['run2d','epochflag']: + # for {run2d}{epochflag} as epochflag is calculated automatically + pdict = {keys[0]:value.replace('-epoch','')} + elif keys == ['coadd', 'obs']: + value = value.split('_') + if len(value) == 1: + value.append('') + pdict = {keys[0]: value[0], keys[1]: value[1]} elif keys[0] in ['rc', 'br', 'filter', 'camrow']: # for {camrow}{camcol}, {filter}{camcol}, {br}{id}, etc pdict = {keys[0]: value[0], keys[1]: value[1:]} else: raise ValueError('This case has not yet been accounted for.') path_dict.update(pdict) + elif keys[0] in ['sptypefolder','fieldgrp','spcoaddfolder','spcoaddgrp']: + # supress the keys since they are automatically calculated + continue else: path_dict[keys[0]] = value return path_dict @@ -678,7 +708,7 @@ def full(self, filetype, **kwargs): if not skip_tag_check: template = re.sub(r'tags/(v?[0-9._]+)', r'\1', template, count=1) - return self._check_compression(template) + return self._check_compression(os.path.normpath(template)) @staticmethod def check_modules(template, permanent=None): @@ -1531,8 +1561,184 @@ def pad_fieldid(self, filetype, **kwargs): return str(fieldid) if fieldid.isnumeric(): return str(fieldid).zfill(6) - else: - return fieldid + return fieldid + + def spcoaddfolder(self, filetype, **kwargs): + ''' Returns the reorganized subfolder structure for the BOSS idlspec2d run2d version + + Parameters + --------- + filetype : str + File type parameter + run2d : str + BOSS idlspec2d run2d version + coadd : str + Name of the custom coadd schema + Returns + ------- + sptypefolder : str + ''' + + run2d = kwargs.get('run2d', None) + coaddname = kwargs.get('coadd', None) + + if (not run2d): + return '' + if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or + ('v6_0' in run2d) or ('v6_1' in run2d)): + return '' + if filetype.lower() in ['spall_coadd','spall-lite_coadd','spallline_coadd']: + return 'summary' + if filetype.lower() in ['speclite_coadd','specfull_coadd', + 'spallfield_coadd','spalllinefield_coadd']: + return coaddname + return 'fields' + + + def spcoaddgrp(self, filetype, **kwargs): + ''' Returns the coadd group (field group analog) subfolder structure for the BOSS idlspec2d run2d version + + Parameters + --------- + filetype : str + File type parameter + run2d : str + BOSS idlspec2d run2d version + coadd : str + Name of the custom coadd schema + Returns + ------- + spcoaddgrp : str + ''' + run2d = kwargs.get('run2d', None) + coaddname = kwargs.get('coadd', None) + + if (not run2d): + return '' + if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or + ('v6_0' in run2d) or ('v6_1' in run2d)): + return '' + return coaddname + + def sptypefolder(self, filetype, **kwargs): + ''' Returns the reorganized subfolder structure for the BOSS idlspec2d run2d version + + Parameters + --------- + filetype : str + File type parameter + run2d : str + BOSS idlspec2d run2d version + Returns + ------- + sptypefolder : str + ''' + + run2d = kwargs.get('run2d', None) + + if (not run2d) or ('v5' in run2d) or (str(run2d) in ['26','103','104']): + return '' + if ('v6_0' in run2d) or ('v6_1' in run2d): + if filetype.lower() in ['speclite_epoch','specfull_epoch', + 'spallfield_epoch','spalllinefield_epoch']: + return 'epoch/spectra' + if 'epoch' in filetype.lower(): + return 'epoch' + return '' + if filetype.lower() in ['fieldlist_epoch','spall_epoch', + 'spall-lite_epoch','spallline_epoch']: + return 'summary/epoch' + if filetype.lower() in ['speclite_epoch','specfull_epoch', + 'spallfield_epoch','spalllinefield_epoch']: + return 'spectra/epoch' + if filetype.lower() in ['conflist','fieldlist','spall', + 'spall-lite','spallline']: + return 'summary/daily' + if filetype.lower() in ['speclite','specfull', + 'spallfield','spalllinefield']: + return 'daily' + return 'fields' + + def spcoaddobs(self, filetype, **kwargs): + ''' Returns the formatted observatory flag for custom coadds for the BOSS idlspec2d + + Parameters + ---------- + filetype : str + File type parameter + run2d : str + BOSS idlspec2d run2d version + obs : str + Observatory of observations: LCO, APO, ''(for merged) + + Returns + ------- + obs : str + ''' + + obs = kwargs.get('obs', None) + run2d = kwargs.get('run2d', None) + + if not obs or obs == '': + return '' + if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or + (('v6_0' in run2d) or ('v6_1' in run2d)) and obs.lower()) == 'apo': + return '' + if obs == '*': + return obs + return '_{}'.format(obs.lower()) + + def epochflag(self, filetype, **kwargs): + ''' Returns the flag for epoch coadds for the BOSS idlspec2d + + Parameters + ---------- + filetype : str + File type parameter + run2d : str + BOSS idlspec2d run2d version + + Returns + ------- + epochflag : str + ''' + + run2d = kwargs.get('run2d', None) + if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or + ('v6_0' in run2d) or ('v6_1' in run2d)): + return '' + return '-epoch' + + def fieldgrp(self, filetype, **kwargs): + ''' Returns the fieldid group for the BOSS idlspec2d run2d version + + Parameters + --------- + filetype : str + File type parameter + run2d : str + BOSS idlspec2d run2d version + fieldid : str or int + Field ID number. Will be converted to str internally. + + Returns + ------- + fieldgrp : str + ''' + + fieldid = kwargs.get('fieldid', None) + run2d = kwargs.get('run2d', None) + + if (not fieldid): + return '' + + if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or + ('v6_0' in run2d) or ('v6_1' in run2d)): + return '' + fieldid = str(fieldid) + if fieldid.isnumeric(): + return '{:0>3d}XXX'.format(int(fieldid) // 1000) + return fieldid def tilegrp(self, filetype, **kwargs): ''' Returns LVM tile id group subdirectory diff --git a/tests/conftest.py b/tests/conftest.py index 9eb1585..442df87 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -80,7 +80,7 @@ def expdata(datapath): # full source file location source = os.path.join(base, sas_module, location) # full final file location - destination = os.path.join(os.getenv('SAS_BASE_DIR'), sas_module, location) + destination = os.path.normpath(os.path.join(os.getenv('SAS_BASE_DIR'), sas_module, location)) # combined dict result = {'name': datapath['name'], 'params': datapath['params'], 'base': base, 'sas_module': sas_module, 'location': location, 'source': source, diff --git a/tests/path/test_sdss5.py b/tests/path/test_sdss5.py index 9b56c87..baf05bd 100644 --- a/tests/path/test_sdss5.py +++ b/tests/path/test_sdss5.py @@ -59,15 +59,28 @@ def test_apogee_paths(self, path, name, special, keys, exp): 'v6_0_4/1234p/spFrame-b1-00005432.fits.gz'), ('spField', '@pad_fieldid', {'run2d': 'v6_1_1', 'mjd': '59630', 'fieldid': '*'}, 'v6_1_1/*/spField-*-59630.fits'), + ('spCOADDS','@spcoaddfolder',{'run2d':'v6_2_0','survey':'BHM','coadd':'allepoch','phase':'SDSSV'}, + 'v6_2_0/fields/SDSSV_BHM_COADDS.par'), + ('spField','@sptypefolder',{'run2d':'v6_2_0','mjd': '59630', 'fieldid': '123456'}, + 'v6_2_0/fields/123XXX/123456/spField-123456-59630.fits'), + ('spField','@fieldgrp',{'run2d':'v6_2_0','mjd': '59630', 'fieldid': '123456'}, + 'v6_2_0/fields/123XXX/123456/spField-123456-59630.fits'), + ('spFullsky','@spcoaddobs',{'run2d':'v6_2_0','mjd': '59630','coadd':'allepoch','obs':'apo'}, + 'v6_2_0/fields/allepoch/allepoch_apo/spFullsky-allepoch_apo-59630.fits'), + ('spAll_epoch','@epochflag',{'run2d':'v6_2_0'}, + 'v6_2_0/summary/epoch/spAll-v6_2_0-epoch.fits'), + ('spAll_coadd','@spcoaddgrp',{'run2d':'v6_2_0','coadd':'allepoch'}, + 'v6_2_0/summary/allepoch/spAll-v6_2_0-allepoch.fits'), ('lvm_frame', '@tilegrp', {'drpver': 'master', 'mjd': 60235, 'tileid': 1055360, 'kind': 'CFrame', 'expnum': 6817}, '1055XX/1055360/60235/lvmCFrame-00006817.fits')], ids=['configgrp', 'apgprefix-apo', 'apgprefix-lco', 'apgprefix-ins', - 'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', 'frame-nopadp', - 'pad_fieldid-*', 'lvm-tileid']) + 'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', + 'frame-nopadp', 'pad_fieldid-*','spcoaddfolder', 'sptypefolder', + 'fieldgrp','spcoaddobs','epochflag','spcoaddgrp','lvm-tileid']) def test_special_function(self, path, name, special, keys, exp): assert special in path.templates[name] - full = path.full(name, **keys) + full = os.path.normpath(path.full(name, **keys)) assert exp in full @pytest.mark.parametrize('name, keys', [('specLite', ['fieldid', 'catalogid', 'run2d', 'mjd']),