Merge branch 'main' into newrel

sdss · Nov 2, 2023 · ef2dc3a · ef2dc3a
2 parents 4addd40 + 13320c7
commit ef2dc3a
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 5 deletions.
diff --git a/python/sdss_access/path/path.py b/python/sdss_access/path/path.py
@@ -345,6 +345,8 @@ def extract(self, name, example):
             template = re.sub('@component_default[|]', '{component_default}', template)
         if re.search('@cat_id_groups[|]', template):
             template = re.sub('@cat_id_groups[|]', '{cat_id_groups}', template)
+        if re.search('@sdss_id_groups[|]', template):
+            template = re.sub('@sdss_id_groups[|]', '{sdss_id_groups}', template)
 
         # check if template has any brackets
         haskwargs = re.search('[{}]', template)
@@ -1257,6 +1259,29 @@ def cat_id_groups(self, filetype, **kwargs):
             cat_id = int(kwargs['cat_id'])
         return f"{(cat_id // k) % k:0>2.0f}/{cat_id % k:0>2.0f}"
 
+    def sdss_id_groups(self, filetype, **kwargs):
+        '''
+        Return a folder structure to group data together based on their SDSS
+        identifier so that we don't have too many files in any one folder.
+
+        Parameters
+        ----------
+        filetype : str
+            File type parameter.
+        sdss_id : int or str
+            SDSS-V identifier
+
+        Returns
+        -------
+        sdssid_groups : str
+            A set of folders.
+        '''        
+        # with k = 100 then even with 10 M sources, each folder will have ~1,000 files
+        k = 100
+        sdss_id = int(kwargs["sdss_id"])
+        return f"{(sdss_id // k) % k:0>2.0f}/{sdss_id % k:0>2.0f}"
+
+
     def component_default(self, filetype, **kwargs):
         ''' Return the component name, if given.
 
@@ -1407,7 +1432,7 @@ def isplate(self, filetype, **kwargs):
         Parameters
         ---------
         filetype : str
-            File type paramter
+            File type parameter
         run2d : str
             BOSS idlspec2d run2d version
 
@@ -1430,7 +1455,7 @@ def pad_fieldid(self, filetype, **kwargs):
         Parameters
         ---------
         filetype : str
-            File type paramter
+            File type parameter
         run2d : str
             BOSS idlspec2d run2d version
         fieldid : str or int
@@ -1447,9 +1472,13 @@ def pad_fieldid(self, filetype, **kwargs):
 
         if (not run2d) & (not fieldid):
             return ''
+        fieldid = str(fieldid)
         if run2d in ['v6_0_1','v6_0_2', 'v6_0_3', 'v6_0_4']:
             return str(fieldid)
-        return str(fieldid).zfill(6)
+        if fieldid.isnumeric():
+            return str(fieldid).zfill(6)
+        else:
+            return fieldid
 
 
 class AccessError(Exception):

diff --git a/tests/path/test_sdss5.py b/tests/path/test_sdss5.py
@@ -56,9 +56,11 @@ def test_apogee_paths(self, path, name, special, keys, exp):
                               ('spFrame', '@pad_fieldid', {'run2d': 'v6_0_8', 'br': 'b', 'id': '1', 'frame': '5432', 'fieldid':'1234'},
                                 'v6_0_8/001234/spFrame-b1-00005432.fits.gz'),
                               ('spFrame', '@pad_fieldid', {'run2d': 'v6_0_4', 'br': 'b', 'id': '1', 'frame': '5432', 'fieldid':'1234'},
-                                'v6_0_4/1234p/spFrame-b1-00005432.fits.gz')],
+                                'v6_0_4/1234p/spFrame-b1-00005432.fits.gz'),
+                              ('spField', '@pad_fieldid', {'run2d': 'v6_1_1', 'mjd': '59630', 'fieldid': '*'},
+                                'v6_1_1/*/spField-*-59630.fits')],
                              ids=['configgrp', 'apgprefix-apo', 'apgprefix-lco', 'apgprefix-ins',
-                                  'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', 'frame-nopadp'])
+                                  'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', 'frame-nopadp', 'pad_fieldid-*'])
     def test_special_function(self, path, name, special, keys, exp):
         assert special in path.templates[name]
         full = path.full(name, **keys)