Merge branch 'main' into albireox/coord-is-sky

CHANGELOG.md

@@ -2,6 +2,24 @@
 
 ## Next version
 
+# 1.3.14 - August 12 2024
+
+### 🚀 New
+
+* add cartons to target_selection plan 1.2.15
+
+## 1.3.13 - August 10, 2024
+
+### 🚀 New
+
+* add cartons to target_selection plan 1.2.14
+
+## 1.3.12 - August 1, 2024
+
+### 🚀 New
+
+* add cartons to target_selection plan 1.2.13
+
 ## 1.3.11 - August 1, 2024
 
 ### 🚀 New

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sdss-target-selection"
-version = "1.3.12.a0"
+version = "1.3.15.a0"
 description = "Code to perform target selection for BHM/MWM using catalogdb"
 authors = ["José Sánchez-Gallego <[email protected]>"]
 license = "BSD 3-Clause"

python/target_selection/cartons/bhm_galaxies.py

@@ -6,6 +6,8 @@
 # @Filename: bhm_galaxies.py
 # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause)
 
+import numpy
+import pandas
 import peewee
 
 from sdssdb.peewee.sdss5db.catalogdb import (
@@ -65,8 +67,8 @@ def build_query(self, version_id, query_region=None):
         instrument = peewee.Value(self.instrument)
         # cadence = peewee.Value(self.parameters.get('cadence', self.cadence))
 
-        dered_flux_z_min = AB2nMgy(self.parameters["dered_mag_z_max"])
-        dered_fiberflux_z_min = AB2nMgy(self.parameters["dered_fibermag_z_max"])
+        # dered_flux_z_min = AB2nMgy(self.parameters["dered_mag_z_max"])  # moved to post_process
+        # dered_fiberflux_z_min = AB2nMgy(self.parameters["dered_fibermag_z_max"])  # ditto
         fiberflux_z_min = AB2nMgy(self.parameters["fibermag_z_max"])
         fiberflux_z_max = AB2nMgy(self.parameters["fibermag_z_min"])
         fiberflux_r_min = AB2nMgy(self.parameters["fibermag_r_max"])
@@ -169,6 +171,7 @@ def build_query(self, version_id, query_region=None):
         )
 
         # compute the abs(Galactic latitude):
+        # we return this in the query result, and then and test aginst it in post_process
         gal_lat = peewee.fn.abs(
             90.0 - peewee.fn.q3c_dist(north_gal_pole_ra, north_gal_pole_dec, c.ra, c.dec)
         )
@@ -206,10 +209,12 @@ def build_query(self, version_id, query_region=None):
                 ls.flux_r.alias("ls10_flux_r"),  # extra
                 ls.flux_i.alias("ls10_flux_i"),  # extra
                 ls.flux_z.alias("ls10_flux_z"),  # extra
+                ls.fiberflux_z.alias("ls10_fiberflux_z"),  # extra
                 ls.ebv.alias("ls10_ebv"),  # extra
+                ls.mw_transmission_z.alias("ls10_mw_transmission_z"),  # extra
+                ls.shape_r.alias("ls10_shape_r"),  # extra
                 ls.maskbits.alias("ls10_maskbits"),  # extra
                 ls.fitbits.alias("ls10_fitbits"),  # extra
-                ls.mw_transmission_z.alias("ls10_mw_transmission_z"),  # extra
                 gal_lat.alias("abs_gal_lat"),  # extra
             )
             .join(c2ls)
@@ -219,28 +224,30 @@ def build_query(self, version_id, query_region=None):
                 c2ls.version_id == version_id,
                 ls.type != "PSF",
                 ls.parallax <= 0.0,
-                ls.flux_z > dered_flux_z_min * ls.mw_transmission_z,
+                # ls.flux_z > dered_flux_z_min * ls.mw_transmission_z,  # moved to post_process
+                # ls.fiberflux_z > dered_fiberflux_z_min * ls.mw_transmission_z,  # ditto
                 ls.fiberflux_g > fiberflux_g_min,
                 ls.fiberflux_r > fiberflux_r_min,
                 ls.fiberflux_z > fiberflux_z_min,
-                ls.fiberflux_z > dered_fiberflux_z_min * ls.mw_transmission_z,
                 ls.fiberflux_g < fiberflux_g_max,
                 ls.fiberflux_r < fiberflux_r_max,
                 ls.fiberflux_z < fiberflux_z_max,
                 # safety check using Gaia mags to avoid bad ls photometry
                 ~(ls.gaia_phot_g_mean_mag.between(0.1, self.parameters["gaia_g_mag_limit"])),
                 ~(ls.gaia_phot_rp_mean_mag.between(0.1, self.parameters["gaia_rp_mag_limit"])),
                 ls.shape_r >= self.parameters["shape_r_min"],
-                gal_lat > self.parameters["min_gal_lat"],
+                # gal_lat > self.parameters["min_gal_lat"],  # moved to post_process
                 ls.ebv < self.parameters["max_ebv"],
                 (ls.maskbits.bin_and(maskbits_mask) == 0),  # avoid bad ls data
                 # (ls.fitbits.bin_and(fitbits_mask) == 0),  # avoid bad ls fits
             )
+            # .where(c.catalogid.between(63050396500000000,   # debug
+            #                            63050396550000000))  # debug
             .distinct(c.catalogid)
         )
 
         if self.only_faintest_cadence:
-            query = query.where(cadence == cadence3)
+            query = query.where(ls.fiberflux_r <= fiberflux_r_min_for_cadence2)
 
         # query_region[0] is ra of center of the region, degrees
         # query_region[1] is dec of center of the region, degrees
@@ -254,6 +261,54 @@ def build_query(self, version_id, query_region=None):
 
         return query
 
+    def post_process(self, model, **kwargs):
+        """Runs post-process."""
+        dered_flux_z_min = AB2nMgy(self.parameters["dered_mag_z_max"])
+        dered_fiberflux_z_min = AB2nMgy(self.parameters["dered_fibermag_z_max"])
+
+        data = pandas.read_sql(
+            (
+                "SELECT catalogid,abs_gal_lat,ls10_flux_z,"
+                "ls10_fiberflux_z,ls10_mw_transmission_z "
+                f"from {self.path}"
+            ),
+            self.database,
+        )
+
+        valid = numpy.where(
+            (data["abs_gal_lat"] > self.parameters["min_gal_lat"])
+            & (data["ls10_flux_z"] > dered_flux_z_min * data["ls10_mw_transmission_z"])
+            & (data["ls10_fiberflux_z"] > dered_fiberflux_z_min * data["ls10_mw_transmission_z"]),
+            True,
+            False,
+        )
+
+        print(
+            "During post-processing we down-selected "
+            f"{numpy.count_nonzero(valid)}/{len(data)} rows"
+        )
+
+        data = data[valid]
+
+        valid_cids = data.catalogid.values
+
+        # This way seems faster than updating from a list of values.
+        values_cids = ",".join(f"({vc})" for vc in valid_cids)
+        self.database.execute_sql(
+            "CREATE TEMP TABLE valid_cids AS SELECT * "
+            f"FROM (VALUES {values_cids}) "
+            "AS t (catalogid)"
+        )
+        self.database.execute_sql("CREATE INDEX ON valid_cids (catalogid);")
+        self.database.execute_sql(f"UPDATE {self.path} SET selected = false")
+        self.database.execute_sql(
+            f"UPDATE {self.path} SET selected = true "
+            "FROM valid_cids vc "
+            f"WHERE {self.path}.catalogid = vc.catalogid"
+        )
+
+        return super().post_process(model, **kwargs)
+
 
 class BhmColrGalaxiesLsdr10D3Carton(BhmColrGalaxiesLsdr10Carton):
     name = "bhm_colr_galaxies_lsdr10_d3"

python/target_selection/cartons/mwm_galactic.py

@@ -6,6 +6,8 @@
 # @Filename: mwm_galactic.py
 # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause)
 
+import random
+
 import peewee
 
 from sdssdb.peewee.sdss5db.catalogdb import (
@@ -280,3 +282,162 @@ def build_query(self, version_id, query_region=None):
             )
 
         return query
+
+
+class MWM_Galactic_Core_Dist_apogee_sparse_Carton(MWM_Galactic_Core_Dist_apogee_Carton):
+    """mwm_galactic_core_dist_apogee_sparse
+    Short description: Sparse-sampled Galactic Genesis sample.
+    First do an initial selection as in mwm_galactic_core_dist_apogee and order by catalogid.
+    Then randomly select exactly 2/3 of the sources using a fixed random seed.
+    Metadata:
+    Priority: 2710
+    Cadence: bright_1x1
+    Instrument: APOGEE
+    can_offset = True
+    Lead contact: Michael Blanton
+    """
+
+    name = "mwm_galactic_core_dist_apogee_sparse"
+    category = "science"
+    instrument = "APOGEE"
+    cadence = "bright_1x1"
+    priority = 2710
+    program = "mwm_galactic"
+    mapper = "MWM"
+    can_offset = True
+
+    def build_query(self, version_id, query_region=None):
+        query = super().build_query(version_id, query_region)
+        return query
+
+    def post_process(self, model):
+        cursor = self.database.execute_sql(
+            "update sandbox.temp_mwm_galactic_core_dist_apogee_sparse " + "set selected = false;"
+        )
+
+        cursor = self.database.execute_sql(
+            "select count(1) from " + " sandbox.temp_mwm_galactic_core_dist_apogee_sparse ;"
+        )
+
+        output = cursor.fetchall()
+
+        total_num_rows = output[0][0]
+
+        # This selected_fraction must be the same as in
+        # mwm_galactic_core_dist_apogee_extra
+        selected_fraction = 2.0 / 3.0
+        selected_num_rows = int(selected_fraction * total_num_rows)
+
+        b = [True] * selected_num_rows
+        c = [False] * (total_num_rows - selected_num_rows)
+        is_selected = b + c
+
+        # This random seed must be the same as in
+        # mwm_galactic_core_dist_apogee_extra
+        random.seed(6789)
+        random.shuffle(is_selected)
+
+        # The below "order by catalogid" ensures that the random selection
+        # further below gives the same result every time we run this carton.
+        cursor = self.database.execute_sql(
+            "select catalogid from "
+            + " sandbox.temp_mwm_galactic_core_dist_apogee_sparse "
+            + " order by catalogid;"
+        )
+
+        output = cursor.fetchall()
+
+        for i in range(len(output)):
+            current_catalogid = output[i][0]
+
+            # This condition is opposite of the condition in
+            # mwm_galactic_core_dist_apogee_extra
+            if is_selected[i] is True:
+                self.database.execute_sql(
+                    " update sandbox.temp_mwm_galactic_core_dist_apogee_sparse "
+                    + " set selected = true "
+                    + " where catalogid = "
+                    + str(current_catalogid)
+                    + ";"
+                )
+
+
+class MWM_Galactic_Core_Dist_apogee_extra_Carton(MWM_Galactic_Core_Dist_apogee_Carton):
+    """mwm_galactic_core_dist_apogee_extra
+    Short description: Complement of above carton mwm_galactic_core_dist_apogee_sparse.
+    First do an initial selection as in mwm_galactic_core_dist_apogee and order by catalogid.
+    Then select those targets which were NOT selected in
+    above carton mwm_galactic_core_dist_apogee_sparse.
+    Metadata:
+    Priority: 2711
+    Cadence: bright_1x1
+    Instrument: APOGEE
+    can_offset = True
+    Lead contact: Michael Blanton
+    """
+
+    name = "mwm_galactic_core_dist_apogee_extra"
+    category = "science"
+    instrument = "APOGEE"
+    cadence = "bright_1x1"
+    priority = 2711
+    program = "mwm_galactic"
+    mapper = "MWM"
+    can_offset = True
+
+    def build_query(self, version_id, query_region=None):
+        query = super().build_query(version_id, query_region)
+        return query
+
+    def post_process(self, model):
+        cursor = self.database.execute_sql(
+            "update sandbox.temp_mwm_galactic_core_dist_apogee_extra " + "set selected = false;"
+        )
+
+        cursor = self.database.execute_sql(
+            "select count(1) from " + " sandbox.temp_mwm_galactic_core_dist_apogee_extra ;"
+        )
+
+        output = cursor.fetchall()
+
+        total_num_rows = output[0][0]
+
+        # This selected_fraction must be the same as in
+        # mwm_galactic_core_dist_apogee_sparse
+        selected_fraction = 2.0 / 3.0
+        selected_num_rows = int(selected_fraction * total_num_rows)
+
+        b = [True] * selected_num_rows
+        c = [False] * (total_num_rows - selected_num_rows)
+        is_selected = b + c
+
+        # This random seed must be the same as in
+        # mwm_galactic_core_dist_apogee_sparse
+        random.seed(6789)
+        random.shuffle(is_selected)
+
+        # The below "order by catalogid" ensures that the random selection
+        # further below gives the same result every time we run this carton.
+        cursor = self.database.execute_sql(
+            "select catalogid from "
+            + " sandbox.temp_mwm_galactic_core_dist_apogee_extra "
+            + " order by catalogid;"
+        )
+
+        output = cursor.fetchall()
+
+        for i in range(len(output)):
+            current_catalogid = output[i][0]
+
+            # select those sources which were not selected in
+            # mwm_galactic_core_dist_apogee_sparse.
+            # Hence, this condition is opposite of the condition in
+            # mwm_galactic_core_dist_apogee_sparse
+            if is_selected[i] is False:
+                self.database.execute_sql(
+                    " update sandbox.temp_mwm_galactic_core_dist_apogee_extra "
+                    + " set selected = true "
+                    + " where catalogid = "
+                    + str(current_catalogid)
+                    + ";"
+                )