diff --git a/examples/example_HSC_DRP-RC2.yaml b/examples/example_HSC_DRP-RC2.yaml index 0dd394d0..da74543e 100644 --- a/examples/example_HSC_DRP-RC2.yaml +++ b/examples/example_HSC_DRP-RC2.yaml @@ -6,7 +6,7 @@ name: HSC_DRP-RC2_campaign includes: ["base_campaign"] collections: - campaign_source: HSC/raw/RC2 + campaign_source: "HSC/runs/RC2/w_2024_38/DM-46429,HSC/raw/RC2" campaign_ancillary_inputs: - HSC/calib - HSC/masks @@ -14,49 +14,9 @@ - refcats - skymaps steps: - - Step: - name: step1 - spec_block: HSC_DRP-RC2_step1 - child_config: - split_method: split_by_query - split_min_groups: 3 - - - Step: - name: step2a - spec_block: HSC_DRP-RC2_step2a - prerequisites: ['step1'] - child_config: - split_method: no_split - - Step: - name: step2b - spec_block: HSC_DRP-RC2_step2b - prerequisites: ['step2a'] - child_config: - split_method: split_by_vals - split_vals: - - 9615 - - 9697 - - 9813 - - Step: - name: step2cde - spec_block: HSC_DRP-RC2_step2cde - prerequisites: ['step2b'] - child_config: - split_method: no_split - - Step: - name: step3 - spec_block: HSC_DRP-RC2_step3 - prerequisites: ['step2cde'] - child_config: - split_method: split_by_vals - split_vals: - - 9615 - - 9697 - - 9813 - Step: name: injected_step3 spec_block: HSC_DRP-RC2_injected_step3 - prerequisites: ['step3'] child_config: split_method: split_by_vals split_vals: @@ -73,46 +33,8 @@ - 9615 - 9697 - 9813 - - Step: - name: step4 - spec_block: HSC_DRP-RC2_step4 - prerequisites: ['step3'] - child_config: - split_method: split_by_query - split_min_groups: 3 - data: - extra_qgraph_options: > - -c parameters:sasquatch_dataset_identifier=HSC/RC2 - -c parameters:sasquatch_timestamp_version=reference_package_timestamp - --dataset-query-constraint finalVisitSummary - - - Step: - name: step5 - spec_block: HSC_DRP-RC2_step5 - prerequisites: ['step4'] - child_config: - split_method: split_by_vals - split_vals: - - 9615 - - 9697 - - 9813 - - Step: - name: step6 - spec_block: HSC_DRP-RC2_step6 - prerequisites: ['step4'] - child_config: - split_method: split_by_query - split_min_groups: 3 - split_dataset: calexp - split_field: visit - - Step: - name: step7 - spec_block: HSC_DRP-RC2_step7 - prerequisites: ['step3'] - child_config: - split_method: no_split data: - butler_repo: '/repo/main+sasquatch_dev' + butler_repo: '/repo/main' prod_area: 'output/archive' # Change this to /sdf/group/shared/campaigns data_query: "instrument = 'HSC' and skymap = 'hsc_rings_v1' and band in ('g', 'r', 'i', 'z', 'y')" bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml" diff --git a/examples/example_HSC_DRP-RC2_steps.yaml b/examples/example_HSC_DRP-RC2_steps.yaml index 3d4aaef7..cd57a448 100644 --- a/examples/example_HSC_DRP-RC2_steps.yaml +++ b/examples/example_HSC_DRP-RC2_steps.yaml @@ -1,44 +1,3 @@ -- SpecBlock: - name: HSC_DRP-RC2_step1 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step1" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_dataset: raw - split_field: exposure -- SpecBlock: - name: HSC_DRP-RC2_step2a - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2a" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_method: no_split -- SpecBlock: - name: HSC_DRP-RC2_step2b - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2b" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_field: tract -- SpecBlock: - name: HSC_DRP-RC2_step2cde - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2cde" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_method: no_split -- SpecBlock: - name: HSC_DRP-RC2_step3 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step3" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_field: tract - SpecBlock: name: HSC_DRP-RC2_injected_step3 includes: ['step'] @@ -55,37 +14,3 @@ child_config: base_query: "instrument='HSC' and skymap='hsc_rings_v1'" split_field: tract -- SpecBlock: - name: HSC_DRP-RC2_step4 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step4" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_dataset: calexp - split_field: visit -- SpecBlock: - name: HSC_DRP-RC2_step5 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step5" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_field: tract -- SpecBlock: - name: HSC_DRP-RC2_step6 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step6" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_dataset: calexp - split_field: visit -- SpecBlock: - name: HSC_DRP-RC2_step7 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step7" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_method: no_split diff --git a/examples/example_start_RC2_template.yaml b/examples/example_start_RC2_template.yaml index 4fed4b00..eadc16c3 100644 --- a/examples/example_start_RC2_template.yaml +++ b/examples/example_start_RC2_template.yaml @@ -1,22 +1,22 @@ Production: name: HSC_DRP-RC2 Campaign: - name: w_2024_26_DM-45061 - spec_name: HSC_DRP-RC2_panda + name: test_source_injection + spec_name: HSC_DRP-RC2_htcondor collections: - out: HSC/runs/RC2/w_2024_26/DM-45061 - campaign_source: HSC/raw/RC2 + out: u/eiger/DM-41828 + campaign_source: "HSC/runs/RC2/w_2024_38/DM-46429,HSC/raw/RC2" data: - lsst_version: w_2024_26 + lsst_version: w_2024_38 prod_area: /sdf/group/rubin/shared/campaigns/HSC-RC2/output/archive -# custom_lsst_setup: setup -j -r /sdf/group/rubin/u/yusra/lsst_devel/ctrl_bps -# -# butler_repo: '/repo/main+sasquatch_dev' -# data_query: "instrument = 'HSC' and skymap = 'hsc_rings_v1'" -# bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml" -# bps_wms_resources_file: "${DRP_PIPE_DIR}/bps/resources/HSC/DRP-RC2.yaml" -# + custom_lsst_setup: "setup -j -r /sdf/home/m/mccann/repos/drp_pipe;setup -j -r /sdf/home/m/mccann/repos/source_injection;" + + butler_repo: '/repo/main' + data_query: "instrument = 'HSC' and skymap = 'hsc_rings_v1'" + bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml" + bps_wms_resources_file: "${DRP_PIPE_DIR}/bps/resources/HSC/DRP-RC2.yaml" + # Examples of how to override step specific stuff # child_config: # step1: diff --git a/src/lsst/cmservice/models/queue.py b/src/lsst/cmservice/models/queue.py index 65e9cb02..bd9ebbda 100644 --- a/src/lsst/cmservice/models/queue.py +++ b/src/lsst/cmservice/models/queue.py @@ -68,3 +68,8 @@ class QueueUpdate(QueueBase): time_updated: datetime | None = None # When processing of this element completed time_finished: datetime | None = None + + # Id of node to add + node_id: int + # Which type of node to add + node_level: LevelEnum