Merge branch 'columnflow:master' into run3_working_branch

analysis_templates/cms_minimal/law.cfg

@@ -56,6 +56,7 @@ skip_ensure_proxy: False
 # some remote workflow parameter defaults
 htcondor_flavor: $CF_HTCONDOR_FLAVOR
 htcondor_share_software: False
+htcondor_disk: -1
 slurm_flavor: $CF_SLURM_FLAVOR
 slurm_partition: $CF_SLURM_PARTITION
 
@@ -91,8 +92,9 @@ lfn_sources: wlcg_fs_infn_redirector, wlcg_fs_global_redirector
 # the config name, the task family, the dataset name, or the shift name
 # (see AnalysisTask.get_config_lookup_keys() - and subclasses - for the exact order)
 # values can have the following format:
-# for local targets : "local[, LOCAL_FS_NAME or STORE_PATH][, store_parts_modifier]"
-# for remote targets: "wlcg[, WLCG_FS_NAME][, store_parts_modifier]"
+# for local targets   : "local[, LOCAL_FS_NAME or STORE_PATH][, store_parts_modifier]"
+# for remote targets  : "wlcg[, WLCG_FS_NAME][, store_parts_modifier]"
+# for mirrored targets: "wlcg_mirrored, LOCAL_FS_NAME, WLCG_FS_NAME[, store_parts_modifier]"
 # (when WLCG_FS_NAME is empty, the tasks' "default_wlcg_fs" attribute is used)
 # the "store_parts_modifiers" can be the name of a function in the "store_parts_modifiers" aux dict
 # of the analysis instance, which is called with an output's store parts of an output to modify them
@@ -131,7 +133,7 @@ remote_lcg_setup_el7: /cvmfs/grid.cern.ch/centos7-ui-200122/etc/profile.d/setup-
 remote_lcg_setup_el9: /cvmfs/grid.cern.ch/alma9-ui-test/etc/profile.d/setup-alma9-test.sh
 
 # whether the loading of the remove lcg setup file is enforced
-# otherwise is might be skipped in case gfal-ls, etc., are already available
+# otherwise this might be skipped in case gfal-ls, etc., are already available
 remote_lcg_setup_force: True
 
 

analysis_templates/cms_minimal/setup.sh

@@ -134,6 +134,9 @@ setup___cf_short_name_lc__() {
         # source law's bash completion scipt
         source "$( law completion )" ""
 
+        # add completion to the claw command
+        complete -o bashdefault -o default -F _law_complete claw
+
         # silently index
         law index -q
     fi

bin/claw

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Executable that conveniently triggers "cf_sandbox SANDBOX law ..." commands.
+# The SANDBOX is determined with the following precedence:
+# 1. CLAW_SANDBOX (env)
+# 2. analysis.default_columnar_sandbox (law.cfg)
+# 3. venv_columnar_dev (default)
+
+action() {
+    # get the target sandbox
+    local sandbox
+    if [ ! -z "${CLAW_SANDBOX}" ]; then
+        sandbox="${CLAW_SANDBOX}"
+    fi
+    if [ -z "${sandbox}" ]; then
+        local sandbox_tmp="$( law config analysis.default_columnar_sandbox 2>/dev/null )"
+        if [ "$?" = "0" ]; then
+            # extract the name of the sandbox, remove file extension, potentially add '_dev' suffix
+            sandbox_tmp="$( basename "${sandbox_tmp}" )"
+            sandbox_tmp="${sandbox_tmp%.*}"
+            [[ "${sandbox_tmp}" = *_dev ]] || sandbox_tmp="${sandbox_tmp}_dev"
+            sandbox="${sandbox_tmp}"
+        fi
+    fi
+    if [ -z "${sandbox}" ]; then
+        sandbox="venv_columnar_dev"
+    fi
+    # echo "sandbox '${sandbox}'"
+
+    # run the command
+    cf_sandbox "${sandbox}" law "$@"
+}
+action "$@"

columnflow/calibration/__init__.py

@@ -106,6 +106,7 @@ def update_cls_dict(cls_name, cls_dict, get_attr):
                             "data_only, nominal_only or shifts_only are set",
                         )
 
+                if "skip_func" not in cls_dict:
                     def skip_func(self):
                         # check mc_only and data_only
                         if getattr(self, "dataset_inst", None):

columnflow/columnar_util.py

@@ -1565,6 +1565,17 @@ def PRODUCES(cls) -> IOFlagged:
         """
         return cls.IOFlagged(cls, cls.IOFlag.PRODUCES)
 
+    @classmethod
+    def call(cls, func: Callable[[Any, ...], Any]) -> None:
+        """
+        Decorator to wrap a function *func* that should be registered as :py:meth:`call_func`
+        which defines the main callable for processing chunks of data. The function should accept
+        arbitrary arguments and can return arbitrary objects.
+
+        The decorator does not return the wrapped function.
+        """
+        cls.call_func = func
+
     @classmethod
     def init(cls, func: Callable[[], None]) -> None:
         """