htcondorcern: fix Singularity and Kerberos tokens

Fixes CERN HTCondor compute backend for Singularity unpacked images
execution mode where jobs couldn't access restricted EOS directories due
to inaccessible Kerberos credentials.

Dockerfile

@@ -1,5 +1,5 @@
 # This file is part of REANA.
-# Copyright (C) 2017, 2018, 2019, 2020, 2021 CERN.
+# Copyright (C) 2017, 2018, 2019, 2020, 2021, 2022 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.

reana_job_controller/htcondorcern_job_manager.py

@@ -146,6 +146,7 @@ def execute(self):
             job_ad["MaxRunTime"] = 3600
         if self.htcondor_accounting_group:
             job_ad["AccountingGroup"] = self.htcondor_accounting_group
+        job_ad["MY.SendCredential"] = True
         future = current_app.htcondor_executor.submit(self._submit, job_ad)
         clusterid = future.result()
         return clusterid
@@ -229,12 +230,15 @@ def _copy_wrapper_file(self):
                 )
             else:
                 template = (
-                    "#!/bin/bash \n"
+                    "#!/bin/bash\n"
+                    'SINGULARITY_KRB5CCNAME="FILE:/srv/$(basename $KRB5CCNAME)"\n'
                     "singularity exec "
+                    "--contain "
+                    "--ipc "
+                    "--pid "
                     "--home $PWD:/srv "
-                    "--bind $PWD:/srv "
                     "--bind /cvmfs "
-                    "--bind /eos "
+                    "--env KRB5CCNAME=$SINGULARITY_KRB5CCNAME "
                     "{DOCKER_IMG} {CMD}".format(
                         DOCKER_IMG=self.docker_img,
                         CMD=self._format_arguments() + " | bash",
@@ -253,15 +257,15 @@ def _copy_wrapper_file(self):
     def _submit(self, job_ad):
         """Execute submission transaction."""
         ads = []
-        schedd = HTCondorJobManagerCERN._get_schedd()
+        schedd, credd = HTCondorJobManagerCERN._get_schedd()
         logging.info("Submiting job - {}".format(job_ad))
         clusterid = schedd.submit(job_ad, 1, True, ads)
         HTCondorJobManagerCERN._spool_input(ads)
         return clusterid
 
     @retry(stop_max_attempt_number=MAX_NUM_RETRIES, wait_fixed=RETRY_WAIT_TIME)
     def _spool_input(ads):
-        schedd = HTCondorJobManagerCERN._get_schedd()
+        schedd, credd = HTCondorJobManagerCERN._get_schedd()
         logging.info("Spooling job inputs - {}".format(ads))
         schedd.spool(ads)
 
@@ -274,12 +278,19 @@ def _get_schedd():
                 thread_local, "MONITOR_THREAD_SCHEDD", htcondor.Schedd()  # noqa: F821
             )
         logging.info("Getting schedd: {}".format(thread_local.MONITOR_THREAD_SCHEDD))
-        return thread_local.MONITOR_THREAD_SCHEDD
+        credd = getattr(thread_local, "MONITOR_THREAD_CREDD", None)
+        if credd is None:
+            setattr(
+                thread_local, "MONITOR_THREAD_CREDD", htcondor.Credd()  # noqa: F821
+            )
+        thread_local.MONITOR_THREAD_CREDD.add_user_cred(htcondor.CredTypes.Kerberos, None)
+        logging.info("Getting credd: {}".format(thread_local.MONITOR_THREAD_CREDD))
+        return thread_local.MONITOR_THREAD_SCHEDD, thread_local.MONITOR_THREAD_CREDD
 
     def stop(backend_job_id):
         """Stop HTCondor job execution."""
         try:
-            schedd = HTCondorJobManagerCERN._get_schedd()
+            schedd, credd = HTCondorJobManagerCERN._get_schedd()
             schedd.act(
                 htcondor.JobAction.Remove,  # noqa: F821
                 "ClusterId=={}".format(backend_job_id),
@@ -290,7 +301,7 @@ def stop(backend_job_id):
     @retry(stop_max_attempt_number=MAX_NUM_RETRIES, wait_fixed=RETRY_WAIT_TIME)
     def spool_output(backend_job_id):
         """Transfer job output."""
-        schedd = HTCondorJobManagerCERN._get_schedd()
+        schedd, credd = HTCondorJobManagerCERN._get_schedd()
         logging.info("Spooling jobs {} output.".format(backend_job_id))
         schedd.retrieve("ClusterId == {}".format(backend_job_id))
 
@@ -316,7 +327,7 @@ def get_logs(backend_job_id, workspace):
 
     def find_job_in_history(backend_job_id):
         """Return job if present in condor history."""
-        schedd = HTCondorJobManagerCERN._get_schedd()
+        schedd, credd = HTCondorJobManagerCERN._get_schedd()
         ads = ["ClusterId", "JobStatus", "ExitCode", "RemoveReason"]
         condor_it = schedd.history(
             "ClusterId == {0}".format(backend_job_id), ads, match=1

reana_job_controller/job_monitor.py

@@ -528,7 +528,7 @@ def query_condor_jobs(app, backend_job_ids):
     ads = ["ClusterId", "JobStatus", "ExitCode", "ExitStatus", "HoldReasonCode"]
     query = format_condor_job_que_query(backend_job_ids)
     htcondorcern_job_manager_cls = COMPUTE_BACKENDS["htcondorcern"]()
-    schedd = htcondorcern_job_manager_cls._get_schedd()
+    schedd, credd = htcondorcern_job_manager_cls._get_schedd()
     logging.info("Querying jobs {}".format(backend_job_ids))
     condor_jobs = schedd.xquery(requirements=query, projection=ads)
     return condor_jobs