test: running on nodes with CVMFS

fstagni · Nov 3, 2023 · 3010de9 · 3010de9
1 parent db7a973
commit 3010de9
Show file tree

Hide file tree

Showing 4 changed files with 89 additions and 56 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -8,7 +8,7 @@ jobs:
 
     strategy:
       matrix:
-        python: 
+        python_version: 
           - 2.7.18
           - 3.6.15
           - 3.9.17
@@ -22,10 +22,21 @@ jobs:
           - dteam
           - gridpp
 
-    container: python:${{ matrix.python }}-slim
+    # container: python:${{ matrix.python }}-slim
 
     steps:
       - uses: actions/checkout@v4
+      - uses: cvmfs-contrib/github-action-cvmfs@v3
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python_version }}
+
+      - name: Check Python version
+        run: python --version
+
+      - name: Test CernVM-FS
+        run: ls /cvmfs/dirac.egi.eu
+
       - name: Retrieve the secret and decode it to a file
         env:
           HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
@@ -135,46 +146,51 @@ jobs:
 
 
 
-  matching:
-    runs-on: ubuntu-latest
+  # matching:
+  #   runs-on: ubuntu-latest
 
-    strategy:
-      matrix:
-        python: 
-          - 2.7.18
-          - 3.6.15
-          - 3.9.17
-        pilot_schema:
-          - pilot_oldSchema.json
-          - pilot_newSchema.json
+  #   strategy:
+  #     matrix:
+  #       python: 
+  #         - 2.7.18
+  #         - 3.6.15
+  #         - 3.9.17
+  #       pilot_schema:
+  #         - pilot_oldSchema.json
+  #         - pilot_newSchema.json
 
-    container: python:${{ matrix.python }}-slim
+  #   container: python:${{ matrix.python }}-slim
 
-    steps:
-      - uses: actions/checkout@v4
-      - name: Retrieve the secret and decode it to a file
-        env:
-          HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
-          HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }}
-        run: |
-          cd Pilot
-          mkdir -p etc/grid-security
-          echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem
-          echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem
-          chmod 440 etc/grid-security/hostcert.pem
-          chmod 400 etc/grid-security/hostkey.pem
-      - name: tests
-        run: |
-          cd Pilot
-          cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
-          sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
-          sed -i "s/VAR_JENKINS_CE/jenkins-full.cern.ch/g" pilot.json
-          sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
-          sed -i "s/VAR_DIRAC_VERSION/integration/g" pilot.json
-          sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
-          sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
-          sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: cvmfs-contrib/github-action-cvmfs@v3
+
+  #     - name: Test CernVM-FS
+  #       run: ls /cvmfs/dirac.egi.eu
+
+  #     - name: Retrieve the secret and decode it to a file
+  #       env:
+  #         HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
+  #         HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }}
+  #       run: |
+  #         cd Pilot
+  #         mkdir -p etc/grid-security
+  #         echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem
+  #         echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem
+  #         chmod 440 etc/grid-security/hostcert.pem
+  #         chmod 400 etc/grid-security/hostkey.pem
+  #     - name: tests
+  #       run: |
+  #         cd Pilot
+  #         cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
+  #         sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
+  #         sed -i "s/VAR_JENKINS_CE/jenkins-full.cern.ch/g" pilot.json
+  #         sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
+  #         sed -i "s/VAR_DIRAC_VERSION/integration/g" pilot.json
+  #         sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
+  #         sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
+  #         sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
+  #         python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug
 
 
   integration-cvmfs_matching:

diff --git a/Pilot/tests/Test_Pilot.py b/Pilot/tests/Test_Pilot.py
@@ -3,18 +3,18 @@
 
 from __future__ import absolute_import, division, print_function
 
-# pylint: disable=protected-access, missing-docstring, invalid-name, line-too-long
-
-# imports
-import unittest
 import json
-import stat
-import sys
 import os
 import shutil
+import stat
+import sys
+
+# pylint: disable=protected-access, missing-docstring, invalid-name, line-too-long
+# imports
+import unittest
 
-from Pilot.pilotTools import PilotParams
 from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes
+from Pilot.pilotTools import PilotParams
 
 
 class PilotTestCase(unittest.TestCase):
@@ -39,6 +39,10 @@ def setUp(self):
                 },
                 fp,
             )
+        os.environ["X509_CERT_DIR"] = os.getcwd()
+        os.environ["X509_VOMS_DIR"] = os.getcwd()
+        os.environ["X509_VOMSES"] = os.getcwd()
+        os.environ["X509_USER_PROXY"] = os.getcwd()
 
     def tearDown(self):
         for fileProd in [

diff --git a/Pilot/tests/Test_proxyTools.py b/Pilot/tests/Test_proxyTools.py
@@ -1,12 +1,10 @@
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
 
-import subprocess
-import shlex
-import sys
 import os
+import shlex
 import shutil
+import subprocess
+import sys
 import unittest
 
 ############################
@@ -49,7 +47,7 @@ def test_getVOPopenFails(self, popenMock):
             data = fp.read()
         os.remove(cert)
         with self.assertRaises(Exception) as exc:
-            vo = getVO(data)
+            getVO(data)
 
         self.assertEqual(str(exc.exception), msg)
 
@@ -58,7 +56,7 @@ def test_getVOPopenFails(self, popenMock):
         msg = "command not found: openssl"
         popenMock.side_effect = OSError(msg)
         with self.assertRaises(OSError) as exc:
-            res = getVO(data)
+            getVO(data)
         self.assertEqual(str(exc.exception), msg)
 
     @patch("Pilot.proxyTools.Popen")
@@ -70,7 +68,7 @@ def test_parseASN1Fails(self, popenMock):
         msg = "Error when invoking openssl asn1parse"
 
         with self.assertRaises(Exception) as exc:
-            res = parseASN1("Anything")
+            parseASN1("Anything")
 
         self.assertEqual(str(exc.exception), msg)
 
@@ -80,7 +78,7 @@ def test_parseASN1Fails(self, popenMock):
         msg = "command not found: openssl"
         popenMock.side_effect = OSError(msg)
         with self.assertRaises(OSError) as exc:
-            res = parseASN1("Anything")
+            parseASN1("Anything")
         self.assertEqual(str(exc.exception), msg)
 
     def test_createFakeProxy(self):

diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py
@@ -34,6 +34,10 @@ def test_pilotParamsInit(self, argvmock):
             "tests/pilot.json",
         ]
 
+        os.environ["X509_CERT_DIR"] = os.getcwd()
+        os.environ["X509_VOMS_DIR"] = os.getcwd()
+        os.environ["X509_VOMSES"] = os.getcwd()
+        os.environ["X509_USER_PROXY"] = os.getcwd()
         pp = PilotParams()
 
         argvmock.__getitem__.assert_called()
@@ -89,6 +93,10 @@ def test_pilotOptions(self, argvmock, mockPaths):
             "/%s/Pilot" % vo
         ]
         mockPaths.return_value = paths
+        os.environ["X509_CERT_DIR"] = os.getcwd()
+        os.environ["X509_VOMS_DIR"] = os.getcwd()
+        os.environ["X509_VOMSES"] = os.getcwd()
+        os.environ["X509_USER_PROXY"] = os.getcwd()
         pp = PilotParams()
         lTESTcommands = "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements"
 
@@ -112,6 +120,9 @@ def setUp(self):
         # These temporary files, opened in a binary mode, will act as standard stream pipes for `Popen`
         self.stdout_mock = tempfile.NamedTemporaryFile(mode="rb+", delete=False)
         self.stderr_mock = tempfile.NamedTemporaryFile(mode="rb+", delete=False)
+        os.environ["X509_CERT_DIR"] = '/some/thing/'
+        os.environ["X509_USER_PROXY"] = '/some/thing'
+
 
     def tearDown(self):
         # At the end of the test, we'll close and remove the created files
@@ -122,6 +133,10 @@ def tearDown(self):
     @patch(("sys.argv"))
     @patch("subprocess.Popen")
     def test_executeAndGetOutput(self, popenMock, argvmock):
+        os.environ["X509_CERT_DIR"] = os.getcwd()
+        os.environ["X509_VOMS_DIR"] = os.getcwd()
+        os.environ["X509_VOMSES"] = os.getcwd()
+        os.environ["X509_USER_PROXY"] = os.getcwd()
         argvmock.__getitem__.return_value = [
             "-d",
             "-g",