set environ(OS_ARCH) (#378)

* set environ(OS_ARCH) * <bot> update requirements-docs.txt * <bot> update requirements-tests.txt * <bot> update requirements.txt * fix tests * <bot> update requirements-docs.txt * <bot> update requirements-tests.txt * <bot> update requirements.txt --------- Co-authored-by: github-actions <[email protected]>
WIPACrepo · Oct 3, 2024 · 0b0012a · 0b0012a
1 parent d06bac7
commit 0b0012a
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 26 deletions.
diff --git a/iceprod/core/exe.py b/iceprod/core/exe.py
@@ -148,7 +148,14 @@ def scope_env(cfg: ConfigParser, obj: dict, upperenv: Optional[Env] = None, logg
         upperenv: previous scope's env output
         logger: a logger object, for localized logging
     """
-    env: Env = {'parameters': {}, 'input_files': set(), 'output_files': set()}
+    env: Env = {
+        'parameters': {},
+        'input_files': set(),
+        'output_files': set(),
+        'environment': {
+            'OS_ARCH': '$OS_ARCH',
+        }
+    }
     if upperenv:
         env['parameters'].update(upperenv['parameters'])
         env['input_files'] = upperenv['input_files']
@@ -382,6 +389,9 @@ async def convert(self):
             for field in self.options:
                 print(f'# {field}={self.options[field]}', file=f)
             print('', file=f)
+            print('# set some env vars for expansion', file=f)
+            print('OS_ARCH=$(/cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/os_arch.sh)', file=f)
+            print('', file=f)
             with scope_env(self.cfgparser, self.task.dataset.config['steering'], logger=self.logger) as globalenv:
                 task = self.task.get_task_config()
                 if self.task.task_files:

diff --git a/requirements-docs.txt b/requirements-docs.txt
@@ -16,9 +16,9 @@ attrs==24.2.0
     #   referencing
 babel==2.16.0
     # via sphinx
-boto3==1.35.32
+boto3==1.35.33
     # via iceprod (setup.py)
-botocore==1.35.32
+botocore==1.35.33
     # via
     #   boto3
     #   s3transfer
@@ -50,7 +50,7 @@ exceptiongroup==1.2.2
     # via anyio
 h11==0.14.0
     # via httpcore
-htcondor==23.9.6
+htcondor==23.10.1
     # via iceprod (setup.py)
 httpcore==1.0.6
     # via httpx

diff --git a/requirements-tests.txt b/requirements-tests.txt
@@ -14,11 +14,11 @@ attrs==24.2.0
     #   referencing
 beautifulsoup4==4.12.3
     # via iceprod (setup.py)
-boto3==1.35.32
+boto3==1.35.33
     # via
     #   iceprod (setup.py)
     #   moto
-botocore==1.35.32
+botocore==1.35.33
     # via
     #   boto3
     #   moto
@@ -60,7 +60,7 @@ flexmock==0.12.1
     # via iceprod (setup.py)
 h11==0.14.0
     # via httpcore
-htcondor==23.9.6
+htcondor==23.10.1
     # via iceprod (setup.py)
 httpcore==1.0.6
     # via httpx

diff --git a/requirements.txt b/requirements.txt
@@ -12,9 +12,9 @@ attrs==24.2.0
     # via
     #   jsonschema
     #   referencing
-boto3==1.35.32
+boto3==1.35.33
     # via iceprod (setup.py)
-botocore==1.35.32
+botocore==1.35.33
     # via
     #   boto3
     #   s3transfer
@@ -44,7 +44,7 @@ exceptiongroup==1.2.2
     # via anyio
 h11==0.14.0
     # via httpcore
-htcondor==23.9.6
+htcondor==23.10.1
     # via iceprod (setup.py)
 httpcore==1.0.6
     # via httpx

diff --git a/tests/core/exe_test.py b/tests/core/exe_test.py
@@ -321,7 +321,7 @@ async def test_write_to_script_module_src(tmp_path):
     assert not ws.outfiles
     script = open(scriptpath).read()
     lines = [line for line in script.split('\n') if not (not line.strip() or line.startswith('#') or line.startswith('set '))]
-    assert lines == ['python foo.py']
+    assert lines[-1] == 'python foo.py'
 
 
 async def test_write_to_script_module_shell(tmp_path):
@@ -344,7 +344,7 @@ async def test_write_to_script_module_shell(tmp_path):
     assert not ws.outfiles
     script = open(scriptpath).read()
     lines = [line for line in script.split('\n') if not (not line.strip() or line.startswith('#') or line.startswith('set '))]
-    assert lines == ['/bin/sh foo.sh']
+    assert lines[-1] == '/bin/sh foo.sh'
 
 
 async def test_write_to_script_module_binary(tmp_path):
@@ -367,7 +367,7 @@ async def test_write_to_script_module_binary(tmp_path):
     assert not ws.outfiles
     script = open(scriptpath).read()
     lines = [line for line in script.split('\n') if not (not line.strip() or line.startswith('#') or line.startswith('set '))]
-    assert lines == ['./foo']
+    assert lines[-1] == './foo'
 
 async def test_write_to_script_module_binary_fullpath(tmp_path):
     t = get_task({
@@ -389,7 +389,30 @@ async def test_write_to_script_module_binary_fullpath(tmp_path):
     assert not ws.outfiles
     script = open(scriptpath).read()
     lines = [line for line in script.split('\n') if not (not line.strip() or line.startswith('#') or line.startswith('set '))]
-    assert lines == ['/cvmfs/foo']
+    assert lines[-1] == '/cvmfs/foo'
+
+
+async def test_write_to_script_os_arch(tmp_path):
+    t = get_task({
+        'tasks': [{
+            'name': 'foo',
+            'trays': [{
+                'modules': [{
+                    'env_clear': False,
+                    'src': '/cvmfs/foo/$environ(OS_ARCH)'
+                }]
+            }],
+        }]
+    })
+
+    ws = iceprod.core.exe.WriteToScript(t, workdir=tmp_path, logger=logger)
+    scriptpath = await ws.convert()
+
+    assert not ws.infiles
+    assert not ws.outfiles
+    script = open(scriptpath).read()
+    lines = [line for line in script.split('\n') if not (not line.strip() or line.startswith('#') or line.startswith('set '))]
+    assert lines[-1] == '/cvmfs/foo/$OS_ARCH'
 
 
 async def test_write_to_script_tray_iter(tmp_path):
@@ -415,7 +438,7 @@ async def test_write_to_script_tray_iter(tmp_path):
     script = open(scriptpath).read()
     logging.debug('script: \n%s', script)
     lines = [line for line in script.split('\n') if not (not line.strip() or line.startswith('#') or line.startswith('set '))]
-    assert lines == [
+    assert lines[-3:] == [
         'python foo.py 0',
         'python foo.py 1',
         'python foo.py 2',
@@ -442,9 +465,8 @@ async def test_write_to_script_module_env_clear(tmp_path):
     assert not ws.outfiles
     script = open(scriptpath).read()
     lines = [line for line in script.split('\n') if not (not line.strip() or line.startswith('#') or line.startswith('set '))]
-    assert len(lines) == 1
-    assert lines[0].startswith('env -i ')
-    assert lines[0].endswith(' python foo.py')
+    assert lines[-1].startswith('env -i ')
+    assert lines[-1].endswith(' python foo.py')
 
 
 async def test_write_to_script_module_env_shell(tmp_path):
@@ -468,7 +490,7 @@ async def test_write_to_script_module_env_shell(tmp_path):
     assert not ws.outfiles
     script = open(scriptpath).read()
     lines = [line for line in script.split('\n') if not (not line.strip() or line.startswith('#') or line.startswith('set '))]
-    assert lines == ['/foo/bar/baz.sh python foo.py']
+    assert lines[-1] == '/foo/bar/baz.sh python foo.py'
 
 
 async def test_write_to_script_data(tmp_path):
@@ -503,7 +525,7 @@ async def test_write_to_script_data(tmp_path):
     assert ws.outfiles == {Data('https://foo.bar/1234', '1234', Transfer.TRUE)}
     script = open(scriptpath).read()
     lines = [line for line in script.split('\n') if not (not line.strip() or line.startswith('#') or line.startswith('set '))]
-    assert lines == ['python foo.py']
+    assert lines[-1] == 'python foo.py'
 
 
 async def test_write_to_script_data_task_files(tmp_path):
@@ -592,9 +614,6 @@ async def test_write_to_script_data_dups(tmp_path):
 
     assert ws.infiles == {Data('https://foo.bar/baz', 'baz', Transfer.TRUE)}
     assert ws.outfiles == {Data('https://foo.bar/1234', '1234', Transfer.TRUE)}
-    script = open(scriptpath).read()
-    lines = [line for line in script.split('\n') if not (not line.strip() or line.startswith('#') or line.startswith('set '))]
-    assert lines == ['python foo.py']
 
 
 async def test_write_to_script_data_iterations(tmp_path):
@@ -630,6 +649,3 @@ async def test_write_to_script_data_iterations(tmp_path):
 
     assert ws.infiles == {Data('https://foo.bar/foo_bar_baz', 'foo_bar_baz', Transfer.TRUE)}
     assert ws.outfiles == set()
-    script = open(scriptpath).read()
-    lines = [line for line in script.split('\n') if not (not line.strip() or line.startswith('#') or line.startswith('set '))]
-    assert lines == ['python foo.py']