add run_command

dpgen2/op/collect_data.py

@@ -24,13 +24,13 @@ def get_input_sign(cls):
         return OPIOSign({
             "name" : str,
             "labeled_data" : Artifact(List[Path]),
-            "iter_data" : Artifact(Set[Path]),
+            "iter_data" : Artifact(List[Path]),
         })
 
     @classmethod
     def get_output_sign(cls):
         return OPIOSign({
-            "iter_data" : Artifact(Set[Path]),
+            "iter_data" : Artifact(List[Path]),
         })
 
     @OP.exec_sign_check
@@ -53,7 +53,7 @@ def execute(
         -------
             Output dict with components:
         
-            - `iter_data`: (`Artifact(set[Path])`) The paths of iteration data, added with labeled data generated by this iteration.
+            - `iter_data`: (`Artifact(List[Path])`) The paths of iteration data, added with labeled data generated by this iteration.
         
         """
         raise NotImplementedError

dpgen2/op/run_dp_train.py

@@ -25,7 +25,7 @@ def get_input_sign(cls):
             "task_path" : Artifact(Path),
             "init_model" : Artifact(Path),
             "init_data" : Artifact(Set[Path]),
-            "iter_data" : Artifact(Set[Path]),
+            "iter_data" : Artifact(List[Path]),
         })
 
     @classmethod
@@ -53,7 +53,7 @@ def execute(
             - `task_path`: (`Artifact(Path)`) The path that contains all input files prepareed by `PrepDPTrain`.
             - `init_model`: (`Artifact(Path)`) A frozen model to initialize the training.
             - `init_data`: (`Artifact(set[Path])`) Initial training data.
-            - `iter_data`: (`Artifact(set[Path])`) Training data generated in the DPGEN iterations.
+            - `iter_data`: (`Artifact(List[Path])`) Training data generated in the DPGEN iterations.
 
         Returns
         -------

dpgen2/utils/run_command.py

@@ -0,0 +1,18 @@
+import sys, subprocess
+
+def run_command(
+        cmd,
+        shell = None,
+):
+    pp = subprocess.Popen(
+        cmd, 
+        stdout=subprocess.PIPE, 
+        stderr=subprocess.PIPE,
+        shell=shell,
+    )
+    out, err = pp.communicate()
+    return_code = pp.poll()
+    out = out.decode(sys.stdin.encoding)
+    err = err.decode(sys.stdin.encoding)
+    return return_code, out, err
+

tests/mocked_ops.py

@@ -357,12 +357,12 @@ def execute(
         labeled_data = ip['labeled_data']
         iter_data = ip['iter_data']
 
-        new_iter_data = set()
+        new_iter_data = []
         # copy iter_data
         for ii in iter_data:
             iiname = ii.name
             shutil.copytree(ii, iiname)
-            new_iter_data.add(Path(iiname))
+            new_iter_data.append(Path(iiname))
 
         # collect labled data
         name = Path(name)
@@ -371,7 +371,7 @@ def execute(
         for ii in labeled_data:
             iiname = ii.name
             shutil.copytree(ii, name/iiname)
-        new_iter_data.add(name)
+        new_iter_data.append(name)
 
         return OPIO({
             "iter_data" : new_iter_data,

tests/test_collect_data.py

@@ -41,8 +41,8 @@
 
 class TestMockedCollectData(unittest.TestCase):
     def setUp(self): 
-        self.iter_data = set(('foo/iter0', 'bar/iter1'))
-        self.iter_data = set([Path(ii) for ii in self.iter_data])
+        self.iter_data = ['foo/iter0', 'bar/iter1']
+        self.iter_data = [Path(ii) for ii in self.iter_data]
         self.name = 'outdata'
         self.labeled_data = ['d0', 'd1']
         self.labeled_data = [Path(ii) for ii in self.labeled_data]

tests/test_prep_run_dp_train.py

@@ -168,7 +168,7 @@ def setUp(self):
             ii.mkdir(exist_ok=True, parents=True)
             (ii/'a').write_text('data a')
             (ii/'b').write_text('data b')
-        self.iter_data = set(tmp_iter_data)
+        self.iter_data = tmp_iter_data
 
         self.template_script = mocked_template_script.copy()
 
@@ -225,7 +225,7 @@ def setUp (self) :
 
         tmp_init_data = make_mocked_init_data()
         self.init_data = upload_artifact(tmp_init_data)
-        self.path_init_data = set(tmp_init_data)
+        self.path_init_data = tmp_init_data
 
         tmp_iter_data = [Path('iter_data/foo'), Path('iter_data/bar')]
         for ii in tmp_iter_data:

tests/test_run_command.py

@@ -0,0 +1,48 @@
+import context
+import numpy as np
+import unittest, json, shutil, os
+from pathlib import Path
+from dpgen2.utils.run_command import run_command
+
+class TestRunCommand(unittest.TestCase):
+    def setUp(self):
+        self.work_path = Path('work_path')
+        self.work_path.mkdir(exist_ok=True)
+        (self.work_path/'foo').write_text('foo')
+        (self.work_path/'bar').write_text('foo')
+
+    def tearDown(self):
+        if self.work_path.is_dir():
+            shutil.rmtree(self.work_path)
+
+    def test_success_shell(self):
+        os.chdir(self.work_path)
+        ret, out, err = run_command(['ls | sort'], shell=True)
+        self.assertEqual(ret, 0)
+        self.assertEqual(out, 'bar\nfoo\n')
+        self.assertEqual(err, '')
+        os.chdir('..')
+
+    def test_success(self):
+        os.chdir(self.work_path)
+        ret, out, err = run_command(['ls'])
+        self.assertEqual(ret, 0)
+        self.assertEqual(out, 'bar\nfoo\n')
+        self.assertEqual(err, '')
+        os.chdir('..')
+
+    def test_success_foo(self):
+        os.chdir(self.work_path)
+        ret, out, err = run_command(['ls', 'foo'])
+        self.assertEqual(ret, 0)
+        self.assertEqual(out, 'foo\n')
+        self.assertEqual(err, '')
+        os.chdir('..')
+
+    def test_failed(self):
+        os.chdir(self.work_path)
+        ret, out, err = run_command(['ls', 'tar'])
+        self.assertEqual(ret, 2)
+        self.assertEqual(out, '')
+        self.assertEqual(err, "ls: cannot access 'tar': No such file or directory\n")
+        os.chdir('..')