[tests] Make pytest debugging a bit more user-friendly

* use --skip-teardown to prevent removal of artifacts
* for tamper tests, run all of them instead of abort on first failure

Signed-off-by: g2flyer <[email protected]>

libos/test/fs/README.md

@@ -20,3 +20,13 @@ How to execute
 
 Encrypted file tests assume that Gramine was built with SGX enabled (see comment
 in `test_enc.py`).
+
+This test suite automatically creates files-under-test on startup and removes
+them afterwards. When some test fails and you want to debug this failure, it's
+more convenient to skip this automatic removal of files (and manually
+investigate the test scenario, e.g. with the help of GDB). In such cases, use
+the pytest option `--skip-teardown`. Note that running with that option will
+(a) work successfully only when restricting tests (using option `-k`) to an individual
+Module/TestCase family, i.e., `EncryptedFiles`, `FileSystem`, `Sync` and `Tmpfs`, and
+(b) only for the first run. Otherwise, the lack of teardown will cause various errors
+until you reset the test environment with a `rm -rf tmp`.

libos/test/fs/conftest.py

@@ -0,0 +1,10 @@
+import pytest
+
+option = None
+
+def pytest_addoption(parser):
+    parser.addoption("--skip-teardown", action='store_true')
+
+def pytest_configure(config):
+    global option
+    option = config.option

libos/test/fs/test_enc.py

@@ -179,17 +179,21 @@ def test_500_invalid(self):
         # prepare valid encrypted file (largest one for maximum possible corruptions)
         original_input = self.OUTPUT_FILES[-1]
         self.__encrypt_file(self.INPUT_FILES[-1], original_input)
+        # Save for debugging as we overwrite original below
+        shutil.copy(original_input, original_input + '.orig')
         # generate invalid files based on the above
         self.__corrupt_file(original_input, invalid_dir)
 
         # try to decrypt invalid files
+        failed = False
         for name in os.listdir(invalid_dir):
             invalid = os.path.join(invalid_dir, name)
             output_path = os.path.join(self.OUTPUT_DIR, name)
             input_path = os.path.join(invalid_dir, os.path.basename(original_input))
             # copy the file so it has the original file name (for allowed path check)
             shutil.copy(invalid, input_path)
 
+            # test decryption using the pf-crypt tool
             try:
                 args = ['decrypt', '-V', '-w', self.WRAP_KEY, '-i', input_path, '-o', output_path]
                 self.__pf_crypt(args)
@@ -198,4 +202,6 @@ def test_500_invalid(self):
                 self.assertEqual(exc.returncode, 255)
             else:
                 print('[!] Fail: successfully decrypted file: ' + name)
-                self.fail()
+                failed = True
+        if failed:
+            self.fail()

libos/test/fs/test_fs.py

@@ -2,6 +2,7 @@
 import os
 import shutil
 import unittest
+import conftest
 
 from graminelibos.regression import (
     HAS_SGX,
@@ -31,7 +32,8 @@ def setUpClass(cls):
 
     @classmethod
     def tearDownClass(cls):
-        shutil.rmtree(cls.TEST_DIR)
+        if not conftest.option.skip_teardown:
+            shutil.rmtree(cls.TEST_DIR)
 
     def setUp(self):
         # clean output for each test
@@ -346,7 +348,8 @@ def setUp(self):
         os.mkdir(self.TEST_DIR)
 
     def tearDown(self):
-        shutil.rmtree(self.TEST_DIR)
+        if not conftest.option.skip_teardown:
+            shutil.rmtree(self.TEST_DIR)
 
     def _test_multiple_writers(self, n_lines, n_processes, n_threads):
         output_path = os.path.join(self.TEST_DIR, 'output.txt')