Adds lock to FM provider (#882)

When running experiments on slurm, it can happen that multiple nodes fetch the feature model repository at the same time. As nodes may use a shared feature model repository (E.g. on /scratch/) this can lead to issues.

This is no issue if the feature model repository is already up-to-date, however if there are any upstream changes this leads to errors in the fetch on some nodes and subsequent failure of the slurm jobs.


---------

Co-authored-by: Lukas Abelt <[email protected]>
Co-authored-by: Florian Sattler <[email protected]>

tests/utils/test_filesystem_util.py

@@ -0,0 +1,48 @@
+"""Test the filesystem utils of VaRA-TS."""
+import errno
+import os
+import unittest
+import uuid
+from fcntl import flock, LOCK_EX, LOCK_NB, LOCK_UN
+
+from varats.utils.filesystem_util import lock_file
+
+
+class TestFileLock(unittest.TestCase):
+    """Tests whether the lock context manager works correctly."""
+
+    def test_file_locking(self):
+        """Test that the file is locked when in a context manager."""
+        tmp_lock_file = "/tmp/lock-test.lock"
+
+        with lock_file(tmp_lock_file):
+            # File should automatically be created
+            self.assertTrue(os.path.exists(tmp_lock_file))
+
+            f = os.open(tmp_lock_file, os.O_RDONLY)
+
+            with self.assertRaises(OSError) as context:
+                # A non-blocking attempt to lock the file again should fail immediately
+                flock(f, LOCK_EX | LOCK_NB)
+            os.close(f)
+            self.assertEqual(context.exception.errno, errno.EWOULDBLOCK)
+
+        # Attempting to lock the file and immediately unlocking should now work
+        f = os.open(tmp_lock_file, os.O_RDONLY)
+        flock(f, LOCK_EX | LOCK_NB)
+        flock(f, LOCK_UN)
+        os.close(f)
+
+    def test_lock_file_new_folder(self):
+        """Test that the lock context manager works correctly when the lock file
+        is in a new folder."""
+        tmp_lock_file = f"/tmp/{uuid.uuid4()}"
+
+        while os.path.isdir(tmp_lock_file):
+            tmp_lock_file = f"/tmp/{uuid.uuid4()}"
+
+        tmp_lock_file += "/lock-test.lock"
+
+        with lock_file(tmp_lock_file):
+            # File should automatically be created
+            self.assertTrue(os.path.exists(tmp_lock_file))

varats-core/varats/provider/feature/feature_model_provider.py

@@ -7,6 +7,7 @@
 from benchbuild.source.base import target_prefix
 
 from varats.provider.provider import Provider
+from varats.utils.filesystem_util import lock_file
 
 
 class FeatureModelNotFound(FileNotFoundError):
@@ -90,6 +91,9 @@ def _get_feature_model_repository_path() -> Path:
             refspec="origin/HEAD",
             limit=1,
         )
-        fm_source.fetch()
+
+        lock_path = Path(target_prefix()) / "fm_provider.lock"
+        with lock_file(lock_path):
+            fm_source.fetch()
 
         return Path(Path(target_prefix()) / fm_source.local)

varats-core/varats/utils/filesystem_util.py

@@ -20,6 +20,9 @@ def __init__(self, folder: tp.Union[Path, str]) -> None:
 @contextmanager
 def lock_file(lock_path: Path,
               lock_mode: int = fcntl.LOCK_EX) -> tp.Generator[None, None, None]:
+    # Create directories until lock file if required
+    os.makedirs(os.path.dirname(lock_path), exist_ok=True)
+
     open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC
     lock_fd = os.open(lock_path, open_mode)
     try: