From d84d5d35e6703f69e98d977aa11eb6b632ce9d34 Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Tue, 26 Mar 2024 12:55:36 +0100 Subject: [PATCH 1/5] - Added lock to feature model provider to avoid parallel fetches when running in cluster mode --- .../varats/provider/feature/feature_model_provider.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/varats-core/varats/provider/feature/feature_model_provider.py b/varats-core/varats/provider/feature/feature_model_provider.py index 90c9bf4d8..3471cbfed 100644 --- a/varats-core/varats/provider/feature/feature_model_provider.py +++ b/varats-core/varats/provider/feature/feature_model_provider.py @@ -5,6 +5,7 @@ import benchbuild as bb from benchbuild.project import Project from benchbuild.source.base import target_prefix +from utils.filesystem_util import lock_file from varats.provider.provider import Provider @@ -90,6 +91,9 @@ def _get_feature_model_repository_path() -> Path: refspec="origin/HEAD", limit=1, ) - fm_source.fetch() + lock_path = Path(target_prefix()) / "fm_provider.lock" + + with lock_file(lock_path): + fm_source.fetch() return Path(Path(target_prefix()) / fm_source.local) From a3e50b5789a13c873840c42a35f10669871bd9a2 Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Thu, 28 Mar 2024 15:08:32 +0100 Subject: [PATCH 2/5] - Fix imports --- varats-core/varats/provider/feature/feature_model_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varats-core/varats/provider/feature/feature_model_provider.py b/varats-core/varats/provider/feature/feature_model_provider.py index 3471cbfed..67d38c009 100644 --- a/varats-core/varats/provider/feature/feature_model_provider.py +++ b/varats-core/varats/provider/feature/feature_model_provider.py @@ -5,9 +5,9 @@ import benchbuild as bb from benchbuild.project import Project from benchbuild.source.base import target_prefix -from utils.filesystem_util import lock_file from varats.provider.provider import Provider +from varats.utils.filesystem_util import lock_file class FeatureModelNotFound(FileNotFoundError): From 6bfa515e2db89f6fda25ced0a0c19dea2fdf0742 Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Wed, 3 Apr 2024 09:45:11 +0200 Subject: [PATCH 3/5] Include suggestions from review Co-authored-by: Florian Sattler --- varats-core/varats/provider/feature/feature_model_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varats-core/varats/provider/feature/feature_model_provider.py b/varats-core/varats/provider/feature/feature_model_provider.py index 67d38c009..69d2d7cca 100644 --- a/varats-core/varats/provider/feature/feature_model_provider.py +++ b/varats-core/varats/provider/feature/feature_model_provider.py @@ -91,8 +91,8 @@ def _get_feature_model_repository_path() -> Path: refspec="origin/HEAD", limit=1, ) - lock_path = Path(target_prefix()) / "fm_provider.lock" + lock_path = Path(target_prefix()) / "fm_provider.lock" with lock_file(lock_path): fm_source.fetch() From 4a578e665a05092838984352dcd04a13d01603e2 Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Mon, 22 Apr 2024 10:37:44 +0200 Subject: [PATCH 4/5] * Add tests for lock context manager --- tests/utils/test_filesystem_util.py | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/utils/test_filesystem_util.py diff --git a/tests/utils/test_filesystem_util.py b/tests/utils/test_filesystem_util.py new file mode 100644 index 000000000..4e444976a --- /dev/null +++ b/tests/utils/test_filesystem_util.py @@ -0,0 +1,48 @@ +"""Test the filesystem utils of VaRA-TS.""" +import errno +import os +import unittest +import uuid +from fcntl import flock, LOCK_EX, LOCK_NB, LOCK_UN + +from varats.utils.filesystem_util import lock_file + + +class TestFileLock(unittest.TestCase): + """Tests whether the lock context manager works correctly.""" + + def test_file_locking(self): + """Test that the file is locked when in a context manager.""" + tmp_lock_file = "/tmp/lock-test.lock" + + with lock_file(tmp_lock_file): + # File should automatically be created + self.assertTrue(os.path.exists(tmp_lock_file)) + + f = os.open(tmp_lock_file, os.O_RDONLY) + + with self.assertRaises(OSError) as context: + # A non-blocking attempt to lock the file again should fail immediately + flock(f, LOCK_EX | LOCK_NB) + os.close(f) + self.assertEqual(context.exception.errno, errno.EWOULDBLOCK) + + # Attempting to lock the file and immediately unlocking should now work + f = os.open(tmp_lock_file, os.O_RDONLY) + flock(f, LOCK_EX | LOCK_NB) + flock(f, LOCK_UN) + os.close(f) + + def test_lock_file_new_folder(self): + """Test that the lock context manager works correctly when the lock file + is in a new folder.""" + tmp_lock_file = f"/tmp/{uuid.uuid4()}" + + while os.path.isdir(tmp_lock_file): + tmp_lock_file = f"/tmp/{uuid.uuid4()}" + + tmp_lock_file += "/lock-test.lock" + + with lock_file(tmp_lock_file): + # File should automatically be created + self.assertTrue(os.path.exists(tmp_lock_file)) From 4880d324fcc8b2c787fb0a1a587cdbc14fc595e2 Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Mon, 22 Apr 2024 10:39:05 +0200 Subject: [PATCH 5/5] * Change lock context manager to create directories if required --- varats-core/varats/utils/filesystem_util.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/varats-core/varats/utils/filesystem_util.py b/varats-core/varats/utils/filesystem_util.py index 258fb5e27..4f6ff107d 100644 --- a/varats-core/varats/utils/filesystem_util.py +++ b/varats-core/varats/utils/filesystem_util.py @@ -20,6 +20,9 @@ def __init__(self, folder: tp.Union[Path, str]) -> None: @contextmanager def lock_file(lock_path: Path, lock_mode: int = fcntl.LOCK_EX) -> tp.Generator[None, None, None]: + # Create directories until lock file if required + os.makedirs(os.path.dirname(lock_path), exist_ok=True) + open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC lock_fd = os.open(lock_path, open_mode) try: