From b87a13c4d9007251ef12b51caa8178e24fd9833d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20B=C3=B6hm?= Date: Wed, 18 Sep 2024 09:43:24 +0200 Subject: [PATCH] Improve handling of git operations (#890) Our handling of git operations in the tool-suite is a mess; some functions use pygit2, others use command-line git and all have different interfaces and conventions and I am very annoyed that I always have the incorrect parameters at hand when I need to use some git functionality. This PR aims to improve this situation by introducing a lightweight RepositoryHandle class that allows easy and uniform access to both, pygit2 and command-line git`. Since git is ubiquitous in the tool-suite, there are a lot of changes, but most of them are very minor. As a result of the refactoring, git_util no longer depends on project_util, but the other way round. In my opinion this makes more sense since projects make use of git, while git can happily be used without projects. One open point that is remains is to sort out what belongs in git_util vs. git_commands since that is currently not very strict and at least one function in git_commands uses a function in git_util, which imo it should not do. --- .pre-commit-config.yaml | 8 +- tests/data/test_commit_report.py | 6 +- tests/paper_mgmt/test_case_study.py | 6 +- tests/provider/test_bug_provider.py | 25 +- tests/provider/test_patch_provider.py | 10 +- .../test_driver_gen_benchbuild_config.py | 2 +- tests/utils/test_git_util.py | 117 ++-- tests/utils/test_project_util.py | 10 +- varats-core/varats/experiment/steps/patch.py | 9 +- varats-core/varats/mapping/author_map.py | 10 +- varats-core/varats/mapping/commit_map.py | 79 ++- varats-core/varats/project/project_util.py | 300 +++++++--- varats-core/varats/provider/bug/bug.py | 6 +- varats-core/varats/provider/cve/cve.py | 3 +- varats-core/varats/provider/cve/cve_map.py | 62 +- .../varats/provider/cve/cve_provider.py | 4 +- .../varats/provider/patch/patch_provider.py | 30 +- .../provider/release/release_provider.py | 5 +- varats-core/varats/utils/git_commands.py | 72 +-- varats-core/varats/utils/git_util.py | 557 ++++++------------ .../databases/blame_diff_metrics_database.py | 17 +- .../blame_interaction_degree_database.py | 4 +- .../databases/szz_quality_metrics_database.py | 8 +- .../data/reports/blame_interaction_graph.py | 30 +- .../experiments/vara/blame_ast_experiment.py | 10 +- .../vara/blame_report_experiment.py | 6 +- .../vara/commit_report_experiment.py | 1 + .../experiments/vara/feature_experiment.py | 2 +- .../gui/cs_gen/case_study_generation.py | 26 +- varats/varats/paper_mgmt/artefacts.py | 4 +- varats/varats/paper_mgmt/case_study.py | 24 +- .../varats/paper_mgmt/paper_config_manager.py | 5 +- .../plots/blame_interaction_graph_plots.py | 8 +- .../plots/blame_interaction_scatter_plots.py | 12 +- varats/varats/plots/blame_lorenz_curve.py | 4 +- varats/varats/plots/bug_fixing_relations.py | 4 +- varats/varats/plots/case_study_overview.py | 8 +- varats/varats/plots/code_centrality_plots.py | 15 +- varats/varats/plots/paper_config_overview.py | 6 +- varats/varats/plots/repository_churn.py | 10 +- varats/varats/projects/c_projects/asterisk.py | 9 +- varats/varats/projects/c_projects/bison.py | 7 +- varats/varats/projects/c_projects/bitlbee.py | 7 +- varats/varats/projects/c_projects/brotli.py | 30 +- varats/varats/projects/c_projects/busybox.py | 7 +- varats/varats/projects/c_projects/bzip2.py | 11 +- .../varats/projects/c_projects/coreutils.py | 9 +- varats/varats/projects/c_projects/curl.py | 7 +- varats/varats/projects/c_projects/file.py | 7 +- varats/varats/projects/c_projects/gawk.py | 7 +- varats/varats/projects/c_projects/git.py | 7 +- varats/varats/projects/c_projects/glib.py | 7 +- varats/varats/projects/c_projects/glibc.py | 7 +- varats/varats/projects/c_projects/gravity.py | 22 +- varats/varats/projects/c_projects/grep.py | 7 +- varats/varats/projects/c_projects/gzip.py | 11 +- varats/varats/projects/c_projects/htop.py | 11 +- varats/varats/projects/c_projects/hypre.py | 11 +- varats/varats/projects/c_projects/irssi.py | 7 +- .../projects/c_projects/libjpeg_turbo.py | 9 +- varats/varats/projects/c_projects/libpng.py | 7 +- .../varats/projects/c_projects/libsigrok.py | 9 +- varats/varats/projects/c_projects/libssh.py | 28 +- varats/varats/projects/c_projects/libtiff.py | 7 +- varats/varats/projects/c_projects/libvpx.py | 7 +- varats/varats/projects/c_projects/libxml2.py | 9 +- varats/varats/projects/c_projects/lrzip.py | 7 +- varats/varats/projects/c_projects/lz4.py | 7 +- varats/varats/projects/c_projects/openssl.py | 22 +- varats/varats/projects/c_projects/openvpn.py | 7 +- varats/varats/projects/c_projects/opus.py | 7 +- varats/varats/projects/c_projects/picosat.py | 30 +- varats/varats/projects/c_projects/qemu.py | 7 +- varats/varats/projects/c_projects/redis.py | 7 +- varats/varats/projects/c_projects/tig.py | 7 +- varats/varats/projects/c_projects/tmux.py | 7 +- varats/varats/projects/c_projects/vim.py | 7 +- varats/varats/projects/c_projects/x264.py | 22 +- varats/varats/projects/c_projects/xz.py | 22 +- varats/varats/projects/cpp_projects/clasp.py | 13 +- .../varats/projects/cpp_projects/doxygen.py | 7 +- varats/varats/projects/cpp_projects/dune.py | 7 +- varats/varats/projects/cpp_projects/ect.py | 7 +- .../projects/cpp_projects/fast_downward.py | 13 +- varats/varats/projects/cpp_projects/hyteg.py | 7 +- varats/varats/projects/cpp_projects/lepton.py | 7 +- varats/varats/projects/cpp_projects/libzmq.py | 7 +- .../varats/projects/cpp_projects/mongodb.py | 7 +- varats/varats/projects/cpp_projects/opencv.py | 7 +- .../varats/projects/cpp_projects/poppler.py | 7 +- ...nteraction_discrete_libs_single_project.py | 7 +- varats/varats/projects/cpp_projects/z3.py | 11 +- .../perf_tests/feature_perf_cs_collection.py | 59 +- .../perf_tests/feature_perf_regression.py | 11 +- .../test_projects/bug_provider_test_repos.py | 7 +- .../test_projects/example_test_repo.py | 7 +- .../test_projects/feature_test_repo.py | 9 +- .../multi_author_coordination.py | 7 +- .../projects/test_projects/taint_tests.py | 9 +- .../projects/test_projects/test_suite.py | 7 +- .../tables/blame_interaction_graph_table.py | 8 +- .../varats/tables/case_study_metrics_table.py | 16 +- varats/varats/tables/code_centrality_table.py | 12 +- .../varats/tables/feature_perf_precision.py | 55 +- varats/varats/tools/bb_config.py | 4 +- varats/varats/tools/driver_casestudy.py | 23 +- .../tools/research_tools/development.py | 15 +- varats/varats/tools/research_tools/phasar.py | 2 +- .../tools/research_tools/research_tool.py | 65 +- .../tools/research_tools/szz_unleashed.py | 2 +- varats/varats/tools/research_tools/vara.py | 6 +- .../tools/research_tools/vara_manager.py | 24 +- varats/varats/ts_utils/cli_util.py | 6 +- 113 files changed, 1195 insertions(+), 1217 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 72648f848..b5821b530 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -13,17 +13,17 @@ repos: args: ['--django'] exclude: 'tests/helper_utils.py' - repo: https://github.com/timothycrosley/isort.git - rev: 5.11.5 + rev: 5.13.2 hooks: - id: isort args: ['--nis'] - repo: https://github.com/myint/docformatter.git - rev: v1.4 + rev: v1.7.5 hooks: - id: docformatter args: ['--in-place', '--wrap-summaries=80', '--wrap-descriptions=80', '--pre-summary-newline'] - repo: https://github.com/MarcoGorelli/auto-walrus - rev: v0.2.2 + rev: 0.3.4 hooks: - id: auto-walrus - repo: https://github.com/pre-commit/mirrors-yapf diff --git a/tests/data/test_commit_report.py b/tests/data/test_commit_report.py index a05edc915..5f67fdc6e 100644 --- a/tests/data/test_commit_report.py +++ b/tests/data/test_commit_report.py @@ -16,7 +16,7 @@ generate_interactions, ) from varats.mapping.commit_map import CommitMap -from varats.project.project_util import get_local_project_git_path +from varats.project.project_util import get_local_project_repo from varats.projects.discover_projects import initialize_projects from varats.report.report import FileStatusExtension, ReportFilename from varats.utils.git_util import FullCommitHash, ShortCommitHash @@ -355,9 +355,9 @@ def testing_gen_commit_map() -> CommitMap: """Generate a local commit map for testing.""" initialize_projects() - xz_repo_path = get_local_project_git_path("xz") + xz_repo = get_local_project_repo("xz") return CommitMap( - xz_repo_path, + xz_repo, start="923bf96b55e5216a6c8df9d8331934f54784390e", end="80a1a8bb838842a2be343bd88ad1462c21c5e2c9" ) diff --git a/tests/paper_mgmt/test_case_study.py b/tests/paper_mgmt/test_case_study.py index 7a70b3325..2de18ab04 100644 --- a/tests/paper_mgmt/test_case_study.py +++ b/tests/paper_mgmt/test_case_study.py @@ -17,7 +17,7 @@ from varats.paper.paper_config import get_paper_config, load_paper_config from varats.plot.plots import PlotConfig from varats.plots.case_study_overview import CaseStudyOverviewPlot -from varats.project.project_util import get_local_project_git_path +from varats.project.project_util import get_local_project_repo from varats.projects.discover_projects import initialize_projects from varats.report.report import FileStatusExtension, ReportFilename from varats.utils.git_util import FullCommitHash, ShortCommitHash @@ -342,12 +342,12 @@ def test_extend_with_revs_per_year(self) -> None: random.seed(42) cs = CaseStudy("xz", 0) - git_path = get_local_project_git_path("xz") + repo = get_local_project_repo("xz") cmap = get_commit_map( "xz", end="c5c7ceb08a011b97d261798033e2c39613a69eb7" ) - MCS.extend_with_revs_per_year(cs, cmap, 0, True, str(git_path), 2, True) + MCS.extend_with_revs_per_year(cs, cmap, 0, True, repo, 2, True) self.assertEqual(cs.num_stages, 17) self.assertEqual(len(cs.revisions), 31) self.assertEqual( diff --git a/tests/provider/test_bug_provider.py b/tests/provider/test_bug_provider.py index e3f49beb9..f57233eaa 100644 --- a/tests/provider/test_bug_provider.py +++ b/tests/provider/test_bug_provider.py @@ -23,6 +23,7 @@ _filter_commit_message_bugs, ) from varats.provider.bug.bug_provider import BugProvider +from varats.utils.git_util import RepositoryHandle class DummyIssueData: @@ -164,8 +165,10 @@ def get(commit_id: str) -> pygit2.Commit: return mock_commit # pygit2 dummy repo - self.mock_repo = mock.create_autospec(pygit2.Repository) - self.mock_repo.get = get + self.mock_pygit = mock.create_autospec(pygit2.Repository) + self.mock_pygit.get = get + self.mock_repo_handle = mock.create_autospec(RepositoryHandle) + self.mock_repo_handle.pygit_repo = self.mock_pygit def test_issue_events_closing_bug(self) -> None: """Test identifying issue events that close a bug related issue, with @@ -257,7 +260,7 @@ def test_pygit_bug_creation(self, mock_pydriller_git) -> None: mock_pydriller_git.return_value = DummyPydrillerRepo("") pybug = _create_corresponding_bug( - self.mock_repo.get(issue_event.commit_id), self.mock_repo, + self.mock_pygit.get(issue_event.commit_id), self.mock_pygit, issue_event.issue.number ) @@ -265,9 +268,9 @@ def test_pygit_bug_creation(self, mock_pydriller_git) -> None: self.assertEqual(issue_event.issue.number, pybug.issue_id) @mock.patch('varats.provider.bug.bug.pydriller.Git') - @mock.patch('varats.provider.bug.bug.get_local_project_git') + @mock.patch('varats.provider.bug.bug.get_local_project_repo') def test_filter_issue_bugs( - self, mock_get_local_project_git, mock_pydriller_git + self, mock_get_local_project_repo, mock_pydriller_git ) -> None: """Test on a set of IssueEvents whether the corresponding set of bugs is created correctly.""" @@ -302,7 +305,7 @@ def test_filter_issue_bugs( event_close_second, event_close_first ] - mock_get_local_project_git.return_value = self.mock_repo + mock_get_local_project_repo.return_value = self.mock_repo_handle mock_pydriller_git.return_value = DummyPydrillerRepo("") # issue filter method for pygit bugs @@ -346,9 +349,9 @@ def accept_pybugs( self.assertEqual(expected_second_bug_intro_ids, intro_second_bug) @mock.patch('varats.provider.bug.bug.pydriller.Git') - @mock.patch('varats.provider.bug.bug.get_local_project_git') + @mock.patch('varats.provider.bug.bug.get_local_project_repo') def test_filter_commit_message_bugs( - self, mock_get_local_project_git, mock_pydriller_git + self, mock_get_local_project_repo, mock_pydriller_git ) -> None: """Test on the commit history of a project whether the corresponding set of bugs is created correctly.""" @@ -377,18 +380,18 @@ def mock_walk(_start_id: str, _sort_mode: int): ]) # customize walk method of mock repo - self.mock_repo.walk = mock.create_autospec( + self.mock_pygit.walk = mock.create_autospec( pygit2.Repository.walk, side_effect=mock_walk ) - mock_get_local_project_git.return_value = self.mock_repo + mock_get_local_project_repo.return_value = self.mock_repo_handle mock_pydriller_git.return_value = DummyPydrillerRepo("") # commit filter method for pygit bugs def accept_pybugs(repo: pygit2.Repository, commit: pygit2.Commit) -> tp.Optional[PygitBug]: if _is_closing_message(commit.message): - return _create_corresponding_bug(commit, self.mock_repo) + return _create_corresponding_bug(commit, self.mock_pygit) return None pybug_ids = set( diff --git a/tests/provider/test_patch_provider.py b/tests/provider/test_patch_provider.py index f19dae922..898406f31 100644 --- a/tests/provider/test_patch_provider.py +++ b/tests/provider/test_patch_provider.py @@ -4,7 +4,6 @@ import benchbuild as bb from benchbuild.source.base import target_prefix -from benchbuild.utils.revision_ranges import _get_git_for_path from tests.helper_utils import TEST_INPUTS_DIR from varats.projects.perf_tests.feature_perf_cs_collection import ( @@ -15,6 +14,7 @@ ShortCommitHash, get_all_revisions_between, get_initial_commit, + RepositoryHandle, ) @@ -76,12 +76,14 @@ def setUpClass(cls) -> None: project_git_source.fetch() - repo_git_path = Path(target_prefix() + "/FeaturePerfCSCollection") + repo = RepositoryHandle( + Path(target_prefix() + "/FeaturePerfCSCollection") + ) cls.all_revisions = set( get_all_revisions_between( - get_initial_commit(repo_git_path).hash, "", ShortCommitHash, - repo_git_path + repo, + get_initial_commit(repo).hash, "", ShortCommitHash ) ) diff --git a/tests/tools/test_driver_gen_benchbuild_config.py b/tests/tools/test_driver_gen_benchbuild_config.py index edd403bb5..301e49a39 100644 --- a/tests/tools/test_driver_gen_benchbuild_config.py +++ b/tests/tools/test_driver_gen_benchbuild_config.py @@ -14,7 +14,7 @@ class TestDriverGenBenchbuildConfig(unittest.TestCase): @run_in_test_environment() def test_gen_bbconfig(self): - """basic tests for the `gen-bbconfig` command.""" + """Basic tests for the `gen-bbconfig` command.""" runner = CliRunner() Path(vara_cfg()["benchbuild_root"].value + "/.benchbuild.yml").unlink() runner.invoke(driver_gen_benchbuild_config.main, []) diff --git a/tests/utils/test_git_util.py b/tests/utils/test_git_util.py index c11818665..f8db4b253 100644 --- a/tests/utils/test_git_util.py +++ b/tests/utils/test_git_util.py @@ -5,9 +5,9 @@ from benchbuild.utils.revision_ranges import RevisionRange, SingleRevision from varats.project.project_util import ( - get_local_project_git, - get_local_project_git_path, BinaryType, + get_local_project_repo, + RevisionBinaryMap, ) from varats.projects.discover_projects import initialize_projects from varats.utils.git_util import ( @@ -24,7 +24,6 @@ get_all_revisions_between, get_current_branch, get_initial_commit, - RevisionBinaryMap, get_submodule_head, calc_code_churn_range, RepositoryAtCommit, @@ -56,17 +55,17 @@ def test_is_commit_hash(self) -> None: def test_get_current_branch(self): """Check if we can correctly retrieve the current branch of a repo.""" - repo = get_local_project_git("brotli") + repo = get_local_project_repo("brotli") - repo.checkout(repo.lookup_branch('master')) + repo.pygit_repo.checkout(repo.pygit_repo.lookup_branch('master')) - self.assertEqual(get_current_branch(repo.workdir), 'master') + self.assertEqual(get_current_branch(repo), 'master') def test_get_initial_commit(self) -> None: """Check if we can correctly retrieve the inital commit of a repo.""" - repo_path = get_local_project_git_path("FeaturePerfCSCollection") + repo = get_local_project_repo("FeaturePerfCSCollection") - inital_commit = get_initial_commit(repo_path) + inital_commit = get_initial_commit(repo) self.assertEqual( FullCommitHash("4d84c8f80ec2db3aaa880d323f7666752c4be51d"), @@ -76,7 +75,7 @@ def test_get_initial_commit(self) -> None: def test_get_initial_commit_with_specified_path(self) -> None: """Check if we can correctly retrieve the inital commit of a repo.""" inital_commit = get_initial_commit( - get_local_project_git_path("FeaturePerfCSCollection") + get_local_project_repo("FeaturePerfCSCollection") ) self.assertEqual( @@ -86,11 +85,12 @@ def test_get_initial_commit_with_specified_path(self) -> None: def test_get_all_revisions_between_full(self): """Check if the correct all revisions are correctly found.""" - repo_path = get_local_project_git_path("brotli") + repo = get_local_project_repo("brotli") revs = get_all_revisions_between( + repo, '5692e422da6af1e991f9182345d58df87866bc5e', - '2f9277ff2f2d0b4113b1ffd9753cc0f6973d354a', FullCommitHash, - repo_path + '2f9277ff2f2d0b4113b1ffd9753cc0f6973d354a', + FullCommitHash, ) self.assertSetEqual( @@ -104,11 +104,12 @@ def test_get_all_revisions_between_full(self): def test_get_all_revisions_between_short(self): """Check if the correct all revisions are correctly found.""" - repo_path = get_local_project_git_path("brotli") + repo = get_local_project_repo("brotli") revs = get_all_revisions_between( + repo, '5692e422da6af1e991f9182345d58df87866bc5e', - '2f9277ff2f2d0b4113b1ffd9753cc0f6973d354a', ShortCommitHash, - repo_path + '2f9277ff2f2d0b4113b1ffd9753cc0f6973d354a', + ShortCommitHash, ) self.assertSetEqual( @@ -123,9 +124,11 @@ def test_get_all_revisions_between_short(self): def test_get_submodule_head(self): """Check if correct submodule commit is retrieved.""" repo_head = FullCommitHash("cb15dfa4b2d7fba0d50e87b49f979c7f996b8ebc") + repo = get_local_project_repo("grep") + submodule = get_local_project_repo("grep", "gnulib") - with RepositoryAtCommit("grep", repo_head.to_short_commit_hash()): - submodule_head = get_submodule_head("grep", "gnulib", repo_head) + with RepositoryAtCommit(repo, repo_head.to_short_commit_hash()): + submodule_head = get_submodule_head(repo, submodule, repo_head) self.assertEqual( submodule_head, FullCommitHash("f44eb378f7239eadac38d02463019a8a6b935525") @@ -134,18 +137,21 @@ def test_get_submodule_head(self): def test_get_submodule_head_main_repo(self): """Check if correct main repo commit is retrieved.""" repo_head = FullCommitHash("cb15dfa4b2d7fba0d50e87b49f979c7f996b8ebc") + correct_submodule_head = FullCommitHash( + "f44eb378f7239eadac38d02463019a8a6b935525" + ) + repo = get_local_project_repo("grep") + submodule = get_local_project_repo("grep", "gnulib") - with RepositoryAtCommit("grep", repo_head.to_short_commit_hash()): - submodule_head = get_submodule_head("grep", "grep", repo_head) - self.assertEqual(submodule_head, repo_head) + with RepositoryAtCommit(repo, repo_head.to_short_commit_hash()): + submodule_head = get_submodule_head(repo, submodule, repo_head) + self.assertEqual(submodule_head, correct_submodule_head) def test_get_commits_before_timestamp(self) -> None: """Check if we can correctly determine the commits before a specific timestamp.""" - project_repo = get_local_project_git_path('brotli') - brotli_commits_after = get_commits_before_timestamp( - '2013-10-24', project_repo - ) + repo = get_local_project_repo('brotli') + brotli_commits_after = get_commits_before_timestamp(repo, '2013-10-24') # newest found commit should be self.assertEqual( @@ -161,10 +167,8 @@ def test_get_commits_before_timestamp(self) -> None: def test_get_commits_after_timestamp(self) -> None: """Check if we can correctly determine the commits after a specific timestamp.""" - project_repo = get_local_project_git_path('brotli') - brotli_commits_after = get_commits_after_timestamp( - '2021-01-01', project_repo - ) + repo = get_local_project_repo('brotli') + brotli_commits_after = get_commits_after_timestamp(repo, '2021-01-01') # oldest found commit should be self.assertEqual( @@ -180,49 +184,55 @@ def test_get_commits_after_timestamp(self) -> None: def test_contains_source_code_without(self) -> None: """Check if we can correctly identify commits with source code.""" churn_conf = ChurnConfig.create_c_style_languages_config() - project_git_path = get_local_project_git_path('brotli') + repo = get_local_project_repo('brotli') self.assertFalse( contains_source_code( + repo, ShortCommitHash('f4153a09f87cbb9c826d8fc12c74642bb2d879ea'), - project_git_path, churn_conf + churn_conf ) ) self.assertFalse( contains_source_code( + repo, ShortCommitHash('e83c7b8e8fb8b696a1df6866bc46cbb76d7e0348'), - project_git_path, churn_conf + churn_conf ) ) self.assertFalse( contains_source_code( + repo, ShortCommitHash('698e3a7f9d3000fa44174f5be415bf713f71bd0e'), - project_git_path, churn_conf + churn_conf ) ) def test_contains_source_code_with(self) -> None: """Check if we can correctly identify commits without source code.""" churn_conf = ChurnConfig.create_c_style_languages_config() - project_git_path = get_local_project_git_path('brotli') + repo = get_local_project_repo('brotli') self.assertTrue( contains_source_code( + repo, ShortCommitHash('62662f87cdd96deda90ac817de94e3c4af75226a'), - project_git_path, churn_conf + churn_conf ) ) self.assertTrue( contains_source_code( + repo, ShortCommitHash('27dd7265403d8e8fed99a854b9c3e1db7d79525f'), - project_git_path, churn_conf + churn_conf ) ) # Merge commit of the previous one self.assertTrue( contains_source_code( + repo, ShortCommitHash('4ec67035c0d97c270c1c73038cc66fc5fcdfc120'), - project_git_path, churn_conf + churn_conf ) ) @@ -405,11 +415,10 @@ def setUpClass(cls): def test_one_commit_diff(self): """Check if we get the correct code churn for a single commit.""" - repo_path = get_local_project_git_path("brotli") + repo = get_local_project_repo("brotli") files_changed, insertions, deletions = calc_commit_code_churn( - repo_path, - FullCommitHash("0c5603e07bed1d5fbb45e38f9bdf0e4560fde3f0"), + repo, FullCommitHash("0c5603e07bed1d5fbb45e38f9bdf0e4560fde3f0"), ChurnConfig.create_c_style_languages_config() ) @@ -420,11 +429,10 @@ def test_one_commit_diff(self): def test_one_commit_diff_2(self): """Check if we get the correct code churn for a single commit.""" - repo_path = get_local_project_git_path("brotli") + repo = get_local_project_repo("brotli") files_changed, insertions, deletions = calc_commit_code_churn( - repo_path, - FullCommitHash("fc823290a76a260b7ba6f47ab5f52064a0ce19ff"), + repo, FullCommitHash("fc823290a76a260b7ba6f47ab5f52064a0ce19ff"), ChurnConfig.create_c_style_languages_config() ) @@ -435,11 +443,10 @@ def test_one_commit_diff_2(self): def test_one_commit_diff_3(self): """Check if we get the correct code churn for a single commit.""" - repo_path = get_local_project_git_path("brotli") + repo = get_local_project_repo("brotli") files_changed, insertions, deletions = calc_commit_code_churn( - repo_path, - FullCommitHash("924b2b2b9dc54005edbcd85a1b872330948cdd9e"), + repo, FullCommitHash("924b2b2b9dc54005edbcd85a1b872330948cdd9e"), ChurnConfig.create_c_style_languages_config() ) @@ -451,11 +458,10 @@ def test_one_commit_diff_ignore_non_c_cpp_files(self): """Check if we get the correct code churn for a single commit but only consider code changes.""" - repo_path = get_local_project_git_path("brotli") + repo = get_local_project_repo("brotli") files_changed, insertions, deletions = calc_commit_code_churn( - repo_path, - FullCommitHash("f503cb709ca181dbf5c73986ebac1b18ac5c9f63"), + repo, FullCommitHash("f503cb709ca181dbf5c73986ebac1b18ac5c9f63"), ChurnConfig.create_c_style_languages_config() ) @@ -466,10 +472,10 @@ def test_one_commit_diff_ignore_non_c_cpp_files(self): def test_start_with_initial_commit(self): """Check if the initial commit is handled correctly.""" - repo_path = get_local_project_git_path("brotli") + repo = get_local_project_repo("brotli") churn = calc_code_churn_range( - repo_path, ChurnConfig.create_c_style_languages_config(), + repo, ChurnConfig.create_c_style_languages_config(), FullCommitHash("8f30907d0f2ef354c2b31bdee340c2b11dda0fb0"), FullCommitHash("8f30907d0f2ef354c2b31bdee340c2b11dda0fb0") ) @@ -483,10 +489,10 @@ def test_start_with_initial_commit(self): def test_end_only(self): """Check if churn is correct if only end range is set.""" - repo_path = get_local_project_git_path("brotli") + repo = get_local_project_repo("brotli") churn = calc_code_churn_range( - repo_path, ChurnConfig.create_c_style_languages_config(), None, + repo, ChurnConfig.create_c_style_languages_config(), None, FullCommitHash("645552217219c2877780ba4d7030044ec62d8255") ) @@ -506,11 +512,10 @@ def test_end_only(self): def test_commit_range(self): """Check if we get the correct code churn for commit range.""" - repo_path = get_local_project_git_path("brotli") + repo = get_local_project_repo("brotli") files_changed, insertions, deletions = calc_code_churn( - repo_path, - FullCommitHash("36ac0feaf9654855ee090b1f042363ecfb256f31"), + repo, FullCommitHash("36ac0feaf9654855ee090b1f042363ecfb256f31"), FullCommitHash("924b2b2b9dc54005edbcd85a1b872330948cdd9e"), ChurnConfig.create_c_style_languages_config() ) @@ -527,7 +532,7 @@ class TestRevisionBinaryMap(unittest.TestCase): def setUp(self) -> None: self.rv_map = RevisionBinaryMap( - get_local_project_git_path("FeaturePerfCSCollection") + get_local_project_repo("FeaturePerfCSCollection") ) def test_specification_of_always_valid_binaries(self) -> None: diff --git a/tests/utils/test_project_util.py b/tests/utils/test_project_util.py index 0cf3734a2..5bec0bddd 100644 --- a/tests/utils/test_project_util.py +++ b/tests/utils/test_project_util.py @@ -14,7 +14,7 @@ ProjectBinaryWrapper, BinaryType, get_tagged_commits, - get_local_project_git_path, + get_local_project_repo, ) from varats.projects.c_projects.gravity import Gravity from varats.projects.discover_projects import initialize_projects @@ -277,8 +277,8 @@ def test_get_tagged_commits_lightweight(self) -> None: """Check if we can get list of tagged commits from a project when lightweight tags are used.""" fast_downward_tagged_commits = set(get_tagged_commits("FastDownward")) - fast_downward_repo_loc = get_local_project_git_path("FastDownward") - with local.cwd(fast_downward_repo_loc): + fast_downward_repo = get_local_project_repo("FastDownward") + with local.cwd(fast_downward_repo.worktree_path): for (hash_value, _) in fast_downward_tagged_commits: self.assertTrue(self.hash_belongs_to_commit(hash_value)) @@ -298,8 +298,8 @@ def test_get_tagged_commits_annotated(self) -> None: """Check if we can get list of tagged commits from a project when annotated tags are used.""" xz_tagged_commits = set(get_tagged_commits("xz")) - xz_repo_loc = get_local_project_git_path("xz") - with local.cwd(xz_repo_loc): + xz_repo = get_local_project_repo("xz") + with local.cwd(xz_repo.worktree_path): for (hash_value, _) in xz_tagged_commits: self.assertTrue(self.hash_belongs_to_commit(hash_value)) diff --git a/varats-core/varats/experiment/steps/patch.py b/varats-core/varats/experiment/steps/patch.py index 461cf13e7..4d687c754 100644 --- a/varats-core/varats/experiment/steps/patch.py +++ b/varats-core/varats/experiment/steps/patch.py @@ -8,6 +8,7 @@ from varats.project.varats_project import VProject from varats.provider.patch.patch_provider import Patch from varats.utils.git_commands import apply_patch, revert_patch +from varats.utils.git_util import RepositoryHandle class ApplyPatch(actions.ProjectStep): @@ -27,7 +28,10 @@ def __call__(self) -> StepResult: f"Applying {self.__patch.shortname} to " f"{self.project.source_of_primary}" ) - apply_patch(Path(self.project.source_of_primary), self.__patch.path) + apply_patch( + RepositoryHandle(Path(self.project.source_of_primary)), + self.__patch.path + ) except ProcessExecutionError: self.status = StepResult.ERROR @@ -59,7 +63,8 @@ def __call__(self) -> StepResult: f"{self.project.source_of_primary}" ) revert_patch( - Path(self.project.source_of_primary), self.__patch.path + RepositoryHandle(Path(self.project.source_of_primary)), + self.__patch.path ) except ProcessExecutionError: diff --git a/varats-core/varats/mapping/author_map.py b/varats-core/varats/mapping/author_map.py index b64af1b76..8ec35c72b 100644 --- a/varats-core/varats/mapping/author_map.py +++ b/varats-core/varats/mapping/author_map.py @@ -5,10 +5,7 @@ import typing as tp from functools import reduce -from benchbuild.utils.cmd import git - -from varats.project.project_util import get_local_project_git_path -from varats.utils.git_util import __get_git_path_arg +from varats.project.project_util import get_local_project_repo LOG = logging.getLogger(__name__) @@ -163,10 +160,9 @@ def __repr__(self) -> str: def generate_author_map(project_name: str) -> AuthorMap: """Generate an AuthorMap for the repository at the given path.""" - path = get_local_project_git_path(project_name) + repo = get_local_project_repo(project_name) author_map = AuthorMap() - test = git[__get_git_path_arg(path), "shortlog", "-sne", - "--all"]().strip().split("\n") + test = repo("shortlog", "-sne", "--all").strip().split("\n") for line in test: match = NAME_REGEX.match(line) if not match: diff --git a/varats-core/varats/mapping/commit_map.py b/varats-core/varats/mapping/commit_map.py index ed7b24388..b83c5b5e9 100644 --- a/varats-core/varats/mapping/commit_map.py +++ b/varats-core/varats/mapping/commit_map.py @@ -2,20 +2,18 @@ import logging import typing as tp from collections.abc import ItemsView -from pathlib import Path -from benchbuild.utils.cmd import git, mkdir -from plumbum import local from pygtrie import CharTrie from varats.project.project_util import ( - get_local_project_git_path, get_primary_project_source, + get_local_project_repo, ) from varats.utils.git_util import ( get_current_branch, FullCommitHash, ShortCommitHash, + RepositoryHandle, ) LOG = logging.getLogger(__name__) @@ -30,12 +28,12 @@ class CommitMap(): def __init__( self, - git_path: Path, + repo: RepositoryHandle, end: str = "HEAD", start: tp.Optional[str] = None, refspec: str = "HEAD" ) -> None: - self.git_path = git_path + self.repo = repo self.end = end self.start = start self.refspec = refspec @@ -60,38 +58,39 @@ def generate_hash_to_id(self, master: bool = False) -> CharTrie: search_range += self.start + ".." search_range += self.end - with local.cwd(self.git_path): - old_head = get_current_branch() - git("checkout", self.refspec) - full_out = git("--no-pager", "log", "--all", "--pretty=format:'%H'") - if master: - wanted_out = git( - "--no-pager", "log", "--pretty=format:'%H'", search_range - ) - else: - wanted_out = git( - "--no-pager", "log", "--all", "--pretty=format:'%H'", - search_range - ) - - def format_stream() -> tp.Generator[str, None, None]: - wanted_cm = set() - for line in wanted_out.split('\n'): - wanted_cm.add(line[1:-1]) - - for number, line in enumerate(reversed(full_out.split('\n'))): - line = line[1:-1] - if line in wanted_cm: - yield f"{number}, {line}\n" - - hash_to_id: CharTrie = CharTrie() - for line in format_stream(): - slices = line.strip().split(', ') - hash_to_id[slices[1]] = int(slices[0]) - - git("checkout", old_head) - - return hash_to_id + old_head = get_current_branch(self.repo) + self.repo("checkout", self.refspec) + full_out = self.repo( + "--no-pager", "log", "--all", "--pretty=format:'%H'" + ) + if master: + wanted_out = self.repo( + "--no-pager", "log", "--pretty=format:'%H'", search_range + ) + else: + wanted_out = self.repo( + "--no-pager", "log", "--all", "--pretty=format:'%H'", + search_range + ) + + def format_stream() -> tp.Generator[str, None, None]: + wanted_cm = set() + for line in wanted_out.split('\n'): + wanted_cm.add(line[1:-1]) + + for number, line in enumerate(reversed(full_out.split('\n'))): + line = line[1:-1] + if line in wanted_cm: + yield f"{number}, {line}\n" + + hash_to_id: CharTrie = CharTrie() + for line in format_stream(): + slices = line.strip().split(', ') + hash_to_id[slices[1]] = int(slices[0]) + + self.repo("checkout", old_head) + + return hash_to_id def convert_to_full_or_warn( self, short_commit: ShortCommitHash @@ -222,11 +221,11 @@ def get_commit_map( Returns: a bidirectional commit map from commits to time IDs """ - project_git_path = get_local_project_git_path(project_name) + project_repo = get_local_project_repo(project_name) primary_source = get_primary_project_source(project_name) if refspec is None and hasattr(primary_source, "refspec"): refspec = primary_source.refspec elif refspec is None: refspec = "HEAD" - return CommitMap(project_git_path, end, start, refspec) + return CommitMap(project_repo, end, start, refspec) diff --git a/varats-core/varats/project/project_util.py b/varats-core/varats/project/project_util.py index a4c27d74d..5df259498 100644 --- a/varats-core/varats/project/project_util.py +++ b/varats-core/varats/project/project_util.py @@ -2,16 +2,31 @@ import logging import os import typing as tp +from collections import defaultdict from enum import Enum +from itertools import chain from pathlib import Path import benchbuild as bb import pygit2 +from _operator import attrgetter from benchbuild.source import Git -from benchbuild.utils.cmd import git +from benchbuild.utils.revision_ranges import AbstractRevisionRange from plumbum import local from plumbum.commands.base import BoundCommand +from varats.utils.git_util import ( + RepositoryHandle, + FullCommitHash, + num_commits, + get_submodule_commits, + get_authors, + calc_repo_loc, + CommitHash, + ShortCommitHash, + CommitRepoPair, + CommitLookupTy, +) from varats.utils.settings import bb_cfg LOG = logging.getLogger(__name__) @@ -52,23 +67,9 @@ def get_primary_project_source(project_name: str) -> bb.source.FetchableSource: return bb.source.primary(*project_cls.SOURCE) -def get_local_project_git_path( +def get_local_project_repo( project_name: str, git_name: tp.Optional[str] = None -) -> Path: - """ - Get the path to the local download location of a git repository for a given - benchbuild project. - - Args: - project_name: name of the given benchbuild project - git_name: name of the git repository, i.e., the name of the repository - folder. If no git_name is provided, the name of the primary - source is used. - - Returns: - Path to the local download location of the git repository. - """ - +) -> RepositoryHandle: if git_name: source = get_extended_commit_lookup_source(project_name, git_name) else: @@ -83,7 +84,32 @@ def get_local_project_git_path( git_path = base / source.local.replace(os.sep, "-") if not git_path.exists(): git_path = Path(source.fetch()) - return git_path + return RepositoryHandle(git_path) + + +def get_local_project_repos( + project_name: str +) -> tp.Dict[str, RepositoryHandle]: + """ + Get the all git repositories for a given benchbuild project. + + Args: + project_name: name of the given benchbuild project + + Returns: + dict with the repository handles for the project's git sources + """ + repos: tp.Dict[str, RepositoryHandle] = {} + project_cls = get_project_cls_by_name(project_name) + + for source in project_cls.SOURCE: + if isinstance(source, Git): + source_name = os.path.basename(source.local) + repos[source_name] = get_local_project_repo( + project_name, source_name + ) + + return repos def get_extended_commit_lookup_source( @@ -111,91 +137,132 @@ def get_extended_commit_lookup_source( ) -def get_local_project_git( - project_name: str, git_name: tp.Optional[str] = None -) -> pygit2.Repository: +def create_project_commit_lookup_helper(project_name: str) -> CommitLookupTy: """ - Get the git repository for a given benchbuild project. + Creates a commit lookup function for project repositories. Args: project_name: name of the given benchbuild project - git_name: name of the git repository Returns: - git repository that matches the given git_name. + a Callable that maps a commit hash and repository name to the + corresponding commit. """ - git_path = get_local_project_git_path(project_name, git_name) - repo_path = pygit2.discover_repository(str(git_path)) - return pygit2.Repository(repo_path) + repos = get_local_project_repos(project_name) -def get_local_project_gits( - project_name: str -) -> tp.Dict[str, pygit2.Repository]: + def get_commit(crp: CommitRepoPair) -> pygit2.Commit: + """ + Gets the commit from a given ``CommitRepoPair``. + + Args: + crp: the ``CommitRepoPair`` for the commit to get + + Returns: + the commit corresponding to the given CommitRepoPair + """ + commit = repos[crp.repository_name].pygit_repo.get(crp.commit_hash.hash) + if not commit: + raise LookupError( + f"Could not find commit {crp} for project {project_name}." + ) + + return commit + + return get_commit + + +def get_tagged_commits(project_name: str) -> tp.List[tp.Tuple[str, str]]: + """Get a list of all tagged commits along with their respective tags.""" + repo = get_local_project_repo(project_name) + # --dereference resolves tag IDs into commits for annotated tags + # These lines are indicated by the suffix '^{}' (see man git-show-ref) + ref_list: tp.List[str] = repo("show-ref", "--tags", + "--dereference").strip().split("\n") + + # Only keep dereferenced or leightweight tags (i.e., only keep commits) + # and strip suffix, if necessary + refs: tp.List[tp.Tuple[str, str]] = [ + (ref_split[0], ref_split[1][10:].replace('^{}', '')) + for ref_split in [ref.strip().split() for ref in ref_list] + if repo("cat-file", "-t", ref_split[1][10:]).replace('\n', '' + ) == 'commit' + ] + + return refs + + +def num_project_commits(project_name: str, revision: FullCommitHash) -> int: """ - Get the all git repositories for a given benchbuild project. + Calculate the number of commits of a project including submodules. Args: - project_name: name of the given benchbuild project + project_name: name of the project to calculate commits for + revision: revision to calculate commits at Returns: - dict with the git repositories for the project's sources + the number of commits in the project """ - repos: tp.Dict[str, pygit2.Repository] = {} - project_cls = get_project_cls_by_name(project_name) - - for source in project_cls.SOURCE: - if isinstance(source, Git): - source_name = os.path.basename(source.local) - repos[source_name] = get_local_project_git( - project_name, source_name - ) - - return repos + project_repos = get_local_project_repos(project_name) + main_repo = get_local_project_repo(project_name) + + commits = num_commits(main_repo, revision.hash) + for submodule, sub_rev in get_submodule_commits(main_repo, + revision.hash).items(): + if submodule not in project_repos: + LOG.warning("Ignoring unknown submodule %s",) + continue + commits += num_commits(project_repos[submodule], sub_rev.hash) + return commits -def get_local_project_git_paths(project_name: str) -> tp.Dict[str, Path]: +def num_project_authors(project_name: str, revision: FullCommitHash) -> int: """ - Get the all paths to the git repositories for a given benchbuild project. + Calculate the number of authors of a project including submodules. Args: - project_name: name of the given benchbuild project + project_name: name of the project to calculate authors for + revision: revision to authors commits at Returns: - dict with the paths to the git repositories for the project's sources + the number of authors in the project """ - repos: tp.Dict[str, Path] = {} - project_cls = get_project_cls_by_name(project_name) - for source in project_cls.SOURCE: - if isinstance(source, Git): - source_name = os.path.basename(source.local) - repos[source_name] = get_local_project_git_path( - project_name, source_name - ) + project_repos = get_local_project_repos(project_name) + main_repo = get_local_project_repo(project_name) - return repos + authors = get_authors(main_repo, revision.hash) + for submodule, sub_rev in get_submodule_commits(main_repo, + revision.hash).items(): + if submodule not in project_repos: + LOG.warning("Ignoring unknown submodule %s", submodule) + continue + authors.update(get_authors(project_repos[submodule], sub_rev.hash)) + return len(authors) -def get_tagged_commits(project_name: str) -> tp.List[tp.Tuple[str, str]]: - """Get a list of all tagged commits along with their respective tags.""" - repo_loc = get_local_project_git_path(project_name) - with local.cwd(repo_loc): - # --dereference resolves tag IDs into commits for annotated tags - # These lines are indicated by the suffix '^{}' (see man git-show-ref) - ref_list: tp.List[str] = git("show-ref", "--tags", - "--dereference").strip().split("\n") +def calc_project_loc(project_name: str, revision: FullCommitHash) -> int: + """ + Calculate the LOC for a project including submodules at the given revision. - # Only keep dereferenced or leightweight tags (i.e., only keep commits) - # and strip suffix, if necessary - refs: tp.List[tp.Tuple[str, str]] = [ - (ref_split[0], ref_split[1][10:].replace('^{}', '')) - for ref_split in [ref.strip().split() for ref in ref_list] - if git("cat-file", "-t", ref_split[1][10:]).replace('\n', '' - ) == 'commit' - ] + Args: + project_name: name of the project to calculate LOC for + revision: revision to calculate LOC at - return refs + Returns: + the LOC in the project + """ + project_repos = get_local_project_repos(project_name) + main_repo = get_local_project_repo(project_name) + + loc = calc_repo_loc(main_repo, revision.hash) + for submodule, sub_rev in get_submodule_commits(main_repo, + revision.hash).items(): + if submodule not in project_repos: + LOG.warning("Ignoring unknown submodule %s", submodule) + continue + loc += calc_repo_loc(project_repos[submodule], sub_rev.hash) + return loc def is_git_source(source: bb.source.FetchableSource) -> bool: @@ -340,6 +407,91 @@ def verify_binaries(project: bb.Project) -> None: raise BinaryNotFound.create_error_for_binary(binary) +class RevisionBinaryMap(tp.Container[str]): + """A special map that specifies for which revision ranges a binaries is + valid.""" + + def __init__(self, repo: RepositoryHandle) -> None: + self.__repo_location = repo.worktree_path + self.__revision_specific_mappings: tp.Dict[ + 'AbstractRevisionRange', + tp.List[ProjectBinaryWrapper]] = defaultdict(list) + self.__always_valid_mappings: tp.List[ProjectBinaryWrapper] = [] + + def specify_binary( + self, location: str, binary_type: BinaryType, **kwargs: tp.Any + ) -> 'RevisionBinaryMap': + """ + Add a binary to the map. + + Args: + location: where the binary can be found, relative to the + project-source root + binary_type: the type of binary that is produced + override_binary_name: overrides the used binary name + override_entry_point: overrides the executable entry point + only_valid_in: additionally specifies a validity range that + specifies in which revision range this binary is + produced + + Returns: + self for builder-style usage + """ + binary_location_path = Path(location) + binary_name: str = kwargs.get( + "override_binary_name", binary_location_path.stem + ) + override_entry_point = kwargs.get("override_entry_point", None) + if override_entry_point: + override_entry_point = Path(override_entry_point) + validity_range: tp.Optional[AbstractRevisionRange] = kwargs.get( + "only_valid_in", None + ) + valid_exit_codes = kwargs.get("valid_exit_codes", None) + + wrapped_binary = ProjectBinaryWrapper( + binary_name, binary_location_path, binary_type, + override_entry_point, valid_exit_codes + ) + + if validity_range: + validity_range.init_cache(str(self.__repo_location)) + self.__revision_specific_mappings[validity_range].append( + wrapped_binary + ) + else: + self.__always_valid_mappings.append(wrapped_binary) + + return self + + def __getitem__(self, + revision: CommitHash) -> tp.List[ProjectBinaryWrapper]: + revision = revision.to_short_commit_hash() + revision_specific_binaries = [] + + for validity_range, wrapped_binaries \ + in self.__revision_specific_mappings.items(): + if revision in map(ShortCommitHash, validity_range): + revision_specific_binaries.extend(wrapped_binaries) + + revision_specific_binaries.extend(self.__always_valid_mappings) + + return sorted( + revision_specific_binaries, key=attrgetter("name", "path") + ) + + def __contains__(self, binary_name: object) -> bool: + if isinstance(binary_name, str): + for binary in chain( + self.__always_valid_mappings, + *self.__revision_specific_mappings.values() + ): + if binary.name == binary_name: + return True + + return False + + def copy_renamed_git_to_dest(src_dir: Path, dest_dir: Path) -> None: """ Renames git files that were made git_storable (e.g., .gitted) back to their diff --git a/varats-core/varats/provider/bug/bug.py b/varats-core/varats/provider/bug/bug.py index f54843af5..a9431f092 100644 --- a/varats-core/varats/provider/bug/bug.py +++ b/varats-core/varats/provider/bug/bug.py @@ -9,8 +9,8 @@ from github.IssueEvent import IssueEvent from varats.project.project_util import ( - get_local_project_git, get_project_cls_by_name, + get_local_project_repo, ) from varats.utils.git_util import FullCommitHash from varats.utils.github_util import ( @@ -294,7 +294,7 @@ def _find_corresponding_pygit_suspect_tuple( A PygitSuspectTuple if the issue event represents the closing of a bug, None otherwise """ - pygit_repo: pygit2.Repository = get_local_project_git(project_name) + pygit_repo = get_local_project_repo(project_name).pygit_repo pydrill_repo = pydriller.Git(pygit_repo.path) if _has_closed_a_bug(issue_event) and issue_event.commit_id: @@ -400,7 +400,7 @@ def _filter_commit_message_bugs( the set of bugs created by the given filter """ filtered_bugs = set() - project_repo = get_local_project_git(project_name) + project_repo = get_local_project_repo(project_name).pygit_repo for commit in project_repo.walk( project_repo.head.target, pygit2.GIT_SORT_TIME diff --git a/varats-core/varats/provider/cve/cve.py b/varats-core/varats/provider/cve/cve.py index 58bcc50f8..482a41cc5 100644 --- a/varats-core/varats/provider/cve/cve.py +++ b/varats-core/varats/provider/cve/cve.py @@ -25,7 +25,8 @@ class CVE: CVE representation with the major fields. Mainly a data object to store everything. Uses the API at - https://cve.circl.lu/api/search/ to find entries. + https://cve.circl.lu/api/search/ + to find entries. """ def __init__( diff --git a/varats-core/varats/provider/cve/cve_map.py b/varats-core/varats/provider/cve/cve_map.py index f7f4c9f23..adc4c3de3 100644 --- a/varats-core/varats/provider/cve/cve_map.py +++ b/varats-core/varats/provider/cve/cve_map.py @@ -45,7 +45,7 @@ find_cve, find_cwe, ) -from varats.utils.git_util import FullCommitHash +from varats.utils.git_util import FullCommitHash, RepositoryHandle LOG = logging.getLogger(__name__) @@ -234,7 +234,7 @@ def __merge_results(result_list: tp.List[CVEDict]) -> CVEDict: def generate_cve_map( - path: Path, + repo: RepositoryHandle, products: tp.List[tp.Tuple[str, str]], end: str = "HEAD", start: tp.Optional[str] = None, @@ -273,34 +273,30 @@ def split_commit_info(commit_info: str) -> tp.Tuple[FullCommitHash, str]: search_range += start + ".." search_range += end - with local.cwd(path): - commits = git( - "--no-pager", "log", "--pretty=format:'%H %d %s'", search_range - ) - wanted_out = list( - map(split_commit_info, [x[1:-1] for x in commits.split('\n')]) - ) - - def get_results_for_product(vendor: str, product: str) -> CVEDict: - cve_list = find_all_cve(vendor=vendor, product=product) - cve_maps = [ - __collect_via_commit_mgs(commits=wanted_out), - __collect_via_references( - commits=wanted_out, - cve_list=cve_list, - vendor=vendor, - product=product - ) - ] - if not only_precise: - cve_maps.append( - __collect_via_version( - commits=wanted_out, cve_list=cve_list - ), - ) - return __merge_results(cve_maps) - - return __merge_results([ - get_results_for_product(vendor, product) - for vendor, product in products - ]) + commits = repo( + "--no-pager", "log", "--pretty=format:'%H %d %s'", search_range + ) + wanted_out = list( + map(split_commit_info, [x[1:-1] for x in commits.split('\n')]) + ) + + def get_results_for_product(vendor: str, product: str) -> CVEDict: + cve_list = find_all_cve(vendor=vendor, product=product) + cve_maps = [ + __collect_via_commit_mgs(commits=wanted_out), + __collect_via_references( + commits=wanted_out, + cve_list=cve_list, + vendor=vendor, + product=product + ) + ] + if not only_precise: + cve_maps.append( + __collect_via_version(commits=wanted_out, cve_list=cve_list), + ) + return __merge_results(cve_maps) + + return __merge_results([ + get_results_for_product(vendor, product) for vendor, product in products + ]) diff --git a/varats-core/varats/provider/cve/cve_provider.py b/varats-core/varats/provider/cve/cve_provider.py index 6bec46601..376e51f89 100644 --- a/varats-core/varats/provider/cve/cve_provider.py +++ b/varats-core/varats/provider/cve/cve_provider.py @@ -4,7 +4,7 @@ from benchbuild.project import Project -from varats.project.project_util import get_local_project_git_path +from varats.project.project_util import get_local_project_repo from varats.provider.cve.cve import CVE from varats.provider.cve.cve_map import generate_cve_map, CVEDict from varats.provider.provider import Provider @@ -38,7 +38,7 @@ def __init__(self, project: tp.Type[Project]) -> None: project_name = project.NAME if issubclass(project, CVEProviderHook): self.__cve_map: CVEDict = generate_cve_map( - get_local_project_git_path(project_name), + get_local_project_repo(project_name), project.get_cve_product_info() ) else: diff --git a/varats-core/varats/provider/patch/patch_provider.py b/varats-core/varats/provider/patch/patch_provider.py index 9cc802da4..c69ce51db 100644 --- a/varats-core/varats/provider/patch/patch_provider.py +++ b/varats-core/varats/provider/patch/patch_provider.py @@ -16,7 +16,7 @@ from benchbuild.source.base import target_prefix from yaml import YAMLError -from varats.project.project_util import get_local_project_git_path +from varats.project.project_util import get_local_project_repo from varats.provider.provider import Provider, ProviderType from varats.utils.filesystem_util import lock_file from varats.utils.git_commands import pull_current_branch, fetch_repository @@ -25,6 +25,7 @@ ShortCommitHash, get_all_revisions_between, get_initial_commit, + RepositoryHandle, ) @@ -80,7 +81,7 @@ def from_yaml(yaml_path: Path) -> 'Patch': tags = yaml_dict.get("tags") feature_tags = yaml_dict.get("feature_tags") - project_git_path = get_local_project_git_path(project_name) + project_repo = get_local_project_repo(project_name) def parse_revisions( rev_dict: tp.Dict[str, tp.Any] @@ -106,8 +107,8 @@ def parse_revisions( end_rev = "" res.update( get_all_revisions_between( - rev_range["start"], end_rev, ShortCommitHash, - project_git_path + project_repo, rev_range["start"], end_rev, + ShortCommitHash ) ) @@ -119,8 +120,8 @@ def parse_revisions( else: include_revisions = set( get_all_revisions_between( - get_initial_commit(project_git_path).hash, "", - ShortCommitHash, project_git_path + project_repo, + get_initial_commit(project_repo).hash, "", ShortCommitHash ) ) @@ -285,9 +286,9 @@ def __init__(self, project: tp.Type[Project]): super().__init__(project) self._update_local_patches_repo() - repo_path = self._get_patches_repository_path() + repo = self._get_patches_repository() - patches_project_dir = repo_path / self.project.NAME + patches_project_dir = repo.worktree_path / self.project.NAME if not patches_project_dir.is_dir(): warnings.warn( @@ -298,8 +299,8 @@ def __init__(self, project: tp.Type[Project]): self.__patches: tp.Set[Patch] = set() # Update repository to have all upstream changes - project_git_path = get_local_project_git_path(self.project.NAME) - fetch_repository(project_git_path) + project_repo = get_local_project_repo(self.project.NAME) + fetch_repository(project_repo) for root, _, files in os.walk(patches_project_dir): for filename in files: @@ -364,9 +365,10 @@ def create_default_provider( ) @classmethod - def _get_patches_repository_path(cls) -> Path: - # pathlib doesn't have type annotations for '/' - return tp.cast(Path, Path(target_prefix()) / cls.patches_source.local) + def _get_patches_repository(cls) -> RepositoryHandle: + return RepositoryHandle( + Path(target_prefix()) / cls.patches_source.local + ) @classmethod def _update_local_patches_repo(cls) -> None: @@ -374,4 +376,4 @@ def _update_local_patches_repo(cls) -> None: with lock_file(lock_path): cls.patches_source.fetch() - pull_current_branch(cls._get_patches_repository_path()) + pull_current_branch(cls._get_patches_repository()) diff --git a/varats-core/varats/provider/release/release_provider.py b/varats-core/varats/provider/release/release_provider.py index 8190eb792..9b9dbe71e 100644 --- a/varats-core/varats/provider/release/release_provider.py +++ b/varats-core/varats/provider/release/release_provider.py @@ -112,8 +112,9 @@ class ReleaseDefaultProvider(ReleaseProvider): Default implementation of the :class:`ReleaseProvider` for projects that do not need or support their own implementation. - This implementation looks for commits with tags that are `PEP 440 - `_ versions. + This implementation looks for commits with tags that are + `PEP 440 `_ + versions. """ def __init__(self, project: tp.Type[Project]) -> None: diff --git a/varats-core/varats/utils/git_commands.py b/varats-core/varats/utils/git_commands.py index 3eb89b839..88999044a 100644 --- a/varats-core/varats/utils/git_commands.py +++ b/varats-core/varats/utils/git_commands.py @@ -4,39 +4,46 @@ from benchbuild.utils.cmd import git -from varats.utils.git_util import get_current_branch, CommitHash +from varats.utils.git_util import ( + get_current_branch, + CommitHash, + RepositoryHandle, +) -def add_remote(repo_folder: Path, remote: str, url: str) -> None: +def add_remote(repo: RepositoryHandle, remote: str, url: str) -> None: """Adds new remote to the repository.""" - git(["-C", repo_folder.absolute(), "remote", "add", remote, url]) + repo("remote", "add", remote, url) -def show_status(repo_folder: Path) -> None: +def show_status(repo: RepositoryHandle) -> None: """Show git status.""" - git["-C", repo_folder.absolute(), "status"].run_fg() + repo("status") def get_branches( - repo_folder: Path, extra_args: tp.Optional[tp.List[str]] = None + repo: RepositoryHandle, + extra_args: tp.Optional[tp.List[str]] = None ) -> str: """Show git branches.""" args = ["branch"] if extra_args: args += extra_args - return tp.cast(str, git("-C", repo_folder.absolute(), args)) + return tp.cast(str, repo(*args)) -def get_tags(repo_folder: Path, - extra_args: tp.Optional[tp.List[str]] = None) -> tp.List[str]: +def get_tags( + repo: RepositoryHandle, + extra_args: tp.Optional[tp.List[str]] = None +) -> tp.List[str]: """Get the list of available git tags.""" args = ["tag"] if extra_args: args += extra_args - git_tag_string: str = git("-C", repo_folder.absolute(), args) + git_tag_string: str = repo(*args) git_tag_list: tp.List[str] = [] if git_tag_string: @@ -47,13 +54,13 @@ def get_tags(repo_folder: Path, return git_tag_list -def init_all_submodules(folder: Path) -> None: +def init_all_submodules(repo: RepositoryHandle) -> None: """Inits all submodules.""" - git("-C", folder.absolute(), "submodule", "init") + repo("submodule", "init") def update_all_submodules( - folder: Path, recursive: bool = True, init: bool = False + repo: RepositoryHandle, recursive: bool = True, init: bool = False ) -> None: """Updates all submodules.""" git_params = ["submodule", "update"] @@ -61,12 +68,12 @@ def update_all_submodules( git_params.append("--recursive") if init: git_params.append("--init") - git("-C", folder, git_params) + repo(*git_params) def fetch_remote( + repo: RepositoryHandle, remote: tp.Optional[str] = None, - repo_folder: tp.Optional[Path] = None, extra_args: tp.Optional[tp.List[str]] = None ) -> None: """Fetches the new changes from the remote.""" @@ -75,16 +82,16 @@ def fetch_remote( args += extra_args if remote: args.append(remote) - git("-C", repo_folder, args) + repo(*args) -def pull_current_branch(repo_folder: Path) -> None: +def pull_current_branch(repo: RepositoryHandle) -> None: """Pull in changes in a certain branch.""" - git("-C", repo_folder.absolute(), "pull") + repo("pull") def push_current_branch( - repo_folder: tp.Optional[Path] = None, + repo: RepositoryHandle, upstream: tp.Optional[str] = None, branch_name: tp.Optional[str] = None ) -> None: @@ -97,28 +104,25 @@ def push_current_branch( if branch_name is not None: cmd_args.append(branch_name) else: - cmd_args.append(get_current_branch(repo_folder)) + cmd_args.append(get_current_branch(repo)) - if repo_folder is None or repo_folder == Path(""): - git(cmd_args) - else: - git("-C", repo_folder.absolute(), cmd_args) + repo(*cmd_args) -def fetch_repository(repo_folder: tp.Optional[Path] = None) -> None: +def fetch_repository(repo: RepositoryHandle) -> None: """Pull in changes in a certain branch.""" - git("-C", repo_folder, "fetch") + repo("fetch") def checkout_branch_or_commit( - repo_folder: Path, target: tp.Union[str, CommitHash] + repo: RepositoryHandle, target: tp.Union[str, CommitHash] ) -> None: """Checks out a branch or commit in the repository.""" - git("-C", repo_folder.absolute(), "checkout", str(target)) + repo("checkout", str(target)) def checkout_new_branch( - repo_folder: Path, + repo: RepositoryHandle, branch: str, remote_branch: tp.Optional[str] = None ) -> None: @@ -126,7 +130,7 @@ def checkout_new_branch( args = ["checkout", "-b", branch] if remote_branch is not None: args.append(remote_branch) - git("-C", repo_folder.absolute(), args) + repo(*args) def download_repo( @@ -153,11 +157,11 @@ def download_repo( post_out(line) -def apply_patch(repo_folder: Path, patch_file: Path) -> None: +def apply_patch(repo: RepositoryHandle, patch_file: Path) -> None: """Applies a given patch file to the specified git repository.""" - git("-C", repo_folder.absolute(), "apply", str(patch_file)) + repo("apply", str(patch_file)) -def revert_patch(repo_folder: Path, patch_file: Path) -> None: +def revert_patch(repo: RepositoryHandle, patch_file: Path) -> None: """Reverts a given patch file on the specified git repository.""" - git("-C", repo_folder.absolute(), "apply", "-R", str(patch_file)) + repo("apply", "-R", str(patch_file)) diff --git a/varats-core/varats/utils/git_util.py b/varats-core/varats/utils/git_util.py index 6f0cd1c1d..de1b76de7 100644 --- a/varats-core/varats/utils/git_util.py +++ b/varats-core/varats/utils/git_util.py @@ -3,25 +3,16 @@ import logging import re import typing as tp -from collections import defaultdict from enum import Enum -from itertools import chain -from operator import attrgetter from pathlib import Path from types import TracebackType import pygit2 from benchbuild.utils.cmd import git, grep -from plumbum import local, TF, RETCODE - -from varats.project.project_util import ( - get_local_project_gits, - get_primary_project_source, - get_local_project_git_path, - BinaryType, - ProjectBinaryWrapper, - get_local_project_git_paths, -) +from plumbum import TF, RETCODE +from plumbum.commands.base import BoundCommand + +from varats.utils.exceptions import unwrap if tp.TYPE_CHECKING: from benchbuild.utils.revision_ranges import AbstractRevisionRange @@ -137,6 +128,69 @@ def full_commit_hashes_sorted_by_time_id( # Git interaction helpers +class RepositoryHandle: + """Wrapper class providing access to a git repository using either pygit2 or + commandline-git.""" + + def __init__(self, worktree_path: Path): + self.__worktree_path = worktree_path + self.__git: BoundCommand = git["-C", str(self.__worktree_path)] + + self.__repo_path: tp.Optional[Path] = None + self.__libgit_repo: tp.Optional[pygit2.Repository] = None + + def __call__(self, *args: tp.Any, **kwargs: tp.Any) -> tp.Any: + """Call git with the given arguments.""" + return self.__git(*args, **kwargs) + + def __getitem__(self, *args: tp.Any) -> BoundCommand: + """Get a bound git command with the given arguments.""" + return self.__git.bound_command(*args) + + @property + def repo_name(self) -> str: + """Name of the repository, i.e., name of the worktree folder.""" + return self.worktree_path.name + + @property + def worktree_path(self) -> Path: + """Path to the main worktree of the repository, typically the parent of + the .git folder.""" + return self.__worktree_path + + @property + def repo_path(self) -> Path: + """Path to the git repository, i.e., the .git folder.""" + if self.__repo_path is None: + self.__repo_path = Path( + unwrap( + pygit2.discover_repository(str(self.worktree_path)), + f"No git repository found." + ) + ) + + return self.__repo_path + + @property + def pygit_repo(self) -> pygit2.Repository: + """A pygit2 repository instance for the repository.""" + if self.__libgit_repo is None: + self.__libgit_repo = pygit2.Repository(str(self.repo_path)) + + return self.__libgit_repo + + def __eq__(self, other: tp.Any) -> bool: + if not isinstance(other, RepositoryHandle): + return False + return self.repo_path == other.repo_path + + def __repr__(self) -> str: + return f"RepositoryHandle({self.repo_path})" + + def __str__(self) -> str: + return self.repo_name + + def is_commit_hash(value: str) -> bool: """ Checks if a string is a valid git (sha1) hash. @@ -147,71 +201,50 @@ def is_commit_hash(value: str) -> bool: return re.search("^[a-fA-F0-9]{1,40}$", value) is not None -def __get_git_path_arg(repo_folder: tp.Optional[Path] = None) -> tp.List[str]: - if repo_folder is None or repo_folder == Path(''): - return [] - - return ["-C", f"{repo_folder}"] - - -def get_current_branch(repo_folder: tp.Optional[Path] = None) -> str: +def get_current_branch(repo: RepositoryHandle) -> str: """ Get the current branch of a repository, e.g., HEAD. Args: - repo_folder: where the git repository is located + repo: git repository handle Returns: branch name """ - return tp.cast( - str, - git( - __get_git_path_arg(repo_folder), "rev-parse", "--abbrev-ref", "HEAD" - ).strip() - ) + return tp.cast(str, repo("rev-parse", "--abbrev-ref", "HEAD").strip()) -def get_head_commit(repo_folder: tp.Optional[Path] = None) -> FullCommitHash: +def get_head_commit(repo: RepositoryHandle) -> FullCommitHash: """ Get the current HEAD commit. Args: - repo_folder:where the git repository is located + repo: git repository handle Returns: head commit hash """ - return FullCommitHash( - git(__get_git_path_arg(repo_folder), "rev-parse", "HEAD").strip() - ) + return FullCommitHash(repo("rev-parse", "HEAD").strip()) -def get_initial_commit(repo_folder: tp.Optional[Path] = None) -> FullCommitHash: +def get_initial_commit(repo: RepositoryHandle) -> FullCommitHash: """ Get the initial commit of a repository, i.e., the first commit made. Args: - repo_folder: where the git repository is located + repo: git repository handle Returns: initial commit hash """ - return FullCommitHash( - git( - __get_git_path_arg(repo_folder), "rev-list", "--max-parents=0", - "HEAD" - ).strip() - ) + return FullCommitHash(repo("rev-list", "--max-parents=0", "HEAD").strip()) -def get_submodule_commits( - c_head: str = "HEAD", - repo_folder: tp.Optional[Path] = None -) -> tp.Dict[str, FullCommitHash]: +def get_submodule_commits(repo: RepositoryHandle, + c_head: str = "HEAD") -> tp.Dict[str, FullCommitHash]: """ Get the revisions of all submodules of a repo at a given commit. Args: + repo: repository to get the submodules for c_head: the commit to look at - repo_folder: the repo to get the submodules for Returns: a mapping from submodule name to commit @@ -220,7 +253,7 @@ def get_submodule_commits( r"\d{6} commit (?P[\da-f]{40})\s+(?P.+)$" ) - ls_tree_result = git(__get_git_path_arg(repo_folder), "ls-tree", c_head) + ls_tree_result = repo("ls-tree", c_head) result: tp.Dict[str, FullCommitHash] = {} for line in ls_tree_result.splitlines(): match = submodule_regex.match(line) @@ -230,10 +263,8 @@ def get_submodule_commits( def get_all_revisions_between( - c_start: str, - c_end: str, - hash_type: tp.Type[CommitHashTy], - repo_folder: tp.Optional[Path] = None + repo: RepositoryHandle, c_start: str, c_end: str, + hash_type: tp.Type[CommitHashTy] ) -> tp.List[CommitHashTy]: """ Returns a list of all revisions between two commits c_start and c_end (both @@ -242,16 +273,16 @@ def get_all_revisions_between( It is assumed that the current working directory is the git repository. Args: - c_start: first commit of the range c_end: last commit of the range - short: shorten revision hashes repo_folder: where the git repository is - located + repo: git repository handle + c_start: first commit of the range + c_end: last commit of the range + hash_type: type of the commit hash to return """ result = [c_start] result.extend( reversed( - git( - __get_git_path_arg(repo_folder), "log", "--pretty=%H", - "--ancestry-path", f"{c_start}..{c_end}" + repo( + "log", "--pretty=%H", "--ancestry-path", f"{c_start}..{c_end}" ).strip().split() ) ) @@ -259,29 +290,27 @@ def get_all_revisions_between( def typed_revision_range( - rev_range: 'AbstractRevisionRange', repo_path: Path, + repo: RepositoryHandle, rev_range: 'AbstractRevisionRange', hash_type: tp.Type[CommitHashTy] ) -> tp.Iterator[CommitHashTy]: """ Typed iterator for revision ranges. Args: + repo: git repository handle rev_range: the revision range to iterate - repo_path: the path to the git repo hash_type: the commit type to use for iteration Returns: an iterator over the typed commits in the range """ - rev_range.init_cache(str(repo_path)) + rev_range.init_cache(str(repo.repo_path)) for revision in rev_range: yield hash_type(revision) -def get_commits_before_timestamp( - timestamp: str, - repo_folder: tp.Optional[Path] = None -) -> tp.List[FullCommitHash]: +def get_commits_before_timestamp(repo: RepositoryHandle, + timestamp: str) -> tp.List[FullCommitHash]: """ Get all commits before a specific timestamp (given as a git date format). @@ -289,55 +318,49 @@ def get_commits_before_timestamp( default to today. Args: + repo: git repository handle timestamp: before which commits should be collected - repo_folder: where the git repository is located Returns: list[last_commit_before_timestamp, ..., initial_commits] """ return [ - FullCommitHash(hash_val) for hash_val in git( - __get_git_path_arg(repo_folder), "rev-list", - f"--before={timestamp}", "HEAD" - ).split() + FullCommitHash(hash_val) for hash_val in + repo("rev-list", f"--before={timestamp}", "HEAD").split() ] -def get_commits_after_timestamp( - timestamp: str, - repo_folder: tp.Optional[Path] = None -) -> tp.List[FullCommitHash]: +def get_commits_after_timestamp(repo: RepositoryHandle, + timestamp: str) -> tp.List[FullCommitHash]: """ Get all commits after a specific timestamp (given as a git date format). - Note: for imprecise timestamps (e.g., only 2020), the day and month will - default to today. + Note: for imprecise timestamps (e.g., only 2020), the day and month will + default to today. - Args: - repo_folder: where the git repository is located - timestamp: after which commits should be collected + Args: + repo: git repository handle + timestamp: after which commits should be collected Returns: list[newest_commit, ..., last_commit_after_timestamp] """ return [ - FullCommitHash(hash_val) for hash_val in git( - __get_git_path_arg(repo_folder), "rev-list", f"--after={timestamp}", - "HEAD" - ).split() + FullCommitHash(hash_val) + for hash_val in repo("rev-list", f"--after={timestamp}", "HEAD").split() ] def contains_source_code( + repo: RepositoryHandle, commit: ShortCommitHash, - repo_folder: tp.Optional[Path] = None, churn_config: tp.Optional['ChurnConfig'] = None ) -> bool: """ - Check if a commit contains source code of any language specifyed with the + Check if a commit contains source code of any language specified with the churn config. Args: + repo: git repository handle commit: to check - repo_folder: of the commits repository churn_config: to specify the files that should be considered Returns: True, if source code of a language, specified in the churn @@ -346,9 +369,13 @@ def contains_source_code( if not churn_config: churn_config = ChurnConfig.create_c_style_languages_config() - return_code = git[__get_git_path_arg(repo_folder), "show", "--exit-code", - "-m", "--quiet", commit.hash, "--", - churn_config.get_extensions_repr('*.')] & RETCODE + git_show_args = ["show", "--exit-code", "-m", "--quiet", commit.hash, "--"] + git_show_args += churn_config.get_extensions_repr('*.') + # There should be a '*' in front of 'git_show_args' to unpack the list. + # However, yapf and sphinx are unable to parse this. + # Fortunately, plumbum seems to unpack this correctly before running. + # Still: fix when possible. + return_code = repo[git_show_args] & RETCODE if return_code == 0: return False @@ -359,103 +386,57 @@ def contains_source_code( raise RuntimeError(f"git diff failed with retcode={return_code}") -def num_commits( - c_start: str = "HEAD", repo_folder: tp.Optional[Path] = None -) -> int: +def num_commits(repo: RepositoryHandle, c_start: str = "HEAD") -> int: """ Count the commits in a git repo starting from the given commit back to the initial commit. Args: + repo: git repository handle c_start: commit to start counting at - repo_folder: path to the git repo Returns: the number of commits """ - return int( - git(__get_git_path_arg(repo_folder), "rev-list", "--count", c_start) - ) + return int(repo("rev-list", "--count", c_start)) -def num_authors( - c_start: str = "HEAD", repo_folder: tp.Optional[Path] = None -) -> int: +def num_authors(repo: RepositoryHandle, c_start: str = "HEAD") -> int: """ Count the authors in a git repo starting from the given commit back to the initial commit. Args: + repo: git repository handle c_start: commit to start counting at - repo_folder: path to the git repo Returns: the number of authors """ - return len( - git(__get_git_path_arg(repo_folder), "shortlog", "-s", - c_start).splitlines() - ) + return len(repo("shortlog", "-s", c_start).splitlines()) -def num_project_commits(project_name: str, revision: FullCommitHash) -> int: +def get_authors(repo: RepositoryHandle, c_start: str = "HEAD") -> tp.Set[str]: """ - Calculate the number of commits of a project including submodules. - - Args: - project_name: name of the project to calculate commits for - revision: revision to calculate commits at - - Returns: - the number of commits in the project - """ - project_repos = get_local_project_git_paths(project_name) - main_repo = get_local_project_git_path(project_name) - - commits = num_commits(revision.hash, main_repo) - for submodule, sub_rev in get_submodule_commits(revision.hash, - main_repo).items(): - if submodule not in project_repos: - LOG.warning("Ignoring unknown submodule {}", submodule) - continue - commits += num_commits(sub_rev.hash, project_repos[submodule]) - return commits - - -def num_project_authors(project_name: str, revision: FullCommitHash) -> int: - """ - Calculate the number of authors of a project including submodules. + Get the authors in a git repo starting from the given commit back to the + initial commit. Args: - project_name: name of the project to calculate authors for - revision: revision to authors commits at + repo: git repository handle + c_start: commit to start counting at Returns: - the number of authors in the project + the number of authors """ author_regex = re.compile(r"\s*\d+\s+(?P.+)$") - def get_authors(repo_path: Path, rev: str) -> tp.Set[str]: - lines = git(__get_git_path_arg(repo_path), "shortlog", "-s", - rev).splitlines() - result = set() - for line in lines: - match = author_regex.match(line) - if match: - result.add(match.group("author")) - return result - - project_repos = get_local_project_git_paths(project_name) - main_repo = get_local_project_git_path(project_name) - - authors = get_authors(main_repo, revision.hash) - for submodule, sub_rev in get_submodule_commits(revision.hash, - main_repo).items(): - if submodule not in project_repos: - LOG.warning("Ignoring unknown submodule {}", submodule) - continue - authors.update(get_authors(project_repos[submodule], sub_rev.hash)) - return len(authors) + lines = repo("shortlog", "-s", c_start).splitlines() + result = set() + for line in lines: + match = author_regex.match(line) + if match: + result.add(match.group("author")) + return result ################################################################################ @@ -637,74 +618,35 @@ def __repr__(self) -> str: return str(self) -CommitLookupTy = tp.Callable[[CommitRepoPair], pygit2.Commit] - - -def create_commit_lookup_helper(project_name: str) -> CommitLookupTy: - """ - Creates a commit lookup function for project repositories. - - Args: - project_name: name of the given benchbuild project - - Returns: - a Callable that maps a commit hash and repository name to the - corresponding commit. - """ - - repos = get_local_project_gits(project_name) - - def get_commit(crp: CommitRepoPair) -> pygit2.Commit: - """ - Gets the commit from a given ``CommitRepoPair``. - - Args: - crp: the ``CommitRepoPair`` for the commit to get - - - Returns: - the commit corresponding to the given CommitRepoPair - """ - commit = repos[crp.repository_name].get(crp.commit_hash.hash) - if not commit: - raise LookupError( - f"Could not find commit {crp} for project {project_name}." - ) - - return commit - - return get_commit - - def get_submodule_head( - project_name: str, submodule_name: str, commit: FullCommitHash + repo: RepositoryHandle, submodule: RepositoryHandle, commit: FullCommitHash ) -> FullCommitHash: """ Retrieve the checked out commit for a submodule of a project. Args: - project_name: name of the project - submodule_name: name of the submodule + repo: main project repository handle + submodule: submodule repository handle commit: commit of the project's main repo Returns: checked out commit of the submodule """ - if submodule_name == get_primary_project_source(project_name).local: + if submodule.repo_name == repo.repo_name: return commit - main_repo = get_local_project_git_path(project_name) - submodule_status = git(__get_git_path_arg(main_repo), "ls-tree", commit) + submodule_status = repo("ls-tree", commit) commit_pattern = re.compile( - r"[0-9]* commit ([0-9abcdef]*)\t" + submodule_name + r"[0-9]* commit ([0-9abcdef]*)\t" + submodule.repo_name ) - if (match := commit_pattern.search(submodule_status)): + if match := commit_pattern.search(submodule_status): return FullCommitHash(match.group(1)) - raise AssertionError(f"Unknown submodule {submodule_name}") + raise AssertionError(f"Unknown submodule {submodule.repo_name}") MappedCommitResultType = tp.TypeVar("MappedCommitResultType") +CommitLookupTy = tp.Callable[[CommitRepoPair], pygit2.Commit] def map_commits( @@ -734,7 +676,7 @@ def map_commits( def __calc_code_churn_range_impl( - repo_path: Path, + repo: RepositoryHandle, churn_config: ChurnConfig, start_range: tp.Optional[FullCommitHash] = None, end_range: tp.Optional[FullCommitHash] = None @@ -748,14 +690,14 @@ def __calc_code_churn_range_impl( git log --pretty=format:'%H' --date=short --shortstat -- ':*.[enabled_exts]' Args: - repo_path: path to the git repository + repo: git repository handle churn_config: churn config to customize churn generation start_range: begin churn calculation at start commit end_range: end churn calculation at end commit """ churn_values: tp.Dict[FullCommitHash, tp.Tuple[int, int, int]] = {} - if start_range and start_range == get_initial_commit(repo_path): + if start_range and start_range == get_initial_commit(repo): start_range = None if start_range is None and end_range is None: @@ -767,7 +709,6 @@ def __calc_code_churn_range_impl( else: revision_range = f"{start_range.hash}~..{end_range.hash}" - repo_git = git[__get_git_path_arg(repo_path)] log_base_params = ["log", "--pretty=%H"] diff_base_params = ["log", "--pretty=format:'%H'", "--shortstat", "-l0"] if revision_range: @@ -780,8 +721,8 @@ def __calc_code_churn_range_impl( diff_base_params = diff_base_params + \ churn_config.get_extensions_repr('*.') - stdout = repo_git(diff_base_params) - revs = repo_git(log_base_params).strip().split() + stdout = repo(diff_base_params) + revs = repo(log_base_params).strip().split() # initialize with 0 as otherwise commits without changes would be # missing from the churn data @@ -804,7 +745,7 @@ def value_or_zero(match_result: tp.Any) -> int: def calc_code_churn_range( - repo_path: Path, + repo: RepositoryHandle, churn_config: tp.Optional[ChurnConfig] = None, start_range: tp.Optional[FullCommitHash] = None, end_range: tp.Optional[FullCommitHash] = None @@ -816,7 +757,7 @@ def calc_code_churn_range( calculated. Args: - repo_path: path to git repository + repo: git repository handle churn_config: churn config to customize churn generation Returns: @@ -825,12 +766,12 @@ def calc_code_churn_range( """ churn_config = ChurnConfig.init_as_default_if_none(churn_config) return __calc_code_churn_range_impl( - repo_path, churn_config, start_range, end_range + repo, churn_config, start_range, end_range ) def calc_commit_code_churn( - repo_path: Path, + repo: RepositoryHandle, commit_hash: FullCommitHash, churn_config: tp.Optional[ChurnConfig] = None ) -> tp.Tuple[int, int, int]: @@ -838,7 +779,7 @@ def calc_commit_code_churn( Calculates churn of a specific commit. Args: - repo_path: path to git repository + repo: git repository handle commit_hash: commit hash to get churn for churn_config: churn config to customize churn generation @@ -847,13 +788,12 @@ def calc_commit_code_churn( (files changed, insertions, deletions) """ churn_config = ChurnConfig.init_as_default_if_none(churn_config) - return calc_code_churn_range( - repo_path, churn_config, commit_hash, commit_hash - )[commit_hash] + return calc_code_churn_range(repo, churn_config, commit_hash, + commit_hash)[commit_hash] def calc_code_churn( - repo_path: Path, + repo: RepositoryHandle, commit_a: FullCommitHash, commit_b: FullCommitHash, churn_config: tp.Optional[ChurnConfig] = None @@ -862,7 +802,7 @@ def calc_code_churn( Calculates churn between two commits. Args: - repo: git repository + repo: git repository handle commit_a: base commit for diff calculation commit_b: target commit for diff calculation churn_config: churn config to customize churn generation @@ -882,10 +822,10 @@ def calc_code_churn( diff_base_params = diff_base_params + \ churn_config.get_extensions_repr('*.') - stdout = git(__get_git_path_arg(repo_path), diff_base_params) + stdout = repo(*diff_base_params) # initialize with 0 as otherwise commits without changes would be # missing from the churn data - if (match := GIT_DIFF_MATCHER.match(stdout)): + if match := GIT_DIFF_MATCHER.match(stdout): def value_or_zero(match_result: tp.Any) -> int: if match_result is not None: @@ -901,14 +841,14 @@ def value_or_zero(match_result: tp.Any) -> int: def calc_repo_code_churn( - repo_path: Path, + repo: RepositoryHandle, churn_config: tp.Optional[ChurnConfig] = None ) -> tp.Dict[FullCommitHash, tp.Tuple[int, int, int]]: """ Calculates code churn for a repository. Args: - repo_path: path to git repository + repo: git repository handle churn_config: churn config to customize churn generation Returns: @@ -916,18 +856,20 @@ def calc_repo_code_churn( (files changed, insertions, deletions) """ churn_config = ChurnConfig.init_as_default_if_none(churn_config) - return calc_code_churn_range(repo_path, churn_config) + return calc_code_churn_range(repo, churn_config) def __print_calc_repo_code_churn( - repo: pygit2.Repository, + repo: RepositoryHandle, churn_config: tp.Optional[ChurnConfig] = None ) -> None: """Prints calc repo code churn data like git log would do.""" churn_config = ChurnConfig.init_as_default_if_none(churn_config) - churn_map = calc_repo_code_churn(Path(repo.path), churn_config) + churn_map = calc_repo_code_churn(repo, churn_config) - for commit in repo.walk(repo.head.target, pygit2.GIT_SORT_TIME): + for commit in repo.pygit_repo.walk( + repo.pygit_repo.head.target, pygit2.GIT_SORT_TIME + ): commit_hash = FullCommitHash.from_pygit_commit(commit) print(commit_hash) @@ -965,12 +907,12 @@ def __print_calc_repo_code_churn( print() -def calc_repo_loc(repo_path: Path, rev_range: str) -> int: +def calc_repo_loc(repo: RepositoryHandle, rev_range: str) -> int: """ Calculate the LOC for a repository. Args: - repo_path: path to the repository to calculate the LOC for + repo: handle for the repository to calculate the LOC for rev_range: the revision range to use for LOC calculation Returns: @@ -982,162 +924,49 @@ def calc_repo_loc(repo_path: Path, rev_range: str) -> int: ) loc: int = 0 - with local.cwd(repo_path): - files = git( - "ls-tree", - "-r", - "--name-only", - rev_range, - ).splitlines() - - for file in files: - if file_pattern.match(file): - lines = git("show", f"{rev_range}:{file}").splitlines() - loc += len([line for line in lines if line]) + files = repo( + "ls-tree", + "-r", + "--name-only", + rev_range, + ).splitlines() + + for file in files: + if file_pattern.match(file): + lines = repo("show", f"{rev_range}:{file}").splitlines() + loc += len([line for line in lines if line]) return loc -def calc_project_loc(project_name: str, revision: FullCommitHash) -> int: - """ - Calculate the LOC for a project including submodules at the given revision. - - Args: - project_name: name of the project to calculate LOC for - revision: revision to calculate LOC at - - Returns: - the LOC in the project - """ - project_repos = get_local_project_git_paths(project_name) - main_repo = get_local_project_git_path(project_name) - - loc = calc_repo_loc(main_repo, revision.hash) - for submodule, sub_rev in get_submodule_commits(revision.hash, - main_repo).items(): - if submodule not in project_repos: - LOG.warning("Ignoring unknown submodule {}", submodule) - continue - loc += calc_repo_loc(project_repos[submodule], sub_rev.hash) - return loc - - ################################################################################ # Special git-specific classes -class RevisionBinaryMap(tp.Container[str]): - """A special map that specifies for which revision ranges a binaries is - valid.""" - - def __init__(self, repo_location: Path) -> None: - self.__repo_location = repo_location - self.__revision_specific_mappings: tp.Dict[ - 'AbstractRevisionRange', - tp.List[ProjectBinaryWrapper]] = defaultdict(list) - self.__always_valid_mappings: tp.List[ProjectBinaryWrapper] = [] - - def specify_binary( - self, location: str, binary_type: BinaryType, **kwargs: tp.Any - ) -> 'RevisionBinaryMap': - """ - Add a binary to the map. - - Args: - location: where the binary can be found, relative to the - project-source root - binary_type: the type of binary that is produced - override_binary_name: overrides the used binary name - override_entry_point: overrides the executable entry point - only_valid_in: additionally specifies a validity range that - specifies in which revision range this binary is - produced - - Returns: - self for builder-style usage - """ - binary_location_path = Path(location) - binary_name: str = kwargs.get( - "override_binary_name", binary_location_path.stem - ) - override_entry_point = kwargs.get("override_entry_point", None) - if override_entry_point: - override_entry_point = Path(override_entry_point) - validity_range: AbstractRevisionRange = kwargs.get( - "only_valid_in", None - ) - valid_exit_codes = kwargs.get("valid_exit_codes", None) - - wrapped_binary = ProjectBinaryWrapper( - binary_name, binary_location_path, binary_type, - override_entry_point, valid_exit_codes - ) - - if validity_range: - validity_range.init_cache(self.__repo_location) - self.__revision_specific_mappings[validity_range].append( - wrapped_binary - ) - else: - self.__always_valid_mappings.append(wrapped_binary) - - return self - - def __getitem__(self, - revision: CommitHash) -> tp.List[ProjectBinaryWrapper]: - revision = revision.to_short_commit_hash() - revision_specific_binaries = [] - - for validity_range, wrapped_binaries \ - in self.__revision_specific_mappings.items(): - if revision in map(ShortCommitHash, validity_range): - revision_specific_binaries.extend(wrapped_binaries) - - revision_specific_binaries.extend(self.__always_valid_mappings) - - return sorted( - revision_specific_binaries, key=attrgetter("name", "path") - ) - - def __contains__(self, binary_name: object) -> bool: - if isinstance(binary_name, str): - for binary in chain( - self.__always_valid_mappings, - *self.__revision_specific_mappings.values() - ): - if binary.name == binary_name: - return True - - return False - - -def has_branch(repo_folder: Path, branch_name: str) -> bool: +def has_branch(repo: RepositoryHandle, branch_name: str) -> bool: """Checks if a branch exists in the local repository.""" - exit_code = git["-C", - repo_folder.absolute(), "rev-parse", "--verify", - branch_name] & TF + exit_code = repo["rev-parse", "--verify", branch_name] & TF return tp.cast(bool, exit_code) -def has_remote_branch(repo_folder: Path, branch_name: str, remote: str) -> bool: +def has_remote_branch( + repo: RepositoryHandle, branch_name: str, remote: str +) -> bool: """Checks if a remote branch of a repository exists.""" exit_code = ( - git["-C", - repo_folder.absolute(), "ls-remote", "--heads", remote, branch_name] - | grep[branch_name] + repo["ls-remote", "--heads", remote, branch_name] | grep[branch_name] ) & RETCODE return tp.cast(bool, exit_code == 0) def branch_has_upstream( - repo_folder: Path, branch_name: str, upstream: str = 'origin' + repo: RepositoryHandle, branch_name: str, upstream: str = 'origin' ) -> bool: """Check if a branch has an upstream remote.""" exit_code = ( - git["-C", - repo_folder.absolute(), "rev-parse", "--abbrev-ref", - branch_name + "@{upstream}"] | grep[upstream] + repo["rev-parse", "--abbrev-ref", branch_name + "@{upstream}"] | + grep[upstream] ) & RETCODE return tp.cast(bool, exit_code == 0) @@ -1146,10 +975,10 @@ class RepositoryAtCommit(): """Context manager to work with a repository at a specific revision, without duplicating the repository.""" - def __init__(self, project_name: str, revision: ShortCommitHash) -> None: - self.__repo = pygit2.Repository( - get_local_project_git_path(project_name) - ) + def __init__( + self, repo: RepositoryHandle, revision: ShortCommitHash + ) -> None: + self.__repo = repo.pygit_repo self.__initial_head = self.__repo.head self.__revision = self.__repo.get(revision.hash) diff --git a/varats/varats/data/databases/blame_diff_metrics_database.py b/varats/varats/data/databases/blame_diff_metrics_database.py index eff4a3232..1b33649fe 100644 --- a/varats/varats/data/databases/blame_diff_metrics_database.py +++ b/varats/varats/data/databases/blame_diff_metrics_database.py @@ -26,7 +26,10 @@ from varats.mapping.commit_map import CommitMap from varats.paper.case_study import CaseStudy from varats.paper_mgmt.case_study import get_case_study_file_name_filter -from varats.project.project_util import get_local_project_git +from varats.project.project_util import ( + get_local_project_repo, + create_project_commit_lookup_helper, +) from varats.report.report import ReportFilepath from varats.revision.revisions import ( get_processed_revisions_files, @@ -36,7 +39,6 @@ from varats.utils.git_util import ( ChurnConfig, calc_code_churn, - create_commit_lookup_helper, ShortCommitHash, FullCommitHash, ) @@ -300,8 +302,9 @@ def _load_dataframe( cls, project_name: str, commit_map: CommitMap, case_study: tp.Optional[CaseStudy], **kwargs: tp.Any ) -> pd.DataFrame: - repo = get_local_project_git(project_name) - commit_lookup = create_commit_lookup_helper(project_name) + repo = get_local_project_repo(project_name) + pygit_repo = repo.pygit_repo + commit_lookup = create_project_commit_lookup_helper(project_name) def create_dataframe_layout() -> pd.DataFrame: df_layout = pd.DataFrame(columns=cls.COLUMNS) @@ -314,15 +317,15 @@ def create_data_frame_for_report( # Look-up commit and infos about the HEAD commit of the report head_report = load_blame_report(report_paths[0]) pred_report = load_blame_report(report_paths[1]) - commit = repo.get(head_report.head_commit.hash) + commit = pygit_repo.get(head_report.head_commit.hash) commit_date = datetime.utcfromtimestamp(commit.commit_time) - pred_commit = repo.get(pred_report.head_commit.hash) + pred_commit = pygit_repo.get(pred_report.head_commit.hash) diff_between_head_pred = BlameReportDiff(head_report, pred_report) # Calculate the total churn between pred and base commit code_churn = calc_code_churn( - Path(repo.path), FullCommitHash.from_pygit_commit(pred_commit), + repo, FullCommitHash.from_pygit_commit(pred_commit), FullCommitHash.from_pygit_commit(commit), ChurnConfig.create_c_style_languages_config() ) diff --git a/varats/varats/data/databases/blame_interaction_degree_database.py b/varats/varats/data/databases/blame_interaction_degree_database.py index 56aa5f920..883e37bf0 100644 --- a/varats/varats/data/databases/blame_interaction_degree_database.py +++ b/varats/varats/data/databases/blame_interaction_degree_database.py @@ -20,12 +20,12 @@ from varats.mapping.commit_map import CommitMap from varats.paper.case_study import CaseStudy from varats.paper_mgmt.case_study import get_case_study_file_name_filter +from varats.project.project_util import create_project_commit_lookup_helper from varats.report.report import ReportFilepath from varats.revision.revisions import ( get_failed_revisions_files, get_processed_revisions_files, ) -from varats.utils.git_util import create_commit_lookup_helper MAX_TIME_BUCKET_SIZE = 1 AVG_TIME_BUCKET_SIZE = 1 @@ -88,7 +88,7 @@ def _load_dataframe( cls, project_name: str, commit_map: CommitMap, case_study: tp.Optional[CaseStudy], **kwargs: tp.Any ) -> pd.DataFrame: - commit_lookup = create_commit_lookup_helper(project_name) + commit_lookup = create_project_commit_lookup_helper(project_name) def create_dataframe_layout() -> pd.DataFrame: df_layout = pd.DataFrame(columns=cls.COLUMNS) diff --git a/varats/varats/data/databases/szz_quality_metrics_database.py b/varats/varats/data/databases/szz_quality_metrics_database.py index 5fdb64fec..ec821092d 100644 --- a/varats/varats/data/databases/szz_quality_metrics_database.py +++ b/varats/varats/data/databases/szz_quality_metrics_database.py @@ -29,12 +29,14 @@ from varats.jupyterhelper.file import load_blame_report from varats.mapping.commit_map import CommitMap, get_commit_map from varats.paper.case_study import CaseStudy -from varats.project.project_util import get_primary_project_source +from varats.project.project_util import ( + get_primary_project_source, + create_project_commit_lookup_helper, +) from varats.report.report import ReportFilepath from varats.revision.revisions import get_processed_revisions_files from varats.utils.git_util import ( CommitRepoPair, - create_commit_lookup_helper, ShortCommitHash, FullCommitHash, ) @@ -111,7 +113,7 @@ def _load_dataframe_for_report( project_name: str, cache_id: str, columns: tp.List[str], commit_map: CommitMap, szz_report: SZZReport ) -> pd.DataFrame: - commit_lookup = create_commit_lookup_helper(project_name) + commit_lookup = create_project_commit_lookup_helper(project_name) commit_map = get_commit_map(project_name) prj_src = get_primary_project_source(project_name) diff --git a/varats/varats/data/reports/blame_interaction_graph.py b/varats/varats/data/reports/blame_interaction_graph.py index dc490b9b4..aa726c1eb 100644 --- a/varats/varats/data/reports/blame_interaction_graph.py +++ b/varats/varats/data/reports/blame_interaction_graph.py @@ -8,7 +8,6 @@ from typing import TypedDict import networkx as nx -from benchbuild.utils.cmd import git from varats.data.cache_helper import build_cached_graph from varats.data.reports.blame_report import ( @@ -18,14 +17,14 @@ ) from varats.jupyterhelper.file import load_blame_report from varats.project.project_util import ( - get_local_project_git_path, - get_local_project_gits, + get_local_project_repo, + get_local_project_repos, + create_project_commit_lookup_helper, ) from varats.report.report import ReportFilename, ReportFilepath from varats.revision.revisions import get_processed_revisions_files from varats.utils.git_util import ( CommitRepoPair, - create_commit_lookup_helper, ChurnConfig, UNCOMMITTED_COMMIT_HASH, FullCommitHash, @@ -181,7 +180,7 @@ def author_interaction_graph(self) -> nx.DiGraph: the author interaction graph """ interaction_graph = self._interaction_graph() - commit_lookup = create_commit_lookup_helper(self.project_name) + commit_lookup = create_project_commit_lookup_helper(self.project_name) def partition(node_u: BIGNodeTy, node_v: BIGNodeTy) -> bool: if UNCOMMITTED_COMMIT_HASH in ( @@ -256,7 +255,7 @@ def commit_author_interaction_graph( the commit-author interaction graph """ commit_interaction_graph = self.commit_interaction_graph() - commit_lookup = create_commit_lookup_helper(self.project_name) + commit_lookup = create_project_commit_lookup_helper(self.project_name) commit_author_mapping = { commit: commit_lookup(commit).author.name @@ -415,7 +414,8 @@ def __init__(self, project_name: str, head_commit: FullCommitHash): def _interaction_graph(self) -> nx.DiGraph: def create_graph() -> nx.DiGraph: - repos = get_local_project_gits(self.project_name) + main_repo = get_local_project_repo(self.project_name) + repos = get_local_project_repos(self.project_name) interaction_graph = nx.DiGraph() churn_config = ChurnConfig.create_c_style_languages_config() file_pattern = re.compile( @@ -426,28 +426,24 @@ def create_graph() -> nx.DiGraph: blame_regex = re.compile(r"^([0-9a-f]+)\s+(?:.+\s+)?[\d]+\) ?(.*)$") - for repo_name in repos: - repo_path = get_local_project_git_path( - self.project_name, repo_name - ) - project_git = git["-C", str(repo_path)] + for repo_name, repo in repos.items(): head_commit = get_submodule_head( - self.project_name, repo_name, self.__head_commit + main_repo, repo, self.__head_commit ) - file_names = project_git( + file_names = repo( "ls-tree", "--full-tree", "--name-only", "-r", head_commit ).split("\n") files: tp.List[Path] = [ - repo_path / path + repo.worktree_path / path for path in file_names if file_pattern.search(path) ] for file in files: nodes: tp.Set[BIGNodeTy] = set() - blame_lines: str = project_git( + blame_lines: str = repo( "blame", "-w", "-s", "-l", "--root", head_commit, "--", - str(file.relative_to(repo_path)) + str(file.relative_to(repo.worktree_path)) ) for line in blame_lines.strip().split("\n"): diff --git a/varats/varats/experiments/vara/blame_ast_experiment.py b/varats/varats/experiments/vara/blame_ast_experiment.py index ace3264df..d9ef77f40 100644 --- a/varats/varats/experiments/vara/blame_ast_experiment.py +++ b/varats/varats/experiments/vara/blame_ast_experiment.py @@ -14,8 +14,8 @@ from benchbuild.utils.cmd import opt import varats.experiments.vara.blame_experiment as BE -from varats.data.reports.blame_annotations import BlameComparisonReport as BCR from varats.data.reports.blame_annotations import BlameAnnotations as BA +from varats.data.reports.blame_annotations import BlameComparisonReport as BCR from varats.data.reports.blame_annotations import compare_blame_annotations from varats.experiment.experiment_util import ( ExperimentHandle, @@ -32,7 +32,7 @@ _create_default_bc_file_creation_actions, get_cached_bc_file_path, ) -from varats.project.project_util import get_local_project_git_paths +from varats.project.project_util import get_local_project_repos from varats.project.varats_project import VProject from varats.report.report import ReportSpecification @@ -60,7 +60,7 @@ def __call__(self) -> actions.StepResult: def analyze(self) -> actions.StepResult: """ This step generates a blame annotation report given a binary annotated - with blame informatin. + with blame information. Flags used: * -vara-BA: to run a blame annotation report @@ -76,8 +76,8 @@ def analyze(self) -> actions.StepResult: "--enable-new-pm=0", "-vara-BD", "-vara-BA", "-vara-init-commits", "-vara-rewriteMD", "-vara-git-mappings=" + ",".join([ - f'{repo}:{path}' for repo, path in - get_local_project_git_paths(self.project.name).items() + f'{repo_name}:{repo.repo_path}' for repo_name, repo in + get_local_project_repos(self.project.name).items() ]), "-vara-use-phasar", f"-vara-report-outfile={result_file}", get_cached_bc_file_path( self.project, binary, self.__file_extensions diff --git a/varats/varats/experiments/vara/blame_report_experiment.py b/varats/varats/experiments/vara/blame_report_experiment.py index 747e8d313..917beea8e 100644 --- a/varats/varats/experiments/vara/blame_report_experiment.py +++ b/varats/varats/experiments/vara/blame_report_experiment.py @@ -25,7 +25,7 @@ create_new_success_result_filepath, ) from varats.experiment.wllvm import get_cached_bc_file_path, BCFileExtensions -from varats.project.project_util import get_local_project_git_paths +from varats.project.project_util import get_local_project_repos from varats.project.varats_project import VProject from varats.report.report import ReportSpecification @@ -71,8 +71,8 @@ def analyze(self) -> actions.StepResult: "--enable-new-pm=0", "-vara-BD", "-vara-BR", "-vara-init-commits", "-vara-rewriteMD", "-vara-git-mappings=" + ",".join([ - f'{repo}:{path}' for repo, path in - get_local_project_git_paths(self.project.name).items() + f'{repo_name}:{repo.repo_path}' for repo_name, repo in + get_local_project_repos(self.project.name).items() ]), "-vara-use-phasar", f"-vara-blame-taint-scope={self.__blame_taint_scope.name}", f"-vara-report-outfile={result_file}", diff --git a/varats/varats/experiments/vara/commit_report_experiment.py b/varats/varats/experiments/vara/commit_report_experiment.py index e0ef3fc4c..7ad01f6b7 100644 --- a/varats/varats/experiments/vara/commit_report_experiment.py +++ b/varats/varats/experiments/vara/commit_report_experiment.py @@ -61,6 +61,7 @@ def __call__(self) -> actions.StepResult: def analyze(self) -> actions.StepResult: """ This step performs the actual analysis with the correct flags. + Flags: -vara-CR: to run a commit flow report -vara-report-outfile=: specify the path to store the results diff --git a/varats/varats/experiments/vara/feature_experiment.py b/varats/varats/experiments/vara/feature_experiment.py index 3eacff35e..fbe9a6a14 100644 --- a/varats/varats/experiments/vara/feature_experiment.py +++ b/varats/varats/experiments/vara/feature_experiment.py @@ -51,7 +51,7 @@ class FeatureInstrType(Enum): """Print tracing information to stdout.""" PERF_INFLUENCE_TRACE = "perf_infl_trace" - """Produce a performance influence trace file""" + """Produce a performance influence trace file.""" TEF = "trace_event" """Produce a trace file.""" diff --git a/varats/varats/gui/cs_gen/case_study_generation.py b/varats/varats/gui/cs_gen/case_study_generation.py index e5c83c707..a1bb53261 100644 --- a/varats/varats/gui/cs_gen/case_study_generation.py +++ b/varats/varats/gui/cs_gen/case_study_generation.py @@ -33,9 +33,13 @@ ) from varats.project.project_util import ( get_loaded_vara_projects, - get_local_project_git_path, get_project_cls_by_name, get_primary_project_source, + get_local_project_repo, + num_project_commits, + num_project_authors, + calc_project_loc, + create_project_commit_lookup_helper, ) from varats.projects.discover_projects import initialize_projects from varats.report.report import FileStatusExtension @@ -47,15 +51,8 @@ get_initial_commit, get_all_revisions_between, ShortCommitHash, - create_commit_lookup_helper, FullCommitHash, CommitRepoPair, - num_commits, - num_authors, - calc_repo_loc, - calc_project_loc, - num_project_commits, - num_project_authors, ) from varats.utils.settings import vara_cfg @@ -173,7 +170,7 @@ def gen(self) -> None: ) == GenerationStrategy.REVS_PER_YEAR.value: extend_with_revs_per_year( case_study, cmap, 0, True, - get_local_project_git_path(self.selected_project), + get_local_project_repo(self.selected_project), self.revs_per_year.value(), self.seperate.checkState() ) @@ -189,8 +186,7 @@ def show_project_data(self, index: QModelIndex) -> None: if self.selected_project != project_name: self.selected_project = project_name project = get_project_cls_by_name(project_name) - git_path = get_local_project_git_path(project_name) - repo = pygit2.Repository(pygit2.discover_repository(git_path)) + repo = get_local_project_repo(project_name).pygit_repo last_pygit_commit: pygit2.Commit = repo[repo.head.target] last_commit = FullCommitHash.from_pygit_commit(last_pygit_commit) @@ -223,12 +219,12 @@ def revisions_of_project(self) -> None: self.revision_details.repaint() # Update the local project git get_primary_project_source(self.selected_project).fetch() - git_path = get_local_project_git_path(self.selected_project) - initial_commit = get_initial_commit(git_path).hash + repo = get_local_project_repo(self.selected_project) + initial_commit = get_initial_commit(repo).hash commits = get_all_revisions_between( - initial_commit, 'HEAD', FullCommitHash, git_path + repo, initial_commit, 'HEAD', FullCommitHash ) - commit_lookup_helper = create_commit_lookup_helper( + commit_lookup_helper = create_project_commit_lookup_helper( self.selected_project ) project = get_project_cls_by_name(self.selected_project) diff --git a/varats/varats/paper_mgmt/artefacts.py b/varats/varats/paper_mgmt/artefacts.py index f8c7873db..486f248c1 100644 --- a/varats/varats/paper_mgmt/artefacts.py +++ b/varats/varats/paper_mgmt/artefacts.py @@ -165,9 +165,7 @@ def __init__( @property def artefacts(self) -> tp.Iterable[Artefact]: - r""" - An iterator of the :class:`Artefact`\ s in this collection. - """ + r"""An iterator of the :class:`Artefact`\ s in this collection.""" return self.__artefacts.values() def get_artefact(self, name: str) -> tp.Optional[Artefact]: diff --git a/varats/varats/paper_mgmt/case_study.py b/varats/varats/paper_mgmt/case_study.py index 556441cde..c5f69de0e 100644 --- a/varats/varats/paper_mgmt/case_study.py +++ b/varats/varats/paper_mgmt/case_study.py @@ -29,7 +29,7 @@ from varats.plot.plot import Plot from varats.project.project_util import ( get_project_cls_by_name, - get_local_project_git_path, + get_local_project_repo, ) from varats.provider.bug.bug import RawBug from varats.provider.bug.bug_provider import BugProvider @@ -58,6 +58,7 @@ FullCommitHash, contains_source_code, ChurnConfig, + RepositoryHandle, ) if tp.TYPE_CHECKING: @@ -412,7 +413,7 @@ def get_unique_cs_name(case_studies: tp.List[CaseStudy]) -> tp.List[str]: ############################################################################### def extend_with_latest_rev( case_study: CaseStudy, cmap: CommitMap, merge_stage: int, - ignore_blocked: bool, git_path: str + ignore_blocked: bool, repo: RepositoryHandle ) -> None: """ Extend a case_study with the latest revision. @@ -422,11 +423,11 @@ def extend_with_latest_rev( cmap: commit map to map revisions to unique IDs merge_stage: stage to add the new revisions to ignore_blocked: ignore blocked revisions' - git_path: git path to the project + repo: git repository handle """ - repo = pygit2.Repository(pygit2.discover_repository(git_path)) + pygit2_repo = repo.pygit_repo - last_pygit_commit: pygit2.Commit = repo[repo.head.target] + last_pygit_commit: pygit2.Commit = pygit2_repo[pygit2_repo.head.target] last_commit = FullCommitHash.from_pygit_commit(last_pygit_commit) while ignore_blocked and is_revision_blocked( @@ -462,7 +463,8 @@ def extend_with_extra_revs( def extend_with_revs_per_year( case_study: CaseStudy, cmap: CommitMap, merge_stage: int, - ignore_blocked: bool, git_path: str, revs_per_year: int, revs_year_sep: bool + ignore_blocked: bool, repo: RepositoryHandle, revs_per_year: int, + revs_year_sep: bool ) -> None: """ Extend a case_study with ``revs_per_year`` revisions per year. @@ -508,13 +510,13 @@ def get_or_create_stage_for_year(year: int) -> int: case_study.name_stage(num_stages, str(year)) return num_stages - repo = pygit2.Repository(pygit2.discover_repository(git_path)) - last_commit = repo[repo.head.target] + pygit_repo = repo.pygit_repo + last_commit = pygit_repo[pygit_repo.head.target] commits: tp.DefaultDict[int, tp.List[FullCommitHash]] = defaultdict( list ) # maps year -> list of commits - for commit in repo.walk(last_commit.id, pygit2.GIT_SORT_TIME): + for commit in pygit_repo.walk(last_commit.id, pygit2.GIT_SORT_TIME): commit_date = datetime.utcfromtimestamp(commit.commit_time) commits[commit_date.year].append( FullCommitHash.from_pygit_commit(commit) @@ -572,10 +574,10 @@ def extend_with_distrib_sampling( is_code_commit: tp.Callable[[ShortCommitHash], bool] = lambda rev: True if only_code_commits: churn_conf = ChurnConfig.create_c_style_languages_config() - project_git_path = get_local_project_git_path(case_study.project_name) + repo = get_local_project_repo(case_study.project_name) def is_c_cpp_code_commit(commit: ShortCommitHash) -> bool: - return contains_source_code(commit, project_git_path, churn_conf) + return contains_source_code(repo, commit, churn_conf) is_code_commit = is_c_cpp_code_commit diff --git a/varats/varats/paper_mgmt/paper_config_manager.py b/varats/varats/paper_mgmt/paper_config_manager.py index 14c6d0d21..220661d76 100644 --- a/varats/varats/paper_mgmt/paper_config_manager.py +++ b/varats/varats/paper_mgmt/paper_config_manager.py @@ -94,8 +94,9 @@ def show_status_of_case_studies( def get_revision_list(case_study: CaseStudy) -> str: - """Returns a string with a list of revsion from the case-study, group by - case- study stages. + """ + Returns a string with a list of revsion from the case-study, group by case- + study stages. Args: case_study: to print revisions for diff --git a/varats/varats/plots/blame_interaction_graph_plots.py b/varats/varats/plots/blame_interaction_graph_plots.py index 7d06288a7..98ebd9b45 100644 --- a/varats/varats/plots/blame_interaction_graph_plots.py +++ b/varats/varats/plots/blame_interaction_graph_plots.py @@ -35,6 +35,7 @@ ArcPlotEdgeInfo, ArcPlotNodeInfo, ) +from varats.project.project_util import create_project_commit_lookup_helper from varats.ts_utils.cli_util import CLIOptionTy, make_cli_option from varats.ts_utils.click_param_types import ( REQUIRE_REVISION, @@ -43,7 +44,6 @@ from varats.utils.exceptions import UnsupportedOperation from varats.utils.git_util import ( CommitRepoPair, - create_commit_lookup_helper, UNCOMMITTED_COMMIT_HASH, FullCommitHash, ShortCommitHash, @@ -107,7 +107,7 @@ def _prepare_cig_plotly( EdgeInfoTy] ) -> tp.Tuple[tp.List[tp.Tuple[NodeTy, NodeInfoTy]], tp.List[tp.Tuple[ NodeTy, NodeTy, EdgeInfoTy]]]: - commit_lookup = create_commit_lookup_helper(project_name) + commit_lookup = create_project_commit_lookup_helper(project_name) cig = create_blame_interaction_graph( project_name, revision, BlameReportExperiment ).commit_interaction_graph() @@ -314,7 +314,9 @@ def plot(self, view_mode: bool) -> None: cig = create_blame_interaction_graph( case_study.project_name, revision, BlameReportExperiment ).commit_interaction_graph() - commit_lookup = create_commit_lookup_helper(case_study.project_name) + commit_lookup = create_project_commit_lookup_helper( + case_study.project_name + ) def filter_nodes(node: CommitRepoPair) -> bool: if node.commit_hash == UNCOMMITTED_COMMIT_HASH: diff --git a/varats/varats/plots/blame_interaction_scatter_plots.py b/varats/varats/plots/blame_interaction_scatter_plots.py index 8b6c034ae..d0fb35900 100644 --- a/varats/varats/plots/blame_interaction_scatter_plots.py +++ b/varats/varats/plots/blame_interaction_scatter_plots.py @@ -21,14 +21,14 @@ from varats.plot.plots import PlotGenerator from varats.plots.scatter_plot_utils import multivariate_grid from varats.project.project_util import ( - get_local_project_gits, - get_local_project_git_path, + get_local_project_repos, + get_local_project_repo, + create_project_commit_lookup_helper, ) from varats.ts_utils.cli_util import CLIOptionTy, make_cli_option from varats.ts_utils.click_param_types import REQUIRE_MULTI_CASE_STUDY from varats.utils.exceptions import UnsupportedOperation from varats.utils.git_util import ( - create_commit_lookup_helper, CommitRepoPair, ChurnConfig, calc_repo_code_churn, @@ -67,11 +67,11 @@ def plot(self, view_mode: bool) -> None: project_name, revision, BlameReportExperiment ).commit_interaction_graph() - commit_lookup = create_commit_lookup_helper(project_name) - repo_lookup = get_local_project_gits(project_name) + commit_lookup = create_project_commit_lookup_helper(project_name) + repo_lookup = get_local_project_repos(project_name) code_churn_lookup = { repo_name: calc_repo_code_churn( - get_local_project_git_path(project_name, repo_name), + get_local_project_repo(project_name, repo_name), ChurnConfig.create_c_style_languages_config() ) for repo_name, _ in repo_lookup.items() } diff --git a/varats/varats/plots/blame_lorenz_curve.py b/varats/varats/plots/blame_lorenz_curve.py index c27b075a8..5e06cee5f 100644 --- a/varats/varats/plots/blame_lorenz_curve.py +++ b/varats/varats/plots/blame_lorenz_curve.py @@ -19,7 +19,7 @@ build_repo_churn_table, draw_code_churn, ) -from varats.project.project_util import get_local_project_git +from varats.project.project_util import get_local_project_repo from varats.ts_utils.click_param_types import REQUIRE_MULTI_CASE_STUDY from varats.utils.exceptions import UnsupportedOperation from varats.utils.git_util import ( @@ -124,7 +124,7 @@ def filter_non_code_changes( Returns: filtered data frame without rows related to non code changes """ - repo = get_local_project_git(project_name) + repo = get_local_project_repo(project_name) code_related_changes = [ x.hash for x in calc_repo_code_churn( repo, ChurnConfig.create_c_style_languages_config() diff --git a/varats/varats/plots/bug_fixing_relations.py b/varats/varats/plots/bug_fixing_relations.py index 2bc008153..30dd3c9eb 100644 --- a/varats/varats/plots/bug_fixing_relations.py +++ b/varats/varats/plots/bug_fixing_relations.py @@ -20,7 +20,7 @@ from varats.plot.plots import PlotGenerator, PlotConfig from varats.project.project_util import ( get_project_cls_by_name, - get_local_project_git, + get_local_project_repo, ) from varats.provider.bug.bug import PygitBug, as_pygit_bug from varats.provider.bug.bug_provider import BugProvider @@ -502,7 +502,7 @@ def supports_stage_separation() -> bool: def plot(self, view_mode: bool) -> None: """Plots bug plot for the whole project.""" project_name = self.plot_kwargs['case_study'].project_name - project_repo = get_local_project_git(project_name) + project_repo = get_local_project_repo(project_name).pygit_repo bug_provider = BugProvider.get_provider_for_project( get_project_cls_by_name(project_name) diff --git a/varats/varats/plots/case_study_overview.py b/varats/varats/plots/case_study_overview.py index 3ee5db9b5..3912fde35 100644 --- a/varats/varats/plots/case_study_overview.py +++ b/varats/varats/plots/case_study_overview.py @@ -15,7 +15,7 @@ from varats.plot.plots import PlotGenerator from varats.project.project_util import ( get_project_cls_by_name, - get_local_project_git_path, + get_local_project_repo, ) from varats.report.report import FileStatusExtension from varats.ts_utils.cli_util import CLIOptionTy, make_cli_option @@ -204,9 +204,9 @@ def get_commit_hash(row: tp.Any) -> ShortCommitHash: return ShortCommitHash(str(row["revision"])) return find_missing_revisions( - revision_df.iterrows(), get_local_project_git_path(project_name), - commit_map, should_insert_revision, get_commit_hash, - head_cm_neighbours + revision_df.iterrows(), + get_local_project_repo(project_name).repo_path, commit_map, + should_insert_revision, get_commit_hash, head_cm_neighbours ) diff --git a/varats/varats/plots/code_centrality_plots.py b/varats/varats/plots/code_centrality_plots.py index e19af9170..dda6f48f8 100644 --- a/varats/varats/plots/code_centrality_plots.py +++ b/varats/varats/plots/code_centrality_plots.py @@ -1,7 +1,6 @@ """Module for code centrality plots.""" import logging import typing as tp -from pathlib import Path import matplotlib.pyplot as plt import pandas as pd @@ -18,12 +17,14 @@ ) from varats.plot.plot import Plot, PlotDataEmpty from varats.plot.plots import PlotGenerator -from varats.project.project_util import get_local_project_gits +from varats.project.project_util import ( + get_local_project_repos, + create_project_commit_lookup_helper, +) from varats.ts_utils.click_param_types import REQUIRE_CASE_STUDY from varats.utils.exceptions import UnsupportedOperation from varats.utils.git_util import ( CommitRepoPair, - create_commit_lookup_helper, ChurnConfig, calc_commit_code_churn, UNCOMMITTED_COMMIT_HASH, @@ -58,8 +59,8 @@ def plot(self, view_mode: bool) -> None: cig = create_blame_interaction_graph( project_name, revision, BlameReportExperiment ).commit_interaction_graph() - commit_lookup = create_commit_lookup_helper(project_name) - repo_lookup = get_local_project_gits(project_name) + commit_lookup = create_project_commit_lookup_helper(project_name) + repo_lookup = get_local_project_repos(project_name) def filter_nodes(node: CommitRepoPair) -> bool: if node.commit_hash == UNCOMMITTED_COMMIT_HASH: @@ -73,8 +74,8 @@ def filter_nodes(node: CommitRepoPair) -> bool: if not filter_nodes(commit): continue _, insertions, _ = calc_commit_code_churn( - Path(repo_lookup[commit.repository_name].path), - commit.commit_hash, churn_config + repo_lookup[commit.repository_name], commit.commit_hash, + churn_config ) if insertions == 0: LOG.warning(f"Churn for commit {commit} is 0.") diff --git a/varats/varats/plots/paper_config_overview.py b/varats/varats/plots/paper_config_overview.py index d90d7dd3d..ba4697bf7 100644 --- a/varats/varats/plots/paper_config_overview.py +++ b/varats/varats/plots/paper_config_overview.py @@ -15,7 +15,7 @@ from varats.paper_mgmt.case_study import get_revisions_status_for_case_study from varats.plot.plot import Plot from varats.plot.plots import PlotGenerator, PlotConfig -from varats.project.project_util import get_local_project_git +from varats.project.project_util import get_local_project_repo from varats.report.report import FileStatusExtension # colors taken from seaborn's default palette from varats.ts_utils.click_param_types import REQUIRE_EXPERIMENT_TYPE @@ -48,13 +48,13 @@ def _load_projects_ordered_by_year( case_study, experiment_type ) - repo = get_local_project_git(case_study.project_name) + pygit_repo = get_local_project_repo(case_study.project_name).pygit_repo revisions: tp.Dict[int, tp.List[tp.Tuple[ ShortCommitHash, FileStatusExtension]]] = defaultdict(list) # dict: year -> [ (revision: str, status: FileStatusExtension) ] for rev, status in processed_revisions: - commit = repo.get(rev.hash) + commit = pygit_repo.get(rev.hash) commit_date = datetime.utcfromtimestamp(commit.commit_time) revisions[commit_date.year].append((rev, status)) diff --git a/varats/varats/plots/repository_churn.py b/varats/varats/plots/repository_churn.py index 44219c5d6..5cc997036 100644 --- a/varats/varats/plots/repository_churn.py +++ b/varats/varats/plots/repository_churn.py @@ -14,7 +14,7 @@ from varats.paper.case_study import CaseStudy from varats.plot.plot import Plot from varats.plot.plots import PlotGenerator -from varats.project.project_util import get_local_project_git_path +from varats.project.project_util import get_local_project_repo from varats.ts_utils.click_param_types import REQUIRE_MULTI_CASE_STUDY from varats.utils.exceptions import UnsupportedOperation from varats.utils.git_util import ( @@ -54,10 +54,10 @@ def create_dataframe_layout() -> pd.DataFrame: df_layout.changed_files = df_layout.changed_files.astype('int64') return df_layout - repo_path = get_local_project_git_path(project_name) # By default, we only look at c-style code files code_churn = calc_repo_code_churn( - repo_path, ChurnConfig.create_c_style_languages_config() + get_local_project_repo(project_name), + ChurnConfig.create_c_style_languages_config() ) churn_data = pd.DataFrame({ "revision": list(code_churn), @@ -105,13 +105,13 @@ def create_dataframe_layout() -> pd.DataFrame: df_layout.changed_files = df_layout.changed_files.astype('int64') return df_layout - repo_path = get_local_project_git_path(project_name) + repo = get_local_project_repo(project_name) revision_pairs = zip(*(islice(revisions, i, None) for i in range(2))) code_churn = [(0, 0, 0)] code_churn.extend([ calc_code_churn( - repo_path, a, b, ChurnConfig.create_c_style_languages_config() + repo, a, b, ChurnConfig.create_c_style_languages_config() ) for a, b in revision_pairs ]) churn_data = pd.DataFrame({ diff --git a/varats/varats/projects/c_projects/asterisk.py b/varats/varats/projects/c_projects/asterisk.py index 4b104c08c..e36f7bac7 100644 --- a/varats/varats/projects/c_projects/asterisk.py +++ b/varats/varats/projects/c_projects/asterisk.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -46,9 +47,7 @@ class Asterisk(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap( - get_local_project_git_path(Asterisk.NAME) - ) + binary_map = RevisionBinaryMap(get_local_project_repo(Asterisk.NAME)) binary_map.specify_binary('./main/asterisk', BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/bison.py b/varats/varats/projects/c_projects/bison.py index 814aab110..1e3ef737b 100644 --- a/varats/varats/projects/c_projects/bison.py +++ b/varats/varats/projects/c_projects/bison.py @@ -15,11 +15,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -62,7 +63,7 @@ class Bison(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Bison.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Bison.NAME)) binary_map.specify_binary('./src/bison', BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/bitlbee.py b/varats/varats/projects/c_projects/bitlbee.py index 587fe1247..8ebd1ce85 100644 --- a/varats/varats/projects/c_projects/bitlbee.py +++ b/varats/varats/projects/c_projects/bitlbee.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -50,7 +51,7 @@ class Bitlbee(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Bitlbee.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Bitlbee.NAME)) binary_map.specify_binary('bitlbee', BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/brotli.py b/varats/varats/projects/c_projects/brotli.py index 2c9603ad6..1f9526b23 100644 --- a/varats/varats/projects/c_projects/brotli.py +++ b/varats/varats/projects/c_projects/brotli.py @@ -17,15 +17,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, -) -from varats.project.varats_project import VProject -from varats.utils.git_util import ( - ShortCommitHash, RevisionBinaryMap, - get_all_revisions_between, ) +from varats.project.varats_project import VProject +from varats.utils.git_util import ShortCommitHash, get_all_revisions_between from varats.utils.settings import bb_cfg @@ -66,7 +63,7 @@ class Brotli(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Brotli.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Brotli.NAME)) binary_map.specify_binary( "out/brotli", @@ -107,17 +104,16 @@ def run_tests(self) -> None: def compile(self) -> None: """Compile the project.""" brotli_version_source = local.path(self.source_of_primary) - brotli_git_path = get_local_project_git_path(self.NAME) + brotli_repo = get_local_project_repo(self.NAME) brotli_version = ShortCommitHash(self.version_of_primary) - with local.cwd(brotli_git_path): - configure_revisions = get_all_revisions_between( - "f9ab24a7aaee93d5932ba212e5e3d32e4306f748", - "5814438791fb2d4394b46e5682a96b68cd092803", ShortCommitHash - ) - simple_make_revisions = get_all_revisions_between( - "e1739826c04a9944672b99b98249dda021bdeb36", - "378485b097fd7b80a5e404a3cb912f7b18f78cdb", ShortCommitHash - ) + configure_revisions = get_all_revisions_between( + brotli_repo, "f9ab24a7aaee93d5932ba212e5e3d32e4306f748", + "5814438791fb2d4394b46e5682a96b68cd092803", ShortCommitHash + ) + simple_make_revisions = get_all_revisions_between( + brotli_repo, "e1739826c04a9944672b99b98249dda021bdeb36", + "378485b097fd7b80a5e404a3cb912f7b18f78cdb", ShortCommitHash + ) c_compiler = bb.compiler.cc(self) if brotli_version in simple_make_revisions: diff --git a/varats/varats/projects/c_projects/busybox.py b/varats/varats/projects/c_projects/busybox.py index 499a6bd09..301f40087 100644 --- a/varats/varats/projects/c_projects/busybox.py +++ b/varats/varats/projects/c_projects/busybox.py @@ -12,10 +12,11 @@ ProjectBinaryWrapper, BinaryType, verify_binaries, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -41,7 +42,7 @@ class Busybox(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Busybox.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Busybox.NAME)) binary_map.specify_binary("PLEASE_REPLACE_ME", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/bzip2.py b/varats/varats/projects/c_projects/bzip2.py index 150e364a8..183a28c85 100644 --- a/varats/varats/projects/c_projects/bzip2.py +++ b/varats/varats/projects/c_projects/bzip2.py @@ -16,18 +16,15 @@ from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, BinaryType, verify_binaries, + RevisionBinaryMap, ) from varats.project.sources import FeatureSource from varats.project.varats_command import VCommand from varats.project.varats_project import VProject -from varats.utils.git_util import ( - ShortCommitHash, - RevisionBinaryMap, - typed_revision_range, -) +from varats.utils.git_util import ShortCommitHash, typed_revision_range from varats.utils.settings import bb_cfg @@ -137,7 +134,7 @@ class Bzip2(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Bzip2.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Bzip2.NAME)) binary_map.specify_binary( 'build/bzip2', diff --git a/varats/varats/projects/c_projects/coreutils.py b/varats/varats/projects/c_projects/coreutils.py index a3653fd92..db9185dba 100644 --- a/varats/varats/projects/c_projects/coreutils.py +++ b/varats/varats/projects/c_projects/coreutils.py @@ -11,11 +11,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -42,9 +43,7 @@ class Coreutils(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap( - get_local_project_git_path(Coreutils.NAME) - ) + binary_map = RevisionBinaryMap(get_local_project_repo(Coreutils.NAME)) binary_map.specify_binary('src/uniq', BinaryType.EXECUTABLE) binary_map.specify_binary('src/dircolors', BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/curl.py b/varats/varats/projects/c_projects/curl.py index 48361b294..78f50e519 100644 --- a/varats/varats/projects/c_projects/curl.py +++ b/varats/varats/projects/c_projects/curl.py @@ -14,10 +14,11 @@ ProjectBinaryWrapper, BinaryType, verify_binaries, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -60,7 +61,7 @@ class Curl(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Curl.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Curl.NAME)) binary_map.specify_binary('src/.libs/curl', BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/file.py b/varats/varats/projects/c_projects/file.py index cde4ff34c..edcf56978 100644 --- a/varats/varats/projects/c_projects/file.py +++ b/varats/varats/projects/c_projects/file.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -47,7 +48,7 @@ class File(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(File.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(File.NAME)) binary_map.specify_binary("src/.libs/file", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/gawk.py b/varats/varats/projects/c_projects/gawk.py index 9056e907b..f57b5a2ed 100644 --- a/varats/varats/projects/c_projects/gawk.py +++ b/varats/varats/projects/c_projects/gawk.py @@ -13,10 +13,11 @@ ProjectBinaryWrapper, BinaryType, verify_binaries, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -51,7 +52,7 @@ class Gawk(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Gawk.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Gawk.NAME)) binary_map.specify_binary('gawk', BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/git.py b/varats/varats/projects/c_projects/git.py index c2ab0ddae..00c44ee04 100755 --- a/varats/varats/projects/c_projects/git.py +++ b/varats/varats/projects/c_projects/git.py @@ -15,10 +15,11 @@ ProjectBinaryWrapper, BinaryType, verify_binaries, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -48,7 +49,7 @@ class Git(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Git.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Git.NAME)) binary_map.specify_binary("git", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/glib.py b/varats/varats/projects/c_projects/glib.py index b7b371d8d..23ce85eb7 100644 --- a/varats/varats/projects/c_projects/glib.py +++ b/varats/varats/projects/c_projects/glib.py @@ -13,10 +13,11 @@ ProjectBinaryWrapper, BinaryType, verify_binaries, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -53,7 +54,7 @@ class Glib(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Glib.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Glib.NAME)) binary_map.specify_binary( 'build/glib/libglib-2.0.so', BinaryType.SHARED_LIBRARY diff --git a/varats/varats/projects/c_projects/glibc.py b/varats/varats/projects/c_projects/glibc.py index a0afb4f8f..3a024f1f1 100644 --- a/varats/varats/projects/c_projects/glibc.py +++ b/varats/varats/projects/c_projects/glibc.py @@ -10,11 +10,12 @@ from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, BinaryType, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -40,7 +41,7 @@ class Glibc(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Glibc.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Glibc.NAME)) binary_map.specify_binary("PLEASE_REPLACE_ME", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/gravity.py b/varats/varats/projects/c_projects/gravity.py index 0acb83a32..e212dd99e 100644 --- a/varats/varats/projects/c_projects/gravity.py +++ b/varats/varats/projects/c_projects/gravity.py @@ -17,16 +17,13 @@ from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, BinaryType, verify_binaries, -) -from varats.project.varats_project import VProject -from varats.utils.git_util import ( - ShortCommitHash, RevisionBinaryMap, - get_all_revisions_between, ) +from varats.project.varats_project import VProject +from varats.utils.git_util import ShortCommitHash, get_all_revisions_between from varats.utils.settings import bb_cfg @@ -83,7 +80,7 @@ class Gravity(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Gravity.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Gravity.NAME)) binary_map.specify_binary("gravity", BinaryType.EXECUTABLE) @@ -94,16 +91,15 @@ def run_tests(self) -> None: def compile(self) -> None: """Compile the project.""" - gravity_git_path = get_local_project_git_path(self.NAME) + gravity_repo = get_local_project_repo(self.NAME) gravity_version = self.version_of_primary # commit 46133fb47d6da1f0dec27ae23db1d633bc72e9e3 introduced # cmake as build system - with local.cwd(gravity_git_path): - cmake_revisions = get_all_revisions_between( - "dbb4d61fc2ebb9aca44e8e6bb978efac4a6def87", "master", - ShortCommitHash - ) + cmake_revisions = get_all_revisions_between( + gravity_repo, "dbb4d61fc2ebb9aca44e8e6bb978efac4a6def87", "master", + ShortCommitHash + ) if gravity_version in cmake_revisions: self.__compile_cmake() diff --git a/varats/varats/projects/c_projects/grep.py b/varats/varats/projects/c_projects/grep.py index ce95dace0..7e4f6ef7d 100644 --- a/varats/varats/projects/c_projects/grep.py +++ b/varats/varats/projects/c_projects/grep.py @@ -16,10 +16,11 @@ ProjectBinaryWrapper, BinaryType, verify_binaries, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -58,7 +59,7 @@ class Grep(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Grep.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Grep.NAME)) binary_map.specify_binary("src/grep", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/gzip.py b/varats/varats/projects/c_projects/gzip.py index f3b2a106d..beffeb4a3 100644 --- a/varats/varats/projects/c_projects/gzip.py +++ b/varats/varats/projects/c_projects/gzip.py @@ -21,19 +21,16 @@ get_tagged_commits, ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject from varats.provider.release.release_provider import ( ReleaseProviderHook, ReleaseType, ) -from varats.utils.git_util import ( - FullCommitHash, - ShortCommitHash, - RevisionBinaryMap, -) +from varats.utils.git_util import FullCommitHash, ShortCommitHash from varats.utils.settings import bb_cfg @@ -116,7 +113,7 @@ class Gzip(VProject, ReleaseProviderHook): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Gzip.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Gzip.NAME)) binary_map.specify_binary("build/gzip", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/htop.py b/varats/varats/projects/c_projects/htop.py index c9e87b8b8..bd9a881b0 100644 --- a/varats/varats/projects/c_projects/htop.py +++ b/varats/varats/projects/c_projects/htop.py @@ -15,14 +15,11 @@ ProjectBinaryWrapper, BinaryType, verify_binaries, - get_local_project_git_path, -) -from varats.project.varats_project import VProject -from varats.utils.git_util import ( - ShortCommitHash, + get_local_project_repo, RevisionBinaryMap, - typed_revision_range, ) +from varats.project.varats_project import VProject +from varats.utils.git_util import ShortCommitHash, typed_revision_range from varats.utils.settings import bb_cfg @@ -53,7 +50,7 @@ class Htop(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Htop.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Htop.NAME)) binary_map.specify_binary('htop', BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/hypre.py b/varats/varats/projects/c_projects/hypre.py index c7082f467..c2f020882 100644 --- a/varats/varats/projects/c_projects/hypre.py +++ b/varats/varats/projects/c_projects/hypre.py @@ -16,19 +16,16 @@ BinaryType, get_tagged_commits, ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject from varats.provider.release.release_provider import ( ReleaseProviderHook, ReleaseType, ) -from varats.utils.git_util import ( - RevisionBinaryMap, - ShortCommitHash, - FullCommitHash, -) +from varats.utils.git_util import ShortCommitHash, FullCommitHash from varats.utils.settings import bb_cfg @@ -64,7 +61,7 @@ class Hypre(VProject, ReleaseProviderHook): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Hypre.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Hypre.NAME)) binary_map.specify_binary('src/cmbuild/test/ij', BinaryType.EXECUTABLE) return binary_map[revision] diff --git a/varats/varats/projects/c_projects/irssi.py b/varats/varats/projects/c_projects/irssi.py index 594326c7d..160378387 100644 --- a/varats/varats/projects/c_projects/irssi.py +++ b/varats/varats/projects/c_projects/irssi.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -51,7 +52,7 @@ class Irssi(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Irssi.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Irssi.NAME)) binary_map.specify_binary('./src/fe-text/irssi', BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/libjpeg_turbo.py b/varats/varats/projects/c_projects/libjpeg_turbo.py index 049bf0e84..0df7ace02 100644 --- a/varats/varats/projects/c_projects/libjpeg_turbo.py +++ b/varats/varats/projects/c_projects/libjpeg_turbo.py @@ -13,15 +13,16 @@ ProjectBinaryWrapper, BinaryType, verify_binaries, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg class LibjpegTurbo(VProject): - """libjpeg-turbo is a JPEG image codec.""" + """Libjpeg-turbo is a JPEG image codec.""" NAME = 'libjpeg_turbo' GROUP = 'c_projects' @@ -46,7 +47,7 @@ def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(LibjpegTurbo.NAME) + get_local_project_repo(LibjpegTurbo.NAME) ) binary_map.specify_binary("libjpeg.so", BinaryType.SHARED_LIBRARY) diff --git a/varats/varats/projects/c_projects/libpng.py b/varats/varats/projects/c_projects/libpng.py index 2f53b3371..87e7b262f 100644 --- a/varats/varats/projects/c_projects/libpng.py +++ b/varats/varats/projects/c_projects/libpng.py @@ -17,11 +17,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -146,7 +147,7 @@ class Libpng(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Libpng.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Libpng.NAME)) binary_map.specify_binary('build/libpng.so', BinaryType.SHARED_LIBRARY) diff --git a/varats/varats/projects/c_projects/libsigrok.py b/varats/varats/projects/c_projects/libsigrok.py index 6790b217f..0a57f1e2c 100644 --- a/varats/varats/projects/c_projects/libsigrok.py +++ b/varats/varats/projects/c_projects/libsigrok.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -52,9 +53,7 @@ class Libsigrok(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap( - get_local_project_git_path(Libsigrok.NAME) - ) + binary_map = RevisionBinaryMap(get_local_project_repo(Libsigrok.NAME)) binary_map.specify_binary( '.libs/libsigrok.so', BinaryType.SHARED_LIBRARY diff --git a/varats/varats/projects/c_projects/libssh.py b/varats/varats/projects/c_projects/libssh.py index 1f95fa28a..e37abc5de 100644 --- a/varats/varats/projects/c_projects/libssh.py +++ b/varats/varats/projects/c_projects/libssh.py @@ -16,16 +16,13 @@ from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, BinaryType, verify_binaries, -) -from varats.project.varats_project import VProject -from varats.utils.git_util import ( - ShortCommitHash, - get_all_revisions_between, RevisionBinaryMap, ) +from varats.project.varats_project import VProject +from varats.utils.git_util import ShortCommitHash, get_all_revisions_between from varats.utils.settings import bb_cfg @@ -65,7 +62,7 @@ class Libssh(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Libssh.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Libssh.NAME)) binary_map.specify_binary( 'build/src/libssh.so', @@ -86,11 +83,11 @@ def run_tests(self) -> None: def compile(self) -> None: """Compile the project.""" - libssh_source = local.path(self.source_of(self.primary_source)) + libssh_repo = get_local_project_repo(self.NAME) libssh_version = ShortCommitHash(self.version_of_primary) cmake_revisions = get_all_revisions_between( - "0151b6e17041c56813c882a3de6330c82acc8d93", "master", - ShortCommitHash, libssh_source + libssh_repo, "0151b6e17041c56813c882a3de6330c82acc8d93", "master", + ShortCommitHash ) if libssh_version in cmake_revisions: self.__compile_cmake() @@ -112,17 +109,20 @@ def __compile_cmake(self) -> None: verify_binaries(self) def __compile_make(self) -> None: + libssh_repo = get_local_project_repo(self.NAME) libssh_source = local.path(self.source_of(self.primary_source)) libssh_version = ShortCommitHash(self.version_of_primary) autoconf_revisions = get_all_revisions_between( + libssh_repo, "5e02c25291d594e01a910fce097a3fc5084fd68f", - "21e639cc3fd54eb3d59568744c9627beb26e07ed", ShortCommitHash, - libssh_source + "21e639cc3fd54eb3d59568744c9627beb26e07ed", + ShortCommitHash, ) autogen_revisions = get_all_revisions_between( + libssh_repo, "ca32b0aa146b31d7772f27d16098845e615432aa", - "ee54acb417c5589a8dc9dab0676f34b3d40a182b", ShortCommitHash, - libssh_source + "ee54acb417c5589a8dc9dab0676f34b3d40a182b", + ShortCommitHash, ) compiler = bb.compiler.cc(self) with local.cwd(libssh_source): diff --git a/varats/varats/projects/c_projects/libtiff.py b/varats/varats/projects/c_projects/libtiff.py index d1359463b..36ed2fbea 100644 --- a/varats/varats/projects/c_projects/libtiff.py +++ b/varats/varats/projects/c_projects/libtiff.py @@ -17,11 +17,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -111,7 +112,7 @@ class Libtiff(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Libtiff.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Libtiff.NAME)) binary_map.specify_binary( "libtiff/.libs/libtiff.so", BinaryType.SHARED_LIBRARY diff --git a/varats/varats/projects/c_projects/libvpx.py b/varats/varats/projects/c_projects/libvpx.py index a9741da40..04a6d9182 100644 --- a/varats/varats/projects/c_projects/libvpx.py +++ b/varats/varats/projects/c_projects/libvpx.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -45,7 +46,7 @@ class Libvpx(VProject): def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Libvpx.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Libvpx.NAME)) binary_map.specify_binary("vpxdec", BinaryType.EXECUTABLE) binary_map.specify_binary("vpxenc", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/libxml2.py b/varats/varats/projects/c_projects/libxml2.py index 29f460153..91ec8d7c4 100644 --- a/varats/varats/projects/c_projects/libxml2.py +++ b/varats/varats/projects/c_projects/libxml2.py @@ -14,16 +14,17 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg class Libxml2(VProject): - """libxml2 is a software library for parsing XML documents.""" + """Libxml2 is a software library for parsing XML documents.""" NAME = 'libxml2' GROUP = 'c_projects' @@ -52,7 +53,7 @@ class Libxml2(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Libxml2.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Libxml2.NAME)) binary_map.specify_binary("libxml2.so", BinaryType.SHARED_LIBRARY) diff --git a/varats/varats/projects/c_projects/lrzip.py b/varats/varats/projects/c_projects/lrzip.py index ce1cccb48..f6d458341 100644 --- a/varats/varats/projects/c_projects/lrzip.py +++ b/varats/varats/projects/c_projects/lrzip.py @@ -15,11 +15,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -70,7 +71,7 @@ class Lrzip(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Lrzip.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Lrzip.NAME)) binary_map.specify_binary("lrzip", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/lz4.py b/varats/varats/projects/c_projects/lz4.py index 979ae7ecb..bce8d4bcc 100644 --- a/varats/varats/projects/c_projects/lz4.py +++ b/varats/varats/projects/c_projects/lz4.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -48,7 +49,7 @@ class Lz4(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Lz4.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Lz4.NAME)) binary_map.specify_binary('lz4', BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/openssl.py b/varats/varats/projects/c_projects/openssl.py index d8b294262..28ecebbd9 100644 --- a/varats/varats/projects/c_projects/openssl.py +++ b/varats/varats/projects/c_projects/openssl.py @@ -12,15 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, -) -from varats.project.varats_project import VProject -from varats.utils.git_util import ( - ShortCommitHash, RevisionBinaryMap, - get_all_revisions_between, ) +from varats.project.varats_project import VProject +from varats.utils.git_util import ShortCommitHash, get_all_revisions_between from varats.utils.settings import bb_cfg @@ -49,7 +46,7 @@ class OpenSSL(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(OpenSSL.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(OpenSSL.NAME)) binary_map.specify_binary("apps/openssl", BinaryType.EXECUTABLE) # binary_map.specify_binary("libssl.so", BinaryType.SHARED_LIBRARY) @@ -61,15 +58,14 @@ def run_tests(self) -> None: def compile(self) -> None: """Compile the project.""" - openssl_git_path = get_local_project_git_path(self.NAME) + openssl_repo = get_local_project_repo(self.NAME) openssl_version = ShortCommitHash(self.version_of_primary) openssl_source = local.path(self.source_of_primary) - with local.cwd(openssl_git_path): - configure_bug_revisions = get_all_revisions_between( - "486f149131e94c970da1b89ebe8c66ab88e5d343", - "5723a8ec514930c7c49d080cd7a2b17a8f8c7afa", ShortCommitHash - ) + configure_bug_revisions = get_all_revisions_between( + openssl_repo, "486f149131e94c970da1b89ebe8c66ab88e5d343", + "5723a8ec514930c7c49d080cd7a2b17a8f8c7afa", ShortCommitHash + ) compiler = bb.compiler.cc(self) with local.cwd(openssl_source): diff --git a/varats/varats/projects/c_projects/openvpn.py b/varats/varats/projects/c_projects/openvpn.py index 7e4cbef9e..118ca618b 100644 --- a/varats/varats/projects/c_projects/openvpn.py +++ b/varats/varats/projects/c_projects/openvpn.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -54,7 +55,7 @@ class OpenVPN(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(OpenVPN.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(OpenVPN.NAME)) binary_map.specify_binary( './src/openvpn/openvpn', BinaryType.EXECUTABLE diff --git a/varats/varats/projects/c_projects/opus.py b/varats/varats/projects/c_projects/opus.py index 1d6a46f12..3b89f51cd 100644 --- a/varats/varats/projects/c_projects/opus.py +++ b/varats/varats/projects/c_projects/opus.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -47,7 +48,7 @@ class Opus(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Opus.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Opus.NAME)) binary_map.specify_binary(".libs/libopus.so", BinaryType.SHARED_LIBRARY) diff --git a/varats/varats/projects/c_projects/picosat.py b/varats/varats/projects/c_projects/picosat.py index 569f38a2f..500fa98e7 100644 --- a/varats/varats/projects/c_projects/picosat.py +++ b/varats/varats/projects/c_projects/picosat.py @@ -21,8 +21,9 @@ BinaryType, get_tagged_commits, ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.sources import FeatureSource from varats.project.varats_command import VCommand @@ -32,7 +33,6 @@ ReleaseType, ) from varats.utils.git_util import ( - RevisionBinaryMap, ShortCommitHash, FullCommitHash, get_all_revisions_between, @@ -41,7 +41,7 @@ class PicoSAT(VProject, ReleaseProviderHook): - """picoSAT is a SAT solver.""" + """PicoSAT is a SAT solver.""" NAME = 'picosat' GROUP = 'c_projects' @@ -148,7 +148,7 @@ class PicoSAT(VProject, ReleaseProviderHook): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(PicoSAT.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(PicoSAT.NAME)) binary_map.specify_binary( 'picosat', BinaryType.EXECUTABLE, valid_exit_codes=[0, 10, 20] ) @@ -160,15 +160,16 @@ def run_tests(self) -> None: def compile(self) -> None: """Compile the project.""" + picosat_repo = get_local_project_repo(self.NAME) picosat_source = local.path(self.source_of(self.primary_source)) c_compiler = bb.compiler.cc(self) cxx_compiler = bb.compiler.cxx(self) - with local.cwd(picosat_source): - revisions_with_new_config_name = get_all_revisions_between( - "33c685e82213228726364980814f0183e435de78", "", ShortCommitHash - ) + revisions_with_new_config_name = get_all_revisions_between( + picosat_repo, "33c685e82213228726364980814f0183e435de78", "", + ShortCommitHash + ) picosat_version = ShortCommitHash(self.version_of_primary) if picosat_version in revisions_with_new_config_name: config_script_name = "./configure.sh" @@ -282,7 +283,7 @@ def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(PicoSATLoadTime.NAME) + get_local_project_repo(PicoSATLoadTime.NAME) ) binary_map.specify_binary( 'picosat', BinaryType.EXECUTABLE, valid_exit_codes=[0, 10, 20] @@ -295,15 +296,16 @@ def run_tests(self) -> None: def compile(self) -> None: """Compile the project.""" - picosat_source = local.path(self.source_of(self.primary_source)) + picosat_repo = get_local_project_repo(self.NAME) + picosat_source = local.path(self.source_of_primary) c_compiler = bb.compiler.cc(self) cxx_compiler = bb.compiler.cxx(self) - with local.cwd(picosat_source): - revisions_with_new_config_name = get_all_revisions_between( - "33c685e82213228726364980814f0183e435de78", "", ShortCommitHash - ) + revisions_with_new_config_name = get_all_revisions_between( + picosat_repo, "33c685e82213228726364980814f0183e435de78", "", + ShortCommitHash + ) picosat_version = ShortCommitHash(self.version_of_primary) if picosat_version in revisions_with_new_config_name: config_script_name = "./configure.sh" diff --git a/varats/varats/projects/c_projects/qemu.py b/varats/varats/projects/c_projects/qemu.py index 558669d53..292a72139 100644 --- a/varats/varats/projects/c_projects/qemu.py +++ b/varats/varats/projects/c_projects/qemu.py @@ -11,11 +11,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -45,7 +46,7 @@ class Qemu(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Qemu.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Qemu.NAME)) binary_map.specify_binary( "build/x86_64-softmmu/qemu-system-x86_64", BinaryType.EXECUTABLE diff --git a/varats/varats/projects/c_projects/redis.py b/varats/varats/projects/c_projects/redis.py index aa61fe251..e15c0d999 100644 --- a/varats/varats/projects/c_projects/redis.py +++ b/varats/varats/projects/c_projects/redis.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -48,7 +49,7 @@ class Redis(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Redis.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Redis.NAME)) binary_map.specify_binary( 'src/redis-server', diff --git a/varats/varats/projects/c_projects/tig.py b/varats/varats/projects/c_projects/tig.py index 659930819..9fdd507d9 100644 --- a/varats/varats/projects/c_projects/tig.py +++ b/varats/varats/projects/c_projects/tig.py @@ -11,12 +11,13 @@ from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, BinaryType, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -46,7 +47,7 @@ class Tig(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Tig.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Tig.NAME)) binary_map.specify_binary("src/tig", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/tmux.py b/varats/varats/projects/c_projects/tmux.py index 9d0174c61..cf77a551a 100644 --- a/varats/varats/projects/c_projects/tmux.py +++ b/varats/varats/projects/c_projects/tmux.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -47,7 +48,7 @@ class Tmux(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Tmux.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Tmux.NAME)) binary_map.specify_binary("tmux", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/vim.py b/varats/varats/projects/c_projects/vim.py index 2acd3a5ec..7d52ce159 100644 --- a/varats/varats/projects/c_projects/vim.py +++ b/varats/varats/projects/c_projects/vim.py @@ -11,11 +11,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -41,7 +42,7 @@ class Vim(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Vim.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Vim.NAME)) binary_map.specify_binary("src/vim", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/c_projects/x264.py b/varats/varats/projects/c_projects/x264.py index e0b1dcee0..6baef6949 100644 --- a/varats/varats/projects/c_projects/x264.py +++ b/varats/varats/projects/c_projects/x264.py @@ -12,16 +12,13 @@ from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, BinaryType, verify_binaries, -) -from varats.project.varats_project import VProject -from varats.utils.git_util import ( - ShortCommitHash, - get_all_revisions_between, RevisionBinaryMap, ) +from varats.project.varats_project import VProject +from varats.utils.git_util import ShortCommitHash, get_all_revisions_between from varats.utils.settings import bb_cfg @@ -55,7 +52,7 @@ class X264(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(X264.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(X264.NAME)) binary_map.specify_binary("x264", BinaryType.EXECUTABLE) @@ -66,18 +63,21 @@ def run_tests(self) -> None: def compile(self) -> None: """Compile the project.""" + x264_repo = get_local_project_repo(self.NAME) x264_version_source = local.path(self.source_of_primary) x264_version = ShortCommitHash(self.version_of_primary) fpic_revisions = get_all_revisions_between( + x264_repo, "5dc0aae2f900064d1f58579929a2285ab289a436", - "290de9638e5364c37316010ac648a6c959f6dd26", ShortCommitHash, - x264_version_source + "290de9638e5364c37316010ac648a6c959f6dd26", + ShortCommitHash, ) ldflags_revisions = get_all_revisions_between( + x264_repo, "6490f4398d9e28e65d7517849e729e14eede8c5b", - "275ef5332dffec445a0c5a78dbc00c3e0766011d", ShortCommitHash, - x264_version_source + "275ef5332dffec445a0c5a78dbc00c3e0766011d", + ShortCommitHash, ) if x264_version in fpic_revisions: diff --git a/varats/varats/projects/c_projects/xz.py b/varats/varats/projects/c_projects/xz.py index 9f0f1e73e..0ba48b062 100644 --- a/varats/varats/projects/c_projects/xz.py +++ b/varats/varats/projects/c_projects/xz.py @@ -19,18 +19,15 @@ from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, BinaryType, verify_binaries, + RevisionBinaryMap, ) from varats.project.sources import FeatureSource from varats.project.varats_command import VCommand from varats.project.varats_project import VProject -from varats.utils.git_util import ( - ShortCommitHash, - get_all_revisions_between, - RevisionBinaryMap, -) +from varats.utils.git_util import ShortCommitHash, get_all_revisions_between from varats.utils.settings import bb_cfg @@ -120,7 +117,7 @@ class Xz(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Xz.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Xz.NAME)) binary_map.specify_binary( 'src/xz/xz', @@ -142,18 +139,17 @@ def run_tests(self) -> None: def compile(self) -> None: """Compile the project.""" - xz_git_path = get_local_project_git_path(self.NAME) + xz_repo = get_local_project_repo(self.NAME) xz_version_source = local.path(self.source_of_primary) xz_version = self.version_of_primary # dynamic linking is off by default until # commit f9907503f882a745dce9d84c2968f6c175ba966a # (fda4724 is its parent) - with local.cwd(xz_git_path): - revisions_wo_dynamic_linking = get_all_revisions_between( - "5d018dc03549c1ee4958364712fb0c94e1bf2741", - "fda4724d8114fccfa31c1839c15479f350c2fb4c", ShortCommitHash - ) + revisions_wo_dynamic_linking = get_all_revisions_between( + xz_repo, "5d018dc03549c1ee4958364712fb0c94e1bf2741", + "fda4724d8114fccfa31c1839c15479f350c2fb4c", ShortCommitHash + ) self.cflags += ["-fPIC"] diff --git a/varats/varats/projects/cpp_projects/clasp.py b/varats/varats/projects/cpp_projects/clasp.py index e32759cfb..f7fd6ed4a 100644 --- a/varats/varats/projects/cpp_projects/clasp.py +++ b/varats/varats/projects/cpp_projects/clasp.py @@ -13,24 +13,21 @@ BinaryType, get_tagged_commits, ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject from varats.provider.release.release_provider import ( ReleaseProviderHook, ReleaseType, ) -from varats.utils.git_util import ( - RevisionBinaryMap, - ShortCommitHash, - FullCommitHash, -) +from varats.utils.git_util import ShortCommitHash, FullCommitHash from varats.utils.settings import bb_cfg class Clasp(VProject, ReleaseProviderHook): - """clasp is an answer set solver for (extended) normal and disjunctive logic + """Clasp is an answer set solver for (extended) normal and disjunctive logic programs.""" NAME = 'clasp' @@ -52,7 +49,7 @@ class Clasp(VProject, ReleaseProviderHook): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Clasp.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Clasp.NAME)) binary_map.specify_binary('build/bin/clasp', BinaryType.EXECUTABLE) return binary_map[revision] diff --git a/varats/varats/projects/cpp_projects/doxygen.py b/varats/varats/projects/cpp_projects/doxygen.py index c49df9a99..adc16631a 100644 --- a/varats/varats/projects/cpp_projects/doxygen.py +++ b/varats/varats/projects/cpp_projects/doxygen.py @@ -14,11 +14,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -62,7 +63,7 @@ class Doxygen(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Doxygen.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Doxygen.NAME)) binary_map.specify_binary('doxygen', BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/cpp_projects/dune.py b/varats/varats/projects/cpp_projects/dune.py index 4d3f58a91..1caadd8e3 100644 --- a/varats/varats/projects/cpp_projects/dune.py +++ b/varats/varats/projects/cpp_projects/dune.py @@ -12,14 +12,15 @@ from varats.paper.paper_config import PaperConfigSpecificGit from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( - get_local_project_git_path, + get_local_project_repo, BinaryType, ProjectBinaryWrapper, + RevisionBinaryMap, ) from varats.project.sources import FeatureSource from varats.project.varats_command import VCommand from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash class DunePerfRegression(VProject): @@ -130,7 +131,7 @@ def binaries_for_revision( revision: ShortCommitHash ) -> tp.List['ProjectBinaryWrapper']: binary_map = RevisionBinaryMap( - get_local_project_git_path(DunePerfRegression.NAME) + get_local_project_repo(DunePerfRegression.NAME) ) rev_range = RevisionRange( diff --git a/varats/varats/projects/cpp_projects/ect.py b/varats/varats/projects/cpp_projects/ect.py index 13f04a1b6..42c853705 100644 --- a/varats/varats/projects/cpp_projects/ect.py +++ b/varats/varats/projects/cpp_projects/ect.py @@ -11,11 +11,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash class Ect(VProject): @@ -48,7 +49,7 @@ class Ect(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Ect.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Ect.NAME)) binary_map.specify_binary("build/ect", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/cpp_projects/fast_downward.py b/varats/varats/projects/cpp_projects/fast_downward.py index b7be8c68b..c71e4c1e3 100644 --- a/varats/varats/projects/cpp_projects/fast_downward.py +++ b/varats/varats/projects/cpp_projects/fast_downward.py @@ -13,20 +13,17 @@ from varats.project.project_util import ( BinaryType, ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, get_tagged_commits, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject from varats.provider.release.release_provider import ( ReleaseProviderHook, ReleaseType, ) -from varats.utils.git_util import ( - FullCommitHash, - RevisionBinaryMap, - ShortCommitHash, -) +from varats.utils.git_util import FullCommitHash, ShortCommitHash from varats.utils.settings import bb_cfg @@ -57,7 +54,7 @@ def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(FastDownward.NAME) + get_local_project_repo(FastDownward.NAME) ) binary_map.specify_binary('build/bin/downward', BinaryType.EXECUTABLE) @@ -88,7 +85,7 @@ def compile(self) -> None: def get_release_revisions( cls, release_type: ReleaseType ) -> tp.List[tp.Tuple[FullCommitHash, str]]: - repo_loc = get_local_project_git_path(cls.NAME) + repo_loc = get_local_project_repo(cls.NAME) with local.cwd(repo_loc): # Before 2019_07, there were no real releases, but the following # commits were identified as suitable. diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py index ecd90ca5b..37768d41f 100644 --- a/varats/varats/projects/cpp_projects/hyteg.py +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -13,15 +13,16 @@ from varats.paper.paper_config import PaperConfigSpecificGit from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( - get_local_project_git_path, + get_local_project_repo, BinaryType, ProjectBinaryWrapper, + RevisionBinaryMap, ) from varats.project.sources import FeatureSource from varats.project.varats_command import VCommand from varats.project.varats_project import VProject from varats.utils.git_commands import update_all_submodules -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash LOG = logging.getLogger(__name__) @@ -90,7 +91,7 @@ class HyTeg(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List['ProjectBinaryWrapper']: - binaries = RevisionBinaryMap(get_local_project_git_path(HyTeg.NAME)) + binaries = RevisionBinaryMap(get_local_project_repo(HyTeg.NAME)) binaries.specify_binary( "ProfilingApp", diff --git a/varats/varats/projects/cpp_projects/lepton.py b/varats/varats/projects/cpp_projects/lepton.py index cd6f23a73..479e88e1b 100644 --- a/varats/varats/projects/cpp_projects/lepton.py +++ b/varats/varats/projects/cpp_projects/lepton.py @@ -11,11 +11,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash class Lepton(VProject): @@ -45,7 +46,7 @@ class Lepton(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Lepton.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Lepton.NAME)) binary_map.specify_binary("build/lepton", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/cpp_projects/libzmq.py b/varats/varats/projects/cpp_projects/libzmq.py index fe59c823f..50a823790 100644 --- a/varats/varats/projects/cpp_projects/libzmq.py +++ b/varats/varats/projects/cpp_projects/libzmq.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -49,7 +50,7 @@ class Libzmq(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Libzmq.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Libzmq.NAME)) binary_map.specify_binary( "build/lib/libzmq.so", BinaryType.SHARED_LIBRARY diff --git a/varats/varats/projects/cpp_projects/mongodb.py b/varats/varats/projects/cpp_projects/mongodb.py index 8ae87b8bd..24f6becd0 100644 --- a/varats/varats/projects/cpp_projects/mongodb.py +++ b/varats/varats/projects/cpp_projects/mongodb.py @@ -11,11 +11,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -46,7 +47,7 @@ class MongoDB(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(MongoDB.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(MongoDB.NAME)) # TODO: please add correct binary names binary_map.specify_binary("MISSING", BinaryType.EXECUTABLE) diff --git a/varats/varats/projects/cpp_projects/opencv.py b/varats/varats/projects/cpp_projects/opencv.py index f21474248..bb49fde39 100644 --- a/varats/varats/projects/cpp_projects/opencv.py +++ b/varats/varats/projects/cpp_projects/opencv.py @@ -12,11 +12,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -47,7 +48,7 @@ class OpenCV(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(OpenCV.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(OpenCV.NAME)) binary_map.specify_binary( 'build/lib/libopencv_core.so', BinaryType.SHARED_LIBRARY diff --git a/varats/varats/projects/cpp_projects/poppler.py b/varats/varats/projects/cpp_projects/poppler.py index b5ad4a0ab..3471e0e00 100644 --- a/varats/varats/projects/cpp_projects/poppler.py +++ b/varats/varats/projects/cpp_projects/poppler.py @@ -12,12 +12,13 @@ from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, BinaryType, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -57,7 +58,7 @@ class Poppler(VProject): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Poppler.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Poppler.NAME)) binary_map.specify_binary("libpoppler.so", BinaryType.SHARED_LIBRARY) diff --git a/varats/varats/projects/cpp_projects/two_libs_one_project_interaction_discrete_libs_single_project.py b/varats/varats/projects/cpp_projects/two_libs_one_project_interaction_discrete_libs_single_project.py index 66721b6e5..6fb7a6467 100644 --- a/varats/varats/projects/cpp_projects/two_libs_one_project_interaction_discrete_libs_single_project.py +++ b/varats/varats/projects/cpp_projects/two_libs_one_project_interaction_discrete_libs_single_project.py @@ -12,16 +12,17 @@ from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, BinaryType, verify_binaries, + RevisionBinaryMap, ) from varats.project.varats_project import VProject from varats.ts_utils.project_sources import ( VaraTestRepoSource, VaraTestRepoSubmodule, ) -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -92,7 +93,7 @@ def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path( + get_local_project_repo( TwoLibsOneProjectInteractionDiscreteLibsSingleProject.NAME ) ) diff --git a/varats/varats/projects/cpp_projects/z3.py b/varats/varats/projects/cpp_projects/z3.py index d5894107c..777d4f8cd 100644 --- a/varats/varats/projects/cpp_projects/z3.py +++ b/varats/varats/projects/cpp_projects/z3.py @@ -14,20 +14,17 @@ from varats.project.project_util import ( BinaryType, ProjectBinaryWrapper, - get_local_project_git_path, + get_local_project_repo, verify_binaries, get_tagged_commits, + RevisionBinaryMap, ) from varats.project.varats_project import VProject from varats.provider.release.release_provider import ( ReleaseProviderHook, ReleaseType, ) -from varats.utils.git_util import ( - RevisionBinaryMap, - ShortCommitHash, - FullCommitHash, -) +from varats.utils.git_util import ShortCommitHash, FullCommitHash from varats.utils.settings import bb_cfg @@ -55,7 +52,7 @@ class Z3(VProject, ReleaseProviderHook): def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(Z3.NAME)) + binary_map = RevisionBinaryMap(get_local_project_repo(Z3.NAME)) binary_map.specify_binary('build/z3', BinaryType.EXECUTABLE) return binary_map[revision] diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index 07e1fadb0..60305dbb3 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -22,14 +22,15 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, verify_binaries, + RevisionBinaryMap, ) from varats.project.sources import FeatureSource from varats.project.varats_command import VCommand from varats.project.varats_project import VProject from varats.utils.git_commands import init_all_submodules, update_all_submodules -from varats.utils.git_util import RevisionBinaryMap, ShortCommitHash +from varats.utils.git_util import ShortCommitHash, RepositoryHandle from varats.utils.settings import bb_cfg @@ -37,23 +38,23 @@ def _do_feature_perf_cs_collection_compile( project: VProject, cmake_flag: str ) -> None: """Common compile function for FeaturePerfCSCollection projects.""" - feature_perf_source = local.path(project.source_of(project.primary_source)) + feature_perf_repo = RepositoryHandle(Path(project.source_of_primary)) cc_compiler = bb.compiler.cc(project) cxx_compiler = bb.compiler.cxx(project) - mkdir("-p", feature_perf_source / "build") + mkdir("-p", feature_perf_repo.worktree_path / "build") - init_all_submodules(Path(feature_perf_source)) - update_all_submodules(Path(feature_perf_source)) + init_all_submodules(feature_perf_repo) + update_all_submodules(feature_perf_repo) - with local.cwd(feature_perf_source / "build"): + with local.cwd(feature_perf_repo.worktree_path / "build"): with local.env(CC=str(cc_compiler), CXX=str(cxx_compiler)): bb.watch(cmake)("..", "-G", "Unix Makefiles", f"-D{cmake_flag}=ON") bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) - with local.cwd(feature_perf_source): + with local.cwd(feature_perf_repo.worktree_path): verify_binaries(project) @@ -123,7 +124,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(FeaturePerfCSCollection.NAME) + get_local_project_repo(FeaturePerfCSCollection.NAME) ) binary_map.specify_binary( @@ -206,7 +207,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthSAFlowSensitivity.NAME) + get_local_project_repo(SynthSAFlowSensitivity.NAME) ) binary_map.specify_binary( @@ -270,7 +271,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthSAContextSensitivity.NAME) + get_local_project_repo(SynthSAContextSensitivity.NAME) ) binary_map.specify_binary( @@ -331,7 +332,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthSAWholeProgram.NAME) + get_local_project_repo(SynthSAWholeProgram.NAME) ) binary_map.specify_binary( @@ -394,7 +395,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthDADynamicDispatch.NAME) + get_local_project_repo(SynthDADynamicDispatch.NAME) ) binary_map.specify_binary( @@ -455,7 +456,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthDARecursion.NAME) + get_local_project_repo(SynthDARecursion.NAME) ) binary_map.specify_binary( @@ -516,7 +517,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthOVInsideLoop.NAME) + get_local_project_repo(SynthOVInsideLoop.NAME) ) binary_map.specify_binary( @@ -579,7 +580,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthFeatureInteraction.NAME) + get_local_project_repo(SynthFeatureInteraction.NAME) ) binary_map.specify_binary( @@ -644,7 +645,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthFeatureHigherOrderInteraction.NAME) + get_local_project_repo(SynthFeatureHigherOrderInteraction.NAME) ) binary_map.specify_binary( @@ -859,7 +860,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: return RevisionBinaryMap( - get_local_project_git_path(SynthIPRuntime.NAME) + get_local_project_repo(SynthIPRuntime.NAME) ).specify_binary( "build/bin/Runtime", BinaryType.EXECUTABLE, @@ -907,7 +908,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: return RevisionBinaryMap( - get_local_project_git_path(SynthIPTemplate.NAME) + get_local_project_repo(SynthIPTemplate.NAME) ).specify_binary( "build/bin/Template", BinaryType.EXECUTABLE, @@ -955,7 +956,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: return RevisionBinaryMap( - get_local_project_git_path(SynthIPTemplate2.NAME) + get_local_project_repo(SynthIPTemplate2.NAME) ).specify_binary( "build/bin/Template2", BinaryType.EXECUTABLE, @@ -1003,7 +1004,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: return RevisionBinaryMap( - get_local_project_git_path(SynthIPCombined.NAME) + get_local_project_repo(SynthIPCombined.NAME) ).specify_binary( "build/bin/Combined", BinaryType.EXECUTABLE, @@ -1053,7 +1054,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthSAFieldSensitivity.NAME) + get_local_project_repo(SynthSAFieldSensitivity.NAME) ) binary_map.specify_binary( @@ -1111,7 +1112,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthCTTraitBased.NAME) + get_local_project_repo(SynthCTTraitBased.NAME) ) binary_map.specify_binary( @@ -1170,7 +1171,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthCTPolicies.NAME) + get_local_project_repo(SynthCTPolicies.NAME) ) binary_map.specify_binary( @@ -1226,9 +1227,7 @@ class SynthCTCRTP(VProject): def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthCTCRTP.NAME) - ) + binary_map = RevisionBinaryMap(get_local_project_repo(SynthCTCRTP.NAME)) binary_map.specify_binary( "build/bin/CTCRTP", @@ -1286,7 +1285,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SynthCTTemplateSpecialization.NAME) + get_local_project_repo(SynthCTTemplateSpecialization.NAME) ) binary_map.specify_binary( @@ -1348,7 +1347,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: return RevisionBinaryMap( - get_local_project_git_path(SynthFeatureLargeConfigSpace.NAME) + get_local_project_repo(SynthFeatureLargeConfigSpace.NAME) ).specify_binary( "build/bin/LargeConfigSpace", BinaryType.EXECUTABLE, @@ -1406,7 +1405,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: return RevisionBinaryMap( - get_local_project_git_path(SynthFeatureRestrictedConfigSpace.NAME) + get_local_project_repo(SynthFeatureRestrictedConfigSpace.NAME) ).specify_binary( "build/bin/RestrictedConfigSpace", BinaryType.EXECUTABLE, diff --git a/varats/varats/projects/perf_tests/feature_perf_regression.py b/varats/varats/projects/perf_tests/feature_perf_regression.py index c1b8961f7..33257b8f7 100644 --- a/varats/varats/projects/perf_tests/feature_perf_regression.py +++ b/varats/varats/projects/perf_tests/feature_perf_regression.py @@ -14,9 +14,10 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, verify_binaries, get_tagged_commits, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.sources import FeatureSource from varats.project.varats_project import VProject @@ -24,11 +25,7 @@ ReleaseProviderHook, ReleaseType, ) -from varats.utils.git_util import ( - RevisionBinaryMap, - ShortCommitHash, - FullCommitHash, -) +from varats.utils.git_util import ShortCommitHash, FullCommitHash from varats.utils.settings import bb_cfg @@ -66,7 +63,7 @@ def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(FeaturePerfRegression.NAME) + get_local_project_repo(FeaturePerfRegression.NAME) ) binary_map.specify_binary( 'build/bin/CompressionTool', BinaryType.EXECUTABLE diff --git a/varats/varats/projects/test_projects/bug_provider_test_repos.py b/varats/varats/projects/test_projects/bug_provider_test_repos.py index 51ab69aba..1f7478421 100644 --- a/varats/varats/projects/test_projects/bug_provider_test_repos.py +++ b/varats/varats/projects/test_projects/bug_provider_test_repos.py @@ -8,11 +8,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.varats_project import VProject from varats.ts_utils.project_sources import VaraTestRepoSource -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash class BasicBugDetectionTestRepo(VProject): @@ -38,7 +39,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(BasicBugDetectionTestRepo.NAME) + get_local_project_repo(BasicBugDetectionTestRepo.NAME) ).specify_binary("main", BinaryType.EXECUTABLE) return binary_map[revision] diff --git a/varats/varats/projects/test_projects/example_test_repo.py b/varats/varats/projects/test_projects/example_test_repo.py index 739d73cdd..df4745e3e 100644 --- a/varats/varats/projects/test_projects/example_test_repo.py +++ b/varats/varats/projects/test_projects/example_test_repo.py @@ -8,11 +8,12 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.varats_project import VProject from varats.ts_utils.project_sources import VaraTestRepoSource -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash class ExampleTestRepo(VProject): @@ -38,7 +39,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(ExampleTestRepo.NAME) + get_local_project_repo(ExampleTestRepo.NAME) ).specify_binary("main", BinaryType.EXECUTABLE) return binary_map[revision] diff --git a/varats/varats/projects/test_projects/feature_test_repo.py b/varats/varats/projects/test_projects/feature_test_repo.py index ca2788b80..5d26c6e03 100644 --- a/varats/varats/projects/test_projects/feature_test_repo.py +++ b/varats/varats/projects/test_projects/feature_test_repo.py @@ -11,12 +11,13 @@ ProjectBinaryWrapper, verify_binaries, BinaryType, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.sources import FeatureSource from varats.project.varats_project import VProject from varats.ts_utils.project_sources import VaraTestRepoSource -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash class FeatureTestRepo(VProject): @@ -51,7 +52,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(FeatureTestRepo.NAME) + get_local_project_repo(FeatureTestRepo.NAME) ).specify_binary("main", BinaryType.EXECUTABLE) return binary_map[revision] @@ -95,7 +96,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(CommitFeatureInteractionExample.NAME) + get_local_project_repo(CommitFeatureInteractionExample.NAME) ).specify_binary("main", BinaryType.EXECUTABLE) return binary_map[revision] diff --git a/varats/varats/projects/test_projects/multi_author_coordination.py b/varats/varats/projects/test_projects/multi_author_coordination.py index 1b6f13a13..bd7fc404d 100644 --- a/varats/varats/projects/test_projects/multi_author_coordination.py +++ b/varats/varats/projects/test_projects/multi_author_coordination.py @@ -11,11 +11,12 @@ ProjectBinaryWrapper, verify_binaries, BinaryType, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.varats_project import VProject from varats.ts_utils.project_sources import VaraTestRepoSource -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg @@ -42,7 +43,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(MultiAuthorCoordination.NAME) + get_local_project_repo(MultiAuthorCoordination.NAME) ).specify_binary("build/computeService", BinaryType.EXECUTABLE) return binary_map[revision] diff --git a/varats/varats/projects/test_projects/taint_tests.py b/varats/varats/projects/test_projects/taint_tests.py index 18d8e9679..b8e7702fe 100644 --- a/varats/varats/projects/test_projects/taint_tests.py +++ b/varats/varats/projects/test_projects/taint_tests.py @@ -8,10 +8,11 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import RevisionBinaryMap, ShortCommitHash +from varats.utils.git_util import ShortCommitHash class TaintTests(VProject): @@ -50,9 +51,7 @@ def binaries_for_revision( revision: ShortCommitHash ) -> tp.List[ProjectBinaryWrapper]: """Return a list of binaries generated by the project.""" - binary_map = RevisionBinaryMap( - get_local_project_git_path(TaintTests.NAME) - ) + binary_map = RevisionBinaryMap(get_local_project_repo(TaintTests.NAME)) for file_name in TaintTests.CPP_FILES: binary_map.specify_binary( file_name.replace('.cpp', ''), BinaryType.EXECUTABLE diff --git a/varats/varats/projects/test_projects/test_suite.py b/varats/varats/projects/test_projects/test_suite.py index 12fee5d85..cab6ec013 100644 --- a/varats/varats/projects/test_projects/test_suite.py +++ b/varats/varats/projects/test_projects/test_suite.py @@ -9,10 +9,11 @@ from varats.project.project_util import ( ProjectBinaryWrapper, BinaryType, - get_local_project_git_path, + get_local_project_repo, + RevisionBinaryMap, ) from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.git_util import ShortCommitHash class SVFPointsToAnalysisBenchmark(VProject): @@ -493,7 +494,7 @@ def binaries_for_revision( revision: ShortCommitHash # pylint: disable=W0613 ) -> tp.List[ProjectBinaryWrapper]: binary_map = RevisionBinaryMap( - get_local_project_git_path(SVFPointsToAnalysisBenchmark.NAME) + get_local_project_repo(SVFPointsToAnalysisBenchmark.NAME) ) for file_name in SVFPointsToAnalysisBenchmark.FILE_PATHS: diff --git a/varats/varats/tables/blame_interaction_graph_table.py b/varats/varats/tables/blame_interaction_graph_table.py index ef77ba21a..6ef554e49 100644 --- a/varats/varats/tables/blame_interaction_graph_table.py +++ b/varats/varats/tables/blame_interaction_graph_table.py @@ -17,7 +17,7 @@ from varats.paper_mgmt.case_study import ( newest_processed_revision_for_case_study, ) -from varats.project.project_util import get_local_project_git_path +from varats.project.project_util import get_local_project_repo from varats.table.table import Table, TableDataEmpty from varats.table.table_utils import dataframe_to_table from varats.table.tables import TableFormat, TableGenerator @@ -32,7 +32,7 @@ def _generate_graph_table( degree_data: tp.List[pd.DataFrame] = [] for case_study in get_paper_config().get_all_case_studies(): project_name = case_study.project_name - project_git_path = get_local_project_git_path(project_name) + project_repo = get_local_project_repo(project_name) revision = newest_processed_revision_for_case_study( case_study, BlameReportExperiment ) @@ -54,9 +54,9 @@ def _generate_graph_table( pd.DataFrame.from_dict({ project_name: { ("commits", ""): - num_commits(revision.hash, project_git_path), + num_commits(project_repo, revision.hash), ("authors", ""): - num_authors(revision.hash, project_git_path), + num_authors(project_repo, revision.hash), ("nodes", ""): len(graph.nodes), ("edges", ""): diff --git a/varats/varats/tables/case_study_metrics_table.py b/varats/varats/tables/case_study_metrics_table.py index a292da3f3..e6a5b72d3 100644 --- a/varats/varats/tables/case_study_metrics_table.py +++ b/varats/varats/tables/case_study_metrics_table.py @@ -8,17 +8,15 @@ from varats.paper.paper_config import get_loaded_paper_config from varats.project.project_util import ( get_project_cls_by_name, - get_local_project_git_path, + get_local_project_repo, + num_project_commits, + num_project_authors, + calc_project_loc, ) from varats.table.table import Table from varats.table.table_utils import dataframe_to_table from varats.table.tables import TableFormat, TableGenerator -from varats.utils.git_util import ( - calc_project_loc, - num_project_commits, - num_project_authors, - calc_repo_loc, -) +from varats.utils.git_util import calc_repo_loc LOG = logging.Logger(__name__) @@ -35,14 +33,14 @@ def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: project_name = case_study.project_name commit_map = get_commit_map(project_name) project_cls = get_project_cls_by_name(project_name) - project_git_path = get_local_project_git_path(project_name) + project_repo = get_local_project_repo(project_name) revisions = sorted( case_study.revisions, key=commit_map.time_id, reverse=True ) revision = revisions[0] - repo_loc = calc_repo_loc(project_git_path, revision.hash) + repo_loc = calc_repo_loc(project_repo, revision.hash) project_loc = calc_project_loc(project_name, revision) commits = num_project_commits(project_name, revision) authors = num_project_authors(project_name, revision) diff --git a/varats/varats/tables/code_centrality_table.py b/varats/varats/tables/code_centrality_table.py index a59d25fd0..cc50d28fb 100644 --- a/varats/varats/tables/code_centrality_table.py +++ b/varats/varats/tables/code_centrality_table.py @@ -16,7 +16,10 @@ from varats.paper_mgmt.case_study import ( newest_processed_revision_for_case_study, ) -from varats.project.project_util import get_local_project_gits +from varats.project.project_util import ( + get_local_project_repos, + create_project_commit_lookup_helper, +) from varats.table.table import Table, TableDataEmpty from varats.table.table_utils import dataframe_to_table from varats.table.tables import TableFormat, TableGenerator @@ -25,7 +28,6 @@ from varats.utils.git_util import ( ChurnConfig, calc_commit_code_churn, - create_commit_lookup_helper, CommitRepoPair, UNCOMMITTED_COMMIT_HASH, FullCommitHash, @@ -42,8 +44,8 @@ def _collect_cig_node_data( cig = create_blame_interaction_graph( project_name, revision, experiment_type ).commit_interaction_graph() - commit_lookup = create_commit_lookup_helper(project_name) - repo_lookup = get_local_project_gits(project_name) + commit_lookup = create_project_commit_lookup_helper(project_name) + repo_lookup = get_local_project_repos(project_name) def filter_nodes(node: CommitRepoPair) -> bool: if node.commit_hash == UNCOMMITTED_COMMIT_HASH: @@ -57,7 +59,7 @@ def filter_nodes(node: CommitRepoPair) -> bool: if not filter_nodes(commit): continue _, insertions, _ = calc_commit_code_churn( - Path(repo_lookup[commit.repository_name].path), commit.commit_hash, + repo_lookup[commit.repository_name], commit.commit_hash, churn_config ) if insertions == 0: diff --git a/varats/varats/tables/feature_perf_precision.py b/varats/varats/tables/feature_perf_precision.py index 1cff1b825..17a5c84ff 100644 --- a/varats/varats/tables/feature_perf_precision.py +++ b/varats/varats/tables/feature_perf_precision.py @@ -1,7 +1,6 @@ """Module for the FeaturePerfPrecision tables.""" import re import typing as tp -from pathlib import Path import matplotlib.pyplot as plt import numpy as np @@ -9,7 +8,6 @@ import pandas as pd from benchbuild.utils.cmd import git from matplotlib import colors -from plumbum import local from pylatex import Document, Package from varats.data.databases.feature_perf_precision_database import ( @@ -29,12 +27,12 @@ from varats.paper.case_study import CaseStudy from varats.paper.paper_config import get_loaded_paper_config from varats.project.project_domain import ProjectDomains -from varats.project.project_util import get_local_project_git_path +from varats.project.project_util import get_local_project_repo from varats.table.table import Table from varats.table.table_utils import dataframe_to_table from varats.table.tables import TableFormat, TableGenerator from varats.ts_utils.cli_util import make_cli_option -from varats.utils.git_util import calc_repo_loc, ChurnConfig +from varats.utils.git_util import calc_repo_loc, ChurnConfig, RepositoryHandle SYNTH_CATEGORIES = [ "Static Analysis", "Dynamic Analysis", "Configurability", @@ -492,32 +490,33 @@ class FeaturePerfMetricsOverviewTable(Table, table_name="fperf_overview"): studies.""" @staticmethod - def _calc_folder_locs(repo_path: Path, rev_range: str, folder: str) -> int: + def _calc_folder_locs( + repo: RepositoryHandle, rev_range: str, folder: str + ) -> int: churn_config = ChurnConfig.create_c_style_languages_config() file_pattern = re.compile( "|".join(churn_config.get_extensions_repr(r"^.*\.", r"$")) ) loc: int = 0 - with local.cwd(repo_path): - files = git( - "ls-tree", - "-r", - "--name-only", - rev_range, - ).splitlines() - - for file in files: - if not file.startswith(folder): - continue - if file_pattern.match(file): - lines = git("show", f"{rev_range}:{file}").splitlines() - loc += len([line for line in lines if line]) + files = repo( + "ls-tree", + "-r", + "--name-only", + rev_range, + ).splitlines() + + for file in files: + if not file.startswith(folder): + continue + if file_pattern.match(file): + lines = git("show", f"{rev_range}:{file}").splitlines() + loc += len([line for line in lines if line]) return loc @staticmethod - def _calc_folder_locs_dune(repo_path: Path, rev_range: str) -> int: + def _calc_folder_locs_dune(repo: RepositoryHandle, rev_range: str) -> int: dune_sub_projects = [ "dune-alugrid", "dune-common", "dune-functions", "dune-geometry", "dune-grid", "dune-istl", "dune-localfunctions", @@ -526,11 +525,13 @@ def _calc_folder_locs_dune(repo_path: Path, rev_range: str) -> int: ] total_locs = 0 - total_locs += calc_repo_loc(repo_path, rev_range) + total_locs += calc_repo_loc(repo, rev_range) for sub_project in dune_sub_projects: - sub_project_path = repo_path / sub_project - locs = calc_repo_loc(sub_project_path, "HEAD") + sub_project_repo = RepositoryHandle( + repo.worktree_path / sub_project + ) + locs = calc_repo_loc(sub_project_repo, "HEAD") total_locs += locs return total_locs @@ -545,7 +546,7 @@ def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: for case_study in case_studies: project_name = case_study.project_name rev = case_study.revisions[0] - project_git_path = get_local_project_git_path(project_name) + project_repo = get_local_project_repo(project_name) cs_precision_data = df_precision[df_precision['CaseStudy'] == project_name] @@ -559,12 +560,12 @@ def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: ): src_folder = "projects/SynthCTSpecialization" locs = self._calc_folder_locs( - project_git_path, rev.hash, src_folder + project_repo, rev.hash, src_folder ) elif case_study.project_cls.NAME == "DunePerfRegression": - locs = self._calc_folder_locs_dune(project_git_path, rev.hash) + locs = self._calc_folder_locs_dune(project_repo, rev.hash) else: - locs = calc_repo_loc(project_git_path, rev.hash) + locs = calc_repo_loc(project_repo, rev.hash) cs_dict = { project_name: { diff --git a/varats/varats/tools/bb_config.py b/varats/varats/tools/bb_config.py index 3b3ecff37..773e21ee2 100644 --- a/varats/varats/tools/bb_config.py +++ b/varats/varats/tools/bb_config.py @@ -18,7 +18,7 @@ def update_projects( bb_cfg: s.Configuration, include_test_projects: bool = False ) -> None: - """update the projects entry in the benchbuild config to contain our + """Update the projects entry in the benchbuild config to contain our projects.""" projects_conf = bb_cfg["plugins"]["projects"] # If we want later to use default BB projects @@ -98,7 +98,7 @@ def update_projects( def update_experiments(bb_cfg: s.Configuration) -> None: - """update the given benchbuild config to contain our experiments.""" + """Update the given benchbuild config to contain our experiments.""" projects_conf = bb_cfg["plugins"]["experiments"] projects_conf.value[:] = [] projects_conf.value[:] += [ diff --git a/varats/varats/tools/driver_casestudy.py b/varats/varats/tools/driver_casestudy.py index 364ddc69d..20fe6cfee 100644 --- a/varats/varats/tools/driver_casestudy.py +++ b/varats/varats/tools/driver_casestudy.py @@ -41,17 +41,12 @@ from varats.plot.plots import PlotGenerator, PlotConfig, PlotGeneratorFailed from varats.plots.discover_plots import initialize_plots from varats.project.project_util import ( - get_local_project_git_path, get_primary_project_source, + get_local_project_repo, ) from varats.projects.discover_projects import initialize_projects from varats.provider.release.release_provider import ReleaseType -from varats.report.report import ( - FileStatusExtension, - BaseReport, - ReportFilename, - ReportFilepath, -) +from varats.report.report import FileStatusExtension, BaseReport, ReportFilepath from varats.revision.revisions import ( get_all_revisions_files, get_failed_revisions_files, @@ -211,7 +206,7 @@ def __casestudy_gen( ctx.obj['path'] = Path( vara_cfg()["paper_config"]["folder"].value ) / (paper_config + f"/{project}_{version}.case_study") - ctx.obj['git_path'] = get_local_project_git_path(project) + ctx.obj['repo'] = get_local_project_repo(project) if update: get_primary_project_source(project).fetch() @@ -280,7 +275,7 @@ def __gen_latest(ctx: click.Context) -> None: cmap, merge_stage=ctx.obj['merge_stage'], ignore_blocked=ctx.obj['ignore_blocked'], - git_path=ctx.obj["git_path"] + repo=ctx.obj["repo"] ) store_case_study(case_study, ctx.obj['path']) @@ -337,16 +332,16 @@ def __gen_sample( distribution )() - project_repo_path = get_local_project_git_path(ctx.obj['project']) + project_repo = get_local_project_repo(ctx.obj['project']) if end != "HEAD" and not is_commit_hash(end): - end = get_commits_before_timestamp(end, project_repo_path)[0].hash + end = get_commits_before_timestamp(project_repo, end)[0].hash if start is not None and not is_commit_hash(start): - commits_before = get_commits_before_timestamp(start, project_repo_path) + commits_before = get_commits_before_timestamp(project_repo, start) if commits_before: start = commits_before[0].hash else: - start = get_initial_commit(project_repo_path).hash + start = get_initial_commit(project_repo).hash cmap = get_commit_map(ctx.obj['project'], end, start) extend_with_distrib_sampling( @@ -373,7 +368,7 @@ def __gen_per_year( cmap = get_commit_map(ctx.obj['project'], refspec='HEAD') extend_with_revs_per_year( ctx.obj['case_study'], cmap, ctx.obj['merge_stage'], - ctx.obj['ignore_blocked'], ctx.obj['git_path'], revs_per_year, separate + ctx.obj['ignore_blocked'], ctx.obj['repo'], revs_per_year, separate ) store_case_study(ctx.obj['case_study'], ctx.obj['path']) diff --git a/varats/varats/tools/research_tools/development.py b/varats/varats/tools/research_tools/development.py index 5fcf38f9e..900dc7abf 100644 --- a/varats/varats/tools/research_tools/development.py +++ b/varats/varats/tools/research_tools/development.py @@ -34,22 +34,23 @@ def __quickfix_dev_branches(branch_name: str, sub_project: SubProject) -> str: Test: >>> import re - >>> fixed_branch_name = __quickfix_dev_branches(\ - 'vara-dev', SubProject(None, "vara-llvm-project", "", "", "")) + >>> from pathlib import Path + >>> fixed_branch_name = __quickfix_dev_branches('vara-dev', \ + SubProject(Path("/dev/null"), "vara-llvm-project", "", "", "")) >>> re.match(r'vara-\\d+-dev', fixed_branch_name) is not None True - >>> fixed_branch_name = __quickfix_dev_branches(\ - 'vara', SubProject(None, "vara-llvm-project", "", "", "")) + >>> fixed_branch_name = __quickfix_dev_branches('vara', \ + SubProject(Path("/dev/null"), "vara-llvm-project", "", "", "")) >>> re.match(r'vara-\\d+', fixed_branch_name) is not None True - >>> __quickfix_dev_branches(\ - "f-FooBar", SubProject(None, "vara-llvm-project", "", "", "")) + >>> __quickfix_dev_branches("f-FooBar", \ + SubProject(Path("/dev/null"), "vara-llvm-project", "", "", "")) 'f-FooBar' >>> __quickfix_dev_branches(\ - "vara-dev", SubProject(None, "vara", "", "", "")) + "vara-dev", SubProject(Path("/dev/null"), "vara", "", "", "")) 'vara-dev' """ if sub_project.name == "vara-llvm-project": diff --git a/varats/varats/tools/research_tools/phasar.py b/varats/varats/tools/research_tools/phasar.py index 278dae85d..9880f5a36 100644 --- a/varats/varats/tools/research_tools/phasar.py +++ b/varats/varats/tools/research_tools/phasar.py @@ -35,7 +35,7 @@ class PhasarCodeBase(CodeBase): def __init__(self, base_dir: Path) -> None: sub_projects = [ SubProject( - self, "phasar", + base_dir, "phasar", "https://github.com/secure-software-engineering/phasar.git", "origin", "phasar" ) diff --git a/varats/varats/tools/research_tools/research_tool.py b/varats/varats/tools/research_tools/research_tool.py index 8f1035e9d..e07168729 100644 --- a/varats/varats/tools/research_tools/research_tool.py +++ b/varats/varats/tools/research_tools/research_tool.py @@ -31,6 +31,7 @@ has_branch, has_remote_branch, branch_has_upstream, + RepositoryHandle, ) from varats.utils.logger_util import log_without_linesep @@ -169,7 +170,7 @@ class SubProject(): def __init__( self, - parent_code_base: 'CodeBase', + parent_base_dir: Path, name: str, url: str, remote: str, @@ -177,11 +178,12 @@ def __init__( is_submodule: bool = False ): self.__name = name - self.__parent_code_base = parent_code_base + self.__parent_base_dir = parent_base_dir self.__url = url self.__remote = remote self.__sub_path = Path(sub_path) self.__is_submodule = is_submodule + self.__repo = RepositoryHandle(self.__parent_base_dir / self.__sub_path) @property def name(self) -> str: @@ -228,20 +230,18 @@ def init_and_update_submodules(self) -> None: Args: cb_base_dir: base directory for the ``CodeBase`` """ - init_all_submodules(self.__parent_code_base.base_dir / self.path) - update_all_submodules(self.__parent_code_base.base_dir / self.path) + init_all_submodules(self.__repo) + update_all_submodules(self.__repo) def clone(self) -> None: """Clone the sub project into the specified folder relative to the base dir of the ``CodeBase``.""" - print(f"Cloning {self.name} into {self.__parent_code_base.base_dir}") - if (self.__parent_code_base.base_dir / self.path).exists(): - raise FolderAlreadyPresentError( - self.__parent_code_base.base_dir / self.path - ) + print(f"Cloning {self.name} into {self.__parent_base_dir}") + if self.__repo.worktree_path.exists(): + raise FolderAlreadyPresentError(self.__repo.worktree_path) download_repo( - self.__parent_code_base.base_dir / self.path.parent, self.url, - self.path.name, self.remote, log_without_linesep(print) + self.__repo.worktree_path.parent, self.url, self.path.name, + self.remote, log_without_linesep(print) ) def has_branch( @@ -261,11 +261,10 @@ def has_branch( Returns: True, if the branch exists """ - absl_repo_path = self.__parent_code_base.base_dir / self.path if remote_to_check is None: - return has_branch(absl_repo_path, branch_name) + return has_branch(self.__repo, branch_name) - return has_remote_branch(absl_repo_path, branch_name, remote_to_check) + return has_remote_branch(self.__repo, branch_name, remote_to_check) def get_branches(self, extra_args: tp.Optional[tp.List[str] @@ -279,9 +278,7 @@ def get_branches(self, Returns: list of branch names """ - return get_branches( - self.__parent_code_base.base_dir / self.path, extra_args - ).split() + return get_branches(self.__repo, extra_args).split() def add_remote(self, remote: str, url: str) -> None: """ @@ -291,8 +288,8 @@ def add_remote(self, remote: str, url: str) -> None: remote: name of the new remote url: to the remote """ - add_remote(self.__parent_code_base.base_dir / self.path, remote, url) - fetch_remote(remote, self.__parent_code_base.base_dir / self.path) + add_remote(self.__repo, remote, url) + fetch_remote(self.__repo, remote) def checkout_branch(self, branch_name: str) -> None: """ @@ -301,9 +298,7 @@ def checkout_branch(self, branch_name: str) -> None: Args: branch_name: name of the branch, should exists in the repo """ - checkout_branch_or_commit( - self.__parent_code_base.base_dir / self.path, branch_name - ) + checkout_branch_or_commit(self.__repo, branch_name) def checkout_new_branch( self, branch_name: str, remote_branch: tp.Optional[str] = None @@ -314,10 +309,7 @@ def checkout_new_branch( Args: branch_name: name of the new branch, should not exists in the repo """ - checkout_new_branch( - self.__parent_code_base.base_dir / self.path, branch_name, - remote_branch - ) + checkout_new_branch(self.__repo, branch_name, remote_branch) def fetch( self, @@ -325,27 +317,24 @@ def fetch( extra_args: tp.Optional[tp.List[str]] = None ) -> None: """Fetch updates from the remote.""" - fetch_remote( - remote, self.__parent_code_base.base_dir / self.path, extra_args - ) + fetch_remote(self.__repo, remote, extra_args) def pull(self) -> None: """Pull updates from the remote of the current branch into the sub project.""" - pull_current_branch(self.__parent_code_base.base_dir / self.path) + pull_current_branch(self.__repo) def push(self) -> None: """Push updates from the current branch to the remote branch.""" - absl_repo_path = self.__parent_code_base.base_dir / self.path - branch_name = get_current_branch(absl_repo_path) - if branch_has_upstream(absl_repo_path, branch_name): - push_current_branch(absl_repo_path) + branch_name = get_current_branch(self.__repo) + if branch_has_upstream(self.__repo, branch_name): + push_current_branch(self.__repo) else: - push_current_branch(absl_repo_path, "origin", branch_name) + push_current_branch(self.__repo, "origin", branch_name) def show_status(self) -> None: """Show the current status of the sub project.""" - show_status(self.__parent_code_base.base_dir / self.path) + show_status(self.__repo) def __str__(self) -> str: return f"{self.name} [{self.url}:{self.remote}] {self.path}" @@ -353,9 +342,7 @@ def __str__(self) -> str: def get_tags(self, extra_args: tp.Optional[tp.List[str]] = None) -> tp.List[str]: """Get the list of available git tags.""" - tag_list = get_tags( - self.__parent_code_base.base_dir / self.path, extra_args - ) + tag_list = get_tags(self.__repo, extra_args) return tag_list diff --git a/varats/varats/tools/research_tools/szz_unleashed.py b/varats/varats/tools/research_tools/szz_unleashed.py index e841fb48c..bd3728e3d 100644 --- a/varats/varats/tools/research_tools/szz_unleashed.py +++ b/varats/varats/tools/research_tools/szz_unleashed.py @@ -29,7 +29,7 @@ def __init__(self, base_dir: Path): super().__init__( base_dir, [ SubProject( - self, "SZZUnleashed", + base_dir, "SZZUnleashed", "https://github.com/boehmseb/SZZUnleashed.git", "origin", "szzunleashed" ) diff --git a/varats/varats/tools/research_tools/vara.py b/varats/varats/tools/research_tools/vara.py index 0cd9f0e5a..3ffff601a 100644 --- a/varats/varats/tools/research_tools/vara.py +++ b/varats/varats/tools/research_tools/vara.py @@ -51,16 +51,16 @@ class VaRACodeBase(CodeBase): def __init__(self, base_dir: Path) -> None: sub_projects = [ SubProject( - self, "vara-llvm-project", + base_dir, "vara-llvm-project", "https://github.com/llvm/llvm-project.git", "upstream", "vara-llvm-project" ), SubProject( - self, "VaRA", "git@github.com:se-sic/VaRA.git", "origin", + base_dir, "VaRA", "git@github.com:se-sic/VaRA.git", "origin", "vara-llvm-project/vara" ), SubProject( - self, + base_dir, "phasar", "https://github.com/secure-software-engineering/phasar.git", "origin", diff --git a/varats/varats/tools/research_tools/vara_manager.py b/varats/varats/tools/research_tools/vara_manager.py index 538f3e4f8..adaa15d33 100644 --- a/varats/varats/tools/research_tools/vara_manager.py +++ b/varats/varats/tools/research_tools/vara_manager.py @@ -20,6 +20,7 @@ from varats.utils.exceptions import ProcessTerminatedError from varats.utils.git_commands import fetch_remote +from varats.utils.git_util import RepositoryHandle from varats.utils.settings import vara_cfg LOG = logging.getLogger(__name__) @@ -119,18 +120,17 @@ def get_llvm_project_status( ) -> GitStatus: """Retrieve the git status of a llvm project.""" try: - with local.cwd(llvm_folder / project_folder): - fetch_remote('origin') - git_status = git['status'] - stdout = git_status('-sb') - for line in stdout.split('\n'): - if line.startswith( - '## vara-' + str(vara_cfg()['version']) + '-dev' - ): - match = re.match(r".*\[(.*)\]", line) - if match is not None: - return GitStatus(GitState.BEHIND, match.group(1)) - return GitStatus(GitState.OK) + repo = RepositoryHandle(llvm_folder / project_folder) + fetch_remote(repo, 'origin') + stdout = repo('status', '-sb') + for line in stdout.split('\n'): + if line.startswith( + '## vara-' + str(vara_cfg()['version']) + '-dev' + ): + match = re.match(r".*\[(.*)\]", line) + if match is not None: + return GitStatus(GitState.BEHIND, match.group(1)) + return GitStatus(GitState.OK) except ProcessTerminatedError: pass diff --git a/varats/varats/ts_utils/cli_util.py b/varats/varats/ts_utils/cli_util.py index 21eb97dee..cd825f681 100644 --- a/varats/varats/ts_utils/cli_util.py +++ b/varats/varats/ts_utils/cli_util.py @@ -133,9 +133,9 @@ class CLIOptionConverter(abc.ABC, tp.Generic[ConversionTy_co]): Converter for CLI option declarations. Converters are required for CLI options that are converted to complex types - by click so that they can still be properly stored in an artefact file. - In general, a converter should implement a mapping from the complex type to - a string value as it would be provided on the command line. + by click so that they can still be properly stored in an artefact file. In + general, a converter should implement a mapping from the complex type to a + string value as it would be provided on the command line. A converter can be attached to a CLI option using the function/decorator :func:`convert_value()`.