diff --git a/varats-core/varats/project/project_domain.py b/varats-core/varats/project/project_domain.py index 071ed9605..dcf78d864 100644 --- a/varats-core/varats/project/project_domain.py +++ b/varats-core/varats/project/project_domain.py @@ -31,6 +31,8 @@ class ProjectDomains(Enum): UNIX_TOOLS = "UNIX utils" VERSION_CONTROL = "Version control" WEB_TOOLS = "Web tools" + RUNTIME = "Runtime" + MPI = "Message Passing Interface" def __str__(self) -> str: return str(self.value) diff --git a/varats/varats/plots/blame_interaction_graph_plots.py b/varats/varats/plots/blame_interaction_graph_plots.py index 7d06288a7..498533c96 100644 --- a/varats/varats/plots/blame_interaction_graph_plots.py +++ b/varats/varats/plots/blame_interaction_graph_plots.py @@ -1,5 +1,6 @@ """Module for BlameInteractionGraph plots.""" - +import configparser +import subprocess import typing as tp from datetime import datetime from pathlib import Path @@ -9,6 +10,7 @@ import networkx as nx import pandas as pd import plotly.offline as offply +import pygit2 from varats.data.reports.blame_interaction_graph import ( create_blame_interaction_graph, @@ -18,7 +20,7 @@ CAIGNodeAttrs, ) from varats.experiments.vara.blame_report_experiment import ( - BlameReportExperiment, + BlameReportExperiment, BlameReportExperimentRegion, ) from varats.mapping.commit_map import get_commit_map from varats.paper_mgmt.case_study import ( @@ -64,7 +66,7 @@ def save(self, plot_dir: Path, filetype: str = 'svg') -> None: revision = commit_map.convert_to_full_or_warn(short_revision) cig = create_blame_interaction_graph( - project_name, revision, BlameReportExperiment + project_name, revision, BlameReportExperimentRegion ).commit_interaction_graph() nx.set_node_attributes( cig, @@ -109,7 +111,7 @@ def _prepare_cig_plotly( NodeTy, NodeTy, EdgeInfoTy]]]: commit_lookup = create_commit_lookup_helper(project_name) cig = create_blame_interaction_graph( - project_name, revision, BlameReportExperiment + project_name, revision, BlameReportExperimentRegion ).commit_interaction_graph() def filter_nodes(node: CommitRepoPair) -> bool: @@ -280,6 +282,138 @@ def generate(self) -> tp.List[Plot]: ) +def get_gitmodules_info(): + repo = pygit2.Repository(REPO_PATH) + # Get the .gitmodules blob object + commit = repo[repo.head.target] + tree = commit.tree + gitmodules_entry = tree[".gitmodules"] + gitmodules_blob = repo[gitmodules_entry.oid] + + # Parse the .gitmodules file contents + gitmodules_data = gitmodules_blob.data + gitmodules_config = configparser.ConfigParser() + gitmodules_config.read_string(gitmodules_data.decode('utf-8')) + + # Collect the submodule path and URL information + submodules = [] + for section in gitmodules_config.sections(): + if section.startswith('submodule '): + submodule_name = section[len('submodule '):] + submodule_path = gitmodules_config.get(section, 'path') + submodule_url = gitmodules_config.get(section, 'url') + submodules.append((submodule_name, submodule_path, submodule_url)) + + return submodules + + +def pull_commit_history_information(submodules): + + # Get the tree object for the specified commit + repo = pygit2.Repository(REPO_PATH) + + commits = [] + i = 0 + for submodule in submodules: + walker = repo.walk(repo.head.target, pygit2.GIT_SORT_TOPOLOGICAL | pygit2.GIT_SORT_REVERSE) + subdirectory_path = submodule[1] + for commit in walker: + # Initialize a flag to check if the 'third_party' has changes in this commit + # has_changes = False + # Compare the tree of this commit with its parent + if len(commit.parents) == 0: + continue + diff = repo.diff(commit.parents[0], commit) + # Loop through all patches in the diff + files_changed = [patch.delta.new_file.path for patch in diff] + if any([file.startswith(subdirectory_path) for file in files_changed]): + i += 1 + print(i) # total 1864 times commits is for third party + commits.append({ + 'author_name': commit.author.name, + 'author_email': commit.author.email, + # 'message': commit.message, + 'commit_hash': commit.hex, + 'commit_time': datetime.datetime.fromtimestamp(commit.commit_time) + # 'changed_folder': get_change_folder(commit) + }) + + df = pd.DataFrame(commits) + # df.to_csv(TASK_PATH + 'openssl_commit_history2.csv', index=False) + + return df + + +class SubCommitInteractionGraphPlot(Plot, plot_name='sub-cig-plot'): + submodule_info = get_gitmodules_info() + df = pull_commit_history_information(submodule_info) + + def is_submodule_change(self, commit_hash): + for index, row in self.df.iterrows(): + if row['commit_hash'] == commit_hash: + return True + else: + return False + + def plot(self, view_mode: bool) -> None: + project_name = self.plot_kwargs["case_study"].project_name + commit_map = get_commit_map(project_name) + short_revision = ShortCommitHash(self.plot_kwargs["revision"]) + revision = commit_map.convert_to_full_or_warn(short_revision) + + cig = create_blame_interaction_graph( + project_name, revision, BlameReportExperimentRegion + ).commit_interaction_graph() + nx.set_node_attributes( + cig, + {node: cig.nodes[node]["commit"].commit_hash for node in cig.nodes}, + "label" + ) + + # Find nodes representing commits that changed submodules + submodule_commit_nodes = [node for node in cig.nodes if + self.is_submodule_change(cig.nodes[node]["commit"].commit_hash)] + + # Count the number of unique authors for main repository and submodule changes + main_repo_authors = set() + submodule_change_authors = set() + + for node in cig.nodes: + commit_info = cig.nodes[node]["commit"] + author = commit_info.author_name # Assuming the existence of the author_name attribute, modify as needed + + main_repo_authors.add(author) + + if node in submodule_commit_nodes: + submodule_change_authors.add(author) + + main_repo_author_count = len(main_repo_authors) + submodule_change_author_count = len(submodule_change_authors) + + print(f"Main Repository Authors: {main_repo_author_count}") + print(f"Authors with Submodule Changes: {submodule_change_author_count}") + + + + def calc_missing_revisions( + self, boundary_gradient: float + ) -> tp.Set[FullCommitHash]: + raise UnsupportedOperation + + +class SubCIGPlotGenerator( + PlotGenerator, + generator_name="sub-cig-plot", + options=[REQUIRE_CASE_STUDY, REQUIRE_REVISION] +): + """Generates a plot for a submodule's commit interaction graph.""" + + def generate(self) -> tp.List[Plot]: + return [ + SubCommitInteractionGraphPlot(self.plot_config, **self.plot_kwargs) + ] + + class CommitInteractionGraphNodeDegreePlot(Plot, plot_name='cig_node_degrees'): """ Plot node degrees of a commit interaction graph. diff --git a/varats/varats/projects/c_projects/bcc.py b/varats/varats/projects/c_projects/bcc.py new file mode 100644 index 000000000..2db00ba1a --- /dev/null +++ b/varats/varats/projects/c_projects/bcc.py @@ -0,0 +1,101 @@ +"""Project file for the bcc.""" +import typing as tp + +import benchbuild as bb +from benchbuild.utils.cmd import make, mkdir, cmake, sudo +from benchbuild.utils.settings import get_number_of_jobs +from plumbum import local + +from varats.containers.containers import get_base_image, ImageBase +from varats.paper.paper_config import ( + project_filter_generator, + PaperConfigSpecificGit, +) +from varats.project.project_domain import ProjectDomains +from varats.project.project_util import ( + ProjectBinaryWrapper, + BinaryType, + verify_binaries, + get_local_project_git_path, +) +from varats.project.varats_project import VProject +from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.settings import bb_cfg + + +class Bcc(VProject): + + NAME = 'bcc' + GROUP = 'c_projects' + DOMAIN = ProjectDomains.UNIX_TOOLS + + SOURCE = [ + PaperConfigSpecificGit( + project_name="bcc", + remote="https://github.com/iovisor/bcc", + local="bcc", + refspec="origin/HEAD", + limit=None, + shallow=False + ), + bb.source.GitSubmodule( + remote="https://github.com/libbpf/libbpf.git", + local="bcc/src/cc/libbpf", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("bcc") + ), + bb.source.GitSubmodule( + remote="https://github.com/libbpf/bpftool", + local="bcc/libbpf-tools/bpftool", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("bcc") + ), + bb.source.GitSubmodule( + remote="https://github.com/libbpf/blazesym", + local="bcc/ibbpf-tools/blazesym", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("bcc") + ) + ] + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash + ) -> tp.List[ProjectBinaryWrapper]: + binary_map = RevisionBinaryMap(get_local_project_git_path(Bcc.NAME)) + + binary_map.specify_binary("build/src/cc/libbcc.so", BinaryType.SHARED_LIBRARY) + + return binary_map[revision] + + def compile(self) -> None: + """Compile the BCC project.""" + bcc_source = local.path(self.source_of_primary) + mkdir("-p", bcc_source / "build") + + cc_compiler = bb.compiler.cc(self) + cxx_compiler = bb.compiler.cxx(self) + with local.cwd(bcc_source / 'build'): + with local.env(CC=str(cc_compiler), CXX=cxx_compiler): + # llvm_path = "/lib/llvm-14/lib/cmake/llvm/" + bb.watch(cmake)("..", + # "-DCMAKE_BUILD_TYPE=Release", + # f"-DLLVM_DIR={llvm_path}", + # "-DCMAKE_POSITION_INDEPENDENT_CODE=ON", + "-DCMAKE_C_FLAGS=-fPIE", "-DCMAKE_CXX_FLAGS=-fPIE", + # "-DCMAKE_EXE_LINKER_FLAGS=-pie" + ) # Configuring the project + # Compiling the project with make, specifying the number of jobs + bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) + + with local.cwd(bcc_source): + verify_binaries(self) + + + diff --git a/varats/varats/projects/c_projects/crun.py b/varats/varats/projects/c_projects/crun.py new file mode 100644 index 000000000..fb41967b8 --- /dev/null +++ b/varats/varats/projects/c_projects/crun.py @@ -0,0 +1,72 @@ +"""Project file for the Crun.""" +import typing as tp + +import benchbuild as bb +from benchbuild.utils.cmd import make +from benchbuild.utils.settings import get_number_of_jobs +from plumbum import local + +from varats.containers.containers import get_base_image, ImageBase +from varats.paper.paper_config import ( + project_filter_generator, + PaperConfigSpecificGit, +) +from varats.project.project_domain import ProjectDomains +from varats.project.project_util import ( + ProjectBinaryWrapper, + BinaryType, + verify_binaries, + get_local_project_git_path, +) +from varats.project.varats_project import VProject +from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.settings import bb_cfg + + +class Crun(VProject): + + NAME = 'crun' + GROUP = 'c_projects' + DOMAIN = ProjectDomains.RUNTIME + + SOURCE = [ + PaperConfigSpecificGit( + project_name="crun", + remote="https://github.com/containers/crun", + local="crun", + refspec="origin/HEAD", + limit=None, + shallow=False + ), + bb.source.GitSubmodule( + remote="https://github.com/containers/libocispec.git", + local="crun/libocispec", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("crun") + ) + ] + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash + ) -> tp.List[ProjectBinaryWrapper]: + binary_map = RevisionBinaryMap(get_local_project_git_path(Crun.NAME)) + + binary_map.specify_binary("crun", BinaryType.EXECUTABLE) + + return binary_map[revision] + + def compile(self) -> None: + crun_source = local.path(self.source_of_primary) + compiler = bb.compiler.cc(self) + with local.cwd(crun_source): + with local.env(CC=str(compiler)): + bb.watch(local["./autogen.sh"])() + bb.watch(local["./configure"])("--disable-gcc-warnings") + + bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) + + verify_binaries(self) + diff --git a/varats/varats/projects/c_projects/ompi.py b/varats/varats/projects/c_projects/ompi.py new file mode 100644 index 000000000..0945666db --- /dev/null +++ b/varats/varats/projects/c_projects/ompi.py @@ -0,0 +1,93 @@ +"""Project file for the open-mpi.""" +import typing as tp + +import benchbuild as bb +from benchbuild.utils.cmd import make, mkdir +from benchbuild.utils.settings import get_number_of_jobs +from plumbum import local + +from varats.containers.containers import get_base_image, ImageBase +from varats.paper.paper_config import ( + project_filter_generator, + PaperConfigSpecificGit, +) +from varats.project.project_domain import ProjectDomains +from varats.project.project_util import ( + ProjectBinaryWrapper, + BinaryType, + verify_binaries, + get_local_project_git_path, +) +from varats.project.varats_project import VProject +from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.settings import bb_cfg + + +class Ompi(VProject): + + NAME = 'ompi' + GROUP = 'c_projects' + DOMAIN = ProjectDomains.MPI + + SOURCE = [ + PaperConfigSpecificGit( + project_name="ompi", + remote="https://github.com/open-mpi/ompi", + local="ompi", + refspec="origin/HEAD", + limit=None, + shallow=False + ), + bb.source.GitSubmodule( + remote="https://github.com/openpmix/prrte.git", + local="ompi/3rd-party/prrte", + refspec="heads/master", + limit=None, + shallow=False, + version_filter=project_filter_generator("ompi") + ), + bb.source.GitSubmodule( + remote="https://github.com/openpmix/openpmix.git", + local="ompi/3rd-party/openpmix", + refspec="heads/master", + limit=None, + shallow=False, + version_filter=project_filter_generator("ompi") + ), + bb.source.GitSubmodule( + remote="https://github.com/open-mpi/oac", + local="ompi/config/oac", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("ompi") + ) + ] + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash + ) -> tp.List[ProjectBinaryWrapper]: + binary_map = RevisionBinaryMap(get_local_project_git_path(Ompi.NAME)) + + binary_map.specify_binary("build/ompi", BinaryType.EXECUTABLE) + + return binary_map[revision] + + + def compile(self) -> None: + ompi_source = local.path(self.source_of_primary) + mkdir("-p", ompi_source / "build") + + cc_compiler = bb.compiler.cc(self) + cxx_compiler = bb.compiler.cxx(self) + with local.cwd(ompi_source): + with local.env(CC=str(cc_compiler), CXX=cxx_compiler): + bb.watch(local["./autogen.pl"])() + with local.cwd(ompi_source / "build"): + bb.watch(local["../configure"])("--disable-mpi-fortran") + bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) + + with local.cwd(ompi_source): + verify_binaries(self) + diff --git a/varats/varats/tools/bb_config.py b/varats/varats/tools/bb_config.py index 482f8ae02..150da6be1 100644 --- a/varats/varats/tools/bb_config.py +++ b/varats/varats/tools/bb_config.py @@ -66,6 +66,9 @@ def update_projects( 'varats.projects.c_projects.vim', 'varats.projects.c_projects.x264', 'varats.projects.c_projects.xz', + 'varats.projects.c_projects.bcc', + 'varats.projects.c_projects.crun', + 'varats.projects.c_projects.ompi', 'varats.projects.cpp_projects.clasp', 'varats.projects.cpp_projects.fast_downward', 'varats.projects.cpp_projects.libzmq',