Merge branch 'vara-dev' into f-IterIDE

.github/workflows/ci.yml

@@ -5,6 +5,8 @@ on:
     branches: [ vara, vara-dev ]
   pull_request:
     branches: [ vara, vara-dev ]
+  merge_group:
+    branches: [ vara, vara-dev ]
 
 env:
   BB_TMP_DIR: $(pwd)/benchbuild/tmp

.github/workflows/docs.yml

@@ -3,6 +3,8 @@ name: Docs CI
 on:
   pull_request:
     branches: [ vara, vara-dev ]
+  merge_group:
+    branches: [ vara, vara-dev ]
 
 jobs:
   build:

.github/workflows/pre-commit.yml

@@ -2,6 +2,7 @@ name: pre-commit
 
 on:
   pull_request:
+  merge_group:
 
 jobs:
   pre-commit:

.github/workflows/reviewdog.yml

@@ -3,6 +3,8 @@ name: reviewdog
 on:
   pull_request:
     branches: [ vara, vara-dev ]
+  merge_group:
+    branches: [ vara, vara-dev ]
 
 jobs:
   reviewdog:

.github/workflows/tag.yml

@@ -17,6 +17,11 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.11'
+
       - uses: ./.github/actions/dependenciesActions
       - name: Create and Publish Release
         run: |

.gitignore

@@ -21,3 +21,4 @@ docs/source/.varats.yaml
 docs/**/*.inc
 .vscode/
 .cache/
+pre-commit-env

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
     rev: 5.11.5
     hooks:
     - id: isort
-      args: ['-nis']
+      args: ['--nis']
 -   repo: https://github.com/myint/docformatter.git
     rev: v1.4
     hooks:

docs/source/conf.py

@@ -87,6 +87,10 @@
 # modules that require this module before setting the type checking flag.
 import scipy.stats  # isort:skip
 
+# Matplotlib >=3.8 has a type-checking-flag-guarded import of a symbol that does
+# not exist in the shipped version.
+import matplotlib.pyplot  # isort:skip
+
 # The autodocs typehints plugin does not resolve circular imports caused by type
 # annotations, so we have to manually break the circles.
 import rich.console  # isort:skip
@@ -95,6 +99,7 @@
 import cryptography.exceptions  # isort:skip
 import click  # isort:skip
 import git  # isort:skip
+import github  # isort:skip
 import urllib3.exceptions  # isort:skip
 
 # Some packages use new syntax for type checking that isn't available to us

docs/source/research_tool_docs/vara/add_setup_infos/clion_setup.rst

@@ -5,7 +5,7 @@ How to set up VaRA/LLVM in CLion
 --------------------------------
 1. Use ``vara-buildsetup vara -i`` to correctly clone and checkout the VaRa/LLVM repos (cf. :ref:`How to setup VaRA`)
 
-2. Start CLion and from the menu select **File | Open** and point to ``<varats_root>/vara-llvm-project/llvm/CMakeLists.txt``.
+2. Start CLion and from the menu select **File | Open** and point to ``<varats_root>/tools_src/vara-llvm-project/llvm/CMakeLists.txt``.
    In the dialog that opens, click **Open as Project**.
 
 3. Go to **Settings/Preferences | Build, Execution, Deployment | CMake** to configure the CMake project.
@@ -20,8 +20,8 @@ How to set up VaRA/LLVM in CLion
 
            -DBUILD_CLAR=OFF
            -DBUILD_SHARED_LIBS=ON
-           -DCMAKE_C_FLAGS_DEBUG=-O2 -g -fno-omit-frame-pointer
-           -DCMAKE_CXX_FLAGS_DEBUG=-O2 -g -fno-omit-frame-pointer
+           -DCMAKE_C_FLAGS_DEBUG="-O2 -g -fno-omit-frame-pointer"
+           -DCMAKE_CXX_FLAGS_DEBUG="-O2 -g -fno-omit-frame-pointer"
            -DCMAKE_CXX_STANDARD=17
            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
            -DCMAKE_INSTALL_PREFIX=<varats_root>/tools/VaRA
@@ -55,8 +55,8 @@ How to set up VaRA/LLVM in CLion
 
            -DBUILD_CLAR=OFF
            -DBUILD_SHARED_LIBS=ON
-           -DCMAKE_C_FLAGS_RELEASE=-O3 -DNDEBUG -march=native -fno-omit-frame-pointer -gmlt
-           -DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -march=native -fno-omit-frame-pointer -gmlt
+           -DCMAKE_C_FLAGS_RELEASE="-O3 -DNDEBUG -march=native -fno-omit-frame-pointer -gmlt"
+           -DCMAKE_CXX_FLAGS_RELEASE="-O3 -DNDEBUG -march=native -fno-omit-frame-pointer -gmlt"
            -DCMAKE_CXX_STANDARD=17
            -DCMAKE_INSTALL_PREFIX=<varats_root>/tools/VaRA
            -DLLVM_ENABLE_ASSERTIONS=OFF

docs/source/research_tool_docs/vara/instrumentations/perf_infl_trace.rst

@@ -0,0 +1,4 @@
+Performance-Influence Model Trace (``perf_infl_trace``)
+=======================================================
+
+VaRA can automatically insert instrumentation markers for measuring the cumulative time that's spent within a region.

docs/source/research_tool_docs/vara/instrumentations/usdt.rst

@@ -1,4 +1,6 @@
-USDT Probe Instrumentations (``usdt``)
-======================================
+USDT Probe Instrumentations (``usdt``/``usdt_raw``)
+===================================================
 
-TODO: add docs
+Insert USDT probes around regions.
+VaRA inserts in the ``usdt`` mode one probe that handles all start/end events,
+in ``usdt_raw`` mode it inserts a separate probe for each region start/end is added.

docs/source/research_tool_docs/vara/usages/feature_performance_analysis.rst

@@ -34,6 +34,11 @@ The different instrumentation options can be found :ref:`here<Instrumentations>`
     CXX_FLAGS="$CXX_FLAGS -fvara-feature -fsanitize=vara -fvara-instr=trace_event"
 
 
+.. note::
+
+    By default, regions smaller than 100 instructions are ignored. To manually set the instruction threshold, add ``-vara-instruction-threshold=X`` as a ``C/CXX_FLAG``, where ``X`` defines amount of instructions that need to be contained in a region.
+
+
 (Recommended) Configure your project to use link-time optimization (LTO) for more precise analysis results.
 Without LTO, the analysis can only run within a translation unit, hence, features that are used across different translation units can not be correctly analyzed.
 

docs/source/research_tool_docs/vara/using.rst

@@ -55,5 +55,6 @@ To enable this, specify a region detection strategy, enable the tracing code wit
    instrumentations/print_instr
    instrumentations/clock_instr.rst
    instrumentations/trace_event_instr.rst
+   instrumentations/perf_infl_trace.rst
    instrumentations/instr_verify.rst
    instrumentations/usdt.rst

docs/source/tutorials/configuration_specific_experiments.rst

@@ -21,6 +21,7 @@ One just needs to extend the case-study file of a project with a yaml document t
 .. code-block:: yaml
 
   ---
+  config_type: PlainCommandlineConfiguration
   0: '["--foo", "--bar"]'
   1: '["--foo"]'
   ...

docs/source/vara-ts-api/tools/vara-cs-gui.rst

@@ -10,6 +10,7 @@ The gui is started by::
 
 The gui provides 3 Strategies to generate case studies:
     - Manual revision selection: Select revision from the revision history of a project. Multiple revisions can be selected by holding `ctrl` and ranges by holding `shift`. Revisions which are blocked because of bugs in the compilation of the project are marked blue.
+
     .. figure:: vara-cs-gui-manual.png
 
     - Random Sampling: Sample a number of revisions using a random a Normal or HalfNormal Distribution.

icons/icons.qrc

@@ -28,6 +28,7 @@
     <file>breeze/light/document-save.svg</file>
     <file>breeze/light/application-exit.svg</file>
     <file>breeze/light/document-open.svg</file>
+    <file>breeze/light/vcs-commit.svg</file>
     <file>operators/and-operator.svg</file>
     <file>operators/or-operator.svg</file>
     <file>operators/not-operator.svg</file>

icons/update-icons.sh

@@ -3,4 +3,4 @@
 script=$(readlink -f "$0")
 scriptpath=$(dirname "${script}")
 
-pyrcc5 icons.qrc -o "${scriptpath}"/../varats/gui/icons_rc.py
+pyrcc5 icons.qrc -o "${scriptpath}"/../varats/varats/gui/icons_rc.py

requirements.txt

@@ -1,12 +1,13 @@
-benchbuild>=6.6.4
+benchbuild>=6.8
 click>=8.1.3
+cliffs-delta>=1.0.0
 distro>=1.5.0
 graphviz>=0.14.2
 ijson>=3.1.4
 Jinja2>=3.1.2
 jupyter>=1.0.0
 kaleido>=0.2.1
-matplotlib>=3.7.1
+matplotlib>=3.8.2
 networkx>=3.0
 numpy>=1.24.2
 packaging>=20.1
@@ -28,7 +29,7 @@ PyYAML>=6.0
 requests>=2.28.2
 rich>=12.6
 scikit-learn>=1.2.2
-seaborn>=0.12.2
+seaborn>=0.13.0
 tabulate>=0.9
 types-PyYAML
 types-requests

tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPRuntime_0.case_study

@@ -0,0 +1,25 @@
+---
+DocType: CaseStudy
+Version: 1
+...
+---
+project_name: SynthIPRuntime
+stages:
+- revisions:
+  - commit_hash: 793035062810ea3a2d9a10f831cd199fbbb82090
+    commit_id: 64
+    config_ids:
+    - 0
+    - 1
+    - 2
+    - 3
+    - 4
+version: 0
+...
+---
+config_type: PlainCommandlineConfiguration
+0: '["-d"]'
+1: '["-c"]'
+2: '["-c", "-1"]'
+3: '["-c", "-2"]'
+4: '["-c", "-1", "-2"]'

tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPTemplate_0.case_study

@@ -0,0 +1,25 @@
+---
+DocType: CaseStudy
+Version: 1
+...
+---
+project_name: SynthIPTemplate
+stages:
+- revisions:
+  - commit_hash: 793035062810ea3a2d9a10f831cd199fbbb82090
+    commit_id: 64
+    config_ids:
+    - 0
+    - 1
+    - 2
+    - 3
+    - 4
+version: 0
+...
+---
+config_type: PatchConfiguration
+0: '["Decompress"]'
+1: '["Compress", "no_fastmode", "no_smallmode"]'
+2: '["Compress", "fastmode", "no_smallmode"]'
+3: '["Compress", "no_fastmode", "smallmode"]'
+4: '["Compress", "fastmode", "smallmode"]'

tests/TEST_INPUTS/paper_configs/test_config_ids/SynthSAContextSensitivity_0.case_study

@@ -0,0 +1,20 @@
+---
+DocType: CaseStudy
+Version: 1
+...
+---
+project_name: SynthSAContextSensitivity
+stages:
+- revisions:
+  - commit_hash: 06eac0edb6886a7e487867c8d5629cb2409b54fd
+    commit_id: 57
+    config_ids:
+    - 0
+    - 1
+version: 0
+...
+---
+config_type: PlainCommandlineConfiguration
+0: '["--compress", "--mem", "10", "8"]'
+1: '["--compress", "--mem", "300", "8"]'
+...

tests/TEST_INPUTS/paper_configs/test_config_ids/xz_0.case_study

@@ -10,5 +10,6 @@ stages:
     config_ids: [1]
 version: 0
 ---
+config_type: PlainCommandlineConfiguration
 0: '["--foo", "--bar"]'
 1: '["--foo"]'

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/exclude-revision-range.info

@@ -0,0 +1,8 @@
+description: Patch that excludes a range of commits (Otherwise includes all)
+exclude_revisions:
+  revision_range:
+    end: 8ca5cc28e6746eef7340064b5d843631841bf31e
+    start: 01f9f1f07bef22d4248e8349aba4f0c1f204607e
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: exclude-revision-range

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/exclude-single-and-revision-range.info

@@ -0,0 +1,10 @@
+description: Patch that excludes a certain range and individual commits (Otherwise
+  includes all)
+exclude_revisions:
+  revision_range:
+    end: 8ca5cc28e6746eef7340064b5d843631841bf31e
+    start: 01f9f1f07bef22d4248e8349aba4f0c1f204607e
+  single_revision: 27f17080376e409860405c40744887d81d6b3f34
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: exclude-single-and-revision-range

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/exclude-single-revision.info

@@ -0,0 +1,6 @@
+description: Patch that is valid for all commits except a single one
+exclude_revisions:
+  single_revision: 8ca5cc28e6746eef7340064b5d843631841bf31e
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: exclude-single-revision

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/include-range-exclude-range.info

@@ -0,0 +1,12 @@
+description: Patch valid for a range of commits where a subrange is explicitly excluded
+exclude_revisions:
+  revision_range:
+    end: a94fb35ca49719028a1c50bdbc2fb82122043f46
+    start: c051e44a973ee31b3baa571407694467a513ba68
+include_revisions:
+  revision_range:
+    end: 4300ea495e7f013f68e785fdde5c4ead81297999
+    start: 01f9f1f07bef22d4248e8349aba4f0c1f204607e
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: include-range-exclude-range

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/include-range-exclude-single.info

@@ -0,0 +1,10 @@
+description: Patch valid for a range of commits where a single one is excluded
+exclude_revisions:
+  single_revision: 162db88346b06be20faac6976f1ff9bad986accf
+include_revisions:
+  revision_range:
+    end: 8ca5cc28e6746eef7340064b5d843631841bf31e
+    start: 01f9f1f07bef22d4248e8349aba4f0c1f204607e
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: include-range-exclude-single

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/include-revision-range.info

@@ -0,0 +1,8 @@
+description: Patch that is valid for a range of commits
+include_revisions:
+  revision_range:
+    end: 8ca5cc28e6746eef7340064b5d843631841bf31e
+    start: 01f9f1f07bef22d4248e8349aba4f0c1f204607e
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: include-revision-range

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/include-single-and-revision-range.info

@@ -0,0 +1,9 @@
+description: Patch that is valid for a revision range AND another single commit
+include_revisions:
+  revision_range:
+    end: 8ca5cc28e6746eef7340064b5d843631841bf31e
+    start: 01f9f1f07bef22d4248e8349aba4f0c1f204607e
+  single_revision: 27f17080376e409860405c40744887d81d6b3f34
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: include-single-and-revision-range

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/include-single-revision.info

@@ -0,0 +1,6 @@
+description: Patch that is valid for a single revision
+include_revisions:
+  single_revision: 8ca5cc28e6746eef7340064b5d843631841bf31e
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: include-single-revision

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/regression-severity.info

@@ -0,0 +1,5 @@
+description: Patch with a regression severity
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: regression-severity
+regression_severity: 1000

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/unrestricted-range.info

@@ -0,0 +1,5 @@
+description: Patch describing an unrestricted range of commits, meaning it should
+  include all commits
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: unrestricted-range