Merge remote-tracking branch 'origin/master' into sdp_concat_optimize…

…d_states

.github/actions/smart-ci/action.yml

@@ -32,6 +32,14 @@ inputs:
     description: "Path to labeler configuration file"
     required: false
     default: ".github/labeler.yml"
+  skip_when_only_listed_labels_set:
+    description: "Comma-separated list of labels. If PR has only these labels set, 
+                  return indicator that CI can be skipped"
+    required: false
+  skip_when_only_listed_files_changed:
+    description: "Comma-separated list of patterns (fnmatch-style). If PR has only matching files changed, 
+                  return indicator that CI can be skipped"
+    required: false
 
 outputs:
   all_components:
@@ -40,6 +48,9 @@ outputs:
   affected_components:
     description: "Affected components to run validation for and their validation scope"
     value: ${{ steps.smart_ci.outputs.affected_components }}
+  skip_workflow:
+    description: "Whether the workflow should be run with Smart CI rules applied or skipped completely"
+    value: ${{ steps.smart_ci.outputs.skip_workflow }}
 
 runs:
   using: "composite"
@@ -80,7 +91,9 @@ runs:
           -p "${{ inputs.component_pattern }}" \
           -c "${{ inputs.components_config }}" \
           -m "${{ inputs.components_config_schema }}" \
-          -l "${{ inputs.labeler_config }}"
+          -l "${{ inputs.labeler_config }}" \
+          --skip-when-only-listed-labels-set "${{ inputs.skip_when_only_listed_labels_set }}" \
+          --skip-when-only-listed-files-changed "${{ inputs.skip_when_only_listed_files_changed }}"
       shell: bash
       env:
         GITHUB_TOKEN: ${{ inputs.repo_token }}

.github/actions/smart-ci/smart_ci.py

@@ -7,6 +7,7 @@
 import logging
 from pathlib import Path
 from ghapi.all import GhApi
+from fnmatch import fnmatch
 
 
 class ComponentConfig:
@@ -121,6 +122,12 @@ def parse_args():
                         help='Path to the schema file for components config')
     parser.add_argument('-l', '--labeler-config', default='.github/labeler.yml',
                         help='Path to PR labeler config file')
+    parser.add_argument('--skip-when-only-listed-labels-set',
+                        help="Comma-separated list of labels. If PR has only these labels set, "
+                             "return indicator that CI can be skipped")
+    parser.add_argument('--skip-when-only-listed-files-changed',
+                        help="Comma-separated list of patterns (fnmatch-style). If PR has only matching files changed, "
+                             "return indicator that CI can be skipped")
     args = parser.parse_args()
     return args
 
@@ -189,25 +196,25 @@ def main():
     cfg = ComponentConfig(components_config, schema, all_possible_components)
     affected_components = cfg.get_affected_components(changed_component_names)
 
-    # We don't need to run workflow if changes were only in documentation
-    # This is the case when we have no product labels set except "docs"
-    # or only if md files were matched (the latter covers cases where *.md is outside docs directory)
-    only_docs_changes = False
+    skip_workflow = False
     if args.pr and not run_full_scope:
-        # We don't want to add helper labels to labeler config for now, so handling it manually
-        docs_label_only = changed_component_names - {'docs'} == set()
-        if docs_label_only:
-            only_docs_changes = True
-        else:
+        if args.skip_when_only_listed_labels_set:
+            excepted_labels = set(args.skip_when_only_listed_labels_set.split(','))
+            excepted_labels_only = changed_component_names - excepted_labels == set()
+            skip_workflow = excepted_labels_only
+
+        if not skip_workflow and args.skip_when_only_listed_files_changed:
             # To avoid spending extra API requests running step below only if necessary
             changed_files = gh_api.pulls.list_files(args.pr)
-            doc_suffixes = ['.md', '.rst', '.png', '.jpg', '.svg']
-            only_docs_changes = all([Path(f.filename).suffix in doc_suffixes for f in changed_files])
-            logger.debug(f"doc files only: {only_docs_changes}")
+            patterns = set(args.skip_when_only_listed_files_changed.split(','))
+
+            matched_files_only = all(any(fnmatch(f.filename, pattern) for pattern in patterns) for f in changed_files)
+            logger.debug(f"matched files only: {matched_files_only}")
+            skip_workflow = matched_files_only
 
-    if only_docs_changes:
-        logger.info(f"Changes are documentation only, workflow may be skipped")
-        affected_components['docs_only'] = ComponentConfig.FullScope
+    if skip_workflow:
+        logger.info(f"All changes are marked for skip, workflow may be skipped")
+        set_github_output("skip_workflow", str(skip_workflow))
 
     # Syntactic sugar for easier use in GHA pipeline
     affected_components_output = {name: {s: True for s in scope} for name, scope in affected_components.items()}

.github/workflows/android_arm64.yml

@@ -2,21 +2,7 @@ name: Android ARM64 with vcpkg
 on:
   workflow_dispatch:
   pull_request:
-    paths-ignore:
-      - '**/docs/**'
-      - 'docs/**'
-      - '**/**.md'
-      - '**.md'
-      - '**/layer_tests_summary/**'
-      - '**/conformance/**'
   push:
-    paths-ignore:
-      - '**/docs/**'
-      - 'docs/**'
-      - '**/**.md'
-      - '**.md'
-      - '**/layer_tests_summary/**'
-      - '**/conformance/**'
     branches:
       - master
 
@@ -26,12 +12,36 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  Smart_CI:
+    runs-on: ubuntu-latest
+    outputs:
+      affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
+      skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
+    steps:
+      - name: checkout action
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/actions/smart-ci
+
+      - name: Get affected components
+        id: smart_ci
+        uses: ./.github/actions/smart-ci
+        with:
+          repository: ${{ github.repository }}
+          pr: ${{ github.event.number }}
+          commit_sha: ${{ github.sha }}
+          component_pattern: "category: (.*)"
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          skip_when_only_listed_labels_set: 'docs'
+          skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*'
+
   Build:
+    needs: Smart_CI
     timeout-minutes: 150
     defaults:
       run:
         shell: bash
-    runs-on: aks-linux-16-cores
+    runs-on: aks-linux-16-cores-32gb
     container:
       image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
       volumes:
@@ -52,6 +62,7 @@ jobs:
       VCPKG_DEFAULT_BINARY_CACHE: '/mount/caches/ccache/android_arm64/vcpkg_cache'
       VCPKG_FORCE_SYSTEM_BINARIES: '1'
       SCCACHE_AZURE_KEY_PREFIX: android_arm64
+    if: "!needs.smart_ci.outputs.skip_workflow"
     steps:
       - name: Install git
         run: apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates

.github/workflows/build_doc.yml

@@ -4,7 +4,6 @@ on:
 
 env:
   DOXY_VER: '1.9.6'
-  DOXYREST_VER: '2.1.3'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
@@ -40,19 +39,18 @@ jobs:
           python3 -m pip install -r docs/requirements.txt
           (cd docs/openvino_sphinx_theme && python3 setup.py install)
 
-      - name: Download and install doxygen && doxyrest
+      - name: Download and install doxygen
         run: |
-          # install doxyrest
-          wget https://github.com/vovkos/doxyrest/releases/download/doxyrest-$DOXYREST_VER/doxyrest-$DOXYREST_VER-linux-amd64.tar.xz
-          tar -xf doxyrest-$DOXYREST_VER-linux-amd64.tar.xz
-          echo "$(pwd)/doxyrest-$DOXYREST_VER-linux-amd64/bin/" >> $GITHUB_PATH
           # install doxygen
           wget https://www.doxygen.nl/files/doxygen-$DOXY_VER.linux.bin.tar.gz
           tar -xzf doxygen-$DOXY_VER.linux.bin.tar.gz
           echo "$(pwd)/doxygen-$DOXY_VER/bin/" >> $GITHUB_PATH
 
-      - name: CMake configure
-        run: cmake -DENABLE_DOCS=ON -B build
+      - name: Build docs
+        run: |
+          rm -rf build && mkdir build && cd build
+          cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_DOCS=ON -DENABLE_CPP_API=ON
+          cmake --build . --target sphinx_docs
 
       - name: Cache documentation
         id: cache_sphinx_docs
@@ -61,13 +59,6 @@ jobs:
           path: build/docs/_build/.doctrees
           key: sphinx-docs-cache
 
-      - name: Get number of CPU cores
-        uses: SimenB/github-actions-cpu-cores@v2
-        id: cpu-cores
-
-      - name: Build docs
-        run: cmake --build build --target sphinx_docs --parallel ${{ steps.cpu-cores.outputs.count }}
-
       - name: Archive docs HTML
         run: (cd build/docs && zip -r openvino_docs_html.zip _build)
 
@@ -97,14 +88,13 @@ jobs:
       - name: Run Pytest
         run: |
           pytest --doxygen="./build/docs/doxygen.log" \
-                 --include_pot \
                  --sphinx="./build/docs/sphinx.log" \
-                 --suppress-warnings="./docs/suppress_warnings.txt" \
+                 --suppress-warnings="./docs/scripts/tests/suppress_warnings.txt" \
                  --confcutdir="./docs/scripts/tests/" \
                  --html="./build/docs/_artifacts/doc-generation.html" \
                  --doxygen-strip="$(pwd)" \
-                 --sphinx-strip="$(pwd)/build/docs/rst" \
-                 --doxygen-xfail="./docs/doxygen-xfail.txt" \
+                 --sphinx-strip="$(pwd)/build/docs/sphinx_source" \
+                 --xfail="./docs/scripts/tests/xfail.txt" \
                  --self-contained-html ./docs/scripts/tests/test_docs.py
 
       - name: 'Upload test results'

.github/workflows/fedora.yml

@@ -2,21 +2,7 @@ name: Fedora (RHEL), Python 3.9
 on:
   workflow_dispatch:
   pull_request:
-    paths-ignore:
-      - '**/docs/**'
-      - 'docs/**'
-      - '**/**.md'
-      - '**.md'
-      - '**/layer_tests_summary/**'
-      - '**/conformance/**'
   push:
-    paths-ignore:
-      - '**/docs/**'
-      - 'docs/**'
-      - '**/**.md'
-      - '**.md'
-      - '**/layer_tests_summary/**'
-      - '**/conformance/**'
     branches:
       - master
       - 'releases/**'
@@ -27,12 +13,36 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  Smart_CI:
+    runs-on: ubuntu-latest
+    outputs:
+      affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
+      skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
+    steps:
+      - name: checkout action
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/actions/smart-ci
+
+      - name: Get affected components
+        id: smart_ci
+        uses: ./.github/actions/smart-ci
+        with:
+          repository: ${{ github.repository }}
+          pr: ${{ github.event.number }}
+          commit_sha: ${{ github.sha }}
+          component_pattern: "category: (.*)"
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          skip_when_only_listed_labels_set: 'docs'
+          skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*'
+
   Build:
+    needs: Smart_CI
     timeout-minutes: 150
     defaults:
       run:
         shell: bash
-    runs-on: aks-linux-16-cores
+    runs-on: aks-linux-16-cores-32gb
     container:
       image: fedora:33
       volumes:
@@ -49,6 +59,7 @@ jobs:
       INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install
       BUILD_DIR: /__w/openvino/openvino/openvino_build
       SCCACHE_AZURE_KEY_PREFIX: fedora33_x86_64_Release
+    if: "!needs.smart_ci.outputs.skip_workflow"
     steps:
       - name: Install git
         run: yum update -y && yum install -y git

.github/workflows/linux.yml

@@ -24,6 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
+      skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
     steps:
       - name: checkout action
         uses: actions/checkout@v4
@@ -39,19 +40,16 @@ jobs:
           commit_sha: ${{ github.sha }}
           component_pattern: "category: (.*)"
           repo_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Show affected components
-        run: |
-          echo "${{ toJSON(steps.smart_ci.outputs.affected_components) }}"
-        shell: bash
+          skip_when_only_listed_labels_set: 'docs'
+          skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg'
 
   Build:
     needs: Smart_CI
     timeout-minutes: 150
     defaults:
       run:
         shell: bash
-    runs-on: aks-linux-16-cores
+    runs-on: aks-linux-16-cores-32gb
     container:
       image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
       volumes:
@@ -72,7 +70,7 @@ jobs:
       BUILD_DIR: /__w/openvino/openvino/openvino_build
       SCCACHE_AZURE_KEY_PREFIX: ubuntu20_x86_64_Release
       ONNX_RUNTIME_UTILS: /__w/openvino/openvino/openvino/.ci/azure/ci_utils/onnxruntime
-    if: "!fromJSON(needs.smart_ci.outputs.affected_components).docs_only"
+    if: "!needs.smart_ci.outputs.skip_workflow"
 
     steps:
       - name: Install git
@@ -529,7 +527,7 @@ jobs:
     defaults:
       run:
         shell: bash
-    runs-on: aks-linux-16-cores
+    runs-on: aks-linux-16-cores-32gb
     container:
       image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
       volumes:
@@ -1278,7 +1276,7 @@ jobs:
       run:
         shell: bash
     runs-on: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores'}}
-    timeout-minutes: ${{ github.event_name == 'schedule' && 400 || 5 }}
+    timeout-minutes: ${{ github.event_name == 'schedule' && 400 || 25 }}
     # TODO: Switch back to self-hosted runners
     # container:
     #   image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
@@ -1364,7 +1362,7 @@ jobs:
       run:
         shell: bash
     runs-on: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores'}}
-    timeout-minutes: ${{ github.event_name == 'schedule' && 400 || 5 }}
+    timeout-minutes: ${{ github.event_name == 'schedule' && 400 || 25 }}
     env:
       OPENVINO_REPO: ${{ github.workspace }}/openvino
       INSTALL_DIR: ${{ github.workspace }}/install
@@ -1550,7 +1548,7 @@ jobs:
     defaults:
       run:
         shell: bash
-    runs-on: aks-linux-16-cores
+    runs-on: aks-linux-16-cores-32gb
     container:
       image: openvinogithubactions.azurecr.io/dockerhub/nvidia/cuda:11.8.0-runtime-ubuntu20.04
       volumes: