Merge branch 'master' into fix_test04

krai · Dec 9, 2024 · 2a0aeb2 · 2a0aeb2
2 parents b40a42c + dd9be96
commit 2a0aeb2
Show file tree

Hide file tree

Showing 589 changed files with 63,486 additions and 18,344 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,3 +1,5 @@
 # These owners will be the default owners for everything in the repo.
 # Unless a later match takes precedence,they will be requested for review when someone opens a pull request.
 * @mlcommons/wg-inference
+
+/CODEOWNERS @mlcommons/staff
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -3,37 +3,141 @@ name: Build loadgen wheels and release them into PYPI
 on:
   release:
     types: [published]
+  push:
+    branches:
+      - master
+      - loadgen-release
+      - dev
+    paths:
+      - loadgen/**
 
 jobs:
+  update_version:
+    name: Update version only on ubuntu but used by windows and macos
+    if: github.repository_owner == 'mlcommons'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+          ssh-key: ${{ secrets.DEPLOY_KEY }}
+
+      # Check if VERSION.txt file has changed in this push
+      - name: Check if VERSION.txt file has changed
+        id: version_changed
+        run: |
+          echo "version_changed=false" >> $GITHUB_ENV
+          echo "new_version=" >> $GITHUB_ENV  # Initialize with empty value
+          if git diff --name-only HEAD~1 | grep -q "VERSION.txt"; then
+            echo "VERSION.txt file has been modified"
+            echo "version_changed=true" >> $GITHUB_ENV
+            new_version=$(cat loadgen/VERSION.txt)
+            echo "new_version=$new_version" >> $GITHUB_ENV
+          else
+            echo "VERSION.txt file has NOT been modified"
+          fi
+
+      # Step 4: Increment version if VERSION.txt was not changed
+      - name: Increment version if necessary
+        id: do_version_increment
+        if: env.version_changed == 'false'
+        run: |
+          cd loadgen
+          # Check if VERSION file exists, else initialize it
+          if [ ! -f VERSION.txt ]; then
+            echo "0.0.0" > VERSION.txt
+          fi
+
+          version=$(cat VERSION.txt)
+          IFS='.' read -r major minor patch <<< "$version"
+          patch=$((patch + 1))
+          new_version="$major.$minor.$patch"
+          echo $new_version > VERSION.txt
+          echo "New version: $new_version"
+          echo "new_version=$new_version" >> $GITHUB_ENV
+
+      # Step 5: Commit the updated version to the repository
+      - name: Commit updated version
+        if: env.version_changed == 'false'
+        run: |
+          cd loadgen
+          git config --global user.name "${{ github.actor }}"
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          git add VERSION.txt
+          git commit -m "Increment version to $new_version"
+          git push
+  
   build_wheels:
     name: Build wheels on ${{ matrix.os }}
+    needs: update_version
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
 
     steps:
       - uses: actions/checkout@v3
 
       - uses: actions/setup-python@v3
 
       - name: Install requirements
-        run: python -m pip install cibuildwheel==2.11.4 twine==4.0.2
+        run: python -m pip install cibuildwheel twine build
 
-      - name: Init pybind11 submodule
+      - name: Build src dist
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         run: |
-          git submodule init third_party/pybind
-          git submodule update third_party/pybind
+          python -m build --sdist --outdir wheels loadgen
 
       - name: Build wheels
-        run: python -m cibuildwheel loadgen/ --output-dir wheels
-        env:
-          CIBW_ENVIRONMENT: "CFLAGS='-std=c++14'"
-          CIBW_BUILD: 'cp3{6,7,8,9,10}-*'
+        run: git pull && python -m cibuildwheel loadgen/ --output-dir wheels
 
-      - uses: actions/upload-artifact@v3
+       # Save wheels as artifacts
+      - name: Upload built wheels
+        uses: actions/upload-artifact@v4
         with:
-          path: ./wheels/*.whl 
+          name: wheels-${{ matrix.os }}
+          path: wheels
 
-      - name: Publish package to PyPI
-        run: python -m twine upload wheels/* -u __token__ -p ${{ secrets.PYPI_API_TOKEN }}
+  publish_wheels:
+    needs: build_wheels  # Wait for the build_wheels job to complete
+    runs-on: ubuntu-latest  # Only run this job on Linux
+    environment: release
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+    steps:
+      - uses: actions/checkout@v3
+
+      # Download the built wheels from ubuntu
+      - name: Download Ubuntu wheels
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels-ubuntu-latest
+          path: wheels
+      # Download the built wheels from macOS-latest
+      - name: Download macOS-latest wheels
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels-macos-latest
+          path: wheels
+      # Download the built wheels from macOS-13 (x86)
+      - name: Download macOS-13 (x86) wheels
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels-macos-13
+          path: wheels
+      # Download the built wheels from Windows
+      - name: Download Windows wheels
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels-windows-latest
+          path: wheels
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          verify-metadata: true
+          skip-existing: true
+          packages-dir: wheels
+          repository-url: https://upload.pypi.org/legacy/
+          verbose: true
diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml
@@ -8,6 +8,7 @@ on:
 
 jobs:
   cla-check:
+    if: github.repository_owner == 'mlcommons'
     runs-on: ubuntu-latest
     steps:
       - name: "MLCommons CLA bot check"

diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml
@@ -0,0 +1,60 @@
+# Automatic code formatting
+name: "Code formatting"
+on:
+  push:
+    branches:
+    - "**"
+
+env:
+  python_version: "3.9"
+
+jobs:
+  format-code:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ssh-key: ${{ secrets.DEPLOY_KEY }}
+      - name: Set up Python ${{ env.python_version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ env.python_version }}
+
+      - name: Format modified python files
+        env:
+          filter: ${{ github.event.before }}
+        run: |
+          python3 -m pip install autopep8
+          for FILE in $(git diff --name-only $filter | grep -E '.*\.py$')
+          do
+            # Check if the file still exists in the working tree
+            if [ -f "$FILE" ]; then
+              autopep8 --in-place -a "$FILE"
+              git add "$FILE"
+            fi
+          done
+
+      - name: Format modified C++ files
+        env:
+          filter: ${{ github.event.before }}
+        run: |
+          for FILE in $(git diff --name-only $filter | grep -E '.*\.(cc|cpp|h|hpp)$')
+          do
+            # Check if the file still exists in the working tree
+            if [ -f "$FILE" ]; then
+              clang-format -i -style=file $FILE
+              git add $FILE
+            fi
+          done
+
+      - name: Commit and push changes
+        run: |
+          HAS_CHANGES=$(git diff --staged --name-only)
+          if [ ${#HAS_CHANGES} -gt 0 ]; then
+            git config --global user.name mlcommons-bot
+            git config --global user.email "[email protected]"
+            # Commit changes
+            git commit -m '[Automated Commit] Format Codebase'
+            git push
+          fi 
diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -0,0 +1,33 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Publish site
+
+
+on:
+  release:
+    types: [published]
+  push:
+    branches:
+      - master
+      - docs
+
+jobs:
+
+  publish:
+    name: Publish the site
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository normally
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install Mkdocs
+        run: pip install -r docs/requirements.txt
+
+      - name: Run Mkdocs deploy
+        run: mkdocs gh-deploy --force
diff --git a/.github/workflows/test-bert.yml b/.github/workflows/test-bert.yml
@@ -12,6 +12,9 @@ on:
       - .github/workflows/test-bert.yml
       - '!**.md'
 
+env:
+  PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
+
 jobs:
   build:
 
@@ -30,9 +33,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python3 -m pip install cmind
-        cm pull repo mlcommons@ck
-        cm run script --quiet --tags=get,sys-utils-cm
+        python3 -m pip install cm4mlops
     - name: Test BERT and end to end submission generation
       run: |
-        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=bert-99 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.env.CM_GIT_CHECKOUT=${{ github.event.pull_request.head.ref }} --adr.inference-src.env.CM_GIT_URL=${{ github.event.pull_request.head.repo.html_url }} --target_qps=1
+        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=bert-99 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.$PR_HEAD_REF --adr.loadgen.version=custom
diff --git a/.github/workflows/test-loadgen.yml b/.github/workflows/test-loadgen.yml
@@ -11,14 +11,17 @@ on:
       - .github/workflows/test-loadgen.yml
       - '!**.md'
 
+env:
+  PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
+
 jobs:
   build:
 
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3
@@ -28,8 +31,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python3 -m pip install cmind
-        cm pull repo mlcommons@ck
+        python3 -m pip install cm4mlops
     - name: Test Loadgen
       run: |
-        cm run script --tags=get,mlperf,inference,loadgen --quiet --version=custom --adr.inference-src.env.CM_GIT_CHECKOUT=${{ github.event.pull_request.head.ref }} --adr.inference-src.env.CM_GIT_URL=${{ github.event.pull_request.head.repo.html_url }}
+        cm run script --tags=get,mlperf,inference,loadgen --quiet --version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.$PR_HEAD_REF --adr.loadgen.tags=_no-compilation-warnings
diff --git a/.github/workflows/test-resnet50.yml b/.github/workflows/test-resnet50.yml
@@ -8,10 +8,14 @@ on:
     branches: [ "master", "dev" ]
     paths:
       - vision/classification_and_detection/**
+      - loadgen/**
       - tools/submission/**
       - .github/workflows/test-resnet50.yml
       - '!**.md'
 
+env:
+  PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
+
 jobs:
   build:
 
@@ -21,6 +25,7 @@ jobs:
       matrix:
         python-version: [ "3.9" ]
         backend: [ "onnxruntime", "tf" ]
+        loadgen-flag: [ "", "--adr.loadgen.tags=_from-pip --pip_loadgen=yes" ]
 
     steps:
     - uses: actions/checkout@v3
@@ -30,9 +35,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python3 -m pip install cmind
-        cm pull repo mlcommons@ck
-        cm run script --quiet --tags=get,sys-utils-cm
+        python3 -m pip install cm4mlops
     - name: Test Resnet50 and end to end submission generation
       run: |
-        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.env.CM_GIT_CHECKOUT=${{ github.event.pull_request.head.ref }} --adr.inference-src.version=custom --adr.inference-src.env.CM_GIT_URL=${{ github.event.pull_request.head.repo.html_url }}
+        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }}
diff --git a/.github/workflows/test-retinanet.yml b/.github/workflows/test-retinanet.yml
@@ -12,6 +12,9 @@ on:
       - .github/workflows/test-retinanet.yml
       - '!**.md'
 
+env:
+  PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
+
 jobs:
   build:
 
@@ -30,9 +33,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python3 -m pip install cmind
-        cm pull repo mlcommons@ck
-        cm run script --quiet --tags=get,sys-utils-cm
+        python3 -m pip install cm4mlops
     - name: Test Retinanet and end to end submission generation
       run: |
-        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=retinanet --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=10 --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.env.CM_GIT_CHECKOUT=${{ github.event.pull_request.head.ref }} --adr.inference-src.env.CM_GIT_URL=${{ github.event.pull_request.head.repo.html_url }}
+        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=retinanet --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=10 --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.$PR_HEAD_REF --adr.loadgen.version=custom
diff --git a/.github/workflows/test-rnnt.yml b/.github/workflows/test-rnnt.yml
@@ -5,13 +5,16 @@ name: Test for MLPerf inference rnnt submission generation using CM script autom
 
 on:
   pull_request:
-    branches: [ "master", "dev" ]
+    branches: [ "master-retired", "dev-retired" ]
     paths:
       - speech_recognition/rnnt/**
       - tools/submission/**
       - .github/workflows/test-rnnt.yml
       - '!**.md'
 
+env:
+  PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
+
 jobs:
   build:
 
@@ -36,4 +39,4 @@ jobs:
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test RNNT and end to end submission generation
       run: |
-        cm run script --tags=run,mlperf,inference,generate-run-cmds,_performance-only --quiet --submitter="MLCommons" --hw_name=default --model=rnnt --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --precision=${{ matrix.precision }} --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.env.CM_GIT_CHECKOUT=${{ github.event.pull_request.head.ref }} --adr.inference-src.env.CM_GIT_URL=${{ github.event.pull_request.head.repo.html_url }} --adr.ml-engine-pytorch.version=1.13.0 --adr.ml-engine-torchvision.version=0.14.1 --adr.librosa.version_max=0.9.1
+        cm run script --tags=run,mlperf,inference,generate-run-cmds,_performance-only --quiet --submitter="MLCommons" --hw_name=default --model=rnnt --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --precision=${{ matrix.precision }} --adr.compiler.tags=gcc  --adr.inference-src.version=custom --adr.inference-src.env.CM_GIT_CHECKOUT=$PR_HEAD_REF --adr.inference-src.env.CM_GIT_URL=${{ github.event.pull_request.head.repo.html_url }} --adr.ml-engine-pytorch.version=1.13.0 --adr.ml-engine-torchvision.version=0.14.1 --adr.librosa.version_max=0.9.1