ci(bazel): simplify scripts and configuration in advance of additions (…

…#2918) * ci(bazel): simplify scripts and configuration in advance of additions Simplify the Bazel-related CI scripts and BUILD configuration in preparation for adding several more for the compression feature. - Move CC=clang to .bazelrc - Define CXX in case it's already defined in the environment, in which case it overrides deferring to CC - Replace --repo_env, which doesn't affect build, with --action_env - Move commented-out ubsan invocation to its own CI job. Since ubsan was commented out, Bazel invocations have moved to their own jobs so they can run in parallel. - Consistently test all targets, `...`, since TFLM is no longer confined to the tensorflow/lite/micro directory. Filter out inappropriate new targets with no.san tags. - Don't run `bazel build ...` before `bazel test ...`; the latter already builds all non-test targets. - Remove obsolete filter for `no_oss`, there were no more `no_oss` tags in the tree. - Stop using readable_run: it has diverged from its original meaning and now only redirects stderr to stdout. This is not needed. BUG=see description * refactor: clarify intention to run at top and test entire project Clarify that the scripts run from the root directory of the project by restoring the change of directory at the top of the script, even though CI runs from the root directory of the project. Also clarify the intention that all tests in the project be run by specifying `//...`, rather than `...`, which means all targets at or below the working directory.
tensorflow · Nov 21, 2024 · 124db96 · 124db96
1 parent 31b0d73
commit 124db96
Show file tree

Hide file tree

Showing 9 changed files with 94 additions and 46 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -34,17 +34,21 @@ build --cxxopt -std=c++17
 build --copt -Werror
 
 # Common options for --config=ci
+build:ci --action_env=CC=clang
+build:ci --action_env=CXX=clang++
 build:ci --curses=no 
 build:ci --color=no 
 build:ci --noshow_progress
 build:ci --noshow_loading_progress
 build:ci --show_timestamps
 build:ci --terminal_columns=0
 build:ci --verbose_failures
+build:ci --test_output=errors
 
 # When building with the address sanitizer
 # E.g., bazel build --config asan
-build:asan --repo_env CC=clang
+build:asan --action_env=CC=clang
+build:asan --action_env=CXX=clang++
 build:asan --strip=never
 build:asan --copt -fsanitize=address
 build:asan --copt -DADDRESS_SANITIZER
@@ -55,7 +59,8 @@ build:asan --linkopt -fsanitize=address
 
 # When building with the memory sanitizer
 # E.g., bazel build --config msan
-build:msan --repo_env CC=clang
+build:msan --action_env=CC=clang
+build:msan --action_env=CXX=clang++
 build:msan --strip=never
 build:msan --copt -fsanitize=memory
 build:msan --copt -DADDRESS_SANITIZER
@@ -66,7 +71,8 @@ build:msan --linkopt -fsanitize=memory
 
 # When building with the undefined behavior sanitizer
 # E.g., bazel build --config ubsan
-build:ubsan --repo_env CC=clang
+build:ubsan --action_env=CC=clang
+build:ubsan --action_env=CXX=clang++
 build:ubsan --strip=never
 build:ubsan --copt -fsanitize=undefined
 build:ubsan --copt -g

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -90,6 +90,32 @@ jobs:
         run: |
           tensorflow/lite/micro/tools/ci_build/test_bazel_asan.sh
 
+  # TODO(b/178621680): enable ubsan once bazel + clang + ubsan errors are fixed.
+  # bazel_ubsan:
+  #   runs-on: ubuntu-latest
+  #
+  #   name: Bazel ubsan (presubmit)
+  #   steps:
+  #     - uses: actions/setup-python@v4
+  #       with:
+  #         python-version: '3.10'
+  #     - uses: actions/checkout@v3
+  #       with:
+  #         ref: ${{ inputs.trigger-sha }}
+  #     - name: Install dependencies
+  #       run: |
+  #         sudo ci/install_bazelisk.sh
+  #         pip3 install Pillow
+  #         pip3 install numpy
+  #     - name: Fix kernel mmap rnd bits
+  #       # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with
+  #       # high-entropy ASLR in much newer kernels that GitHub runners are
+  #       # using leading to random crashes: https://reviews.llvm.org/D148280
+  #       run: sudo sysctl vm.mmap_rnd_bits=28
+  #     - name: Test
+  #       run: |
+  #         tensorflow/lite/micro/tools/ci_build/test_bazel_ubsan.sh
+
   cortex_m_bluepill_release:
     runs-on: ubuntu-latest
 

diff --git a/python/tests/BUILD b/python/tests/BUILD
@@ -4,6 +4,9 @@ cc_test(
     srcs = [
         "cc_deps_link_test.cc",
     ],
+    tags = [
+        "nomsan",  # avoid, because tensorflow library fails msan
+    ],
     deps = [
         "@tensorflow_cc_deps//:cc_library",
     ],

diff --git a/python/tflite_micro/BUILD b/python/tflite_micro/BUILD
@@ -251,6 +251,9 @@ sh_test(
         ":whl",
     ],
     tags = [
+        "noasan",
+        "nomsan",
         "notap",  # See http://b/294278650#comment4 for more details.
+        "noubsan",
     ],
 )
diff --git a/python/tflite_micro/signal/BUILD b/python/tflite_micro/signal/BUILD
@@ -68,6 +68,11 @@ py_test(
     srcs = ["ops/delay_op_test.py"],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":delay_op",
         requirement("numpy"),
@@ -96,6 +101,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":energy_op",
         requirement("numpy"),
@@ -124,6 +134,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":fft_ops",
         requirement("numpy"),
@@ -158,6 +173,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":filter_bank_ops",
         requirement("numpy"),
@@ -186,6 +206,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":framer_op",
         requirement("numpy"),
@@ -211,6 +236,11 @@ py_test(
     srcs = ["ops/overlap_add_op_test.py"],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":overlap_add_op",
         "@absl_py//absl/testing:parameterized",
@@ -273,6 +303,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":stacker_op",
         requirement("numpy"),
@@ -300,6 +335,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":window_op",
         requirement("numpy"),

diff --git a/python/tflite_micro/signal/utils/BUILD b/python/tflite_micro/signal/utils/BUILD
@@ -18,6 +18,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
 )
 
 py_library(

diff --git a/tensorflow/lite/micro/tools/ci_build/test_bazel.sh b/tensorflow/lite/micro/tools/ci_build/test_bazel.sh
@@ -21,20 +21,5 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 ROOT_DIR=${SCRIPT_DIR}/../../../../..
 cd "${ROOT_DIR}"
 
-source tensorflow/lite/micro/tools/ci_build/helper_functions.sh
-
-# We are using a bazel build followed by bazel test to make sure that the CI
-# covers non-test binary targets as well. These were previousbly covered by
-# having build_test but that was removed with #194.
-
-CC=clang readable_run bazel build ... \
-  --config=ci \
-  --build_tag_filters=-no_oss
-CC=clang readable_run bazel test ... \
-  --config=ci \
-  --test_tag_filters=-no_oss --build_tag_filters=-no_oss \
-  --test_output=errors
-
-# TODO(b/178621680): enable ubsan once bazel + clang + ubsan errors are fixed.
-#CC=clang readable_run bazel test tensorflow/lite/micro/... --config=ubsan --test_tag_filters=-no_oss,-noubsan --build_tag_filters=-no_oss,-noubsan
-
+bazel test //... \
+  --config=ci
diff --git a/tensorflow/lite/micro/tools/ci_build/test_bazel_asan.sh b/tensorflow/lite/micro/tools/ci_build/test_bazel_asan.sh
@@ -21,18 +21,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 ROOT_DIR=${SCRIPT_DIR}/../../../../..
 cd "${ROOT_DIR}"
 
-source tensorflow/lite/micro/tools/ci_build/helper_functions.sh
-
-# We are using a bazel build followed by bazel test to make sure that the CI
-# covers non-test binary targets as well. These were previousbly covered by
-# having build_test but that was removed with #194.
-
-CC=clang readable_run bazel build tensorflow/lite/micro/... \
-  --config=ci \
-  --config=asan --build_tag_filters=-no_oss,-noasan
-
-CC=clang readable_run bazel test tensorflow/lite/micro/... \
+bazel test //... \
   --config=ci \
   --config=asan \
-  --test_tag_filters=-no_oss,-noasan --build_tag_filters=-no_oss,-noasan \
-  --test_output=errors
+  --build_tag_filters=-noasan \
+  --test_tag_filters=-noasan
diff --git a/tensorflow/lite/micro/tools/ci_build/test_bazel_msan.sh b/tensorflow/lite/micro/tools/ci_build/test_bazel_msan.sh
@@ -21,18 +21,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 ROOT_DIR=${SCRIPT_DIR}/../../../../..
 cd "${ROOT_DIR}"
 
-source tensorflow/lite/micro/tools/ci_build/helper_functions.sh
-
-# We are using a bazel build followed by bazel test to make sure that the CI
-# covers non-test binary targets as well. These were previousbly covered by
-# having build_test but that was removed with #194.
-
-CC=clang readable_run bazel build tensorflow/lite/micro/... \
-  --config=ci \
-  --config=msan --build_tag_filters=-no_oss,-nomsan
-
-CC=clang readable_run bazel test tensorflow/lite/micro/... \
+bazel test //... \
   --config=ci \
   --config=msan \
-  --test_tag_filters=-no_oss,-nomsan --build_tag_filters=-no_oss,-nomsan \
-  --test_output=errors
+  --build_tag_filters=-nomsan \
+  --test_tag_filters=-nomsan