ci(bazel): simplify scripts and configuration in advance of additions

Simplify the Bazel-related CI scripts and BUILD configuration in preparation for adding several more for the compression feature. - Move CC=clang to .bazelrc - Define CXX in case it's already defined in the environment, in which case it overrides deferring to CC - Replace --repo_env, which doesn't affect build, with --action_env - Move commented-out ubsan invocation to its own CI job. Since ubsan was commented out, Bazel invocations have moved to their own jobs so they can run in parallel. - Consistently test all targets, `...`, since TFLM is no longer confined to the tensorflow/lite/micro directory. Filter out inappropriate new targets with no.san tags. - Don't run `bazel build ...` before `bazel test ...`; the latter already builds all non-test targets. - Remove obsolete filter for `no_oss`, there were no more `no_oss` tags in the tree. - Stop using readable_run: it has diverged from its original meaning and now only redirects stderr to stdout. This is not needed. BUG=see description
tensorflow · Nov 20, 2024 · 5dd1af8 · 5dd1af8
1 parent 31b0d73
commit 5dd1af8
Show file tree

Hide file tree

Showing 9 changed files with 94 additions and 58 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -34,17 +34,21 @@ build --cxxopt -std=c++17
 build --copt -Werror
 
 # Common options for --config=ci
+build:ci --action_env=CC=clang
+build:ci --action_env=CXX=clang++
 build:ci --curses=no 
 build:ci --color=no 
 build:ci --noshow_progress
 build:ci --noshow_loading_progress
 build:ci --show_timestamps
 build:ci --terminal_columns=0
 build:ci --verbose_failures
+build:ci --test_output=errors
 
 # When building with the address sanitizer
 # E.g., bazel build --config asan
-build:asan --repo_env CC=clang
+build:asan --action_env=CC=clang
+build:asan --action_env=CXX=clang++
 build:asan --strip=never
 build:asan --copt -fsanitize=address
 build:asan --copt -DADDRESS_SANITIZER
@@ -55,7 +59,8 @@ build:asan --linkopt -fsanitize=address
 
 # When building with the memory sanitizer
 # E.g., bazel build --config msan
-build:msan --repo_env CC=clang
+build:msan --action_env=CC=clang
+build:msan --action_env=CXX=clang++
 build:msan --strip=never
 build:msan --copt -fsanitize=memory
 build:msan --copt -DADDRESS_SANITIZER
@@ -66,7 +71,8 @@ build:msan --linkopt -fsanitize=memory
 
 # When building with the undefined behavior sanitizer
 # E.g., bazel build --config ubsan
-build:ubsan --repo_env CC=clang
+build:ubsan --action_env=CC=clang
+build:ubsan --action_env=CXX=clang++
 build:ubsan --strip=never
 build:ubsan --copt -fsanitize=undefined
 build:ubsan --copt -g

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -90,6 +90,32 @@ jobs:
         run: |
           tensorflow/lite/micro/tools/ci_build/test_bazel_asan.sh
 
+  # TODO(b/178621680): enable ubsan once bazel + clang + ubsan errors are fixed.
+  # bazel_ubsan:
+  #   runs-on: ubuntu-latest
+  #
+  #   name: Bazel ubsan (presubmit)
+  #   steps:
+  #     - uses: actions/setup-python@v4
+  #       with:
+  #         python-version: '3.10'
+  #     - uses: actions/checkout@v3
+  #       with:
+  #         ref: ${{ inputs.trigger-sha }}
+  #     - name: Install dependencies
+  #       run: |
+  #         sudo ci/install_bazelisk.sh
+  #         pip3 install Pillow
+  #         pip3 install numpy
+  #     - name: Fix kernel mmap rnd bits
+  #       # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with
+  #       # high-entropy ASLR in much newer kernels that GitHub runners are
+  #       # using leading to random crashes: https://reviews.llvm.org/D148280
+  #       run: sudo sysctl vm.mmap_rnd_bits=28
+  #     - name: Test
+  #       run: |
+  #         tensorflow/lite/micro/tools/ci_build/test_bazel_ubsan.sh
+
   cortex_m_bluepill_release:
     runs-on: ubuntu-latest
 

diff --git a/python/tests/BUILD b/python/tests/BUILD
@@ -4,6 +4,9 @@ cc_test(
     srcs = [
         "cc_deps_link_test.cc",
     ],
+    tags = [
+        "nomsan",  # avoid, because tensorflow library fails msan
+    ],
     deps = [
         "@tensorflow_cc_deps//:cc_library",
     ],

diff --git a/python/tflite_micro/BUILD b/python/tflite_micro/BUILD
@@ -251,6 +251,9 @@ sh_test(
         ":whl",
     ],
     tags = [
+        "noasan",
+        "nomsan",
         "notap",  # See http://b/294278650#comment4 for more details.
+        "noubsan",
     ],
 )
diff --git a/python/tflite_micro/signal/BUILD b/python/tflite_micro/signal/BUILD
@@ -68,6 +68,11 @@ py_test(
     srcs = ["ops/delay_op_test.py"],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":delay_op",
         requirement("numpy"),
@@ -96,6 +101,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":energy_op",
         requirement("numpy"),
@@ -124,6 +134,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":fft_ops",
         requirement("numpy"),
@@ -158,6 +173,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":filter_bank_ops",
         requirement("numpy"),
@@ -186,6 +206,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":framer_op",
         requirement("numpy"),
@@ -211,6 +236,11 @@ py_test(
     srcs = ["ops/overlap_add_op_test.py"],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":overlap_add_op",
         "@absl_py//absl/testing:parameterized",
@@ -273,6 +303,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":stacker_op",
         requirement("numpy"),
@@ -300,6 +335,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
     deps = [
         ":window_op",
         requirement("numpy"),

diff --git a/python/tflite_micro/signal/utils/BUILD b/python/tflite_micro/signal/utils/BUILD
@@ -18,6 +18,11 @@ py_test(
     ],
     python_version = "PY3",
     srcs_version = "PY3",
+    tags = [
+        "noasan",
+        "nomsan",
+        "noubsan",
+    ],
 )
 
 py_library(

diff --git a/tensorflow/lite/micro/tools/ci_build/test_bazel.sh b/tensorflow/lite/micro/tools/ci_build/test_bazel.sh
@@ -17,24 +17,5 @@
 set -e
 set -x
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ROOT_DIR=${SCRIPT_DIR}/../../../../..
-cd "${ROOT_DIR}"
-
-source tensorflow/lite/micro/tools/ci_build/helper_functions.sh
-
-# We are using a bazel build followed by bazel test to make sure that the CI
-# covers non-test binary targets as well. These were previousbly covered by
-# having build_test but that was removed with #194.
-
-CC=clang readable_run bazel build ... \
-  --config=ci \
-  --build_tag_filters=-no_oss
-CC=clang readable_run bazel test ... \
-  --config=ci \
-  --test_tag_filters=-no_oss --build_tag_filters=-no_oss \
-  --test_output=errors
-
-# TODO(b/178621680): enable ubsan once bazel + clang + ubsan errors are fixed.
-#CC=clang readable_run bazel test tensorflow/lite/micro/... --config=ubsan --test_tag_filters=-no_oss,-noubsan --build_tag_filters=-no_oss,-noubsan
-
+bazel test ... \
+  --config=ci
diff --git a/tensorflow/lite/micro/tools/ci_build/test_bazel_asan.sh b/tensorflow/lite/micro/tools/ci_build/test_bazel_asan.sh
@@ -17,22 +17,8 @@
 set -e
 set -x
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ROOT_DIR=${SCRIPT_DIR}/../../../../..
-cd "${ROOT_DIR}"
-
-source tensorflow/lite/micro/tools/ci_build/helper_functions.sh
-
-# We are using a bazel build followed by bazel test to make sure that the CI
-# covers non-test binary targets as well. These were previousbly covered by
-# having build_test but that was removed with #194.
-
-CC=clang readable_run bazel build tensorflow/lite/micro/... \
-  --config=ci \
-  --config=asan --build_tag_filters=-no_oss,-noasan
-
-CC=clang readable_run bazel test tensorflow/lite/micro/... \
+bazel test ... \
   --config=ci \
   --config=asan \
-  --test_tag_filters=-no_oss,-noasan --build_tag_filters=-no_oss,-noasan \
-  --test_output=errors
+  --build_tag_filters=-noasan \
+  --test_tag_filters=-noasan
diff --git a/tensorflow/lite/micro/tools/ci_build/test_bazel_msan.sh b/tensorflow/lite/micro/tools/ci_build/test_bazel_msan.sh
@@ -17,22 +17,8 @@
 set -e
 set -x
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ROOT_DIR=${SCRIPT_DIR}/../../../../..
-cd "${ROOT_DIR}"
-
-source tensorflow/lite/micro/tools/ci_build/helper_functions.sh
-
-# We are using a bazel build followed by bazel test to make sure that the CI
-# covers non-test binary targets as well. These were previousbly covered by
-# having build_test but that was removed with #194.
-
-CC=clang readable_run bazel build tensorflow/lite/micro/... \
-  --config=ci \
-  --config=msan --build_tag_filters=-no_oss,-nomsan
-
-CC=clang readable_run bazel test tensorflow/lite/micro/... \
+bazel test ... \
   --config=ci \
   --config=msan \
-  --test_tag_filters=-no_oss,-nomsan --build_tag_filters=-no_oss,-nomsan \
-  --test_output=errors
+  --build_tag_filters=-nomsan \
+  --test_tag_filters=-nomsan