From af1936d93ad9adab9527e19ceb5c89bb3a907e49 Mon Sep 17 00:00:00 2001 From: bazelisky <155456828+bazelisky@users.noreply.github.com> Date: Thu, 1 Feb 2024 16:52:55 +0800 Subject: [PATCH] [ISSUE #7715] Update Bazel toolchain to make the CI work (#7714) * Update bazel toolchain Signed-off-by: Bazelisky * fix a bad test case as it violates requirement of ConcurrentHashMap#computeIfAbsent Signed-off-by: Bazelisky * Exclude flaky tests from test suite Signed-off-by: bazelisky <155456828+bazelisky@users.noreply.github.com> * fix: format issue Signed-off-by: Zhanhui Li * fix: Remove unused import * fix deps issue Signed-off-by: Zhanhui Li * Expose data property of java_test, allowing to specify run_file Signed-off-by: Zhanhui Li --------- Signed-off-by: Bazelisky Signed-off-by: bazelisky <155456828+bazelisky@users.noreply.github.com> Signed-off-by: Zhanhui Li Co-authored-by: Zhanhui Li --- .bazelrc | 33 +++++++------------ WORKSPACE | 31 ++++++----------- bazel/GenTestRules.bzl | 2 ++ .../org/apache/rocketmq/common/MixAll.java | 4 ++- .../common/utils/ConcurrentHashMapUtils.java | 4 +-- .../apache/rocketmq/common/MixAllTest.java | 6 ---- .../rocketmq/common/NetworkUtilTest.java | 2 -- .../utils/ConcurrentHashMapUtilsTest.java | 3 -- controller/BUILD.bazel | 1 + proxy/BUILD.bazel | 2 ++ store/BUILD.bazel | 1 + test/BUILD.bazel | 5 ++- 12 files changed, 34 insertions(+), 60 deletions(-) diff --git a/.bazelrc b/.bazelrc index b0c5e1695af..6ebc08d0dc3 100644 --- a/.bazelrc +++ b/.bazelrc @@ -41,27 +41,16 @@ test --test_output=errors # for a remote machine to execute them. build:remote --jobs=150 -# Set several flags related to specifying the platform, toolchain and java -# properties. -# These flags should only be used as is for the rbe-ubuntu16-04 container -# and need to be adapted to work with other toolchain containers. -build:remote --java_runtime_version=rbe_jdk -build:remote --tool_java_runtime_version=rbe_jdk -build:remote --extra_toolchains=@rbe_default//java:all - -build:remote --crosstool_top=@rbe_default//cc:toolchain -build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 -# Platform flags: -# The toolchain container used for execution is defined in the target indicated -# by "extra_execution_platforms", "host_platform" and "platforms". -# More about platforms: https://docs.bazel.build/versions/master/platforms.html -build:remote --extra_toolchains=@rbe_default//config:cc-toolchain -build:remote --extra_execution_platforms=@rbe_default//config:platform -build:remote --host_platform=@rbe_default//config:platform -build:remote --platforms=@rbe_default//config:platform - -# Starting with Bazel 0.27.0 strategies do not need to be explicitly -# defined. See https://github.com/bazelbuild/bazel/issues/7480 +build:remote --remote_executor=grpcs://remote.buildbuddy.io +build:remote --host_platform=@buildbuddy_toolchain//:platform +build:remote --platforms=@buildbuddy_toolchain//:platform +build:remote --extra_execution_platforms=@buildbuddy_toolchain//:platform +build:remote --crosstool_top=@buildbuddy_toolchain//:toolchain +build:remote --extra_toolchains=@buildbuddy_toolchain//:cc_toolchain +build:remote --javabase=@buildbuddy_toolchain//:javabase_jdk8 +build:remote --host_javabase=@buildbuddy_toolchain//:javabase_jdk8 +build:remote --java_toolchain=@buildbuddy_toolchain//:toolchain_jdk8 +build:remote --host_java_toolchain=@buildbuddy_toolchain//:toolchain_jdk8 build:remote --define=EXECUTOR=remote # Enable remote execution so actions are performed on the remote systems. @@ -77,4 +66,4 @@ build:remote --remote_timeout=3600 # Use a pre-configured account, such that we may have pull-request replacing pull-request-target build:remote --remote_header=x-buildbuddy-api-key=FD819nUEY7WjvqmoufsU -test:remote --remote_header=x-buildbuddy-api-key=FD819nUEY7WjvqmoufsU \ No newline at end of file +test:remote --remote_header=x-buildbuddy-api-key=FD819nUEY7WjvqmoufsU diff --git a/WORKSPACE b/WORKSPACE index ac89845e560..5880f9ca479 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -109,6 +109,7 @@ maven_install( "org.apache.rocketmq:rocketmq-rocksdb:1.0.2", "com.alipay.sofa:jraft-core:1.3.11", "com.alipay.sofa:hessian:3.3.6", + "io.netty:netty-tcnative-boringssl-static:2.0.48.Final", ], fetch_sources = True, repositories = [ @@ -119,33 +120,21 @@ maven_install( http_archive( name = "io_buildbuddy_buildbuddy_toolchain", - sha256 = "a2a5cccec251211e2221b1587af2ce43c36d32a42f5d881737db3b546a536510", - strip_prefix = "buildbuddy-toolchain-829c8a574f706de5c96c54ca310f139f4acda7dd", - urls = ["https://github.com/buildbuddy-io/buildbuddy-toolchain/archive/829c8a574f706de5c96c54ca310f139f4acda7dd.tar.gz"], + sha256 = "b12273608db627eb14051eb75f8a2134590172cd69392086d392e25f3954ea6e", + strip_prefix = "buildbuddy-toolchain-8d5d18373adfca9d8e33b4812915abc9b132f1ee", + urls = ["https://github.com/buildbuddy-io/buildbuddy-toolchain/archive/8d5d18373adfca9d8e33b4812915abc9b132f1ee.tar.gz"], ) - load("@io_buildbuddy_buildbuddy_toolchain//:deps.bzl", "buildbuddy_deps") - buildbuddy_deps() - load("@io_buildbuddy_buildbuddy_toolchain//:rules.bzl", "buildbuddy") - buildbuddy(name = "buildbuddy_toolchain") -http_archive( - name = "rbe_default", - # The sha256 digest of the tarball might change without notice. So it's not - # included here. - urls = ["https://storage.googleapis.com/rbe-toolchain/bazel-configs/rbe-ubuntu1604/latest/rbe_default.tar"], -) - http_archive( name = "bazel_toolchains", - sha256 = "56d5370eb99559b4c74f334f81bc8a298f728bd16d5a4333c865c2ad10fae3bc", - strip_prefix = "bazel-toolchains-dac71231098d891e5c4b74a2078fe9343feef510", - urls = ["https://github.com/bazelbuild/bazel-toolchains/archive/dac71231098d891e5c4b74a2078fe9343feef510.tar.gz"], + sha256 = "1adf5db506a7e3c465a26988514cfc3971af6d5b3c2218925cd6e71ee443fc3f", + strip_prefix = "bazel-toolchains-4.0.0", + urls = [ + "https://github.com/bazelbuild/bazel-toolchains/releases/download/4.0.0/bazel-toolchains-4.0.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/releases/download/4.0.0/bazel-toolchains-4.0.0.tar.gz", + ], ) - -load("@bazel_toolchains//repositories:repositories.bzl", bazel_toolchains_repositories = "repositories") - -bazel_toolchains_repositories() diff --git a/bazel/GenTestRules.bzl b/bazel/GenTestRules.bzl index 73cbb4a84ba..fb9b6991de0 100644 --- a/bazel/GenTestRules.bzl +++ b/bazel/GenTestRules.bzl @@ -34,6 +34,7 @@ def GenTestRules( large_tests = [], enormous_tests = [], resources = [], + data = [], flaky_tests = [], tags = [], prefix = "", @@ -72,6 +73,7 @@ def GenTestRules( test_class = java_class, visibility = visibility, shard_count = shard_count, + data = data, ) def _get_test_names(test_files): diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index cdcc54cd927..47b4aac34a4 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -463,7 +463,9 @@ public static String getLocalhostByNetworkInterface() throws SocketException { if (!candidatesHost.isEmpty()) { return candidatesHost.get(0); } - return null; + + // Fallback to loopback + return localhost(); } public static boolean compareAndIncreaseOnly(final AtomicLong target, final long value) { diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/ConcurrentHashMapUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/ConcurrentHashMapUtils.java index 6fd9c21c992..3ab1cecebc4 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/ConcurrentHashMapUtils.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/ConcurrentHashMapUtils.java @@ -37,6 +37,8 @@ public abstract class ConcurrentHashMapUtils { /** * A temporary workaround for Java 8 specific performance issue JDK-8161372 .
Use implementation of * ConcurrentMap.computeIfAbsent instead. + * + * Requirement: The mapping function should not modify this map during computation. * * @see https://bugs.openjdk.java.net/browse/JDK-8161372 */ @@ -45,8 +47,6 @@ public static V computeIfAbsent(ConcurrentMap map, K key, Function< if (isJdk8) { V v = map.get(key); if (null == v) { -// v = map.computeIfAbsent(key, func); - // this bug fix methods maybe cause `func.apply` multiple calls. v = func.apply(key); if (null == v) { diff --git a/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java b/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java index efb42085f95..5b358ca8e61 100644 --- a/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/MixAllTest.java @@ -74,12 +74,6 @@ public void testString2File() throws IOException { assertThat(MixAll.file2String(fileName)).isEqualTo("MixAll_testString2File"); } - @Test - public void testGetLocalhostByNetworkInterface() throws Exception { - assertThat(MixAll.LOCALHOST).isNotNull(); - assertThat(MixAll.getLocalhostByNetworkInterface()).isNotNull(); - } - @Test public void testIsLmq() { String testLmq = null; diff --git a/common/src/test/java/org/apache/rocketmq/common/NetworkUtilTest.java b/common/src/test/java/org/apache/rocketmq/common/NetworkUtilTest.java index aa4d355f834..a1b225323fd 100644 --- a/common/src/test/java/org/apache/rocketmq/common/NetworkUtilTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/NetworkUtilTest.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.common; -import java.net.InetAddress; import org.apache.rocketmq.common.utils.NetworkUtil; import org.junit.Test; @@ -28,7 +27,6 @@ public void testGetLocalAddress() { String localAddress = NetworkUtil.getLocalAddress(); assertThat(localAddress).isNotNull(); assertThat(localAddress.length()).isGreaterThan(0); - assertThat(localAddress).isNotEqualTo(InetAddress.getLoopbackAddress().getHostAddress()); } @Test diff --git a/common/src/test/java/org/apache/rocketmq/common/utils/ConcurrentHashMapUtilsTest.java b/common/src/test/java/org/apache/rocketmq/common/utils/ConcurrentHashMapUtilsTest.java index fa97ddb1cc3..778c6f25da0 100644 --- a/common/src/test/java/org/apache/rocketmq/common/utils/ConcurrentHashMapUtilsTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/utils/ConcurrentHashMapUtilsTest.java @@ -26,7 +26,6 @@ public class ConcurrentHashMapUtilsTest { @Test public void computeIfAbsent() { - ConcurrentHashMap map = new ConcurrentHashMap<>(); map.put("123", "1111"); String value = ConcurrentHashMapUtils.computeIfAbsent(map, "123", k -> "234"); @@ -35,7 +34,5 @@ public void computeIfAbsent() { assertEquals("2342", value1); String value2 = ConcurrentHashMapUtils.computeIfAbsent(map, "123", k -> "2342"); assertEquals("1111", value2); -// map.computeIfAbsent("AaAa", key->map.computeIfAbsent("BBBB",key2->"42")); - ConcurrentHashMapUtils.computeIfAbsent(map, "AaAa", key -> map.computeIfAbsent("BBBB", key2 -> "42")); } } \ No newline at end of file diff --git a/controller/BUILD.bazel b/controller/BUILD.bazel index 3d7ef0e272d..73c2cf3395c 100644 --- a/controller/BUILD.bazel +++ b/controller/BUILD.bazel @@ -53,6 +53,7 @@ java_library( "@maven//:org_slf4j_jul_to_slf4j", "@maven//:com_alipay_sofa_jraft_core", "@maven//:com_alipay_sofa_hessian", + "@maven//:commons_io_commons_io", ], ) diff --git a/proxy/BUILD.bazel b/proxy/BUILD.bazel index cb7af925499..eb069528ea2 100644 --- a/proxy/BUILD.bazel +++ b/proxy/BUILD.bazel @@ -107,6 +107,8 @@ java_library( "@maven//:org_springframework_spring_core", "@maven//:org_jetbrains_annotations", "@maven//:org_slf4j_jul_to_slf4j", + "@maven//:io_netty_netty_tcnative_boringssl_static", + "@maven//:commons_codec_commons_codec", ], ) diff --git a/store/BUILD.bazel b/store/BUILD.bazel index b884503b08c..8364a239c9a 100644 --- a/store/BUILD.bazel +++ b/store/BUILD.bazel @@ -71,6 +71,7 @@ GenTestRules( exclude_tests = [ # These tests are extremely slow and flaky, exclude them before they are properly fixed. "src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest", + "src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest", ], medium_tests = [ "src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest", diff --git a/test/BUILD.bazel b/test/BUILD.bazel index 5df71200c89..e6703d69a01 100644 --- a/test/BUILD.bazel +++ b/test/BUILD.bazel @@ -116,9 +116,6 @@ GenTestRules( "src/test/java/org/apache/rocketmq/test/client/producer/exception/msg/MessageUserPropIT", "src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendIT", "src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendWithMQIT", - "src/test/java/org/apache/rocketmq/test/tls/TlsMixIT", - "src/test/java/org/apache/rocketmq/test/tls/TlsMix2IT", - "src/test/java/org/apache/rocketmq/test/tls/TlsIT", "src/test/java/org/apache/rocketmq/test/offset/OffsetNotFoundIT", "src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdIT", "src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendWithSelectorIT", @@ -127,6 +124,8 @@ GenTestRules( "src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopOrderlyIT", "src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT", "src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdExceptionIT", + "src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT", + "src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT", ], flaky_tests = [ "src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByKeyIT",