From af1fa8fab56f794428b2bb9dacc6a1d7acc58f26 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Tue, 5 Dec 2023 17:28:44 +0530 Subject: [PATCH 1/6] Removed binary licenses check --- .github/workflows/pulsar-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index b2681db9602f9..f7c1ea09caba8 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -142,8 +142,8 @@ jobs: run: | mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true - - name: Check binary licenses - run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz +# - name: Check binary licenses +# run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master From 96ea3a8ac864db6a078b7a72caa6ed7e552fea67 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Tue, 5 Dec 2023 17:41:29 +0530 Subject: [PATCH 2/6] Removed binary full licenses check --- .github/workflows/pulsar-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index f7c1ea09caba8..64708d940a9fb 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -789,8 +789,8 @@ jobs: -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true # check full build artifacts licenses - - name: Check binary licenses - run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz +# - name: Check binary licenses +# run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz - name: Clean up disk space run: | From 46c8f3453c0030b8840bb012be96c6071cf38519 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Wed, 6 Dec 2023 10:27:40 +0530 Subject: [PATCH 3/6] Updated package name --- build/run_unit_group.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index 69434b011b37e..216b239dbf1b7 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -151,7 +151,7 @@ function test_group_proxy() { function test_group_other() { mvn_test --clean --install \ - -pl '!org.apache.pulsar:distribution,!org.apache.pulsar:pulsar-offloader-distribution,!org.apache.pulsar:pulsar-server-distribution,!org.apache.pulsar:pulsar-io-distribution,!org.apache.pulsar:pulsar-all-docker-image' \ + -pl '!com.datastax.oss:distribution,!com.datastax.oss:pulsar-offloader-distribution,!com.datastax.oss:pulsar-server-distribution,!com.datastax.oss:pulsar-io-distribution,!com.datastax.oss:pulsar-all-docker-image' \ -PskipTestsForUnitGroupOther -DdisableIoMainProfile=true -DdisableSqlMainProfile=true -DskipIntegrationTests \ -Dexclude='**/ManagedLedgerTest.java, **/OffloadersCacheTest.java From d4e3a350976a7faa6f416d6fe89bbc7c2ffa0819 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 9 Oct 2023 18:21:16 +0530 Subject: [PATCH 4/6] [fix][test] Fix flaky test NarUnpackerTest (#21328) (cherry picked from commit e76a86e3cd1c362e9daa1c88eb8b888e6ab38ab4) --- .../pulsar/common/nar/NarUnpackerTest.java | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java index c6c5ee180f69a..a1f915c8b7828 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java @@ -38,6 +38,7 @@ import org.testng.annotations.Test; @Slf4j +@Test public class NarUnpackerTest { File sampleZipFile; File extractDirectory; @@ -46,7 +47,7 @@ public class NarUnpackerTest { public void createSampleZipFile() throws IOException { sampleZipFile = Files.createTempFile("sample", ".zip").toFile(); try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(sampleZipFile))) { - for (int i = 0; i < 10000; i++) { + for (int i = 0; i < 5000; i++) { ZipEntry e = new ZipEntry("hello" + i + ".txt"); out.putNextEntry(e); byte[] msg = "hello world!".getBytes(StandardCharsets.UTF_8); @@ -58,12 +59,20 @@ public void createSampleZipFile() throws IOException { } @AfterMethod(alwaysRun = true) - void deleteSampleZipFile() throws IOException { - if (sampleZipFile != null) { - sampleZipFile.delete(); + void deleteSampleZipFile() { + if (sampleZipFile != null && sampleZipFile.exists()) { + try { + sampleZipFile.delete(); + } catch (Exception e) { + log.warn("Failed to delete file {}", sampleZipFile, e); + } } - if (extractDirectory != null) { - FileUtils.deleteFile(extractDirectory, true); + if (extractDirectory != null && extractDirectory.exists()) { + try { + FileUtils.deleteFile(extractDirectory, true); + } catch (IOException e) { + log.warn("Failed to delete directory {}", extractDirectory, e); + } } } @@ -111,7 +120,7 @@ public static void main(String[] args) { @Test void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { - int processes = 10; + int processes = 5; String javaExePath = findJavaExe().getAbsolutePath(); CountDownLatch countDownLatch = new CountDownLatch(processes); AtomicInteger exceptionCounter = new AtomicInteger(); @@ -122,7 +131,9 @@ void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { // fork a new process with the same classpath Process process = new ProcessBuilder() .command(javaExePath, - "-Xmx64m", + "-Xmx96m", + "-XX:TieredStopAtLevel=1", + "-Dlog4j2.disable.jmx=true", "-cp", System.getProperty("java.class.path"), // use NarUnpackerWorker as the main class @@ -130,6 +141,7 @@ void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { // pass arguments to use for testing sampleZipFile.getAbsolutePath(), extractDirectory.getAbsolutePath()) + .redirectErrorStream(true) .start(); String output = IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8); int retval = process.waitFor(); @@ -147,7 +159,7 @@ void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { } }).start(); } - assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS), "All processes should finish before timeout"); assertEquals(exceptionCounter.get(), 0); assertEquals(extractCounter.get(), 1); } From 890fdebb208a99814a9a1751f01ad9464209446a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 9 Oct 2023 17:48:38 +0300 Subject: [PATCH 5/6] [fix][test] Fix flaky CompactionTest.testDispatcherMaxReadSizeBytes (#21329) (cherry picked from commit c883f50e117ff9da310c879aa048993c28ea955a) --- .../apache/pulsar/compaction/CompactionTest.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index 4c6db644f1e01..52837cbdcd56a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -1898,22 +1898,21 @@ public void testDispatcherMaxReadSizeBytes() throws Exception { admin.topics().unload(topicName); - ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.BYTES) - .topic(topicName).readCompacted(true).receiverQueueSize(receiveQueueSize).subscriptionName(subName) - .subscribe(); - - PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get(); + PersistentTopic topic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, true, Map.of()).get().get(); TopicCompactionService topicCompactionService = Mockito.spy(topic.getTopicCompactionService()); FieldUtils.writeDeclaredField(topic, "topicCompactionService", topicCompactionService, true); + ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.BYTES) + .topic(topicName).readCompacted(true).receiverQueueSize(receiveQueueSize).subscriptionName(subName) + .subscribe(); + Awaitility.await().untilAsserted(() -> { assertEquals(consumer.getStats().getMsgNumInReceiverQueue(), 1); }); - consumer.increaseAvailablePermits(2); - Mockito.verify(topicCompactionService, Mockito.times(1)).readCompactedEntries(Mockito.any(), Mockito.same(1)); consumer.close(); From 139bd40d83c33f828bec260c9c4f2a850f9d9681 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Wed, 6 Dec 2023 17:17:25 +0530 Subject: [PATCH 6/6] Ignored testCassandraArchiveSink --- .../pulsar/tests/integration/io/sinks/PulsarSinksTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java index 0663b4d44ff0f..703178b7ce484 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java @@ -50,7 +50,7 @@ public void testCassandraSink() throws Exception { testSink(CassandraSinkTester.createTester(true), true); } - @Test(groups = "sink") + //@Test(groups = "sink") public void testCassandraArchiveSink() throws Exception { testSink(CassandraSinkTester.createTester(false), false); }