[improve][misc] Sync commits from apache into 3.1_ds

.gitignore

@@ -99,3 +99,5 @@ test-reports/
 .mvn/.gradle-enterprise/
 # Gradle Develocity
 .mvn/.develocity/
+.vscode
+effective-pom.xml

.mvn/wrapper/maven-wrapper.properties

@@ -5,14 +5,15 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.5/apache-maven-3.8.5-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar
+wrapperVersion=3.3.2
+distributionType=only-script
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip

distribution/server/src/assemble/LICENSE.bin.txt

@@ -382,25 +382,25 @@ The Apache Software License, Version 2.0
     - org.asynchttpclient-async-http-client-2.12.1.jar
     - org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar
  * Jetty
-    - org.eclipse.jetty-jetty-client-9.4.54.v20240208.jar
-    - org.eclipse.jetty-jetty-continuation-9.4.54.v20240208.jar
-    - org.eclipse.jetty-jetty-http-9.4.54.v20240208.jar
-    - org.eclipse.jetty-jetty-io-9.4.54.v20240208.jar
-    - org.eclipse.jetty-jetty-proxy-9.4.54.v20240208.jar
-    - org.eclipse.jetty-jetty-security-9.4.54.v20240208.jar
-    - org.eclipse.jetty-jetty-server-9.4.54.v20240208.jar
-    - org.eclipse.jetty-jetty-servlet-9.4.54.v20240208.jar
-    - org.eclipse.jetty-jetty-servlets-9.4.54.v20240208.jar
-    - org.eclipse.jetty-jetty-util-9.4.54.v20240208.jar
-    - org.eclipse.jetty-jetty-util-ajax-9.4.54.v20240208.jar
-    - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.54.v20240208.jar
-    - org.eclipse.jetty.websocket-websocket-api-9.4.54.v20240208.jar
-    - org.eclipse.jetty.websocket-websocket-client-9.4.54.v20240208.jar
-    - org.eclipse.jetty.websocket-websocket-common-9.4.54.v20240208.jar
-    - org.eclipse.jetty.websocket-websocket-server-9.4.54.v20240208.jar
-    - org.eclipse.jetty.websocket-websocket-servlet-9.4.54.v20240208.jar
-    - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.54.v20240208.jar
-    - org.eclipse.jetty-jetty-alpn-server-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-client-9.4.56.v20240826.jar
+    - org.eclipse.jetty-jetty-continuation-9.4.56.v20240826.jar
+    - org.eclipse.jetty-jetty-http-9.4.56.v20240826.jar
+    - org.eclipse.jetty-jetty-io-9.4.56.v20240826.jar
+    - org.eclipse.jetty-jetty-proxy-9.4.56.v20240826.jar
+    - org.eclipse.jetty-jetty-security-9.4.56.v20240826.jar
+    - org.eclipse.jetty-jetty-server-9.4.56.v20240826.jar
+    - org.eclipse.jetty-jetty-servlet-9.4.56.v20240826.jar
+    - org.eclipse.jetty-jetty-servlets-9.4.56.v20240826.jar
+    - org.eclipse.jetty-jetty-util-9.4.56.v20240826.jar
+    - org.eclipse.jetty-jetty-util-ajax-9.4.56.v20240826.jar
+    - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.56.v20240826.jar
+    - org.eclipse.jetty.websocket-websocket-api-9.4.56.v20240826.jar
+    - org.eclipse.jetty.websocket-websocket-client-9.4.56.v20240826.jar
+    - org.eclipse.jetty.websocket-websocket-common-9.4.56.v20240826.jar
+    - org.eclipse.jetty.websocket-websocket-server-9.4.56.v20240826.jar
+    - org.eclipse.jetty.websocket-websocket-servlet-9.4.56.v20240826.jar
+    - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.56.v20240826.jar
+    - org.eclipse.jetty-jetty-alpn-server-9.4.56.v20240826.jar
  * SnakeYaml -- org.yaml-snakeyaml-2.0.jar
  * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar
  * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar

distribution/shell/src/assemble/LICENSE.bin.txt

@@ -395,14 +395,14 @@ The Apache Software License, Version 2.0
     - async-http-client-2.12.1.jar
     - async-http-client-netty-utils-2.12.1.jar
  * Jetty
-    - jetty-client-9.4.54.v20240208.jar
-    - jetty-http-9.4.54.v20240208.jar
-    - jetty-io-9.4.54.v20240208.jar
-    - jetty-util-9.4.54.v20240208.jar
-    - javax-websocket-client-impl-9.4.54.v20240208.jar
-    - websocket-api-9.4.54.v20240208.jar
-    - websocket-client-9.4.54.v20240208.jar
-    - websocket-common-9.4.54.v20240208.jar
+    - jetty-client-9.4.56.v20240826.jar
+    - jetty-http-9.4.56.v20240826.jar
+    - jetty-io-9.4.56.v20240826.jar
+    - jetty-util-9.4.56.v20240826.jar
+    - javax-websocket-client-impl-9.4.56.v20240826.jar
+    - websocket-api-9.4.56.v20240826.jar
+    - websocket-client-9.4.56.v20240826.jar
+    - websocket-common-9.4.56.v20240826.jar
  * SnakeYaml -- snakeyaml-2.0.jar
  * Google Error Prone Annotations - error_prone_annotations-2.5.1.jar
  * Javassist -- javassist-3.25.0-GA.jar

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java

@@ -400,6 +400,7 @@ public void initializeFailed(ManagedLedgerException e) {
 
                     // Clean the map if initialization fails
                     ledgers.remove(name, future);
+                                entryCacheManager.removeEntryCache(name);
 
                     if (pendingInitializeLedgers.remove(name, pendingLedger)) {
                         pendingLedger.ledger.asyncClose(new CloseCallback() {

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java

@@ -126,7 +126,7 @@ public static OffloaderDefinition getOffloaderDefinition(String narPath, String
 
     public static Offloaders searchForOffloaders(String offloadersPath, String narExtractionDirectory)
             throws IOException {
-        Path path = Paths.get(offloadersPath).toAbsolutePath();
+        Path path = Paths.get(offloadersPath).toAbsolutePath().normalize();
         log.info("Searching for offloaders in {}", path);
 
         Offloaders offloaders = new Offloaders();

managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java

@@ -31,12 +31,14 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import lombok.Cleanup;
 import org.apache.bookkeeper.client.BKException;
 import org.apache.bookkeeper.client.BookKeeper;
 import org.apache.bookkeeper.client.LedgerHandle;
 import org.apache.bookkeeper.client.api.DigestType;
+import org.apache.bookkeeper.mledger.AsyncCallbacks;
 import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback;
 import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback;
 import org.apache.bookkeeper.mledger.Entry;
@@ -509,6 +511,35 @@ public void recoverAfterWriteError() throws Exception {
         entries.forEach(Entry::release);
     }
 
+    @Test
+    public void recoverAfterOpenManagedLedgerFail() throws Exception {
+        ManagedLedger ledger = factory.open("recoverAfterOpenManagedLedgerFail");
+        Position position = ledger.addEntry("entry".getBytes());
+        ledger.close();
+        bkc.failAfter(0, BKException.Code.BookieHandleNotAvailableException);
+        try {
+            factory.open("recoverAfterOpenManagedLedgerFail");
+        } catch (Exception e) {
+            // ok
+        }
+
+        ledger = factory.open("recoverAfterOpenManagedLedgerFail");
+        CompletableFuture<byte[]> future = new CompletableFuture<>();
+        ((ManagedLedgerImpl)ledger).asyncReadEntry((PositionImpl)position, new AsyncCallbacks.ReadEntryCallback() {
+            @Override
+            public void readEntryComplete(Entry entry, Object ctx) {
+                future.complete(entry.getData());
+            }
+
+            @Override
+            public void readEntryFailed(ManagedLedgerException exception, Object ctx) {
+                future.completeExceptionally(exception);
+            }
+        }, null);
+        byte[] bytes = future.get(30, TimeUnit.SECONDS);
+        assertEquals(new String(bytes), "entry");
+    }
+
     @Test
     public void recoverLongTimeAfterMultipleWriteErrors() throws Exception {
         ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("recoverLongTimeAfterMultipleWriteErrors");

managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java

@@ -4358,9 +4358,10 @@ public void testDeleteCurrentLedgerWhenItIsClosed(boolean closeLedgerByAddEntry)
             // Detect the current ledger is full by the timed task. (Imitate: the timed task `checkLedgerRollTask` call
             // `rollCurrentLedgerIfFull` periodically).
             ml.rollCurrentLedgerIfFull();
-            // the ledger closing in the `rollCurrentLedgerIfFull` is async, so the wait is needed.
-            Awaitility.await().untilAsserted(() -> assertEquals(ml.ledgers.size(), 2));
         }
+        // wait the new ledger create
+        Awaitility.await().untilAsserted(() -> assertEquals(ml.ledgers.size(), 2));
+
         // Act: Trigger trimming to delete the previous current ledger.
         ml.internalTrimLedgers(false, Futures.NULL_PROMISE);
         // Verify: A new ledger will be opened after the current ledger is closed and the previous current ledger can be