[improve][build, test] Sync commits from apache/pulsar branch-3.1 into 3.1_ds

conf/functions_worker.yml

@@ -43,6 +43,16 @@ metadataStoreOperationTimeoutSeconds: 30
 # Metadata store cache expiry time in seconds
 metadataStoreCacheExpirySeconds: 300
 
+# Specifies if the function worker should use classloading for validating submissions for built-in
+# connectors and functions. This is required for validateConnectorConfig to take effect.
+# Default is false.
+enableClassloadingOfBuiltinFiles: false
+
+# Specifies if the function worker should use classloading for validating submissions for external
+# connectors and functions. This is required for validateConnectorConfig to take effect.
+# Default is false.
+enableClassloadingOfExternalFiles: false
+
 ################################
 # Function package management
 ################################
@@ -398,9 +408,20 @@ saslJaasServerRoleTokenSignerSecretPath:
 ########################
 
 connectorsDirectory: ./connectors
+# Whether to enable referencing connectors directory files by file url in connector (sink/source) creation
+enableReferencingConnectorDirectoryFiles: true
+# Regex patterns for enabling creation of connectors by referencing packages in matching http/https urls
+additionalEnabledConnectorUrlPatterns: []
 functionsDirectory: ./functions
-
-# Should connector config be validated during submission
+# Whether to enable referencing functions directory files by file url in functions creation
+enableReferencingFunctionsDirectoryFiles: true
+# Regex patterns for enabling creation of functions by referencing packages in matching http/https urls
+additionalEnabledFunctionsUrlPatterns: []
+
+# Enables extended validation for connector config with fine-grain annotation based validation
+# during submission. Classloading with either enableClassloadingOfExternalFiles or
+# enableClassloadingOfBuiltinFiles must be enabled on the worker for this to take effect.
+# Default is false.
 validateConnectorConfig: false
 
 # Whether to initialize distributed log metadata by runtime.

conf/proxy.conf

@@ -28,17 +28,19 @@ metadataStoreUrl=
 # The metadata store URL for the configuration data. If empty, we fall back to use metadataStoreUrl
 configurationMetadataStoreUrl=
 
-# If Service Discovery is Disabled this url should point to the discovery service provider.
+# If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the discovery service
+# provider, and does not support multi urls yet.
 # The URL must begin with pulsar:// for plaintext or with pulsar+ssl:// for TLS.
 brokerServiceURL=
 brokerServiceURLTLS=
 
-# These settings are unnecessary if `zookeeperServers` is specified
+# If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the discovery service
+# provider, and does not support multi urls yet.
 brokerWebServiceURL=
 brokerWebServiceURLTLS=
 
-# If function workers are setup in a separate cluster, configure the following 2 settings
-# to point to the function workers cluster
+# If function workers are setup in a separate cluster, configure the following 2 settings. This url should point to
+# the discovery service provider of the function workers cluster, and does not support multi urls yet.
 functionWorkerWebServiceURL=
 functionWorkerWebServiceURLTLS=
 

distribution/server/src/assemble/LICENSE.bin.txt

@@ -285,7 +285,7 @@ The Apache Software License, Version 2.0
     - commons-lang-commons-lang-2.6.jar
     - commons-logging-commons-logging-1.1.1.jar
     - org.apache.commons-commons-collections4-4.4.jar
-    - org.apache.commons-commons-compress-1.21.jar
+    - org.apache.commons-commons-compress-1.26.0.jar
     - org.apache.commons-commons-lang3-3.11.jar
     - org.apache.commons-commons-text-1.10.0.jar
  * Netty
@@ -345,34 +345,34 @@ The Apache Software License, Version 2.0
     - net.java.dev.jna-jna-jpms-5.12.1.jar
     - net.java.dev.jna-jna-platform-jpms-5.12.1.jar
  * BookKeeper
-    - org.apache.bookkeeper-bookkeeper-common-4.16.3.jar
-    - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.3.jar
-    - org.apache.bookkeeper-bookkeeper-proto-4.16.3.jar
-    - org.apache.bookkeeper-bookkeeper-server-4.16.3.jar
-    - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.3.jar
-    - org.apache.bookkeeper-circe-checksum-4.16.3.jar
-    - org.apache.bookkeeper-cpu-affinity-4.16.3.jar
-    - org.apache.bookkeeper-statelib-4.16.3.jar
-    - org.apache.bookkeeper-stream-storage-api-4.16.3.jar
-    - org.apache.bookkeeper-stream-storage-common-4.16.3.jar
-    - org.apache.bookkeeper-stream-storage-java-client-4.16.3.jar
-    - org.apache.bookkeeper-stream-storage-java-client-base-4.16.3.jar
-    - org.apache.bookkeeper-stream-storage-proto-4.16.3.jar
-    - org.apache.bookkeeper-stream-storage-server-4.16.3.jar
-    - org.apache.bookkeeper-stream-storage-service-api-4.16.3.jar
-    - org.apache.bookkeeper-stream-storage-service-impl-4.16.3.jar
-    - org.apache.bookkeeper.http-http-server-4.16.3.jar
-    - org.apache.bookkeeper.http-vertx-http-server-4.16.3.jar
-    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.3.jar
-    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.3.jar
-    - org.apache.distributedlog-distributedlog-common-4.16.3.jar
-    - org.apache.distributedlog-distributedlog-core-4.16.3-tests.jar
-    - org.apache.distributedlog-distributedlog-core-4.16.3.jar
-    - org.apache.distributedlog-distributedlog-protocol-4.16.3.jar
-    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.3.jar
-    - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.3.jar
-    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.3.jar
-    - org.apache.bookkeeper-native-io-4.16.3.jar
+    - org.apache.bookkeeper-bookkeeper-common-4.16.4.jar
+    - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.4.jar
+    - org.apache.bookkeeper-bookkeeper-proto-4.16.4.jar
+    - org.apache.bookkeeper-bookkeeper-server-4.16.4.jar
+    - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.4.jar
+    - org.apache.bookkeeper-circe-checksum-4.16.4.jar
+    - org.apache.bookkeeper-cpu-affinity-4.16.4.jar
+    - org.apache.bookkeeper-statelib-4.16.4.jar
+    - org.apache.bookkeeper-stream-storage-api-4.16.4.jar
+    - org.apache.bookkeeper-stream-storage-common-4.16.4.jar
+    - org.apache.bookkeeper-stream-storage-java-client-4.16.4.jar
+    - org.apache.bookkeeper-stream-storage-java-client-base-4.16.4.jar
+    - org.apache.bookkeeper-stream-storage-proto-4.16.4.jar
+    - org.apache.bookkeeper-stream-storage-server-4.16.4.jar
+    - org.apache.bookkeeper-stream-storage-service-api-4.16.4.jar
+    - org.apache.bookkeeper-stream-storage-service-impl-4.16.4.jar
+    - org.apache.bookkeeper.http-http-server-4.16.4.jar
+    - org.apache.bookkeeper.http-vertx-http-server-4.16.4.jar
+    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.4.jar
+    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.4.jar
+    - org.apache.distributedlog-distributedlog-common-4.16.4.jar
+    - org.apache.distributedlog-distributedlog-core-4.16.4-tests.jar
+    - org.apache.distributedlog-distributedlog-core-4.16.4.jar
+    - org.apache.distributedlog-distributedlog-protocol-4.16.4.jar
+    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.4.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.4.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.4.jar
+    - org.apache.bookkeeper-native-io-4.16.4.jar
   * Apache HTTP Client
     - org.apache.httpcomponents-httpclient-4.5.13.jar
     - org.apache.httpcomponents-httpcore-4.4.15.jar
@@ -382,25 +382,25 @@ The Apache Software License, Version 2.0
     - org.asynchttpclient-async-http-client-2.12.1.jar
     - org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar
  * Jetty
-    - org.eclipse.jetty-jetty-client-9.4.53.v20231009.jar
-    - org.eclipse.jetty-jetty-continuation-9.4.53.v20231009.jar
-    - org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar
-    - org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar
-    - org.eclipse.jetty-jetty-proxy-9.4.53.v20231009.jar
-    - org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar
-    - org.eclipse.jetty-jetty-server-9.4.53.v20231009.jar
-    - org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar
-    - org.eclipse.jetty-jetty-servlets-9.4.53.v20231009.jar
-    - org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar
-    - org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar
-    - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.53.v20231009.jar
-    - org.eclipse.jetty.websocket-websocket-api-9.4.53.v20231009.jar
-    - org.eclipse.jetty.websocket-websocket-client-9.4.53.v20231009.jar
-    - org.eclipse.jetty.websocket-websocket-common-9.4.53.v20231009.jar
-    - org.eclipse.jetty.websocket-websocket-server-9.4.53.v20231009.jar
-    - org.eclipse.jetty.websocket-websocket-servlet-9.4.53.v20231009.jar
-    - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.53.v20231009.jar
-    - org.eclipse.jetty-jetty-alpn-server-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-client-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-continuation-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-http-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-io-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-proxy-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-security-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-server-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-servlet-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-servlets-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-util-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-util-ajax-9.4.54.v20240208.jar
+    - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.54.v20240208.jar
+    - org.eclipse.jetty.websocket-websocket-api-9.4.54.v20240208.jar
+    - org.eclipse.jetty.websocket-websocket-client-9.4.54.v20240208.jar
+    - org.eclipse.jetty.websocket-websocket-common-9.4.54.v20240208.jar
+    - org.eclipse.jetty.websocket-websocket-server-9.4.54.v20240208.jar
+    - org.eclipse.jetty.websocket-websocket-servlet-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.54.v20240208.jar
+    - org.eclipse.jetty-jetty-alpn-server-9.4.54.v20240208.jar
  * SnakeYaml -- org.yaml-snakeyaml-2.0.jar
  * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar
  * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar
@@ -445,6 +445,10 @@ The Apache Software License, Version 2.0
   * Jodah
     - net.jodah-typetools-0.5.0.jar
     - net.jodah-failsafe-2.4.4.jar
+  * Byte Buddy
+    - net.bytebuddy-byte-buddy-1.14.12.jar
+  * zt-zip
+    - org.zeroturnaround-zt-zip-1.17.jar
   * Apache Avro
     - org.apache.avro-avro-1.11.3.jar
     - org.apache.avro-avro-protobuf-1.11.3.jar
@@ -581,28 +585,28 @@ Bouncy Castle License
     - org.bouncycastle-bcutil-jdk18on-1.75.jar
 
 Datastax
-    - com.datastax.oss-managed-ledger-3.1.0.2.jar
-    - com.datastax.oss-pulsar-broker-3.1.0.2.jar
-    - com.datastax.oss-pulsar-broker-auth-oidc-3.1.0.2.jar
-    - com.datastax.oss-pulsar-broker-auth-sasl-3.1.0.2.jar
-    - com.datastax.oss-pulsar-broker-common-3.1.0.2.jar
-    - com.datastax.oss-pulsar-config-validation-3.1.0.2.jar
-    - com.datastax.oss-pulsar-docs-tools-3.1.0.2.jar
-    - com.datastax.oss-pulsar-functions-api-3.1.0.2.jar
-    - com.datastax.oss-pulsar-functions-instance-3.1.0.2.jar
-    - com.datastax.oss-pulsar-functions-local-runner-original-3.1.0.2.jar
-    - com.datastax.oss-pulsar-functions-proto-3.1.0.2.jar
-    - com.datastax.oss-pulsar-functions-runtime-3.1.0.2.jar
-    - com.datastax.oss-pulsar-functions-secrets-3.1.0.2.jar
-    - com.datastax.oss-pulsar-functions-utils-3.1.0.2.jar
-    - com.datastax.oss-pulsar-functions-worker-3.1.0.2.jar
-    - com.datastax.oss-pulsar-io-core-3.1.0.2.jar
-    - com.datastax.oss-pulsar-metadata-3.1.0.2.jar
-    - com.datastax.oss-pulsar-proxy-3.1.0.2.jar
-    - com.datastax.oss-pulsar-testclient-3.1.0.2.jar
-    - com.datastax.oss-pulsar-transaction-common-3.1.0.2.jar
-    - com.datastax.oss-pulsar-transaction-coordinator-3.1.0.2.jar
-    - com.datastax.oss-testmocks-3.1.0.2.jar
+    - com.datastax.oss-managed-ledger-3.1.3.0.jar
+    - com.datastax.oss-pulsar-broker-3.1.3.0.jar
+    - com.datastax.oss-pulsar-broker-auth-oidc-3.1.3.0.jar
+    - com.datastax.oss-pulsar-broker-auth-sasl-3.1.3.0.jar
+    - com.datastax.oss-pulsar-broker-common-3.1.3.0.jar
+    - com.datastax.oss-pulsar-config-validation-3.1.3.0.jar
+    - com.datastax.oss-pulsar-docs-tools-3.1.3.0.jar
+    - com.datastax.oss-pulsar-functions-api-3.1.3.0.jar
+    - com.datastax.oss-pulsar-functions-instance-3.1.3.0.jar
+    - com.datastax.oss-pulsar-functions-local-runner-original-3.1.3.0.jar
+    - com.datastax.oss-pulsar-functions-proto-3.1.3.0.jar
+    - com.datastax.oss-pulsar-functions-runtime-3.1.3.0.jar
+    - com.datastax.oss-pulsar-functions-secrets-3.1.3.0.jar
+    - com.datastax.oss-pulsar-functions-utils-3.1.3.0.jar
+    - com.datastax.oss-pulsar-functions-worker-3.1.3.0.jar
+    - com.datastax.oss-pulsar-io-core-3.1.3.0.jar
+    - com.datastax.oss-pulsar-metadata-3.1.3.0.jar
+    - com.datastax.oss-pulsar-proxy-3.1.3.0.jar
+    - com.datastax.oss-pulsar-testclient-3.1.3.0.jar
+    - com.datastax.oss-pulsar-transaction-common-3.1.3.0.jar
+    - com.datastax.oss-pulsar-transaction-coordinator-3.1.3.0.jar
+    - com.datastax.oss-testmocks-3.1.3.0.jar
 
 ------------------------
 

distribution/shell/src/assemble/LICENSE.bin.txt

@@ -342,7 +342,7 @@ The Apache Software License, Version 2.0
     - commons-logging-1.2.jar
     - commons-lang3-3.11.jar
     - commons-text-1.10.0.jar
-    - commons-compress-1.21.jar
+    - commons-compress-1.26.0.jar
  * Netty
     - netty-buffer-4.1.100.Final.jar
     - netty-codec-4.1.100.Final.jar
@@ -386,23 +386,23 @@ The Apache Software License, Version 2.0
     - log4j-web-2.18.0.jar
 
  * BookKeeper
-    - bookkeeper-common-allocator-4.16.3.jar
-    - cpu-affinity-4.16.3.jar
-    - circe-checksum-4.16.3.jar
+    - bookkeeper-common-allocator-4.16.4.jar
+    - cpu-affinity-4.16.4.jar
+    - circe-checksum-4.16.4.jar
   * AirCompressor
      - aircompressor-0.20.jar
  * AsyncHttpClient
     - async-http-client-2.12.1.jar
     - async-http-client-netty-utils-2.12.1.jar
  * Jetty
-    - jetty-client-9.4.53.v20231009.jar
-    - jetty-http-9.4.53.v20231009.jar
-    - jetty-io-9.4.53.v20231009.jar
-    - jetty-util-9.4.53.v20231009.jar
-    - javax-websocket-client-impl-9.4.53.v20231009.jar
-    - websocket-api-9.4.53.v20231009.jar
-    - websocket-client-9.4.53.v20231009.jar
-    - websocket-common-9.4.53.v20231009.jar
+    - jetty-client-9.4.54.v20240208.jar
+    - jetty-http-9.4.54.v20240208.jar
+    - jetty-io-9.4.54.v20240208.jar
+    - jetty-util-9.4.54.v20240208.jar
+    - javax-websocket-client-impl-9.4.54.v20240208.jar
+    - websocket-api-9.4.54.v20240208.jar
+    - websocket-client-9.4.54.v20240208.jar
+    - websocket-common-9.4.54.v20240208.jar
  * SnakeYaml -- snakeyaml-2.0.jar
  * Google Error Prone Annotations - error_prone_annotations-2.5.1.jar
  * Javassist -- javassist-3.25.0-GA.jar

docker/pulsar/Dockerfile

@@ -36,10 +36,14 @@ COPY scripts/install-pulsar-client.sh /pulsar/bin
 
 # The final image needs to give the root group sufficient permission for Pulsar components
 # to write to specific directories within /pulsar
+# The ownership is changed to uid 10000 to allow using a different root group. This is necessary when running the
+# container when gid=0 is prohibited. In that case, the container must be run with uid 10000 with
+# any group id != 0 (for example 10001).
 # The file permissions are preserved when copying files from this builder image to the target image.
 RUN for SUBDIRECTORY in conf data download logs; do \
      [ -d /pulsar/$SUBDIRECTORY ] || mkdir /pulsar/$SUBDIRECTORY; \
-     chmod -R g+w /pulsar/$SUBDIRECTORY; \
+     chmod -R ug+w /pulsar/$SUBDIRECTORY; \
+     chown -R 10000:0 /pulsar/$SUBDIRECTORY; \
      done
 
 # Trino writes logs to this directory (at least during tests), so we need to give the process permission
@@ -54,6 +58,7 @@ FROM ubuntu:22.04
 ARG DEBIAN_FRONTEND=noninteractive
 ARG UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt
 ARG UBUNTU_SECURITY_MIRROR=http://security.ubuntu.com/ubuntu/
+ARG DEFAULT_USERNAME=pulsar
 
 # Install some utilities
 RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-mirror://mirrors.ubuntu.com/mirrors.txt}|g" \
@@ -104,4 +109,5 @@ RUN chmod +x /pulsar/bin/install-pulsar-client.sh
 RUN /pulsar/bin/install-pulsar-client.sh
 
 # The UID must be non-zero. Otherwise, it is arbitrary. No logic should rely on its specific value.
+RUN useradd ${DEFAULT_USERNAME} -u 10000 -g 0
 USER 10000