From 367bca9d6dd03b5914f5275c36f4bbeebc9898f6 Mon Sep 17 00:00:00 2001 From: Michael Koepf <47541996+michaelkoepf@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:54:02 +0100 Subject: [PATCH 1/4] [Hotfix] Handle cases where schema differs from plugin name in lakehouse compaction service Issue #178 --- fluss-dist/src/main/fluss-bin/bin/lakehouse.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/fluss-dist/src/main/fluss-bin/bin/lakehouse.sh b/fluss-dist/src/main/fluss-bin/bin/lakehouse.sh index 437fac2d..2fbc0be5 100755 --- a/fluss-dist/src/main/fluss-bin/bin/lakehouse.sh +++ b/fluss-dist/src/main/fluss-bin/bin/lakehouse.sh @@ -41,11 +41,22 @@ get_schema() { fi } +# handles cases where the schema differs from the plugin name +get_plugin_name_from_schema() { + schema="$1" + if [ "$schema" == "hdfs" ]; then + echo "hadoop" + else + echo "$schema" + fi +} + addition_jars="" # for fluss client to access remote filesystem, we need to prepare filesystem plugins fluss_filesystem_scheme=$(get_schema "$REMOTE_DATA_DIR") -fluss_plugin_jars=$(constructPluginJars ${fluss_filesystem_scheme}) +fluss_plugin_name=$(get_plugin_name_from_schema ${fluss_filesystem_scheme}) +fluss_plugin_jars=$(constructPluginJars ${fluss_plugin_name}) if [ ! -z "$fluss_plugin_jars" ]; then addition_jars="${fluss_plugin_jars}" From b255ada83485eb6e22a0bc0be1ac8654dcafb2f9 Mon Sep 17 00:00:00 2001 From: Michael Koepf <47541996+michaelkoepf@users.noreply.github.com> Date: Tue, 17 Dec 2024 01:08:16 +0100 Subject: [PATCH 2/4] Revert "[Hotfix] Handle cases where schema differs from plugin name in lakehouse compaction service" This reverts commit 367bca9d6dd03b5914f5275c36f4bbeebc9898f6. --- fluss-dist/src/main/fluss-bin/bin/lakehouse.sh | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/fluss-dist/src/main/fluss-bin/bin/lakehouse.sh b/fluss-dist/src/main/fluss-bin/bin/lakehouse.sh index 2fbc0be5..437fac2d 100755 --- a/fluss-dist/src/main/fluss-bin/bin/lakehouse.sh +++ b/fluss-dist/src/main/fluss-bin/bin/lakehouse.sh @@ -41,22 +41,11 @@ get_schema() { fi } -# handles cases where the schema differs from the plugin name -get_plugin_name_from_schema() { - schema="$1" - if [ "$schema" == "hdfs" ]; then - echo "hadoop" - else - echo "$schema" - fi -} - addition_jars="" # for fluss client to access remote filesystem, we need to prepare filesystem plugins fluss_filesystem_scheme=$(get_schema "$REMOTE_DATA_DIR") -fluss_plugin_name=$(get_plugin_name_from_schema ${fluss_filesystem_scheme}) -fluss_plugin_jars=$(constructPluginJars ${fluss_plugin_name}) +fluss_plugin_jars=$(constructPluginJars ${fluss_filesystem_scheme}) if [ ! -z "$fluss_plugin_jars" ]; then addition_jars="${fluss_plugin_jars}" From a415964c5bf1c47418fe0c4f01087dff91fa48b5 Mon Sep 17 00:00:00 2001 From: Michael Koepf <47541996+michaelkoepf@users.noreply.github.com> Date: Tue, 17 Dec 2024 01:50:16 +0100 Subject: [PATCH 3/4] [dist][hotfix] Fix name of Hadoop filesystem plugin Name Hadoop filesystem plugin JAR according to its schema ("hdfs") Issue #178 --- fluss-dist/src/main/assemblies/plugins.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fluss-dist/src/main/assemblies/plugins.xml b/fluss-dist/src/main/assemblies/plugins.xml index 849f46ad..6e1a5fc1 100644 --- a/fluss-dist/src/main/assemblies/plugins.xml +++ b/fluss-dist/src/main/assemblies/plugins.xml @@ -27,10 +27,11 @@ + ../fluss-filesystems/fluss-fs-hadoop/target/fluss-fs-hadoop-${project.version}-with-dependencies.jar - plugins/hadoop/ - fluss-fs-hadoop-${project.version}.jar + plugins/hdfs/ + fluss-fs-hdfs-${project.version}.jar 0644 From afbde4415b5add8458921df851dbceb901afea6c Mon Sep 17 00:00:00 2001 From: Michael Koepf <47541996+michaelkoepf@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:44:08 +0100 Subject: [PATCH 4/4] [dist][hotfix] Fix name of Hadoop filesystem plugin Do not rename JAR file Issue #178 --- fluss-dist/src/main/assemblies/plugins.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fluss-dist/src/main/assemblies/plugins.xml b/fluss-dist/src/main/assemblies/plugins.xml index 6e1a5fc1..ddfd8f8e 100644 --- a/fluss-dist/src/main/assemblies/plugins.xml +++ b/fluss-dist/src/main/assemblies/plugins.xml @@ -27,11 +27,11 @@ - + ../fluss-filesystems/fluss-fs-hadoop/target/fluss-fs-hadoop-${project.version}-with-dependencies.jar plugins/hdfs/ - fluss-fs-hdfs-${project.version}.jar + fluss-fs-hadoop-${project.version}.jar 0644