Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fix](kerberos) pass kerberos_krb5_conf_path to be jvm option #47431

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
+377 −9
Open

[fix](kerberos) pass kerberos_krb5_conf_path to be jvm option #47431

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d8a481b
push kerb5ConfPath when createVM in be
suxiaogang223 Jan 24, 2025
9a1b527
add regression for spark hudi jni scanner
suxiaogang223 Jan 24, 2025
bb877c3
flush out
suxiaogang223 Jan 25, 2025
94be24d
flush test_hudi_incremental
suxiaogang223 Jan 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions be/src/util/jni-util.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,10 @@ const std::string GetDorisJNIClasspathOption() {
}
}

const std::string GetKerb5ConfPath() {
return "-Djava.security.krb5.conf=" + config::kerberos_krb5_conf_path;
}

[[maybe_unused]] void SetEnvIfNecessary() {
const auto* doris_home = getenv("DORIS_HOME");
DCHECK(doris_home) << "Environment variable DORIS_HOME is not set.";
Expand Down Expand Up @@ -136,6 +140,7 @@ const std::string GetDorisJNIClasspathOption() {
std::istringstream stream(java_opts);
options = std::vector<std::string>(std::istream_iterator<std::string> {stream},
std::istream_iterator<std::string>());
options.push_back(GetKerb5ConfPath());
options.push_back(GetDorisJNIClasspathOption());
}
std::unique_ptr<JavaVMOption[]> jvm_options(new JavaVMOption[options.size()]);
Expand Down
174 changes: 174 additions & 0 deletions regression-test/data/external_table_p2/hudi/test_hudi_incremental.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -521,3 +521,177 @@
-- !incremental_9_10 --
1000

-- !incremental_1_end --
9000

-- !incremental_earliest_1 --
1000

-- !incremental_2_end --
8000

-- !incremental_earliest_2 --
2000

-- !incremental_1_2 --
1000

-- !incremental_3_end --
7000

-- !incremental_earliest_3 --
3000

-- !incremental_2_3 --
1000

-- !incremental_4_end --
6000

-- !incremental_earliest_4 --
4000

-- !incremental_3_4 --
1000

-- !incremental_5_end --
5000

-- !incremental_earliest_5 --
5000

-- !incremental_4_5 --
1000

-- !incremental_6_end --
4000

-- !incremental_earliest_6 --
6000

-- !incremental_5_6 --
1000

-- !incremental_7_end --
3000

-- !incremental_earliest_7 --
7000

-- !incremental_6_7 --
1000

-- !incremental_8_end --
2000

-- !incremental_earliest_8 --
8000

-- !incremental_7_8 --
1000

-- !incremental_9_end --
1000

-- !incremental_earliest_9 --
9000

-- !incremental_8_9 --
1000

-- !incremental_10_end --
0

-- !incremental_earliest_10 --
10000

-- !incremental_9_10 --
1000

-- !incremental_1_end --
9000

-- !incremental_earliest_1 --
1000

-- !incremental_2_end --
8000

-- !incremental_earliest_2 --
2000

-- !incremental_1_2 --
1000

-- !incremental_3_end --
7000

-- !incremental_earliest_3 --
3000

-- !incremental_2_3 --
1000

-- !incremental_4_end --
6000

-- !incremental_earliest_4 --
4000

-- !incremental_3_4 --
1000

-- !incremental_5_end --
5000

-- !incremental_earliest_5 --
5000

-- !incremental_4_5 --
1000

-- !incremental_6_end --
4000

-- !incremental_earliest_6 --
6000

-- !incremental_5_6 --
1000

-- !incremental_7_end --
3000

-- !incremental_earliest_7 --
7000

-- !incremental_6_7 --
1000

-- !incremental_8_end --
2000

-- !incremental_earliest_8 --
8000

-- !incremental_7_8 --
1000

-- !incremental_9_end --
1000

-- !incremental_earliest_9 --
9000

-- !incremental_8_9 --
1000

-- !incremental_10_end --
0

-- !incremental_earliest_10 --
10000

-- !incremental_9_10 --
1000

32 changes: 32 additions & 0 deletions regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,3 +63,35 @@
20241118012149007 20241118012149007_0_4 5 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 5 Eva {"age":31.5, "address":"Chengdu"}
20241118012149007 20241118012149007_0_5 6 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 6 Frank {"age":29.2, "address":"Wuhan"}

-- !adding_simple_columns_table --
20241118012126237 20241118012126237_0_1 1 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 1 Alice \N
20241118012126237 20241118012126237_0_0 2 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 2 Bob \N
20241118012126237 20241118012126237_0_2 3 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 3 Cathy \N
20241118012132306 20241118012132306_0_3 4 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 4 David 25
20241118012132306 20241118012132306_0_4 5 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 5 Eva 30
20241118012132306 20241118012132306_0_5 6 5166112a-90d8-4ba8-8646-337fbeb2a375-0_0-35-121_20241118012132306.parquet 6 Frank 28

-- !altering_simple_columns_table --
20241118012136512 20241118012136512_0_0 1 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 1 Alice 25.0
20241118012136512 20241118012136512_0_2 2 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 2 Bob 30.0
20241118012136512 20241118012136512_0_1 3 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 3 Cathy 28.0
20241118012138287 20241118012138287_0_3 4 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 4 David 26.0
20241118012138287 20241118012138287_0_4 5 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 5 Eva 31.5
20241118012138287 20241118012138287_0_5 6 203f0f43-ae9d-4c17-8d5d-834f0dbc62c9-0_0-78-246_20241118012138287.parquet 6 Frank 29.2

-- !adding_complex_columns_table --
20241118012144831 20241118012144831_0_1 1 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 1 Alice {"age":25, "address":"Guangzhou", "email":null}
20241118012144831 20241118012144831_0_0 2 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 2 Bob {"age":30, "address":"Shanghai", "email":null}
20241118012144831 20241118012144831_0_2 3 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 3 Cathy {"age":28, "address":"Beijing", "email":null}
20241118012146150 20241118012146150_0_3 4 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 4 David {"age":25, "address":"Shenzhen", "email":"[email protected]"}
20241118012146150 20241118012146150_0_4 5 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 5 Eva {"age":30, "address":"Chengdu", "email":"[email protected]"}
20241118012146150 20241118012146150_0_5 6 3c038df9-a652-4878-9b8a-221ae443448e-0_0-165-497_20241118012146150.parquet 6 Frank {"age":28, "address":"Wuhan", "email":"[email protected]"}

-- !altering_complex_columns_table --
20241118012147879 20241118012147879_0_0 1 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 1 Alice {"age":25, "address":"Guangzhou"}
20241118012147879 20241118012147879_0_2 2 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 2 Bob {"age":30, "address":"Shanghai"}
20241118012147879 20241118012147879_0_1 3 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 3 Cathy {"age":28, "address":"Beijing"}
20241118012149007 20241118012149007_0_3 4 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 4 David {"age":26, "address":"Shenzhen"}
20241118012149007 20241118012149007_0_4 5 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 5 Eva {"age":31.5, "address":"Chengdu"}
20241118012149007 20241118012149007_0_5 6 185d101f-a484-45ce-b236-03ccd33c521b-0_0-208-622_20241118012149007.parquet 6 Frank {"age":29.2, "address":"Wuhan"}

Binary file modified regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out
View file
Open in desktop
Binary file not shown.
120 changes: 120 additions & 0 deletions regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,3 +239,123 @@
-- !timetravel10 --
10000

-- !timetravel1 --
1000

-- !timetravel2 --
2000

-- !timetravel3 --
3000

-- !timetravel4 --
4000

-- !timetravel5 --
5000

-- !timetravel6 --
6000

-- !timetravel7 --
7000

-- !timetravel8 --
8000

-- !timetravel9 --
9000

-- !timetravel10 --
10000

-- !timetravel1 --
1000

-- !timetravel2 --
2000

-- !timetravel3 --
3000

-- !timetravel4 --
4000

-- !timetravel5 --
5000

-- !timetravel6 --
6000

-- !timetravel7 --
7000

-- !timetravel8 --
8000

-- !timetravel9 --
9000

-- !timetravel10 --
10000

-- !timetravel1 --
1000

-- !timetravel2 --
2000

-- !timetravel3 --
3000

-- !timetravel4 --
4000

-- !timetravel5 --
5000

-- !timetravel6 --
6000

-- !timetravel7 --
7000

-- !timetravel8 --
8000

-- !timetravel9 --
9000

-- !timetravel10 --
10000

-- !timetravel1 --
1000

-- !timetravel2 --
2000

-- !timetravel3 --
3000

-- !timetravel4 --
4000

-- !timetravel5 --
5000

-- !timetravel6 --
6000

-- !timetravel7 --
7000

-- !timetravel8 --
8000

-- !timetravel9 --
9000

-- !timetravel10 --
10000

18 changes: 13 additions & 5 deletions regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,17 +104,25 @@ suite("test_hudi_incremental", "p2,external,hudi,external_remote,external_remote
"20241114152334111",
]

test_hudi_incremental_querys("user_activity_log_cow_non_partition", timestamps_cow_non_partition)
test_hudi_incremental_querys("user_activity_log_cow_partition", timestamps_cow_partition)
sql """set force_jni_scanner=true;"""
sql """set hudi_jni_scanner='hadoop';"""
// TODO: @suxiaogang223 don't support incremental query for cow table by jni reader
// test_hudi_incremental_querys("user_activity_log_cow_non_partition", timestamps_cow_non_partition)
// test_hudi_incremental_querys("user_activity_log_cow_partition", timestamps_cow_partition)
test_hudi_incremental_querys("user_activity_log_mor_non_partition", timestamps_mor_non_partition)
test_hudi_incremental_querys("user_activity_log_mor_partition", timestamps_mor_partition)
sql """set force_jni_scanner=true;"""
// don't support incremental query for cow table by jni reader

sql """set hudi_jni_scanner='spark';"""
// test_hudi_incremental_querys("user_activity_log_cow_non_partition", timestamps_cow_non_partition)
// test_hudi_incremental_querys("user_activity_log_cow_partition", timestamps_cow_partition)
test_hudi_incremental_querys("user_activity_log_mor_non_partition", timestamps_mor_non_partition)
test_hudi_incremental_querys("user_activity_log_mor_partition", timestamps_mor_partition)
// sql """set force_jni_scanner=false;"""

sql """set force_jni_scanner=false;"""
test_hudi_incremental_querys("user_activity_log_cow_non_partition", timestamps_cow_non_partition)
test_hudi_incremental_querys("user_activity_log_cow_partition", timestamps_cow_partition)
test_hudi_incremental_querys("user_activity_log_mor_non_partition", timestamps_mor_non_partition)
test_hudi_incremental_querys("user_activity_log_mor_partition", timestamps_mor_partition)

sql """drop catalog if exists ${catalog_name};"""
}
Loading
Loading