Skip to content

Commit

Permalink
[enhance](regression-test) fix hudi incremental query bug and add hud…
Browse files Browse the repository at this point in the history
…i p2 cases (#44003)

### What problem does this PR solve?

Problem Summary:
fix hudi incremental query bug and add hudi p2 cases
  • Loading branch information
suxiaogang223 authored and Your Name committed Nov 22, 2024
1 parent 7d123ed commit 52617d3
Show file tree
Hide file tree
Showing 13 changed files with 929 additions and 197 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public COWIncrementalRelation(Map<String, String> optParams, Configuration confi
if (!metaClient.getTableConfig().populateMetaFields()) {
throw new HoodieException("Incremental queries are not supported when meta fields are disabled");
}
HoodieInstant lastInstant = commitTimeline.lastInstant().get();

String startInstantTime = optParams.get("hoodie.datasource.read.begin.instanttime");
if (startInstantTime == null) {
throw new HoodieException("Specify the begin instant time to pull from using "
Expand All @@ -89,16 +89,18 @@ public COWIncrementalRelation(Map<String, String> optParams, Configuration confi
startInstantTime = "000";
}
String endInstantTime = optParams.getOrDefault("hoodie.datasource.read.end.instanttime",
lastInstant.getTimestamp());
hollowCommitHandling == HollowCommitHandling.USE_TRANSITION_TIME
? commitTimeline.lastInstant().get().getStateTransitionTime()
: commitTimeline.lastInstant().get().getTimestamp());
startInstantArchived = commitTimeline.isBeforeTimelineStarts(startInstantTime);
endInstantArchived = commitTimeline.isBeforeTimelineStarts(endInstantTime);

HoodieTimeline commitsTimelineToReturn;
if (hollowCommitHandling == HollowCommitHandling.USE_TRANSITION_TIME) {
commitsTimelineToReturn = commitTimeline.findInstantsInRangeByStateTransitionTime(startInstantTime,
lastInstant.getStateTransitionTime());
endInstantTime);
} else {
commitsTimelineToReturn = commitTimeline.findInstantsInRange(startInstantTime, lastInstant.getTimestamp());
commitsTimelineToReturn = commitTimeline.findInstantsInRange(startInstantTime, endInstantTime);
}
List<HoodieInstant> commitsToReturn = commitsTimelineToReturn.getInstants();

Expand Down
3 changes: 3 additions & 0 deletions regression-test/conf/regression-conf.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,9 @@ extEsPort = 9200
extEsUser = "*******"
extEsPassword = "***********"

enableExternalHudiTest = false
hudiEmrCatalog = "***********"

enableObjStorageTest=false
enableMaxComputeTest=false
aliYunAk="***********"
Expand Down
349 changes: 349 additions & 0 deletions regression-test/data/external_table_p2/hudi/test_hudi_incremental.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,349 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !incremental_1_end --
9000

-- !incremental_earliest_1 --
1000

-- !incremental_2_end --
8000

-- !incremental_earliest_2 --
2000

-- !incremental_1_2 --
1000

-- !incremental_3_end --
7000

-- !incremental_earliest_3 --
3000

-- !incremental_2_3 --
1000

-- !incremental_4_end --
6000

-- !incremental_earliest_4 --
4000

-- !incremental_3_4 --
1000

-- !incremental_5_end --
5000

-- !incremental_earliest_5 --
5000

-- !incremental_4_5 --
1000

-- !incremental_6_end --
4000

-- !incremental_earliest_6 --
6000

-- !incremental_5_6 --
1000

-- !incremental_7_end --
3000

-- !incremental_earliest_7 --
7000

-- !incremental_6_7 --
1000

-- !incremental_8_end --
2000

-- !incremental_earliest_8 --
8000

-- !incremental_7_8 --
1000

-- !incremental_9_end --
1000

-- !incremental_earliest_9 --
9000

-- !incremental_8_9 --
1000

-- !incremental_10_end --
0

-- !incremental_earliest_10 --
10000

-- !incremental_9_10 --
1000

-- !incremental_1_end --
9000

-- !incremental_earliest_1 --
1000

-- !incremental_2_end --
8000

-- !incremental_earliest_2 --
2000

-- !incremental_1_2 --
1000

-- !incremental_3_end --
7000

-- !incremental_earliest_3 --
3000

-- !incremental_2_3 --
1000

-- !incremental_4_end --
6000

-- !incremental_earliest_4 --
4000

-- !incremental_3_4 --
1000

-- !incremental_5_end --
5000

-- !incremental_earliest_5 --
5000

-- !incremental_4_5 --
1000

-- !incremental_6_end --
4000

-- !incremental_earliest_6 --
6000

-- !incremental_5_6 --
1000

-- !incremental_7_end --
3000

-- !incremental_earliest_7 --
7000

-- !incremental_6_7 --
1000

-- !incremental_8_end --
2000

-- !incremental_earliest_8 --
8000

-- !incremental_7_8 --
1000

-- !incremental_9_end --
1000

-- !incremental_earliest_9 --
9000

-- !incremental_8_9 --
1000

-- !incremental_10_end --
0

-- !incremental_earliest_10 --
10000

-- !incremental_9_10 --
1000

-- !incremental_1_end --
9000

-- !incremental_earliest_1 --
1000

-- !incremental_2_end --
8000

-- !incremental_earliest_2 --
2000

-- !incremental_1_2 --
1000

-- !incremental_3_end --
7000

-- !incremental_earliest_3 --
3000

-- !incremental_2_3 --
1000

-- !incremental_4_end --
6000

-- !incremental_earliest_4 --
4000

-- !incremental_3_4 --
1000

-- !incremental_5_end --
5000

-- !incremental_earliest_5 --
5000

-- !incremental_4_5 --
1000

-- !incremental_6_end --
4000

-- !incremental_earliest_6 --
6000

-- !incremental_5_6 --
1000

-- !incremental_7_end --
3000

-- !incremental_earliest_7 --
7000

-- !incremental_6_7 --
1000

-- !incremental_8_end --
2000

-- !incremental_earliest_8 --
8000

-- !incremental_7_8 --
1000

-- !incremental_9_end --
1000

-- !incremental_earliest_9 --
9000

-- !incremental_8_9 --
1000

-- !incremental_10_end --
0

-- !incremental_earliest_10 --
10000

-- !incremental_9_10 --
1000

-- !incremental_1_end --
9000

-- !incremental_earliest_1 --
1000

-- !incremental_2_end --
8000

-- !incremental_earliest_2 --
2000

-- !incremental_1_2 --
1000

-- !incremental_3_end --
7000

-- !incremental_earliest_3 --
3000

-- !incremental_2_3 --
1000

-- !incremental_4_end --
6000

-- !incremental_earliest_4 --
4000

-- !incremental_3_4 --
1000

-- !incremental_5_end --
5000

-- !incremental_earliest_5 --
5000

-- !incremental_4_5 --
1000

-- !incremental_6_end --
4000

-- !incremental_earliest_6 --
6000

-- !incremental_5_6 --
1000

-- !incremental_7_end --
3000

-- !incremental_earliest_7 --
7000

-- !incremental_6_7 --
1000

-- !incremental_8_end --
2000

-- !incremental_earliest_8 --
8000

-- !incremental_7_8 --
1000

-- !incremental_9_end --
1000

-- !incremental_earliest_9 --
9000

-- !incremental_8_9 --
1000

-- !incremental_10_end --
0

-- !incremental_earliest_10 --
10000

-- !incremental_9_10 --
1000

Loading

0 comments on commit 52617d3

Please sign in to comment.