From 878bde75e9e78629e6c32f47867bb57ad87840a9 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Wed, 18 Dec 2024 15:06:57 +0800 Subject: [PATCH] update --- ...t_key_bounds_truncation_read_scenarios.out | 30 ++++++ .../test_key_bounds_truncation_basic.groovy | 11 --- ...ey_bounds_truncation_read_scenarios.groovy | 92 +++++++++++++++++++ ..._bounds_truncation_write_scenarios.groovy} | 4 +- 4 files changed, 124 insertions(+), 13 deletions(-) create mode 100644 regression-test/data/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_read_scenarios.out create mode 100644 regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_read_scenarios.groovy rename regression-test/suites/unique_with_mow_p0/key_bounds/{test_key_bounds_truncation_scenarios.groovy => test_key_bounds_truncation_write_scenarios.groovy} (98%) diff --git a/regression-test/data/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_read_scenarios.out b/regression-test/data/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_read_scenarios.out new file mode 100644 index 000000000000000..30087ad3e53eed7 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_read_scenarios.out @@ -0,0 +1,30 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +aaaaaaaabbb1 1 1 1 +aaaaaaaabbb2 1 1 1 +aaaaaaaabbb3 1 1 1 +aaaaaaaabbb4 1 1 1 +aaaaaaaabbb5 1 1 1 +aaaaaaaabbb6 1 1 1 +aaaaaaaabbb7 1 1 1 +aaaaaaaabbb8 1 1 1 +aaaaaaaabbb9 1 1 1 +aaaaaaaaccc1 2 2 2 +aaaaaaaaccc2 2 2 2 +aaaaaaaaccc3 2 2 2 +aaaaaaaaccc4 2 2 2 +aaaaaaaaccc5 2 2 2 +aaaaaaaaccc6 2 2 2 +aaaaaaaaccc7 2 2 2 +aaaaaaaaccc8 2 2 2 +aaaaaaaaccc9 2 2 2 +aaaaaaaaddd1 3 3 3 +aaaaaaaaddd2 3 3 3 +aaaaaaaaddd3 3 3 3 +aaaaaaaaddd4 3 3 3 +aaaaaaaaddd5 3 3 3 +aaaaaaaaddd6 3 3 3 +aaaaaaaaddd7 3 3 3 +aaaaaaaaddd8 3 3 3 +aaaaaaaaddd9 3 3 3 + diff --git a/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_basic.groovy b/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_basic.groovy index 34445de4fc2007e..cdf28c5d28b993a 100644 --- a/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_basic.groovy +++ b/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_basic.groovy @@ -45,17 +45,6 @@ suite("test_key_bounds_truncation_basic", "nonConcurrent") { } } - def getSegmentKeyBounds = { int version -> - def metaUrl = sql_return_maparray("show tablets from ${tableName};").get(0).MetaUrl - def jsonMeta = Http.GET(metaUrl, true, false) - for (def meta : jsonMeta.rs_metas) { - int end_version = meta.end_version - if (end_version == version) { - return meta.segments_key_bounds - } - } - } - def truncateString = { String s, int l -> if (s.size() > l) { return s.substring(0, l) diff --git a/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_read_scenarios.groovy b/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_read_scenarios.groovy new file mode 100644 index 000000000000000..8e142f6aeaf7e28 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_read_scenarios.groovy @@ -0,0 +1,92 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.google.common.collect.Maps +import org.apache.commons.lang.RandomStringUtils +import org.apache.doris.regression.util.Http +import java.util.concurrent.TimeUnit +import org.awaitility.Awaitility + +suite("test_key_bounds_truncation_read_scenarios", "nonConcurrent") { + + def tableName = "test_key_bounds_truncation_read_scenarios" + sql """ DROP TABLE IF EXISTS ${tableName} force;""" + sql """ CREATE TABLE ${tableName} ( + `k` varchar(65533) NOT NULL, + `v1` int, + v2 int, + v3 int ) + UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES("replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true"); """ + + def getRowsetMetas = { int version -> + def metaUrl = sql_return_maparray("show tablets from ${tableName};").get(0).MetaUrl + def jsonMeta = Http.GET(metaUrl, true, false) + for (def meta : jsonMeta.rs_metas) { + int end_version = meta.end_version + if (end_version == version) { + return meta + } + } + } + + def checkKeyBounds = { int version, int length, boolean turnedOn -> + def rowsetMeta = getRowsetMetas(version) + def keyBounds = rowsetMeta.segments_key_bounds + + logger.info("\nsegments_key_bounds_truncated=${rowsetMeta.segments_key_bounds_truncated}, turnedOn=${turnedOn}") + assertEquals(turnedOn, rowsetMeta.segments_key_bounds_truncated) + + for (def bounds : keyBounds) { + String min_key = bounds.min_key + String max_key = bounds.max_key + logger.info("\nmin_key=${min_key}, size=${min_key.size()}\nmax_key=${max_key}, size=${max_key.size()}") + assertTrue(min_key.size() <= length) + assertTrue(max_key.size() <= length) + } + } + + + def customBeConfig = [ + enable_segments_key_bounds_truncation : true, + segments_key_bounds_truncation_threshold : 2 + ] + + setBeConfigTemporary(customBeConfig) { + // 1. mow load + for (int j=1;j<=10;j++) { + for (int i=1;i<=9;i++) { + String k1 = "aaaaaaaabbb${i}" + String k2 = "aaaaaaaaccc${i}" + String k3 = "aaaaaaaaddd${i}" + sql """insert into ${tableName} values("${k1}",1,1,1),("${k2}",2,2,2),("${k3}",3,3,3);""" + } + } + (2..91).each { idx -> + checkKeyBounds(idx, 2, true) + } + qt_sql "select * from ${tableName} order by k;" + + + // 2. point lookup on mow table + + + } +} diff --git a/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_scenarios.groovy b/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_write_scenarios.groovy similarity index 98% rename from regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_scenarios.groovy rename to regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_write_scenarios.groovy index c21ac5dd1cb2c75..9f71c61bf6adf26 100644 --- a/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_scenarios.groovy +++ b/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_write_scenarios.groovy @@ -22,9 +22,9 @@ import org.apache.doris.regression.util.Http import java.util.concurrent.TimeUnit import org.awaitility.Awaitility -suite("test_key_bounds_truncation_scenarios", "nonConcurrent") { +suite("test_key_bounds_truncation_write_scenarios", "nonConcurrent") { - def tableName = "test_key_bounds_truncation_scenarios" + def tableName = "test_key_bounds_truncation_write_scenarios" sql """ DROP TABLE IF EXISTS ${tableName} force;""" sql """ CREATE TABLE ${tableName} ( `k` varchar(65533) NOT NULL,