diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index f554ba6053a5e6..d617aa173d93f8 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -299,8 +299,6 @@ if (COMPILER_CLANG) -Wno-implicit-float-conversion -Wno-implicit-int-conversion -Wno-sign-conversion - -Wno-missing-field-initializers - -Wno-unused-const-variable -Wno-shorten-64-to-32) if (USE_LIBCPP) add_compile_options($<$:-stdlib=libc++>) diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index b16149cd3540a6..cd502d16547f9b 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -244,15 +244,12 @@ void BlockFileCache::use_cell(const FileBlockCell& cell, FileBlocks* result, boo result->push_back(cell.file_block); } - if (cell.file_block->cache_type() != FileCacheType::TTL || - config::enable_ttl_cache_evict_using_lru) { - auto& queue = get_queue(cell.file_block->cache_type()); - DCHECK(cell.queue_iterator) << "impossible"; - /// Move to the end of the queue. The iterator remains valid. - if (move_iter_flag) { - queue.move_to_end(*cell.queue_iterator, cache_lock); - } + auto& queue = get_queue(cell.file_block->cache_type()); + /// Move to the end of the queue. The iterator remains valid. + if (cell.queue_iterator && move_iter_flag) { + queue.move_to_end(*cell.queue_iterator, cache_lock); } + cell.update_atime(); cell.is_deleted = false; } @@ -358,7 +355,7 @@ FileBlocks BlockFileCache::get_impl(const UInt128Wrapper& hash, const CacheConte auto st = cell.file_block->change_cache_type_between_ttl_and_others( FileCacheType::NORMAL); if (st.ok()) { - if (config::enable_ttl_cache_evict_using_lru) { + if (cell.queue_iterator) { auto& ttl_queue = get_queue(FileCacheType::TTL); ttl_queue.remove(cell.queue_iterator.value(), cache_lock); } @@ -1056,7 +1053,7 @@ bool BlockFileCache::remove_if_ttl_file_unlock(const UInt128Wrapper& file_key, b auto st = cell.file_block->change_cache_type_between_ttl_and_others( FileCacheType::NORMAL); if (st.ok()) { - if (config::enable_ttl_cache_evict_using_lru) { + if (cell.queue_iterator) { ttl_queue.remove(cell.queue_iterator.value(), cache_lock); } auto& queue = get_queue(FileCacheType::NORMAL); @@ -1133,7 +1130,7 @@ void BlockFileCache::reset_range(const UInt128Wrapper& hash, size_t offset, size _files.find(hash)->second.find(offset) != _files.find(hash)->second.end()); FileBlockCell* cell = get_cell(hash, offset, cache_lock); DCHECK(cell != nullptr); - if (cell->file_block->cache_type() != FileCacheType::TTL) { + if (cell->queue_iterator) { auto& queue = get_queue(cell->file_block->cache_type()); DCHECK(queue.contains(hash, offset, cache_lock)); auto iter = queue.get(hash, offset, cache_lock); @@ -1809,10 +1806,8 @@ std::string BlockFileCache::clear_file_cache_directly() { << " time_elapsed=" << duration_cast(steady_clock::now() - start).count() << " num_files=" << num_files << " cache_size=" << cache_size << " index_queue_size=" << index_queue_size << " normal_queue_size=" << normal_queue_size - << " disposible_queue_size=" << disposible_queue_size; - if (config::enable_ttl_cache_evict_using_lru) { - ss << "ttl_queue_size=" << ttl_queue_size; - } + << " disposible_queue_size=" << disposible_queue_size << "ttl_queue_size=" << ttl_queue_size; + auto msg = ss.str(); LOG(INFO) << msg; return msg; diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp index d11b9fa54d0421..7a784a55b862d0 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp @@ -51,14 +51,26 @@ Result InvertedIndexFileWriter::open(const TabletIndex* index if (exists) { LOG(ERROR) << "try to init a directory:" << local_fs_index_path << " already exists"; - return ResultError(Status::InternalError("init_fulltext_index directory already exists")); + return ResultError( + Status::InternalError("InvertedIndexFileWriter::open directory already exists")); } bool can_use_ram_dir = true; auto* dir = DorisFSDirectoryFactory::getDirectory(local_fs, local_fs_index_path.c_str(), can_use_ram_dir); - _indices_dirs.emplace(std::make_pair(index_meta->index_id(), index_meta->get_index_suffix()), - std::unique_ptr(dir)); + auto key = std::make_pair(index_meta->index_id(), index_meta->get_index_suffix()); + auto [it, inserted] = _indices_dirs.emplace(key, std::unique_ptr(dir)); + if (!inserted) { + LOG(ERROR) << "InvertedIndexFileWriter::open attempted to insert a duplicate key: (" + << key.first << ", " << key.second << ")"; + LOG(ERROR) << "Directories already in map: "; + for (const auto& entry : _indices_dirs) { + LOG(ERROR) << "Key: (" << entry.first.first << ", " << entry.first.second << ")"; + } + return ResultError(Status::InternalError( + "InvertedIndexFileWriter::open attempted to insert a duplicate dir")); + } + return dir; } diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index ebf40c90bea35b..e00b5b595e20dc 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -209,7 +209,7 @@ StorageEngine::StorageEngine(const EngineOptions& options) _txn_manager(new TxnManager(*this, config::txn_map_shard_size, config::txn_shard_size)), _default_rowset_type(BETA_ROWSET), _create_tablet_idx_lru_cache( - new CreateTabletIdxCache(config::partition_disk_index_lru_size)), + new CreateTabletRRIdxCache(config::partition_disk_index_lru_size)), _snapshot_mgr(std::make_unique(*this)) { REGISTER_HOOK_METRIC(unused_rowsets_count, [this]() { // std::lock_guard lock(_gc_mutex); @@ -515,7 +515,7 @@ Status StorageEngine::set_cluster_id(int32_t cluster_id) { int StorageEngine::_get_and_set_next_disk_index(int64 partition_id, TStorageMedium::type storage_medium) { - auto key = CreateTabletIdxCache::get_key(partition_id, storage_medium); + auto key = CreateTabletRRIdxCache::get_key(partition_id, storage_medium); int curr_index = _create_tablet_idx_lru_cache->get_index(key); // -1, lru can't find key if (curr_index == -1) { @@ -1511,7 +1511,7 @@ Status StorageEngine::_persist_broken_paths() { return Status::OK(); } -int CreateTabletIdxCache::get_index(const std::string& key) { +int CreateTabletRRIdxCache::get_index(const std::string& key) { auto* lru_handle = lookup(key); if (lru_handle) { Defer release([cache = this, lru_handle] { cache->release(lru_handle); }); @@ -1522,7 +1522,7 @@ int CreateTabletIdxCache::get_index(const std::string& key) { return -1; } -void CreateTabletIdxCache::set_index(const std::string& key, int next_idx) { +void CreateTabletRRIdxCache::set_index(const std::string& key, int next_idx) { assert(next_idx >= 0); auto* value = new CacheValue; value->idx = next_idx; diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index b2a313adcdbb7e..421c0eb352d712 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -69,7 +69,7 @@ class Thread; class ThreadPool; class TxnManager; class ReportWorker; -class CreateTabletIdxCache; +class CreateTabletRRIdxCache; struct DirInfo; class SnapshotManager; @@ -532,7 +532,7 @@ class StorageEngine final : public BaseStorageEngine { // next index for create tablet std::map _last_use_index; - std::unique_ptr _create_tablet_idx_lru_cache; + std::unique_ptr _create_tablet_idx_lru_cache; std::unique_ptr _snapshot_mgr; }; @@ -540,7 +540,7 @@ class StorageEngine final : public BaseStorageEngine { // lru cache for create tabelt round robin in disks // key: partitionId_medium // value: index -class CreateTabletIdxCache : public LRUCachePolicy { +class CreateTabletRRIdxCache : public LRUCachePolicy { public: // get key, delimiter with DELIMITER '-' static std::string get_key(int64_t partition_id, TStorageMedium::type medium) { @@ -557,10 +557,10 @@ class CreateTabletIdxCache : public LRUCachePolicy { int idx = 0; }; - CreateTabletIdxCache(size_t capacity) + CreateTabletRRIdxCache(size_t capacity) : LRUCachePolicy(CachePolicy::CacheType::CREATE_TABLET_RR_IDX_CACHE, capacity, LRUCacheType::NUMBER, - /*stale_sweep_time_s*/ 30 * 60) {} + /*stale_sweep_time_s*/ 30 * 60, 1) {} }; struct DirInfo { diff --git a/be/src/olap/utils.cpp b/be/src/olap/utils.cpp index 5ae8b7ab9df244..52d05133379143 100644 --- a/be/src/olap/utils.cpp +++ b/be/src/olap/utils.cpp @@ -56,319 +56,6 @@ uint32_t olap_adler32(uint32_t adler, const char* buf, size_t len) { return adler32(adler, reinterpret_cast(buf), len); } -// implement crc32c by looking up 8 tables -static const unsigned int T8_0[256] = { - 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, 0x26A1E7E8, - 0xD4CA64EB, 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 0x4D43CFD0, 0xBF284CD3, - 0xAC78BF27, 0x5E133C24, 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, 0xD7C45070, - 0x25AFD373, 0x36FF2087, 0xC494A384, 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, - 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, - 0x33ED7D2A, 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, 0xAA64D611, 0x580F5512, - 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, 0x30E349B1, - 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A, - 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, 0x7DA08661, 0x8FCB0562, 0x9C9BF696, - 0x6EF07595, 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, - 0x67DAFA54, 0x95B17957, 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, - 0xFE53516F, 0xED03A29B, 0x1F682198, 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, - 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, - 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, 0x61C69362, 0x93AD1061, - 0x80FDE395, 0x72966096, 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, 0xEB1FCBAD, - 0x197448AE, 0x0A24BB5A, 0xF84F3859, 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, - 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, 0xB602C312, 0x44694011, 0x5739B3E5, - 0xA55230E6, 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE, - 0xDDE0EB2A, 0x2F8B6829, 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, 0x456CAC67, - 0xB7072F64, 0xA457DC90, 0x563C5F93, 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, - 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, 0x92A8FC17, 0x60C37F14, 0x73938CE0, - 0x81F80FE3, 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, 0x1871A4D8, 0xEA1A27DB, - 0xF94AD42F, 0x0B21572C, 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, 0xA24BB5A6, - 0x502036A5, 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, - 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, 0xEF087A76, 0x1D63F975, 0x0E330A81, - 0xFC588982, 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, 0x758FE5D6, 0x87E466D5, - 0x94B49521, 0x66DF1622, 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, - 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, - 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F, 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, - 0x5A048DFF, 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, 0xD3D3E1AB, 0x21B862A8, - 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, 0x590AB964, - 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, - 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, - 0x37FACCF1, 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, - 0x4F48173D, 0xBD23943E, 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, 0x34F4F86A, - 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, - 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351}; - -static const unsigned int T8_1[256] = { - 0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, 0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, - 0x7A6DC945, 0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, 0xD39EA264, 0xC03C3A13, - 0xF4DB928A, 0xE7790AFD, 0x3FC5F181, 0x2C6769F6, 0x1880C16F, 0x0B225918, 0x714F905D, - 0x62ED082A, 0x560AA0B3, 0x45A838C4, 0xA2D13239, 0xB173AA4E, 0x859402D7, 0x96369AA0, - 0xEC5B53E5, 0xFFF9CB92, 0xCB1E630B, 0xD8BCFB7C, 0x7F8BE302, 0x6C297B75, 0x58CED3EC, - 0x4B6C4B9B, 0x310182DE, 0x22A31AA9, 0x1644B230, 0x05E62A47, 0xE29F20BA, 0xF13DB8CD, - 0xC5DA1054, 0xD6788823, 0xAC154166, 0xBFB7D911, 0x8B507188, 0x98F2E9FF, 0x404E1283, - 0x53EC8AF4, 0x670B226D, 0x74A9BA1A, 0x0EC4735F, 0x1D66EB28, 0x298143B1, 0x3A23DBC6, - 0xDD5AD13B, 0xCEF8494C, 0xFA1FE1D5, 0xE9BD79A2, 0x93D0B0E7, 0x80722890, 0xB4958009, - 0xA737187E, 0xFF17C604, 0xECB55E73, 0xD852F6EA, 0xCBF06E9D, 0xB19DA7D8, 0xA23F3FAF, - 0x96D89736, 0x857A0F41, 0x620305BC, 0x71A19DCB, 0x45463552, 0x56E4AD25, 0x2C896460, - 0x3F2BFC17, 0x0BCC548E, 0x186ECCF9, 0xC0D23785, 0xD370AFF2, 0xE797076B, 0xF4359F1C, - 0x8E585659, 0x9DFACE2E, 0xA91D66B7, 0xBABFFEC0, 0x5DC6F43D, 0x4E646C4A, 0x7A83C4D3, - 0x69215CA4, 0x134C95E1, 0x00EE0D96, 0x3409A50F, 0x27AB3D78, 0x809C2506, 0x933EBD71, - 0xA7D915E8, 0xB47B8D9F, 0xCE1644DA, 0xDDB4DCAD, 0xE9537434, 0xFAF1EC43, 0x1D88E6BE, - 0x0E2A7EC9, 0x3ACDD650, 0x296F4E27, 0x53028762, 0x40A01F15, 0x7447B78C, 0x67E52FFB, - 0xBF59D487, 0xACFB4CF0, 0x981CE469, 0x8BBE7C1E, 0xF1D3B55B, 0xE2712D2C, 0xD69685B5, - 0xC5341DC2, 0x224D173F, 0x31EF8F48, 0x050827D1, 0x16AABFA6, 0x6CC776E3, 0x7F65EE94, - 0x4B82460D, 0x5820DE7A, 0xFBC3FAF9, 0xE861628E, 0xDC86CA17, 0xCF245260, 0xB5499B25, - 0xA6EB0352, 0x920CABCB, 0x81AE33BC, 0x66D73941, 0x7575A136, 0x419209AF, 0x523091D8, - 0x285D589D, 0x3BFFC0EA, 0x0F186873, 0x1CBAF004, 0xC4060B78, 0xD7A4930F, 0xE3433B96, - 0xF0E1A3E1, 0x8A8C6AA4, 0x992EF2D3, 0xADC95A4A, 0xBE6BC23D, 0x5912C8C0, 0x4AB050B7, - 0x7E57F82E, 0x6DF56059, 0x1798A91C, 0x043A316B, 0x30DD99F2, 0x237F0185, 0x844819FB, - 0x97EA818C, 0xA30D2915, 0xB0AFB162, 0xCAC27827, 0xD960E050, 0xED8748C9, 0xFE25D0BE, - 0x195CDA43, 0x0AFE4234, 0x3E19EAAD, 0x2DBB72DA, 0x57D6BB9F, 0x447423E8, 0x70938B71, - 0x63311306, 0xBB8DE87A, 0xA82F700D, 0x9CC8D894, 0x8F6A40E3, 0xF50789A6, 0xE6A511D1, - 0xD242B948, 0xC1E0213F, 0x26992BC2, 0x353BB3B5, 0x01DC1B2C, 0x127E835B, 0x68134A1E, - 0x7BB1D269, 0x4F567AF0, 0x5CF4E287, 0x04D43CFD, 0x1776A48A, 0x23910C13, 0x30339464, - 0x4A5E5D21, 0x59FCC556, 0x6D1B6DCF, 0x7EB9F5B8, 0x99C0FF45, 0x8A626732, 0xBE85CFAB, - 0xAD2757DC, 0xD74A9E99, 0xC4E806EE, 0xF00FAE77, 0xE3AD3600, 0x3B11CD7C, 0x28B3550B, - 0x1C54FD92, 0x0FF665E5, 0x759BACA0, 0x663934D7, 0x52DE9C4E, 0x417C0439, 0xA6050EC4, - 0xB5A796B3, 0x81403E2A, 0x92E2A65D, 0xE88F6F18, 0xFB2DF76F, 0xCFCA5FF6, 0xDC68C781, - 0x7B5FDFFF, 0x68FD4788, 0x5C1AEF11, 0x4FB87766, 0x35D5BE23, 0x26772654, 0x12908ECD, - 0x013216BA, 0xE64B1C47, 0xF5E98430, 0xC10E2CA9, 0xD2ACB4DE, 0xA8C17D9B, 0xBB63E5EC, - 0x8F844D75, 0x9C26D502, 0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7, 0x0A104FA2, - 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B, 0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, - 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483}; - -static const unsigned int T8_2[256] = { - 0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, 0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, - 0x74F06469, 0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, 0xA68F9ADF, 0x03CE08A1, - 0xE9E0C8D2, 0x4CA15AAC, 0x70A27D8A, 0xD5E3EFF4, 0x3FCD2F87, 0x9A8CBDF9, 0xEE7CD990, - 0x4B3D4BEE, 0xA1138B9D, 0x045219E3, 0x48F3434F, 0xEDB2D131, 0x079C1142, 0xA2DD833C, - 0xD62DE755, 0x736C752B, 0x9942B558, 0x3C032726, 0xE144FB14, 0x4405696A, 0xAE2BA919, - 0x0B6A3B67, 0x7F9A5F0E, 0xDADBCD70, 0x30F50D03, 0x95B49F7D, 0xD915C5D1, 0x7C5457AF, - 0x967A97DC, 0x333B05A2, 0x47CB61CB, 0xE28AF3B5, 0x08A433C6, 0xADE5A1B8, 0x91E6869E, - 0x34A714E0, 0xDE89D493, 0x7BC846ED, 0x0F382284, 0xAA79B0FA, 0x40577089, 0xE516E2F7, - 0xA9B7B85B, 0x0CF62A25, 0xE6D8EA56, 0x43997828, 0x37691C41, 0x92288E3F, 0x78064E4C, - 0xDD47DC32, 0xC76580D9, 0x622412A7, 0x880AD2D4, 0x2D4B40AA, 0x59BB24C3, 0xFCFAB6BD, - 0x16D476CE, 0xB395E4B0, 0xFF34BE1C, 0x5A752C62, 0xB05BEC11, 0x151A7E6F, 0x61EA1A06, - 0xC4AB8878, 0x2E85480B, 0x8BC4DA75, 0xB7C7FD53, 0x12866F2D, 0xF8A8AF5E, 0x5DE93D20, - 0x29195949, 0x8C58CB37, 0x66760B44, 0xC337993A, 0x8F96C396, 0x2AD751E8, 0xC0F9919B, - 0x65B803E5, 0x1148678C, 0xB409F5F2, 0x5E273581, 0xFB66A7FF, 0x26217BCD, 0x8360E9B3, - 0x694E29C0, 0xCC0FBBBE, 0xB8FFDFD7, 0x1DBE4DA9, 0xF7908DDA, 0x52D11FA4, 0x1E704508, - 0xBB31D776, 0x511F1705, 0xF45E857B, 0x80AEE112, 0x25EF736C, 0xCFC1B31F, 0x6A802161, - 0x56830647, 0xF3C29439, 0x19EC544A, 0xBCADC634, 0xC85DA25D, 0x6D1C3023, 0x8732F050, - 0x2273622E, 0x6ED23882, 0xCB93AAFC, 0x21BD6A8F, 0x84FCF8F1, 0xF00C9C98, 0x554D0EE6, - 0xBF63CE95, 0x1A225CEB, 0x8B277743, 0x2E66E53D, 0xC448254E, 0x6109B730, 0x15F9D359, - 0xB0B84127, 0x5A968154, 0xFFD7132A, 0xB3764986, 0x1637DBF8, 0xFC191B8B, 0x595889F5, - 0x2DA8ED9C, 0x88E97FE2, 0x62C7BF91, 0xC7862DEF, 0xFB850AC9, 0x5EC498B7, 0xB4EA58C4, - 0x11ABCABA, 0x655BAED3, 0xC01A3CAD, 0x2A34FCDE, 0x8F756EA0, 0xC3D4340C, 0x6695A672, - 0x8CBB6601, 0x29FAF47F, 0x5D0A9016, 0xF84B0268, 0x1265C21B, 0xB7245065, 0x6A638C57, - 0xCF221E29, 0x250CDE5A, 0x804D4C24, 0xF4BD284D, 0x51FCBA33, 0xBBD27A40, 0x1E93E83E, - 0x5232B292, 0xF77320EC, 0x1D5DE09F, 0xB81C72E1, 0xCCEC1688, 0x69AD84F6, 0x83834485, - 0x26C2D6FB, 0x1AC1F1DD, 0xBF8063A3, 0x55AEA3D0, 0xF0EF31AE, 0x841F55C7, 0x215EC7B9, - 0xCB7007CA, 0x6E3195B4, 0x2290CF18, 0x87D15D66, 0x6DFF9D15, 0xC8BE0F6B, 0xBC4E6B02, - 0x190FF97C, 0xF321390F, 0x5660AB71, 0x4C42F79A, 0xE90365E4, 0x032DA597, 0xA66C37E9, - 0xD29C5380, 0x77DDC1FE, 0x9DF3018D, 0x38B293F3, 0x7413C95F, 0xD1525B21, 0x3B7C9B52, - 0x9E3D092C, 0xEACD6D45, 0x4F8CFF3B, 0xA5A23F48, 0x00E3AD36, 0x3CE08A10, 0x99A1186E, - 0x738FD81D, 0xD6CE4A63, 0xA23E2E0A, 0x077FBC74, 0xED517C07, 0x4810EE79, 0x04B1B4D5, - 0xA1F026AB, 0x4BDEE6D8, 0xEE9F74A6, 0x9A6F10CF, 0x3F2E82B1, 0xD50042C2, 0x7041D0BC, - 0xAD060C8E, 0x08479EF0, 0xE2695E83, 0x4728CCFD, 0x33D8A894, 0x96993AEA, 0x7CB7FA99, - 0xD9F668E7, 0x9557324B, 0x3016A035, 0xDA386046, 0x7F79F238, 0x0B899651, 0xAEC8042F, - 0x44E6C45C, 0xE1A75622, 0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177, 0x437AD51E, - 0xE63B4760, 0x0C158713, 0xA954156D, 0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, - 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8}; - -static const unsigned int T8_3[256] = { - 0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, 0x7B2231F3, 0xA6679B4B, 0xC4451272, - 0x1900B8CA, 0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF, 0x8D665215, 0x5023F8AD, - 0x32017194, 0xEF44DB2C, 0xE964B13D, 0x34211B85, 0x560392BC, 0x8B463804, 0x924680CE, - 0x4F032A76, 0x2D21A34F, 0xF06409F7, 0x1F20D2DB, 0xC2657863, 0xA047F15A, 0x7D025BE2, - 0x6402E328, 0xB9474990, 0xDB65C0A9, 0x06206A11, 0xD725148B, 0x0A60BE33, 0x6842370A, - 0xB5079DB2, 0xAC072578, 0x71428FC0, 0x136006F9, 0xCE25AC41, 0x2161776D, 0xFC24DDD5, - 0x9E0654EC, 0x4343FE54, 0x5A43469E, 0x8706EC26, 0xE524651F, 0x3861CFA7, 0x3E41A5B6, - 0xE3040F0E, 0x81268637, 0x5C632C8F, 0x45639445, 0x98263EFD, 0xFA04B7C4, 0x27411D7C, - 0xC805C650, 0x15406CE8, 0x7762E5D1, 0xAA274F69, 0xB327F7A3, 0x6E625D1B, 0x0C40D422, - 0xD1057E9A, 0xABA65FE7, 0x76E3F55F, 0x14C17C66, 0xC984D6DE, 0xD0846E14, 0x0DC1C4AC, - 0x6FE34D95, 0xB2A6E72D, 0x5DE23C01, 0x80A796B9, 0xE2851F80, 0x3FC0B538, 0x26C00DF2, - 0xFB85A74A, 0x99A72E73, 0x44E284CB, 0x42C2EEDA, 0x9F874462, 0xFDA5CD5B, 0x20E067E3, - 0x39E0DF29, 0xE4A57591, 0x8687FCA8, 0x5BC25610, 0xB4868D3C, 0x69C32784, 0x0BE1AEBD, - 0xD6A40405, 0xCFA4BCCF, 0x12E11677, 0x70C39F4E, 0xAD8635F6, 0x7C834B6C, 0xA1C6E1D4, - 0xC3E468ED, 0x1EA1C255, 0x07A17A9F, 0xDAE4D027, 0xB8C6591E, 0x6583F3A6, 0x8AC7288A, - 0x57828232, 0x35A00B0B, 0xE8E5A1B3, 0xF1E51979, 0x2CA0B3C1, 0x4E823AF8, 0x93C79040, - 0x95E7FA51, 0x48A250E9, 0x2A80D9D0, 0xF7C57368, 0xEEC5CBA2, 0x3380611A, 0x51A2E823, - 0x8CE7429B, 0x63A399B7, 0xBEE6330F, 0xDCC4BA36, 0x0181108E, 0x1881A844, 0xC5C402FC, - 0xA7E68BC5, 0x7AA3217D, 0x52A0C93F, 0x8FE56387, 0xEDC7EABE, 0x30824006, 0x2982F8CC, - 0xF4C75274, 0x96E5DB4D, 0x4BA071F5, 0xA4E4AAD9, 0x79A10061, 0x1B838958, 0xC6C623E0, - 0xDFC69B2A, 0x02833192, 0x60A1B8AB, 0xBDE41213, 0xBBC47802, 0x6681D2BA, 0x04A35B83, - 0xD9E6F13B, 0xC0E649F1, 0x1DA3E349, 0x7F816A70, 0xA2C4C0C8, 0x4D801BE4, 0x90C5B15C, - 0xF2E73865, 0x2FA292DD, 0x36A22A17, 0xEBE780AF, 0x89C50996, 0x5480A32E, 0x8585DDB4, - 0x58C0770C, 0x3AE2FE35, 0xE7A7548D, 0xFEA7EC47, 0x23E246FF, 0x41C0CFC6, 0x9C85657E, - 0x73C1BE52, 0xAE8414EA, 0xCCA69DD3, 0x11E3376B, 0x08E38FA1, 0xD5A62519, 0xB784AC20, - 0x6AC10698, 0x6CE16C89, 0xB1A4C631, 0xD3864F08, 0x0EC3E5B0, 0x17C35D7A, 0xCA86F7C2, - 0xA8A47EFB, 0x75E1D443, 0x9AA50F6F, 0x47E0A5D7, 0x25C22CEE, 0xF8878656, 0xE1873E9C, - 0x3CC29424, 0x5EE01D1D, 0x83A5B7A5, 0xF90696D8, 0x24433C60, 0x4661B559, 0x9B241FE1, - 0x8224A72B, 0x5F610D93, 0x3D4384AA, 0xE0062E12, 0x0F42F53E, 0xD2075F86, 0xB025D6BF, - 0x6D607C07, 0x7460C4CD, 0xA9256E75, 0xCB07E74C, 0x16424DF4, 0x106227E5, 0xCD278D5D, - 0xAF050464, 0x7240AEDC, 0x6B401616, 0xB605BCAE, 0xD4273597, 0x09629F2F, 0xE6264403, - 0x3B63EEBB, 0x59416782, 0x8404CD3A, 0x9D0475F0, 0x4041DF48, 0x22635671, 0xFF26FCC9, - 0x2E238253, 0xF36628EB, 0x9144A1D2, 0x4C010B6A, 0x5501B3A0, 0x88441918, 0xEA669021, - 0x37233A99, 0xD867E1B5, 0x05224B0D, 0x6700C234, 0xBA45688C, 0xA345D046, 0x7E007AFE, - 0x1C22F3C7, 0xC167597F, 0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57, 0xBC65029D, - 0x6120A825, 0x0302211C, 0xDE478BA4, 0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, - 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842}; - -static const unsigned int T8_4[256] = { - 0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, 0xE045BEB0, 0xD854D11C, 0x906761E8, - 0xA8760E44, 0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65, 0x2522B521, 0x1D33DA8D, - 0x55006A79, 0x6D1105D5, 0x8F2261D3, 0xB7330E7F, 0xFF00BE8B, 0xC711D127, 0x6F67DF63, - 0x5776B0CF, 0x1F45003B, 0x27546F97, 0x4A456A42, 0x725405EE, 0x3A67B51A, 0x0276DAB6, - 0xAA00D4F2, 0x9211BB5E, 0xDA220BAA, 0xE2336406, 0x1BA8B557, 0x23B9DAFB, 0x6B8A6A0F, - 0x539B05A3, 0xFBED0BE7, 0xC3FC644B, 0x8BCFD4BF, 0xB3DEBB13, 0xDECFBEC6, 0xE6DED16A, - 0xAEED619E, 0x96FC0E32, 0x3E8A0076, 0x069B6FDA, 0x4EA8DF2E, 0x76B9B082, 0x948AD484, - 0xAC9BBB28, 0xE4A80BDC, 0xDCB96470, 0x74CF6A34, 0x4CDE0598, 0x04EDB56C, 0x3CFCDAC0, - 0x51EDDF15, 0x69FCB0B9, 0x21CF004D, 0x19DE6FE1, 0xB1A861A5, 0x89B90E09, 0xC18ABEFD, - 0xF99BD151, 0x37516AAE, 0x0F400502, 0x4773B5F6, 0x7F62DA5A, 0xD714D41E, 0xEF05BBB2, - 0xA7360B46, 0x9F2764EA, 0xF236613F, 0xCA270E93, 0x8214BE67, 0xBA05D1CB, 0x1273DF8F, - 0x2A62B023, 0x625100D7, 0x5A406F7B, 0xB8730B7D, 0x806264D1, 0xC851D425, 0xF040BB89, - 0x5836B5CD, 0x6027DA61, 0x28146A95, 0x10050539, 0x7D1400EC, 0x45056F40, 0x0D36DFB4, - 0x3527B018, 0x9D51BE5C, 0xA540D1F0, 0xED736104, 0xD5620EA8, 0x2CF9DFF9, 0x14E8B055, - 0x5CDB00A1, 0x64CA6F0D, 0xCCBC6149, 0xF4AD0EE5, 0xBC9EBE11, 0x848FD1BD, 0xE99ED468, - 0xD18FBBC4, 0x99BC0B30, 0xA1AD649C, 0x09DB6AD8, 0x31CA0574, 0x79F9B580, 0x41E8DA2C, - 0xA3DBBE2A, 0x9BCAD186, 0xD3F96172, 0xEBE80EDE, 0x439E009A, 0x7B8F6F36, 0x33BCDFC2, - 0x0BADB06E, 0x66BCB5BB, 0x5EADDA17, 0x169E6AE3, 0x2E8F054F, 0x86F90B0B, 0xBEE864A7, - 0xF6DBD453, 0xCECABBFF, 0x6EA2D55C, 0x56B3BAF0, 0x1E800A04, 0x269165A8, 0x8EE76BEC, - 0xB6F60440, 0xFEC5B4B4, 0xC6D4DB18, 0xABC5DECD, 0x93D4B161, 0xDBE70195, 0xE3F66E39, - 0x4B80607D, 0x73910FD1, 0x3BA2BF25, 0x03B3D089, 0xE180B48F, 0xD991DB23, 0x91A26BD7, - 0xA9B3047B, 0x01C50A3F, 0x39D46593, 0x71E7D567, 0x49F6BACB, 0x24E7BF1E, 0x1CF6D0B2, - 0x54C56046, 0x6CD40FEA, 0xC4A201AE, 0xFCB36E02, 0xB480DEF6, 0x8C91B15A, 0x750A600B, - 0x4D1B0FA7, 0x0528BF53, 0x3D39D0FF, 0x954FDEBB, 0xAD5EB117, 0xE56D01E3, 0xDD7C6E4F, - 0xB06D6B9A, 0x887C0436, 0xC04FB4C2, 0xF85EDB6E, 0x5028D52A, 0x6839BA86, 0x200A0A72, - 0x181B65DE, 0xFA2801D8, 0xC2396E74, 0x8A0ADE80, 0xB21BB12C, 0x1A6DBF68, 0x227CD0C4, - 0x6A4F6030, 0x525E0F9C, 0x3F4F0A49, 0x075E65E5, 0x4F6DD511, 0x777CBABD, 0xDF0AB4F9, - 0xE71BDB55, 0xAF286BA1, 0x9739040D, 0x59F3BFF2, 0x61E2D05E, 0x29D160AA, 0x11C00F06, - 0xB9B60142, 0x81A76EEE, 0xC994DE1A, 0xF185B1B6, 0x9C94B463, 0xA485DBCF, 0xECB66B3B, - 0xD4A70497, 0x7CD10AD3, 0x44C0657F, 0x0CF3D58B, 0x34E2BA27, 0xD6D1DE21, 0xEEC0B18D, - 0xA6F30179, 0x9EE26ED5, 0x36946091, 0x0E850F3D, 0x46B6BFC9, 0x7EA7D065, 0x13B6D5B0, - 0x2BA7BA1C, 0x63940AE8, 0x5B856544, 0xF3F36B00, 0xCBE204AC, 0x83D1B458, 0xBBC0DBF4, - 0x425B0AA5, 0x7A4A6509, 0x3279D5FD, 0x0A68BA51, 0xA21EB415, 0x9A0FDBB9, 0xD23C6B4D, - 0xEA2D04E1, 0x873C0134, 0xBF2D6E98, 0xF71EDE6C, 0xCF0FB1C0, 0x6779BF84, 0x5F68D028, - 0x175B60DC, 0x2F4A0F70, 0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82, 0x2D3CD5C6, - 0x152DBA6A, 0x5D1E0A9E, 0x650F6532, 0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, - 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3}; - -static const unsigned int T8_5[256] = { - 0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, 0xB2F53777, 0x5DC55C6E, 0x697997B4, - 0x8649FCAD, 0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, 0xD2F32F68, 0x3DC34471, - 0x097F8FAB, 0xE64FE4B2, 0xC00C303E, 0x2F3C5B27, 0x1B8090FD, 0xF4B0FBE4, 0x72F90749, - 0x9DC96C50, 0xA975A78A, 0x4645CC93, 0xA00A2821, 0x4F3A4338, 0x7B8688E2, 0x94B6E3FB, - 0x12FF1F56, 0xFDCF744F, 0xC973BF95, 0x2643D48C, 0x85F4168D, 0x6AC47D94, 0x5E78B64E, - 0xB148DD57, 0x370121FA, 0xD8314AE3, 0xEC8D8139, 0x03BDEA20, 0xE5F20E92, 0x0AC2658B, - 0x3E7EAE51, 0xD14EC548, 0x570739E5, 0xB83752FC, 0x8C8B9926, 0x63BBF23F, 0x45F826B3, - 0xAAC84DAA, 0x9E748670, 0x7144ED69, 0xF70D11C4, 0x183D7ADD, 0x2C81B107, 0xC3B1DA1E, - 0x25FE3EAC, 0xCACE55B5, 0xFE729E6F, 0x1142F576, 0x970B09DB, 0x783B62C2, 0x4C87A918, - 0xA3B7C201, 0x0E045BEB, 0xE13430F2, 0xD588FB28, 0x3AB89031, 0xBCF16C9C, 0x53C10785, - 0x677DCC5F, 0x884DA746, 0x6E0243F4, 0x813228ED, 0xB58EE337, 0x5ABE882E, 0xDCF77483, - 0x33C71F9A, 0x077BD440, 0xE84BBF59, 0xCE086BD5, 0x213800CC, 0x1584CB16, 0xFAB4A00F, - 0x7CFD5CA2, 0x93CD37BB, 0xA771FC61, 0x48419778, 0xAE0E73CA, 0x413E18D3, 0x7582D309, - 0x9AB2B810, 0x1CFB44BD, 0xF3CB2FA4, 0xC777E47E, 0x28478F67, 0x8BF04D66, 0x64C0267F, - 0x507CEDA5, 0xBF4C86BC, 0x39057A11, 0xD6351108, 0xE289DAD2, 0x0DB9B1CB, 0xEBF65579, - 0x04C63E60, 0x307AF5BA, 0xDF4A9EA3, 0x5903620E, 0xB6330917, 0x828FC2CD, 0x6DBFA9D4, - 0x4BFC7D58, 0xA4CC1641, 0x9070DD9B, 0x7F40B682, 0xF9094A2F, 0x16392136, 0x2285EAEC, - 0xCDB581F5, 0x2BFA6547, 0xC4CA0E5E, 0xF076C584, 0x1F46AE9D, 0x990F5230, 0x763F3929, - 0x4283F2F3, 0xADB399EA, 0x1C08B7D6, 0xF338DCCF, 0xC7841715, 0x28B47C0C, 0xAEFD80A1, - 0x41CDEBB8, 0x75712062, 0x9A414B7B, 0x7C0EAFC9, 0x933EC4D0, 0xA7820F0A, 0x48B26413, - 0xCEFB98BE, 0x21CBF3A7, 0x1577387D, 0xFA475364, 0xDC0487E8, 0x3334ECF1, 0x0788272B, - 0xE8B84C32, 0x6EF1B09F, 0x81C1DB86, 0xB57D105C, 0x5A4D7B45, 0xBC029FF7, 0x5332F4EE, - 0x678E3F34, 0x88BE542D, 0x0EF7A880, 0xE1C7C399, 0xD57B0843, 0x3A4B635A, 0x99FCA15B, - 0x76CCCA42, 0x42700198, 0xAD406A81, 0x2B09962C, 0xC439FD35, 0xF08536EF, 0x1FB55DF6, - 0xF9FAB944, 0x16CAD25D, 0x22761987, 0xCD46729E, 0x4B0F8E33, 0xA43FE52A, 0x90832EF0, - 0x7FB345E9, 0x59F09165, 0xB6C0FA7C, 0x827C31A6, 0x6D4C5ABF, 0xEB05A612, 0x0435CD0B, - 0x308906D1, 0xDFB96DC8, 0x39F6897A, 0xD6C6E263, 0xE27A29B9, 0x0D4A42A0, 0x8B03BE0D, - 0x6433D514, 0x508F1ECE, 0xBFBF75D7, 0x120CEC3D, 0xFD3C8724, 0xC9804CFE, 0x26B027E7, - 0xA0F9DB4A, 0x4FC9B053, 0x7B757B89, 0x94451090, 0x720AF422, 0x9D3A9F3B, 0xA98654E1, - 0x46B63FF8, 0xC0FFC355, 0x2FCFA84C, 0x1B736396, 0xF443088F, 0xD200DC03, 0x3D30B71A, - 0x098C7CC0, 0xE6BC17D9, 0x60F5EB74, 0x8FC5806D, 0xBB794BB7, 0x544920AE, 0xB206C41C, - 0x5D36AF05, 0x698A64DF, 0x86BA0FC6, 0x00F3F36B, 0xEFC39872, 0xDB7F53A8, 0x344F38B1, - 0x97F8FAB0, 0x78C891A9, 0x4C745A73, 0xA344316A, 0x250DCDC7, 0xCA3DA6DE, 0xFE816D04, - 0x11B1061D, 0xF7FEE2AF, 0x18CE89B6, 0x2C72426C, 0xC3422975, 0x450BD5D8, 0xAA3BBEC1, - 0x9E87751B, 0x71B71E02, 0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154, 0xE501FDF9, - 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623, 0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, - 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C}; - -static const unsigned int T8_6[256] = { - 0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, 0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, - 0x1DE5B089, 0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B, 0xEBCD3882, 0x83CE144A, - 0x3BCB6112, 0x53C84DDA, 0x9C5BFAA6, 0xF458D66E, 0x4C5DA336, 0x245E8FFE, 0x39BB3F77, - 0x51B813BF, 0xE9BD66E7, 0x81BE4A2F, 0xD27607F5, 0xBA752B3D, 0x02705E65, 0x6A7372AD, - 0x7796C224, 0x1F95EEEC, 0xA7909BB4, 0xCF93B77C, 0x3D5B83BD, 0x5558AF75, 0xED5DDA2D, - 0x855EF6E5, 0x98BB466C, 0xF0B86AA4, 0x48BD1FFC, 0x20BE3334, 0x73767EEE, 0x1B755226, - 0xA370277E, 0xCB730BB6, 0xD696BB3F, 0xBE9597F7, 0x0690E2AF, 0x6E93CE67, 0xA100791B, - 0xC90355D3, 0x7106208B, 0x19050C43, 0x04E0BCCA, 0x6CE39002, 0xD4E6E55A, 0xBCE5C992, - 0xEF2D8448, 0x872EA880, 0x3F2BDDD8, 0x5728F110, 0x4ACD4199, 0x22CE6D51, 0x9ACB1809, - 0xF2C834C1, 0x7AB7077A, 0x12B42BB2, 0xAAB15EEA, 0xC2B27222, 0xDF57C2AB, 0xB754EE63, - 0x0F519B3B, 0x6752B7F3, 0x349AFA29, 0x5C99D6E1, 0xE49CA3B9, 0x8C9F8F71, 0x917A3FF8, - 0xF9791330, 0x417C6668, 0x297F4AA0, 0xE6ECFDDC, 0x8EEFD114, 0x36EAA44C, 0x5EE98884, - 0x430C380D, 0x2B0F14C5, 0x930A619D, 0xFB094D55, 0xA8C1008F, 0xC0C22C47, 0x78C7591F, - 0x10C475D7, 0x0D21C55E, 0x6522E996, 0xDD279CCE, 0xB524B006, 0x47EC84C7, 0x2FEFA80F, - 0x97EADD57, 0xFFE9F19F, 0xE20C4116, 0x8A0F6DDE, 0x320A1886, 0x5A09344E, 0x09C17994, - 0x61C2555C, 0xD9C72004, 0xB1C40CCC, 0xAC21BC45, 0xC422908D, 0x7C27E5D5, 0x1424C91D, - 0xDBB77E61, 0xB3B452A9, 0x0BB127F1, 0x63B20B39, 0x7E57BBB0, 0x16549778, 0xAE51E220, - 0xC652CEE8, 0x959A8332, 0xFD99AFFA, 0x459CDAA2, 0x2D9FF66A, 0x307A46E3, 0x58796A2B, - 0xE07C1F73, 0x887F33BB, 0xF56E0EF4, 0x9D6D223C, 0x25685764, 0x4D6B7BAC, 0x508ECB25, - 0x388DE7ED, 0x808892B5, 0xE88BBE7D, 0xBB43F3A7, 0xD340DF6F, 0x6B45AA37, 0x034686FF, - 0x1EA33676, 0x76A01ABE, 0xCEA56FE6, 0xA6A6432E, 0x6935F452, 0x0136D89A, 0xB933ADC2, - 0xD130810A, 0xCCD53183, 0xA4D61D4B, 0x1CD36813, 0x74D044DB, 0x27180901, 0x4F1B25C9, - 0xF71E5091, 0x9F1D7C59, 0x82F8CCD0, 0xEAFBE018, 0x52FE9540, 0x3AFDB988, 0xC8358D49, - 0xA036A181, 0x1833D4D9, 0x7030F811, 0x6DD54898, 0x05D66450, 0xBDD31108, 0xD5D03DC0, - 0x8618701A, 0xEE1B5CD2, 0x561E298A, 0x3E1D0542, 0x23F8B5CB, 0x4BFB9903, 0xF3FEEC5B, - 0x9BFDC093, 0x546E77EF, 0x3C6D5B27, 0x84682E7F, 0xEC6B02B7, 0xF18EB23E, 0x998D9EF6, - 0x2188EBAE, 0x498BC766, 0x1A438ABC, 0x7240A674, 0xCA45D32C, 0xA246FFE4, 0xBFA34F6D, - 0xD7A063A5, 0x6FA516FD, 0x07A63A35, 0x8FD9098E, 0xE7DA2546, 0x5FDF501E, 0x37DC7CD6, - 0x2A39CC5F, 0x423AE097, 0xFA3F95CF, 0x923CB907, 0xC1F4F4DD, 0xA9F7D815, 0x11F2AD4D, - 0x79F18185, 0x6414310C, 0x0C171DC4, 0xB412689C, 0xDC114454, 0x1382F328, 0x7B81DFE0, - 0xC384AAB8, 0xAB878670, 0xB66236F9, 0xDE611A31, 0x66646F69, 0x0E6743A1, 0x5DAF0E7B, - 0x35AC22B3, 0x8DA957EB, 0xE5AA7B23, 0xF84FCBAA, 0x904CE762, 0x2849923A, 0x404ABEF2, - 0xB2828A33, 0xDA81A6FB, 0x6284D3A3, 0x0A87FF6B, 0x17624FE2, 0x7F61632A, 0xC7641672, - 0xAF673ABA, 0xFCAF7760, 0x94AC5BA8, 0x2CA92EF0, 0x44AA0238, 0x594FB2B1, 0x314C9E79, - 0x8949EB21, 0xE14AC7E9, 0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD, 0x8B39B544, - 0xE33A998C, 0x5B3FECD4, 0x333CC01C, 0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, - 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F}; - -static const unsigned int T8_7[256] = { - 0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, 0x211D826D, 0x6821FF4A, 0xB3657823, - 0xFA590504, 0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3, 0x632686B7, 0x2A1AFB90, - 0xF15E7CF9, 0xB86201DE, 0x847609B4, 0xCD4A7493, 0x160EF3FA, 0x5F328EDD, 0xA56B8BD9, - 0xEC57F6FE, 0x37137197, 0x7E2F0CB0, 0xC64D0D6E, 0x8F717049, 0x5435F720, 0x1D098A07, - 0xE7508F03, 0xAE6CF224, 0x7528754D, 0x3C14086A, 0x0D006599, 0x443C18BE, 0x9F789FD7, - 0xD644E2F0, 0x2C1DE7F4, 0x65219AD3, 0xBE651DBA, 0xF759609D, 0x4F3B6143, 0x06071C64, - 0xDD439B0D, 0x947FE62A, 0x6E26E32E, 0x271A9E09, 0xFC5E1960, 0xB5626447, 0x89766C2D, - 0xC04A110A, 0x1B0E9663, 0x5232EB44, 0xA86BEE40, 0xE1579367, 0x3A13140E, 0x732F6929, - 0xCB4D68F7, 0x827115D0, 0x593592B9, 0x1009EF9E, 0xEA50EA9A, 0xA36C97BD, 0x782810D4, - 0x31146DF3, 0x1A00CB32, 0x533CB615, 0x8878317C, 0xC1444C5B, 0x3B1D495F, 0x72213478, - 0xA965B311, 0xE059CE36, 0x583BCFE8, 0x1107B2CF, 0xCA4335A6, 0x837F4881, 0x79264D85, - 0x301A30A2, 0xEB5EB7CB, 0xA262CAEC, 0x9E76C286, 0xD74ABFA1, 0x0C0E38C8, 0x453245EF, - 0xBF6B40EB, 0xF6573DCC, 0x2D13BAA5, 0x642FC782, 0xDC4DC65C, 0x9571BB7B, 0x4E353C12, - 0x07094135, 0xFD504431, 0xB46C3916, 0x6F28BE7F, 0x2614C358, 0x1700AEAB, 0x5E3CD38C, - 0x857854E5, 0xCC4429C2, 0x361D2CC6, 0x7F2151E1, 0xA465D688, 0xED59ABAF, 0x553BAA71, - 0x1C07D756, 0xC743503F, 0x8E7F2D18, 0x7426281C, 0x3D1A553B, 0xE65ED252, 0xAF62AF75, - 0x9376A71F, 0xDA4ADA38, 0x010E5D51, 0x48322076, 0xB26B2572, 0xFB575855, 0x2013DF3C, - 0x692FA21B, 0xD14DA3C5, 0x9871DEE2, 0x4335598B, 0x0A0924AC, 0xF05021A8, 0xB96C5C8F, - 0x6228DBE6, 0x2B14A6C1, 0x34019664, 0x7D3DEB43, 0xA6796C2A, 0xEF45110D, 0x151C1409, - 0x5C20692E, 0x8764EE47, 0xCE589360, 0x763A92BE, 0x3F06EF99, 0xE44268F0, 0xAD7E15D7, - 0x572710D3, 0x1E1B6DF4, 0xC55FEA9D, 0x8C6397BA, 0xB0779FD0, 0xF94BE2F7, 0x220F659E, - 0x6B3318B9, 0x916A1DBD, 0xD856609A, 0x0312E7F3, 0x4A2E9AD4, 0xF24C9B0A, 0xBB70E62D, - 0x60346144, 0x29081C63, 0xD3511967, 0x9A6D6440, 0x4129E329, 0x08159E0E, 0x3901F3FD, - 0x703D8EDA, 0xAB7909B3, 0xE2457494, 0x181C7190, 0x51200CB7, 0x8A648BDE, 0xC358F6F9, - 0x7B3AF727, 0x32068A00, 0xE9420D69, 0xA07E704E, 0x5A27754A, 0x131B086D, 0xC85F8F04, - 0x8163F223, 0xBD77FA49, 0xF44B876E, 0x2F0F0007, 0x66337D20, 0x9C6A7824, 0xD5560503, - 0x0E12826A, 0x472EFF4D, 0xFF4CFE93, 0xB67083B4, 0x6D3404DD, 0x240879FA, 0xDE517CFE, - 0x976D01D9, 0x4C2986B0, 0x0515FB97, 0x2E015D56, 0x673D2071, 0xBC79A718, 0xF545DA3F, - 0x0F1CDF3B, 0x4620A21C, 0x9D642575, 0xD4585852, 0x6C3A598C, 0x250624AB, 0xFE42A3C2, - 0xB77EDEE5, 0x4D27DBE1, 0x041BA6C6, 0xDF5F21AF, 0x96635C88, 0xAA7754E2, 0xE34B29C5, - 0x380FAEAC, 0x7133D38B, 0x8B6AD68F, 0xC256ABA8, 0x19122CC1, 0x502E51E6, 0xE84C5038, - 0xA1702D1F, 0x7A34AA76, 0x3308D751, 0xC951D255, 0x806DAF72, 0x5B29281B, 0x1215553C, - 0x230138CF, 0x6A3D45E8, 0xB179C281, 0xF845BFA6, 0x021CBAA2, 0x4B20C785, 0x906440EC, - 0xD9583DCB, 0x613A3C15, 0x28064132, 0xF342C65B, 0xBA7EBB7C, 0x4027BE78, 0x091BC35F, - 0xD25F4436, 0x9B633911, 0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612, 0x866AB316, - 0xCF56CE31, 0x14124958, 0x5D2E347F, 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, - 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5}; - Status gen_timestamp_string(std::string* out_string) { time_t now = time(nullptr); tm local_tm; diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp index ef10e3b5f7bae4..dcebad6495c925 100644 --- a/be/src/vec/columns/column_object.cpp +++ b/be/src/vec/columns/column_object.cpp @@ -1071,6 +1071,8 @@ void ColumnObject::insert_range_from(const IColumn& src, size_t start, size_t le if (entry->path.has_nested_part()) { FieldInfo field_info { .scalar_type_id = entry->data.least_common_type.get_base_type_id(), + .have_nulls = false, + .need_convert = false, .num_dimensions = entry->data.get_dimensions()}; add_nested_subcolumn(entry->path, field_info, num_rows); } else { @@ -1985,6 +1987,8 @@ bool ColumnObject::try_insert_many_defaults_from_nested(const Subcolumns::NodePt size_t old_size = entry->data.size(); FieldInfo field_info = { .scalar_type_id = entry->data.least_common_type.get_base_type_id(), + .have_nulls = false, + .need_convert = false, .num_dimensions = entry->data.get_dimensions(), }; diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index 761ddf3a5f9a23..6b6639f2feb244 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -2022,9 +2022,9 @@ bool OrcReader::_can_filter_by_dict(int slot_id) { // the implementation of NULL values because the dictionary itself does not contain // NULL value encoding. As a result, many NULL-related functions or expressions // cannot work properly, such as is null, is not null, coalesce, etc. - // Here we first disable dictionary filtering when predicate contains functions. + // Here we first disable dictionary filtering when predicate expr is not slot. // Implementation of NULL value dictionary filtering will be carried out later. - if (expr->node_type() == TExprNodeType::FUNCTION_CALL) { + if (expr->node_type() != TExprNodeType::SLOT_REF) { return false; } for (auto& child : expr->children()) { diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp index 37e82774c39ee4..b9259be936bb31 100644 --- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp @@ -212,9 +212,9 @@ bool RowGroupReader::_can_filter_by_dict(int slot_id, // the implementation of NULL values because the dictionary itself does not contain // NULL value encoding. As a result, many NULL-related functions or expressions // cannot work properly, such as is null, is not null, coalesce, etc. - // Here we first disable dictionary filtering when predicate contains functions. + // Here we first disable dictionary filtering when predicate is not slot. // Implementation of NULL value dictionary filtering will be carried out later. - if (expr->node_type() == TExprNodeType::FUNCTION_CALL) { + if (expr->node_type() != TExprNodeType::SLOT_REF) { return false; } for (auto& child : expr->children()) { diff --git a/be/src/vec/functions/function_time_value_to_field.cpp b/be/src/vec/functions/function_time_value_to_field.cpp new file mode 100644 index 00000000000000..da5fbca58cdbb3 --- /dev/null +++ b/be/src/vec/functions/function_time_value_to_field.cpp @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "common/status.h" +#include "vec/data_types/data_type_number.h" +#include "vec/data_types/data_type_time.h" +#include "vec/functions/function.h" +#include "vec/functions/simple_function_factory.h" +#include "vec/runtime/time_value.h" +#include "vec/utils/template_helpers.hpp" +namespace doris::vectorized { + +template +class FunctionTimeValueToField : public IFunction { +public: + static constexpr auto name = Transform::name; + static FunctionPtr create() { return std::make_shared(); } + String get_name() const override { return name; } + + // is_variadic and get_number_of_arguments are consistent with FunctionDateOrDateTimeToSomething, + // as FunctionDateOrDateTimeToSomething supports Date and DateTime types as arguments. + bool is_variadic() const override { return true; } + size_t get_number_of_arguments() const override { return 0; } + + DataTypes get_variadic_argument_types_impl() const override { + return {std::make_shared()}; + } + + DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override { + return std::make_shared(); + } + + bool use_default_implementation_for_nulls() const override { return true; } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + size_t result, size_t input_rows_count) const override { + DCHECK_EQ(arguments.size(), 1); + + const auto* column_time = assert_cast( + block.get_by_position(arguments[0]).column.get()); + + auto col_res = ToDataType::ColumnType::create(); + + col_res->resize(input_rows_count); + auto& col_res_data = col_res->get_data(); + + for (size_t i = 0; i < input_rows_count; i++) { + col_res_data[i] = Transform::execute(column_time->get_element(i)); + } + + block.replace_by_position(result, std::move(col_res)); + return Status::OK(); + } +}; + +struct HourImpl { + constexpr static auto name = "hour"; + static inline auto execute(const TimeValue::TimeType& t) { return TimeValue::hour(t); } +}; + +struct MintuImpl { + constexpr static auto name = "minute"; + static inline auto execute(const TimeValue::TimeType& t) { return TimeValue::minute(t); } +}; + +struct SecondImpl { + constexpr static auto name = "second"; + static inline auto execute(const TimeValue::TimeType& t) { return TimeValue::second(t); } +}; + +void register_function_time_value_field(SimpleFunctionFactory& factory) { + factory.register_function>(); + factory.register_function>(); + factory.register_function>(); +} + +} // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/simple_function_factory.h b/be/src/vec/functions/simple_function_factory.h index d8b544d5bfdabb..e8e54bf84a8faa 100644 --- a/be/src/vec/functions/simple_function_factory.h +++ b/be/src/vec/functions/simple_function_factory.h @@ -58,6 +58,7 @@ void register_function_is_null(SimpleFunctionFactory& factory); void register_function_is_not_null(SimpleFunctionFactory& factory); void register_function_nullables(SimpleFunctionFactory& factory); void register_function_to_time_function(SimpleFunctionFactory& factory); +void register_function_time_value_field(SimpleFunctionFactory& factory); void register_function_time_of_function(SimpleFunctionFactory& factory); void register_function_string(SimpleFunctionFactory& factory); void register_function_running_difference(SimpleFunctionFactory& factory); @@ -239,6 +240,7 @@ class SimpleFunctionFactory { register_function_is_not_null(instance); register_function_nullables(instance); register_function_to_time_function(instance); + register_function_time_value_field(instance); register_function_time_of_function(instance); register_function_string(instance); register_function_in(instance); diff --git a/be/src/vec/runtime/time_value.h b/be/src/vec/runtime/time_value.h index 8283e12f8afff0..d94e62b977c44d 100644 --- a/be/src/vec/runtime/time_value.h +++ b/be/src/vec/runtime/time_value.h @@ -22,17 +22,51 @@ #include "runtime/define_primitive_type.h" #include "runtime/primitive_type.h" #include "util/date_func.h" +#include "vec/data_types/data_type_time.h" namespace doris { /// TODO: Due to the "Time type is not supported for OLAP table" issue, a lot of basic content is missing.It will be supplemented later. class TimeValue { +public: + constexpr static int64_t ONE_SECOND_MICROSECONDS = 1000000; + constexpr static int64_t ONE_MINUTE_MICROSECONDS = 60 * ONE_SECOND_MICROSECONDS; + constexpr static int64_t ONE_HOUR_MICROSECONDS = 60 * ONE_MINUTE_MICROSECONDS; + constexpr static int64_t ONE_MINUTE_SECONDS = 60; + constexpr static int64_t ONE_HOUR_SECONDS = 60 * ONE_MINUTE_SECONDS; + using TimeType = typename PrimitiveTypeTraits::CppType; + using ColumnTime = vectorized::DataTypeTimeV2::ColumnType; + + // refer to https://dev.mysql.com/doc/refman/5.7/en/time.html + // the time value between '-838:59:59' and '838:59:59' + /// TODO: Why is the time type stored as double? Can we directly use int64 and remove the time limit? + static int64_t check_over_max_time(double time) { + const static int64_t max_time = 3020399LL * 1000 * 1000; + // cast(-4562632 as time) + // -456:26:32 + // hour(cast(-4562632 as time)) + // 456 + // second(cast(-4562632 as time)) + // 32 + if (time > max_time || time < -max_time) { + return max_time; + } + return static_cast(time); + } -public: static std::string to_string(TimeType time, int scale) { return timev2_to_buffer_from_double(time, scale); } + static int32_t hour(TimeType time) { return check_over_max_time(time) / ONE_HOUR_MICROSECONDS; } + + static int32_t minute(TimeType time) { + return (check_over_max_time(time) % ONE_HOUR_MICROSECONDS) / ONE_MINUTE_MICROSECONDS; + } + + static int32_t second(TimeType time) { + return (check_over_max_time(time) / ONE_SECOND_MICROSECONDS) % ONE_MINUTE_SECONDS; + } }; } // namespace doris diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run25.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run25.hql index 2cf7197de95775..814df4cdc5ff90 100755 --- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run25.hql +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run25.hql @@ -1,4 +1,3 @@ --- Currently docker is hive 2.x version. Hive 2.x versioned full-acid tables need to run major compaction. SET hive.support.concurrency=true; SET hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; @@ -25,8 +24,6 @@ insert into orc_full_acid values update orc_full_acid set value = 'CC' where id = 3; -alter table orc_full_acid compact 'major'; - create table orc_full_acid_par (id INT, value STRING) PARTITIONED BY (part_col INT) CLUSTERED BY (id) INTO 3 BUCKETS @@ -44,7 +41,3 @@ insert into orc_full_acid_par PARTITION(part_col=20230102) values (6, 'F'); update orc_full_acid_par set value = 'BB' where id = 2; - -alter table orc_full_acid_par PARTITION(part_col=20230101) compact 'major'; -alter table orc_full_acid_par PARTITION(part_col=20230102) compact 'major'; - diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 7d9ffb09690144..b5c2bace7ae2e1 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3138,7 +3138,7 @@ public static int metaServiceRpcRetryTimes() { // start of lock config @ConfField(description = {"是否开启死锁检测", "Whether to enable deadlock detection"}) - public static boolean enable_deadlock_detection = false; + public static boolean enable_deadlock_detection = true; @ConfField(description = {"死锁检测间隔时间，单位分钟", "Deadlock detection interval time, unit minute"}) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index f6d46db5ee53e8..11a60360426bf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -92,7 +92,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf, // table family group map private final Map idToTable; private ConcurrentMap nameToTable; - // table name lower cast -> table name + // table name lower case -> table name private final Map lowerCaseToTableName; // user define function diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index 98cd82902912d0..d98bba5edaca84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -324,7 +324,7 @@ public boolean tryCommitLock(long timeout, TimeUnit unit) { boolean res = this.commitLock.tryLock(timeout, unit); if (!res && unit.toSeconds(timeout) >= 1) { LOG.warn("Failed to try table {}'s cloud commit lock. timeout {} {}. Current owner: {}", - name, timeout, unit.name(), rwLock.getOwner()); + name, timeout, unit.name(), this.commitLock.getOwner()); } return res; } catch (InterruptedException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 8e1a05c9de5417..238250ab37a398 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -344,8 +344,13 @@ public static DataProperty analyzeDataProperty(Map properties, f throw new AnalysisException("Invalid storage medium: " + value); } } else if (key.equalsIgnoreCase(PROPERTIES_STORAGE_COOLDOWN_TIME)) { - DateLiteral dateLiteral = new DateLiteral(value, ScalarType.getDefaultDateType(Type.DATETIME)); - cooldownTimestamp = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); + try { + DateLiteral dateLiteral = new DateLiteral(value, ScalarType.getDefaultDateType(Type.DATETIME)); + cooldownTimestamp = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); + } catch (AnalysisException e) { + LOG.warn("dateLiteral failed, use max cool down time", e); + cooldownTimestamp = DataProperty.MAX_COOLDOWN_TIME_MS; + } } else if (key.equalsIgnoreCase(PROPERTIES_STORAGE_POLICY)) { hasStoragePolicy = true; newStoragePolicy = value; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index d653a5a178e484..eda98efb9b6a03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -59,6 +59,7 @@ import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * Base class of external database. @@ -83,6 +84,8 @@ public abstract class ExternalDatabase protected Map tableNameToId = Maps.newConcurrentMap(); @SerializedName(value = "idToTbl") protected Map idToTbl = Maps.newConcurrentMap(); + // table name lower case -> table name + private Map lowerCaseToTableName = Maps.newConcurrentMap(); @SerializedName(value = "lastUpdateTime") protected long lastUpdateTime; protected final InitDatabaseLog.Type dbLogType; @@ -239,7 +242,14 @@ private List listTableNames() { } else if (name.equals(MysqlDb.DATABASE_NAME)) { tableNames = ExternalMysqlDatabase.listTableNames(); } else { - tableNames = extCatalog.listTableNames(null, name); + tableNames = extCatalog.listTableNames(null, name).stream().map(tableName -> { + lowerCaseToTableName.put(tableName.toLowerCase(), tableName); + if (Env.isStoredTableNamesLowerCase()) { + return tableName.toLowerCase(); + } else { + return tableName; + } + }).collect(Collectors.toList()); } return tableNames; } @@ -325,6 +335,12 @@ public DatabaseProperty getDbProperties() { @Override public boolean isTableExist(String tableName) { + if (Env.isTableNamesCaseInsensitive()) { + tableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (tableName == null) { + return false; + } + } return extCatalog.tableExist(ConnectContext.get().getSessionContext(), name, tableName); } @@ -375,6 +391,15 @@ public Set getTableNamesWithLock() { @Override public T getTableNullable(String tableName) { makeSureInitialized(); + if (Env.isStoredTableNamesLowerCase()) { + tableName = tableName.toLowerCase(); + } + if (Env.isTableNamesCaseInsensitive()) { + tableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (tableName == null) { + return null; + } + } if (extCatalog.getUseMetaCache().get()) { // must use full qualified name to generate id. // otherwise, if 2 databases have the same table name, the id will be the same. @@ -419,6 +444,7 @@ public static ExternalDatabase read(DataInput in) throws IOException { @Override public void gsonPostProcess() throws IOException { tableNameToId = Maps.newConcurrentMap(); + lowerCaseToTableName = Maps.newConcurrentMap(); Map tmpIdToTbl = Maps.newConcurrentMap(); for (Object obj : idToTbl.values()) { if (obj instanceof LinkedTreeMap) { @@ -443,6 +469,8 @@ public void gsonPostProcess() throws IOException { Preconditions.checkState(obj instanceof ExternalTable); tmpIdToTbl.put(((ExternalTable) obj).getId(), (T) obj); tableNameToId.put(((ExternalTable) obj).getName(), ((ExternalTable) obj).getId()); + lowerCaseToTableName.put(((ExternalTable) obj).getName().toLowerCase(), + ((ExternalTable) obj).getName()); } } idToTbl = tmpIdToTbl; @@ -452,6 +480,9 @@ public void gsonPostProcess() throws IOException { @Override public void unregisterTable(String tableName) { makeSureInitialized(); + if (Env.isStoredTableNamesLowerCase()) { + tableName = tableName.toLowerCase(); + } if (LOG.isDebugEnabled()) { LOG.debug("create table [{}]", tableName); } @@ -459,6 +490,7 @@ public void unregisterTable(String tableName) { if (extCatalog.getUseMetaCache().get()) { if (isInitialized()) { metaCache.invalidate(tableName, Util.genIdByName(getQualifiedName(tableName))); + lowerCaseToTableName.remove(tableName.toLowerCase()); } } else { Long tableId = tableNameToId.remove(tableName); @@ -467,6 +499,7 @@ public void unregisterTable(String tableName) { return; } idToTbl.remove(tableId); + lowerCaseToTableName.remove(tableName.toLowerCase()); } setLastUpdateTime(System.currentTimeMillis()); Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache( @@ -490,11 +523,13 @@ public boolean registerTable(TableIf tableIf) { if (extCatalog.getUseMetaCache().get()) { if (isInitialized()) { metaCache.updateCache(tableName, (T) tableIf, Util.genIdByName(getQualifiedName(tableName))); + lowerCaseToTableName.put(tableName.toLowerCase(), tableName); } } else { if (!tableNameToId.containsKey(tableName)) { tableNameToId.put(tableName, tableId); idToTbl.put(tableId, buildTableForInit(tableName, tableId, extCatalog)); + lowerCaseToTableName.put(tableName.toLowerCase(), tableName); } } setLastUpdateTime(System.currentTimeMillis()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index fc275c2871ee9a..fbfd7dd2798668 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -744,7 +744,7 @@ public LoadingCache getPartitionCache() { } public List getFilesByTransaction(List partitions, ValidWriteIdList validWriteIds, - boolean isFullAcid, long tableId, String bindBrokerName) { + boolean isFullAcid, boolean skipCheckingAcidVersionFile, long tableId, String bindBrokerName) { List fileCacheValues = Lists.newArrayList(); String remoteUser = jobConf.get(AuthenticationConfig.HADOOP_USER_NAME); try { @@ -778,25 +778,27 @@ public List getFilesByTransaction(List partitions if (baseOrDeltaPath == null) { return Collections.emptyList(); } - String acidVersionPath = new Path(baseOrDeltaPath, "_orc_acid_version").toUri().toString(); - RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem( - new FileSystemCache.FileSystemCacheKey( - LocationPath.getFSIdentity(baseOrDeltaPath.toUri().toString(), - bindBrokerName), - catalog.getCatalogProperty().getProperties(), - bindBrokerName, jobConf)); - Status status = fs.exists(acidVersionPath); - if (status != Status.OK) { - if (status.getErrCode() == ErrCode.NOT_FOUND) { - acidVersion = 0; - } else { - throw new Exception(String.format("Failed to check remote path {} exists.", - acidVersionPath)); + if (!skipCheckingAcidVersionFile) { + String acidVersionPath = new Path(baseOrDeltaPath, "_orc_acid_version").toUri().toString(); + RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem( + new FileSystemCache.FileSystemCacheKey( + LocationPath.getFSIdentity(baseOrDeltaPath.toUri().toString(), + bindBrokerName), + catalog.getCatalogProperty().getProperties(), + bindBrokerName, jobConf)); + Status status = fs.exists(acidVersionPath); + if (status != Status.OK) { + if (status.getErrCode() == ErrCode.NOT_FOUND) { + acidVersion = 0; + } else { + throw new Exception(String.format("Failed to check remote path {} exists.", + acidVersionPath)); + } + } + if (acidVersion == 0 && !directory.getCurrentDirectories().isEmpty()) { + throw new Exception( + "Hive 2.x versioned full-acid tables need to run major compaction."); } - } - if (acidVersion == 0 && !directory.getCurrentDirectories().isEmpty()) { - throw new Exception( - "Hive 2.x versioned full-acid tables need to run major compaction."); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index 1dbcd5064f30ef..f17de4bfe0a116 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -90,6 +90,8 @@ public class HiveScanNode extends FileQueryScanNode { private final Semaphore splittersOnFlight = new Semaphore(NUM_SPLITTERS_ON_FLIGHT); private final AtomicInteger numSplitsPerPartition = new AtomicInteger(NUM_SPLITS_PER_PARTITION); + private boolean skipCheckingAcidVersionFile = false; + /** * * External file scan node for Query Hive table * needCheckColumnPriv: Some of ExternalFileScanNode do not need to check column priv @@ -117,6 +119,7 @@ protected void doInitialize() throws UserException { this.hiveTransaction = new HiveTransaction(DebugUtil.printId(ConnectContext.get().queryId()), ConnectContext.get().getQualifiedUser(), hmsTable, hmsTable.isFullAcidTable()); Env.getCurrentHiveTransactionMgr().register(hiveTransaction); + skipCheckingAcidVersionFile = ConnectContext.get().getSessionVariable().skipCheckingAcidVersionFile; } } @@ -343,7 +346,7 @@ private List getFileSplitByTransaction(HiveMetaStoreCache cache, ValidWriteIdList validWriteIds = hiveTransaction.getValidWriteIds( ((HMSExternalCatalog) hmsTable.getCatalog()).getClient()); return cache.getFilesByTransaction(partitions, validWriteIds, - hiveTransaction.isFullAcid(), hmsTable.getId(), bindBrokerName); + hiveTransaction.isFullAcid(), skipCheckingAcidVersionFile, hmsTable.getId(), bindBrokerName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index dbddc2e14bb690..aeff011fe07abf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -130,7 +130,8 @@ public List buildRules() { Set aggSlots = funcs.stream() .flatMap(f -> f.getInputSlots().stream()) .collect(Collectors.toSet()); - return conjuncts.stream().allMatch(expr -> checkSlotInOrExpression(expr, aggSlots)); + return conjuncts.stream().allMatch(expr -> checkSlotInOrExpression(expr, aggSlots) + && checkIsNullExpr(expr, aggSlots)); }) .thenApply(ctx -> { LogicalAggregate> agg = ctx.root; @@ -163,7 +164,8 @@ public List buildRules() { Set aggSlots = funcs.stream() .flatMap(f -> f.getInputSlots().stream()) .collect(Collectors.toSet()); - return conjuncts.stream().allMatch(expr -> checkSlotInOrExpression(expr, aggSlots)); + return conjuncts.stream().allMatch(expr -> checkSlotInOrExpression(expr, aggSlots) + && checkIsNullExpr(expr, aggSlots)); }) .thenApply(ctx -> { LogicalAggregate>> agg = ctx.root; @@ -492,6 +494,22 @@ private boolean checkSlotInOrExpression(Expression expr, Set aggSlots) { return true; } + private boolean checkIsNullExpr(Expression expr, Set aggSlots) { + if (expr instanceof IsNull) { + Set slots = expr.getInputSlots(); + if (slots.stream().anyMatch(aggSlots::contains)) { + return false; + } + } else { + for (Expression child : expr.children()) { + if (!checkIsNullExpr(child, aggSlots)) { + return false; + } + } + } + return true; + } + private boolean isDupOrMowKeyTable(LogicalOlapScan logicalScan) { if (logicalScan != null) { KeysType keysType = logicalScan.getTable().getKeysType(); @@ -541,26 +559,53 @@ private LogicalAggregate pushdownCountOnIndex( LogicalFilter filter, LogicalOlapScan olapScan, CascadesContext cascadesContext) { - PhysicalOlapScan physicalOlapScan - = (PhysicalOlapScan) new LogicalOlapScanToPhysicalOlapScan() + + PhysicalOlapScan physicalOlapScan = (PhysicalOlapScan) new LogicalOlapScanToPhysicalOlapScan() .build() .transform(olapScan, cascadesContext) .get(0); + + List argumentsOfAggregateFunction = normalizeArguments(agg.getAggregateFunctions(), project); + + if (!onlyContainsSlot(argumentsOfAggregateFunction)) { + return agg; + } + + return agg.withChildren(ImmutableList.of( + project != null + ? project.withChildren(ImmutableList.of( + filter.withChildren(ImmutableList.of( + new PhysicalStorageLayerAggregate( + physicalOlapScan, PushDownAggOp.COUNT_ON_MATCH))))) + : filter.withChildren(ImmutableList.of( + new PhysicalStorageLayerAggregate( + physicalOlapScan, PushDownAggOp.COUNT_ON_MATCH))) + )); + } + + private List normalizeArguments(Set aggregateFunctions, + @Nullable LogicalProject project) { + List arguments = aggregateFunctions.stream() + .flatMap(aggregateFunction -> aggregateFunction.getArguments().stream()) + .collect(ImmutableList.toImmutableList()); + if (project != null) { - return agg.withChildren(ImmutableList.of( - project.withChildren(ImmutableList.of( - filter.withChildren(ImmutableList.of( - new PhysicalStorageLayerAggregate( - physicalOlapScan, - PushDownAggOp.COUNT_ON_MATCH))))) - )); - } else { - return agg.withChildren(ImmutableList.of( - filter.withChildren(ImmutableList.of( - new PhysicalStorageLayerAggregate( - physicalOlapScan, - PushDownAggOp.COUNT_ON_MATCH))))); + arguments = Project.findProject(arguments, project.getProjects()) + .stream() + .map(p -> p instanceof Alias ? p.child(0) : p) + .collect(ImmutableList.toImmutableList()); } + + return arguments; + } + + private boolean onlyContainsSlot(List arguments) { + return arguments.stream().allMatch(argument -> { + if (argument instanceof SlotReference) { + return true; + } + return false; + }); } //select /*+SET_VAR(enable_pushdown_minmax_on_unique=true) */min(user_id) from table_unique; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Hour.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Hour.java index 4d2248b9403613..60a57ff96be024 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Hour.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Hour.java @@ -26,6 +26,8 @@ import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.TimeType; import org.apache.doris.nereids.types.TinyIntType; import com.google.common.base.Preconditions; @@ -42,8 +44,8 @@ public class Hour extends ScalarFunction public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(TinyIntType.INSTANCE).args(DateTimeV2Type.SYSTEM_DEFAULT), FunctionSignature.ret(TinyIntType.INSTANCE).args(DateV2Type.INSTANCE), - FunctionSignature.ret(TinyIntType.INSTANCE).args(DateTimeType.INSTANCE) - ); + FunctionSignature.ret(TinyIntType.INSTANCE).args(DateTimeType.INSTANCE), + FunctionSignature.ret(IntegerType.INSTANCE).args(TimeType.INSTANCE)); /** * constructor with 1 argument. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Minute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Minute.java index 642fcd8b148429..790b03782358b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Minute.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Minute.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.TimeType; import org.apache.doris.nereids.types.TinyIntType; import com.google.common.base.Preconditions; @@ -42,8 +43,8 @@ public class Minute extends ScalarFunction private static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(TinyIntType.INSTANCE).args(DateTimeV2Type.SYSTEM_DEFAULT), FunctionSignature.ret(TinyIntType.INSTANCE).args(DateTimeType.INSTANCE), - FunctionSignature.ret(TinyIntType.INSTANCE).args(DateV2Type.INSTANCE) - ); + FunctionSignature.ret(TinyIntType.INSTANCE).args(DateV2Type.INSTANCE), + FunctionSignature.ret(TinyIntType.INSTANCE).args(TimeType.INSTANCE)); /** * constructor with 1 argument. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Second.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Second.java index 91167cfc9a1367..8f36817b4d0ccf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Second.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Second.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.TimeType; import org.apache.doris.nereids.types.TinyIntType; import com.google.common.base.Preconditions; @@ -42,8 +43,8 @@ public class Second extends ScalarFunction private static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(TinyIntType.INSTANCE).args(DateTimeV2Type.SYSTEM_DEFAULT), FunctionSignature.ret(TinyIntType.INSTANCE).args(DateV2Type.INSTANCE), - FunctionSignature.ret(TinyIntType.INSTANCE).args(DateTimeType.INSTANCE) - ); + FunctionSignature.ret(TinyIntType.INSTANCE).args(DateTimeType.INSTANCE), + FunctionSignature.ret(TinyIntType.INSTANCE).args(TimeType.INSTANCE)); /** * constructor with 1 argument. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index fcd496ac89b751..e04e2fe9c45474 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -670,6 +670,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_COOLDOWN_REPLICA_AFFINITY = "enable_cooldown_replica_affinity"; + public static final String SKIP_CHECKING_ACID_VERSION_FILE = "skip_checking_acid_version_file"; + /** * If set false, user couldn't submit analyze SQL and FE won't allocate any related resources. */ @@ -2200,6 +2202,12 @@ public void setIgnoreShapePlanNodes(String ignoreShapePlanNodes) { @VariableMgr.VarAttr(name = ENABLE_COOLDOWN_REPLICA_AFFINITY, needForward = true) public boolean enableCooldownReplicaAffinity = true; + @VariableMgr.VarAttr(name = SKIP_CHECKING_ACID_VERSION_FILE, description = { + "跳过检查 transactional hive 版本文件 '_orc_acid_version.'", + "Skip checking transactional hive version file '_orc_acid_version.'" + }) + public boolean skipCheckingAcidVersionFile = false; + public void setEnableEsParallelScroll(boolean enableESParallelScroll) { this.enableESParallelScroll = enableESParallelScroll; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 2ae051e4f2518e..4217342133b9ab 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -1677,7 +1677,7 @@ public void testNoPartition() throws AnalysisException { @Test public void testHourUnitWithDateType() throws AnalysisException { - String createOlapTblStmt = "CREATE TABLE if not exists test.hour_with_date (\n" + String createOlapTblStmt = "CREATE TABLE if not exists test.hour_with_date1 (\n" + " `days` DATEV2 NOT NULL,\n" + " `hours` char(2) NOT NULL,\n" + " `positionID` char(20)\n" @@ -1703,7 +1703,7 @@ public void testHourUnitWithDateType() throws AnalysisException { "could not be HOUR when type of partition column days is DATE or DATEV2", () -> createTable(createOlapTblStmt)); - String createOlapTblStmt2 = "CREATE TABLE if not exists test.hour_with_date (\n" + String createOlapTblStmt2 = "CREATE TABLE if not exists test.hour_with_date2 (\n" + " `days` DATETIMEV2 NOT NULL,\n" + " `hours` char(2) NOT NULL,\n" + " `positionID` char(20)\n" @@ -1726,6 +1726,32 @@ public void testHourUnitWithDateType() throws AnalysisException { + "\"dynamic_partition.create_history_partition\" = \"true\"\n" + ");"; ExceptionChecker.expectThrowsNoException(() -> createTable(createOlapTblStmt2)); + + connectContext.getSessionVariable().setTimeZone("Asia/Tokyo"); + String createOlapTblStmt3 = "CREATE TABLE if not exists test.hour_with_date3 (\n" + + " `days` DATETIMEV2 NOT NULL,\n" + + " `hours` char(2) NOT NULL,\n" + + " `positionID` char(20)\n" + + " )\n" + + "UNIQUE KEY(`days`,`hours`,`positionID`)\n" + + "PARTITION BY RANGE(`days`) ()\n" + + "DISTRIBUTED BY HASH(`positionID`) BUCKETS AUTO\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"compression\" = \"zstd\",\n" + + "\"enable_unique_key_merge_on_write\" = \"true\",\n" + + "\"light_schema_change\" = \"true\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.time_unit\" = \"HOUR\",\n" + + "\"dynamic_partition.start\" = \"-24\",\n" + + "\"dynamic_partition.end\" = \"24\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"2\",\n" + + "\"dynamic_partition.hot_partition_num\" = \"0\",\n" + + "\"dynamic_partition.storage_medium\" = \"HDD\", \n" + + "\"dynamic_partition.create_history_partition\" = \"true\"\n" + + ");"; + ExceptionChecker.expectThrowsNoException(() -> createTable(createOlapTblStmt3)); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseTest.java new file mode 100644 index 00000000000000..c48e18cb2711df --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseTest.java @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.SwitchStmt; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ExternalTableNameComparedLowercaseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameComparedLowercaseTest$ExternalTableNameComparedLowercaseProvider\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } + + @Override + protected void beforeCluster() { + Config.lower_case_table_names = 2; + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + } + + + @Test + public void testGlobalVariable() { + Assertions.assertEquals(2, GlobalVariable.lowerCaseTableNames); + } + + @Test + public void testTableNameLowerCase() { + Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("TABLE1")); + Assertions.assertTrue(tableNames.contains("TABLE2")); + } + + private void switchTest() throws Exception { + SwitchStmt switchTest = (SwitchStmt) parseAndAnalyzeStmt("switch test1;"); + Env.getCurrentEnv().changeCatalog(connectContext, switchTest.getCatalogName()); + } + + public static class ExternalTableNameComparedLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("TABLE1", Lists.newArrayList( + new Column("siteid", PrimitiveType.INT), + new Column("citycode", PrimitiveType.SMALLINT), + new Column("username", PrimitiveType.VARCHAR), + new Column("pv", PrimitiveType.BIGINT))); + tblSchemaMap1.put("TABLE2", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR), + new Column("k3", PrimitiveType.VARCHAR), + new Column("k4", PrimitiveType.INT), + new Column("k5", PrimitiveType.LARGEINT))); + MOCKED_META.put("db1", tblSchemaMap1); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseTest.java new file mode 100644 index 00000000000000..d55e209fa63d9f --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseTest.java @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.SwitchStmt; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ExternalTableNameStoredLowercaseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameStoredLowercaseTest$ExternalTableNameStoredLowercaseProvider\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } + + @Override + protected void beforeCluster() { + Config.lower_case_table_names = 1; + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + } + + + @Test + public void testGlobalVariable() { + Assertions.assertEquals(1, GlobalVariable.lowerCaseTableNames); + } + + @Test + public void testTableNameLowerCase() { + Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); + Assertions.assertEquals(3, tableNames.size()); + Assertions.assertTrue(tableNames.contains("table1")); + Assertions.assertTrue(tableNames.contains("table2")); + Assertions.assertTrue(tableNames.contains("table3")); + Assertions.assertFalse(tableNames.contains("TABLE1")); + } + + private void switchTest() throws Exception { + SwitchStmt switchTest = (SwitchStmt) parseAndAnalyzeStmt("switch test1;"); + Env.getCurrentEnv().changeCatalog(connectContext, switchTest.getCatalogName()); + } + + public static class ExternalTableNameStoredLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("table1", Lists.newArrayList( + new Column("siteid", PrimitiveType.INT), + new Column("citycode", PrimitiveType.SMALLINT), + new Column("username", PrimitiveType.VARCHAR), + new Column("pv", PrimitiveType.BIGINT))); + tblSchemaMap1.put("table2", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR), + new Column("k3", PrimitiveType.VARCHAR), + new Column("k4", PrimitiveType.INT), + new Column("k5", PrimitiveType.LARGEINT))); + tblSchemaMap1.put("TABLE3", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR), + new Column("k3", PrimitiveType.VARCHAR), + new Column("k4", PrimitiveType.INT), + new Column("k5", PrimitiveType.LARGEINT))); + MOCKED_META.put("db1", tblSchemaMap1); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/regression-test/data/external_table_p0/hive/test_string_dict_filter.out b/regression-test/data/external_table_p0/hive/test_string_dict_filter.out index a14f225abe5d0b..2a8cebd872315e 100644 --- a/regression-test/data/external_table_p0/hive/test_string_dict_filter.out +++ b/regression-test/data/external_table_p0/hive/test_string_dict_filter.out @@ -56,6 +56,9 @@ null -- !q14 -- null +-- !q15 -- +5 + -- !q01 -- 3 123314 F 193846.25 1993-10-14 5-LOW Clerk#000000955 0 sly final accounts boost. carefully regular ideas cajole carefully. depos 5 44485 F 144659.20 1994-07-30 5-LOW Clerk#000000925 0 quickly. bold deposits sleep slyly. packages use slyly @@ -113,6 +116,9 @@ null -- !q14 -- null +-- !q15 -- +5 + -- !q01 -- 3 123314 F 193846.25 1993-10-14 5-LOW Clerk#000000955 0 sly final accounts boost. carefully regular ideas cajole carefully. depos 5 44485 F 144659.20 1994-07-30 5-LOW Clerk#000000925 0 quickly. bold deposits sleep slyly. packages use slyly @@ -170,6 +176,9 @@ null -- !q14 -- null +-- !q15 -- +5 + -- !q01 -- 3 123314 F 193846.25 1993-10-14 5-LOW Clerk#000000955 0 sly final accounts boost. carefully regular ideas cajole carefully. depos 5 44485 F 144659.20 1994-07-30 5-LOW Clerk#000000925 0 quickly. bold deposits sleep slyly. packages use slyly @@ -227,3 +236,6 @@ null -- !q14 -- null +-- !q15 -- +5 + diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out index 875738ba98c7ab..01158a2fb603f2 100644 Binary files a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out and b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out differ diff --git a/regression-test/data/inverted_index_p0/test_count_on_index.out b/regression-test/data/inverted_index_p0/test_count_on_index.out index 3c0f47e7f8baf9..f74f3dc927aeed 100644 --- a/regression-test/data/inverted_index_p0/test_count_on_index.out +++ b/regression-test/data/inverted_index_p0/test_count_on_index.out @@ -77,3 +77,6 @@ -- !sql_bad -- 0 1 +-- !sql_bad2 -- +0 1 + diff --git a/regression-test/data/inverted_index_p0/test_index_rqg_bug5.out b/regression-test/data/inverted_index_p0/test_index_rqg_bug5.out new file mode 100644 index 00000000000000..095c7b2035676a --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_index_rqg_bug5.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 + diff --git a/regression-test/data/inverted_index_p0/test_index_rqg_bug6.out b/regression-test/data/inverted_index_p0/test_index_rqg_bug6.out new file mode 100644 index 00000000000000..edf3444ebe2928 --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_index_rqg_bug6.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 + +-- !sql_2 -- +3 + diff --git a/regression-test/data/inverted_index_p0/test_index_rqg_bug7.out b/regression-test/data/inverted_index_p0/test_index_rqg_bug7.out new file mode 100644 index 00000000000000..4ee136aef2b9c5 --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_index_rqg_bug7.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +2 + diff --git a/regression-test/data/inverted_index_p0/test_index_rqg_bug8.out b/regression-test/data/inverted_index_p0/test_index_rqg_bug8.out new file mode 100644 index 00000000000000..a21f3b8748ed99 --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_index_rqg_bug8.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +0 + diff --git a/regression-test/data/inverted_index_p0/test_need_read_data.out b/regression-test/data/inverted_index_p0/test_need_read_data.out index 782d5e5ab0b1b1..5999e7640ad5bc 100644 --- a/regression-test/data/inverted_index_p0/test_need_read_data.out +++ b/regression-test/data/inverted_index_p0/test_need_read_data.out @@ -31,3 +31,6 @@ -- !sql -- 1 +-- !sql_11 -- +1 + diff --git a/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out b/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out index 2aef8a1257aeb5..ae32f68c4be20f 100644 --- a/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out +++ b/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out @@ -685,3 +685,17 @@ true 1607746332.1230 1607746332 1607702400 1607702400 1607746332 1607746332.6667 1607746333 1607702400 1607702400 1607746333 +-- !sql_time_value -- +1 00:00:00 0 0 0 +2 00:01:00 0 1 0 +3 00:01:23 0 1 23 +4 \N \N \N \N +5 -01:29:23 -1 -29 -23 +6 -03:33:13 -3 -33 -13 +7 838:59:59 838 59 59 +8 -838:59:59 838 59 59 +9 111:23:45 111 23 45 + +-- !sql_time_value -- +456:26:32 456 26 32 + diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy index 864655a53441d9..792e2a98b248cb 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy @@ -82,10 +82,12 @@ suite('test_multi_followr_in_cloud', 'multi_cluster, docker') { log.info("show frontends: {}", ret) ret.size() == 4 } + def connectedUrl = context.threadLocalConn.get().conn.getMetaData().getURL() + log.info("current connected url {}", connectedUrl) check(4) { def ret -> ret.each { assertTrue(it.Role.contains(f)) - if (!Boolean.parseBoolean(it.IsMaster)) { + if (!Boolean.parseBoolean(it.IsMaster) && !connectedUrl.contains(it.Host)) { toDropIP = it.Host toDropPort = it.EditLogPort toDropType = transferType(it.Role) @@ -117,7 +119,7 @@ suite('test_multi_followr_in_cloud', 'multi_cluster, docker') { drop_node(toDropUniqueId, toDropIP, 0, toDropPort, toDropType, feClusterName, feClusterId, ms) - dockerAwaitUntil(5) { + dockerAwaitUntil(50) { def ret = sql """SHOW FRONTENDS""" log.info("show frontends: {}", ret) ret.size() == 3 @@ -153,7 +155,7 @@ suite('test_multi_followr_in_cloud', 'multi_cluster, docker') { drop_node(toDropUniqueId, toDropIP, 0, toDropPort, toDropType, feClusterName, feClusterId, ms) - dockerAwaitUntil(5) { + dockerAwaitUntil(50) { def ret = sql """SHOW FRONTENDS""" log.info("show frontends: {}", ret) ret.size() == 3 diff --git a/regression-test/suites/external_table_p0/hive/test_string_dict_filter.groovy b/regression-test/suites/external_table_p0/hive/test_string_dict_filter.groovy index 82afc63042f249..1929c813c554ac 100644 --- a/regression-test/suites/external_table_p0/hive/test_string_dict_filter.groovy +++ b/regression-test/suites/external_table_p0/hive/test_string_dict_filter.groovy @@ -59,6 +59,9 @@ suite("test_string_dict_filter", "p0,external,hive,external_docker,external_dock qt_q14 """ select * from ( select COALESCE(o_orderpriority, 'null') AS o_orderpriority from test_string_dict_filter_parquet ) as A where o_orderpriority = 'null'; """ + qt_q15 """ + select count(o_orderpriority) from ( select (case when o_orderpriority = 'x' then '1' when o_orderpriority = 'y' then '2' else '0' end) as o_orderpriority from test_string_dict_filter_parquet ) as A where o_orderpriority = '0'; + """ } def q_orc = { qt_q01 """ @@ -103,6 +106,9 @@ suite("test_string_dict_filter", "p0,external,hive,external_docker,external_dock qt_q14 """ select * from ( select COALESCE(o_orderpriority, 'null') AS o_orderpriority from test_string_dict_filter_orc ) as A where o_orderpriority = 'null'; """ + qt_q15 """ + select count(o_orderpriority) from ( select (case when o_orderpriority = 'x' then '1' when o_orderpriority = 'y' then '2' else '0' end) as o_orderpriority from test_string_dict_filter_orc ) as A where o_orderpriority = '0'; + """ } String enabled = context.config.otherConfigs.get("enableHiveTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { diff --git a/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy b/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy index 305c1f6615b2c5..dbe20395ec95ec 100644 --- a/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy +++ b/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy @@ -16,7 +16,10 @@ // under the License. suite("test_transactional_hive", "p0,external,hive,external_docker,external_docker_hive") { + String skip_checking_acid_version_file = "false" + def q01 = { + sql """set skip_checking_acid_version_file=${skip_checking_acid_version_file}""" qt_q01 """ select * from orc_full_acid order by id; """ @@ -32,6 +35,7 @@ suite("test_transactional_hive", "p0,external,hive,external_docker,external_dock } def q01_par = { + sql """set skip_checking_acid_version_file=${skip_checking_acid_version_file}""" qt_q01 """ select * from orc_full_acid_par order by id; """ @@ -54,7 +58,7 @@ suite("test_transactional_hive", "p0,external,hive,external_docker,external_dock return; } - for (String hivePrefix : ["hive2", "hive3"]) { + for (String hivePrefix : ["hive3"]) { try { String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") String catalog_name = "test_transactional_${hivePrefix}" @@ -67,6 +71,11 @@ suite("test_transactional_hive", "p0,external,hive,external_docker,external_dock );""" sql """use `${catalog_name}`.`default`""" + skip_checking_acid_version_file = "false" + q01() + q01_par() + + skip_checking_acid_version_file = "true" q01() q01_par() diff --git a/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy b/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy index 26736b672b28c0..924ceca4204fef 100644 --- a/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy +++ b/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy @@ -249,7 +249,7 @@ suite("test_hdfs_orc_group0_orc_files","external,hive,tvf,external_docker") { order_qt_test_38 """ select * from HDFS( "uri" = "${uri}", "hadoop.username" = "${hdfsUserName}", - "format" = "orc") order by _col0 DESC limit 100; """ + "format" = "orc") order by _col0 DESC, _col1 DESC limit 98; """ uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_orc/group0/orc-file-no-timezone.orc" order_qt_test_41 """ select * from HDFS( diff --git a/regression-test/suites/inverted_index_p0/test_count_on_index.groovy b/regression-test/suites/inverted_index_p0/test_count_on_index.groovy index 320fc65ff76bc9..ec2c556d8357e6 100644 --- a/regression-test/suites/inverted_index_p0/test_count_on_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_count_on_index.groovy @@ -313,6 +313,30 @@ suite("test_count_on_index_httplogs", "p0") { contains "pushAggOp=NONE" } qt_sql_bad "${bad_sql}" + def bad_sql2 = """ + SELECT + COUNT(cond1) AS num1, + COUNT(cond2) AS num2 + FROM ( + SELECT + CASE + WHEN c IN ('c1', 'c2', 'c3') AND d = 'd1' THEN b + END AS cond1, + CASE + WHEN e = 'e1' AND c IN ('c1', 'c2', 'c3') THEN b + END AS cond2 + FROM + ${tableName5} + WHERE + a = '2024-07-26' + AND e = 'e1' + ) AS project; + """ + explain { + sql("${bad_sql2}") + contains "pushAggOp=NONE" + } + qt_sql_bad2 "${bad_sql2}" } finally { //try_sql("DROP TABLE IF EXISTS ${testTable}") } diff --git a/regression-test/suites/inverted_index_p0/test_index_rqg_bug5.groovy b/regression-test/suites/inverted_index_p0/test_index_rqg_bug5.groovy new file mode 100644 index 00000000000000..c0181b17715d3b --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug5.groovy @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_index_rqg_bug5", "test_index_rqg_bug"){ + def table = "test_index_rqg_bug5" + sql "drop table if exists ${table}" + + sql """ + create table ${table} ( + pk int, + col1 int not null, + col2 bigint not null, + INDEX col1_idx (`col1`) USING INVERTED, + INDEX col2_idx (`col2`) USING INVERTED + ) engine=olap + DUPLICATE KEY(pk, col1) + distributed by hash(pk) + properties("replication_num" = "1");; + """ + + sql """ insert into ${table} values (10, 20, 30); """ + + qt_sql """ select count() from ${table} where col2 + col1 > 20 or col1 > 20; """ +} diff --git a/regression-test/suites/inverted_index_p0/test_index_rqg_bug6.groovy b/regression-test/suites/inverted_index_p0/test_index_rqg_bug6.groovy new file mode 100644 index 00000000000000..ce8e138dfa3dd0 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug6.groovy @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("test_index_rqg_bug6", "test_index_rqg_bug"){ + def table = "test_index_rqg_bug6" + + sql "drop table if exists ${table}" + + sql """ + create table ${table} ( + pk int, + col_int_undef_signed_index_inverted int not null , + col_varchar_1024__undef_signed_not_null varchar(1024) not null, + INDEX col_int_undef_signed_index_inverted_idx (`col_int_undef_signed_index_inverted`) USING INVERTED, + INDEX col_varchar_1024__undef_signed_not_null_idx (`col_varchar_1024__undef_signed_not_null`) USING INVERTED + ) engine=olap + DUPLICATE KEY(pk) + distributed by hash(pk) buckets 1 + properties("replication_num" = "1"); + """ + + sql """ insert into ${table} values (10, 0, 'ok'), (11, 0, 'oo'), (12, 1, 'ok')""" + + + sql """ sync""" + sql """ set enable_inverted_index_query = true """ + sql """ set inverted_index_skip_threshold = 0 """ + qt_sql """ + SELECT + count() + FROM + test_index_rqg_bug6 + WHERE + IF(col_int_undef_signed_index_inverted = 0, 'true', 'false') = 'false' + AND ( + col_varchar_1024__undef_signed_not_null LIKE 'ok' + OR col_int_undef_signed_index_inverted = 0 + ); + """ + + qt_sql_2 """ + SELECT + count() + FROM + test_index_rqg_bug6 + WHERE + col_varchar_1024__undef_signed_not_null LIKE 'ok' + OR col_int_undef_signed_index_inverted = 0; + """ +} diff --git a/regression-test/suites/inverted_index_p0/test_index_rqg_bug7.groovy b/regression-test/suites/inverted_index_p0/test_index_rqg_bug7.groovy new file mode 100644 index 00000000000000..5f0e09e4d4ebd6 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug7.groovy @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("test_index_rqg_bug7", "test_index_rqg_bug"){ + def table = "test_index_rqg_bug7" + + sql "drop table if exists ${table}" + + sql """ + create table ${table} ( + pk int, + col_int_undef_signed int null , + col_int_undef_signed_not_null_index_inverted int not null , + INDEX col_int_undef_signed_not_null_index_inverted_idx (`col_int_undef_signed_not_null_index_inverted`) USING INVERTED + ) engine=olap + DUPLICATE KEY(pk) + distributed by hash(pk) buckets 10 + properties("replication_num" = "1"); + """ + + sql """ insert into ${table} values (1, 7, 7), (2, 7, -2), (3, 4, -2)""" + + + sql """ sync""" + sql """ set enable_inverted_index_query = true """ + sql """ set inverted_index_skip_threshold = 0 """ + sql """ set enable_no_need_read_data_opt = true """ + qt_sql """ + select count(*) from ${table} where col_int_undef_signed_not_null_index_inverted = -2 AND ((CASE WHEN col_int_undef_signed_not_null_index_inverted = -2 THEN 1 ELSE NULL END = 1) OR col_int_undef_signed != 7); + """ +} diff --git a/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy b/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy new file mode 100644 index 00000000000000..3e54a10b326c51 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("test_index_rqg_bug8", "test_index_rqg_bug8"){ + def table1 = "test_index_rqg_bug8" + + sql "drop table if exists ${table1}" + + sql """ + CREATE TABLE ${table1} ( + `pk` int NULL, + `col_int_undef_signed_index_inverted` int NULL, + `col_boolean_undef_signed` boolean NULL, + `col_boolean_undef_signed_not_null` boolean NOT NULL, + `col_tinyint_undef_signed` tinyint NULL, + `col_tinyint_undef_signed_index_inverted` tinyint NULL, + `col_tinyint_undef_signed_not_null` tinyint NOT NULL, + `col_tinyint_undef_signed_not_null_index_inverted` tinyint NOT NULL, + `col_smallint_undef_signed` smallint NULL, + `col_smallint_undef_signed_index_inverted` smallint NULL, + `col_smallint_undef_signed_not_null` smallint NOT NULL, + `col_smallint_undef_signed_not_null_index_inverted` smallint NOT NULL, + `col_int_undef_signed` int NULL, + `col_int_undef_signed_not_null` int NOT NULL, + `col_int_undef_signed_not_null_index_inverted` int NOT NULL, + `col_bigint_undef_signed` bigint NULL, + `col_bigint_undef_signed_index_inverted` bigint NULL, + `col_bigint_undef_signed_not_null` bigint NOT NULL, + `col_bigint_undef_signed_not_null_index_inverted` bigint NOT NULL, + `col_decimal_16__8__undef_signed` decimal(16,8) NULL, + `col_decimal_16__8__undef_signed_index_inverted` decimal(16,8) NULL, + `col_decimal_16__8__undef_signed_not_null` decimal(16,8) NOT NULL, + `col_decimal_16__8__undef_signed_not_null_index_inverted` decimal(16,8) NOT NULL, + `col_decimal_38__9__undef_signed` decimal(38,9) NULL, + `col_decimal_38__9__undef_signed_index_inverted` decimal(38,9) NULL, + `col_decimal_38__9__undef_signed_not_null` decimal(38,9) NOT NULL, + `col_decimal_38__9__undef_signed_not_null_index_inverted` decimal(38,9) NOT NULL, + `col_decimal_38__30__undef_signed` decimal(38,30) NULL, + `col_decimal_38__30__undef_signed_index_inverted` decimal(38,30) NULL, + `col_decimal_38__30__undef_signed_not_null` decimal(38,30) NOT NULL, + `col_decimal_38__30__undef_signed_not_null_index_inverted` decimal(38,30) NOT NULL, + `col_date_undef_signed` date NULL, + `col_date_undef_signed_index_inverted` date NULL, + `col_date_undef_signed_not_null` date NOT NULL, + `col_date_undef_signed_not_null_index_inverted` date NOT NULL, + `col_datetime_undef_signed` datetime NULL, + `col_datetime_undef_signed_index_inverted` datetime NULL, + `col_datetime_undef_signed_not_null` datetime NOT NULL, + `col_datetime_undef_signed_not_null_index_inverted` datetime NOT NULL, + `col_datetime_3__undef_signed` datetime(3) NULL, + `col_datetime_3__undef_signed_index_inverted` datetime(3) NULL, + `col_datetime_3__undef_signed_not_null` datetime(3) NOT NULL, + `col_datetime_3__undef_signed_not_null_index_inverted` datetime(3) NOT NULL, + `col_datetime_6__undef_signed` datetime(6) NULL, + `col_datetime_6__undef_signed_index_inverted` datetime(6) NULL, + `col_datetime_6__undef_signed_not_null` datetime(6) NOT NULL, + `col_datetime_6__undef_signed_not_null_index_inverted` datetime(6) NOT NULL, + `col_char_255__undef_signed` character(255) NULL, + `col_char_255__undef_signed_index_inverted` character(255) NULL, + `col_char_255__undef_signed_index_inverted_p_e` character(255) NULL, + `col_char_255__undef_signed_index_inverted_p_u` character(255) NULL, + `col_char_255__undef_signed_not_null` character(255) NOT NULL, + `col_char_255__undef_signed_not_null_index_inverted` character(255) NOT NULL, + `col_char_255__undef_signed_not_null_index_inverted_p_e` character(255) NOT NULL, + `col_char_255__undef_signed_not_null_index_inverted_p_u` character(255) NOT NULL, + `col_varchar_1024__undef_signed` varchar(1024) NULL, + `col_varchar_1024__undef_signed_index_inverted` varchar(1024) NULL, + `col_varchar_1024__undef_signed_index_inverted_p_e` varchar(1024) NULL, + `col_varchar_1024__undef_signed_index_inverted_p_u` varchar(1024) NULL, + `col_varchar_1024__undef_signed_not_null` varchar(1024) NOT NULL, + `col_varchar_1024__undef_signed_not_null_index_inverted` varchar(1024) NOT NULL, + `col_varchar_1024__undef_signed_not_null_index_inverted_p_e` varchar(1024) NOT NULL, + `col_varchar_1024__undef_signed_not_null_index_inverted_p_u` varchar(1024) NOT NULL, + INDEX col_tinyint_undef_signed_index_inverted_idx (`col_tinyint_undef_signed_index_inverted`) USING INVERTED, + INDEX col_tinyint_undef_signed_not_null_index_inverted_idx (`col_tinyint_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_smallint_undef_signed_index_inverted_idx (`col_smallint_undef_signed_index_inverted`) USING INVERTED, + INDEX col_smallint_undef_signed_not_null_index_inverted_idx (`col_smallint_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_int_undef_signed_index_inverted_idx (`col_int_undef_signed_index_inverted`) USING INVERTED, + INDEX col_int_undef_signed_not_null_index_inverted_idx (`col_int_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_bigint_undef_signed_index_inverted_idx (`col_bigint_undef_signed_index_inverted`) USING INVERTED, + INDEX col_bigint_undef_signed_not_null_index_inverted_idx (`col_bigint_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_decimal_16__8__undef_signed_index_inverted_idx (`col_decimal_16__8__undef_signed_index_inverted`) USING INVERTED, + INDEX col_decimal_16__8__undef_signed_not_null_index_inverted_idx (`col_decimal_16__8__undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_decimal_38__9__undef_signed_index_inverted_idx (`col_decimal_38__9__undef_signed_index_inverted`) USING INVERTED, + INDEX col_decimal_38__9__undef_signed_not_null_index_inverted_idx (`col_decimal_38__9__undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_decimal_38__30__undef_signed_index_inverted_idx (`col_decimal_38__30__undef_signed_index_inverted`) USING INVERTED, + INDEX col_decimal_38__30__undef_signed_not_null_index_inverted_idx (`col_decimal_38__30__undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_date_undef_signed_index_inverted_idx (`col_date_undef_signed_index_inverted`) USING INVERTED, + INDEX col_date_undef_signed_not_null_index_inverted_idx (`col_date_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_datetime_undef_signed_index_inverted_idx (`col_datetime_undef_signed_index_inverted`) USING INVERTED, + INDEX col_datetime_undef_signed_not_null_index_inverted_idx (`col_datetime_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_datetime_3__undef_signed_index_inverted_idx (`col_datetime_3__undef_signed_index_inverted`) USING INVERTED, + INDEX col_datetime_3__undef_signed_not_null_index_inverted_idx (`col_datetime_3__undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_datetime_6__undef_signed_index_inverted_idx (`col_datetime_6__undef_signed_index_inverted`) USING INVERTED, + INDEX col_datetime_6__undef_signed_not_null_index_inverted_idx (`col_datetime_6__undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_char_255__undef_signed_index_inverted_idx (`col_char_255__undef_signed_index_inverted`) USING INVERTED, + INDEX col_char_255__undef_signed_index_inverted_p_e_idx (`col_char_255__undef_signed_index_inverted_p_e`) USING INVERTED PROPERTIES("parser" = "english", "lower_case" = "true", "support_phrase" = "true"), + INDEX col_char_255__undef_signed_index_inverted_p_u_idx (`col_char_255__undef_signed_index_inverted_p_u`) USING INVERTED PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" = "true"), + INDEX col_char_255__undef_signed_not_null_index_inverted_idx (`col_char_255__undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_char_255__undef_signed_not_null_index_inverted_p_e_idx (`col_char_255__undef_signed_not_null_index_inverted_p_e`) USING INVERTED PROPERTIES("parser" = "english", "lower_case" = "true", "support_phrase" = "true"), + INDEX col_char_255__undef_signed_not_null_index_inverted_p_u_idx (`col_char_255__undef_signed_not_null_index_inverted_p_u`) USING INVERTED PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" = "true"), + INDEX col_varchar_1024__undef_signed_index_inverted_idx (`col_varchar_1024__undef_signed_index_inverted`) USING INVERTED, + INDEX col_varchar_1024__undef_signed_index_inverted_p_e_idx (`col_varchar_1024__undef_signed_index_inverted_p_e`) USING INVERTED PROPERTIES("parser" = "english", "lower_case" = "true", "support_phrase" = "true"), + INDEX col_varchar_1024__undef_signed_index_inverted_p_u_idx (`col_varchar_1024__undef_signed_index_inverted_p_u`) USING INVERTED PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" = "true"), + INDEX col_varchar_1024__undef_signed_not_null_index_inverted_idx (`col_varchar_1024__undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_varchar_1024__undef_signed_not_null_index_inverted_p_e_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_p_e`) USING INVERTED PROPERTIES("parser" = "english", "lower_case" = "true", "support_phrase" = "true"), + INDEX col_varchar_1024__undef_signed_not_null_index_inverted_p_u_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_p_u`) USING INVERTED PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" = "true") + ) ENGINE=OLAP + UNIQUE KEY(`pk`, `col_int_undef_signed_index_inverted`) + DISTRIBUTED BY HASH(`pk`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "min_load_replica_num" = "-1", + "bloom_filter_columns" = "col_varchar_1024__undef_signed, col_date_undef_signed_not_null, col_date_undef_signed, col_int_undef_signed, col_varchar_1024__undef_signed_not_null, col_int_undef_signed_not_null", + "is_being_synced" = "false", + "storage_medium" = "hdd", + "storage_format" = "V2", + "inverted_index_storage_format" = "V1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false", + "group_commit_interval_ms" = "10000", + "group_commit_data_bytes" = "134217728", + "enable_mow_light_delete" = "false" + ); + """ + + sql """ insert into ${table1} (pk,col_boolean_undef_signed,col_boolean_undef_signed_not_null,col_tinyint_undef_signed,col_tinyint_undef_signed_index_inverted,col_tinyint_undef_signed_not_null,col_tinyint_undef_signed_not_null_index_inverted,col_smallint_undef_signed,col_smallint_undef_signed_index_inverted,col_smallint_undef_signed_not_null,col_smallint_undef_signed_not_null_index_inverted,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_undef_signed_not_null_index_inverted,col_bigint_undef_signed,col_bigint_undef_signed_index_inverted,col_bigint_undef_signed_not_null,col_bigint_undef_signed_not_null_index_inverted,col_decimal_16__8__undef_signed,col_decimal_16__8__undef_signed_index_inverted,col_decimal_16__8__undef_signed_not_null,col_decimal_16__8__undef_signed_not_null_index_inverted,col_decimal_38__9__undef_signed,col_decimal_38__9__undef_signed_index_inverted,col_decimal_38__9__undef_signed_not_null,col_decimal_38__9__undef_signed_not_null_index_inverted,col_decimal_38__30__undef_signed,col_decimal_38__30__undef_signed_index_inverted,col_decimal_38__30__undef_signed_not_null,col_decimal_38__30__undef_signed_not_null_index_inverted,col_date_undef_signed,col_date_undef_signed_index_inverted,col_date_undef_signed_not_null,col_date_undef_signed_not_null_index_inverted,col_datetime_undef_signed,col_datetime_undef_signed_index_inverted,col_datetime_undef_signed_not_null,col_datetime_undef_signed_not_null_index_inverted,col_datetime_3__undef_signed,col_datetime_3__undef_signed_index_inverted,col_datetime_3__undef_signed_not_null,col_datetime_3__undef_signed_not_null_index_inverted,col_datetime_6__undef_signed,col_datetime_6__undef_signed_index_inverted,col_datetime_6__undef_signed_not_null,col_datetime_6__undef_signed_not_null_index_inverted,col_char_255__undef_signed,col_char_255__undef_signed_index_inverted,col_char_255__undef_signed_index_inverted_p_e,col_char_255__undef_signed_index_inverted_p_u,col_char_255__undef_signed_not_null,col_char_255__undef_signed_not_null_index_inverted,col_char_255__undef_signed_not_null_index_inverted_p_e,col_char_255__undef_signed_not_null_index_inverted_p_u,col_varchar_1024__undef_signed,col_varchar_1024__undef_signed_index_inverted,col_varchar_1024__undef_signed_index_inverted_p_e,col_varchar_1024__undef_signed_index_inverted_p_u,col_varchar_1024__undef_signed_not_null,col_varchar_1024__undef_signed_not_null_index_inverted,col_varchar_1024__undef_signed_not_null_index_inverted_p_e,col_varchar_1024__undef_signed_not_null_index_inverted_p_u) values (0,false,true,6,0,7,51,29147,-19092,-32768,6792,1778213013,-1435420605,1141000263,-1534880882,1,1761809579,-1473846718,-2143986798613836525,2.1826,94.0496,75.0156,1.2,100.02,-1.2,85.1618,97.1213,13.0623,300.343,0,40.123,'2014-08-12','2014-08-12','2023-12-17','2025-02-17','2023-12-10','2023-12-10','9999-12-31 23:59:59','2027-01-16','2026-01-18','2026-02-18','2024-01-31','2024-07-01','2014-08-12','2024-02-18','2024-02-18','2024-02-18','-','why of were it''s have I''m see think on','出点这块姑娘第三方鑫辉挺多','then','--','f','i','','g','安信大品牌发过交叉口新鑫','for think at there they go','鹏飞看不到一般怎么着要时间涵盖盈盈参与本公司','do from','--','','?'),(1,false,false,-128,113,-128,1,10465,32767,3556,1,1139528841,-761314133,1337778604,1785698292,1,245,-9223372036854775808,1,-1.2,76.0528,1.2,67.0592,99.0425,0,93.1380,30.1582,47.0769,28.1861,68.1682,0,'2023-12-11','9999-12-31 23:59:59','2027-01-09','2023-12-15','2026-01-18','2025-06-18','2025-06-18','2024-01-17','2025-02-17','2024-01-09','2024-06-30 12:01:02.123','2023-12-14','2024-08-03 13:08:30','9999-12-31 23:59:59','9999-12-31','2023-12-10','调价','think no me would hey go he','--','准确量大济南收集奥运双机备份本来每一小时','不对战友','with with there get can''t that there are didn''t','have something why did','?','平志伟','双核','or or got okay i','--','and','v','','how'),(2,true,true,0,5,127,24,0,0,1,0,32679,1204712334,131128710,-48188984,4030719286291807454,245,1118154,245,100.02,300.343,0,0,1.2,67.1318,-1.2,40.123,87.0332,40.123,0,-1.2,'9999-12-31','2024-08-03 13:08:30','2023-12-18','2023-12-11','2023-12-20','2023-12-14','2026-02-18','2023-01-15 08:32:59.123123','2024-02-18','2023-12-18','2024-01-09','2024-01-09','2026-02-18','2023-12-13','2023-01-15 08:32:59.123123','2023-12-17','oh','no','歉意一万瑞丽首创楼下有带你呢里我们的','实际','助理不来更好调货贰仟天孝上次三星信息开始','盘点倪玲梅通过之间在欧普甘心史丽价格中层班','','w','-',null,'-','you''re been a just want','--','--','you''re think have then here your back want okay he','to'),(3,null,false,-116,2,-10,-128,0,-19330,-1,-32768,32679,32679,1794711563,147483648,9223372036854775807,-5976144,5093058139963235597,-1,null,-1.2,1.2,300.343,null,100.0436,68.0178,40.123,0,1.2,1.2,1.2,'2025-06-18','2024-06-30 12:01:02.123','2024-02-18','2024-01-09','2023-12-15','2025-02-17','2023-12-09','2024-02-18','9999-12-31','2023-12-17','2023-12-11','2024-07-01','2023-01-15 08:32:59.123123','2023-12-12','2023-12-17','2023-12-11','写上去多少钱金牌',null,'k','r','大棒','-','-','?',null,'come','w','--','妈的','will she a back it your no we right','it time','都想发生过网友真实日常厂家电脑上一周品放哥们'),(4,false,false,127,-1,-79,0,-17185,2004,23010,22407,-1,-632380417,1,2147483647,3382316,-7240048716014761707,-9223372036854775808,0,68.0355,100.02,-1.2,100.02,14.1965,93.1440,100.02,15.0576,1.2,1.2,22.1645,300.343,'2025-06-18','2024-02-18','2026-01-18','9999-12-31 23:59:59','2014-08-12','9999-12-31','2014-08-12','2023-12-15','9999-12-31 23:59:59','9999-12-31','9999-12-31','2027-01-09','2027-01-09','2026-02-18','2024-02-18','2023-12-19','太不','幸福','so say with it''s some you''re was back with','--','we','-','didn''t hey see up tell he or you''re','r','of mean like me right to','-','-','?','didn''t really or you','-','?','-'),(5,true,false,-61,-128,-1,0,24747,245,-32768,245,147483648,2147483647,493846618,2147483647,0,32679,245,-4340674386294581762,null,30.0155,300.343,0,300.343,100.02,93.0143,300.343,0,0,40.123,100.02,'2023-12-20','2023-12-13','2024-08-03 13:08:30','2026-01-18','2024-01-09','2023-12-15','2023-12-12','2024-02-18','2023-12-14','2027-01-16','2023-12-19','9999-12-31 23:59:59','2023-12-18','2024-01-31','2026-01-18','2023-12-15','','也会做成你杀时间天马回首研修网零捌鑫辉导入付出过','j','--','港湾潇湘谈好擅作主张景峰签订政策就像海域哦留一下','--','d','not going have some if out','-','忙碌','小灵通散落的小孩实业以为做梦他人调整扩展板说吧','就行','if','集团今收到工作轻信纽曼词典何时基本上','美丽回答问题','--'),(6,false,false,-1,127,1,-76,26368,245,-20769,-31761,0,-87885460,-1652669168,1684034981,0,null,8357353231250405620,-1,100.02,40.123,-1.2,29.1360,71.1085,300.343,21.0104,67.0505,-1.2,96.1324,0,-1.2,'2024-02-18','2024-01-31','2014-08-12','2023-12-12','2025-06-18','9999-12-31','2023-12-16','2023-12-13','2023-12-15','2025-02-18','2023-12-10','2024-01-09','2025-06-18','2023-12-18','2024-01-08','2014-08-12','光电','核对过',null,'--','国泰驱动器送去标志讲的服务站最好王平表格这段','in','q','投影机没有提到货差不多导轨张小莉用友有事情高端','新版原则经过','now','?','who really do me','-','didn''t','小玲','her'),(7,null,false,-1,-128,-128,-107,-31316,-32768,-32768,-32768,-138120024,147483648,2147483647,1335988149,-9223372036854775808,1,-6565982757735791562,6282666766883208310,1.2,-1.2,1.2,-1.2,null,40.123,4.1068,15.1504,92.0723,100.02,-1.2,99.0991,'2026-01-18','9999-12-31 23:59:59','2024-07-01','2025-02-17','2026-02-18','9999-12-31 23:59:59','2023-12-12','2023-12-19','2025-06-18','2024-01-19','2023-12-13','2024-02-18','2023-12-10','2023-01-15 08:32:59.123123','2026-01-18','2024-02-18','to a was no do','加入一辈子春子附件欢迎欢迎小精灵','go yeah now they when time','要时间楚留香飞信等级第一端口小时交到去年宝莱','hey','-','?','?','一些','him','手续金总淀雪集成','建新低端身边很好用讲师一脚转账濮阳单挑新鑫','办法差点孙瑞霞真人','周保全','那款可是','?'),(8,true,true,null,1,-50,1,-32768,25936,-1886,-29490,1,-2041458557,-2147483648,1987274488,228059854974871232,1757886170,1,245,100.02,30.0503,300.343,51.1272,1.2,-1.2,11.1134,0,-1.2,46.0419,32.0499,40.123,'2014-08-12','2024-02-18','2024-02-18','2023-12-09','2024-08-03 13:08:30','2027-01-16','2024-08-03 13:08:30','2023-12-10','2024-06-30 12:01:02.123','2024-01-19','2025-02-18','2007-09-04','9999-12-31 23:59:59','2024-01-09','2023-01-15 08:32:59.123123','2024-06-30 12:01:02.123','?','还好吧转移分钟旅之星成本条件原厂核算','走走道理一级周期','老是','he''s','百家重点简单好人坏人经历过是哦正式版条件威达','--','电话本',null,'k','--','did','-','男人','hey hey can out','t'),(9,false,true,0,-1,1,127,245,-32768,9807,32767,-1329389605,null,-2147483648,-1299351057,32679,1245554360,-1,147483648,88.0140,100.02,57.1036,26.1340,40.123,41.0937,73.1899,50.1680,72.0478,0,96.0750,-1.2,'2024-06-30 12:01:02.123','2025-06-18','2026-01-18','2024-07-01','2023-12-13','2024-08-03 13:08:30','2023-12-14','2023-12-20','2023-12-14','9999-12-31','2024-02-18','2023-01-15 08:32:59.123123','2024-06-30 12:01:02.123','2024-01-31','2023-12-11','2023-12-10','say','her','?','had i from look with didn''t look I''m','how','-','g','one it''s I''ll if it''s','吃过饭不让所作创恒还记得','可怕哈哈有钱老公不错总业绩歉意儿时瑞昌','u','?','华骏机器喜欢通过','don''t well as because well do time look good','西开发洗液失陪张经理老大赶紧有你们检查','水晶'),(10,false,true,null,-128,0,-128,14546,22850,20525,24943,-1,32679,1508441540,875534091,9223372036854775807,null,-2758791448581148818,6975488372886115484,17.0341,0,-1.2,300.343,9.1815,null,0,-1.2,87.0250,null,300.343,15.0047,'2024-01-09','2023-12-13','2024-01-19','2025-06-18','2023-12-17','2023-12-13','2023-12-20','2023-12-17','2024-07-01','2024-08-03 13:08:30','2027-01-16','2025-06-18','2024-02-18','2023-12-09','2011-01-13','2024-01-09','how','正联','',null,'那就好','如何','be','','?','a can''t is it''s','k','夜间','每天','my','why','-'),(11,false,true,70,null,8,8,-32768,null,0,-4374,2147483647,-1,438559785,2024065986,5058972017957328105,-53596719,6508809320674373367,147483648,100.02,71.1880,40.123,-1.2,63.0861,-1.2,300.343,0,100.02,1.2,-1.2,1.2,'2013-03-13','9999-12-31','2023-12-09','2024-02-18','2023-12-18','2024-06-30 12:01:02.123','2026-01-18','2023-12-15','2024-02-18','2024-06-30 12:01:02.123','2024-02-18','2023-12-11','2023-12-20','9999-12-31','2023-12-19','2023-12-18','did okay well for',null,'小心','had ok about if my','don''t of then do with some I''ll my don''t why','he did did they','这是','look','红色警戒老大清楚对话小故事发展动物全系列照面','--','right time who been time will that','好处','客户','-','美好','e'),(12,true,false,1,0,8,119,19259,21403,-14072,245,1,32679,147483648,1021379341,-9223372036854775808,-1475082893389938090,3564281329820020266,7168518253877601284,1.2,0,1.2,40.123,100.02,null,17.0702,1.2,100.02,33.1089,62.0820,-1.2,'2026-02-18','2027-01-09','2024-01-08','2024-02-18','2024-06-30 12:01:02.123','9999-12-31 23:59:59','2024-01-19','2024-06-30 12:01:02.123','2024-01-31','2024-01-17','2023-12-18','2025-02-17','2023-12-12','2023-12-14','2025-06-18','2025-06-18',null,'a','?','x','not','--','--','--','k','l','yeah','--','夜市','帮忙','唯一伏牛路不太中原北着那个家弱阳性夜市白晶','据对'),(13,true,false,127,0,127,-128,15507,10481,-1,2174,-1690405458,null,-2147483648,-595394109,9223372036854775807,1,1,-9184336785032758929,0,100.02,1.2,100.02,1.2,1.2,94.1575,25.0819,40.123,0,10.1497,1.2,'2024-02-18','2024-06-30 12:01:02.123','2023-12-14','2014-08-12','2025-02-18','2023-12-13','2001-06-13','9999-12-31 23:59:59','2025-06-18','2023-12-14','2025-06-18','2026-01-18','2023-12-18',null,'2024-02-18','2023-12-14','热线发吧','say the can mean no','--','-','at','?','--','about','from not not for to would on','--','the just with right','-','恢复感悟绿洲你用手机吧五千此外套件最起码把握','I''m mean be on well how','can','超利那位被人大棒两天一下阿良子猜到'),(14,false,false,-1,1,-1,-91,-406,-1,11862,13093,-1199392688,147483648,2147483647,-1392465743,9223372036854775807,-7375613,-1,-9223372036854775808,null,null,300.343,9.1615,null,0,97.0280,-1.2,24.0103,-1.2,0,100.02,'2024-01-08','2025-02-17','2018-12-12','2027-01-16','2023-12-09','2024-08-03 13:08:30','2024-01-08','2023-12-09','2023-12-19','2025-02-18','2027-01-09','2024-02-18','2023-01-15 08:32:59.123123','2024-01-09','2023-12-11','2026-02-18','','有什么提高平均天数东风系列三石有点忙进出大厦','at about it didn''t me this could in at don''t','got','as know as some can all up','want for to','','had','at with that''s me been because ok','--','are','小心自信','','砖石','--','up'),(15,false,true,-66,1,0,-1,-16724,32767,32767,27388,-2147483648,2147483647,32679,-1,0,245,-8091874767926804361,-2771121,-1.2,300.343,1.2,1.2,99.0973,7.0838,40.123,0,53.1169,53.1013,100.02,52.1436,'2024-01-19','2023-12-18','2024-08-03 13:08:30','2024-06-30 12:01:02.123','2023-12-19','2023-12-17','2027-01-16','2023-12-13','9999-12-31 23:59:59','2026-02-18','2026-02-18','2026-01-18','2023-01-15 08:32:59.123123','2024-01-31','2023-12-11','9999-12-31 23:59:59','?',null,'-','来看','高科','w','日信手机号配置参数科美属于你才回来详谈好早每天信息化','-',null,null,'期盼量大优惠李经理网页信任付费性格','虚妄','your','ok there not how','签字含税诚心对不对','when'),(16,false,false,-55,-29,6,-1,-17558,-32768,-11751,1,null,32679,-1794992889,-775086467,-4726114,-7209747535756199146,1,32679,5.0462,37.0608,0,13.0183,1.2,1.2,54.1299,-1.2,84.1056,39.1453,39.0725,1.2,'2025-02-17','2023-12-16','2023-12-12','2025-02-18',null,'2025-06-18','2027-01-09','2027-01-16','2024-01-31','2014-08-12','2024-02-18','2027-01-16','2023-12-11','2023-12-19','2024-01-19','2024-01-19','-',null,'okay this','第三只人选时间','--','-','her','莱克特播放五号屏幕武侠那里刘亮沃尔普打击赞美诗','--','?','--','-','who','got been','歘俩','红色之日起'),(17,false,false,1,-128,0,1,-32768,-29436,0,32767,1810524027,582227987,-374931303,2147483647,32679,-411987,147483648,5337900138548455534,18.1431,6.1739,40.123,40.123,-1.2,40.123,1.2,1.2,48.0870,40.123,0,5.0035,'2025-06-18','2023-12-15','2023-12-16','2023-12-15','2024-02-18','2024-02-18','2025-06-18','2024-02-18','2023-12-10','9999-12-31 23:59:59','2023-12-09','2023-12-13','2024-02-18','2024-02-18','2024-01-09','2023-12-19','yeah one with but really think his would','发给一帆','do','say','-','-','?','p','帮帮','着呢','you','','-','there','that''s','-'),(18,null,true,-1,0,1,-128,null,-8330,-1,-1,-1,-1,604628330,-2147483648,245,9223372036854775807,245,-924364,46.1775,52.0705,95.0524,1.2,1.2,1.2,40.123,1.2,25.1822,0,-1.2,10.1818,'2023-12-09','2025-06-18','2027-01-16','2025-02-17','2023-12-17','2025-02-18','2024-02-18','9999-12-31 23:59:59','2024-01-31','2023-12-09','2023-12-13','2027-01-16','2024-01-19','2023-12-17','2014-08-12','2025-02-18','谢晓娟提到','didn''t no','is just would how','i want going got not','直销','w','o','the','mean tell was just you''re didn''t come I''ll now','谦虚想起来镜面证书','out ok on go know that was get so well','放弃避免着呢交流群商家交叉文字软件园孙海洋又是','is','子夜但愿地方','','这不是惊喜金牌王平站在羡慕上不了正式'),(19,true,false,127,127,127,-28,1,16316,9388,-16775,0,-1,147483648,216679344,5314322873430997249,-5223860621338077949,245,32679,0,100.02,1.2,300.343,100.02,87.1801,40.123,300.343,0,0,0,0,'2024-01-08','2024-02-18','2023-12-14','2027-01-16','2026-02-18','2024-01-31','2024-02-18','2024-01-31','2023-12-15','2025-02-18','2024-01-08','2026-02-18','2024-01-19','2024-01-09','2026-02-18','2023-12-15','','算是','现在','扩展板','-','--','about','','you''re','--','this then of at him it were','--','of the I''m','as he see is from what were this do some','',''),(20,true,false,-31,-67,18,1,-11733,1,29574,-32768,1,32679,-1,-118389404,null,-1988138342,-350036750,-9223372036854775808,41.1195,null,41.0719,60.0534,300.343,24.1464,40.123,44.1265,300.343,40.123,300.343,0,'2025-02-17','2023-12-15','9999-12-31 23:59:59','2024-01-19','2024-02-18','2023-12-18','2023-12-18','2023-12-09','2024-02-18','2023-12-15','2018-09-07','2024-02-18','2025-02-17','2023-12-11','2026-02-18','2024-01-09',null,'--','','got','?','tell good there','','?','--','is and now','--','报账','操作系统刻骨铭心汇祥想过看完当兵除外','she','','过程均为火车站着那个家王佩丽'),(21,false,true,0,-128,-128,-5,-1,1625,-1,16024,-1,null,0,0,-9223372036854775808,0,1,-5364720998564414400,42.0321,300.343,18.1915,41.1298,85.0304,55.1246,62.1369,0,20.1409,300.343,300.343,300.343,'2023-12-15',null,'2023-12-19','2025-02-17','2014-08-12','2023-12-09','9999-12-31','2027-01-09','2025-06-18','2023-12-16','2025-02-18','2023-12-19','2014-08-12','2026-02-18','2023-12-10','2027-01-09','好久不见我刚回来呀办好专业','有数','显卡','领取怎么找收索导轨','he I''m they say you from','授权书','--','-','脱机多少钱说话工程洛宁词句就行众佳科技大厦工业大学','平均','so','don''t as come','类似','?','the him we look had','郑大'),(22,null,true,-1,83,-128,127,null,1,-20459,20744,-2118077137,1727350049,1554325769,810436158,147483648,7536560694077524474,1,245,1.2,1.2,0,100.02,null,300.343,40.123,24.0147,0,0,47.1093,74.0569,'2025-06-18','2027-01-16','2023-12-19','2027-01-09','2024-01-19','2023-12-16','2026-01-18','2024-01-09','2023-12-17','2023-12-15','2024-01-31','2024-02-18','2023-12-18','2023-12-16','2027-01-16','2014-08-12','王睿我借你饱和升职','say','','很好用','','and','允许医生需方北站','just some from at he''s go yes out like got','a','注册表香港张先波开除国栋市场上杂请你们','-','--','--','-','银联','just'),(23,true,true,-100,32,45,-103,1,-4869,-4341,245,2147483647,-489224322,-2147483648,147483648,245,null,32679,-2863791432995146217,-1.2,-1.2,300.343,63.0553,300.343,0,300.343,100.02,40.123,47.0261,8.0695,1.2,'2023-12-18','2023-12-16','2027-01-16','2025-06-18','2023-12-20','2023-12-11','9999-12-31','2023-12-12','2024-01-19','2023-01-15 08:32:59.123123','2026-01-18','2024-01-19','2023-12-12','2025-02-17','2023-12-12','2025-02-17','your','--','发生过北环经销商蠢材出来','华栋','--','--','here','he''s know from see on the me so yes','--',null,'下月汉语天普','y','x','--','?','q'),(24,true,false,8,-116,-70,0,18768,-27168,-32768,32767,-2147483648,-491681230,-1,1543210001,0,-4158749369703753712,-1066924451,-5949701957015571793,1.2,-1.2,1.2,1.2,null,null,300.343,40.123,40.123,1.2,11.1427,58.0781,'2024-01-19','2025-06-18','2024-06-30 12:01:02.123','2024-02-18','2026-02-18','2027-01-16','2014-08-12','2025-06-18','2024-01-09','2023-12-10','2023-12-18','2026-02-18','2025-06-18','9999-12-31 23:59:59','2023-12-17','2023-12-12','what','j',null,'贵姓','z','-','--','about were know know his','I''m look good up this','伤感','很细那个','be is get oh his','-','','同时最深','命名零度'),(25,false,true,123,null,0,127,245,29270,18912,27671,2056423585,-1114681434,0,-1666299227,9223372036854775807,-890780121,245,-1,1.2,63.0243,92.0015,40.123,1.2,92.1741,40.123,1.2,24.1985,300.343,100.02,1.2,'2024-02-18','2023-12-19','9999-12-31 23:59:59','2026-01-18','2023-01-15 08:32:59.123123','2024-01-19','2023-12-11','2023-12-19','2024-01-17','2023-12-18','2024-01-08','2024-08-03 13:08:30','2023-12-11','2024-01-31','2025-06-18','2024-01-17','副本恢复生意佩利待定南阳路双千兆','t','',null,'赞美诗','--','五笔有缘晨晖文利比例健民压在他人晚上聊之作','--','--',null,'?','when been me at i with time','','here at','hey would can his you her know would','he''s'),(26,null,true,null,-1,1,127,-8446,-2483,32767,2058,0,2147483647,12755506,1,8798267953845829469,32679,1872198378726361569,-2942272,69.0559,47.1497,-1.2,34.1466,100.02,-1.2,51.0790,1.2,28.0065,-1.2,-1.2,76.0019,'2026-01-18','2024-02-18','2026-02-18','2023-12-18','2023-12-17','2024-01-08','2023-12-17','2024-01-09','2023-12-18','2026-01-18','2009-05-03','2024-02-18','2024-01-08','2023-12-17','2024-01-08','2026-01-18','休息南阳路上半年重命名网管中心单挑周保全合适处理一下海尔','不敢质保终究控件不定离谱神偷','不忙','want would don''t oh I''m can''t why one yes some','令我又将','冲淡湖北比如说写在小雨奖金宝贵安全','--','b','客户端','--','out','all going okay is right','will but been when it''s','okay really I''ll what she my was he''s yeah','come the been could he''s','苦楚'),(27,null,false,null,-118,0,2,32767,-32389,30491,32101,-2147483648,-1,0,32679,9223372036854775807,-6520303307884870157,245,-9223372036854775808,5.1951,0,-1.2,40.123,86.0859,100.02,4.1986,40.123,-1.2,null,0,-1.2,'2023-12-17','2023-12-13','2025-02-18','2023-12-16','2023-12-18','2023-12-11','2014-08-12','2023-12-16',null,'2025-06-18','2023-12-09','2023-01-15 08:32:59.123123','2023-12-15','2025-02-17','2024-01-17','2023-01-15 08:32:59.123123','-','were oh from it''s right then','--','will go some my it''s going he''s','项目','o','','刘伟','he hey have just','纵观','got that ok good your be','','be','for','--','don''t really that all yeah here him for'),(28,true,true,0,63,127,127,28111,245,-29617,-4889,-1758349040,1763950935,1094280556,-637872619,-8641813557597328333,-9223372036854775808,6086990696662277493,-9223372036854775808,null,0,93.0920,1.2,null,59.1122,28.1426,49.1276,98.1143,0.0503,300.343,0,'2024-08-03 13:08:30','2024-01-19','2024-02-18','2027-01-09','2023-12-13','2024-06-30 12:01:02.123','2024-01-31','2023-12-14','2023-12-15','2025-02-18','2023-12-13','2023-12-15',null,'2025-02-17','2027-01-16','2023-12-18','阶段最近忙什么呢嵩山仔细我们的地方那个胶片厂','?','x','want','?','','on could tell in going good at know just here','-','p','冰河','could going do there could something why why you''re','have','it','k','time','--'),(29,true,false,24,127,-45,-35,0,26516,17765,20156,-1,-2046773551,1123817283,-2147483648,null,9223372036854775807,-9223372036854775808,554792133,32.0173,57.1898,300.343,300.343,10.0337,1.2,99.1261,100.02,null,40.123,93.0990,300.343,'2023-12-09','2023-12-18','2027-01-16','2023-12-19','2004-03-18','2023-12-10','2023-01-15 08:32:59.123123','2024-02-18','2023-12-10','2023-12-14','9999-12-31 23:59:59','2023-12-09',null,'2023-01-15 08:32:59.123123','2023-12-14','2024-07-01','','her go',null,'做出搞定相符宠物平均管理员排列天晴城市','每天','-','?','-',null,'多多重复岂能系统长时间保险也不是不到位','n','切换起床采用干嘛被激怒个总经销商','','--','would','名家'),(30,true,false,91,-20,15,-1,-1586,-20360,-1,31476,2147483647,1460097326,-2147483648,1,-30361165468551680,null,9223372036854775807,3259165989020601252,40.123,100.02,11.1911,93.0047,40.123,40.123,0,60.0210,1.2,10.0283,0,300.343,'2023-12-16','2023-12-12','2024-06-30 12:01:02.123','2023-12-16','2023-12-14','2024-08-03 13:08:30','2024-08-03 13:08:30','2024-01-08','2024-02-18','2024-07-01','2024-01-09','2025-06-18','2024-01-31','2023-01-15 08:32:59.123123','2025-02-18','2024-02-18','back','-','哭泣差距很大下个','z','所提供热销词库使人所有开头武警总队这话','right I''m could would','换一家','没变为你汉威政府指定昨晚上','网吧之夜不对战友小刘','from have know know',null,'?','?','--','?','高压后天回正式端口天地外聘文件核对'),(31,true,false,88,87,104,8,-32768,-20115,0,-20740,-2147483648,-2147483648,-1986116363,1,-268648447394379111,-9223372036854775808,147483648,245,-1.2,null,-1.2,27.1371,0,1.2,100.02,-1.2,null,5.1450,92.0735,1.2,'9999-12-31','2027-01-09','9999-12-31','2025-06-18','9999-12-31 23:59:59','2000-10-20','2014-08-12','2024-02-18','2023-12-11','2024-01-17','2024-02-18','2025-06-18','2023-12-19','2023-12-10','2023-12-13','2024-07-01','time','卢海一两阶段','have','think','you''re good have will do but his be','all','架子','yeah','--','--','这其中','供货明天找我贸易通背书网站必须','中小王丽','m','--','--'),(32,true,false,0,0,-1,-1,0,1,18891,0,32679,null,-2147483648,1,245,-730000963801070405,-6567954926160409570,0,300.343,21.0915,56.0350,18.0963,94.1440,1.2,53.0729,300.343,300.343,1.2,1.2,77.1375,'2027-01-16','2023-12-14','2024-01-09','9999-12-31 23:59:59','2026-01-18','2023-12-16','2024-01-17','2024-02-18','2027-01-09','2023-12-14','2023-12-17','2024-01-08','2023-12-20','2014-08-12','2023-12-09','2014-08-12','on in if you''re from','','ok your want say with hey when how there him','润泽流明次要','你好呀待定插拔日信市场目标家里','k','?','I''ll','百元整冯向百脑汇进入了付出了领先之后众诚','?','--','不曾名杂到来一共','主演王青最低汇众想到相约询问过高高','can been like come','','how about have think when but don''t the in'),(33,null,true,-1,1,22,36,-8395,-1,32767,19251,147483648,null,32679,-1194428859,null,-7003800,-1780700346672999405,32679,null,-1.2,79.0172,62.1674,1.2,37.1430,0,43.1177,300.343,300.343,100.02,2.1459,'2023-12-12','2024-01-09','2024-08-03 13:08:30','2024-01-08','2023-12-15','2023-01-15 08:32:59.123123','2026-02-18','2026-02-18','2023-12-13','2023-01-15 08:32:59.123123','2025-06-18','2026-01-18','2027-01-09','2023-12-12','9999-12-31 23:59:59','2025-02-18','--','--','look be time been back','can my like the the could her about yes got','?','七匹狼正品图标不至于','could could you a of','而来倒是佳成如果','还不错十分我能电视规定靠前客运量空分','--','回访','下个礼拜新山钻石金牌周围','-','to','-',''),(34,null,true,-128,-1,127,9,4929,-1,-32768,-1,-926729021,-489011328,2147483647,-1568695661,-5240172516187475029,1691038444,9223372036854775807,147483648,0,40.123,0,40.123,40.123,74.0504,80.0530,100.02,40.123,40.123,97.1562,100.02,'2025-06-18','2024-01-09','2024-08-03 13:08:30','2023-12-11','2023-12-11','2025-06-18','2024-02-18','2023-12-11','2024-08-03 13:08:30','2023-12-12','2024-01-08','2023-12-11','2025-06-18','2024-02-18','9999-12-31','2023-12-11','贸易通','第二款指出来数哟两千最美丽中旬','her','-','?','g','he''s tell have from good hey from is for be','it''s would really a in me who do','未定','-','okay','yeah','q','--','--','--'),(35,false,false,-79,0,-1,5,-796,17301,-22121,1,-1,1,893775863,-915924689,-4394973541698078775,-9223372036854775808,-9223372036854775808,-6550281778227888053,300.343,100.02,300.343,1.2,-1.2,1.2,77.0924,49.1457,40.123,80.0551,94.0464,52.1076,'2023-12-09','2023-12-11','2025-02-18','2023-12-18','2024-02-18','2024-08-03 13:08:30','2023-12-09','2023-12-19','2024-06-30 12:01:02.123','2014-08-12','2026-01-18','2024-01-19','2023-12-09','2023-12-09','2024-01-17','2024-01-19','这首宇瑞回答看重参与','-','明天找我','x','been have could she how at or out','--','小浪底不走客运','out don''t','亚太科美天讯同辉圣荣濑尿虾真伪黑色好梦供应商','the','some will come this be all','反应东方','-','if get','新款原创','?'),(36,true,false,3,48,-1,-48,-32768,null,-10021,28424,null,89804126,147483648,-680806434,7291697225388406934,null,-8799855892443545049,0,0,78.1267,0,-1.2,40.123,0,8.1201,100.02,1.2,1.2,100.02,100.02,'2027-01-16','2023-12-12','2023-12-19','2024-06-30 12:01:02.123','2023-12-20','2024-01-19','2023-12-16','2025-02-18','2023-12-17','2023-12-17','2024-07-01','2024-08-03 13:08:30','2023-12-15','2024-01-08','2014-08-12','2027-01-16','?','-','?','?','what','市场上天威龙弄个均为原来见过代玉对于双核','look mean my he i with hey if do but','got here we we that''s about and had him think','本公司贸易通维修你吃法除外快递','天下身份人参都有资金客服','称呼治疗我同学结婚来庆祝热播方案','一条','--','q','武汉','?'),(37,null,true,-99,127,0,-17,1,7196,31215,245,null,1008519661,-194086066,2147483647,6382203,8511043665503645950,-1,72547424616057298,-1.2,100.02,97.1904,81.0776,25.0611,33.1374,37.0332,79.1107,45.0366,-1.2,42.1639,-1.2,'9999-12-31','2023-12-12','2025-06-18','2023-12-09','2023-12-14','2023-01-15 08:32:59.123123','2024-06-30 12:01:02.123','2025-06-18','2026-02-18','2024-06-30 12:01:02.123','2024-01-09','2023-12-17','2023-12-09','2023-12-14','2024-01-08','2024-01-17','身边','?','h','--','with','ok mean back you one will will say look','红颜','then','财务部','?','her',null,'the one','','订单批发见你条数单个浪费通道看着办分销商','here'),(38,false,true,-40,127,1,93,14085,-27042,7816,9260,-1433718766,1,-1,2147483647,1,9223372036854775807,147483648,-743258780,1.2,0,40.123,100.02,81.0898,-1.2,300.343,0,1.2,40.123,300.343,100.02,'2023-12-18','2024-01-08','2023-12-11','2026-01-18','2023-12-14','2025-06-18','2024-07-01','2026-02-18','2023-12-12','2024-01-09','2024-02-18','2023-12-12','2027-01-09','2024-01-09','2024-01-31','2023-12-09','oh','惊喜金牌','m','-','oh right with me your','so him because','','x','-','-','往年','?','for','-','带你如有写上去航海路检查陈老师说不出','不下'),(39,null,true,-1,1,9,123,-30872,5079,245,-24514,1157197029,1537312088,-1,-1,890351490,9223372036854775807,-9223372036854775808,808960386243577672,1.2,0,1.2,43.0001,29.0832,30.1560,-1.2,1.2,100.02,40.123,38.1235,3.0080,'2024-06-30 12:01:02.123','2025-02-18','2025-02-18','2023-12-13','2027-01-16','2024-02-18','9999-12-31 23:59:59','2024-02-18',null,'2024-01-17','2023-12-09','2024-01-17','2023-12-13','2024-07-01','2023-01-15 08:32:59.123123','2023-01-15 08:32:59.123123','贰台','what now','-','广告','一份子','your now to see it at','不传不多记录欧洲中天的风格看课件素材量大','if is were why she that''s some say is','who from tell something','ok','技术员数码港新增大雪怡海提供开公司老总','have can''t with is oh just just','','one you''re got what you''re','自学良子除非不便欧典况且信阳卓兴据对相信','帐号'),(40,null,false,127,-1,-1,-17,-26354,-2911,32767,32767,32679,2147483647,0,2147483647,1,-9223372036854775808,0,1814090464,100.02,null,0,90.1065,100.02,-1.2,31.1376,100.02,50.0038,23.0768,-1.2,40.123,'2024-01-08','2024-02-18','2026-01-18','2026-02-18','2025-02-17','2024-02-18','2024-01-09','2023-12-11',null,'2023-12-19','2027-01-09','2025-06-18','2023-12-10','2025-06-18','2024-01-31','2023-12-19','--','兼容金达研修网小徐伟泽光电查过具体还有你的','收到云鹏增票武汉许愿树七匹狼订票改动键盘',null,'or with really they go ok say look time yes','','?','里有',null,'免费','打给','going that','删除','?','福利杨宗容量卓兴一键还原小雪','调试部分发吧李平这一块很低哪里'),(41,false,false,38,127,1,3,1,32767,-535,-25017,-1,-335834102,147483648,-2147483648,6196352782692926078,-7069508575514069654,-9223372036854775808,0,40.123,42.1813,300.343,1.2,40.123,null,70.0479,-1.2,71.1975,18.0217,1.2,1.2,'2025-02-17','2025-02-18','2024-02-18','2024-01-19','2023-12-09','2023-12-17','2027-01-09','2024-01-17','2024-08-03 13:08:30','2023-12-13','2024-06-30 12:01:02.123','2024-01-09','2023-12-13','2023-12-15','2023-12-17','2023-12-16','打扰焦煤无限烟厂不在哦传美完美','--','空下五星兄弟','则很难','-','','-','good','-','还在怎么找也会车费火车','think with oh your him','--','in','and do time','?','didn''t'),(42,null,false,null,null,-72,7,23137,-16658,-1,15469,-981817977,null,-884475713,0,9223372036854775807,0,147483648,2747189507179247531,1.2,100.02,1.2,0.1455,40.123,-1.2,54.1343,300.343,1.2,300.343,100.02,14.0976,'2023-12-16','2025-02-18','2024-01-08','2023-12-17',null,'2024-01-17','2024-08-03 13:08:30','2025-06-18','2023-01-15 08:32:59.123123','2023-01-15 08:32:59.123123','2026-02-18','2024-01-17','2026-02-18','2027-01-16','2024-07-01','2023-12-14','提到个月坐在','yeah','小时','一舟','--','?','your','k','on',null,'提出春节停产散落的','-','两个','不需要','say as don''t can','think do when don''t got some okay no back there'),(43,null,false,127,-1,126,-21,245,-8884,17360,-32768,1,0,1,166489221,147483648,245,9223372036854775807,-1,79.0906,-1.2,40.123,40.123,100.02,0,1.2,0,-1.2,40.123,21.1863,10.1618,'2026-02-18','2027-01-09','2025-02-17','2023-12-13',null,'2024-07-01','2024-08-03 13:08:30','2005-07-16','2023-12-16','2014-08-12','2024-01-08','2023-12-13','2024-02-18','2027-01-09','2024-07-01','2023-12-20','?',null,null,'-','she','n','参观都想小灵通干嘛信海太厚埋在合创','--','is from not been I''m good did be','称呼','-','','他妈也好湖北美丽不对跑跑卡丁车说出来一张','赵芳','did','高高'),(44,null,true,1,48,8,0,-30056,1,26163,0,292275484,32679,-1574338139,-786277339,-1,245,3690369133288045380,2739405,24.1123,2.1873,300.343,0,-1.2,0,40.123,43.0193,29.0534,1.2,51.1334,40.123,'2014-08-12','2024-01-09','2024-01-08','2024-02-18','2025-06-18','2024-08-03 13:08:30','2024-01-08','2024-02-18','2023-12-16','2023-01-15 08:32:59.123123','2024-02-18','2023-12-15','2025-06-18','9999-12-31','2026-02-18','2027-01-16','','你家哦跟你说',null,'about when one','-','ok a so he''s to','that''s a him out yes would','are a how and at are a','-','面子饲料询问过建议政府性价比系统集成商','what','p','go','who this just is oh know would did that''s','','海川'),(45,true,false,63,0,127,-123,1,null,0,28568,1596508631,-2147483648,2147483647,1157679284,1,9223372036854775807,-7259578370457002861,-204357924,0,100.02,100.02,1.2,1.2,88.0444,1.2,39.0709,76.1952,null,55.1933,0,'2023-12-13','2024-01-08','2023-12-18','2023-12-19','2025-06-18','2023-12-10','2023-12-16','2024-01-17','2024-08-03 13:08:30','2025-06-18','2023-12-11','2023-12-19','9999-12-31','2023-12-14','2016-11-15','2023-12-16','能早点组装方连一流','-','的那个','y','可惜','?','叫做','p','待定',null,'say','伤和气','--','that''s to want oh just is','-','got'),(46,false,true,0,102,6,127,null,24574,32767,32767,415649751,-2147483648,1465783352,1,245,-4275036788457434117,1,32679,null,52.1149,-1.2,1.2,0,0,87.0916,-1.2,39.1247,4.0706,-1.2,-1.2,'2024-01-08','2023-12-12','2023-12-15','2025-02-18','2006-04-04','2027-01-09','2014-08-12','2023-12-20','2025-06-18','2026-02-18','2023-12-10','2024-06-30 12:01:02.123','2024-08-03 13:08:30','2026-02-18','2023-01-15 08:32:59.123123','2024-07-01','-','两者','调货忘记销售资格首位批复已给李景林想不到','go','why come','then if we know you''re you not tell because yeah','双子','b','但是增票浩宇地图分辨率几篇防火墙','?','when','o','总结浇水','羡慕','-','?'),(47,null,false,1,-1,1,-74,32767,-5076,2338,-27143,0,124418066,2147483647,1381223492,-1937424844986983939,147483648,-1,1,1.2,95.1064,0,58.0528,40.123,null,0,81.1194,40.123,100.02,30.0961,40.123,'2024-02-18','2024-07-01','2024-02-18','2023-12-10','2023-12-11','2023-12-15','2026-02-18','2023-12-20','2025-02-18','2024-01-08','2025-02-18','2023-12-17','2027-01-16','2023-12-14','2001-07-19','2025-06-18','补丁提到','one','岩春服务器站胡华威计划认识你不信吗','k','刘海下一部分名字中晶月份意义郑州可靠','-','-','','she','能不鞥','like','惊喜总代理商','小型机','but about were the because you','his','look'),(48,true,true,1,0,-1,1,-32768,-32768,-6624,245,32679,2147483647,1111644729,1,147483648,null,147483648,-9061497079404703305,98.1875,38.0903,67.1236,0,4.1641,17.0363,-1.2,10.1437,7.0136,8.1929,35.1616,0,'2024-01-19','2026-02-18','2023-12-18','2027-01-09','2023-01-15 08:32:59.123123','2024-01-31','2024-01-19','2024-01-09','2023-12-10','2024-07-01','2024-01-08','2025-02-17','2023-01-15 08:32:59.123123','2017-10-02','2024-02-18','2024-06-30 12:01:02.123','这不是有项目列表底下煤气前提报账','could him was well really up have see well really','台式机','-','卢海认为不至于医院许愿树河南总代理资质报表','','-','go don''t him yes well as like','说好耀博请重试场景许文及其长期不了一套帐期','名次','写上掏点钱弟子看不出来问过谁知道星星西郊价格','','真正','切换期盼','something there then did if but it''s','?'),(49,false,false,-1,0,81,3,32767,12315,-1,-18884,-1105820371,1,787531014,1603914739,0,32679,1347350265,-1,72.1999,40.123,1.2,-1.2,-1.2,1.2,1.2,1.2,45.0509,94.0948,0,30.1775,'2024-01-19','2024-01-31','2024-01-08','9999-12-31 23:59:59','2024-07-01','2023-12-10','2023-12-15','2023-12-17','2026-02-18',null,'2023-01-15 08:32:59.123123','2024-08-03 13:08:30','2023-12-13','2024-08-03 13:08:30','2026-02-18','2023-12-09','for','-','there','-','y','s','一家','','港湾样品礼拜天没变心连心氛围收藏夹亮度医药上机柜','-','利达汉化补丁也好什么塔式','go want time been there one how is','would','-','短信一开打工的比人韩鸽飞中龙快运郑东排列开机','停产'),(50,null,true,0,1,0,-128,32767,-1,245,245,null,-1,147483648,2147483647,147483648,null,-1,1,40.123,99.0784,1.2,50.1917,56.0902,40.123,57.0612,80.1629,0,0,37.1425,-1.2,'2023-12-16',null,'2024-01-17','2023-12-19','2026-01-18','2027-01-16','2024-06-30 12:01:02.123','2023-12-16','2024-06-30 12:01:02.123','2026-01-18','2024-01-08','2024-01-08','2026-01-18','2026-01-18','2023-12-18','2023-12-13','词库考核列表想不到','?','that''s','can''t','how','-','--','改写太不燕龙','','right','权威','-','','沐浴群殴没错速度黄委会白经理真人青春华栋','?','see'),(51,false,true,7,-1,127,127,null,-28014,21619,-29062,32679,-453263646,1,-571902945,null,8776264616826296648,255479746459051394,1,100.02,300.343,1.2,100.02,0,-1.2,18.0864,0,40.123,300.343,1.2,40.123,'2023-12-10','2015-11-19','2016-10-25','2023-12-13','2024-08-03 13:08:30','9999-12-31 23:59:59','2025-02-18','2023-01-15 08:32:59.123123','2023-12-13','2023-12-13','2014-08-12','2023-12-17','2024-02-18','2027-01-16','2024-02-18','2024-07-01',null,'--','or','been','w','l','杨宗','辅佐','with they one yes yes don''t some why for here','不见拜托才好见过面令我','礼物限价再发白菜不定拿货青青绿色称呼志彬彭伟','一周','文杰很难原厂做到','--','I''ll not you your the your tell he okay','我在政治哦配合雅鹿沙及其办公考虑神秘面对'),(52,true,false,30,-68,11,127,-32768,-30696,245,1,-89768634,871771752,874669061,1683077888,null,-9223372036854775808,-1,-9223372036854775808,94.0219,40.123,1.2,100.02,0,-1.2,57.1140,0,40.123,40.123,40.123,100.02,'2023-12-18','2024-02-18','2023-12-18','2026-02-18','2023-12-11','2023-12-16','2023-12-13','2024-01-08','2023-12-20','2024-06-30 12:01:02.123','9999-12-31 23:59:59','9999-12-31','2024-06-30 12:01:02.123','2023-12-09','2023-12-11','2024-07-01','聊聊伟博普通天下无贼我们见不了吉林依然订货解决','this',null,'can''t','?','because','-','she ok then don''t like that''s we some see','-','推磨刘总出面','i','g','帮忙好处虚妄是吗过的','?','?','--') """ + + qt_sql """ + SELECT count(col_decimal_16__8__undef_signed_index_inverted) FROM ${table1} where col_decimal_16__8__undef_signed_index_inverted is null; + """ + + +} diff --git a/regression-test/suites/inverted_index_p0/test_need_read_data.groovy b/regression-test/suites/inverted_index_p0/test_need_read_data.groovy index 3bff37e261d649..9c80019e46ab95 100644 --- a/regression-test/suites/inverted_index_p0/test_need_read_data.groovy +++ b/regression-test/suites/inverted_index_p0/test_need_read_data.groovy @@ -138,4 +138,26 @@ suite("test_need_read_data", "p0"){ sql "INSERT INTO ${indexTblName3} VALUES (1, 1),(1, -2),(1, -1);" qt_sql "SELECT /*+SET_VAR(enable_common_expr_pushdown=false,inverted_index_skip_threshold=100) */ id FROM ${indexTblName3} WHERE value<0 and abs(value)>1;" qt_sql "SELECT /*+SET_VAR(enable_common_expr_pushdown=true,inverted_index_skip_threshold=100) */ id FROM ${indexTblName3} WHERE value<0 and abs(value)>1;" + + sql "DROP TABLE IF EXISTS tt" + sql """ + CREATE TABLE `tt` ( + `a` int NULL, + `b` varchar(20) NULL, + `c` int NULL, + INDEX idx_source (`b`) USING INVERTED, + ) ENGINE=OLAP + DUPLICATE KEY(`a`, `b`) + COMMENT 'OLAP' + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ insert into tt values (20, 'aa', 30); """ + sql """ insert into tt values (20, null, 30); """ + + qt_sql_11 """ select /*+SET_VAR(enable_count_on_index_pushdown=true) */ count(b) from tt where c = 30; """ + sql """ DROP TABLE IF EXISTS tt """ + } diff --git a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy index ae7489978397c7..5bde4423253744 100644 --- a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy +++ b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy @@ -702,4 +702,22 @@ suite("test_date_function") { ('5', '2020-12-12 12:12:12.666666', '2020-12-12 12:12:12.666666', '2020-12-12 12:12:12.666666', '2020-12-12 12:12:12.666666', '2020-12-12 12:12:12.666666'); """ qt_sql_dt_null_1 """ select unix_timestamp(dtv24), unix_timestamp(dtv20n), unix_timestamp(dv2), unix_timestamp(dv2n), unix_timestamp(str) from dt_null order by k1; """ + + sql """ DROP TABLE IF EXISTS dt_timenull; """ + + sql """ + CREATE TABLE IF NOT EXISTS dt_timenull( + `k1` INT NOT NULL, + `k2` BIGINT NOT NULL + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 5 + properties("replication_num" = "1"); + """ + + sql """ insert into dt_timenull values (1, 0),(2, 100),(3, 123),(4, 219837),(5, -8923),(6, -29313),(7, 2131321231),(8, -21312313),(9,1112345);""" + + qt_sql_time_value """ select k1 , cast(k2 as time) , hour(cast(k2 as time)) , minute(cast(k2 as time)), second(cast(k2 as time)) from dt_timenull order by k1;""" + + + qt_sql_time_value """ select cast(4562632 as time), hour(cast(4562632 as time)) , minute(cast(4562632 as time)) , second(cast(4562632 as time)); """ } diff --git a/regression-test/suites/partition_p0/test_partition_create_tablet_rr.groovy b/regression-test/suites/partition_p0/test_partition_create_tablet_rr.groovy index 836dff938f8c96..f31322facff966 100644 --- a/regression-test/suites/partition_p0/test_partition_create_tablet_rr.groovy +++ b/regression-test/suites/partition_p0/test_partition_create_tablet_rr.groovy @@ -26,7 +26,8 @@ suite("test_partition_create_tablet_rr", "docker") { options.beConfigs += [ 'report_tablet_interval_seconds=1', 'report_disk_state_interval_seconds=1', - "partition_disk_index_lru_size=$partition_disk_index_lru_size" + "partition_disk_index_lru_size=$partition_disk_index_lru_size", + 'sys_log_verbose_modules=*' ] options.beDisks = ['HDD=4','SSD=4'] options.enableDebugPoints()