Merge pull request ByConity#1788 from ByConity/cp-0730-1

Cp 0730 1
edgedelta · Aug 1, 2024 · b1d561a · b1d561a
2 parents 7f78dfc + c48c9ec
commit b1d561a
Show file tree

Hide file tree

Showing 969 changed files with 29,206 additions and 30,792 deletions.
diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt
@@ -60,6 +60,7 @@ option (ENABLE_CLICKHOUSE_PART_TOOLKIT "Enable part_toolkit" ${ENABLE_CLICKHOUSE
 option (ENABLE_CLICKHOUSE_DUMPER "Enable clickhouse-dumper" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_RESOURCE_MANAGER "Service that manage worker resources" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_META_INSPECTOR "Enable meta-inspector in CNCH" ${ENABLE_CLICKHOUSE_ALL})
+option (ENABLE_CLICKHOUSE_STORAGE_TOOLS "Enable storage-tools in CNCH" ${ENABLE_CLICKHOUSE_ALL})
 
 if (NOT USE_NURAFT)
     # RECONFIGURE_MESSAGE_LEVEL should not be used here,
@@ -262,6 +263,7 @@ add_subdirectory (tso)
 add_subdirectory (part-toolkit)
 add_subdirectory (dumper)
 add_subdirectory (meta-inspector)
+add_subdirectory (storage-tools)
 
 if (ENABLE_CLICKHOUSE_KEEPER)
     add_subdirectory (keeper)
@@ -347,6 +349,7 @@ if (CLICKHOUSE_SPLIT_BINARY)
         clickhouse-part-toolkit
         clickhouse-dumper
         clickhouse-meta-inspector
+        clickhouse-storage-tools
     )
 
     if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
@@ -430,6 +433,9 @@ else ()
     if (ENABLE_CLICKHOUSE_META_INSPECTOR)
         clickhouse_target_link_split_lib(clickhouse meta-inspector)
     endif()
+    if (ENABLE_CLICKHOUSE_STORAGE_TOOLS)
+        clickhouse_target_link_split_lib(clickhouse storage-tools)
+    endif()
 
     set (CLICKHOUSE_BUNDLE)
     if (ENABLE_CLICKHOUSE_SERVER)
@@ -538,6 +544,14 @@ else ()
         install (FILES "${CMAKE_CURRENT_BINARY_DIR}/resource_manager" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
         list(APPEND CLICKHOUSE_BUNDLE resource_manager)
     endif ()
+    if (ENABLE_CLICKHOUSE_STORAGE_TOOLS)
+        add_custom_target (clickhouse-storage-tools ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-storage-tools DEPENDS clickhouse)
+        install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-storage-tools" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
+        list(APPEND CLICKHOUSE_BUNDLE clickhouse-storage-tools)
+        add_custom_target (clickhouse-storage_tools ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-storage_tools DEPENDS clickhouse)
+        install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-storage_tools" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
+        list(APPEND CLICKHOUSE_BUNDLE clickhouse-storage_tools)
+    endif ()
 
     install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 

diff --git a/programs/config_tools.h.in b/programs/config_tools.h.in
@@ -24,3 +24,4 @@
 #cmakedefine01 ENABLE_CLICKHOUSE_RESOURCE_MANAGER
 #cmakedefine01 ENABLE_CLICKHOUSE_META_INSPECTOR
 #cmakedefine01 ENABLE_CLICKHOUSE_DUMPER
+#cmakedefine01 ENABLE_CLICKHOUSE_STORAGE_TOOLS
diff --git a/programs/copier/TaskTableAndShard.h b/programs/copier/TaskTableAndShard.h
@@ -310,7 +310,7 @@ inline TaskTable::TaskTable(TaskCluster & parent, const Poco::Util::AbstractConf
     engine_push_str = config.getString(table_prefix + "engine", "rand()");
 
     {
-        ParserStorage parser_storage(ParserSettings::CLICKHOUSE);
+        ParserStorage parser_storage(ParserStorage::TABLE_ENGINE, ParserSettings::CLICKHOUSE);
         engine_push_ast = parseQuery(parser_storage, engine_push_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
         engine_push_partition_key_ast = extractPartitionKey(engine_push_ast);
         primary_key_comma_separated = boost::algorithm::join(extractPrimaryKeyColumnNames(engine_push_ast), ", ");

diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
@@ -36,6 +36,7 @@
 #include <Interpreters/DatabaseCatalog.h>
 #include <Storages/System/attachSystemTables.h>
 #include <Storages/System/attachInformationSchemaTables.h>
+#include <Storages/System/attachMySQLTables.h>
 #include <Interpreters/ProcessList.h>
 #include <Interpreters/executeQuery.h>
 #include <Interpreters/loadMetadata.h>
@@ -341,6 +342,8 @@ try
         attachSystemTablesLocal(*createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE));
         attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
         attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
+        attachMySQL(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::MYSQL));
+        attachMySQL(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::MYSQL_UPPERCASE));
         loadMetadata(global_context);
         DatabaseCatalog::instance().loadDatabases();
         LOG_DEBUG(log, "Loaded metadata.");
@@ -350,6 +353,8 @@ try
         attachSystemTablesLocal(*createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE));
         attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
         attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
+        attachMySQL(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::MYSQL));
+        attachMySQL(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::MYSQL_UPPERCASE));
     }
 
     processQueries();

diff --git a/programs/main.cpp b/programs/main.cpp
@@ -121,6 +121,9 @@ int mainEntryClickHouseResourceManager(int argc, char ** argv);
 #if ENABLE_CLICKHOUSE_TSO_SERVER
 int mainEntryClickHouseTSOServer(int argc, char ** argv);
 #endif
+#if ENABLE_CLICKHOUSE_STORAGE_TOOLS
+int mainEntryStorageTools(int argc, char ** argv);
+#endif
 
 int mainEntryClickHouseHashBinary(int, char **)
 {
@@ -211,6 +214,10 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
     {"tso-server", mainEntryClickHouseTSOServer},
     {"tso_server", mainEntryClickHouseTSOServer},
 #endif
+#if ENABLE_CLICKHOUSE_STORAGE_TOOLS
+    {"storage-tools", mainEntryStorageTools},
+    {"storage_tools", mainEntryStorageTools},
+#endif
 };
 
 

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
@@ -82,6 +82,7 @@
 #include <Storages/StorageReplicatedMergeTree.h>
 #include <Storages/System/attachSystemTables.h>
 #include <Storages/System/attachInformationSchemaTables.h>
+#include <Storages/System/attachMySQLTables.h>
 #include <AggregateFunctions/registerAggregateFunctions.h>
 #include <Functions/registerFunctions.h>
 #include <TableFunctions/registerTableFunctions.h>
@@ -1249,6 +1250,11 @@ int Server::main(const std::vector<std::string> & /*args*/)
     if (global_context->getServerType() == ServerType::cnch_server)
         global_context->setQueryCache(config());
 
+    /// Size of cache for intermediate_result. It is not necessary.
+    size_t intermediate_result_cache_size = config().getUInt64("intermediate_result_cache_size", 1000000000);
+    if (intermediate_result_cache_size)
+        global_context->setIntermediateResultCache(intermediate_result_cache_size);
+
     /// Size of delete bitmap for HaMergeTree engine to be cached in memory; default is 1GB
     size_t delete_bitmap_cache_size = config().getUInt64("delete_bitmap_cache_size", 1073741824);
     global_context->setDeleteBitmapCache(delete_bitmap_cache_size);
@@ -1484,6 +1490,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
         attachSystemTablesServer(*database_catalog.getSystemDatabase(), has_zookeeper);
         attachInformationSchema(global_context, *database_catalog.getDatabase(DatabaseCatalog::INFORMATION_SCHEMA, global_context));
         attachInformationSchema(global_context, *database_catalog.getDatabase(DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE, global_context));
+        attachMySQL(global_context, *database_catalog.getDatabase(DatabaseCatalog::MYSQL, global_context));
+        attachMySQL(global_context, *database_catalog.getDatabase(DatabaseCatalog::MYSQL_UPPERCASE, global_context));
         /// We load temporary database first, because projections need it.
         database_catalog.initializeAndLoadTemporaryDatabase();
         /// Then, load remaining databases

diff --git a/programs/server/config.xml b/programs/server/config.xml
@@ -529,6 +529,7 @@
              executed by any user. You can change this behaviour by setting this to true.
              If it's set to true then this query requires "GRANT SELECT ON information_schema.<table>" just like as for ordinary tables. -->
         <select_from_information_schema_requires_grant>false</select_from_information_schema_requires_grant>
+        <select_from_mysql_requires_grant>false</select_from_mysql_requires_grant>
     </access_control_improvements>
 
     <!-- Default profile of settings. -->

diff --git a/programs/storage-tools/CMakeLists.txt b/programs/storage-tools/CMakeLists.txt
@@ -0,0 +1,7 @@
+set(CLICKHOUSE_STORAGE_TOOLS_SOURCES
+        ${CMAKE_CURRENT_SOURCE_DIR}/StorageTools.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/PartInspector.cpp)
+        # ${CMAKE_CURRENT_SOURCE_DIR}/S3StorageChecker.cpp)
+set(CLICKHOUSE_STORAGE_TOOLS_LINK PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY} ${ROCKSDB_LIBRARY})
+
+clickhouse_program_add(storage-tools)