create branch ed-release/1.0.1

.github/workflows/ci.yml

@@ -2,9 +2,9 @@ name: CI
 on:
   # Triggers the workflow on push or pull request events but only for the "main" branch
   push:
-    branches: [ "master" , "release/0.4.x" ]
+    branches: [ "master" , "release/1.0.x" ]
   pull_request:
-    branches: [ "master" , "release/0.4.x" ]
+    branches: [ "master" , "release/1.0.x" ]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -102,6 +102,7 @@ jobs:
           cp -r $GITHUB_WORKSPACE/tests/clickhouse-test ./
           mkdir queries
           cp -r $GITHUB_WORKSPACE/tests/queries/4_cnch_stateless queries/
+          cp -r $GITHUB_WORKSPACE/tests/queries/8_cnch_S3_only queries/
           cp -r $GITHUB_WORKSPACE/tests/queries/shell_config.sh queries/
           # We need skip-list to skip some tests.
           cp $GITHUB_WORKSPACE/tests/queries/skip_list.json queries/

.github/workflows/release.yml

@@ -11,7 +11,7 @@ jobs:
       - name: Setup Git Proxy.
         if: ${{ runner.name != 'ec2-aws-id4-10.10.129.157' }}
         run: |
-          git config --global http.proxy http://${{ secrets.HTTP_PROXY }}
+          git config --global http.https://github.com.proxy http://${{ secrets.VIP_PROXY }}:3128
       - name: Setup Environment Varialbes
         run: |
           export PROJECT_NAME=byconity-$(cat /etc/hostname)
@@ -48,17 +48,17 @@ jobs:
           tag: ${{ github.ref }}
           overwrite: true
           file_glob: true
+      - name: Login to Docker Hub.
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Build Docker.
         run: |
           cd ByConity/docker/debian/release
           BUILD_TYPE=Release CMAKE_FLAGS="-DENABLE_JAVA_EXTENSIONS=0" TAG=${{ github.ref_name }} make image-github
           cd -
           docker images
-      - name: Login to Docker Hub.
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Push Dockers.
         run: |
           docker push byconity/byconity:${{ github.ref_name }}
@@ -68,6 +68,22 @@ jobs:
           docker tag byconity/byconity:${{ github.ref_name }} byconity/byconity:stable
           docker push byconity/byconity:stable
           docker image rm -f byconity/byconity:stable
+      - name: Login to Volcano Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: byconity-cn-beijing.cr.volces.com
+          username: ${{ secrets.VOLC_CR_USER }}
+          password: ${{ secrets.VOLC_CR_PASS }}
+      - name: Push Dockers to Volcano CR
+        run: |
+          docker tag  byconity/byconity:${{ github.ref_name }} byconity-cn-beijing.cr.volces.com/byconity/byconity:${{ github.ref_name}}
+          docker push byconity-cn-beijing.cr.volces.com/byconity/byconity:${{ github.ref_name }}
+      - name: Tag as Stable then push to Volcano CR
+        if: "!github.event.release.prerelease"
+        run: |
+          docker tag byconity/byconity:${{ github.ref_name }} byconity-cn-beijing.cr.volces.com/byconity/byconity:stable
+          docker push byconity-cn-beijing.cr.volces.com/byconity/byconity:stable
+          docker image rm -f byconity-cn-beijing.cr.volces.com/byconity/byconity:stable
       - name: Cleanup Data.
         if: always()
         run: |

.gitmodules

@@ -286,7 +286,7 @@
 	url = https://github.com/google/benchmark.git
 [submodule "contrib/hivemetastore"]
 	path = contrib/hivemetastore
-	url = https://github.com/ClickHouse/hive-metastore.git
+	url = https://github.com/ByConity/hivemetastore.git
 [submodule "contrib/udns"]
 	path = contrib/udns
 	url = https://github.com/ortclib/udns.git

base/common/ThreadLocal.h

@@ -102,7 +102,10 @@ template<class T, class I>
 class ThreadLocalManagedBase {
 public:
     ThreadLocalManagedBase() noexcept {
-        bthread_key_create(&key, [](void *obj) { delete static_cast<T *>(obj); });
+        bthread_key_create(&key, [](void * obj) {
+            static_cast<T *>(obj)->~T();
+            free(obj);
+        });
     }
 
     ~ThreadLocalManagedBase() noexcept { bthread_key_delete(key); }
@@ -128,9 +131,3 @@ class ThreadLocalManagedBase {
 private:
     bthread_key_t key;
 };
-
-template<class T>
-class ThreadLocalManaged : public ThreadLocalManagedBase<T, ThreadLocalManaged<T>> {
-public:
-    static void *create() { return new T(); }
-};

base/common/bthread_exception.cpp

@@ -1,5 +1,14 @@
 #include <common/ThreadLocal.h>
 
+template<class T>
+class ThreadLocalManagedUntracked : public ThreadLocalManagedBase<T, ThreadLocalManagedUntracked<T>> {
+public:
+    static void *create() {
+        void * p = malloc(sizeof(T));
+        return new (p) T();
+    }
+};
+
 namespace {
 struct __cxa_eh_globals {
     void *   caughtExceptions;
@@ -11,7 +20,7 @@ namespace __cxxabiv1 {
 
 namespace {
     __cxa_eh_globals * __globals () {
-        static ThreadLocalManaged<__cxa_eh_globals> eh_globals;
+        static ThreadLocalManagedUntracked<__cxa_eh_globals> eh_globals;
         return eh_globals.get();
         }
     }

base/common/chrono_io.h

@@ -39,7 +39,7 @@ std::string to_string(const std::chrono::time_point<Clock, Duration> & tp)
     // Don't use DateLUT because it shows weird characters for
     // TimePoint::max(). I wish we could use C++20 format, but it's not
     // there yet.
-    // return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp));
+    // return DateLUT::serverTimezoneInstance().timeToString(std::chrono::system_clock::to_time_t(tp));
 
     auto in_time_t = std::chrono::system_clock::to_time_t(tp);
     return to_string(in_time_t);

base/daemon/BaseDaemon.cpp

@@ -1128,7 +1128,7 @@ void BaseDaemon::shouldSetupWatchdog(char * argv0_)
 void BaseDaemon::setupWatchdog()
 {
     /// Initialize in advance to avoid double initialization in forked processes.
-    DateLUT::instance();
+    DateLUT::serverTimezoneInstance();
 
     std::string original_process_name;
     if (argv0)

ci_scripts/config/users.xml

@@ -21,6 +21,7 @@
             <background_schedule_pool_size>6</background_schedule_pool_size>
             <min_insert_block_size_bytes>8589934592</min_insert_block_size_bytes>
             <cnch_max_cached_storage>50000</cnch_max_cached_storage>
+            <max_query_size>0</max_query_size>
         </default>
 
         <!-- Profile that allows only read queries. -->

contrib/hualloc/hu_alloc.cpp

@@ -71,15 +71,15 @@ void* ReclaimThread(void *args)
 {
     // keep & max can be separate for large & segment spaces
     const char * sleep_second = std::getenv("HUALLOC_CLAIM_INTERVAL");
-    int sleep = 3;
+    int sleep = 1;
     try
     {
         if (sleep_second && std::strlen(sleep_second) > 0)
             sleep = atoi(sleep_second);
     }
     catch(...)
     {
-        sleep = 3;
+        sleep = 1;
     }
 
     yint cached = *(yint *) args;

contrib/hualloc/hu_alloc.h

@@ -1843,7 +1843,7 @@ static void DumpLocalAllocMasksLocked(char *segment)
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////
 static yint ReclaimKeepSize = 1 * 1024 * (1ull << 20);
-static yint ReclaimMaxReclaim = 512 * (1ull << 20);
+static yint ReclaimMaxReclaim = 1* 1024 * (1ull << 20);
 
 static void hu_init()
 {

docker/CI/multi-servers/server.yml

@@ -1,11 +1,15 @@
 logger:
-  level: debug
+  level: trace
   log: /var/log/byconity/out.log
   errorlog: /var/log/byconity/err.log
   testlog: /var/log/byconity/test.log
   size: 1000M
   count: 10
   console: true
+additional_services:
+  GIS: 1
+  VectorSearch: 1
+  FullTextSearch: 1
 http_port: 21557
 rpc_port: 30605
 tcp_port: 52145
@@ -15,6 +19,13 @@ exchange_status_port: 60611
 interserver_http_port: 30491
 mysql_port: 9004
 listen_host: "0.0.0.0"
+prometheus:
+  endpoint: "/metrics"
+  port: 0
+  metrics: true
+  events: true
+  asynchronous_metrics: true
+  part_metrics: false
 cnch_type: server
 max_connections: 4096
 keep_alive_timeout: 3
@@ -89,7 +100,7 @@ service_discovery:
     psm: data.cnch.server
     node:
       - host: server-0
-        hostname: server
+        hostname: server-0
         ports:
           port:
             - name: PORT2

docker/CI/multi-servers/worker.yml

@@ -55,6 +55,11 @@ storage_configuration:
           default: local_disk
           disk: local_disk
 hdfs_addr: "hdfs://hdfs-namenode:9000"
+cnch_unique_table_log:
+  database: cnch_system
+  table: cnch_unique_table_log
+  flush_max_row_count: 10000
+  flush_interval_milliseconds: 7500
 query_log:
   database: system
   table: query_log

docker/CI/multi-workers/server.yml

@@ -5,6 +5,10 @@ logger:
   testlog: /var/log/byconity/test.log
   size: 1000M
   count: 10
+additional_services:
+  GIS: 1
+  VectorSearch: 1
+  FullTextSearch: 1
 http_port: 21557
 rpc_port: 30605
 tcp_port: 52145
@@ -14,6 +18,13 @@ exchange_status_port: 60611
 interserver_http_port: 30491
 mysql_port: 9004
 listen_host: "0.0.0.0"
+prometheus:
+  endpoint: "/metrics"
+  port: 0
+  metrics: true
+  events: true
+  asynchronous_metrics: true
+  part_metrics: false
 cnch_type: server
 max_connections: 4096
 keep_alive_timeout: 3

docker/CI/s3/server.yml

@@ -1,12 +1,16 @@
 # Auto-generated! Please do not modify this file directly. Refer to 'convert-hdfs-configs-to-s3.sh'.
 logger:
-  level: debug
+  level: trace
   log: /var/log/byconity/out.log
   errorlog: /var/log/byconity/err.log
   testlog: /var/log/byconity/test.log
   size: 1000M
   count: 10
   console: true
+additional_services:
+  GIS: 1
+  VectorSearch: 1
+  FullTextSearch: 1
 http_port: 21557
 rpc_port: 30605
 tcp_port: 52145
@@ -16,6 +20,13 @@ exchange_status_port: 60611
 interserver_http_port: 30491
 mysql_port: 9004
 listen_host: "0.0.0.0"
+prometheus:
+  endpoint: "/metrics"
+  port: 0
+  metrics: true
+  events: true
+  asynchronous_metrics: true
+  part_metrics: false
 cnch_type: server
 max_connections: 4096
 keep_alive_timeout: 3
@@ -98,7 +109,7 @@ service_discovery:
     psm: data.cnch.server
     node:
       - host: server-0
-        hostname: server
+        hostname: server-0
         ports:
           port:
             - name: PORT2

docker/CI/s3/worker.yml

@@ -63,6 +63,11 @@ storage_configuration:
           disk: s3_disk
   # To avoid break hard-coded test cases.
   cnch_default_policy: cnch_default_hdfs
+cnch_unique_table_log:
+  database: cnch_system
+  table: cnch_unique_table_log
+  flush_max_row_count: 10000
+  flush_interval_milliseconds: 7500
 query_log:
   database: system
   table: query_log

docker/ci-deploy/config/daemon-manager.yml

@@ -35,6 +35,10 @@ daemon_manager:
       name: DEDUP_WORKER
       interval: 3000
       disable: 0
+    - 
+      name: TXN_GC
+      interval: 3000
+      disable: 0
 
 hdfs_addr: "hdfs://COMPOSE_PROJECT_NAME-hdfs-namenode:9000"
 storage_configuration:

docker/ci-deploy/config/server.yml

@@ -14,6 +14,13 @@ exchange_status_port: 60611
 interserver_http_port: 30491
 mysql_port: 9004
 listen_host: "0.0.0.0"
+prometheus:
+  endpoint: "/metrics"
+  port: 0
+  metrics: true
+  events: true
+  asynchronous_metrics: true
+  part_metrics: false
 cnch_type: server
 max_connections: 4096
 keep_alive_timeout: 3

docker/ci-deploy/config/worker.yml

@@ -13,6 +13,13 @@ exchange_port: 47447
 exchange_status_port: 60611
 interserver_http_port: 30491
 listen_host: "0.0.0.0"
+prometheus:
+  endpoint: "/metrics"
+  port: 0
+  metrics: true
+  events: true
+  asynchronous_metrics: true
+  part_metrics: false
 cnch_type: worker
 max_connections: 4096
 keep_alive_timeout: 3

docker/ci-deploy/config_for_s3_storage/daemon-manager.yml

@@ -35,6 +35,10 @@ daemon_manager:
       name: DEDUP_WORKER
       interval: 3000
       disable: 0
+    - 
+      name: TXN_GC
+      interval: 3000
+      disable: 0
 
 storage_configuration:
   disks:

docker/ci-deploy/config_for_s3_storage/server.yml

@@ -15,6 +15,13 @@ exchange_status_port: 60611
 interserver_http_port: 30491
 mysql_port: 9004
 listen_host: "0.0.0.0"
+prometheus:
+  endpoint: "/metrics"
+  port: 0
+  metrics: true
+  events: true
+  asynchronous_metrics: true
+  part_metrics: false
 cnch_type: server
 max_connections: 4096
 keep_alive_timeout: 3

docker/ci-deploy/docker-compose-s3.yml

@@ -121,6 +121,7 @@ services:
       - SYS_PTRACE
   minio:
     ports:
+      - 19000:9000
       - 19001:9001
     environment:
       - MINIO_ROOT_USER=minio