Merge pull request ByConity#1892 from ByConity/cp-1011

Cp 1011
edgedelta · Oct 11, 2024 · 0bca77a · 0bca77a
2 parents 4d59b68 + 1594e18
commit 0bca77a
Show file tree

Hide file tree

Showing 145 changed files with 5,006 additions and 1,301 deletions.
diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp
@@ -137,6 +137,7 @@ static DB::PipeFDs signal_pipe;
 
 #if USE_BREAKPAD
 static bool use_minidump = true;
+static std::atomic_bool init_minidump{false};
 static std::shared_ptr<google_breakpad::MinidumpDescriptor> descriptor;
 static std::shared_ptr<google_breakpad::ExceptionHandler> eh;
 
@@ -192,7 +193,9 @@ static void call_default_signal_handler(int sig)
 {
     signal(sig, SIG_DFL);
 #if USE_BREAKPAD
-    if (use_minidump)
+    bool not_init = false;
+    /// Only initialize minidump once
+    if (use_minidump && init_minidump.compare_exchange_strong(not_init, true))
         eh = std::shared_ptr<google_breakpad::ExceptionHandler>(
             new google_breakpad::ExceptionHandler(*descriptor, nullptr, dumpCallbackError, nullptr, true, -1));
 #endif

diff --git a/docker/CI/docker-compose-nexusfs.yml b/docker/CI/docker-compose-nexusfs.yml
@@ -0,0 +1,174 @@
+version: "3"
+
+services:
+  # After upgrade to docker-compose v2, we could use `include` instead of `extend`.
+  hdfs-namenode:
+    extends:
+      file: ./common/hdfs.yml
+      service: hdfs-namenode
+  hdfs-datanode:
+    extends:
+      file: ./common/hdfs.yml
+      service: hdfs-datanode
+  fdb:
+    extends:
+      file: ./common/fdb.yml
+      service: fdb
+  my_mysql:
+    extends:
+      file: ./common/mysql.yml
+      service: my_mysql
+  tso:
+    image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1
+    command: bash -c "fdbcli -C /config/fdb.cluster --exec \"configure new single ssd\"; tso-server --config-file /config/tso.yml"
+    depends_on:
+      - fdb
+      - hdfs-namenode
+    volumes:
+      - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro
+      - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro
+      - ./nexusfs/:/config/:ro
+      - ./test_output/tso/:/var/log/byconity/:rw
+    environment: &env
+      LD_LIBRARY_PATH: /opt/byconity/lib
+      PATH: /opt/byconity/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+      ASAN_OPTIONS:
+      TSAN_OPTIONS:
+      IS_CI_ENV: 1
+      CI_PIPELINE_NAME: CI
+    cap_add:
+      - SYS_PTRACE
+    healthcheck:
+      test: ["CMD", "curl", "localhost:18845"]
+      interval: 5s
+
+  server-0:
+    image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1
+    command: bash -c "(udf-manager --config-file /config/server.yml & clickhouse-server --config-file /config/server.yml)"
+    depends_on:
+      tso:
+        condition: service_healthy
+    ports:
+      - "9000:52145"
+      - "127.0.0.1:8123:21557"
+      - "127.0.0.1:9004:9004"
+    environment:
+      <<: *env
+      SERVER_ID: server-0
+    volumes:
+      - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro
+      - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro
+      - ./nexusfs/:/config/:ro
+      - ./test_output/server-0/:/var/log/byconity/:rw
+      - ./queries/:/opt/byconity/queries/:ro
+    cap_add:
+      - SYS_PTRACE
+    healthcheck:
+      test: ["CMD", "curl", "localhost:21557"]
+      interval: 5s
+
+  server-1:
+    image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1
+    command: bash -c "(udf-manager --config-file /config/server.yml & clickhouse-server --config-file /config/server.yml)"
+    depends_on:
+      tso:
+        condition: service_healthy
+    ports:
+      - "9001:52145"
+      - "127.0.0.1:8124:21557"
+    environment:
+      <<: *env
+      SERVER_ID: server-1
+    volumes:
+      - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro
+      - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro
+      - ./nexusfs/:/config/:ro
+      - ./test_output/server-1/:/var/log/byconity/:rw
+      - ./queries/:/opt/byconity/queries/:ro
+    cap_add:
+      - SYS_PTRACE
+    healthcheck:
+      test: ["CMD", "curl", "localhost:52145"]
+      interval: 5s
+
+  worker-write:
+    image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1
+    command: bash -c "clickhouse-server --config-file /config/worker.yml"
+    depends_on:
+      - server-0
+      - server-1
+    ports:
+      - "52149:52145"
+    environment:
+      <<: *env
+      WORKER_GROUP_ID: wg_write
+      VIRTUAL_WAREHOUSE_ID: vw_write
+      WORKER_ID: w0
+    volumes:
+      - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro
+      - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro
+      - ./nexusfs/:/config/:ro
+      - ./test_output/worker-write/:/var/log/byconity/:rw
+      - ./queries/:/opt/byconity/queries/:ro
+    cap_add:
+      - SYS_PTRACE
+  worker-default:
+    image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1
+    command: bash -c "(udf-manager --config-file /config/worker.yml & clickhouse-server --config-file /config/worker.yml)"
+    depends_on:
+      - server-0
+      - server-1
+    environment:
+      <<: *env
+      WORKER_GROUP_ID: wg_default
+      VIRTUAL_WAREHOUSE_ID: vw_default
+      WORKER_ID: r0
+    volumes:
+      - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro
+      - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro
+      - ./nexusfs/:/config/:ro
+      - ./test_output/worker-default/:/var/log/byconity/:rw
+      - ./queries/:/opt/byconity/queries/:ro
+    cap_add:
+      - SYS_PTRACE
+  daemon-manager:
+    image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1
+    command: bash -c "daemon-manager --config-file ./config/daemon-manager.yml"
+    depends_on:
+      server-0:
+        condition: service_healthy
+      server-1:
+        condition: service_healthy
+    environment:
+      <<: *env
+    volumes:
+      - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro
+      - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro
+      - ./nexusfs/:/config/:ro
+      - ./test_output/daemon-manager/:/var/log/byconity/:rw
+    cap_add:
+      - SYS_PTRACE
+    restart: always
+
+  resource-manager:
+    image: hub.byted.org/bytehouse/debian.bullseye.fdb.udf:0.1
+    command: bash -c "resource-manager --config-file /config/resource-manager.yml"
+    depends_on:
+      - tso
+    volumes:
+      - ${CNCH_BINARY_PATH}/:/opt/byconity/bin/:ro
+      - ${CNCH_LIBRARY_PATH}/:/opt/byconity/lib/:ro
+      - ./nexusfs/:/config/:ro
+      - ./test_output/rm/:/var/log/byconity/:rw
+    environment:
+      <<: *env
+    cap_add:
+      - SYS_PTRACE
+
+volumes:
+  fdb-data:
+    external: false
+  hdfs-namenode:
+    external: false
+  hdfs-datanode:
+    external: false
diff --git a/docker/CI/nexusfs/conf.d/catalog.yml b/docker/CI/nexusfs/conf.d/catalog.yml
@@ -0,0 +1,6 @@
+catalog:
+  name_space: default
+catalog_service:
+  type: fdb
+  fdb:
+    cluster_file: /config/fdb.cluster
diff --git a/docker/CI/nexusfs/conf.d/service_discovery.yml b/docker/CI/nexusfs/conf.d/service_discovery.yml
@@ -0,0 +1,115 @@
+service_discovery:
+  mode: local
+  cluster: default
+  disable_cache: false
+  cache_timeout: 5
+  server:
+    psm: data.cnch.server
+    node:
+      - host: server-0
+        hostname: server-0
+        ports:
+          port:
+            - name: PORT2
+              value: 21557
+            - name: PORT1
+              value: 30605
+            - name: PORT0
+              value: 52145
+            - name: PORT4
+              value: 27651
+            - name: PORT3
+              value: 45443
+            - name: PORT5
+              value: 47447
+            - name: PORT6
+              value: 60611
+      - host: server-1
+        hostname: server-1
+        ports:
+          port:
+            - name: PORT2
+              value: 21557
+            - name: PORT1
+              value: 30605
+            - name: PORT0
+              value: 52145
+            - name: PORT4
+              value: 27651
+            - name: PORT3
+              value: 45443
+            - name: PORT5
+              value: 47447
+            - name: PORT6
+              value: 60611
+  tso:
+    psm: data.cnch.tso
+    node:
+      host: tso
+      hostname: tso
+      ports:
+        port:
+          - name: PORT0
+            value: 18845
+          - name: PORT2
+            value: 9181
+  resource_manager:
+    psm: data.cnch.resource_manager
+    node:
+      host: resource-manager
+      hostname: resource-manager
+      ports:
+        port:
+          name: PORT0
+          value: 28989
+  daemon_manager:
+    psm: data.cnch.daemon_manager
+    node:
+      host: daemon-manager
+      hostname: daemon-manager
+      ports:
+        port:
+          name: PORT0
+          value: 17553
+  vw_psm: data.cnch.vw
+  vw:
+    psm: data.cnch.vw
+    node:
+      - host: worker-write
+        hostname: worker-write
+        ports:
+          port:
+            - name: PORT2
+              value: 21557
+            - name: PORT1
+              value: 30605
+            - name: PORT0
+              value: 52145
+            - name: PORT4
+              value: 27651
+            - name: PORT3
+              value: 45443
+            - name: PORT5
+              value: 47447
+            - name: PORT6
+              value: 60611
+        vw_name: vw_write
+      - host: worker-default
+        hostname: worker-default
+        ports:
+          port:
+            - name: PORT2
+              value: 21557
+            - name: PORT1
+              value: 30605
+            - name: PORT0
+              value: 52145
+            - name: PORT4
+              value: 27651
+            - name: PORT3
+              value: 45443
+            - name: PORT5
+              value: 47447
+            - name: PORT6
+              value: 60611
+        vw_name: vw_default
diff --git a/docker/CI/nexusfs/conf.d/storage.yml b/docker/CI/nexusfs/conf.d/storage.yml
@@ -0,0 +1,18 @@
+hdfs_addr: hdfs://hdfs-namenode:9000
+storage_configuration:
+  disks:
+    hdfs_disk:
+      path: /user/clickhouse/
+      type: bytehdfs
+    local_disk:
+      path: /var/byconity/data/
+      type: local
+  policies:
+    default:
+      volumes:
+        hdfs:
+          default: hdfs_disk
+          disk: hdfs_disk
+        local:
+          default: local_disk
+          disk: local_disk
diff --git a/docker/CI/nexusfs/daemon-manager.yml b/docker/CI/nexusfs/daemon-manager.yml
@@ -0,0 +1,63 @@
+logger:
+  level: trace
+  log: /var/log/byconity/out.log
+  errorlog: /var/log/byconity/err.log
+  testlog: /var/log/byconity/test.log
+  size: 1000M
+  count: 10
+http_port: 21557
+rpc_port: 30605
+tcp_port: 52145
+ha_tcp_port: 26247
+exchange_port: 47447
+exchange_status_port: 60611
+interserver_http_port: 30491
+listen_host: "0.0.0.0"
+cnch_type: server
+max_connections: 4096
+keep_alive_timeout: 3
+max_concurrent_queries: 200
+uncompressed_cache_size: 8589934592
+mark_cache_size: 5368709120
+path: /var/byconity/
+tmp_path: /var/byconity/tmp_data/
+users_config: /config/users.yml
+default_profile: default
+default_database: default
+timezone: Europe/Moscow
+mlock_executable: false
+macros:
+  "-incl": macros
+  "-optional": true
+builtin_dictionaries_reload_interval: 3600
+max_session_timeout: 3600
+default_session_timeout: 60
+dictionaries_config: "*_dictionary.xml"
+format_schema_path: /var/byconity/format_schemas/
+perQuery: 1
+daemon_manager:
+  port: 17553
+  daemon_jobs:
+    job:
+      - name: PART_GC
+        interval: 10000
+        disable: 0
+      - name: PART_MERGE
+        interval: 10000
+        disable: 0
+      - name: CONSUMER
+        interval: 10000
+        disable: 0
+      - name: GLOBAL_GC
+        interval: 5000
+        disable: 1
+      - name: PART_CLUSTERING
+        interval: 30000
+        disable: 0
+      - name: DEDUP_WORKER
+        interval: 3000
+        disable: 0
+      # Increasing the frequency of recycling in a test environment
+      - name: TXN_GC
+        interval: 3000
+        disable: 0