[exporter/clickhouse] Update readme documentation (open-telemetry#12993)

gregoryfranklin · Aug 9, 2022 · a552fb7 · a552fb7
1 parent 8ad1dcb
commit a552fb7
Show file tree

Hide file tree

Showing 3 changed files with 102 additions and 41 deletions.
diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md
@@ -25,20 +25,72 @@ Support time-series graph, table and logs.
 
 2. Analyze logs via powerful clickhouse SQL.
 
+- Get log severity count time series.
 ```clickhouse
-/* get error count about my service last 1 hour.*/
-SELECT count(*)
+SELECT toDateTime(toStartOfInterval(Timestamp, INTERVAL 60 second)) as time, SeverityText, count() as count
 FROM otel_logs
-WHERE SeverityText='ERROR' AND Timestamp >= NOW() - INTERVAL 1 HOUR;
-/* find log.*/
-SELECT * 
+WHERE time >= NOW() - INTERVAL 1 HOUR
+GROUP BY SeverityText, time
+ORDER BY time;
+```
+- Find any log.
+```clickhouse
+SELECT Timestamp as log_time, Body
+FROM otel_logs 
+WHERE Timestamp >= NOW() - INTERVAL 1 HOUR
+Limit 100;
+```
+- Find log with specific service.
+```clickhouse
+SELECT Timestamp as log_time, Body
+FROM otel_logs 
+WHERE ServiceName = 'clickhouse-exporter' AND Timestamp >= NOW() - INTERVAL 1 HOUR
+Limit 100;
+```
+- Find log with specific attribute.
+```clickhouse
+SELECT Timestamp as log_time, Body
 FROM otel_logs 
-WHERE Timestamp >= NOW() - INTERVAL 1 HOUR;
-/* find log with specific attribute .*/
-SELECT Body
+WHERE LogAttributes['container_name'] = '/example_flog_1' AND Timestamp >= NOW() - INTERVAL 1 HOUR
+Limit 100;
+```
+- Find log with body contain string token.
+```clickhouse
+SELECT Timestamp as log_time, Body
 FROM otel_logs 
-WHERE LogAttributes.Value[indexOf(LogAttributes.Key, 'http_method')] = 'post' AND Timestamp >= NOW() - INTERVAL 1 HOUR;
+WHERE hasToken(Body, 'http') AND Timestamp >= NOW() - INTERVAL 1 HOUR
+Limit 100;
 ```
+- Find log with body contain string.
+```clickhouse
+SELECT Timestamp as log_time, Body
+FROM otel_logs 
+WHERE Body like '%http%' AND Timestamp >= NOW() - INTERVAL 1 HOUR
+Limit 100;
+```
+- Find log with body regexp match string.
+```clickhouse
+SELECT Timestamp as log_time, Body
+FROM otel_logs 
+WHERE match(Body, 'http') AND Timestamp >= NOW() - INTERVAL 1 HOUR
+Limit 100;
+```
+- Find log with body json extract.
+```clickhouse
+SELECT Timestamp as log_time, Body
+FROM otel_logs 
+WHERE JSONExtractFloat(Body, 'bytes')>1000 AND Timestamp >= NOW() - INTERVAL 1 HOUR
+Limit 100;
+```
+
+## Performance Guide
+
+A single clickhouse instance with 32 CPU cores and 128 GB RAM can handle around 20 TB (20 Billion) logs per day, 
+the data compression ratio is 7 ~ 11, the compressed data store in disk is 1.8 TB ~ 2.85 TB,
+add more clickhouse node to cluster can increase linearly.
+
+The otel-collector with `otlp receiver/batch processor/clickhouse tcp exporter` can process 
+around 40k/s logs entry per CPU cores, add more collector node can increase linearly.
 
 ## Configuration options
 
@@ -68,10 +120,11 @@ receivers:
   examplereceiver:
 processors:
   batch:
-    timeout: 10s
+    timeout: 5s
+    send_batch_size: 100000
 exporters:
   clickhouse:
-    dsn: tcp://127.0.0.1:9000?database=default
+    dsn: tcp://127.0.0.1:9000/default
     ttl_days: 3
     timeout: 5s
     retry_on_failure:
@@ -90,30 +143,30 @@ service:
 ## Schema
 
 ```clickhouse
-CREATE TABLE IF NOT EXISTS otel_logs (
-    Timestamp DateTime CODEC(Delta, ZSTD(1)),
-    TraceId String CODEC(ZSTD(1)),
-    SpanId String CODEC(ZSTD(1)),
-    TraceFlags UInt32,
-    SeverityText LowCardinality(String) CODEC(ZSTD(1)),
-    SeverityNumber Int32,
-    Body String CODEC(ZSTD(1)),
-    ResourceAttributes Nested
-        (
-        Key LowCardinality(String),
-        Value String
-        ) CODEC(ZSTD(1)),
-    LogAttributes Nested
-        (
-        Key LowCardinality(String),
-        Value String
-        ) CODEC(ZSTD(1)),
-INDEX idx_attr_keys ResourceAttributes.Key TYPE bloom_filter(0.01) GRANULARITY 64,
-INDEX idx_res_keys LogAttributes.Key TYPE bloom_filter(0.01) GRANULARITY 64
-) ENGINE MergeTree()
-TTL Timestamp + INTERVAL 3 DAY
-PARTITION BY toDate(Timestamp)
-ORDER BY (toUnixTimestamp(Timestamp));
+CREATE TABLE otel_logs
+(
+    `Timestamp` DateTime64(9) CODEC(Delta, ZSTD(1)),
+    `TraceId` String CODEC(ZSTD(1)),
+    `SpanId` String CODEC(ZSTD(1)),
+    `TraceFlags` UInt32 CODEC(ZSTD(1)),
+    `SeverityText` LowCardinality(String) CODEC(ZSTD(1)),
+    `SeverityNumber` Int32 CODEC(ZSTD(1)),
+    `ServiceName` LowCardinality(String) CODEC(ZSTD(1)),
+    `Body` String CODEC(ZSTD(1)),
+    `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1
+)
+    ENGINE = MergeTree
+        PARTITION BY toDate(Timestamp)
+        ORDER BY (ServiceName, SeverityText, toUnixTimestamp(Timestamp), TraceId)
+        TTL toDateTime(Timestamp) + toIntervalDay(3)
+        SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
 ```
 
 [alpha]:https://github.com/open-telemetry/opentelemetry-collector#alpha

diff --git a/exporter/clickhouseexporter/example/docker-compose.yml b/exporter/clickhouseexporter/example/docker-compose.yml
@@ -13,7 +13,7 @@ services:
     container_name: otel
     command:
       - "--config=/etc/otel-collector-config.yml"
-      - "--set=service.telemetry.logs.level=DEBUG"
+      - "--set=service.telemetry.logs.level=INFO"
     volumes:
       - ./otel-collector-config.yml:/etc/otel-collector-config.yml
     ports:

diff --git a/exporter/clickhouseexporter/example/otel-collector-config.yml b/exporter/clickhouseexporter/example/otel-collector-config.yml
@@ -5,17 +5,25 @@ receivers:
 processors:
   batch:
     send_batch_size: 100000
-    timeout: 2s
+    timeout: 5s
   memory_limiter:
     check_interval: 2s
     limit_mib: 1800
     spike_limit_mib: 500
-
+  resourcedetection/system:
+    detectors: ["system"]
+    system:
+      hostname_sources: ["os"]
+  resource:
+    attributes:
+      - key: service.name
+        value: "clickhouse-exporter"
+        action: upsert
 exporters:
   clickhouse:
-    dsn: tcp://clickhouse:9000?database=default
+    dsn: tcp://clickhouse:9000/default
     ttl_days: 3
-    timeout: 5s
+    timeout: 10s
     sending_queue:
       queue_size: 100
     retry_on_failure:
@@ -35,5 +43,5 @@ service:
   pipelines:
     logs:
       receivers: [ fluentforward ]
-      processors: [ memory_limiter, batch ]
+      processors: [ memory_limiter, resourcedetection/system, resource, batch ]
       exporters: [ clickhouse ]