From a552fb777808e00545d564af7d2849d9b6f285de Mon Sep 17 00:00:00 2001 From: Jimmie Han Date: Tue, 9 Aug 2022 12:16:20 +0800 Subject: [PATCH] [exporter/clickhouse] Update readme documentation (#12993) --- exporter/clickhouseexporter/README.md | 123 +++++++++++++----- .../example/docker-compose.yml | 2 +- .../example/otel-collector-config.yml | 18 ++- 3 files changed, 102 insertions(+), 41 deletions(-) diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md index 2d0682d43b6f..ee995a5b8d4e 100644 --- a/exporter/clickhouseexporter/README.md +++ b/exporter/clickhouseexporter/README.md @@ -25,20 +25,72 @@ Support time-series graph, table and logs. 2. Analyze logs via powerful clickhouse SQL. +- Get log severity count time series. ```clickhouse -/* get error count about my service last 1 hour.*/ -SELECT count(*) +SELECT toDateTime(toStartOfInterval(Timestamp, INTERVAL 60 second)) as time, SeverityText, count() as count FROM otel_logs -WHERE SeverityText='ERROR' AND Timestamp >= NOW() - INTERVAL 1 HOUR; -/* find log.*/ -SELECT * +WHERE time >= NOW() - INTERVAL 1 HOUR +GROUP BY SeverityText, time +ORDER BY time; +``` +- Find any log. +```clickhouse +SELECT Timestamp as log_time, Body +FROM otel_logs +WHERE Timestamp >= NOW() - INTERVAL 1 HOUR +Limit 100; +``` +- Find log with specific service. +```clickhouse +SELECT Timestamp as log_time, Body +FROM otel_logs +WHERE ServiceName = 'clickhouse-exporter' AND Timestamp >= NOW() - INTERVAL 1 HOUR +Limit 100; +``` +- Find log with specific attribute. +```clickhouse +SELECT Timestamp as log_time, Body FROM otel_logs -WHERE Timestamp >= NOW() - INTERVAL 1 HOUR; -/* find log with specific attribute .*/ -SELECT Body +WHERE LogAttributes['container_name'] = '/example_flog_1' AND Timestamp >= NOW() - INTERVAL 1 HOUR +Limit 100; +``` +- Find log with body contain string token. +```clickhouse +SELECT Timestamp as log_time, Body FROM otel_logs -WHERE LogAttributes.Value[indexOf(LogAttributes.Key, 'http_method')] = 'post' AND Timestamp >= NOW() - INTERVAL 1 HOUR; +WHERE hasToken(Body, 'http') AND Timestamp >= NOW() - INTERVAL 1 HOUR +Limit 100; ``` +- Find log with body contain string. +```clickhouse +SELECT Timestamp as log_time, Body +FROM otel_logs +WHERE Body like '%http%' AND Timestamp >= NOW() - INTERVAL 1 HOUR +Limit 100; +``` +- Find log with body regexp match string. +```clickhouse +SELECT Timestamp as log_time, Body +FROM otel_logs +WHERE match(Body, 'http') AND Timestamp >= NOW() - INTERVAL 1 HOUR +Limit 100; +``` +- Find log with body json extract. +```clickhouse +SELECT Timestamp as log_time, Body +FROM otel_logs +WHERE JSONExtractFloat(Body, 'bytes')>1000 AND Timestamp >= NOW() - INTERVAL 1 HOUR +Limit 100; +``` + +## Performance Guide + +A single clickhouse instance with 32 CPU cores and 128 GB RAM can handle around 20 TB (20 Billion) logs per day, +the data compression ratio is 7 ~ 11, the compressed data store in disk is 1.8 TB ~ 2.85 TB, +add more clickhouse node to cluster can increase linearly. + +The otel-collector with `otlp receiver/batch processor/clickhouse tcp exporter` can process +around 40k/s logs entry per CPU cores, add more collector node can increase linearly. ## Configuration options @@ -68,10 +120,11 @@ receivers: examplereceiver: processors: batch: - timeout: 10s + timeout: 5s + send_batch_size: 100000 exporters: clickhouse: - dsn: tcp://127.0.0.1:9000?database=default + dsn: tcp://127.0.0.1:9000/default ttl_days: 3 timeout: 5s retry_on_failure: @@ -90,30 +143,30 @@ service: ## Schema ```clickhouse -CREATE TABLE IF NOT EXISTS otel_logs ( - Timestamp DateTime CODEC(Delta, ZSTD(1)), - TraceId String CODEC(ZSTD(1)), - SpanId String CODEC(ZSTD(1)), - TraceFlags UInt32, - SeverityText LowCardinality(String) CODEC(ZSTD(1)), - SeverityNumber Int32, - Body String CODEC(ZSTD(1)), - ResourceAttributes Nested - ( - Key LowCardinality(String), - Value String - ) CODEC(ZSTD(1)), - LogAttributes Nested - ( - Key LowCardinality(String), - Value String - ) CODEC(ZSTD(1)), -INDEX idx_attr_keys ResourceAttributes.Key TYPE bloom_filter(0.01) GRANULARITY 64, -INDEX idx_res_keys LogAttributes.Key TYPE bloom_filter(0.01) GRANULARITY 64 -) ENGINE MergeTree() -TTL Timestamp + INTERVAL 3 DAY -PARTITION BY toDate(Timestamp) -ORDER BY (toUnixTimestamp(Timestamp)); +CREATE TABLE otel_logs +( + `Timestamp` DateTime64(9) CODEC(Delta, ZSTD(1)), + `TraceId` String CODEC(ZSTD(1)), + `SpanId` String CODEC(ZSTD(1)), + `TraceFlags` UInt32 CODEC(ZSTD(1)), + `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), + `SeverityNumber` Int32 CODEC(ZSTD(1)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), + `Body` String CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1 +) + ENGINE = MergeTree + PARTITION BY toDate(Timestamp) + ORDER BY (ServiceName, SeverityText, toUnixTimestamp(Timestamp), TraceId) + TTL toDateTime(Timestamp) + toIntervalDay(3) + SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; ``` [alpha]:https://github.com/open-telemetry/opentelemetry-collector#alpha diff --git a/exporter/clickhouseexporter/example/docker-compose.yml b/exporter/clickhouseexporter/example/docker-compose.yml index 44619b05a3f8..04fbbeeb7665 100644 --- a/exporter/clickhouseexporter/example/docker-compose.yml +++ b/exporter/clickhouseexporter/example/docker-compose.yml @@ -13,7 +13,7 @@ services: container_name: otel command: - "--config=/etc/otel-collector-config.yml" - - "--set=service.telemetry.logs.level=DEBUG" + - "--set=service.telemetry.logs.level=INFO" volumes: - ./otel-collector-config.yml:/etc/otel-collector-config.yml ports: diff --git a/exporter/clickhouseexporter/example/otel-collector-config.yml b/exporter/clickhouseexporter/example/otel-collector-config.yml index 5c9eb05f00a3..7d9660322831 100644 --- a/exporter/clickhouseexporter/example/otel-collector-config.yml +++ b/exporter/clickhouseexporter/example/otel-collector-config.yml @@ -5,17 +5,25 @@ receivers: processors: batch: send_batch_size: 100000 - timeout: 2s + timeout: 5s memory_limiter: check_interval: 2s limit_mib: 1800 spike_limit_mib: 500 - + resourcedetection/system: + detectors: ["system"] + system: + hostname_sources: ["os"] + resource: + attributes: + - key: service.name + value: "clickhouse-exporter" + action: upsert exporters: clickhouse: - dsn: tcp://clickhouse:9000?database=default + dsn: tcp://clickhouse:9000/default ttl_days: 3 - timeout: 5s + timeout: 10s sending_queue: queue_size: 100 retry_on_failure: @@ -35,5 +43,5 @@ service: pipelines: logs: receivers: [ fluentforward ] - processors: [ memory_limiter, batch ] + processors: [ memory_limiter, resourcedetection/system, resource, batch ] exporters: [ clickhouse ]