enhance telemetry metrics (#304)

* * add some telemetry property: * session uuid * in docker or not * query counter * disable telemetry on CI
timeplus-io · Nov 20, 2023 · aaf5d03 · aaf5d03
1 parent b692b5f
commit aaf5d03
Show file tree

Hide file tree

Showing 8 changed files with 71 additions and 3 deletions.
diff --git a/programs/server/TelemetryCollector.cpp b/programs/server/TelemetryCollector.cpp
@@ -9,9 +9,20 @@
 #include <base/getMemoryAmount.h>
 #include <Common/DateLUT.h>
 #include <Common/getNumberOfPhysicalCPUCores.h>
+#include <Core/ServerUUID.h>
 #include <IO/WriteBufferFromString.h>
 #include <IO/WriteHelpers.h>
 #include <Interpreters/Context.h>
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
+namespace ProfileEvents
+{
+    extern const Event SelectQuery;
+    extern const Event StreamingSelectQuery;
+    extern const Event HistoricalSelectQuery;
+}
 
 namespace DB
 {
@@ -77,6 +88,21 @@ void TelemetryCollector::collect()
 
         Int64 duration_in_minute = UTCMinutes::now() - started_on_in_minutes;
 
+        DB::UUID server_uuid = DB::ServerUUID::get();
+        std::string server_uuid_str = server_uuid != DB::UUIDHelpers::Nil ? DB::toString(server_uuid) : "Unknown";
+
+        /// https://stackoverflow.com/questions/20010199/how-to-determine-if-a-process-runs-inside-lxc-docker
+        bool in_docker = fs::exists("/.dockerenv");
+
+        auto load_counter = [](const auto & event){
+            assert (event < ProfileEvents::end());
+            return ProfileEvents::global_counters[event].load(std::memory_order_relaxed);
+        };
+
+        const auto total_select_query = load_counter(ProfileEvents::SelectQuery);
+        const auto streaming_select_query = load_counter(ProfileEvents::StreamingSelectQuery);
+        const auto historical_select_query = load_counter(ProfileEvents::HistoricalSelectQuery);
+
         std::string data = fmt::format("{{"
             "\"type\": \"track\","
             "\"event\": \"proton_ping\","
@@ -87,9 +113,16 @@ void TelemetryCollector::collect()
             "    \"version\": \"{}\","
             "    \"new_session\": \"{}\","
             "    \"started_on\": \"{}\","
-            "    \"duration_in_minute\": \"{}\""
+            "    \"duration_in_minute\": \"{}\","
+            "    \"server_id\": \"{}\","
+            "    \"docker\": \"{}\","
+            "    \"total_select_query\": \"{}\","
+            "    \"historical_select_query\": \"{}\","
+            "    \"streaming_select_query\": \"{}\""
             "}}"
-        "}}", cpu, memory_in_gb, EDITION, VERSION_STRING, new_session, started_on, duration_in_minute);
+        "}}", cpu, memory_in_gb, EDITION, VERSION_STRING, new_session, started_on, duration_in_minute, server_uuid_str, in_docker, total_select_query, historical_select_query, streaming_select_query);
+
+        LOG_TRACE(log, "Sending telemetry: {}.", data);
 
         new_session = false;
 

diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp
@@ -5,6 +5,8 @@
 #define APPLY_FOR_EVENTS(M) \
     M(Query, "Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by proton itself. Does not count subqueries.") \
     M(SelectQuery, "Same as Query, but only for SELECT queries.") \
+    M(StreamingSelectQuery, "Same as Query, but only for streaming SELECT queries.") \
+    M(HistoricalSelectQuery, "Same as Query, but only for historical SELECT queries.") \
     M(InsertQuery, "Same as Query, but only for INSERT queries.") \
     M(FailedQuery, "Number of failed queries.") \
     M(FailedSelectQuery, "Same as FailedQuery, but only for SELECT queries.") \

diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp
@@ -97,6 +97,8 @@ namespace ProfileEvents
 {
     extern const Event Query;
     extern const Event SelectQuery;
+    extern const Event StreamingSelectQuery;
+    extern const Event HistoricalSelectQuery;
     extern const Event InsertQuery;
 }
 
@@ -123,8 +125,10 @@ std::unique_ptr<IInterpreter> InterpreterFactory::get(ASTPtr & query, ContextMut
     }
     else if (query->as<ASTSelectWithUnionQuery>())
     {
+        auto interpreter = std::make_unique<InterpreterSelectWithUnionQuery>(query, context, options);
         ProfileEvents::increment(ProfileEvents::SelectQuery);
-        return std::make_unique<InterpreterSelectWithUnionQuery>(query, context, options);
+        ProfileEvents::increment(interpreter->isStreamingQuery() ? ProfileEvents::StreamingSelectQuery : ProfileEvents::HistoricalSelectQuery);
+        return std::move(interpreter);
     }
     else if (query->as<ASTSelectIntersectExceptQuery>())
     {

diff --git a/tests/proton_ci/functional_tests_check.py b/tests/proton_ci/functional_tests_check.py
@@ -47,6 +47,7 @@ def get_run_command(check_name, output_path):
     env.append("-e MAX_CONCURRENT_QUERIES=200")
     env.append("-e MAX_CONCURRENT_INSERT_QUERIES=200")
     env.append("-e MAX_CONCURRENT_SELECT_QUERIES=200")
+    env.append("-e TELEMETRY_ENABLED=false")
 
     env_str = " ".join(env)
 

diff --git a/tests/stream/test_stream_smoke/configs/docker-compose-native.yaml b/tests/stream/test_stream_smoke/configs/docker-compose-native.yaml
@@ -27,6 +27,7 @@ services:
       - MAX_CONCURRENT_STREAMING_QUERIES=100   # Default: 100
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -58,6 +59,7 @@ services:
       - MAX_CONCURRENT_STREAMING_QUERIES=100   # Default: 100
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -89,6 +91,7 @@ services:
       - MAX_CONCURRENT_STREAMING_QUERIES=100   # Default: 100
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"       
diff --git a/tests/stream/test_stream_smoke/configs/docker-compose-redp.yaml b/tests/stream/test_stream_smoke/configs/docker-compose-redp.yaml
@@ -28,6 +28,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -62,6 +63,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -97,6 +99,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -131,6 +134,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -165,6 +169,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"

diff --git a/tests/stream/test_stream_smoke/configs/docker-compose.yaml b/tests/stream/test_stream_smoke/configs/docker-compose.yaml
@@ -27,6 +27,7 @@ services:
       - MAX_CONCURRENT_STREAMING_QUERIES=100   # Default: 100
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -58,6 +59,7 @@ services:
       - MAX_CONCURRENT_STREAMING_QUERIES=100   # Default: 100
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"      
@@ -90,6 +92,7 @@ services:
       - MAX_CONCURRENT_STREAMING_QUERIES=100   # Default: 100
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"  
@@ -122,6 +125,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -156,6 +160,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -191,6 +196,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -225,6 +231,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -259,6 +266,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"

diff --git a/tests/stream/test_stream_smoke/configs/docker-compose_11_containers.yaml b/tests/stream/test_stream_smoke/configs/docker-compose_11_containers.yaml
@@ -27,6 +27,7 @@ services:
       - MAX_CONCURRENT_STREAMING_QUERIES=100   # Default: 100
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -58,6 +59,7 @@ services:
       - MAX_CONCURRENT_STREAMING_QUERIES=100   # Default: 100
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"      
@@ -90,6 +92,7 @@ services:
       - MAX_CONCURRENT_STREAMING_QUERIES=100   # Default: 100
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"  
@@ -121,6 +124,7 @@ services:
       - MAX_CONCURRENT_STREAMING_QUERIES=100   # Default: 100
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -152,6 +156,7 @@ services:
       - MAX_CONCURRENT_STREAMING_QUERIES=100   # Default: 100
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"    
@@ -185,6 +190,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -223,6 +229,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -262,6 +269,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -300,6 +308,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -338,6 +347,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -376,6 +386,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"
@@ -414,6 +425,7 @@ services:
       - MAX_SERVER_MEMORY_USAGE_TO_RAM_RATIO=0.9 # Default: 0.9
       - MAX_SERVER_MEMORY_CACHE_TO_RAM_RATIO=0.5 # Default: 0.5
       - STREAM_STORAGE_TYPE=kafka
+      - TELEMETRY_ENABLED=false # Turn off telemetry on smoke test
 
     command: >
       /bin/bash -c "echo sleeping; sleep 2; /entrypoint.sh"