Add a Prometheus Remote Write connector to the exporting engine (netd…

…ata#8292) * Copy files from the Prometheus remote write backend * Update the documentation * Rename backend -> exporting * Add the connector to the Netdata build * Separate files for the remote write connector * Add an initializer and formatters * Read a connector specific configuration option * Add a separate function for header sending * Use labels instead of tags * Separate write request for every instance * Add unit tests
MohameedElsaeid · Mar 12, 2020 · bc0ca9b · bc0ca9b
1 parent bec7b83
commit bc0ca9b
Show file tree

Hide file tree

Showing 35 changed files with 2,407 additions and 168 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -634,6 +634,8 @@ set(EXPORTING_ENGINE_FILES
         exporting/json/json.h
         exporting/opentsdb/opentsdb.c
         exporting/opentsdb/opentsdb.h
+        exporting/prometheus/prometheus.c
+        exporting/prometheus/prometheus.h
         exporting/read_config.c
         exporting/init_connectors.c
         exporting/process_data.c
@@ -642,6 +644,13 @@ set(EXPORTING_ENGINE_FILES
         exporting/send_internal_metrics.c
         )
 
+set(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES
+        exporting/prometheus/remote_write/remote_write.c
+        exporting/prometheus/remote_write/remote_write.h
+        exporting/prometheus/remote_write/remote_write_request.cc
+        exporting/prometheus/remote_write/remote_write_request.h
+        )
+
 set(KINESIS_EXPORTING_FILES
         exporting/aws_kinesis/aws_kinesis.c
         exporting/aws_kinesis/aws_kinesis.h
@@ -753,9 +762,9 @@ IF(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE)
     message(STATUS "prometheus remote write backend: enabled")
 
     find_package(Protobuf REQUIRED)
-    protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS backends/prometheus/remote_write/remote_write.proto)
+    protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS exporting/prometheus/remote_write/remote_write.proto)
 
-    list(APPEND NETDATA_FILES ${PROMETHEUS_REMOTE_WRITE_BACKEND_FILES} ${PROTO_SRCS} ${PROTO_HDRS})
+    list(APPEND NETDATA_FILES ${PROMETHEUS_REMOTE_WRITE_BACKEND_FILES} ${PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES} ${PROTO_SRCS} ${PROTO_HDRS})
     list(APPEND NETDATA_COMMON_LIBRARIES ${PROTOBUF_LIBRARIES} ${SNAPPY_LIBRARIES})
     list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIRS} ${SNAPPY_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR})
     list(APPEND NETDATA_COMMON_CFLAGS ${PROTOBUF_CFLAGS_OTHER} ${SNAPPY_CFLAGS_OTHER})
@@ -1021,6 +1030,17 @@ if(BUILD_TESTING)
         )
     set(TEST_NAME exporting_engine)
     set(KINESIS_LINK_OPTIONS)
+    set(PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS)
+if(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE)
+    list(APPEND EXPORTING_ENGINE_FILES ${PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES} ${PROTO_SRCS} ${PROTO_HDRS})
+    list(
+        APPEND PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS
+        -Wl,--wrap=init_write_request
+        -Wl,--wrap=add_host_info
+        -Wl,--wrap=add_label
+        -Wl,--wrap=add_metric
+    )
+endif()
 if(ENABLE_BACKEND_KINESIS)
     list(APPEND EXPORTING_ENGINE_FILES ${KINESIS_EXPORTING_FILES})
     list(
@@ -1062,6 +1082,7 @@ endif()
         -Wl,--wrap=send
         -Wl,--wrap=connect_to_one_of
         ${KINESIS_LINK_OPTIONS}
+        ${PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS}
     )
     target_link_libraries(${TEST_NAME}_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES})
     add_test(NAME test_${TEST_NAME} COMMAND ${TEST_NAME}_testdriver)

diff --git a/Makefile.am b/Makefile.am
@@ -492,6 +492,8 @@ EXPORTING_ENGINE_FILES = \
     exporting/json/json.h \
     exporting/opentsdb/opentsdb.c \
     exporting/opentsdb/opentsdb.h \
+    exporting/prometheus/prometheus.c \
+    exporting/prometheus/prometheus.h \
     exporting/read_config.c \
     exporting/init_connectors.c \
     exporting/process_data.c \
@@ -500,6 +502,14 @@ EXPORTING_ENGINE_FILES = \
     exporting/send_internal_metrics.c \
     $(NULL)
 
+PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES = \
+    exporting/prometheus/remote_write/remote_write.c \
+    exporting/prometheus/remote_write/remote_write.h \
+    exporting/prometheus/remote_write/remote_write_request.cc \
+    exporting/prometheus/remote_write/remote_write_request.h \
+    exporting/prometheus/remote_write/remote_write.proto \
+    $(NULL)
+
 KINESIS_EXPORTING_FILES = \
     exporting/aws_kinesis/aws_kinesis.c \
     exporting/aws_kinesis/aws_kinesis.h \
@@ -517,7 +527,6 @@ KINESIS_BACKEND_FILES = \
 PROMETHEUS_REMOTE_WRITE_BACKEND_FILES = \
     backends/prometheus/remote_write/remote_write.cc \
     backends/prometheus/remote_write/remote_write.h \
-    backends/prometheus/remote_write/remote_write.proto \
     $(NULL)
 
 MONGODB_BACKEND_FILES = \
@@ -737,16 +746,19 @@ if ENABLE_BACKEND_KINESIS
 endif
 
 if ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE
+if ENABLE_EXPORTING
+    netdata_SOURCES += $(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES)
+endif
     netdata_SOURCES += $(PROMETHEUS_REMOTE_WRITE_BACKEND_FILES)
     netdata_LDADD += $(OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS)
     BUILT_SOURCES = \
-        backends/prometheus/remote_write/remote_write.pb.cc \
-        backends/prometheus/remote_write/remote_write.pb.h \
+        exporting/prometheus/remote_write/remote_write.pb.cc \
+        exporting/prometheus/remote_write/remote_write.pb.h \
         $(NULL)
     nodist_netdata_SOURCES = $(BUILT_SOURCES)
 
-backends/prometheus/remote_write/remote_write.pb.cc \
-backends/prometheus/remote_write/remote_write.pb.h: backends/prometheus/remote_write/remote_write.proto
+exporting/prometheus/remote_write/remote_write.pb.cc \
+exporting/prometheus/remote_write/remote_write.pb.h: exporting/prometheus/remote_write/remote_write.proto
 	$(PROTOC) --proto_path=$(srcdir) --cpp_out=$(builddir) $^
 
 endif
@@ -867,6 +879,17 @@ if ENABLE_UNITTESTS
         $(TEST_LDFLAGS) \
         $(NULL)
     exporting_tests_exporting_engine_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
+if ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE
+    exporting_tests_exporting_engine_testdriver_SOURCES += $(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES)
+    exporting_tests_exporting_engine_testdriver_LDADD += $(OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS)
+    exporting_tests_exporting_engine_testdriver_LDFLAGS += \
+        -Wl,--wrap=init_write_request \
+        -Wl,--wrap=add_host_info \
+        -Wl,--wrap=add_label \
+        -Wl,--wrap=add_metric \
+        $(NULL)
+    nodist_exporting_tests_exporting_engine_testdriver_SOURCES = $(BUILT_SOURCES)
+endif
 if ENABLE_BACKEND_KINESIS
     exporting_tests_exporting_engine_testdriver_SOURCES += $(KINESIS_EXPORTING_FILES)
     exporting_tests_exporting_engine_testdriver_LDADD += $(OPTIONAL_KINESIS_LIBS)

diff --git a/backends/backends.c b/backends/backends.c
@@ -295,7 +295,7 @@ void backend_set_prometheus_variables(int *default_port,
 #endif
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
-    *brc = process_prometheus_remote_write_response;
+    *brc = backends_process_prometheus_remote_write_response;
 #endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */
 }
 
@@ -439,7 +439,7 @@ BACKEND_TYPE backend_select_type(const char *type) {
         return BACKEND_TYPE_JSON;
     }
     else if (!strcmp(type, "prometheus_remote_write")) {
-        return  BACKEND_TYPE_PROMETHEUS;
+        return  BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE;
     }
     else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
         return BACKEND_TYPE_KINESIS;
@@ -557,12 +557,12 @@ void *backends_main(void *ptr) {
             backend_set_opentsdb_http_variables(&default_port,&backend_response_checker,&backend_request_formatter);
             break;
         }
-        case BACKEND_TYPE_PROMETHEUS: {
+        case BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE: {
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
             do_prometheus_remote_write = 1;
 
             http_request_header = buffer_create(1);
-            init_write_request();
+            backends_init_write_request();
 #else
             error("BACKEND: Prometheus remote write support isn't compiled");
 #endif // ENABLE_PROMETHEUS_REMOTE_WRITE
@@ -724,7 +724,7 @@ void *backends_main(void *ptr) {
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
         if(do_prometheus_remote_write)
-            clear_write_request();
+            backends_clear_write_request();
 #endif
         rrd_rdlock();
         RRDHOST *host;
@@ -755,7 +755,7 @@ void *backends_main(void *ptr) {
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
             if(do_prometheus_remote_write) {
-                rrd_stats_remote_write_allmetrics_prometheus(
+                backends_rrd_stats_remote_write_allmetrics_prometheus(
                     host
                     , __hostname
                     , global_backend_prefix
@@ -1048,7 +1048,7 @@ void *backends_main(void *ptr) {
 
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
                 if(do_prometheus_remote_write) {
-                    size_t data_size = get_write_request_size();
+                    size_t data_size = backends_get_write_request_size();
 
                     if(unlikely(!data_size)) {
                         error("BACKEND: write request size is out of range");
@@ -1057,7 +1057,7 @@ void *backends_main(void *ptr) {
 
                     buffer_flush(b);
                     buffer_need_bytes(b, data_size);
-                    if(unlikely(pack_write_request(b->buffer, &data_size))) {
+                    if(unlikely(backends_pack_write_request(b->buffer, &data_size))) {
                         error("BACKEND: cannot pack write request");
                         continue;
                     }
@@ -1137,7 +1137,7 @@ void *backends_main(void *ptr) {
         if(do_prometheus_remote_write && failures) {
             (void) buffer_on_failures;
             failures = 0;
-            chart_lost_bytes = chart_buffered_bytes = get_write_request_size(); // estimated write request size
+            chart_lost_bytes = chart_buffered_bytes = backends_get_write_request_size(); // estimated write request size
             chart_data_lost_events++;
             chart_lost_metrics = chart_buffered_metrics;
         } else
@@ -1209,7 +1209,7 @@ void *backends_main(void *ptr) {
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
     buffer_free(http_request_header);
     if(do_prometheus_remote_write)
-        protocol_buffers_shutdown();
+        backends_protocol_buffers_shutdown();
 #endif
 
 #if HAVE_MONGOC

diff --git a/backends/backends.h b/backends/backends.h
@@ -21,7 +21,7 @@ typedef enum backend_types {
     BACKEND_TYPE_OPENTSDB_USING_TELNET,     // Send data to OpenTSDB using telnet API
     BACKEND_TYPE_OPENTSDB_USING_HTTP,       // Send data to OpenTSDB using HTTP API
     BACKEND_TYPE_JSON,                      // Stores the data using JSON.
-    BACKEND_TYPE_PROMETHEUS,                // The user selected to use Prometheus backend
+    BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE,   // The user selected to use Prometheus backend
     BACKEND_TYPE_KINESIS,                   // Send message to AWS Kinesis
     BACKEND_TYPE_MONGODB,                   // Send data to MongoDB collection
     BACKEND_TYPE_NUM                        // Number of backend types