Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow changing otel endpoint setting without restart #62

Merged
merged 1 commit into from
Sep 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,3 +155,5 @@ pg_tracing.otel_naptime = 2000
```

If an otel endpoint is defined, a background worker will be started and will send spans every $naptime using the OTLP HTTP/JSON protocol.
The endpoint can be changed while the server is running, but if it was not set when the server was started, changing it to non-empty value
requires a restart.
21 changes: 10 additions & 11 deletions src/pg_tracing.c
Original file line number Diff line number Diff line change
Expand Up @@ -123,13 +123,13 @@ static int pg_tracing_buffer_mode = PG_TRACING_KEEP_ON_FULL; /* behaviour on ful
* buffer */
static char *pg_tracing_filter_query_ids = NULL; /* only sample query
* matching query ids */
static char *pg_tracing_otel_endpoint = NULL; /* Otel collector to send
char *pg_tracing_otel_endpoint = NULL; /* Otel collector to send
* spans to */
char *pg_tracing_otel_service_name = NULL; /* Service name set in
* otel traces */
static int pg_tracing_otel_naptime; /* Delay between upload of spans to
int pg_tracing_otel_naptime; /* Delay between upload of spans to
* otel collector */
static int pg_tracing_otel_connect_timeout_ms; /* Connect timeout to the otel
int pg_tracing_otel_connect_timeout_ms; /* Connect timeout to the otel
* collector */
static char *guc_tracecontext_str = NULL; /* Trace context string propagated
* through GUC variable */
Expand Down Expand Up @@ -448,7 +448,7 @@ _PG_init(void)
10000,
1000,
500000,
PGC_POSTMASTER,
PGC_SIGHUP,
0,
NULL,
NULL,
Expand All @@ -461,29 +461,29 @@ _PG_init(void)
1000,
100,
600000,
PGC_POSTMASTER,
PGC_SIGHUP,
0,
NULL,
NULL,
&otel_config_int_assign_hook,
NULL);

DefineCustomStringVariable("pg_tracing.otel_endpoint",
"Otel endpoint to send spans.",
"If unset, no background worker to export to otel is created.",
&pg_tracing_otel_endpoint,
NULL,
PGC_POSTMASTER,
PGC_SIGHUP,
0,
NULL,
NULL,
&otel_config_string_assign_hook,
NULL);

DefineCustomStringVariable("pg_tracing.otel_service_name",
"Service Name to set in traces sent to otel.",
NULL,
&pg_tracing_otel_service_name,
"PostgreSQL_Server",
PGC_POSTMASTER,
PGC_SIGHUP,
0,
NULL,
NULL,
Expand Down Expand Up @@ -536,8 +536,7 @@ _PG_init(void)
if (pg_tracing_otel_endpoint != NULL)
{
elog(INFO, "Starting otel exporter worker on endpoint %s", pg_tracing_otel_endpoint);
pg_tracing_start_worker(pg_tracing_otel_endpoint, pg_tracing_otel_naptime,
pg_tracing_otel_connect_timeout_ms);
pg_tracing_start_worker();
}
}

Expand Down
11 changes: 10 additions & 1 deletion src/pg_tracing.h
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,9 @@ extern pgTracingSpans * shared_spans;
extern char *shared_str;
extern int nested_level;
extern char *pg_tracing_otel_service_name;
extern char *pg_tracing_otel_endpoint;
extern int pg_tracing_otel_naptime;
extern int pg_tracing_otel_connect_timeout_ms;

extern void
store_span(const Span * span);
Expand Down Expand Up @@ -464,6 +467,12 @@ extern Size

/* pg_tracing_otel.c */
extern void
pg_tracing_start_worker(const char *endpoint, int naptime, int connect_timeout_ms);
pg_tracing_start_worker(void);

extern void
otel_config_int_assign_hook(int newval, void *extra);

extern void
otel_config_string_assign_hook(const char *newval, void *extra);

#endif
67 changes: 49 additions & 18 deletions src/pg_tracing_otel.c
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include "pg_tracing.h"
#include "postmaster/bgworker.h"
#include "postmaster/interrupt.h"
#include "utils/guc.h"
#include "utils/memutils.h"
#include "storage/latch.h"
#include "storage/procsignal.h"
Expand All @@ -28,10 +29,7 @@ typedef struct OtelContext
pgTracingSpans *spans; /* A copy of spans to send */
char *spans_str; /* A copy of span text */

const char *endpoint; /* Target otel collector */
int naptime; /* Duration between upload ot spans to the
* otel collector */
int connect_timeout_ms; /* Connection timeout in ms */
bool config_changed;
} OtelContext;

/* State and configuration of the otel exporter */
Expand Down Expand Up @@ -106,8 +104,13 @@ send_json_trace(OtelContext * octx, const char *json_span)
return CURLE_FAILED_INIT;
}
curl_easy_setopt(octx->curl, CURLOPT_HTTPHEADER, octx->headers);
curl_easy_setopt(octx->curl, CURLOPT_URL, octx->endpoint);
curl_easy_setopt(octx->curl, CURLOPT_CONNECTTIMEOUT_MS, octx->connect_timeout_ms);
octx->config_changed = true;
}
if (octx->config_changed)
{
curl_easy_setopt(octx->curl, CURLOPT_URL, pg_tracing_otel_endpoint);
curl_easy_setopt(octx->curl, CURLOPT_CONNECTTIMEOUT_MS, pg_tracing_otel_connect_timeout_ms);
octx->config_changed = false;
}
curl_easy_setopt(octx->curl, CURLOPT_POSTFIELDS, json_span);
curl_easy_setopt(octx->curl, CURLOPT_POSTFIELDSIZE, (long) strlen(json_span));
Expand Down Expand Up @@ -137,7 +140,7 @@ send_json_to_otel_collector(OtelContext * octx, JsonContext * json_ctx)
{
CURLcode ret;

elog(INFO, "Sending %d spans to %s", json_ctx->num_spans, octx->endpoint);
elog(INFO, "Sending %d spans to %s", json_ctx->num_spans, pg_tracing_otel_endpoint);
ret = send_json_trace(octx, json_ctx->str->data);
if (ret == CURLE_OK)
{
Expand Down Expand Up @@ -222,7 +225,7 @@ send_spans_to_otel_collector(OtelContext * octx, JsonContext * json_ctx)
* Register otel exporter background worker
*/
void
pg_tracing_start_worker(const char *endpoint, int naptime, int connect_timeout_ms)
pg_tracing_start_worker(void)
{
BackgroundWorker worker;
BackgroundWorkerHandle *handle;
Expand All @@ -237,13 +240,6 @@ pg_tracing_start_worker(const char *endpoint, int naptime, int connect_timeout_m
strcpy(worker.bgw_name, "pg_tracing otel exporter");
strcpy(worker.bgw_type, "pg_tracing otel exporter");

/* Initialize the otel context struct */
otel_context.headers = NULL;
otel_context.curl = NULL;
otel_context.endpoint = endpoint;
otel_context.naptime = naptime;
otel_context.connect_timeout_ms = connect_timeout_ms;

if (process_shared_preload_libraries_in_progress)
{
RegisterBackgroundWorker(&worker);
Expand Down Expand Up @@ -277,6 +273,10 @@ pg_tracing_otel_exporter(Datum main_arg)

json_ctx.str = NULL;

/* Initialize the otel context struct */
otel_context.headers = NULL;
otel_context.curl = NULL;

/* Establish signal handlers; once that's done, unblock signals. */
pqsignal(SIGTERM, SignalHandlerForShutdownRequest);
pqsignal(SIGHUP, SignalHandlerForConfigReload);
Expand Down Expand Up @@ -321,12 +321,26 @@ pg_tracing_otel_exporter(Datum main_arg)
while (!ShutdownRequestPending)
{
int rc;
int wakeEvents;

/* Clean the latch and wait for the next event */
ResetLatch(MyLatch);
rc = WaitLatch(MyLatch,
WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
otel_context.naptime,

if (ConfigReloadPending)
{
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
}

/*
* If disabled by setting endpoint to empty, sleep until woken up by
* another configuration change.
*/
wakeEvents = WL_LATCH_SET | WL_EXIT_ON_PM_DEATH;
if (pg_tracing_otel_endpoint != NULL && pg_tracing_otel_endpoint[0] != '\0')
wakeEvents |= WL_TIMEOUT;

rc = WaitLatch(MyLatch, wakeEvents, pg_tracing_otel_naptime,
PG_WAIT_EXTENSION);

/* Send spans if we have any */
Expand All @@ -344,3 +358,20 @@ pg_tracing_otel_exporter(Datum main_arg)
}
curl_global_cleanup();
}

/*
* Assign hooks to notice changes to GUCs that need to be also set on the Curl
* handle. (We don't dare to make the changes to the Curl handle here directly,
* in case there's an error.)
*/
void
otel_config_int_assign_hook(int newval, void *extra)
{
otel_context.config_changed = true;
}

void
otel_config_string_assign_hook(const char *newval, void *extra)
{
otel_context.config_changed = true;
}