Enable Collector Otel metrics in Flow (#5045)

grafana · Sep 19, 2023 · dab92d4 · dab92d4
1 parent 9a3162f
commit dab92d4
Show file tree

Hide file tree

Showing 41 changed files with 307 additions and 161 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -77,6 +77,11 @@ Main (unreleased)
 - Add `openstack` config converter to convert OpenStack yaml config (static mode) to river config (flow mode). (@wildum)
 
 
+- Some `otelcol` components will now display their debug metrics via the 
+  Agent's `/metrics` endpoint. Those components include `otelcol.receiver.otlp`, 
+  `otelcol.exporter.otlp` and `otelcol.processor.batch`. There may also be metrics 
+  from other components which are not documented yet. (@ptodev)
+
 ### Other changes
 
 - Use Go 1.21.1 for builds. (@rfratto)

diff --git a/cmd/internal/flowmode/cmd_run.go b/cmd/internal/flowmode/cmd_run.go
@@ -26,6 +26,7 @@ import (
 	"github.com/grafana/agent/service"
 	"github.com/grafana/agent/service/cluster"
 	httpservice "github.com/grafana/agent/service/http"
+	otel_service "github.com/grafana/agent/service/otel"
 	uiservice "github.com/grafana/agent/service/ui"
 	"github.com/grafana/ckit/advertise"
 	"github.com/grafana/ckit/peer"
@@ -233,6 +234,8 @@ func (fr *flowRun) Run(configFile string) error {
 		Cluster:  clusterService.Data().(cluster.Cluster),
 	})
 
+	otelService := otel_service.New()
+
 	f := flow.New(flow.Options{
 		Logger:   l,
 		Tracer:   t,
@@ -242,6 +245,7 @@ func (fr *flowRun) Run(configFile string) error {
 			httpService,
 			uiService,
 			clusterService,
+			otelService,
 		},
 	})
 

diff --git a/component/otelcol/auth/auth.go b/component/otelcol/auth/auth.go
@@ -20,8 +20,6 @@ import (
 	otelextension "go.opentelemetry.io/collector/extension"
 	sdkprometheus "go.opentelemetry.io/otel/exporters/prometheus"
 	"go.opentelemetry.io/otel/sdk/metric"
-
-	_ "github.com/grafana/agent/component/otelcol/internal/featuregate" // Enable needed feature gates
 )
 
 // Arguments is an extension of component.Arguments which contains necessary

diff --git a/component/otelcol/auth/basic/basic.go b/component/otelcol/auth/basic/basic.go
@@ -4,6 +4,7 @@ package basic
 import (
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/otelcol/auth"
+	otel_service "github.com/grafana/agent/service/otel"
 	"github.com/grafana/river/rivertypes"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/basicauthextension"
 	otelcomponent "go.opentelemetry.io/collector/component"
@@ -13,9 +14,10 @@ import (
 
 func init() {
 	component.Register(component.Registration{
-		Name:    "otelcol.auth.basic",
-		Args:    Arguments{},
-		Exports: auth.Exports{},
+		Name:          "otelcol.auth.basic",
+		Args:          Arguments{},
+		Exports:       auth.Exports{},
+		NeedsServices: []string{otel_service.ServiceName},
 
 		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
 			fact := basicauthextension.NewFactory()

diff --git a/component/otelcol/auth/bearer/bearer.go b/component/otelcol/auth/bearer/bearer.go
@@ -4,6 +4,7 @@ package bearer
 import (
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/otelcol/auth"
+	otel_service "github.com/grafana/agent/service/otel"
 	"github.com/grafana/river/rivertypes"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension"
 	otelcomponent "go.opentelemetry.io/collector/component"
@@ -13,9 +14,10 @@ import (
 
 func init() {
 	component.Register(component.Registration{
-		Name:    "otelcol.auth.bearer",
-		Args:    Arguments{},
-		Exports: auth.Exports{},
+		Name:          "otelcol.auth.bearer",
+		Args:          Arguments{},
+		Exports:       auth.Exports{},
+		NeedsServices: []string{otel_service.ServiceName},
 
 		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
 			fact := bearertokenauthextension.NewFactory()

diff --git a/component/otelcol/auth/headers/headers.go b/component/otelcol/auth/headers/headers.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/otelcol/auth"
+	otel_service "github.com/grafana/agent/service/otel"
 	"github.com/grafana/river"
 	"github.com/grafana/river/rivertypes"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/headerssetterextension"
@@ -17,9 +18,10 @@ import (
 
 func init() {
 	component.Register(component.Registration{
-		Name:    "otelcol.auth.headers",
-		Args:    Arguments{},
-		Exports: auth.Exports{},
+		Name:          "otelcol.auth.headers",
+		Args:          Arguments{},
+		Exports:       auth.Exports{},
+		NeedsServices: []string{otel_service.ServiceName},
 
 		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
 			fact := headerssetterextension.NewFactory()

diff --git a/component/otelcol/auth/oauth2/oauth2.go b/component/otelcol/auth/oauth2/oauth2.go
@@ -7,6 +7,7 @@ import (
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/otelcol"
 	"github.com/grafana/agent/component/otelcol/auth"
+	otel_service "github.com/grafana/agent/service/otel"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/oauth2clientauthextension"
 	otelcomponent "go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/config/configopaque"
@@ -15,9 +16,10 @@ import (
 
 func init() {
 	component.Register(component.Registration{
-		Name:    "otelcol.auth.oauth2",
-		Args:    Arguments{},
-		Exports: auth.Exports{},
+		Name:          "otelcol.auth.oauth2",
+		Args:          Arguments{},
+		Exports:       auth.Exports{},
+		NeedsServices: []string{otel_service.ServiceName},
 
 		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
 			fact := oauth2clientauthextension.NewFactory()

diff --git a/component/otelcol/auth/sigv4/sigv4.go b/component/otelcol/auth/sigv4/sigv4.go
@@ -3,16 +3,18 @@ package sigv4
 import (
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/otelcol/auth"
+	otel_service "github.com/grafana/agent/service/otel"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/sigv4authextension"
 	otelcomponent "go.opentelemetry.io/collector/component"
 	otelextension "go.opentelemetry.io/collector/extension"
 )
 
 func init() {
 	component.Register(component.Registration{
-		Name:    "otelcol.auth.sigv4",
-		Args:    Arguments{},
-		Exports: auth.Exports{},
+		Name:          "otelcol.auth.sigv4",
+		Args:          Arguments{},
+		Exports:       auth.Exports{},
+		NeedsServices: []string{otel_service.ServiceName},
 
 		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
 			fact := sigv4authextension.NewFactory()

diff --git a/component/otelcol/connector/connector.go b/component/otelcol/connector/connector.go
@@ -21,8 +21,6 @@ import (
 	otelextension "go.opentelemetry.io/collector/extension"
 	sdkprometheus "go.opentelemetry.io/otel/exporters/prometheus"
 	"go.opentelemetry.io/otel/sdk/metric"
-
-	_ "github.com/grafana/agent/component/otelcol/internal/featuregate" // Enable needed feature gates
 )
 
 const (

diff --git a/component/otelcol/connector/servicegraph/servicegraph.go b/component/otelcol/connector/servicegraph/servicegraph.go
@@ -7,6 +7,7 @@ import (
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/otelcol"
 	"github.com/grafana/agent/component/otelcol/connector"
+	otel_service "github.com/grafana/agent/service/otel"
 	"github.com/grafana/river"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/servicegraphprocessor"
@@ -16,9 +17,10 @@ import (
 
 func init() {
 	component.Register(component.Registration{
-		Name:    "otelcol.connector.servicegraph",
-		Args:    Arguments{},
-		Exports: otelcol.ConsumerExports{},
+		Name:          "otelcol.connector.servicegraph",
+		Args:          Arguments{},
+		Exports:       otelcol.ConsumerExports{},
+		NeedsServices: []string{otel_service.ServiceName},
 
 		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
 			fact := servicegraphconnector.NewFactory()

diff --git a/component/otelcol/connector/spanmetrics/spanmetrics.go b/component/otelcol/connector/spanmetrics/spanmetrics.go
@@ -8,6 +8,7 @@ import (
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/otelcol"
 	"github.com/grafana/agent/component/otelcol/connector"
+	otel_service "github.com/grafana/agent/service/otel"
 	"github.com/grafana/river"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector"
 	otelcomponent "go.opentelemetry.io/collector/component"
@@ -16,9 +17,10 @@ import (
 
 func init() {
 	component.Register(component.Registration{
-		Name:    "otelcol.connector.spanmetrics",
-		Args:    Arguments{},
-		Exports: otelcol.ConsumerExports{},
+		Name:          "otelcol.connector.spanmetrics",
+		Args:          Arguments{},
+		Exports:       otelcol.ConsumerExports{},
+		NeedsServices: []string{otel_service.ServiceName},
 
 		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
 			fact := spanmetricsconnector.NewFactory()

diff --git a/component/otelcol/exporter/exporter.go b/component/otelcol/exporter/exporter.go
@@ -21,8 +21,6 @@ import (
 	otelextension "go.opentelemetry.io/collector/extension"
 	sdkprometheus "go.opentelemetry.io/otel/exporters/prometheus"
 	"go.opentelemetry.io/otel/sdk/metric"
-
-	_ "github.com/grafana/agent/component/otelcol/internal/featuregate" // Enable needed feature gates
 )
 
 // Arguments is an extension of component.Arguments which contains necessary

diff --git a/component/otelcol/exporter/jaeger/jaeger.go b/component/otelcol/exporter/jaeger/jaeger.go
@@ -7,6 +7,7 @@ import (
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/otelcol"
 	"github.com/grafana/agent/component/otelcol/exporter"
+	otel_service "github.com/grafana/agent/service/otel"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/jaegerexporter"
 	otelcomponent "go.opentelemetry.io/collector/component"
 	otelpexporterhelper "go.opentelemetry.io/collector/exporter/exporterhelper"
@@ -15,9 +16,10 @@ import (
 
 func init() {
 	component.Register(component.Registration{
-		Name:    "otelcol.exporter.jaeger",
-		Args:    Arguments{},
-		Exports: otelcol.ConsumerExports{},
+		Name:          "otelcol.exporter.jaeger",
+		Args:          Arguments{},
+		Exports:       otelcol.ConsumerExports{},
+		NeedsServices: []string{otel_service.ServiceName},
 
 		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
 			fact := jaegerexporter.NewFactory()

diff --git a/component/otelcol/exporter/loadbalancing/loadbalancing.go b/component/otelcol/exporter/loadbalancing/loadbalancing.go
@@ -9,6 +9,7 @@ import (
 	"github.com/grafana/agent/component/otelcol"
 	"github.com/grafana/agent/component/otelcol/auth"
 	"github.com/grafana/agent/component/otelcol/exporter"
+	otel_service "github.com/grafana/agent/service/otel"
 	"github.com/grafana/river"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/loadbalancingexporter"
 	otelcomponent "go.opentelemetry.io/collector/component"
@@ -22,9 +23,10 @@ import (
 
 func init() {
 	component.Register(component.Registration{
-		Name:    "otelcol.exporter.loadbalancing",
-		Args:    Arguments{},
-		Exports: otelcol.ConsumerExports{},
+		Name:          "otelcol.exporter.loadbalancing",
+		Args:          Arguments{},
+		Exports:       otelcol.ConsumerExports{},
+		NeedsServices: []string{otel_service.ServiceName},
 
 		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
 			fact := loadbalancingexporter.NewFactory()

diff --git a/component/otelcol/exporter/logging/logging.go b/component/otelcol/exporter/logging/logging.go
@@ -5,6 +5,7 @@ import (
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/otelcol"
 	"github.com/grafana/agent/component/otelcol/exporter"
+	otel_service "github.com/grafana/agent/service/otel"
 	otelcomponent "go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/config/configtelemetry"
 	loggingexporter "go.opentelemetry.io/collector/exporter/loggingexporter"
@@ -13,9 +14,10 @@ import (
 
 func init() {
 	component.Register(component.Registration{
-		Name:    "otelcol.exporter.logging",
-		Args:    Arguments{},
-		Exports: otelcol.ConsumerExports{},
+		Name:          "otelcol.exporter.logging",
+		Args:          Arguments{},
+		Exports:       otelcol.ConsumerExports{},
+		NeedsServices: []string{otel_service.ServiceName},
 
 		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
 			fact := loggingexporter.NewFactory()

diff --git a/component/otelcol/exporter/otlp/otlp.go b/component/otelcol/exporter/otlp/otlp.go
@@ -7,6 +7,7 @@ import (
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/otelcol"
 	"github.com/grafana/agent/component/otelcol/exporter"
+	otel_service "github.com/grafana/agent/service/otel"
 	otelcomponent "go.opentelemetry.io/collector/component"
 	otelpexporterhelper "go.opentelemetry.io/collector/exporter/exporterhelper"
 	"go.opentelemetry.io/collector/exporter/otlpexporter"
@@ -15,9 +16,10 @@ import (
 
 func init() {
 	component.Register(component.Registration{
-		Name:    "otelcol.exporter.otlp",
-		Args:    Arguments{},
-		Exports: otelcol.ConsumerExports{},
+		Name:          "otelcol.exporter.otlp",
+		Args:          Arguments{},
+		Exports:       otelcol.ConsumerExports{},
+		NeedsServices: []string{otel_service.ServiceName},
 
 		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
 			fact := otlpexporter.NewFactory()

diff --git a/component/otelcol/exporter/otlphttp/otlphttp.go b/component/otelcol/exporter/otlphttp/otlphttp.go
@@ -8,16 +8,18 @@ import (
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/otelcol"
 	"github.com/grafana/agent/component/otelcol/exporter"
+	otel_service "github.com/grafana/agent/service/otel"
 	otelcomponent "go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/exporter/otlphttpexporter"
 	otelextension "go.opentelemetry.io/collector/extension"
 )
 
 func init() {
 	component.Register(component.Registration{
-		Name:    "otelcol.exporter.otlphttp",
-		Args:    Arguments{},
-		Exports: otelcol.ConsumerExports{},
+		Name:          "otelcol.exporter.otlphttp",
+		Args:          Arguments{},
+		Exports:       otelcol.ConsumerExports{},
+		NeedsServices: []string{otel_service.ServiceName},
 
 		Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
 			fact := otlphttpexporter.NewFactory()

diff --git a/component/otelcol/extension/extension.go b/component/otelcol/extension/extension.go
@@ -19,8 +19,6 @@ import (
 	otelextension "go.opentelemetry.io/collector/extension"
 	sdkprometheus "go.opentelemetry.io/otel/exporters/prometheus"
 	"go.opentelemetry.io/otel/sdk/metric"
-
-	_ "github.com/grafana/agent/component/otelcol/internal/featuregate" // Enable needed feature gates
 )
 
 // Arguments is an extension of component.Arguments which contains necessary

diff --git a/component/otelcol/internal/featuregate/install.go b/component/otelcol/internal/featuregate/install.go
diff --git a/component/otelcol/internal/featuregate/install_test.go b/component/otelcol/internal/featuregate/install_test.go