loki.source.cloudflare: port latest Promtail changes (#5571)

Signed-off-by: Paschalis Tsilias <[email protected]>

component/loki/source/cloudflare/cloudflare.go

@@ -1,5 +1,10 @@
 package cloudflare
 
+// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868).
+// The cloudflaretarget package is used to configure and run a target that can
+// read from the Cloudflare Logpull API and forward entries to other loki
+// components.
+
 import (
 	"context"
 	"fmt"

component/loki/source/cloudflare/internal/cloudflaretarget/client.go

@@ -1,8 +1,9 @@
 package cloudflaretarget
 
-// This code is copied from Promtail. The cloudflaretarget package is used to
-// configure and run a target that can read from the Cloudflare Logpull API and
-// forward entries to other loki components.
+// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868).
+// The cloudflaretarget package is used to configure and run a target that can
+// read from the Cloudflare Logpull API and forward entries to other loki
+// components.
 
 import (
 	"context"

component/loki/source/cloudflare/internal/cloudflaretarget/fields.go

@@ -1,8 +1,9 @@
 package cloudflaretarget
 
-// This code is copied from Promtail. The cloudflaretarget package is used to
-// configure and run a target that can read from the Cloudflare Logpull API and
-// forward entries to other loki components.
+// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868).
+// The cloudflaretarget package is used to configure and run a target that can
+// read from the Cloudflare Logpull API and forward entries to other loki
+// components.
 
 import (
 	"fmt"
@@ -38,9 +39,9 @@ var (
 		"OriginResponseHTTPExpires", "OriginResponseHTTPLastModified",
 	}...)
 	allFields = append(extendedFields, []string{
-		"BotScore", "BotScoreSrc", "ClientRequestBytes", "ClientSrcPort", "ClientXRequestedWith", "CacheTieredFill", "EdgeResponseCompressionRatio", "EdgeServerIP", "FirewallMatchesSources",
+		"BotScore", "BotScoreSrc", "BotTags", "ClientRequestBytes", "ClientSrcPort", "ClientXRequestedWith", "CacheTieredFill", "EdgeResponseCompressionRatio", "EdgeServerIP", "FirewallMatchesSources",
 		"FirewallMatchesActions", "FirewallMatchesRuleIDs", "OriginResponseBytes", "OriginResponseTime", "ClientDeviceType", "WAFFlags", "WAFMatchedVar", "EdgeColoID",
-		"RequestHeaders", "ResponseHeaders",
+		"RequestHeaders", "ResponseHeaders", "ClientRequestSource",
 	}...)
 )
 

component/loki/source/cloudflare/internal/cloudflaretarget/fields_test.go

@@ -1,5 +1,10 @@
 package cloudflaretarget
 
+// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868).
+// The cloudflaretarget package is used to configure and run a target that can
+// read from the Cloudflare Logpull API and forward entries to other loki
+// components.
+
 import (
 	"testing"
 

component/loki/source/cloudflare/internal/cloudflaretarget/metrics.go

@@ -1,8 +1,9 @@
 package cloudflaretarget
 
-// This code is copied from Promtail. The cloudflaretarget package is used to
-// configure and run a target that can read from the Cloudflare Logpull API and
-// forward entries to other loki components.
+// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868).
+// The cloudflaretarget package is used to configure and run a target that can
+// read from the Cloudflare Logpull API and forward entries to other loki
+// components.
 
 import "github.com/prometheus/client_golang/prometheus"
 

component/loki/source/cloudflare/internal/cloudflaretarget/target.go

@@ -1,8 +1,9 @@
 package cloudflaretarget
 
-// This code is copied from Promtail. The cloudflaretarget package is used to
-// configure and run a target that can read from the Cloudflare Logpull API and
-// forward entries to other loki components.
+// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868).
+// The cloudflaretarget package is used to configure and run a target that can
+// read from the Cloudflare Logpull API and forward entries to other loki
+// components.
 
 import (
 	"context"
@@ -12,11 +13,11 @@ import (
 	"time"
 
 	"github.com/buger/jsonparser"
-	"github.com/cloudflare/cloudflare-go"
 	"github.com/go-kit/log"
 	"github.com/grafana/agent/component/common/loki"
 	"github.com/grafana/agent/component/common/loki/positions"
 	"github.com/grafana/agent/pkg/flow/logging/level"
+	"github.com/grafana/cloudflare-go"
 	"github.com/grafana/dskit/backoff"
 	"github.com/grafana/dskit/concurrency"
 	"github.com/grafana/dskit/multierror"

component/loki/source/cloudflare/internal/cloudflaretarget/target_test.go

@@ -1,8 +1,9 @@
 package cloudflaretarget
 
-// This code is copied from Promtail. The cloudflaretarget package is used to
-// configure and run a target that can read from the Cloudflare Logpull API and
-// forward entries to other loki components.
+// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868).
+// The cloudflaretarget package is used to configure and run a target that can
+// read from the Cloudflare Logpull API and forward entries to other loki
+// components.
 
 import (
 	"context"

component/loki/source/cloudflare/internal/cloudflaretarget/util_test.go

@@ -1,8 +1,9 @@
 package cloudflaretarget
 
-// This code is copied from Promtail. The cloudflaretarget package is used to
-// configure and run a target that can read from the Cloudflare Logpull API and
-// forward entries to other loki components.
+// This code is copied from Promtail (a1c1152b79547a133cc7be520a0b2e6db8b84868).
+// The cloudflaretarget package is used to configure and run a target that can
+// read from the Cloudflare Logpull API and forward entries to other loki
+// components.
 
 import (
 	"context"

docs/sources/flow/reference/components/loki.source.cloudflare.md

@@ -71,7 +71,7 @@ plus any extra fields provided via `additional_fields` argument.
 
 * `all` includes all `extended` fields and adds:
 ```
- "BotScore", "BotScoreSrc", "ClientRequestBytes", "ClientSrcPort", "ClientXRequestedWith", "CacheTieredFill", "EdgeResponseCompressionRatio", "EdgeServerIP", "FirewallMatchesSources", "FirewallMatchesActions", "FirewallMatchesRuleIDs", "OriginResponseBytes", "OriginResponseTime", "ClientDeviceType", "WAFFlags", "WAFMatchedVar", "EdgeColoID", "RequestHeaders", "ResponseHeaders"
+ "BotScore", "BotScoreSrc", "BotTags", "ClientRequestBytes", "ClientSrcPort", "ClientXRequestedWith", "CacheTieredFill", "EdgeResponseCompressionRatio", "EdgeServerIP", "FirewallMatchesSources", "FirewallMatchesActions", "FirewallMatchesRuleIDs", "OriginResponseBytes", "OriginResponseTime", "ClientDeviceType", "WAFFlags", "WAFMatchedVar", "EdgeColoID", "RequestHeaders", "ResponseHeaders", "ClientRequestSource"`
 ```
 plus any extra fields provided via `additional_fields` argument (this is still relevant in this case if new fields are made available via Cloudflare API but are not yet included in `all`).
 
@@ -113,6 +113,7 @@ change the log line format. A sample log looks like this:
     "ClientRequestReferer": "https://www.foo.com/foo/168855/?offset=8625",
     "ClientRequestURI": "/foo/15248108/",
     "ClientRequestUserAgent": "some bot",
+    "ClientRequestSource": "1"
     "ClientSSLCipher": "ECDHE-ECDSA-AES128-GCM-SHA256",
     "ClientSSLProtocol": "TLSv1.2",
     "ClientSrcPort": 39816,
@@ -144,10 +145,10 @@ change the log line format. A sample log looks like this:
     "OriginSSLProtocol": "TLSv1.2",
     "ParentRayID": "00",
     "RayID": "6b0a...",
-  "RequestHeaders": [],
-  "ResponseHeaders": [
-    "x-foo": "bar"
-  ],
+    "RequestHeaders": [],
+    "ResponseHeaders": [
+      "x-foo": "bar"
+    ],
     "SecurityLevel": "med",
     "WAFAction": "unknown",
     "WAFFlags": "0",

go.mod

@@ -23,7 +23,6 @@ require (
 	github.com/burningalchemist/sql_exporter v0.0.0-20221222155641-2ff59aa75200
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/cilium/ebpf v0.11.0 // indirect
-	github.com/cloudflare/cloudflare-go v0.27.0
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/davidmparrott/kafka_exporter/v2 v2.0.1
@@ -683,7 +682,7 @@ replace (
 	github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v36.2.0+incompatible
 	github.com/Azure/azure-storage-blob-go => github.com/MasslessParticle/azure-storage-blob-go v0.14.1-0.20220216145902-b5e698eff68e
 	github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab
-	github.com/cloudflare/cloudflare-go => github.com/cyriltovena/cloudflare-go v0.27.1-0.20211118103540-ff77400bcb93
+	github.com/cloudflare/cloudflare-go => github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2
 	github.com/go-kit/log => github.com/dannykopping/go-kit-log v0.2.2-0.20221002180827-5591c1641b6b
 	github.com/gocql/gocql => github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85
 	github.com/hashicorp/consul => github.com/hashicorp/consul v1.5.1

go.sum

@@ -534,8 +534,6 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
 github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/cyriltovena/cloudflare-go v0.27.1-0.20211118103540-ff77400bcb93 h1:PEBeRA25eDfHWkXNJs0HOnMhjIuKMcxKg/Z3VeuoRbU=
-github.com/cyriltovena/cloudflare-go v0.27.1-0.20211118103540-ff77400bcb93/go.mod h1:sPWL/lIC6biLEdyGZwBQ1rGQKF1FhM7N60fuNiFdYTI=
 github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE=
 github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec=
 github.com/dannykopping/go-kit-log v0.2.2-0.20221002180827-5591c1641b6b h1:G8g9mAKEj9O3RsU6Hd/ow6lIcHarlcUl5omV6sFKEOU=
@@ -2208,7 +2206,6 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK
 github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/vertica/vertica-sql-go v1.3.0 h1:oZL8PgwrpALegtTFhImsaJvg5oQd2G7v7Uai97X9Xp8=
 github.com/vertica/vertica-sql-go v1.3.0/go.mod h1:jnn2GFuv+O2Jcjktb7zyc4Utlbu9YVqpHH/lx63+1M4=
 github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI=
@@ -2818,7 +2815,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=