diff --git a/internal/db/start/start.go b/internal/db/start/start.go index c83e5e2f5..117a00adf 100644 --- a/internal/db/start/start.go +++ b/internal/db/start/start.go @@ -100,14 +100,14 @@ EOF func NewHostConfig() container.HostConfig { hostPort := strconv.FormatUint(uint64(utils.Config.Db.Port), 10) - hostConfig := WithSyslogConfig(container.HostConfig{ + hostConfig := container.HostConfig{ PortBindings: nat.PortMap{"5432/tcp": []nat.PortBinding{{HostPort: hostPort}}}, RestartPolicy: container.RestartPolicy{Name: "always"}, Binds: []string{ utils.DbId + ":/var/lib/postgresql/data", utils.ConfigId + ":/etc/postgresql-custom", }, - }) + } return hostConfig } @@ -184,17 +184,6 @@ func IsUnhealthyError(err error) bool { return ok } -func WithSyslogConfig(hostConfig container.HostConfig) container.HostConfig { - if utils.Config.Analytics.Enabled { - hostConfig.LogConfig.Type = "syslog" - hostConfig.LogConfig.Config = map[string]string{ - "syslog-address": fmt.Sprintf("tcp://%s:%d", utils.Config.Hostname, utils.Config.Analytics.VectorPort), - "tag": "{{.Name}}", - } - } - return hostConfig -} - func initCurrentBranch(fsys afero.Fs) error { // Create _current_branch file to avoid breaking db branch commands if _, err := fsys.Stat(utils.CurrBranchPath); err == nil { diff --git a/internal/functions/deploy/deploy.go b/internal/functions/deploy/deploy.go index be23a96ac..d29662972 100644 --- a/internal/functions/deploy/deploy.go +++ b/internal/functions/deploy/deploy.go @@ -18,7 +18,6 @@ import ( "github.com/go-errors/errors" "github.com/spf13/afero" "github.com/spf13/viper" - "github.com/supabase/cli/internal/db/start" "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/pkg/api" ) @@ -138,9 +137,9 @@ func bundleFunction(ctx context.Context, slug, hostImportMapPath string, fsys af Env: []string{}, Cmd: cmd, }, - start.WithSyslogConfig(container.HostConfig{ + container.HostConfig{ Binds: binds, - }), + }, network.NetworkingConfig{}, "", os.Stdout, diff --git a/internal/functions/download/download.go b/internal/functions/download/download.go index 6e1678acf..71720c4fa 100644 --- a/internal/functions/download/download.go +++ b/internal/functions/download/download.go @@ -15,7 +15,6 @@ import ( "github.com/docker/docker/api/types/network" "github.com/go-errors/errors" "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/start" "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/pkg/api" ) @@ -185,9 +184,9 @@ func extractOne(ctx context.Context, slug, eszipPath string) error { Image: utils.EdgeRuntimeImage, Cmd: []string{"unbundle", "--eszip", dockerEszipPath, "--output", utils.DockerDenoDir}, }, - start.WithSyslogConfig(container.HostConfig{ + container.HostConfig{ Binds: binds, - }), + }, network.NetworkingConfig{}, "", os.Stdout, diff --git a/internal/functions/serve/serve.go b/internal/functions/serve/serve.go index 2de02342f..be14519a7 100644 --- a/internal/functions/serve/serve.go +++ b/internal/functions/serve/serve.go @@ -15,7 +15,6 @@ import ( "github.com/go-errors/errors" "github.com/spf13/afero" "github.com/spf13/viper" - "github.com/supabase/cli/internal/db/start" "github.com/supabase/cli/internal/functions/deploy" "github.com/supabase/cli/internal/secrets/set" "github.com/supabase/cli/internal/utils" @@ -177,10 +176,10 @@ EOF WorkingDir: utils.DockerDenoDir, // No tcp health check because edge runtime logs them as client connection error }, - start.WithSyslogConfig(container.HostConfig{ + container.HostConfig{ Binds: binds, PortBindings: portBindings, - }), + }, network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ utils.NetId: { diff --git a/internal/start/start.go b/internal/start/start.go index b444680f0..a6ec1198e 100644 --- a/internal/start/start.go +++ b/internal/start/start.go @@ -15,6 +15,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/network" + "github.com/docker/docker/client" "github.com/docker/go-connections/nat" "github.com/go-errors/errors" "github.com/jackc/pgconn" @@ -118,6 +119,7 @@ var ( type vectorConfig struct { ApiKey string + VectorId string LogflareId string KongId string GotrueId string @@ -164,6 +166,7 @@ func run(p utils.Program, ctx context.Context, fsys afero.Fs, excludedContainers var vectorConfigBuf bytes.Buffer if err := vectorConfigTemplate.Execute(&vectorConfigBuf, vectorConfig{ ApiKey: utils.Config.Analytics.ApiKey, + VectorId: utils.VectorId, LogflareId: utils.LogflareId, KongId: utils.KongId, GotrueId: utils.GotrueId, @@ -175,14 +178,25 @@ func run(p utils.Program, ctx context.Context, fsys afero.Fs, excludedContainers }); err != nil { return errors.Errorf("failed to exec template: %w", err) } - p.Send(utils.StatusMsg("Starting syslog driver...")) + p.Send(utils.StatusMsg("Starting vector...")) + var binds []string + env := []string{ + "VECTOR_CONFIG=/etc/vector/vector.yaml", + } + host := utils.Docker.DaemonHost() + if parsed, err := client.ParseHostURL(host); err == nil { + if parsed.Scheme == "tcp" { + // Special case for GitLab pipeline + env = append(env, "DOCKER_HOST="+host) + } else { + binds = append(binds, parsed.Host+":/var/run/docker.sock:ro") + } + } if _, err := utils.DockerStart( ctx, container.Config{ Image: utils.VectorImage, - Env: []string{ - "VECTOR_CONFIG=/etc/vector/vector.yaml", - }, + Env: env, Entrypoint: []string{"sh", "-c", `cat <<'EOF' > /etc/vector/vector.yaml && vector ` + vectorConfigBuf.String() + ` EOF @@ -195,10 +209,9 @@ EOF Timeout: 2 * time.Second, Retries: 3, }, - ExposedPorts: nat.PortSet{"9000/tcp": {}}, }, container.HostConfig{ - PortBindings: nat.PortMap{"9000/tcp": []nat.PortBinding{{HostPort: strconv.FormatUint(uint64(utils.Config.Analytics.VectorPort), 10)}}}, + Binds: binds, RestartPolicy: container.RestartPolicy{Name: "always"}, }, network.NetworkingConfig{ @@ -370,11 +383,11 @@ EOF EOF `}, }, - start.WithSyslogConfig(container.HostConfig{ + container.HostConfig{ Binds: binds, PortBindings: nat.PortMap{"8000/tcp": []nat.PortBinding{{HostPort: strconv.FormatUint(uint64(utils.Config.Api.Port), 10)}}}, RestartPolicy: container.RestartPolicy{Name: "always"}, - }), + }, network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ utils.NetId: { @@ -606,9 +619,9 @@ EOF Retries: 3, }, }, - start.WithSyslogConfig(container.HostConfig{ + container.HostConfig{ RestartPolicy: container.RestartPolicy{Name: "always"}, - }), + }, network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ utils.NetId: { @@ -699,9 +712,9 @@ EOF Retries: 3, }, }, - start.WithSyslogConfig(container.HostConfig{ + container.HostConfig{ RestartPolicy: container.RestartPolicy{Name: "always"}, - }), + }, network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ utils.NetId: { @@ -733,9 +746,9 @@ EOF }, // PostgREST does not expose a shell for health check }, - start.WithSyslogConfig(container.HostConfig{ + container.HostConfig{ RestartPolicy: container.RestartPolicy{Name: "always"}, - }), + }, network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ utils.NetId: { @@ -789,10 +802,10 @@ EOF Retries: 3, }, }, - start.WithSyslogConfig(container.HostConfig{ + container.HostConfig{ RestartPolicy: container.RestartPolicy{Name: "always"}, Binds: []string{utils.StorageId + ":" + dockerStoragePath}, - }), + }, network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ utils.NetId: { diff --git a/internal/start/templates/vector.yaml b/internal/start/templates/vector.yaml index daa3b98ae..21dc5b6fa 100644 --- a/internal/start/templates/vector.yaml +++ b/internal/start/templates/vector.yaml @@ -1,28 +1,30 @@ api: enabled: true - address: "0.0.0.0:9001" + address: 0.0.0.0:9001 sources: - docker_syslog: - type: "syslog" - address: "0.0.0.0:9000" - mode: "tcp" - path: "/tmp/socket" + docker_host: + type: docker_logs + exclude_containers: + - "{{ .VectorId }}" transforms: project_logs: type: remap inputs: - - docker_syslog + - docker_host source: |- .project = "default" .event_message = del(.message) - del(.procid) - del(.source_id) + .appname = del(.container_name) + del(.container_created_at) + del(.container_id) del(.source_type) - del(.facility) + del(.stream) + del(.label) + del(.image) del(.host) - del(.id) + del(.stream) router: type: route inputs: @@ -35,7 +37,7 @@ transforms: storage: '.appname == "{{ .StorageId }}"' functions: '.appname == "{{ .EdgeRuntimeId }}"' db: '.appname == "{{ .DbId }}"' - # Kong logs only include api requests + # Ignores non nginx errors since they are related with kong booting up kong_logs: type: remap inputs: @@ -52,7 +54,10 @@ transforms: .metadata.request.protocol = req.protocol .metadata.response.status_code = req.status } - # TODO: create a separate page and filter for kong error logs + if err != null { + abort + } + # Ignores non nginx errors since they are related with kong booting up kong_err: type: remap inputs: @@ -73,6 +78,9 @@ transforms: .metadata.request.protocol = url[2] } } + if err != null { + abort + } # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency. auth_logs: type: remap @@ -126,7 +134,6 @@ transforms: .metadata.context[0].pid = parsed.pid } # Postgres logs some messages to stderr which we map to warning severity level - # TODO: parse raw postgres logs via regex db_logs: type: remap inputs: @@ -134,11 +141,20 @@ transforms: source: |- .metadata.host = "db-default" .metadata.parsed.timestamp = .timestamp - .metadata.parsed.error_severity = replace!(.severity, r'^err$', "warning") + + parsed, err = parse_regex(.event_message, r'.*(?PINFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC?):.*', numeric_groups: true) + + if err != null || parsed == null { + .metadata.parsed.error_severity = "info" + } + if parsed != null { + .metadata.parsed.error_severity = parsed.level + } if .metadata.parsed.error_severity == "info" { .metadata.parsed.error_severity = "log" } - .metadata.parsed.error_severity = upcase(.metadata.parsed.error_severity) + .metadata.parsed.error_severity = upcase!(.metadata.parsed.error_severity) + sinks: logflare_auth: type: "http" @@ -179,34 +195,35 @@ sinks: method: "post" request: retry_max_duration_secs: 10 - # We must route the sink through kong because ingesting logs before logflare is fully initialized will + # We must route the sink through kong because ingesting logs before logflare is fully initialised will # lead to broken queries from studio. This works by the assumption that containers are started in the # following order: vector > db > logflare > kong uri: "http://{{ .KongId }}:8000/analytics/v1/api/logs?source_name=postgres.logs&api_key={{ .ApiKey }}" - logflare_storage: + logflare_functions: type: "http" inputs: - - storage_logs + - router.functions encoding: codec: "json" method: "post" request: retry_max_duration_secs: 10 - uri: "http://{{ .LogflareId }}:4000/api/logs?source_name=storage.logs.prod.2&api_key={{ .ApiKey }}" - logflare_functions: + uri: "http://{{ .LogflareId }}:4000/api/logs?source_name=deno-relay-logs&api_key={{ .ApiKey }}" + logflare_storage: type: "http" inputs: - - router.functions + - storage_logs encoding: codec: "json" method: "post" request: retry_max_duration_secs: 10 - uri: "http://{{ .LogflareId }}:4000/api/logs?source_name=deno-relay-logs&api_key={{ .ApiKey }}" + uri: "http://{{ .LogflareId }}:4000/api/logs?source_name=storage.logs.prod.2&api_key={{ .ApiKey }}" logflare_kong: type: "http" inputs: - kong_logs + - kong_err encoding: codec: "json" method: "post" diff --git a/internal/utils/config.go b/internal/utils/config.go index a7563cd8c..daa5e7fbd 100644 --- a/internal/utils/config.go +++ b/internal/utils/config.go @@ -13,6 +13,7 @@ import ( "time" "github.com/BurntSushi/toml" + "github.com/docker/docker/client" "github.com/docker/go-units" "github.com/go-errors/errors" "github.com/golang-jwt/jwt/v5" @@ -516,11 +517,12 @@ type ( Enabled bool `toml:"enabled"` Port uint16 `toml:"port"` Backend LogflareBackend `toml:"backend"` - VectorPort uint16 `toml:"vector_port"` GcpProjectId string `toml:"gcp_project_id"` GcpProjectNumber string `toml:"gcp_project_number"` GcpJwtPath string `toml:"gcp_jwt_path"` ApiKey string `toml:"-" mapstructure:"api_key"` + // Deprecated together with syslog + VectorPort uint16 `toml:"vector_port"` } experimental struct { @@ -862,6 +864,9 @@ func LoadConfigFS(fsys afero.Fs) error { } // Validate logflare config if Config.Analytics.Enabled { + if Docker.DaemonHost() != client.DefaultDockerHost { + fmt.Fprintln(os.Stderr, Yellow("WARNING:"), "running analytics in docker rootless mode is unsupported.") + } switch Config.Analytics.Backend { case LogflareBigQuery: if len(Config.Analytics.GcpProjectId) == 0 { diff --git a/internal/utils/templates/init_config.test.toml b/internal/utils/templates/init_config.test.toml index 227d37bb9..1a5a7510a 100644 --- a/internal/utils/templates/init_config.test.toml +++ b/internal/utils/templates/init_config.test.toml @@ -176,7 +176,6 @@ inspector_port = 8083 [analytics] enabled = false port = 54327 -vector_port = 54328 # Configure one of the supported backends: `postgres`, `bigquery`. backend = "postgres" diff --git a/internal/utils/templates/init_config.toml b/internal/utils/templates/init_config.toml index f2a9838b5..3853011b0 100644 --- a/internal/utils/templates/init_config.toml +++ b/internal/utils/templates/init_config.toml @@ -176,7 +176,6 @@ inspector_port = 8083 [analytics] enabled = false port = 54327 -vector_port = 54328 # Configure one of the supported backends: `postgres`, `bigquery`. backend = "postgres"