From 7085a0dc5184be7eb80c94670213d68f9d4ea14b Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Tue, 19 Sep 2023 13:26:07 +0200 Subject: [PATCH 01/10] Debug build driver [WIP] --- DEV.md | 1 + packages/env-build-task-driver/Makefile | 8 +++++ .../{env_build_test.go => build_check.go} | 15 ++++---- .../internal/telemetry/console.go | 34 +++++++++++++++++++ packages/env-build-task-driver/main.go | 6 +++- 5 files changed, 55 insertions(+), 9 deletions(-) rename packages/env-build-task-driver/internal/{env_build_test.go => build_check.go} (92%) create mode 100644 packages/env-build-task-driver/internal/telemetry/console.go diff --git a/DEV.md b/DEV.md index b46069753..77498dc7b 100644 --- a/DEV.md +++ b/DEV.md @@ -44,6 +44,7 @@ - parametrize buckets and mount paths - enable simpler debugging - Remove supervisord and just use systemd for both nomad and consul +- Enable terraform cache again later ### API - Add monitoring to the envs routes diff --git a/packages/env-build-task-driver/Makefile b/packages/env-build-task-driver/Makefile index d7e0dcc6a..522edb021 100644 --- a/packages/env-build-task-driver/Makefile +++ b/packages/env-build-task-driver/Makefile @@ -23,4 +23,12 @@ update-build-driver: cp /mnt/disks/envs-pipeline/env-build-task-driver /opt/nomad/plugins/env-build-task-driver && \ chmod +x /opt/nomad/plugins/env-build-task-driver && \ pgrep nomad | xargs kill && \ + exit' + +run-build-check: + make build + gcloud compute scp ./bin/env-build-task-driver root@$$($(client)):/tmp/build-task-driver-test + gcloud compute ssh $$($(client)) -- 'sudo -s && \ + chmod +x /tmp/build-task-driver-test && \ + /tmp/build-task-driver-test && \ exit' \ No newline at end of file diff --git a/packages/env-build-task-driver/internal/env_build_test.go b/packages/env-build-task-driver/internal/build_check.go similarity index 92% rename from packages/env-build-task-driver/internal/env_build_test.go rename to packages/env-build-task-driver/internal/build_check.go index 870d06e6e..a8cdf7bf4 100644 --- a/packages/env-build-task-driver/internal/env_build_test.go +++ b/packages/env-build-task-driver/internal/build_check.go @@ -2,7 +2,6 @@ package internal import ( "context" - "testing" "github.com/docker/docker/client" "go.opentelemetry.io/otel" @@ -15,14 +14,14 @@ import ( //go:embed test-provision-env.ubuntu.sh var provisionEnvScriptFile string -func TestBuildEnv(t *testing.T) { +func BuidlCheck() { ctx := context.Background() tracer := otel.Tracer("test") client, err := client.NewClientWithOpts(client.FromEnv) if err != nil { - t.Fatal(err) + panic(err) } contextsPath := "/mnt/disks/docker-contexts/v1" @@ -49,29 +48,29 @@ func TestBuildEnv(t *testing.T) { err = e.Initialize(ctx, tracer) if err != nil { - t.Fatal(err) + panic(err) } defer e.Cleanup(ctx, tracer) rootfs, err := env.NewRootfs(ctx, tracer, &e, client) if err != nil { - t.Fatal(err) + panic(err) } network, err := env.NewFCNetwork(ctx, tracer, &e) if err != nil { - t.Fatal(err) + panic(err) } defer network.Cleanup(ctx, tracer) snapshot, err := env.NewSnapshot(ctx, tracer, &e, network, rootfs) if err != nil { - t.Fatal(err) + panic(err) } defer snapshot.Cleanup(ctx, tracer) err = e.MoveSnapshotToEnvDir(ctx, tracer) if err != nil { - t.Fatal(err) + panic(err) } } diff --git a/packages/env-build-task-driver/internal/telemetry/console.go b/packages/env-build-task-driver/internal/telemetry/console.go new file mode 100644 index 000000000..227ad2474 --- /dev/null +++ b/packages/env-build-task-driver/internal/telemetry/console.go @@ -0,0 +1,34 @@ +package telemetry + +// import ( +// "io" + +// "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" +// "go.opentelemetry.io/otel/sdk/resource" +// "go.opentelemetry.io/otel/sdk/trace" +// semconv "go.opentelemetry.io/otel/semconv/v1.17.0" +// ) + +// func NewExporter(w io.Writer) (trace.SpanExporter, error) { +// return stdouttrace.New( +// stdouttrace.WithWriter(w), +// // Use human-readable output. +// stdouttrace.WithPrettyPrint(), +// // Do not print timestamps for the demo. +// stdouttrace.WithoutTimestamps(), +// ) +// } + +// // newResource returns a resource describing this application. +// func NewResource(serviceName string) *resource.Resource { +// r, _ := resource.Merge( +// resource.Default(), +// resource.NewWithAttributes( +// semconv.SchemaURL, +// semconv.ServiceName(serviceName), +// semconv.ServiceVersion("v0.1.0"), +// ), +// ) + +// return r +// } diff --git a/packages/env-build-task-driver/main.go b/packages/env-build-task-driver/main.go index c5b870a7e..b400891c7 100644 --- a/packages/env-build-task-driver/main.go +++ b/packages/env-build-task-driver/main.go @@ -20,7 +20,7 @@ const ( otelCollectorGRPCEndpoint = "0.0.0.0:4317" ) -func main() { +func main2() { // Create pprof endpoint for profiling go func() { http.ListenAndServe(":6062", nil) @@ -54,3 +54,7 @@ func main() { func factory(log log.Logger) interface{} { return driver.NewPlugin(log) } + +func main() { + driver.BuidlCheck() +} From ef800abe2c013902b5823697711ee9bddd14cfa2 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 19 Sep 2023 11:40:01 +0000 Subject: [PATCH 02/10] Add make to the instance image --- packages/cluster-disk-image/main.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cluster-disk-image/main.pkr.hcl b/packages/cluster-disk-image/main.pkr.hcl index a143baac3..fbdc8a06c 100644 --- a/packages/cluster-disk-image/main.pkr.hcl +++ b/packages/cluster-disk-image/main.pkr.hcl @@ -52,7 +52,7 @@ build { provisioner "shell" { inline = [ "sudo apt-get update", - "sudo apt-get install -y unzip jq net-tools qemu-utils gcsfuse", + "sudo apt-get install -y unzip jq net-tools qemu-utils gcsfuse make", ] } From 40eb444930fde2ea0e96693e5098d63fd36adc2d Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 19 Sep 2023 11:40:37 +0000 Subject: [PATCH 03/10] Update issues --- DEV.md | 1 + 1 file changed, 1 insertion(+) diff --git a/DEV.md b/DEV.md index 77498dc7b..cbc7c5b48 100644 --- a/DEV.md +++ b/DEV.md @@ -45,6 +45,7 @@ - enable simpler debugging - Remove supervisord and just use systemd for both nomad and consul - Enable terraform cache again later +- Install golang to the vm for easier remote debugging ### API - Add monitoring to the envs routes From 5748b339afe58d1c2906b3fb77c1b16f2164bd74 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 19 Sep 2023 11:54:55 +0000 Subject: [PATCH 04/10] Update issues --- DEV.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DEV.md b/DEV.md index cbc7c5b48..ec202fe5a 100644 --- a/DEV.md +++ b/DEV.md @@ -46,6 +46,10 @@ - Remove supervisord and just use systemd for both nomad and consul - Enable terraform cache again later - Install golang to the vm for easier remote debugging +- add otel logs +- add more otel events +- improve no attributes trace (if branch for zero) +- fix vscode setup errors ### API - Add monitoring to the envs routes @@ -53,6 +57,7 @@ - Add better error if the env was not found - Improve request logging + ### Envd - envd jsonrpc parameters could be objects instead of arrays (compatibility advantages) - Check FC env envd freeze bug (probably OOM) From 4980e05e5614ca028344cd2c6d2f6c90e05bd0b8 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 19 Sep 2023 12:06:53 +0000 Subject: [PATCH 05/10] Fix case sensitive import --- packages/envd/internal/codesnippet/service.go | 7 ++++--- packages/envd/main.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/envd/internal/codesnippet/service.go b/packages/envd/internal/codesnippet/service.go index 9634697fb..17b58b471 100644 --- a/packages/envd/internal/codesnippet/service.go +++ b/packages/envd/internal/codesnippet/service.go @@ -1,13 +1,14 @@ -package codeSnippet +package codesnippet import ( "context" netstat "github.com/drael/GOnetstat" - "github.com/e2b-dev/api/packages/envd/internal/port" - "github.com/e2b-dev/api/packages/envd/internal/subscriber" "github.com/ethereum/go-ethereum/rpc" "go.uber.org/zap" + + "github.com/e2b-dev/api/packages/envd/internal/port" + "github.com/e2b-dev/api/packages/envd/internal/subscriber" ) type Service struct { diff --git a/packages/envd/main.go b/packages/envd/main.go index 52320b656..5c2057b69 100644 --- a/packages/envd/main.go +++ b/packages/envd/main.go @@ -11,7 +11,7 @@ import ( "github.com/gorilla/mux" "go.uber.org/zap" - codesnippet "github.com/e2b-dev/api/packages/envd/internal/codeSnippet" + codesnippet "github.com/e2b-dev/api/packages/envd/internal/codesnippet" "github.com/e2b-dev/api/packages/envd/internal/env" "github.com/e2b-dev/api/packages/envd/internal/filesystem" "github.com/e2b-dev/api/packages/envd/internal/port" From 73932c51fd4c48a8a328b93ae3538415f08fcd91 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 19 Sep 2023 12:11:25 +0000 Subject: [PATCH 06/10] Add testing mode [WIP] --- packages/env-build-task-driver/main.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/env-build-task-driver/main.go b/packages/env-build-task-driver/main.go index b400891c7..188e6a8cf 100644 --- a/packages/env-build-task-driver/main.go +++ b/packages/env-build-task-driver/main.go @@ -20,16 +20,30 @@ const ( otelCollectorGRPCEndpoint = "0.0.0.0:4317" ) -func main2() { +func main() { // Create pprof endpoint for profiling go func() { http.ListenAndServe(":6062", nil) }() - - telemetryAPIKey := flag.String("telemetry-api", "", "api key for telemetry") + + testMode := flag.Bool("test", false, "run in testing mode") flag.Parse() - if *telemetryAPIKey == "" { + if *testMode { + otelLauncher := launcher.ConfigureOpentelemetry( + launcher.WithServiceName(serviceName), + launcher.WithMetricReportingPeriod(10*time.Second), + launcher.WithSpanExporterEndpoint(otelCollectorGRPCEndpoint), + launcher.WithMetricExporterEndpoint(otelCollectorGRPCEndpoint), + launcher.WithMetricExporterInsecure(true), + launcher.WithPropagators([]string{"tracecontext", "baggage"}), + launcher.WithSpanExporterInsecure(true), + ) + defer otelLauncher.Shutdown() + } else { + + } + otelLauncher := launcher.ConfigureOpentelemetry( launcher.WithServiceName(serviceName), launcher.WithMetricReportingPeriod(10*time.Second), From b81b89afe3f558aaedfdbb85178866384db56b71 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Tue, 19 Sep 2023 14:58:24 +0200 Subject: [PATCH 07/10] Change context file name --- .vscode/settings.json | 1 + DEV.md | 1 + packages/api/internal/nomad/env-build.hcl | 2 ++ packages/env-build-task-driver/Makefile | 3 +- .../internal/build_check.go | 6 +++- .../env-build-task-driver/internal/driver.go | 4 +++ .../env-build-task-driver/internal/env/env.go | 9 ++--- .../internal/telemetry/console.go | 34 ------------------- .../internal/telemetry/otel.go | 28 ++++++++++----- packages/env-build-task-driver/main.go | 2 +- 10 files changed, 39 insertions(+), 51 deletions(-) delete mode 100644 packages/env-build-task-driver/internal/telemetry/console.go diff --git a/.vscode/settings.json b/.vscode/settings.json index c76704d4e..50a15b6ec 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -43,6 +43,7 @@ "formatting.local": "github.com/e2b-dev/api", "formatting.gofumpt": true, "ui.completion.usePlaceholders": true, + "experimentalWorkspaceModule": true }, "go.lintTool": "golangci-lint", "go.lintFlags": [ diff --git a/DEV.md b/DEV.md index 77498dc7b..2349766c4 100644 --- a/DEV.md +++ b/DEV.md @@ -45,6 +45,7 @@ - enable simpler debugging - Remove supervisord and just use systemd for both nomad and consul - Enable terraform cache again later +- Secure docker building (+container start problem?) ### API - Add monitoring to the envs routes diff --git a/packages/api/internal/nomad/env-build.hcl b/packages/api/internal/nomad/env-build.hcl index 72d95f2bc..cedf751bd 100644 --- a/packages/api/internal/nomad/env-build.hcl +++ b/packages/api/internal/nomad/env-build.hcl @@ -31,6 +31,8 @@ job "{{ .JobName }}/{{ .EnvID }}" { ENVS_PIPELINE_PATH = "/mnt/disks/envs-pipeline" KERNEL_IMAGE_PATH = "/fc-vm/vmlinux.bin" FIRECRACKER_BINARY_PATH = "/usr/bin/firecracker" + ENVD_NAME = "envd" + CONTEXT_FILE_NAME = "context.tar.gz" } config { diff --git a/packages/env-build-task-driver/Makefile b/packages/env-build-task-driver/Makefile index 522edb021..fd4866eee 100644 --- a/packages/env-build-task-driver/Makefile +++ b/packages/env-build-task-driver/Makefile @@ -30,5 +30,4 @@ run-build-check: gcloud compute scp ./bin/env-build-task-driver root@$$($(client)):/tmp/build-task-driver-test gcloud compute ssh $$($(client)) -- 'sudo -s && \ chmod +x /tmp/build-task-driver-test && \ - /tmp/build-task-driver-test && \ - exit' \ No newline at end of file + /tmp/build-task-driver-test' \ No newline at end of file diff --git a/packages/env-build-task-driver/internal/build_check.go b/packages/env-build-task-driver/internal/build_check.go index a8cdf7bf4..0088572ab 100644 --- a/packages/env-build-task-driver/internal/build_check.go +++ b/packages/env-build-task-driver/internal/build_check.go @@ -14,7 +14,7 @@ import ( //go:embed test-provision-env.ubuntu.sh var provisionEnvScriptFile string -func BuidlCheck() { +func BuildCheck() { ctx := context.Background() tracer := otel.Tracer("test") @@ -30,6 +30,8 @@ func BuidlCheck() { kernelImagePath := "/fc-vm/vmlinux.bin" firecrackerBinaryPath := "/usr/bin/firecracker" envsPipelinePath := "/mnt/disks/envs-pipeline" + envdName := "envd" + contextFileName := "context.tar.gz" e := env.Env{ BuildID: "testing-build-id", @@ -44,6 +46,8 @@ func BuidlCheck() { FirecrackerBinaryPath: firecrackerBinaryPath, ProvisionScript: provisionEnvScriptFile, EnvsPipelinePath: envsPipelinePath, + EnvdName: envdName, + ContextFileName: contextFileName, } err = e.Initialize(ctx, tracer) diff --git a/packages/env-build-task-driver/internal/driver.go b/packages/env-build-task-driver/internal/driver.go index 0417ef68d..9378b6c0c 100644 --- a/packages/env-build-task-driver/internal/driver.go +++ b/packages/env-build-task-driver/internal/driver.go @@ -259,6 +259,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive kernelImagePath := cfg.Env["KERNEL_IMAGE_PATH"] firecrackerBinaryPath := cfg.Env["FIRECRACKER_BINARY_PATH"] envsPipelinePath := cfg.Env["ENVS_PIPELINE_PATH"] + envdName := cfg.Env["ENVD_NAME"] + contextFileName := cfg.Env["CONTEXT_FILE_NAME"] env := env.Env{ BuildID: taskConfig.BuildID, @@ -273,6 +275,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive FirecrackerBinaryPath: firecrackerBinaryPath, ProvisionScript: taskConfig.ProvisionScript, EnvsPipelinePath: envsPipelinePath, + EnvdName: envdName, + ContextFileName: contextFileName, } cancellableContext, cancel := context.WithTimeout(childCtx, envBuildTimeout) diff --git a/packages/env-build-task-driver/internal/env/env.go b/packages/env-build-task-driver/internal/env/env.go index d62be25c1..6df136f38 100644 --- a/packages/env-build-task-driver/internal/env/env.go +++ b/packages/env-build-task-driver/internal/env/env.go @@ -18,8 +18,6 @@ const ( memfileName = "memfile" buildDirName = "builds" - - envdName = "envd" ) type Env struct { @@ -57,16 +55,19 @@ type Env struct { // Path to the directory where files for the envs pipeline are stored. EnvsPipelinePath string + + EnvdName string + ContextFileName string } // Path to the envd. func (e *Env) EnvdPath() string { - return filepath.Join(e.EnvsPipelinePath, envdName) + return filepath.Join(e.EnvsPipelinePath, e.EnvdName) } // Path to the docker context. func (e *Env) DockerContextPath() string { - return filepath.Join(e.DockerContextsPath, e.EnvID) + return filepath.Join(e.DockerContextsPath, e.EnvID, e.ContextFileName) } // Docker tag of the docker image for this env. diff --git a/packages/env-build-task-driver/internal/telemetry/console.go b/packages/env-build-task-driver/internal/telemetry/console.go deleted file mode 100644 index 227ad2474..000000000 --- a/packages/env-build-task-driver/internal/telemetry/console.go +++ /dev/null @@ -1,34 +0,0 @@ -package telemetry - -// import ( -// "io" - -// "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" -// "go.opentelemetry.io/otel/sdk/resource" -// "go.opentelemetry.io/otel/sdk/trace" -// semconv "go.opentelemetry.io/otel/semconv/v1.17.0" -// ) - -// func NewExporter(w io.Writer) (trace.SpanExporter, error) { -// return stdouttrace.New( -// stdouttrace.WithWriter(w), -// // Use human-readable output. -// stdouttrace.WithPrettyPrint(), -// // Do not print timestamps for the demo. -// stdouttrace.WithoutTimestamps(), -// ) -// } - -// // newResource returns a resource describing this application. -// func NewResource(serviceName string) *resource.Resource { -// r, _ := resource.Merge( -// resource.Default(), -// resource.NewWithAttributes( -// semconv.SchemaURL, -// semconv.ServiceName(serviceName), -// semconv.ServiceVersion("v0.1.0"), -// ), -// ) - -// return r -// } diff --git a/packages/env-build-task-driver/internal/telemetry/otel.go b/packages/env-build-task-driver/internal/telemetry/otel.go index 14f711e2a..ef6d4a025 100644 --- a/packages/env-build-task-driver/internal/telemetry/otel.go +++ b/packages/env-build-task-driver/internal/telemetry/otel.go @@ -51,9 +51,13 @@ func ReportEvent(ctx context.Context, name string, attrs ...attribute.KeyValue) fmt.Println(name, attrs) - span.AddEvent(name, - trace.WithAttributes(attrs...), - ) + if len(attrs) == 0 { + span.AddEvent(name) + } else { + span.AddEvent(name, + trace.WithAttributes(attrs...), + ) + } } func ReportCriticalError(ctx context.Context, err error) { @@ -72,10 +76,16 @@ func ReportError(ctx context.Context, err error, attrs ...attribute.KeyValue) { fmt.Fprint(os.Stderr, err.Error()) - span.RecordError(err, - trace.WithStackTrace(true), - trace.WithAttributes( - attrs..., - ), - ) + if len(attrs) == 0 { + span.RecordError(err, + trace.WithStackTrace(true), + ) + } else { + span.RecordError(err, + trace.WithStackTrace(true), + trace.WithAttributes( + attrs..., + ), + ) + } } diff --git a/packages/env-build-task-driver/main.go b/packages/env-build-task-driver/main.go index b400891c7..5fac62f11 100644 --- a/packages/env-build-task-driver/main.go +++ b/packages/env-build-task-driver/main.go @@ -56,5 +56,5 @@ func factory(log log.Logger) interface{} { } func main() { - driver.BuidlCheck() + driver.BuildCheck() } From ca15b6c170cf94f6cf30202ba3f19c424c7becfd Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Tue, 19 Sep 2023 15:09:54 +0200 Subject: [PATCH 08/10] Fix context path; Cleanup --- DEV.md | 1 + .../env-build-task-driver/internal/env/env.go | 2 +- .../{build_check.go => test_build_process.go} | 2 +- packages/env-build-task-driver/main.go | 62 +++++++------------ 4 files changed, 25 insertions(+), 42 deletions(-) rename packages/env-build-task-driver/internal/{build_check.go => test_build_process.go} (98%) diff --git a/DEV.md b/DEV.md index 88b9fe3d4..746956622 100644 --- a/DEV.md +++ b/DEV.md @@ -51,6 +51,7 @@ - add more otel events - improve no attributes trace (if branch for zero) - fix vscode setup errors +- make buckets not public ### API - Add monitoring to the envs routes diff --git a/packages/env-build-task-driver/internal/env/env.go b/packages/env-build-task-driver/internal/env/env.go index 6df136f38..2740d3865 100644 --- a/packages/env-build-task-driver/internal/env/env.go +++ b/packages/env-build-task-driver/internal/env/env.go @@ -67,7 +67,7 @@ func (e *Env) EnvdPath() string { // Path to the docker context. func (e *Env) DockerContextPath() string { - return filepath.Join(e.DockerContextsPath, e.EnvID, e.ContextFileName) + return filepath.Join(e.DockerContextsPath, e.EnvID, e.BuildID, e.ContextFileName) } // Docker tag of the docker image for this env. diff --git a/packages/env-build-task-driver/internal/build_check.go b/packages/env-build-task-driver/internal/test_build_process.go similarity index 98% rename from packages/env-build-task-driver/internal/build_check.go rename to packages/env-build-task-driver/internal/test_build_process.go index 0088572ab..b03eb3f05 100644 --- a/packages/env-build-task-driver/internal/build_check.go +++ b/packages/env-build-task-driver/internal/test_build_process.go @@ -14,7 +14,7 @@ import ( //go:embed test-provision-env.ubuntu.sh var provisionEnvScriptFile string -func BuildCheck() { +func TestBuildProcess() { ctx := context.Background() tracer := otel.Tracer("test") diff --git a/packages/env-build-task-driver/main.go b/packages/env-build-task-driver/main.go index 4f7baaa98..27076c340 100644 --- a/packages/env-build-task-driver/main.go +++ b/packages/env-build-task-driver/main.go @@ -20,55 +20,37 @@ const ( otelCollectorGRPCEndpoint = "0.0.0.0:4317" ) +func configurePlugin() { + otelLauncher := launcher.ConfigureOpentelemetry( + launcher.WithServiceName(serviceName), + launcher.WithMetricReportingPeriod(10*time.Second), + launcher.WithSpanExporterEndpoint(otelCollectorGRPCEndpoint), + launcher.WithMetricExporterEndpoint(otelCollectorGRPCEndpoint), + launcher.WithMetricExporterInsecure(true), + launcher.WithPropagators([]string{"tracecontext", "baggage"}), + launcher.WithSpanExporterInsecure(true), + ) + defer otelLauncher.Shutdown() + + plugins.Serve(factory) +} + +func factory(log log.Logger) interface{} { + return driver.NewPlugin(log) +} + func main() { // Create pprof endpoint for profiling go func() { http.ListenAndServe(":6062", nil) }() - + testMode := flag.Bool("test", false, "run in testing mode") flag.Parse() if *testMode { - otelLauncher := launcher.ConfigureOpentelemetry( - launcher.WithServiceName(serviceName), - launcher.WithMetricReportingPeriod(10*time.Second), - launcher.WithSpanExporterEndpoint(otelCollectorGRPCEndpoint), - launcher.WithMetricExporterEndpoint(otelCollectorGRPCEndpoint), - launcher.WithMetricExporterInsecure(true), - launcher.WithPropagators([]string{"tracecontext", "baggage"}), - launcher.WithSpanExporterInsecure(true), - ) - defer otelLauncher.Shutdown() + configurePlugin() } else { - + driver.TestBuildProcess() } - - otelLauncher := launcher.ConfigureOpentelemetry( - launcher.WithServiceName(serviceName), - launcher.WithMetricReportingPeriod(10*time.Second), - launcher.WithSpanExporterEndpoint(otelCollectorGRPCEndpoint), - launcher.WithMetricExporterEndpoint(otelCollectorGRPCEndpoint), - launcher.WithMetricExporterInsecure(true), - launcher.WithPropagators([]string{"tracecontext", "baggage"}), - launcher.WithSpanExporterInsecure(true), - ) - defer otelLauncher.Shutdown() - } else { - otelLauncher := launcher.ConfigureOpentelemetry( - launcher.WithServiceName(serviceName), - launcher.WithAccessToken(*telemetryAPIKey), - ) - defer otelLauncher.Shutdown() - } - - plugins.Serve(factory) -} - -func factory(log log.Logger) interface{} { - return driver.NewPlugin(log) -} - -func main() { - driver.BuildCheck() } From e51fef64fa7704896cb84ee5ef0c0467dabb3f73 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Tue, 19 Sep 2023 16:11:56 +0200 Subject: [PATCH 09/10] Parametrize testing; Fix flags --- DEV.md | 2 +- go.work.sum | 17 +++++++++++++++++ packages/env-build-task-driver/Makefile | 5 ++--- .../internal/test_build_process.go | 6 +++--- packages/env-build-task-driver/main.go | 10 ++++++---- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/DEV.md b/DEV.md index 746956622..dd8c01371 100644 --- a/DEV.md +++ b/DEV.md @@ -44,7 +44,7 @@ - parametrize buckets and mount paths - enable simpler debugging - Remove supervisord and just use systemd for both nomad and consul -- Enable terraform cache again later +- Enable terraform cache again later (cca 16 blocking seconds saved per GHA run) - Secure docker building (+container start problem?) - Install golang to the vm for easier remote debugging - add otel logs diff --git a/go.work.sum b/go.work.sum index e80413087..f1c4bf879 100644 --- a/go.work.sum +++ b/go.work.sum @@ -412,13 +412,16 @@ github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3 github.com/CloudyKit/jet/v6 v6.2.0/go.mod h1:d3ypHeIRNo2+XyqnGA8s+aphtcVpjP5hPwP/Lzo7Ro4= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM= +github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06/go.mod h1:7erjKLwalezA0k99cWs5L11HWOAPNjdUZ6RxH1BXbbM= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/apparentlymart/go-cidr v1.0.1/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= @@ -458,6 +461,7 @@ github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHq github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= github.com/containerd/go-cni v1.1.9/go.mod h1:XYrZJ1d5W6E2VOvjffL3IZq0Dz6bsVlERHbekNK90PM= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= github.com/coreos/go-oidc/v3 v3.1.0/go.mod h1:rEJ/idjfUyfkBit1eI1fvyr+64/g9dcKpAm8MJMesvo= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -468,6 +472,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeC github.com/digitalocean/godo v1.10.0/go.mod h1:h6faOIcZ8lWIwNQ+DN7b3CgX4Kwby5T+nbpNqkUIozU= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/docker/cli v23.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= @@ -499,6 +505,7 @@ github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg github.com/gomarkdown/markdown v0.0.0-20230716120725-531d2d74bc12/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk= github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= @@ -558,6 +565,8 @@ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbq github.com/lestrrat-go/jwx v1.2.25/go.mod h1:zoNuZymNl5lgdcu6P7K6ie2QRll5HVfF4xwxBBK1NxY= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/linode/linodego v0.7.1/go.mod h1:ga11n3ivecUrPCHN0rANxKmfWBJVkOXfLMZinAbj2sY= +github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -576,6 +585,7 @@ github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo github.com/networkplumbing/go-nft v0.2.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs= github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h1:TLb2Sg7HQcgGdloNxkrmtgDNR9uVYF3lfdFIN4Ro6Sk= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/open-policy-agent/opa v0.42.2/go.mod h1:MrmoTi/BsKWT58kXlVayBb+rYVeaMwuBm3nYAN3923s= github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v1.1.0-rc.3/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= @@ -588,9 +598,11 @@ github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3d github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7/go.mod h1:IToEjHuttnUzwZI5KBSM/LOOW3qLbbrHOEfp3SbECGY= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03/go.mod h1:gRAiPF5C5Nd0eyyRdqIu9qTiFSoZzpTq727b5B8fkkU= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/ryanuber/columnize v2.1.2+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.2.0/go.mod h1:WkKB1DnNtvsMlDmQ50sgwowDJV/hGbJSOvJoEXs1AJQ= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= @@ -613,14 +625,19 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o github.com/urfave/cli/v2 v2.24.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/valyala/fasthttp v1.49.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= +github.com/vektah/gqlparser/v2 v2.4.5/go.mod h1:flJWIR04IMQPGz+BXLrORkrARBxv/rtyIAFvd/MceW0= +github.com/veraison/go-cose v1.0.0-rc.1/go.mod h1:7ziE85vSq4ScFTg6wyoMXjucIGOf4JkFEZi/an96Ct4= github.com/vmihailenco/msgpack v3.3.3+incompatible h1:wapg9xDUZDzGCNFlwc5SqI1rvcciqcxEHac4CYj89xI= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/vmware/govmomi v0.18.0/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yashtewari/glob-intersection v0.1.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok= github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0= github.com/zclconf/go-cty-yaml v1.0.3/go.mod h1:9YLUH4g7lOhVWqUbctnVlZ5KLpg7JAprQNgxSZ1Gyxs= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= diff --git a/packages/env-build-task-driver/Makefile b/packages/env-build-task-driver/Makefile index fd4866eee..c09450e73 100644 --- a/packages/env-build-task-driver/Makefile +++ b/packages/env-build-task-driver/Makefile @@ -27,7 +27,6 @@ update-build-driver: run-build-check: make build - gcloud compute scp ./bin/env-build-task-driver root@$$($(client)):/tmp/build-task-driver-test + gcloud compute scp ./bin/env-build-task-driver root@$$($(client)):/tmp/build-task-driver gcloud compute ssh $$($(client)) -- 'sudo -s && \ - chmod +x /tmp/build-task-driver-test && \ - /tmp/build-task-driver-test' \ No newline at end of file + chmod +x /tmp/build-task-driver' \ No newline at end of file diff --git a/packages/env-build-task-driver/internal/test_build_process.go b/packages/env-build-task-driver/internal/test_build_process.go index b03eb3f05..1a126586c 100644 --- a/packages/env-build-task-driver/internal/test_build_process.go +++ b/packages/env-build-task-driver/internal/test_build_process.go @@ -14,7 +14,7 @@ import ( //go:embed test-provision-env.ubuntu.sh var provisionEnvScriptFile string -func TestBuildProcess() { +func TestBuildProcess(envID, buildID string) { ctx := context.Background() tracer := otel.Tracer("test") @@ -34,8 +34,8 @@ func TestBuildProcess() { contextFileName := "context.tar.gz" e := env.Env{ - BuildID: "testing-build-id", - EnvID: "testing-env-id", + BuildID: buildID, + EnvID: envID, EnvsPath: envsPath, VCpuCount: 1, MemoryMB: 512, diff --git a/packages/env-build-task-driver/main.go b/packages/env-build-task-driver/main.go index 27076c340..db6c961dd 100644 --- a/packages/env-build-task-driver/main.go +++ b/packages/env-build-task-driver/main.go @@ -45,12 +45,14 @@ func main() { http.ListenAndServe(":6062", nil) }() - testMode := flag.Bool("test", false, "run in testing mode") + envID := flag.String("env", "", "env id") + buildID := flag.String("build", "", "build id") + flag.Parse() - if *testMode { - configurePlugin() + if envID != nil && buildID != nil { + driver.TestBuildProcess(*envID, *buildID) } else { - driver.TestBuildProcess() + configurePlugin() } } From 0185e565cee3345d7bb500f98923171e3acfb9b2 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Tue, 19 Sep 2023 16:36:44 +0200 Subject: [PATCH 10/10] Fix build driver init --- packages/env-build-task-driver/internal/env/rootfs.go | 6 +++--- packages/env-build-task-driver/main.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/env-build-task-driver/internal/env/rootfs.go b/packages/env-build-task-driver/internal/env/rootfs.go index 3e6706e15..5e95d74d6 100644 --- a/packages/env-build-task-driver/internal/env/rootfs.go +++ b/packages/env-build-task-driver/internal/env/rootfs.go @@ -81,8 +81,6 @@ func (r *Rootfs) buildDockerImage(ctx context.Context, tracer trace.Tracer) erro }, ) - // Handle fuse mount - // TODO: Stream the logs somewhere _, err = io.Copy(os.Stdout, buildResponse.Body) if err != nil { @@ -148,7 +146,9 @@ func (r *Rootfs) createRootfsFile(ctx context.Context, tracer trace.Tracer) erro telemetry.ReportEvent(childCtx, "opened envd file") // Copy envd to the container - err = r.client.CopyToContainer(childCtx, cont.ID, envdRootfsPath, envdFile, types.CopyToContainerOptions{}) + err = r.client.CopyToContainer(childCtx, cont.ID, envdRootfsPath, envdFile, types.CopyToContainerOptions{ + AllowOverwriteDirWithFile: true, + }) if err != nil { return fmt.Errorf("error copying envd to container %v", err) } diff --git a/packages/env-build-task-driver/main.go b/packages/env-build-task-driver/main.go index db6c961dd..f9530f276 100644 --- a/packages/env-build-task-driver/main.go +++ b/packages/env-build-task-driver/main.go @@ -51,8 +51,8 @@ func main() { flag.Parse() if envID != nil && buildID != nil { - driver.TestBuildProcess(*envID, *buildID) - } else { configurePlugin() + } else { + driver.TestBuildProcess(*envID, *buildID) } }