Experiment with Status Updates over Fluentbit

ls1intum · Aug 6, 2024 · 1216804 · 1216804
1 parent ae075d7
commit 1216804
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 17 deletions.
diff --git a/HadesScheduler/docker/docker.go b/HadesScheduler/docker/docker.go
@@ -92,29 +92,42 @@ func (d *Scheduler) SetFluentdLogging(addr string, max_retries uint) *Scheduler
 func (d Scheduler) ScheduleJob(ctx context.Context, job payload.QueuePayload) error {
 	// Create a custom logger only for this job when fluentd is enabled
 	var job_logger *slog.Logger
+	var status_logger *slog.Logger
 	var container_logs_options container.LogConfig
 	if d.FluentdOptions.Addr != "" {
 		fluentd_client, err := fluentd.GetFluentdClient(d.FluentdOptions)
 		if err != nil {
 			slog.Error("Failed to create fluentd client", slog.Any("error", err))
-			return err
-		}
-		job_logger = slog.New(slogfluentd.Option{
-			Level:  slog.LevelDebug,
-			Client: fluentd_client,
-		}.NewFluentdHandler()).With(slog.String("job_id", job.ID.String()))
-
-		// Configure the handler for the container logs
-		container_logs_options = container.LogConfig{
-			Type: "fluentd",
-			Config: map[string]string{
-				"fluentd-address":     d.FluentdOptions.Addr,
-				"fluentd-max-retries": strconv.FormatUint(uint64(d.FluentdOptions.MaxRetry), 10),
-				"labels":              "job_id",
-			},
+			job_logger = slog.Default().With(slog.String("job_id", job.ID.String()))
+			status_logger = slog.Default().With(slog.String("job_id", job.ID.String()))
+			container_logs_options = container.LogConfig{}
+		} else {
+			job_logger = slog.New(slogfluentd.Option{
+				Level:  slog.LevelDebug,
+				Client: fluentd_client,
+				Tag:    "build_logs",
+			}.NewFluentdHandler()).With(slog.String("job_id", job.ID.String()))
+
+			status_logger = slog.New(slogfluentd.Option{
+				Level:  slog.LevelDebug,
+				Client: fluentd_client,
+				Tag:    "build_status",
+			}.NewFluentdHandler()).With(slog.String("job_id", job.ID.String()))
+
+			// Configure the handler for the container logs
+			container_logs_options = container.LogConfig{
+				Type: "fluentd",
+				Config: map[string]string{
+					"fluentd-address":     d.FluentdOptions.Addr,
+					"fluentd-max-retries": strconv.FormatUint(uint64(d.FluentdOptions.MaxRetry), 10),
+					"labels":              "job_id",
+				},
+			}
+			slog.Info("Using fluentd logger", "fluentd_address", d.FluentdOptions.Addr)
 		}
 	} else {
 		job_logger = slog.Default().With(slog.String("job_id", job.ID.String()))
+		status_logger = slog.Default().With(slog.String("job_id", job.ID.String()))
 		container_logs_options = container.LogConfig{}
 		job_logger.Warn("No fluentd address provided, using default logger")
 	}
@@ -136,14 +149,17 @@ func (d Scheduler) ScheduleJob(ctx context.Context, job payload.QueuePayload) er
 		DockerProps:  jobDockerConfig,
 		QueuePayload: job,
 	}
+	status_logger.Info("Executing")
 	err := docker_job.execute(ctx)
 	if err != nil {
 		job_logger.Error("Failed to execute job", slog.Any("error", err))
+		status_logger.Error("Failed", slog.Any("error", err))
 		return err
 	}
 
 	// Delete the shared volume after the job is done
 	defer func() {
+		status_logger.Info("Cleaning up")
 		time.Sleep(1 * time.Second)
 		if err := deleteSharedVolume(ctx, d.cli, volumeName); err != nil {
 			job_logger.Error("Failed to delete shared volume", slog.Any("error", err))
@@ -152,6 +168,7 @@ func (d Scheduler) ScheduleJob(ctx context.Context, job payload.QueuePayload) er
 		job_logger.Info("Volume deleted", slog.Any("volume", volumeName))
 	}()
 
+	status_logger.Info("Completed")
 	return nil
 }
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -55,10 +55,12 @@ services:
 
   fluentbit:
     container_name: fluentbit
-    image: fluent/fluent-bit
+    image: fluent/fluent-bit:latest
     ports:
       - "24224:24224"
-    command: /fluent-bit/bin/fluent-bit -i forward -o stdout -p format=json_lines -f 1
+    volumes:
+      - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
+    command: /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf
 
 networks:
   hades:
diff --git a/fluent-bit.conf b/fluent-bit.conf
@@ -0,0 +1,39 @@
+[SERVICE]
+    Flush        0.1
+    Log_Level    info
+
+
+[INPUT]
+    Name         forward
+    Listen       0.0.0.0
+    Port         24224
+
+
+[FILTER]
+    Name        modify
+    Match       build_status
+    Rename      message status
+
+
+[OUTPUT]
+    Name         stdout
+    Match        build_status
+    Format       json_lines
+
+# [OUTPUT]
+#     Name         websocket
+#     Match        build_status
+#     Host         host.docker.internal
+#     Port         12345
+#     URI          /build-status
+#     Format       json_lines
+#     tls          off
+
+[OUTPUT]
+    Name         http
+    Match        build_status
+    Host         host.docker.internal
+    Port         3000
+    URI          /build-status
+    Format       json_lines
+    tls          off