From 4dd322cb78a94d6567d5a1254178c8173f1f202d Mon Sep 17 00:00:00 2001 From: Robert Jandow Date: Sat, 23 Sep 2023 22:26:00 +0200 Subject: [PATCH] Add volume support and clone container --- HadesScheduler/docker.go | 48 ---------------- HadesScheduler/docker/container.go | 17 ++++++ HadesScheduler/docker/docker.go | 90 ++++++++++++++++++++++++++++++ HadesScheduler/docker/volume.go | 34 +++++++++++ HadesScheduler/main.go | 3 +- shared/utils/git.go | 22 ++++++++ 6 files changed, 165 insertions(+), 49 deletions(-) delete mode 100644 HadesScheduler/docker.go create mode 100644 HadesScheduler/docker/container.go create mode 100644 HadesScheduler/docker/docker.go create mode 100644 HadesScheduler/docker/volume.go create mode 100644 shared/utils/git.go diff --git a/HadesScheduler/docker.go b/HadesScheduler/docker.go deleted file mode 100644 index 3ba25ea..0000000 --- a/HadesScheduler/docker.go +++ /dev/null @@ -1,48 +0,0 @@ -package main - -import ( - "context" - "github.com/Mtze/HadesCI/shared/payload" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/client" - _ "github.com/docker/docker/client" - log "github.com/sirupsen/logrus" -) - -type DockerScheduler struct{} - -func (d *DockerScheduler) ScheduleJob(job payload.BuildJob) error { - name := job.BuildConfig.ExecutionContainer - ctx := context.Background() - // Create a new Docker client - cli, err := client.NewClientWithOpts(client.FromEnv) - if err != nil { - return err - } - defer cli.Close() - - // Pull the image - _, err = cli.ImagePull(ctx, name, types.ImagePullOptions{}) - if err != nil { - return err - } - - // Create the container - resp, err := cli.ContainerCreate(ctx, &container.Config{ - Image: name, - Cmd: []string{"sleep", "300"}, - }, nil, nil, nil, "") - if err != nil { - return err - } - - // Start the container - err = cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}) - if err != nil { - return err - } - - log.Info("Container %s started with ID: %s\n", name, resp.ID) - return nil -} diff --git a/HadesScheduler/docker/container.go b/HadesScheduler/docker/container.go new file mode 100644 index 0000000..ee3cf86 --- /dev/null +++ b/HadesScheduler/docker/container.go @@ -0,0 +1,17 @@ +package docker + +import ( + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" +) + +var defaultHostConfig = container.HostConfig{ + Mounts: []mount.Mount{ + { + Type: mount.TypeVolume, + Source: sharedVolumeName, + Target: "/shared", + }, + }, + AutoRemove: true, +} diff --git a/HadesScheduler/docker/docker.go b/HadesScheduler/docker/docker.go new file mode 100644 index 0000000..5a3b12e --- /dev/null +++ b/HadesScheduler/docker/docker.go @@ -0,0 +1,90 @@ +package docker + +import ( + "context" + "github.com/Mtze/HadesCI/shared/payload" + "github.com/Mtze/HadesCI/shared/utils" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" + _ "github.com/docker/docker/client" + log "github.com/sirupsen/logrus" +) + +const ( + cloneContainerImage = "alpine/git:latest" + sharedVolumeName = "shared" +) + +type DockerScheduler struct{} + +func (d *DockerScheduler) ScheduleJob(job payload.BuildJob) error { + ctx := context.Background() + // Create a new Docker client + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + return err + } + defer cli.Close() + + // Create the shared volume + err = createSharedVolume(ctx, cli, sharedVolumeName) + if err != nil { + log.WithError(err).Error("Failed to create shared volume") + return err + } + + // Clone the repository + err = cloneRepository(ctx, cli, job.BuildConfig.Repositories...) + if err != nil { + log.WithError(err).Error("Failed to clone repository") + return err + } + + // TODO enable deletion of shared volume + //time.Sleep(5 * time.Second) + //err = deleteSharedVolume(ctx, cli, sharedVolumeName) + //if err != nil { + // log.WithError(err).Error("Failed to delete shared volume") + // return err + //} + + return nil +} + +func cloneRepository(ctx context.Context, client *client.Client, repositories ...payload.Repository) error { + // Pull the image + _, err := client.ImagePull(ctx, cloneContainerImage, types.ImagePullOptions{}) + if err != nil { + return err + } + + // Use the index to modify the slice in place + for i := range repositories { + repositories[i].Path = "/shared" + repositories[i].Path + } + commandStr := utils.BuildCloneCommands(repositories...) + log.Debug(commandStr) + + // Create the container + resp, err := client.ContainerCreate(ctx, &container.Config{ + Image: cloneContainerImage, + Entrypoint: []string{"/bin/sh", "-c"}, + Cmd: []string{commandStr}, + Volumes: map[string]struct{}{ + "/shared": {}, + }, + }, &defaultHostConfig, nil, nil, "") + if err != nil { + return err + } + + // Start the container + err = client.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}) + if err != nil { + return err + } + + log.Infof("Container %s started with ID: %s\n", cloneContainerImage, resp.ID) + return nil +} diff --git a/HadesScheduler/docker/volume.go b/HadesScheduler/docker/volume.go new file mode 100644 index 0000000..5b0f668 --- /dev/null +++ b/HadesScheduler/docker/volume.go @@ -0,0 +1,34 @@ +package docker + +import ( + "context" + "github.com/docker/docker/api/types/volume" + "github.com/docker/docker/client" + log "github.com/sirupsen/logrus" +) + +func createSharedVolume(ctx context.Context, client *client.Client, name string) error { + // Create the volume + _, err := client.VolumeCreate(ctx, volume.CreateOptions{ + Name: name, + }) + if err != nil { + log.WithError(err).Error("Failed to create shared volume") + return err + } + + log.Debugf("Volume %s created\n", name) + return nil +} + +func deleteSharedVolume(ctx context.Context, client *client.Client, name string) error { + // Delete the volume + err := client.VolumeRemove(ctx, name, true) + if err != nil { + log.WithError(err).Error("Failed to delete shared volume") + return err + } + + log.Debugf("Volume %s deleted\n", name) + return nil +} diff --git a/HadesScheduler/main.go b/HadesScheduler/main.go index b17fcf5..34d126d 100644 --- a/HadesScheduler/main.go +++ b/HadesScheduler/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/Mtze/HadesCI/hadesScheduler/docker" "os" "path/filepath" @@ -62,7 +63,7 @@ func main() { var forever chan struct{} - scheduler := DockerScheduler{} + scheduler := docker.DockerScheduler{} BuildQueue.Dequeue(scheduler.ScheduleJob) log.Printf(" [*] Waiting for messages. To exit press CTRL+C") diff --git a/shared/utils/git.go b/shared/utils/git.go new file mode 100644 index 0000000..70eadc7 --- /dev/null +++ b/shared/utils/git.go @@ -0,0 +1,22 @@ +package utils + +import ( + "fmt" + "github.com/Mtze/HadesCI/shared/payload" + "strings" +) + +func BuildCloneCommand(repo payload.Repository) string { + return fmt.Sprintf("git clone %s %s", repo.URL, repo.Path) +} + +func BuildCloneCommands(repos ...payload.Repository) string { + var builder strings.Builder + for i, repo := range repos { + if i > 0 { + builder.WriteString(" && ") + } + builder.WriteString(BuildCloneCommand(repo)) + } + return builder.String() +}