From cf18f90d82ffdda622fadb9d5eb964902ebd60fb Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Thu, 18 May 2023 12:07:34 -0700 Subject: [PATCH] feat: podman as a supported container engine (#809) * feat: podman as a supported container engine in konvoy image builder (#789) * feat: podman as a supported container engine in konvoy image builder * ci: run gcp e2e tests using podman * fix linting error * revert back github action job * ci: run gcp e2e tests using podman * removeme: test github actions flow * build using goreleaser build command * test podman * ci: run sample GHA aws e2e job using podman * run podman as root user * Faiq/kib podman (#801) * fix: move dir to /opt * fix: only mount /etc/grp and /etc/usr * fix: set user and group only in docker * fix: adds nolint * add security opt only for podman engine * ci: run podman kib e2e tests in github actions --------- Co-authored-by: shalin patel * remove setting container engine using env variable * add user id mapping only for docker runner * allow users to specify container engine * inline export KIB_CONTAINER_ENGINE Co-authored-by: Faiq --------- Co-authored-by: Faiq * chore: fix embedded image's tag --------- Co-authored-by: Faiq --- .github/workflows/podman-aws-e2e.yaml | 68 +++++++++ .goreleaser.yml | 1 + Dockerfile | 7 +- Makefile | 11 +- cmd/konvoy-image-wrapper/cmd/wrapper.go | 130 ++++++++++-------- cmd/konvoy-image-wrapper/image/common.go | 15 +- cmd/konvoy-image-wrapper/image/image.go | 23 ++-- .../image/image_not_embedded.go | 12 +- 8 files changed, 160 insertions(+), 107 deletions(-) create mode 100644 .github/workflows/podman-aws-e2e.yaml diff --git a/.github/workflows/podman-aws-e2e.yaml b/.github/workflows/podman-aws-e2e.yaml new file mode 100644 index 000000000..12c6c7ac2 --- /dev/null +++ b/.github/workflows/podman-aws-e2e.yaml @@ -0,0 +1,68 @@ +# Runs Azure tests when pull request opened, repopened or synchronized +name: Podman E2E Tests - Build AWS AMI +on: + workflow_dispatch: + pull_request: + types: [labeled, synchronize] + +permissions: + contents: read + id-token: write + +jobs: + rune2e: + runs-on: ubuntu-latest + continue-on-error: false + if: | + github.event_name == 'pull_request' && + ( + (github.event.action == 'labeled' && (github.event.label.name == 'runs-e2e-tests' || github.event.label.name == 'runs-aws-tests')) || + (github.event.action == 'synchronize' && (contains(github.event.pull_request.labels.*.name, 'runs-e2e-tests') || contains(github.event.pull_request.labels.*.name, 'runs-aws-tests'))) + ) + steps: + - name: Checkout konvoy-image-builder repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ github.ref }} + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version-file: 'go.mod' + cache: true + + - name: Login to dockerhub Registry + uses: docker/login-action@v2 + with: + username: ${{ secrets.NEXUS_USERNAME }} + password: ${{ secrets.NEXUS_PASSWORD }} + + - name: Login to D2iQ's Mirror Registry + uses: docker/login-action@v2 + with: + registry: ${{ secrets.D2IQ_DOCKER_MIRROR_REGISTRY}} + username: ${{ secrets.NEXUS_USERNAME }} + password: ${{ secrets.NEXUS_PASSWORD }} + + - name: Setup buildkit + uses: docker/setup-buildx-action@v2 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/konvoy-image-builder + aws-region: us-west-2 + + - name: Download GoReleaser + run: go install github.com/goreleaser/goreleaser@v1.15.2 + + - name: Build snapshot + run: make build.snapshot + + - name: Run E2E test for AWS centos 7.9 using podman + run: |- + KIB_CONTAINER_ENGINE=podman dist/konvoy-image-wrapper_linux_amd64_v1/konvoy-image build aws images/ami/centos-79.yaml --dry-run + env: + GITHUB_TOKEN: ${{ secrets.MESOSPHERECI_USER_TOKEN }} + KIB_CONTAINER_ENGINE: podman diff --git a/.goreleaser.yml b/.goreleaser.yml index 4ddd5b447..1f7c6b281 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -67,6 +67,7 @@ builds: env: - DOCKER_DEVKIT_DEFAULT_ARGS="--rm" - BUILDARCH={{.Arch }} + - REPO_REV=v{{trimprefix .Version "v"}} archives: - id: konvoy-image-bundle diff --git a/Dockerfile b/Dockerfile index e4ebedcd5..8caf527fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,9 +38,8 @@ COPY --from=devkit /usr/local/bin/govc /usr/local/bin/ COPY --from=devkit /root/.config/packer/plugins/ ${PACKER_PLUGIN_PATH} COPY --from=devkit /usr/share/ansible/collections/ansible_collections/ /usr/share/ansible/collections/ansible_collections/ COPY bin/konvoy-image-${BUILDARCH} /usr/local/bin/konvoy-image -COPY images /root/images -COPY ansible /root/ansible -COPY packer /root/packer +COPY images /opt/images +COPY ansible /opt/ansible -WORKDIR /root +WORKDIR /opt ENTRYPOINT ["/usr/local/bin/konvoy-image"] diff --git a/Makefile b/Makefile index e15af914d..e399798cb 100644 --- a/Makefile +++ b/Makefile @@ -497,19 +497,12 @@ mod-tidy: ## go mod tidy $(call print-target) go mod tidy +GORELEASER_SINGLE_TARGET ?= true .PHONY: build.snapshot build.snapshot: dist/konvoy-image_linux_amd64/konvoy-image build.snapshot: $(call print-target) - # NOTE(jkoelker) shenanigans to get around goreleaser and - # `make release-bundle` being able to share the same - # `Dockerfile`. Unfortunatly goreleaser forbids - # copying the dist folder into the temporary folder - # that it uses as its docker build context ;(. - # NOTE (faiq): does anyone use this target? - mkdir -p bin - cp dist/konvoy-image_linux_$(BUILDARCH)/konvoy-image bin/konvoy-image - goreleaser --parallelism=1 --skip-publish --snapshot --clean + goreleaser build --parallelism=1 --snapshot --single-target=$(GORELEASER_SINGLE_TARGET) --clean .PHONY: diff diff: ## git diff diff --git a/cmd/konvoy-image-wrapper/cmd/wrapper.go b/cmd/konvoy-image-wrapper/cmd/wrapper.go index 2596c8f2c..57ce088ae 100644 --- a/cmd/konvoy-image-wrapper/cmd/wrapper.go +++ b/cmd/konvoy-image-wrapper/cmd/wrapper.go @@ -1,6 +1,7 @@ package cmd import ( + "bytes" "errors" "fmt" "log" @@ -60,8 +61,11 @@ const ( envHTTPProxy = "HTTP_PROXY" envNoProxy = "NO_PROXY" - containerWorkingDir = "/tmp/kib" - windows = "windows" + containerWorkingDir = "/tmp/kib" + windows = "windows" + containerEngineEnv = "KIB_CONTAINER_ENGINE" + containerEngineDocker = "docker" + containerEnginePodman = "podman" ) var ErrEnv = errors.New("manifest not support") @@ -71,8 +75,6 @@ func EnvError(o string) error { } type Runner struct { - version string - usr *user.User usrGroup *user.Group homeDir string @@ -81,6 +83,7 @@ type Runner struct { workingDir string env map[string]string volumes []volume + containerEngine string } type volume struct { @@ -95,11 +98,23 @@ func NewRunner() *Runner { if err != nil { log.Fatalf("error getting user home directory: %v", err) } + var containerEngine string + switch p := os.Getenv(containerEngineEnv); p { + case "": + containerEngine = detectContainerEngine() + case "podman": + containerEngine = containerEnginePodman + case "docker": + containerEngine = containerEngineDocker + default: + log.Printf("ignoring unknown value %q for %s", p, containerEngineEnv) + } return &Runner{ homeDir: home, supplementaryGroupIDs: []int{}, env: map[string]string{}, + containerEngine: containerEngine, } } @@ -289,16 +304,20 @@ func (r *Runner) setupSSHAgent() { } func (r *Runner) dockerRun(args []string) error { + //nolint:gosec // we validate this cmd := exec.Command( - "docker", "run", + r.containerEngine, "run", "--interactive", "--tty=false", "--rm", "--net=host", "-w", containerWorkingDir, ) + if r.containerEngine == containerEnginePodman { + cmd.Args = append(cmd.Args, "--userns=keep-id", "--security-opt", "label=disable") + } - if runtime.GOOS != windows { + if runtime.GOOS != windows && r.containerEngine == containerEngineDocker { cmd.Args = append(cmd.Args, "-u", r.usr.Uid+":"+r.usr.Gid) r.addBindVolume(r.tempDir, r.homeDir) } @@ -341,44 +360,6 @@ func (r *Runner) dockerRun(args []string) error { return nil } -// checkDockerVersion checks whether the docker version is greater than then -// minimum required. -func (r *Runner) checkDockerVersion() error { - _, err := exec.Command("docker", "version", "-f", "{{.Client.Version}}").Output() - if err != nil { - if ee, ok := err.(*exec.ExitError); ok { - os.Stderr.Write(ee.Stderr) - } - return err - } - return nil -} - -// checkDockerRunning checks whether the docker daemon is running. -func (r *Runner) checkDockerRunning() error { - out, err := exec.Command("docker", "info", "-f", "{{json .ServerVersion}}").Output() - if err != nil || len(out) == 0 { - if ee, ok := err.(*exec.ExitError); ok { - os.Stderr.Write(ee.Stderr) - } - return err - } - return nil -} - -func (r *Runner) checkRequirements() error { - err := r.checkDockerVersion() - if err != nil { - return err - } - - err = r.checkDockerRunning() - if err != nil { - return err - } - return nil -} - func (r *Runner) setUserMapping() error { if runtime.GOOS == windows { return nil @@ -474,13 +455,6 @@ func (r *Runner) maskSSHKnownHosts() error { func (r *Runner) Run(args []string) error { // Get the Konvoy image version for marker file var err error - r.version = "" - - err = r.checkRequirements() - if err != nil { - return err - } - // Lookup for current user and its group ID. // This also look for supplementary group IDs to set. err = r.setUserAndGroups() @@ -506,13 +480,15 @@ func (r *Runner) Run(args []string) error { // Setup the user and group mappings in the container so that uid and // gid on the host can be properly resolved in the container too. - err = r.setUserMapping() - if err != nil { - return fmt.Errorf("failed to set user mapping %w", err) - } - err = r.setGroupMapping() - if err != nil { - return fmt.Errorf("failed to set group mapping %w", err) + if r.containerEngine == containerEngineDocker { + err = r.setUserMapping() + if err != nil { + return fmt.Errorf("failed to set user mapping %w", err) + } + err = r.setGroupMapping() + if err != nil { + return fmt.Errorf("failed to set group mapping %w", err) + } } err = r.maskSSHConfig() @@ -541,7 +517,7 @@ func (r *Runner) Run(args []string) error { r.setHTTPProxyEnv() - err = image.LoadImage() + err = image.LoadImage(r.containerEngine) if err != nil { return fmt.Errorf("failed to load image %w", err) } @@ -551,3 +527,39 @@ func (r *Runner) Run(args []string) error { // Run the command in the konvoy docker container. return r.dockerRun(args) } + +// detectContainerEngine determines which container engine should be used. +// if both docker and podman installed then docker engine takes precedence +// if none of them are detected then fallback to existing behavior of using +// docker as a container engine. +func detectContainerEngine() string { + if isDockerAvailable() { + return containerEngineDocker + } else if isPodmanAvailable() { + return containerEnginePodman + } + // fall back to current behavior for backward compatibility + return containerEngineDocker +} + +func isPodmanAvailable() bool { + cmd := exec.Command("podman", "-v") + var buff bytes.Buffer + cmd.Stdout = &buff + err := cmd.Run() + if err != nil { + return false + } + return strings.HasPrefix(buff.String(), "podman version") +} + +func isDockerAvailable() bool { + cmd := exec.Command("docker", "-v") + var buff bytes.Buffer + cmd.Stdout = &buff + err := cmd.Run() + if err != nil { + return false + } + return strings.HasPrefix(buff.String(), "Docker version") +} diff --git a/cmd/konvoy-image-wrapper/image/common.go b/cmd/konvoy-image-wrapper/image/common.go index 24ad04655..1a7fcab7a 100644 --- a/cmd/konvoy-image-wrapper/image/common.go +++ b/cmd/konvoy-image-wrapper/image/common.go @@ -3,26 +3,19 @@ package image import ( "bytes" "fmt" - "os" "os/exec" "github.com/mesosphere/konvoy-image-builder/pkg/version" ) -func imageLoaded(image string) (bool, error) { - cmd := exec.Command("docker", "image", "inspect", image) +func imageLoaded(containerEngine, image string) bool { + cmd := exec.Command(containerEngine, "image", "inspect", image) stdErrBuf := bytes.NewBuffer(make([]byte, 0)) cmd.Stderr = stdErrBuf - if err := cmd.Run(); err != nil { - stdErr := stdErrBuf.Bytes() - if bytes.Index(stdErr, []byte("No such image")) > 0 { - return false, nil - } - fmt.Fprintf(os.Stderr, "docker error: %s", stdErr) - return false, err + return false } - return true, nil + return true } func Tag() string { diff --git a/cmd/konvoy-image-wrapper/image/image.go b/cmd/konvoy-image-wrapper/image/image.go index b75cde72a..a4306cc52 100644 --- a/cmd/konvoy-image-wrapper/image/image.go +++ b/cmd/konvoy-image-wrapper/image/image.go @@ -9,8 +9,6 @@ import ( _ "embed" "os" "os/exec" - - "github.com/pkg/errors" ) const Repository = "mesosphere/konvoy-image-builder" @@ -18,21 +16,16 @@ const Repository = "mesosphere/konvoy-image-builder" //go:embed konvoy-image-builder.tar.gz var konvoyImageTar []byte // memory is cheap, right? -func LoadImage() error { +func LoadImage(containerEngine string) error { image := Tag() - present, err := imageLoaded(image) - if err != nil { - return errors.Wrap(err, "error querying docker for images") + if imageLoaded(containerEngine, image) { + return nil } + cmd := exec.Command(containerEngine, "image", "load") + cmd.Stdin = bytes.NewReader(konvoyImageTar) - if !present { - cmd := exec.Command("docker", "image", "load") - cmd.Stdin = bytes.NewReader(konvoyImageTar) - - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr - return cmd.Run() - } - return nil + return cmd.Run() } diff --git a/cmd/konvoy-image-wrapper/image/image_not_embedded.go b/cmd/konvoy-image-wrapper/image/image_not_embedded.go index 4af6194c3..b78bc26e6 100644 --- a/cmd/konvoy-image-wrapper/image/image_not_embedded.go +++ b/cmd/konvoy-image-wrapper/image/image_not_embedded.go @@ -5,22 +5,16 @@ package image import ( "os/exec" - - "github.com/pkg/errors" ) const Repository = "mesosphere/konvoy-image-builder" -func LoadImage() error { +func LoadImage(containerEngine string) error { image := Tag() - found, err := imageLoaded(image) - if err != nil { - return errors.Wrap(err, "error querying docker for images") - } - if found { + if imageLoaded(containerEngine, image) { return nil } //nolint:gosec // this is necessary - cmd := exec.Command("docker", "pull", Tag()) + cmd := exec.Command(containerEngine, "pull", Tag()) return cmd.Run() }