From bd58cdf98064fa6719ba7808974552ee1f96f495 Mon Sep 17 00:00:00 2001 From: "Philip K. Warren" Date: Thu, 30 Nov 2023 09:06:51 -0600 Subject: [PATCH] Update fetcher to update distroless images (#931) --- internal/cmd/fetcher/main.go | 113 +++--------- internal/docker/baseimage.go | 170 ++++++++++++++++++ internal/docker/baseimage_test.go | 69 +++++++ .../Dockerfile.distroless-java11-debian | 3 + .../Dockerfile.distroless-java17-debian | 3 + .../duplicateversions/Dockerfile.debian11 | 3 + .../duplicateversions/Dockerfile.debian12 | 3 + 7 files changed, 271 insertions(+), 93 deletions(-) create mode 100644 internal/docker/baseimage.go create mode 100644 internal/docker/baseimage_test.go create mode 100644 internal/docker/testdata/duplicatedistroless/Dockerfile.distroless-java11-debian create mode 100644 internal/docker/testdata/duplicatedistroless/Dockerfile.distroless-java17-debian create mode 100644 internal/docker/testdata/duplicateversions/Dockerfile.debian11 create mode 100644 internal/docker/testdata/duplicateversions/Dockerfile.debian12 diff --git a/internal/cmd/fetcher/main.go b/internal/cmd/fetcher/main.go index 9f8832416..d411464ae 100644 --- a/internal/cmd/fetcher/main.go +++ b/internal/cmd/fetcher/main.go @@ -19,6 +19,7 @@ import ( "go.uber.org/multierr" "golang.org/x/mod/semver" + "github.com/bufbuild/plugins/internal/docker" "github.com/bufbuild/plugins/internal/fetchclient" "github.com/bufbuild/plugins/internal/source" ) @@ -144,7 +145,11 @@ func run(ctx context.Context, root string) ([]createdPlugin, error) { defer func() { log.Printf("finished running in: %.2fs\n", time.Since(now).Seconds()) }() - latestBaseImageVersions, err := getLatestBaseImageVersions(root) + baseImageDir, err := docker.FindBaseImageDir(root) + if err != nil { + return nil, err + } + latestBaseImageVersions, err := docker.LoadLatestBaseImages(baseImageDir) if err != nil { return nil, err } @@ -200,89 +205,6 @@ func run(ctx context.Context, root string) ([]createdPlugin, error) { return created, nil } -func getLatestBaseImageVersions(basedir string) (_ map[string]string, retErr error) { - // Walk up from plugins dir to find .github dir - rootDir := basedir - var githubDir string - for { - githubDir = filepath.Join(rootDir, ".github") - if st, err := os.Stat(githubDir); err == nil && st.IsDir() { - break - } - newRootDir := filepath.Dir(filepath.Dir(githubDir)) - if newRootDir == rootDir { - return nil, fmt.Errorf("failed to find .github directory from %s", basedir) - } - rootDir = newRootDir - } - dockerDir := filepath.Join(githubDir, "docker") - d, err := os.Open(dockerDir) - if err != nil { - return nil, err - } - defer func() { - retErr = errors.Join(retErr, d.Close()) - }() - entries, err := d.ReadDir(-1) - if err != nil { - return nil, err - } - latestVersions := make(map[string]string, len(entries)) - for _, entry := range entries { - if !strings.HasPrefix(entry.Name(), "Dockerfile.") { - continue - } - imageName, version, err := parseDockerfileBaseImageNameVersion(filepath.Join(dockerDir, entry.Name())) - if err != nil { - return nil, err - } - latestVersions[imageName] = version - } - return latestVersions, nil -} - -func parseDockerfileBaseImageNameVersion(dockerfile string) (_ string, _ string, retErr error) { - f, err := os.Open(dockerfile) - if err != nil { - return "", "", nil - } - defer func() { - retErr = errors.Join(retErr, f.Close()) - }() - s := bufio.NewScanner(f) - for s.Scan() { - line := strings.TrimSpace(s.Text()) - fields := strings.Fields(line) - if len(fields) < 2 { - continue - } - if !strings.EqualFold(fields[0], "from") { - continue - } - var image string - for i := 1; i < len(fields); i++ { - if strings.HasPrefix(fields[i], "--") { - // Ignore --platform and other args - continue - } - image = fields[i] - break - } - if image == "" { - return "", "", fmt.Errorf("missing image in FROM: %q", line) - } - imageName, version, found := strings.Cut(image, ":") - if !found { - return "", "", fmt.Errorf("invalid FROM line: %q", line) - } - return imageName, version, nil - } - if err := s.Err(); err != nil { - return "", "", err - } - return "", "", fmt.Errorf("failed to detect base image in %s", dockerfile) -} - // copyDirectory copies all files from the source directory to the target, // creating the target directory if it does not exist. // If the source directory contains subdirectories this function returns an error. @@ -291,7 +213,7 @@ func copyDirectory( target string, prevVersion string, newVersion string, - latestBaseImageVersions map[string]string, + latestBaseImages *docker.BaseImages, ) (retErr error) { entries, err := os.ReadDir(source) if err != nil { @@ -314,7 +236,7 @@ func copyDirectory( filepath.Join(target, file.Name()), prevVersion, newVersion, - latestBaseImageVersions, + latestBaseImages, ); err != nil { return err } @@ -326,7 +248,7 @@ func createPluginDir( dir string, previousVersion string, newVersion string, - latestBaseImageVersions map[string]string, + latestBaseImages *docker.BaseImages, ) (retErr error) { if err := os.Mkdir(filepath.Join(dir, newVersion), 0755); err != nil { return err @@ -341,7 +263,7 @@ func createPluginDir( filepath.Join(dir, newVersion), previousVersion, newVersion, - latestBaseImageVersions, + latestBaseImages, ) } @@ -350,7 +272,7 @@ func copyFile( dest string, prevVersion string, newVersion string, - latestBaseImageVersions map[string]string, + latestBaseImages *docker.BaseImages, ) (retErr error) { srcFile, err := os.Open(src) if err != nil { @@ -380,13 +302,17 @@ func copyFile( isDockerfile := strings.HasPrefix(filename, "Dockerfile") prevVersion = strings.TrimPrefix(prevVersion, "v") newVersion = strings.TrimPrefix(newVersion, "v") + latestBazelVersion := latestBaseImages.ImageVersion(bazelImageName) + if latestBazelVersion == "" { + return fmt.Errorf("failed to find latest version for bazel image %q", bazelImageName) + } s := bufio.NewScanner(srcFile) for s.Scan() { line := strings.ReplaceAll(s.Text(), prevVersion, newVersion) line = bazelDownloadRegexp.ReplaceAllString( line, fmt.Sprintf(`bazelbuild/bazel/releases/download/%[1]s/bazel-%[1]s-linux`, - latestBaseImageVersions[bazelImageName], + latestBazelVersion, ), ) if isDockerfile && len(line) > 5 && strings.EqualFold(line[0:5], "from ") { @@ -401,9 +327,10 @@ func copyFile( break } } - if name, _, found := strings.Cut(image, ":"); found { - if newVersion := latestBaseImageVersions[name]; newVersion != "" { - fields[imageIndex] = name + ":" + newVersion + name, _, _ := strings.Cut(image, ":") + if name != "" { + if newImageNameAndVersion := latestBaseImages.ImageNameAndVersion(name); newImageNameAndVersion != "" { + fields[imageIndex] = newImageNameAndVersion line = strings.Join(fields, " ") } } diff --git a/internal/docker/baseimage.go b/internal/docker/baseimage.go new file mode 100644 index 000000000..3816ef435 --- /dev/null +++ b/internal/docker/baseimage.go @@ -0,0 +1,170 @@ +package docker + +import ( + "bufio" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "unicode" +) + +var ( + distrolessNamePrefix = "gcr.io/distroless/" +) + +// BaseImages contains state about the latest versions of base images in the .github/docker directory. +// These are automatically kept up to date by dependabot. +type BaseImages struct { + latestVersions map[string]string + latestDistrolessImageNames map[string]string +} + +// ImageNameAndVersion returns the latest image name and version (if tracked in the .github/docker directory). +// For example, passing "debian" will return "debian:bookworm-yyyyMMdd" (where "yyyyMMdd" is the latest image date). +// If the image is not tracked in the .github/docker directory, returns an empty string. +// This is used to automate updating Dockerfile base image versions when fetching new versions of plugins. +func (b *BaseImages) ImageNameAndVersion(imageName string) string { + latestImageName := imageName + if nameWithoutVersions := distrolessImageNameWithoutVersions(imageName); nameWithoutVersions != "" { + latestImageName = b.latestDistrolessImageNames[nameWithoutVersions] + } + latestVersion, ok := b.latestVersions[latestImageName] + if !ok { + return "" + } + return latestImageName + ":" + latestVersion +} + +// ImageVersion returns the latest version for the image name (if tracked in the .github/docker directory). +// For example, passing "debian" will return "bookworm-yyyyMMdd" (where "yyyyMMdd" is the latest image date). +// If the image is not tracked in the .github/docker directory, returns an empty string. +func (b *BaseImages) ImageVersion(imageName string) string { + latestImageName := imageName + if nameWithoutVersions := distrolessImageNameWithoutVersions(imageName); nameWithoutVersions != "" { + latestImageName = b.latestDistrolessImageNames[nameWithoutVersions] + } + return b.latestVersions[latestImageName] +} + +// FindBaseImageDir looks for the .github/docker folder starting from basedir. +// It continues to search through parent directories till found (or at the root). +func FindBaseImageDir(basedir string) (string, error) { + // Walk up from plugins dir to find .github dir + rootDir, err := filepath.Abs(basedir) + if err != nil { + return "", err + } + var dockerDir string + for { + dockerDir = filepath.Join(rootDir, ".github", "docker") + if st, err := os.Stat(dockerDir); err == nil && st.IsDir() { + break + } + newRootDir := filepath.Dir(rootDir) + if newRootDir == rootDir { + return "", fmt.Errorf("failed to find .github directory from %s", basedir) + } + rootDir = newRootDir + } + return dockerDir, nil +} + +// LoadLatestBaseImages returns the latest base image information from images found in the .github/docker directory. +func LoadLatestBaseImages(baseImageDir string) (_ *BaseImages, retErr error) { + d, err := os.Open(baseImageDir) + if err != nil { + return nil, err + } + defer func() { + retErr = errors.Join(retErr, d.Close()) + }() + entries, err := d.ReadDir(-1) + if err != nil { + return nil, err + } + latestVersions := make(map[string]string, len(entries)) + latestDistrolessImages := make(map[string]string) + for _, entry := range entries { + if entry.IsDir() || !strings.HasPrefix(entry.Name(), "Dockerfile") { + continue + } + imageName, version, err := parseDockerfileBaseImageNameVersion(filepath.Join(baseImageDir, entry.Name())) + if err != nil { + return nil, err + } + if _, ok := latestVersions[imageName]; ok { + return nil, fmt.Errorf("found duplicate dockerfiles for image %q", imageName) + } + latestVersions[imageName] = version + if imageNameWithoutVersions := distrolessImageNameWithoutVersions(imageName); imageNameWithoutVersions != "" { + if _, ok := latestDistrolessImages[imageNameWithoutVersions]; ok { + return nil, fmt.Errorf("found duplicate distroless dockerfiles for image %q", imageNameWithoutVersions) + } + latestDistrolessImages[imageNameWithoutVersions] = imageName + } + } + return &BaseImages{ + latestVersions: latestVersions, + latestDistrolessImageNames: latestDistrolessImages, + }, nil +} + +func parseDockerfileBaseImageNameVersion(dockerfile string) (_ string, _ string, retErr error) { + f, err := os.Open(dockerfile) + if err != nil { + return "", "", nil + } + defer func() { + retErr = errors.Join(retErr, f.Close()) + }() + s := bufio.NewScanner(f) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + if !strings.EqualFold(fields[0], "from") { + continue + } + var image string + for i := 1; i < len(fields); i++ { + if strings.HasPrefix(fields[i], "--") { + // Ignore --platform and other args + continue + } + image = fields[i] + break + } + if image == "" { + return "", "", fmt.Errorf("missing image in FROM: %q", line) + } + imageName, version, found := strings.Cut(image, ":") + if !found { + return "", "", fmt.Errorf("invalid FROM line: %q", line) + } + return imageName, version, nil + } + if err := s.Err(); err != nil { + return "", "", err + } + return "", "", fmt.Errorf("failed to detect base image in %s", dockerfile) +} + +// distrolessImageNameWithoutVersions returns a distroless image name without version numbers. +// If the passed in image name is a distroless image and contains versions, it returns the name without versions. +// Otherwise, it returns an empty string. +func distrolessImageNameWithoutVersions(nameWithVersions string) string { + if !strings.HasPrefix(nameWithVersions, distrolessNamePrefix) || !strings.ContainsFunc(nameWithVersions, unicode.IsDigit) { + return "" + } + var sb strings.Builder + for _, r := range nameWithVersions { + if !unicode.IsDigit(r) { + sb.WriteRune(r) + } + } + return sb.String() +} diff --git a/internal/docker/baseimage_test.go b/internal/docker/baseimage_test.go new file mode 100644 index 000000000..08bb39abd --- /dev/null +++ b/internal/docker/baseimage_test.go @@ -0,0 +1,69 @@ +package docker + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFindBaseImageDir(t *testing.T) { + t.Parallel() + verifyDir := func(basedir, expected string) { + baseImageDir, err := FindBaseImageDir(basedir) + require.NoError(t, err) + expectedAbs, err := filepath.Abs(expected) + require.NoError(t, err) + assert.Equal(t, expectedAbs, baseImageDir) + } + expected := "../../.github/docker" + verifyDir(".", expected) + verifyDir("..", expected) + verifyDir("../..", expected) + verifyDir("../../.github", expected) + verifyDir("../../.github/docker", expected) +} + +func TestBaseImages(t *testing.T) { + t.Parallel() + baseImageDir, err := FindBaseImageDir(".") + require.NoError(t, err) + baseImages, err := LoadLatestBaseImages(baseImageDir) + require.NoError(t, err) + assert.NotEmpty(t, baseImages) + assert.NotEmpty(t, baseImages.ImageNameAndVersion("debian")) + assert.Empty(t, baseImages.ImageNameAndVersion("untracked")) + // Test just returning version + assert.NotEmpty(t, baseImages.ImageVersion("gcr.io/bazel-public/bazel")) + assert.NotContains(t, baseImages.ImageVersion("gcr.io/bazel-public/bazel"), "bazel:") + // Test distroless image upgrades + javaImage := baseImages.ImageNameAndVersion("gcr.io/distroless/java11-debian11") + assert.NotEmpty(t, javaImage) + assert.NotContains(t, javaImage, "java11") // Should be replaced with a later java image + assert.NotContains(t, javaImage, "debian11") // Should be replaced with a later debian image +} + +func TestBaseImagesNoDuplicateVersions(t *testing.T) { + t.Parallel() + _, err := LoadLatestBaseImages("testdata/duplicateversions") + require.ErrorContains(t, err, "found duplicate dockerfiles") +} + +func TestBaseImagesNoDuplicateDistroless(t *testing.T) { + t.Parallel() + _, err := LoadLatestBaseImages("testdata/duplicatedistroless") + require.ErrorContains(t, err, "found duplicate distroless dockerfiles") +} + +func TestDistrolessImageNameWithoutVersions(t *testing.T) { + t.Parallel() + verifyResult := func(imageName, expectedImage string) { + imageWithoutVersions := distrolessImageNameWithoutVersions(imageName) + assert.Equal(t, expectedImage, imageWithoutVersions) + } + verifyResult("gcr.io/distroless/base-debian11", "gcr.io/distroless/base-debian") + verifyResult("gcr.io/distroless/java17-debian11", "gcr.io/distroless/java-debian") + verifyResult("gcr.io/distroless/static", "") + verifyResult("debian", "") +} diff --git a/internal/docker/testdata/duplicatedistroless/Dockerfile.distroless-java11-debian b/internal/docker/testdata/duplicatedistroless/Dockerfile.distroless-java11-debian new file mode 100644 index 000000000..1be27446c --- /dev/null +++ b/internal/docker/testdata/duplicatedistroless/Dockerfile.distroless-java11-debian @@ -0,0 +1,3 @@ +FROM gcr.io/distroless/java11-debian11:latest@sha256:8f80873debfafc0b77dd22d03e6d34d0a716c52404ca4ca19f76f2de11000c55 + +CMD echo this is a dummy file used to automate dependency upgrades for plugins diff --git a/internal/docker/testdata/duplicatedistroless/Dockerfile.distroless-java17-debian b/internal/docker/testdata/duplicatedistroless/Dockerfile.distroless-java17-debian new file mode 100644 index 000000000..8e67609e1 --- /dev/null +++ b/internal/docker/testdata/duplicatedistroless/Dockerfile.distroless-java17-debian @@ -0,0 +1,3 @@ +FROM gcr.io/distroless/java17-debian12:latest@sha256:8f80873debfafc0b77dd22d03e6d34d0a716c52404ca4ca19f76f2de11000c55 + +CMD echo this is a dummy file used to automate dependency upgrades for plugins diff --git a/internal/docker/testdata/duplicateversions/Dockerfile.debian11 b/internal/docker/testdata/duplicateversions/Dockerfile.debian11 new file mode 100644 index 000000000..790b6f642 --- /dev/null +++ b/internal/docker/testdata/duplicateversions/Dockerfile.debian11 @@ -0,0 +1,3 @@ +FROM debian:bullseye-20231120 + +CMD echo this is a dummy file used to automate dependency upgrades for plugins diff --git a/internal/docker/testdata/duplicateversions/Dockerfile.debian12 b/internal/docker/testdata/duplicateversions/Dockerfile.debian12 new file mode 100644 index 000000000..2ba4d75bf --- /dev/null +++ b/internal/docker/testdata/duplicateversions/Dockerfile.debian12 @@ -0,0 +1,3 @@ +FROM debian:bookworm-20231120 + +CMD echo this is a dummy file used to automate dependency upgrades for plugins