diff --git a/go.mod b/go.mod index 523a3a3f..1ae46363 100644 --- a/go.mod +++ b/go.mod @@ -119,6 +119,7 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/theupdateframework/notary v0.7.0 // indirect github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c // indirect diff --git a/go.sum b/go.sum index 5f3b32bd..45bd82d4 100644 --- a/go.sum +++ b/go.sum @@ -422,6 +422,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/mocks/mock_gwclient.go b/mocks/mock_gwclient.go new file mode 100644 index 00000000..3fab007b --- /dev/null +++ b/mocks/mock_gwclient.go @@ -0,0 +1,141 @@ +package mocks + +import ( + "context" + "fmt" + + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/client/llb/sourceresolver" + gwclient "github.com/moby/buildkit/frontend/gateway/client" + "github.com/moby/buildkit/solver/pb" + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/mock" +) + +// Mock for gwclient.Client. +type MockGWClient struct { + mock.Mock +} + +func (m *MockGWClient) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt sourceresolver.Opt) (*sourceresolver.MetaResponse, error) { + args := m.Called(ctx, op, opt) + + metaResponse, ok := args.Get(0).(*sourceresolver.MetaResponse) + if !ok { + return nil, fmt.Errorf("type assertion to *sourceresolver.MetaResponse failed") + } + + return metaResponse, args.Error(1) +} + +//nolint:gocritic +func (m *MockGWClient) Solve(ctx context.Context, req gwclient.SolveRequest) (*gwclient.Result, error) { + args := m.Called(ctx, req) + + result, ok := args.Get(0).(*gwclient.Result) + if !ok { + return nil, fmt.Errorf("type assertion to *gwclient.Result failed") + } + + return result, args.Error(1) +} + +func (m *MockGWClient) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) { + args := m.Called(ctx, ref, opt) + + digestResult, ok1 := args.Get(1).(digest.Digest) + if !ok1 { + return "", digest.Digest(""), nil, fmt.Errorf("type assertion to digest.Digest failed") + } + + byteResult, ok2 := args.Get(2).([]byte) + if !ok2 { + return "", digest.Digest(""), nil, fmt.Errorf("type assertion to []byte failed") + } + + return args.String(0), digestResult, byteResult, args.Error(3) +} + +func (m *MockGWClient) BuildOpts() gwclient.BuildOpts { + args := m.Called() + + buildOpts, ok := args.Get(0).(gwclient.BuildOpts) + if !ok { + panic("type assertion to gwclient.BuildOpts failed") + } + + return buildOpts +} + +func (m *MockGWClient) Inputs(ctx context.Context) (map[string]llb.State, error) { + args := m.Called(ctx) + + stateMap, ok := args.Get(0).(map[string]llb.State) + if !ok { + return nil, fmt.Errorf("type assertion to map[string]llb.State failed") + } + + return stateMap, args.Error(1) +} + +//nolint:gocritic +func (m *MockGWClient) NewContainer(ctx context.Context, req gwclient.NewContainerRequest) (gwclient.Container, error) { + args := m.Called(ctx, req) + + container, ok := args.Get(0).(gwclient.Container) + if !ok { + return nil, fmt.Errorf("type assertion to gwclient.Container failed") + } + + return container, args.Error(1) +} + +//nolint:gocritic +func (m *MockGWClient) Warn(ctx context.Context, dgst digest.Digest, msg string, opts gwclient.WarnOpts) error { + args := m.Called(ctx, dgst, msg, opts) + + warnErr, ok := args.Get(0).(error) + if !ok { + return fmt.Errorf("type assertion to error failed") + } + + return warnErr +} + +// MockReference is a mock of the Reference interface. +type MockReference struct { + mock.Mock +} + +func (m *MockReference) ReadFile(ctx context.Context, req gwclient.ReadRequest) ([]byte, error) { + args := m.Called(ctx, req) + + byteResult, ok := args.Get(0).([]byte) + if !ok { + return nil, fmt.Errorf("type assertion to []byte failed") + } + + return byteResult, args.Error(1) +} + +func (m *MockReference) ToState() (llb.State, error) { + args := m.Called() + + state, ok := args.Get(0).(llb.State) + if !ok { + return state, fmt.Errorf("type assertion to llb.State failed") + } + + return state, args.Error(1) +} + +func (m *MockReference) Evaluate(ctx context.Context) error { + args := m.Called(ctx) + + evalErr, ok := args.Get(0).(error) + if !ok { + return fmt.Errorf("type assertion to error failed") + } + + return evalErr +} diff --git a/pkg/buildkit/buildkit.go b/pkg/buildkit/buildkit.go index 721e3c77..e12e2046 100644 --- a/pkg/buildkit/buildkit.go +++ b/pkg/buildkit/buildkit.go @@ -3,6 +3,8 @@ package buildkit import ( "bytes" "context" + "encoding/json" + "fmt" "github.com/containerd/platforms" "github.com/moby/buildkit/client/llb" @@ -12,11 +14,13 @@ import ( ) type Config struct { - ImageName string - Client gwclient.Client - ConfigData []byte - Platform *ispec.Platform - ImageState llb.State + ImageName string + Client gwclient.Client + ConfigData []byte + PatchedConfigData []byte + Platform *ispec.Platform + ImageState llb.State + PatchedImageState llb.State } type Opts struct { @@ -26,14 +30,14 @@ type Opts struct { KeyPath string } -func InitializeBuildkitConfig(ctx context.Context, c gwclient.Client, image string) (*Config, error) { +func InitializeBuildkitConfig(ctx context.Context, c gwclient.Client, userImage string) (*Config, error) { // Initialize buildkit config for the target image config := Config{ - ImageName: image, + ImageName: userImage, } // Resolve and pull the config for the target image - _, _, configData, err := c.ResolveImageConfig(ctx, image, sourceresolver.Opt{ + _, _, configData, err := c.ResolveImageConfig(ctx, userImage, sourceresolver.Opt{ ImageOpt: &sourceresolver.ResolveImageOpt{ ResolveMode: llb.ResolveModePreferLocal.String(), }, @@ -42,11 +46,15 @@ func InitializeBuildkitConfig(ctx context.Context, c gwclient.Client, image stri return nil, err } - config.ConfigData = configData + var baseImage string + config.ConfigData, config.PatchedConfigData, baseImage, err = updateImageConfigData(ctx, c, configData, userImage) + if err != nil { + return nil, err + } // Load the target image state with the resolved image config in case environment variable settings // are necessary for running apps in the target image for updates - config.ImageState, err = llb.Image(image, + config.ImageState, err = llb.Image(baseImage, llb.ResolveModePreferLocal, llb.WithMetaResolver(c), ).WithImageConfig(config.ConfigData) @@ -54,11 +62,90 @@ func InitializeBuildkitConfig(ctx context.Context, c gwclient.Client, image stri return nil, err } + // Only set PatchedImageState if the user supplied a patched image + // An image is deemed to be a patched image if it contains one of two metadata values + // BaseImage or ispec.AnnotationBaseImageName + if config.PatchedConfigData != nil { + config.PatchedImageState, err = llb.Image(userImage, + llb.ResolveModePreferLocal, + llb.WithMetaResolver(c), + ).WithImageConfig(config.PatchedConfigData) + if err != nil { + return nil, err + } + } + config.Client = c return &config, nil } +func updateImageConfigData(ctx context.Context, c gwclient.Client, configData []byte, image string) ([]byte, []byte, string, error) { + baseImage, userImageConfig, err := setupLabels(image, configData) + if err != nil { + return nil, nil, "", err + } + + if baseImage == "" { + configData = userImageConfig + } else { + patchedImageConfig := userImageConfig + _, _, baseImageConfig, err := c.ResolveImageConfig(ctx, baseImage, sourceresolver.Opt{ + ImageOpt: &sourceresolver.ResolveImageOpt{ + ResolveMode: llb.ResolveModePreferLocal.String(), + }, + }) + if err != nil { + return nil, nil, "", err + } + + _, baseImageWithLabels, _ := setupLabels(baseImage, baseImageConfig) + configData = baseImageWithLabels + + return configData, patchedImageConfig, baseImage, nil + } + + return configData, nil, image, nil +} + +func setupLabels(image string, configData []byte) (string, []byte, error) { + imageConfig := make(map[string]interface{}) + err := json.Unmarshal(configData, &imageConfig) + if err != nil { + return "", nil, err + } + + configMap, ok := imageConfig["config"].(map[string]interface{}) + if !ok { + err := fmt.Errorf("type assertion to map[string]interface{} failed") + return "", nil, err + } + + var baseImage string + labels := configMap["labels"] + if labels == nil { + configMap["labels"] = make(map[string]interface{}) + } + labelsMap, ok := configMap["labels"].(map[string]interface{}) + if !ok { + err := fmt.Errorf("type assertion to map[string]interface{} failed") + return "", nil, err + } + if baseImageValue := labelsMap["BaseImage"]; baseImageValue != nil { + baseImage, ok = baseImageValue.(string) + if !ok { + err := fmt.Errorf("type assertion to string failed") + return "", nil, err + } + } else { + labelsMap["BaseImage"] = image + } + + imageWithLabels, _ := json.Marshal(imageConfig) + + return baseImage, imageWithLabels, nil +} + // Extracts the bytes of the file denoted by `path` from the state `st`. func ExtractFileFromState(ctx context.Context, c gwclient.Client, st *llb.State, path string) ([]byte, error) { // since platform is obtained from host, override it in the case of Darwin diff --git a/pkg/buildkit/buildkit_test.go b/pkg/buildkit/buildkit_test.go index ebd73f09..40b3f677 100644 --- a/pkg/buildkit/buildkit_test.go +++ b/pkg/buildkit/buildkit_test.go @@ -2,12 +2,20 @@ package buildkit import ( "context" + "encoding/json" "errors" "net" "path/filepath" + "reflect" "testing" "time" + "github.com/opencontainers/go-digest" + + "github.com/project-copacetic/copacetic/mocks" + + "github.com/stretchr/testify/mock" + controlapi "github.com/moby/buildkit/api/services/control" types "github.com/moby/buildkit/api/types" gateway "github.com/moby/buildkit/frontend/gateway/pb" @@ -270,3 +278,120 @@ func TestArrayFile(t *testing.T) { }) } } + +func TestSetupLabels(t *testing.T) { + tests := []struct { + testName string + configData []byte + expectBaseImg string + expectImage string + expectError bool + }{ + { + "No labels", + []byte(`{"config": {}}`), + "", + "test_image", + false, + }, + { + "Labels no base", + []byte(`{"config": {"labels": {}}}`), + "", + "test_image", + false, + }, + { + "Labels with base image", + []byte(`{"config": {"labels": {"BaseImage": "existing_base_image"}}}`), + "existing_base_image", + "existing_base_image", + false, + }, + { + "Invalid JSON", + []byte(`{"config": {"labels": {"BaseImage": "existing_base_image"}`), + "", + "", + true, + }, + } + for _, test := range tests { + t.Run(test.testName, func(t *testing.T) { + image := "test_image" + baseImage, updatedConfigData, _ := setupLabels(image, test.configData) + + if test.expectError { + assert.Equal(t, "", baseImage) + assert.Nil(t, updatedConfigData) + } else { + assert.Equal(t, test.expectBaseImg, baseImage) + + var updatedConfig map[string]interface{} + err := json.Unmarshal(updatedConfigData, &updatedConfig) + assert.NoError(t, err) + + labels, ok := updatedConfig["config"].(map[string]interface{})["labels"].(map[string]interface{}) + if !ok { + t.Errorf("type assertion to map[string]interface{} failed") + return + } + assert.Equal(t, test.expectImage, labels["BaseImage"]) + } + }) + } +} + +func TestUpdateImageConfigData(t *testing.T) { + ctx := context.Background() + + t.Run("No base image", func(t *testing.T) { + mockClient := &mocks.MockGWClient{} + configData := []byte(`{"config": {"labels": {"com.example.label": "value"}}}`) + expectedData := []byte(`{"config": {"labels": {"com.example.label": "value"}, {"BaseImage": "myimage:latest"}}}`) + image := "myimage:latest" + + resultConfig, resultPatched, resultImage, err := updateImageConfigData(ctx, mockClient, configData, image) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if reflect.DeepEqual(expectedData, configData) { + t.Errorf("Expected config data to be %s, got %s", configData, resultConfig) + } + + if resultPatched != nil { + t.Errorf("Expected patched config to be nil, got %s", resultPatched) + } + + if resultImage != image { + t.Errorf("Expected image to be %s, got %s", image, resultImage) + } + }) + + t.Run("With base image", func(t *testing.T) { + mockClient := &mocks.MockGWClient{} + mockClient.On("ResolveImageConfig", + mock.Anything, mock.AnythingOfType("string"), mock.Anything). + Return("imageConfigString", digest.Digest("digest"), []byte(`{"config": {"labels": {"BaseImage": "rockylinux:latest"}}}`), nil) + + configData := []byte(`{"config": {"labels": {"BaseImage": "rockylinux:latest"}}}`) + image := "rockylinux:latest" + + resultConfig, _, resultImage, err := updateImageConfigData(ctx, mockClient, configData, image) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + expectedConfig := []byte(`{"config":{"labels":{"BaseImage":"rockylinux:latest"}}}`) + if !reflect.DeepEqual(resultConfig, expectedConfig) { + t.Errorf("Expected config data to be %s, got %s", expectedConfig, resultConfig) + } + + if resultImage != "rockylinux:latest" { + t.Errorf("Expected image to be baseimage:latest, got %s", resultImage) + } + + mockClient.AssertExpectations(t) + }) +} diff --git a/pkg/pkgmgr/apk.go b/pkg/pkgmgr/apk.go index 08cf26bb..53d19fca 100644 --- a/pkg/pkgmgr/apk.go +++ b/pkg/pkgmgr/apk.go @@ -215,9 +215,29 @@ func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.U } } + // If the image has been patched before, diff the base image and patched image to retain previous patches + if am.config.PatchedConfigData != nil { + // Diff the base image and patched image to get previous patches + prevPatchDiff := llb.Diff(am.config.ImageState, am.config.PatchedImageState) + + // Diff the base image and new patches + newPatchDiff := llb.Diff(apkUpdated, apkInstalled) + + // Merging these two diffs will discard everything in the filesystem that hasn't changed + // Doing llb.Scratch ensures we can keep everything in the filesystem that has not changed + combinedPatch := llb.Merge([]llb.State{prevPatchDiff, newPatchDiff}) + squashedPatch := llb.Scratch().File(llb.Copy(combinedPatch, "/", "/")) + + // Merge previous and new patches into the base image + completePatchMerge := llb.Merge([]llb.State{am.config.ImageState, squashedPatch}) + + return &completePatchMerge, resultManifestBytes, nil + } + // Diff the installed updates and merge that into the target image patchDiff := llb.Diff(apkUpdated, apkInstalled) patchMerge := llb.Merge([]llb.State{am.config.ImageState, patchDiff}) + return &patchMerge, resultManifestBytes, nil } diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index 2cef9446..45901971 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -83,7 +83,11 @@ func getAPTImageName(manifest *unversioned.UpdateManifest, osVersion string) str osType := Debian if manifest == nil || manifest.Metadata.OS.Type == Debian { - version = strings.Split(version, ".")[0] + "-slim" + if version > "12" { + version = strings.Split("stable", ".")[0] + "-slim" + } else { + version = strings.Split(version, ".")[0] + "-slim" + } } else { osType = manifest.Metadata.OS.Type } @@ -352,9 +356,29 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U return nil, nil, err } + // If the image has been patched before, diff the base image and patched image to retain previous patches + if dm.config.PatchedConfigData != nil { + // Diff the base image and patched image to get previous patches + prevPatchDiff := llb.Diff(dm.config.ImageState, dm.config.PatchedImageState) + + // Diff the base image and new patches + newPatchDiff := llb.Diff(aptUpdated, aptInstalled) + + // Merging these two diffs will discard everything in the filesystem that hasn't changed + // Doing llb.Scratch ensures we can keep everything in the filesystem that has not changed + combinedPatch := llb.Merge([]llb.State{prevPatchDiff, newPatchDiff}) + squashedPatch := llb.Scratch().File(llb.Copy(combinedPatch, "/", "/")) + + // Merge previous and new patches into the base image + completePatchMerge := llb.Merge([]llb.State{dm.config.ImageState, squashedPatch}) + + return &completePatchMerge, resultsBytes, nil + } + // Diff the installed updates and merge that into the target image patchDiff := llb.Diff(aptUpdated, aptInstalled) patchMerge := llb.Merge([]llb.State{dm.config.ImageState, patchDiff}) + return &patchMerge, resultsBytes, nil } @@ -499,9 +523,29 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers copyStatusCmd := fmt.Sprintf(strings.ReplaceAll(copyStatusTemplate, "\n", ""), dpkgStatusFolder, dm.statusdNames) statusUpdated := fieldsWritten.Dir(resultsPath).Run(llb.Shlex(copyStatusCmd)).Root() + // If the image has been patched before, diff the base image and patched image to retain previous patches + if dm.config.PatchedConfigData != nil { + // Diff the base image and patched image to get previous patches + prevPatchDiff := llb.Diff(dm.config.ImageState, dm.config.PatchedImageState) + + // Diff the manifests for the latest updates + manifestsDiff := llb.Diff(fieldsWritten, statusUpdated) + + // Merging these two diffs will discard everything in the filesystem that hasn't changed + // Doing llb.Scratch ensures we can keep everything in the filesystem that has not changed + combinedPatch := llb.Merge([]llb.State{prevPatchDiff, manifestsDiff, unpackedToRoot}) + squashedPatch := llb.Scratch().File(llb.Copy(combinedPatch, "/", "/")) + + // Merge previous and new patches into the base image + completePatchMerge := llb.Merge([]llb.State{dm.config.ImageState, squashedPatch}) + + return &completePatchMerge, resultsBytes, nil + } + // Diff unpacked packages layers from previous and merge with target statusDiff := llb.Diff(fieldsWritten, statusUpdated) merged := llb.Merge([]llb.State{dm.config.ImageState, unpackedToRoot, statusDiff}) + return &merged, resultsBytes, nil } diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index f01da934..a0e08b06 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -456,9 +456,29 @@ func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.Up } } + // If the image has been patched before, diff the base image and patched image to retain previous patches + if rm.config.PatchedConfigData != nil { + // Diff the base image and pat[]ched image to get previous patches + prevPatchDiff := llb.Diff(rm.config.ImageState, rm.config.PatchedImageState) + + // Diff the base image and new patches + newPatchDiff := llb.Diff(rm.config.ImageState, installed) + + // Merging these two diffs will discard everything in the filesystem that hasn't changed + // Doing llb.Scratch ensures we can keep everything in the filesystem that has not changed + combinedPatch := llb.Merge([]llb.State{prevPatchDiff, newPatchDiff}) + squashedPatch := llb.Scratch().File(llb.Copy(combinedPatch, "/", "/")) + + // Merge previous and new patches into the base image + completePatchMerge := llb.Merge([]llb.State{rm.config.ImageState, squashedPatch}) + + return &completePatchMerge, resultBytes, nil + } + // Diff the installed updates and merge that into the target image patchDiff := llb.Diff(rm.config.ImageState, installed) patchMerge := llb.Merge([]llb.State{rm.config.ImageState, patchDiff}) + return &patchMerge, resultBytes, nil } @@ -600,9 +620,29 @@ func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversi return nil, nil, err } + // If the image has been patched before, diff the base image and patched image to retain previous patches + if rm.config.PatchedConfigData != nil { + // Diff the base image and patched image to get previous patches + prevPatchDiff := llb.Diff(rm.config.ImageState, rm.config.PatchedImageState) + + // Diff the manifests for the latest updates + manifestsDiff := llb.Diff(manifestsUpdated, manifestsPlaced) + + // Merging these two diffs will discard everything in the filesystem that hasn't changed + // Doing llb.Scratch ensures we can keep everything in the filesystem that has not changed + combinedPatch := llb.Merge([]llb.State{prevPatchDiff, manifestsDiff, patchedRoot}) + squashedPatch := llb.Scratch().File(llb.Copy(combinedPatch, "/", "/")) + + // Merge previous and new patches into the base image + completePatchMerge := llb.Merge([]llb.State{rm.config.ImageState, squashedPatch}) + + return &completePatchMerge, resultBytes, nil + } + // Diff unpacked packages layers from previous and merge with target manifestsDiff := llb.Diff(manifestsUpdated, manifestsPlaced) merged := llb.Merge([]llb.State{rm.config.ImageState, patchedRoot, manifestsDiff}) + return &merged, resultBytes, nil }