Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: discard patch layer #689

Merged
Merged
Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
e026382
Add mock config
Jun 22, 2024
66f345d
Revert "Add mock config"
Jun 24, 2024
c7f8c9e
refactor code to unmarshal and modify image metadata
Jun 24, 2024
b7bd8e8
refactor code for readability and maintainability
Jun 25, 2024
da366b0
Refactor updateImageMetadata and setupLabels functions
Jun 25, 2024
20e35df
Add support for patched image data in BuildkitConfig
Jun 25, 2024
051ffd5
Add patch retention for previously patched images, code refactoring
Jun 26, 2024
c6b6c31
Add patch retention for previously patched images, add tests
Jun 26, 2024
3dbb9c0
Refactor tests and rename function in buildkit package
Jun 26, 2024
8f1d79b
Expand test coverage
Jun 27, 2024
53e313d
Merge branch 'project-copacetic:main' into discard-patch-layer
MiahaCybersec Jun 27, 2024
27ca23e
Minor refactor to discard patch layers in all package manager
Jun 28, 2024
218cf8a
Add mock client tests and expand test coverage
Jul 2, 2024
32889ca
Merge branch 'project-copacetic:main' into discard-patch-layer
MiahaCybersec Jul 3, 2024
ec7b989
Merge branch 'project-copacetic:main' into discard-patch-layer
MiahaCybersec Jul 3, 2024
49d6385
Add error handling for type assertions
Jul 3, 2024
6612666
Revert go mod
Jul 3, 2024
eb55041
Remove unused mock methods and unncessary dependency
Jul 5, 2024
7c1a314
Add mock client assertions and suppress gocritic hugeParam error
Jul 5, 2024
2e6afee
Merge branch 'project-copacetic:main' into discard-patch-layer
MiahaCybersec Jul 16, 2024
d9b67ca
Refactor patch merging and apply error handling
Jul 22, 2024
28aa14e
golangci-lint
Jul 22, 2024
c27ac73
golangci-lint
Jul 22, 2024
8314eff
Merge branch 'main' into discard-patch-layer
MiahaCybersec Jul 24, 2024
21d48d5
Merge branch 'project-copacetic:main' into discard-patch-layer
MiahaCybersec Jul 30, 2024
a01880e
Revert first-time patching logic
Jul 30, 2024
503e24b
Revert patching logic regression in DPKG and APK
Jul 31, 2024
efa23d8
Update dpkg version handling with nil manifest
Aug 2, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ require (
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/theupdateframework/notary v0.7.0 // indirect
github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c // indirect
Expand Down
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
Expand Down
141 changes: 141 additions & 0 deletions mocks/mock_gwclient.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
package mocks

import (
"context"
"fmt"

"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/solver/pb"
"github.com/opencontainers/go-digest"
"github.com/stretchr/testify/mock"
)

// Mock for gwclient.Client.
type MockGWClient struct {
mock.Mock
}

func (m *MockGWClient) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt sourceresolver.Opt) (*sourceresolver.MetaResponse, error) {
args := m.Called(ctx, op, opt)

metaResponse, ok := args.Get(0).(*sourceresolver.MetaResponse)
if !ok {
return nil, fmt.Errorf("type assertion to *sourceresolver.MetaResponse failed")
}

return metaResponse, args.Error(1)
}

//nolint:gocritic
func (m *MockGWClient) Solve(ctx context.Context, req gwclient.SolveRequest) (*gwclient.Result, error) {
args := m.Called(ctx, req)

result, ok := args.Get(0).(*gwclient.Result)
if !ok {
return nil, fmt.Errorf("type assertion to *gwclient.Result failed")
}

return result, args.Error(1)
}

func (m *MockGWClient) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
args := m.Called(ctx, ref, opt)

digestResult, ok1 := args.Get(1).(digest.Digest)
if !ok1 {
return "", digest.Digest(""), nil, fmt.Errorf("type assertion to digest.Digest failed")
}

byteResult, ok2 := args.Get(2).([]byte)
if !ok2 {
return "", digest.Digest(""), nil, fmt.Errorf("type assertion to []byte failed")
}

return args.String(0), digestResult, byteResult, args.Error(3)
}

func (m *MockGWClient) BuildOpts() gwclient.BuildOpts {
args := m.Called()

buildOpts, ok := args.Get(0).(gwclient.BuildOpts)
if !ok {
panic("type assertion to gwclient.BuildOpts failed")
}

return buildOpts
}

func (m *MockGWClient) Inputs(ctx context.Context) (map[string]llb.State, error) {
args := m.Called(ctx)

stateMap, ok := args.Get(0).(map[string]llb.State)
if !ok {
return nil, fmt.Errorf("type assertion to map[string]llb.State failed")
}

return stateMap, args.Error(1)
}

//nolint:gocritic
func (m *MockGWClient) NewContainer(ctx context.Context, req gwclient.NewContainerRequest) (gwclient.Container, error) {
args := m.Called(ctx, req)

container, ok := args.Get(0).(gwclient.Container)
if !ok {
return nil, fmt.Errorf("type assertion to gwclient.Container failed")
}

return container, args.Error(1)
}

//nolint:gocritic
func (m *MockGWClient) Warn(ctx context.Context, dgst digest.Digest, msg string, opts gwclient.WarnOpts) error {
args := m.Called(ctx, dgst, msg, opts)

warnErr, ok := args.Get(0).(error)
if !ok {
return fmt.Errorf("type assertion to error failed")
}

return warnErr
}

// MockReference is a mock of the Reference interface.
type MockReference struct {
mock.Mock
}

func (m *MockReference) ReadFile(ctx context.Context, req gwclient.ReadRequest) ([]byte, error) {
args := m.Called(ctx, req)

byteResult, ok := args.Get(0).([]byte)
if !ok {
return nil, fmt.Errorf("type assertion to []byte failed")
}

return byteResult, args.Error(1)
}

func (m *MockReference) ToState() (llb.State, error) {
args := m.Called()

state, ok := args.Get(0).(llb.State)
if !ok {
return state, fmt.Errorf("type assertion to llb.State failed")
}

return state, args.Error(1)
}

func (m *MockReference) Evaluate(ctx context.Context) error {
args := m.Called(ctx)

evalErr, ok := args.Get(0).(error)
if !ok {
return fmt.Errorf("type assertion to error failed")
}

return evalErr
}
107 changes: 97 additions & 10 deletions pkg/buildkit/buildkit.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import (
"bytes"
"context"
"encoding/json"
"fmt"

"github.com/containerd/platforms"
"github.com/moby/buildkit/client/llb"
Expand All @@ -12,11 +14,13 @@
)

type Config struct {
ImageName string
Client gwclient.Client
ConfigData []byte
Platform *ispec.Platform
ImageState llb.State
ImageName string
Client gwclient.Client
ConfigData []byte
PatchedConfigData []byte
Platform *ispec.Platform
ImageState llb.State
PatchedImageState llb.State
MiahaCybersec marked this conversation as resolved.
Show resolved Hide resolved
}

type Opts struct {
Expand All @@ -26,14 +30,14 @@
KeyPath string
}

func InitializeBuildkitConfig(ctx context.Context, c gwclient.Client, image string) (*Config, error) {
func InitializeBuildkitConfig(ctx context.Context, c gwclient.Client, userImage string) (*Config, error) {

Check warning on line 33 in pkg/buildkit/buildkit.go

View check run for this annotation

Codecov / codecov/patch

pkg/buildkit/buildkit.go#L33

Added line #L33 was not covered by tests
// Initialize buildkit config for the target image
config := Config{
ImageName: image,
ImageName: userImage,

Check warning on line 36 in pkg/buildkit/buildkit.go

View check run for this annotation

Codecov / codecov/patch

pkg/buildkit/buildkit.go#L36

Added line #L36 was not covered by tests
}

// Resolve and pull the config for the target image
_, _, configData, err := c.ResolveImageConfig(ctx, image, sourceresolver.Opt{
_, _, configData, err := c.ResolveImageConfig(ctx, userImage, sourceresolver.Opt{

Check warning on line 40 in pkg/buildkit/buildkit.go

View check run for this annotation

Codecov / codecov/patch

pkg/buildkit/buildkit.go#L40

Added line #L40 was not covered by tests
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: llb.ResolveModePreferLocal.String(),
},
Expand All @@ -42,23 +46,106 @@
return nil, err
}

config.ConfigData = configData
var baseImage string
config.ConfigData, config.PatchedConfigData, baseImage, err = updateImageConfigData(ctx, c, configData, userImage)
if err != nil {
return nil, err

Check warning on line 52 in pkg/buildkit/buildkit.go

View check run for this annotation

Codecov / codecov/patch

pkg/buildkit/buildkit.go#L49-L52

Added lines #L49 - L52 were not covered by tests
}

// Load the target image state with the resolved image config in case environment variable settings
// are necessary for running apps in the target image for updates
config.ImageState, err = llb.Image(image,
config.ImageState, err = llb.Image(baseImage,

Check warning on line 57 in pkg/buildkit/buildkit.go

View check run for this annotation

Codecov / codecov/patch

pkg/buildkit/buildkit.go#L57

Added line #L57 was not covered by tests
llb.ResolveModePreferLocal,
llb.WithMetaResolver(c),
).WithImageConfig(config.ConfigData)
if err != nil {
return nil, err
}

// Only set PatchedImageState if the user supplied a patched image
// An image is deemed to be a patched image if it contains one of two metadata values
// BaseImage or ispec.AnnotationBaseImageName
ashnamehrotra marked this conversation as resolved.
Show resolved Hide resolved
if config.PatchedConfigData != nil {
config.PatchedImageState, err = llb.Image(userImage,
llb.ResolveModePreferLocal,
llb.WithMetaResolver(c),
).WithImageConfig(config.PatchedConfigData)
if err != nil {
return nil, err

Check warning on line 74 in pkg/buildkit/buildkit.go

View check run for this annotation

Codecov / codecov/patch

pkg/buildkit/buildkit.go#L68-L74

Added lines #L68 - L74 were not covered by tests
}
}

config.Client = c

return &config, nil
}

func updateImageConfigData(ctx context.Context, c gwclient.Client, configData []byte, image string) ([]byte, []byte, string, error) {
baseImage, userImageConfig, err := setupLabels(image, configData)
if err != nil {
return nil, nil, "", err

Check warning on line 86 in pkg/buildkit/buildkit.go

View check run for this annotation

Codecov / codecov/patch

pkg/buildkit/buildkit.go#L86

Added line #L86 was not covered by tests
}

if baseImage == "" {
configData = userImageConfig
} else {
patchedImageConfig := userImageConfig
_, _, baseImageConfig, err := c.ResolveImageConfig(ctx, baseImage, sourceresolver.Opt{
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: llb.ResolveModePreferLocal.String(),
},
})
if err != nil {
return nil, nil, "", err

Check warning on line 99 in pkg/buildkit/buildkit.go

View check run for this annotation

Codecov / codecov/patch

pkg/buildkit/buildkit.go#L99

Added line #L99 was not covered by tests
}

_, baseImageWithLabels, _ := setupLabels(baseImage, baseImageConfig)
configData = baseImageWithLabels

return configData, patchedImageConfig, baseImage, nil
}

return configData, nil, image, nil
}

func setupLabels(image string, configData []byte) (string, []byte, error) {
imageConfig := make(map[string]interface{})
err := json.Unmarshal(configData, &imageConfig)
if err != nil {
return "", nil, err
}

configMap, ok := imageConfig["config"].(map[string]interface{})
if !ok {
err := fmt.Errorf("type assertion to map[string]interface{} failed")
return "", nil, err

Check warning on line 121 in pkg/buildkit/buildkit.go

View check run for this annotation

Codecov / codecov/patch

pkg/buildkit/buildkit.go#L120-L121

Added lines #L120 - L121 were not covered by tests
}

var baseImage string
labels := configMap["labels"]
if labels == nil {
configMap["labels"] = make(map[string]interface{})
}
labelsMap, ok := configMap["labels"].(map[string]interface{})
if !ok {
err := fmt.Errorf("type assertion to map[string]interface{} failed")
return "", nil, err

Check warning on line 132 in pkg/buildkit/buildkit.go

View check run for this annotation

Codecov / codecov/patch

pkg/buildkit/buildkit.go#L131-L132

Added lines #L131 - L132 were not covered by tests
}
if baseImageValue := labelsMap["BaseImage"]; baseImageValue != nil {
baseImage, ok = baseImageValue.(string)
if !ok {
err := fmt.Errorf("type assertion to string failed")
return "", nil, err

Check warning on line 138 in pkg/buildkit/buildkit.go

View check run for this annotation

Codecov / codecov/patch

pkg/buildkit/buildkit.go#L137-L138

Added lines #L137 - L138 were not covered by tests
}
} else {
labelsMap["BaseImage"] = image
}

imageWithLabels, _ := json.Marshal(imageConfig)

return baseImage, imageWithLabels, nil
}

// Extracts the bytes of the file denoted by `path` from the state `st`.
func ExtractFileFromState(ctx context.Context, c gwclient.Client, st *llb.State, path string) ([]byte, error) {
// since platform is obtained from host, override it in the case of Darwin
Expand Down
Loading
Loading