Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: pull docker images for migration if they don't already exist #45

Merged
merged 5 commits into from
Jul 9, 2024
Merged

fix: pull docker images for migration if they don't already exist #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5ab407f
add ensureImage
colesnodgrass Jul 9, 2024
6dab753
Merge remote-tracking branch 'origin/main' into cole/migrate-pull-images
colesnodgrass Jul 9, 2024
04b931d
Merge remote-tracking branch 'origin/main' into cole/migrate-pull-images
colesnodgrass Jul 9, 2024
0fcb343
read the output to ensure the image pull is complete
colesnodgrass Jul 9, 2024
25154cc
update docker mocks
colesnodgrass Jul 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions internal/cmd/local/check_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/airbytehq/abctl/internal/telemetry"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/api/types/volume"
"github.com/google/go-cmp/cmp"
Expand Down Expand Up @@ -131,6 +132,9 @@ type mockDockerClient struct {
containerExecInspect func(ctx context.Context, execID string) (types.ContainerExecInspect, error)
containerExecStart func(ctx context.Context, execID string, config types.ExecStartCheck) error

imageList func(ctx context.Context, options image.ListOptions) ([]image.Summary, error)
imagePull func(ctx context.Context, refStr string, options image.PullOptions) (io.ReadCloser, error)

serverVersion func(ctx context.Context) (types.Version, error)
volumeInspect func(ctx context.Context, volumeID string) (volume.Volume, error)
}
Expand Down Expand Up @@ -171,6 +175,22 @@ func (m mockDockerClient) ContainerExecStart(ctx context.Context, execID string,
return m.containerExecStart(ctx, execID, config)
}

func (m mockDockerClient) ImageList(ctx context.Context, options image.ListOptions) ([]image.Summary, error) {
// by default return a list with one (empty) item in it
if m.imageList == nil {
return []image.Summary{{}}, nil
}
return m.imageList(ctx, options)
}

func (m mockDockerClient) ImagePull(ctx context.Context, img string, options image.PullOptions) (io.ReadCloser, error) {
// by default return a nop closer (with an empty string)
if m.imagePull == nil {
return io.NopCloser(strings.NewReader("")), nil
}
return m.imagePull(ctx, img, options)
}

func (m mockDockerClient) ServerVersion(ctx context.Context) (types.Version, error) {
return m.serverVersion(ctx)
}
Expand Down
55 changes: 52 additions & 3 deletions internal/cmd/local/docker/docker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"github.com/airbytehq/abctl/internal/cmd/local/paths"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/api/types/volume"
Expand Down Expand Up @@ -47,6 +49,9 @@ type Client interface {
ContainerExecInspect(ctx context.Context, execID string) (types.ContainerExecInspect, error)
ContainerExecStart(ctx context.Context, execID string, config types.ExecStartCheck) error

ImageList(ctx context.Context, options image.ListOptions) ([]image.Summary, error)
ImagePull(ctx context.Context, refStr string, options image.PullOptions) (io.ReadCloser, error)

ServerVersion(ctx context.Context) (types.Version, error)
VolumeInspect(ctx context.Context, volumeID string) (volume.Volume, error)
}
Expand Down Expand Up @@ -176,7 +181,11 @@ func (d *Docker) Port(ctx context.Context, container string) (int, error) {
return 0, errors.New("could not determine port for container")
}

const migratePGDATA = "/var/lib/postgresql/data"
const (
migratePGDATA = "/var/lib/postgresql/data"
imgAlpine = "alpine:3.20"
imgPostgres = "postgres:13-alpine"
)

// MigrateComposeDB handles migrating the existing docker compose database into the abctl managed k8s cluster.
// TODO: move this method out of the the docker class?
Expand All @@ -185,12 +194,19 @@ func (d *Docker) MigrateComposeDB(ctx context.Context, volume string) error {
return errors.New(fmt.Sprintf("volume %s does not exist", volume))
}

if err := d.ensureImage(ctx, imgAlpine); err != nil {
return err
}
if err := d.ensureImage(ctx, imgPostgres); err != nil {
return err
}

// create a container for running the `docker cp` command
// docker run -d -v airbyte_db:/var/lib/postgresql/data alpine:3.20 tail -f /dev/null
conCopy, err := d.Client.ContainerCreate(
ctx,
&container.Config{
Image: "alpine:3.20",
Image: imgAlpine,
Entrypoint: []string{"tail", "-f", "/dev/null"},
},
&container.HostConfig{
Expand Down Expand Up @@ -227,7 +243,7 @@ func (d *Docker) MigrateComposeDB(ctx context.Context, volume string) error {
conTransform, err := d.Client.ContainerCreate(
ctx,
&container.Config{
Image: "postgres:13-alpine",
Image: imgPostgres,
Env: []string{
"POSTGRES_USER=docker",
"POSTGRES_PASSWORD=docker",
Expand Down Expand Up @@ -286,6 +302,39 @@ func (d *Docker) MigrateComposeDB(ctx context.Context, volume string) error {
return nil
}

func (d *Docker) ensureImage(ctx context.Context, img string) error {
// check if an image already exists on the host
pterm.Debug.Println(fmt.Sprintf("Checking if the image '%s' already exists", img))
filter := filters.NewArgs()
filter.Add("reference", img)
imgs, err := d.Client.ImageList(ctx, image.ListOptions{Filters: filter})
if err != nil {
pterm.Error.Println(fmt.Sprintf("Could not list docker images when checking for '%s'", img))
return fmt.Errorf("could not list image '%s': %w", img, err)
}

// if it does exist, there is nothing else to do
if len(imgs) > 0 {
pterm.Debug.Println(fmt.Sprintf("Image '%s' already exists", img))
return nil
}

pterm.Debug.Println(fmt.Sprintf("Image '%s' not found, pulling it", img))
// if we're here, then we need to pull the image
reader, err := d.Client.ImagePull(ctx, img, image.PullOptions{})
if err != nil {
pterm.Error.Println(fmt.Sprintf("Could not pull the docker image '%s'", img))
return fmt.Errorf("could not pull image '%s': %w", img, err)
}
pterm.Debug.Println(fmt.Sprintf("Successfully pulled the docker image '%s'", img))
defer reader.Close()
if _, err := io.Copy(io.Discard, reader); err != nil {
return fmt.Errorf("error fetching output: %w", err)
}

return nil
}

// volumeExists returns the MountPoint of the volumeID (if the volume exists), an empty string otherwise.
func (d *Docker) volumeExists(ctx context.Context, volumeID string) string {
if v, err := d.Client.VolumeInspect(ctx, volumeID); err != nil {
Expand Down
21 changes: 21 additions & 0 deletions internal/cmd/local/docker/docker_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ import (
"github.com/airbytehq/abctl/internal/cmd/local/localerr"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/api/types/volume"
"github.com/docker/docker/client"
"github.com/docker/go-connections/nat"
"github.com/google/go-cmp/cmp"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"io"
"strings"
"testing"
)

Expand Down Expand Up @@ -332,6 +334,9 @@ type mockPinger struct {
containerExecInspect func(ctx context.Context, execID string) (types.ContainerExecInspect, error)
containerExecStart func(ctx context.Context, execID string, config types.ExecStartCheck) error

imageList func(ctx context.Context, options image.ListOptions) ([]image.Summary, error)
imagePull func(ctx context.Context, refStr string, options image.PullOptions) (io.ReadCloser, error)

serverVersion func(ctx context.Context) (types.Version, error)
volumeInspect func(ctx context.Context, volumeID string) (volume.Volume, error)

Expand Down Expand Up @@ -370,6 +375,22 @@ func (m mockPinger) ContainerExecStart(ctx context.Context, execID string, confi
return m.containerExecStart(ctx, execID, config)
}

func (m mockPinger) ImageList(ctx context.Context, options image.ListOptions) ([]image.Summary, error) {
// by default return a list with one (empty) item in it
if m.imageList == nil {
return []image.Summary{{}}, nil
}
return m.imageList(ctx, options)
}

func (m mockPinger) ImagePull(ctx context.Context, img string, options image.PullOptions) (io.ReadCloser, error) {
// by default return a nop closer (with an empty string)
if m.imagePull == nil {
return io.NopCloser(strings.NewReader("")), nil
}
return m.imagePull(ctx, img, options)
}

func (m mockPinger) VolumeInspect(ctx context.Context, volumeID string) (volume.Volume, error) {
return m.volumeInspect(ctx, volumeID)
}
Expand Down