From c51b25c089fd400bf069f349c499724302dd5f5f Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Mon, 21 Sep 2015 17:32:36 +0300 Subject: [PATCH 01/34] vendor update rocker/imagename --- vendor/manifest | 15 +- .../rocker/src/rocker/imagename/dockerhub.go | 165 +++++++++++++++ .../rocker/src/rocker/imagename/imagename.go | 170 ++++++++++++--- .../src/rocker/imagename/imagename_test.go | 194 ++++++++++++++---- 4 files changed, 470 insertions(+), 74 deletions(-) create mode 100644 vendor/src/github.com/grammarly/rocker/src/rocker/imagename/dockerhub.go diff --git a/vendor/manifest b/vendor/manifest index 1747fa0..aa5365e 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -78,13 +78,6 @@ "branch": "master", "path": "/pkg/units" }, - { - "importpath": "github.com/grammarly/rocker/src/rocker/imagename", - "repository": "https://github.com/grammarly/rocker", - "revision": "77d2615f9b36f7c9997ed3a11519767a07f20a5b", - "branch": "master", - "path": "/src/rocker/imagename" - }, { "importpath": "github.com/grammarly/rocker/src/rocker/test", "repository": "https://github.com/grammarly/rocker", @@ -118,6 +111,14 @@ "revision": "b4e13649def6e2852db73477f6cf272b76d47e53", "branch": "master", "path": "/src/rocker/template" + }, + { + "importpath": "github.com/grammarly/rocker/src/rocker/imagename", + "repository": "https://github.com/grammarly/rocker", + "revision": "b4e13649def6e2852db73477f6cf272b76d47e53", + "branch": "master", + "path": "/src/rocker/imagename" + }, } ] } \ No newline at end of file diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/dockerhub.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/dockerhub.go new file mode 100644 index 0000000..787acbd --- /dev/null +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/dockerhub.go @@ -0,0 +1,165 @@ +/*- + * Copyright 2015 Grammarly, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package imagename + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/fsouza/go-dockerclient" +) + +type tags struct { + Name string `json:"name,omitempty"` + Tags []string `json:"tags,omitempty"` +} + +type history struct { + Compatibility string `json:"v1Compatibility,omitempty"` +} + +type manifests struct { + Name string `json:"name,omitempty"` + Tag string `json:"tag,omitempty"` + Architecture string `json:"architecture,omitempty"` + History []*history `json:"history,omitempty"` + SchemaVersion int `json:"schemaVersion,omitempty"` +} + +type hubTags struct { + Count int `json:"count,omitempty"` + Next string `json:"next,omitempty"` + Previous string `json:"previous,omitempty"` + Results []*hubTag `json:"results,omitempty"` +} + +type hubTag struct { + Name string `json:"name,omitempty"` + FullSize int `json:"full_size,omitempty"` + ID int `json:"id,omitempty"` + Repository int `json:"repository,omitempty"` + Creator int `json:"creator,omitempty"` + LastUpdater int `json:"last_updater,omitempty"` + ImageID string `json:"image_id,omitempty"` + V2 bool `json:"v2,omitempty"` +} + +// DockerHub is an facade for communicating with registries +// It is used for getting tag manifests and the list of image tags +type DockerHub struct{} + +// NewDockerHub returns new DockerHub instance +func NewDockerHub() *DockerHub { + return &DockerHub{} +} + +// Get returns docker.Image instance from the information stored in the registry +func (h *DockerHub) Get(image *ImageName) (img *docker.Image, err error) { + manifest := manifests{} + img = &docker.Image{} + + // no cannot get similar info from Hub, just return stub data + if image.Registry == "" { + return + } + + if err = h.doGet(fmt.Sprintf("https://%s/v2/%s/manifests/%s", image.Registry, image.Name, image.Tag), &manifest); err != nil { + return + } + + if len(manifest.History) == 0 { + err = fmt.Errorf("Image doesn't have expected format, history record is empty") + return + } + + last := manifest.History[0] + err = json.Unmarshal([]byte(last.Compatibility), img) + return +} + +// List returns the list of images instances obtained from all tags existing in the registry +func (h *DockerHub) List(image *ImageName) (images []*ImageName, err error) { + if image.Registry != "" { + return h.listRegistry(image) + } + + return h.listHub(image) +} + +// listHub lists image tags from hub.docker.com +func (h *DockerHub) listHub(image *ImageName) (images []*ImageName, err error) { + tg := hubTags{} + if err = h.doGet(fmt.Sprintf("https://hub.docker.com/v2/repositories/library/%s/tags/?page_size=9999&page=1", image.Name), &tg); err != nil { + return + } + + for _, t := range tg.Results { + candidate := New(image.NameWithRegistry(), t.Name) + if image.Contains(candidate) { + images = append(images, candidate) + } + } + return +} + +// listRegistry lists image tags from a private docker registry +func (h *DockerHub) listRegistry(image *ImageName) (images []*ImageName, err error) { + tg := tags{} + if err = h.doGet(fmt.Sprintf("https://%s/v2/%s/tags/list", image.Registry, image.Name), &tg); err != nil { + return + } + + for _, t := range tg.Tags { + candidate := New(image.NameWithRegistry(), t) + if image.Contains(candidate) { + images = append(images, candidate) + } + } + return +} + +// doGet executes HTTP get to a given registry +func (h *DockerHub) doGet(url string, obj interface{}) (err error) { + var res *http.Response + var body []byte + + res, err = http.Get(url) + if err != nil { + err = fmt.Errorf("Request to %s failed with %s\n", url, err) + return + } + + if res.StatusCode == 404 { + err = fmt.Errorf("Not found") + return + } + + if body, err = ioutil.ReadAll(res.Body); err != nil { + err = fmt.Errorf("Response from %s cannot be read due to error %s\n", url, err) + return + } + + if err = json.Unmarshal(body, obj); err != nil { + err = fmt.Errorf("Response from %s cannot be unmarshalled due to error %s, response: %s\n", + url, err, string(body)) + return + } + + return +} diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go index 4842c4c..ab5894c 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go @@ -14,88 +14,198 @@ * limitations under the License. */ +// Package imagename provides docker data structure for docker image names +// It also provides a number of utility functions, related to image name resolving, +// comparison, and semver functionality. package imagename import ( + "encoding/json" "sort" "strings" + + "github.com/wmark/semver" ) -const Latest = "latest" +const ( + // Latest is :latest tag value + Latest = "latest" + + // Wildcards is wildcard value variants + Wildcards = "x*" +) +// ImageName is the data structure with describes docker image name type ImageName struct { Registry string Name string Tag string + Version *semver.Range +} + +// NewFromString parses a given string and returns ImageName +func NewFromString(image string) *ImageName { + n := strings.LastIndex(image, ":") + if n < 0 { + return New(image, "") + } + if tag := image[n+1:]; !strings.Contains(tag, "/") { + return New(image[:n], tag) + } + return New(image, "") +} + +// New parses a given 'image' and 'tag' strings and returns ImageName +func New(image string, tag string) *ImageName { + dockerImage := &ImageName{} + if strings.Contains(image, ".") || len(strings.SplitN(image, "/", 3)) > 2 { + registryAndName := strings.SplitN(image, "/", 2) + dockerImage.Registry = registryAndName[0] + dockerImage.Name = registryAndName[1] + } else { + dockerImage.Name = image + } + if tag != "" { + if rng, err := semver.NewRange(tag); err == nil && rng != nil { + dockerImage.Version = rng + } + dockerImage.Tag = tag + } + return dockerImage +} + +// String returns the string representation of the current image name +func (img ImageName) String() string { + return img.NameWithRegistry() + ":" + img.GetTag() } -func (dockerImage ImageName) GetTag() string { - if dockerImage.HasTag() { - return dockerImage.Tag +// HasTag returns true if tags is specified for the image name +func (img ImageName) HasTag() bool { + return img.Tag != "" +} + +// GetTag returns the tag of the current image name +func (img ImageName) GetTag() string { + if img.HasTag() { + return img.Tag } return Latest } -func (dockerImage ImageName) HasTag() bool { - return dockerImage.Tag != "" +// IsStrict returns true if tag of the current image is specified and contains no fuzzy rules +func (img ImageName) IsStrict() bool { + if img.HasVersionRange() { + return img.TagAsVersion() != nil + } + return img.Tag != "" +} + +// All returns true if tag of the current image refers to any version +func (img ImageName) All() bool { + return strings.Contains(Wildcards, img.Tag) } -func (a ImageName) IsSameKind(b ImageName) bool { - return a.Registry == b.Registry && a.Name == b.Name +// HasVersion returns true if tag of the current image refers to a semver format +func (img ImageName) HasVersion() bool { + return img.TagAsVersion() != nil } -func (dockerImage ImageName) NameWithRegistry() string { +// HasVersionRange returns true if tag of the current image refers to a semver format and is fuzzy +func (img ImageName) HasVersionRange() bool { + return img.Version != nil +} + +// TagAsVersion return semver.Version of the current image tag in case it's in semver format +func (img ImageName) TagAsVersion() (ver *semver.Version) { + ver, _ = semver.NewVersion(strings.TrimPrefix(img.Tag, "v")) + return +} + +// IsSameKind returns true if current image and the given one are same but may have different versions (tags) +func (img ImageName) IsSameKind(b ImageName) bool { + return img.Registry == b.Registry && img.Name == b.Name +} + +// NameWithRegistry returns the [registry/]name of the current image name +func (img ImageName) NameWithRegistry() string { registryPrefix := "" - if dockerImage.Registry != "" { - registryPrefix = dockerImage.Registry + "/" + if img.Registry != "" { + registryPrefix = img.Registry + "/" } - return registryPrefix + dockerImage.Name + return registryPrefix + img.Name } -func (dockerImage ImageName) String() string { - return dockerImage.NameWithRegistry() + ":" + dockerImage.GetTag() -} +// Contains returns true if the current image tag wildcard satisfies a given image version +func (img ImageName) Contains(b *ImageName) bool { + if b == nil { + return false + } -func New(image string) *ImageName { - dockerImage := &ImageName{} - split := strings.SplitN(image, ":", 2) - if strings.Contains(split[0], ".") || len(strings.SplitN(image, "/", 3)) > 2 { - registryAndName := strings.SplitN(split[0], "/", 2) - dockerImage.Registry = registryAndName[0] - dockerImage.Name = registryAndName[1] - } else { - dockerImage.Name = split[0] + if !img.IsSameKind(*b) { + return false } - if len(split) > 1 { - dockerImage.Tag = split[1] + + // semver library has a bug with wildcards, so this checks are + // necessary: empty range (or wildcard range) cannot contain any version, it just fails + if img.All() { + return true } - return dockerImage + + if img.IsStrict() && img.Tag == b.Tag { + return true + } + + if img.HasVersionRange() && b.HasVersion() && img.Version.IsSatisfiedBy(b.TagAsVersion()) { + return true + } + + return img.Tag == "" && !img.HasVersionRange() +} + +// UnmarshalJSON parses JSON string and returns ImageName +func (img *ImageName) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + *img = *NewFromString(s) + return nil +} + +// MarshalJSON serializes ImageName to JSON string +func (img ImageName) MarshalJSON() ([]byte, error) { + return json.Marshal(img.String()) } -// Type structures used for cleaning images -// Able to sort out old tags by creation date +// Tags is a structure used for cleaning images +// Sorts out old tags by creation date type Tags struct { Items []*Tag } +// Tag is a structure used for cleaning images type Tag struct { - Id string + ID string Name ImageName Created int64 } +// Len returns the length of image tags func (tags *Tags) Len() int { return len(tags.Items) } +// Less returns true if item by index[i] is created after of item[j] func (tags *Tags) Less(i, j int) bool { return tags.Items[i].Created > tags.Items[j].Created } +// Swap swaps items by indices [i] and [j] func (tags *Tags) Swap(i, j int) { tags.Items[i], tags.Items[j] = tags.Items[j], tags.Items[i] } +// GetOld returns the list of items older then [keep] newest items sorted by Created date func (tags *Tags) GetOld(keep int) []ImageName { if len(tags.Items) < keep { return nil diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go index 130d2c5..d959234 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go @@ -21,84 +21,204 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/wmark/semver" ) func TestImageParsingWithoutNamespace(t *testing.T) { - img := New("repo/name:1") + img := NewFromString("repo/name:1") assert.Equal(t, "", img.Registry) assert.Equal(t, "1", img.Tag) assert.Equal(t, "repo/name", img.Name) + assert.True(t, img.Contains(NewFromString("repo/name:1"))) +} + +func TestWildcardNamespace(t *testing.T) { + img := NewFromString("repo/name:*") + assert.Empty(t, img.Registry) + assert.Equal(t, "*", img.Tag) + assert.Equal(t, "repo/name", img.Name) + assert.True(t, img.Contains(NewFromString("repo/name:1.0.0"))) +} + +func TestEnvironmentImageName(t *testing.T) { + img := NewFromString("repo/name:1.0.0") + assert.False(t, img.Contains(NewFromString("repo/name:1.0.123"))) } func TestImageRealLifeNamingExample(t *testing.T) { - img := New("quay.io/platform/dockerize:v0.0.1") - assert.Equal(t, "quay.io", img.Registry) - assert.Equal(t, "platform/dockerize", img.Name) + img := NewFromString("docker.io/tools/dockerize:v0.0.1") + assert.Equal(t, "docker.io", img.Registry) + assert.Equal(t, "tools/dockerize", img.Name) assert.Equal(t, "v0.0.1", img.Tag) + assert.True(t, img.Contains(NewFromString("docker.io/tools/dockerize:v0.0.1"))) +} + +func TestRangeContainsPlainVersion(t *testing.T) { + img := NewFromString("docker.io/tools/dockerize:0.0.1") + expected, _ := semver.NewRange("0.0.1") + assert.Equal(t, "docker.io", img.Registry) + assert.Equal(t, "tools/dockerize", img.Name) + assert.Equal(t, "0.0.1", img.Tag) + assert.Equal(t, expected, img.Version) + + v, _ := semver.NewVersion("0.0.1") + assert.True(t, img.Version.Contains(v)) +} + +func TestUpperRangeBounds(t *testing.T) { + img := NewFromString("docker.io/tools/dockerize:~1.2.3") + assert.Equal(t, "docker.io", img.Registry) + assert.Equal(t, "tools/dockerize", img.Name) + assert.False(t, img.IsStrict()) + v, _ := semver.NewVersion("1.2.8") + assert.True(t, img.Version.Contains(v)) +} + +func TestWildcardRangeBounds(t *testing.T) { + img := NewFromString("docker.io/tools/dockerize:1.2.*") + assert.Equal(t, "docker.io", img.Registry) + assert.Equal(t, "tools/dockerize", img.Name) + assert.False(t, img.IsStrict()) + v, _ := semver.NewVersion("1.2.8") + assert.True(t, img.Version.Contains(v)) + v, _ = semver.NewVersion("1.2.0") + assert.True(t, img.Version.Contains(v)) +} + +func TestWildcardContains(t *testing.T) { + img1 := NewFromString("docker.io/tools/dockerize:1.2.*") + img2 := NewFromString("docker.io/tools/dockerize:1.2.1") + assert.False(t, img1.IsStrict()) + assert.True(t, img1.HasVersionRange()) + assert.True(t, img2.IsStrict()) + v, _ := semver.NewVersion("1.2.1") + assert.Equal(t, v, img2.TagAsVersion()) + + assert.True(t, img1.Contains(img2)) + assert.False(t, img2.Contains(img1)) +} + +func TestRangeContains(t *testing.T) { + img1 := NewFromString("docker.io/tools/dockerize:~1.2.1") + img2 := NewFromString("docker.io/tools/dockerize:1.2.999") + assert.True(t, img1.Contains(img2)) + assert.False(t, img2.Contains(img1)) +} + +func TestNilContains(t *testing.T) { + img1 := NewFromString("docker.io/tools/dockerize:~1.2.1") + assert.False(t, img1.Contains(nil)) +} + +func TestRangeNotContains(t *testing.T) { + img1 := NewFromString("docker.io/tools/dockerize:~1.2.1") + img2 := NewFromString("docker.io/tools/dockerize:1.3.1") + assert.False(t, img1.Contains(img2)) + assert.False(t, img2.Contains(img1)) + + img2 = NewFromString("docker.io/xxx/dockerize:1.2.1") + assert.False(t, img1.Contains(img2)) + + img2 = NewFromString("dockerhub.grammarly.com/tools/dockerize:1.2.1") + assert.False(t, img1.Contains(img2)) + + img2 = NewFromString("docker.io/tools/dockerize:1.2.1") + assert.True(t, img1.Contains(img2)) +} + +func TestVersionContains(t *testing.T) { + img1 := NewFromString("docker.io/tools/dockerize:1.2.1") + img2 := NewFromString("docker.io/tools/dockerize:1.2.1") + assert.True(t, img1.Contains(img2)) + assert.True(t, img2.Contains(img1)) +} + +func TestTagContains(t *testing.T) { + img1 := NewFromString("docker.io/tools/dockerize:test1") + img2 := NewFromString("docker.io/tools/dockerize:test1") + assert.True(t, img1.Contains(img2)) + assert.True(t, img2.Contains(img1)) +} + +func TestTagNotContains(t *testing.T) { + img1 := NewFromString("docker.io/tools/dockerize:test1") + img2 := NewFromString("docker.io/tools/dockerize:test2") + assert.False(t, img1.Contains(img2)) + assert.False(t, img2.Contains(img1)) } func TestImageRealLifeNamingExampleWithCapi(t *testing.T) { - img := New("quay.io/common-api") - assert.Equal(t, "quay.io", img.Registry) + img := NewFromString("docker.io/common-api") + assert.Equal(t, "docker.io", img.Registry) assert.Equal(t, "common-api", img.Name) + assert.Equal(t, false, img.HasTag()) assert.Equal(t, "latest", img.GetTag()) + assert.Equal(t, "docker.io/common-api:latest", img.String()) } func TestImageParsingWithNamespace(t *testing.T) { - img := New("hub/ns/name:1") + img := NewFromString("hub/ns/name:1") assert.Equal(t, "hub", img.Registry) assert.Equal(t, "ns/name", img.Name) assert.Equal(t, "1", img.Tag) } func TestImageParsingWithoutTag(t *testing.T) { - img := New("repo/name") + img := NewFromString("repo/name") assert.Equal(t, "", img.Registry) assert.Equal(t, "repo/name", img.Name) - assert.Empty(t, img.Tag) assert.Equal(t, "latest", img.GetTag()) + assert.Equal(t, false, img.HasTag()) + assert.Equal(t, "repo/name:latest", img.String()) } func TestImageLatest(t *testing.T) { - img := New("rocker-build:latest") + img := NewFromString("rocker-build:latest") assert.Equal(t, "", img.Registry, "bag registry value") assert.Equal(t, "rocker-build", img.Name, "bad image name") assert.Equal(t, "latest", img.GetTag(), "bad image tag") } +func TestImageIpRegistry(t *testing.T) { + img := NewFromString("10.0.31.117:5000/golang:1.4") + assert.Equal(t, "127.0.0.1:5000", img.Registry, "bag registry value") + assert.Equal(t, "golang", img.Name, "bad image name") + assert.Equal(t, "1.4", img.GetTag(), "bad image tag") +} + func TestImageIsSameKind(t *testing.T) { - assert.True(t, New("rocker-build").IsSameKind(*New("rocker-build"))) - assert.True(t, New("rocker-build:latest").IsSameKind(*New("rocker-build:latest"))) - assert.True(t, New("rocker-build").IsSameKind(*New("rocker-build:1.2.1"))) - assert.True(t, New("rocker-build:1.2.1").IsSameKind(*New("rocker-build:1.2.1"))) - assert.True(t, New("grammarly/rocker-build").IsSameKind(*New("grammarly/rocker-build"))) - assert.True(t, New("grammarly/rocker-build:3.1").IsSameKind(*New("grammarly/rocker-build:3.1"))) - assert.True(t, New("grammarly/rocker-build").IsSameKind(*New("grammarly/rocker-build:3.1"))) - assert.True(t, New("grammarly/rocker-build:latest").IsSameKind(*New("grammarly/rocker-build:latest"))) - assert.True(t, New("quay.io/rocker-build").IsSameKind(*New("quay.io/rocker-build"))) - assert.True(t, New("quay.io/rocker-build:latest").IsSameKind(*New("quay.io/rocker-build:latest"))) - assert.True(t, New("quay.io/rocker-build:3.2").IsSameKind(*New("quay.io/rocker-build:3.2"))) - assert.True(t, New("quay.io/rocker-build").IsSameKind(*New("quay.io/rocker-build:3.2"))) - assert.True(t, New("quay.io/grammarly/rocker-build").IsSameKind(*New("quay.io/grammarly/rocker-build"))) - assert.True(t, New("quay.io/grammarly/rocker-build:latest").IsSameKind(*New("quay.io/grammarly/rocker-build:latest"))) - assert.True(t, New("quay.io/grammarly/rocker-build:1.2.1").IsSameKind(*New("quay.io/grammarly/rocker-build:1.2.1"))) - assert.True(t, New("quay.io/grammarly/rocker-build").IsSameKind(*New("quay.io/grammarly/rocker-build:1.2.1"))) - - assert.False(t, New("rocker-build").IsSameKind(*New("rocker-build2"))) - assert.False(t, New("rocker-build").IsSameKind(*New("rocker-compose"))) - assert.False(t, New("rocker-build").IsSameKind(*New("grammarly/rocker-build"))) - assert.False(t, New("rocker-build").IsSameKind(*New("quay.io/rocker-build"))) - assert.False(t, New("rocker-build").IsSameKind(*New("quay.io/grammarly/rocker-build"))) + assert.True(t, NewFromString("rocker-build").IsSameKind(*NewFromString("rocker-build"))) + assert.True(t, NewFromString("rocker-build:latest").IsSameKind(*NewFromString("rocker-build:latest"))) + assert.True(t, NewFromString("rocker-build").IsSameKind(*NewFromString("rocker-build:1.2.1"))) + assert.True(t, NewFromString("rocker-build:1.2.1").IsSameKind(*NewFromString("rocker-build:1.2.1"))) + assert.True(t, NewFromString("grammarly/rocker-build").IsSameKind(*NewFromString("grammarly/rocker-build"))) + assert.True(t, NewFromString("grammarly/rocker-build:3.1").IsSameKind(*NewFromString("grammarly/rocker-build:3.1"))) + assert.True(t, NewFromString("grammarly/rocker-build").IsSameKind(*NewFromString("grammarly/rocker-build:3.1"))) + assert.True(t, NewFromString("grammarly/rocker-build:latest").IsSameKind(*NewFromString("grammarly/rocker-build:latest"))) + assert.True(t, NewFromString("quay.io/rocker-build").IsSameKind(*NewFromString("quay.io/rocker-build"))) + assert.True(t, NewFromString("quay.io/rocker-build:latest").IsSameKind(*NewFromString("quay.io/rocker-build:latest"))) + assert.True(t, NewFromString("quay.io/rocker-build:3.2").IsSameKind(*NewFromString("quay.io/rocker-build:3.2"))) + assert.True(t, NewFromString("quay.io/rocker-build").IsSameKind(*NewFromString("quay.io/rocker-build:3.2"))) + assert.True(t, NewFromString("quay.io/grammarly/rocker-build").IsSameKind(*NewFromString("quay.io/grammarly/rocker-build"))) + assert.True(t, NewFromString("quay.io/grammarly/rocker-build:latest").IsSameKind(*NewFromString("quay.io/grammarly/rocker-build:latest"))) + assert.True(t, NewFromString("quay.io/grammarly/rocker-build:1.2.1").IsSameKind(*NewFromString("quay.io/grammarly/rocker-build:1.2.1"))) + assert.True(t, NewFromString("quay.io/grammarly/rocker-build").IsSameKind(*NewFromString("quay.io/grammarly/rocker-build:1.2.1"))) + + assert.False(t, NewFromString("rocker-build").IsSameKind(*NewFromString("rocker-build2"))) + assert.False(t, NewFromString("rocker-build").IsSameKind(*NewFromString("rocker-compose"))) + assert.False(t, NewFromString("rocker-build").IsSameKind(*NewFromString("grammarly/rocker-build"))) + assert.False(t, NewFromString("rocker-build").IsSameKind(*NewFromString("quay.io/rocker-build"))) + assert.False(t, NewFromString("rocker-build").IsSameKind(*NewFromString("quay.io/grammarly/rocker-build"))) } func TestTagsGetOld(t *testing.T) { tags := Tags{ Items: []*Tag{ - &Tag{"1", *New("hub/ns/name:1"), time.Unix(1, 0).Unix()}, - &Tag{"3", *New("hub/ns/name:3"), time.Unix(3, 0).Unix()}, - &Tag{"2", *New("hub/ns/name:2"), time.Unix(2, 0).Unix()}, - &Tag{"5", *New("hub/ns/name:5"), time.Unix(5, 0).Unix()}, - &Tag{"4", *New("hub/ns/name:4"), time.Unix(4, 0).Unix()}, + &Tag{"1", *NewFromString("hub/ns/name:1"), time.Unix(1, 0).Unix()}, + &Tag{"3", *NewFromString("hub/ns/name:3"), time.Unix(3, 0).Unix()}, + &Tag{"2", *NewFromString("hub/ns/name:2"), time.Unix(2, 0).Unix()}, + &Tag{"5", *NewFromString("hub/ns/name:5"), time.Unix(5, 0).Unix()}, + &Tag{"4", *NewFromString("hub/ns/name:4"), time.Unix(4, 0).Unix()}, }, } old := tags.GetOld(2) From 94f15c8c5380be702cf3f47f134d047869a0f9b6 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Mon, 21 Sep 2015 17:33:05 +0300 Subject: [PATCH 02/34] vendor add wmark/semver --- vendor/manifest | 5 + vendor/src/github.com/wmark/semver/LICENSE | 68 ++ vendor/src/github.com/wmark/semver/README.md | 86 +++ vendor/src/github.com/wmark/semver/range.go | 196 ++++++ .../src/github.com/wmark/semver/range_test.go | 610 ++++++++++++++++++ vendor/src/github.com/wmark/semver/semver.go | 173 +++++ .../github.com/wmark/semver/semver_test.go | 251 +++++++ 7 files changed, 1389 insertions(+) create mode 100644 vendor/src/github.com/wmark/semver/LICENSE create mode 100644 vendor/src/github.com/wmark/semver/README.md create mode 100644 vendor/src/github.com/wmark/semver/range.go create mode 100644 vendor/src/github.com/wmark/semver/range_test.go create mode 100644 vendor/src/github.com/wmark/semver/semver.go create mode 100644 vendor/src/github.com/wmark/semver/semver_test.go diff --git a/vendor/manifest b/vendor/manifest index aa5365e..20987cf 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -119,6 +119,11 @@ "branch": "master", "path": "/src/rocker/imagename" }, + { + "importpath": "github.com/wmark/semver", + "repository": "https://github.com/wmark/semver", + "revision": "461c06b538be53cc0339815001a398e29ace025b", + "branch": "master" } ] } \ No newline at end of file diff --git a/vendor/src/github.com/wmark/semver/LICENSE b/vendor/src/github.com/wmark/semver/LICENSE new file mode 100644 index 0000000..4f79d14 --- /dev/null +++ b/vendor/src/github.com/wmark/semver/LICENSE @@ -0,0 +1,68 @@ +Copyright © 2014 Mark Kubacki. +All rights reserved. + +"You" or "Your" means an individual or a legal entity exercising rights +under this license. For legal entities, "You" or "Your" includes any +entity which controls, is controlled by, or is under common control with, +You, where "control" means (a) the power, direct or indirect, to cause +the direction or management of such entity, whether by contract or +otherwise, or (b) ownership of fifty percent (50%) or more of the +outstanding shares or beneficial ownership of such entity. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + + * Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + + * This license will not be construed as creating an agency, +partnership, joint venture, or any other form of legal association +between or among You, the copyright holder, or any contributor, and +You will not represent to the contrary, whether expressly, by +implication, appearance, or otherwise. + + * You do not conduct, endorse, promote, assist in or authorize +offenses against privacy. Nor do you train how to do so, or create, +alter, install, maintain or operate tools with the effect of, intention +to or designed for the purpose of decreasing privacy, with the exception +of a communication you are a sender or receiver thereof, or overtly +monitoring your own premises, or to the absolute unavoidable minimum if +being a service provider for the sender or receiver and having a specific +prior written permission of the sender or receiver. + Offenses against privacy are, but not limited to: + - mass surveillance, "dragnet" surveillance, warrantless wiretapping, + wiretapping for preventive purposes, monitoring or interception + of communication + - collection of communication metadata, collection of or accessing + stored communication, obtaining communications information, + - tampering with private communications, + - decryption of messages, + - aural transfer, + - or storing, transmitting or accessing metadata and/or messages + obtained by a means decreasing and/or offending privacy as expressed + above. +This even applies to telephonic, wired and electronic communication, +including stored messages, even if no reasonable expection of privacy +can be assumed, and even if it is deemed being "lawful" to do so. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/vendor/src/github.com/wmark/semver/README.md b/vendor/src/github.com/wmark/semver/README.md new file mode 100644 index 0000000..3f95e99 --- /dev/null +++ b/vendor/src/github.com/wmark/semver/README.md @@ -0,0 +1,86 @@ +Semantic Versioning for Golang +============================== + +[![Build Status](https://drone.io/github.com/wmark/semver/status.png)](https://drone.io/github.com/wmark/semver/latest) +[![Coverage Status](https://coveralls.io/repos/wmark/semver/badge.png?branch=master)](https://coveralls.io/r/wmark/semver?branch=master) +[![GoDoc](https://godoc.org/github.com/wmark/semver?status.png)](https://godoc.org/github.com/wmark/semver) + +A library for parsing and processing of *Versions* and *Ranges* in: + +* [Semantic Versioning](http://semver.org/) (semver) v2.0.0 notation + * used by npmjs.org, pypi.org… +* Gentoo's ebuild format +* NPM + +Licensed under a [BSD-style license](LICENSE). + +Usage +----- +```bash +$ go get -v github.com/wmark/semver +``` + +```go +import github.com/wmark/semver + +v1, err := semver.NewVersion("1.2.3-beta") +v2, err := semver.NewVersion("2.0.0-alpha20140805.456-rc3+build1800") +v1.Less(v2) + +r1, err := NewRange("~1.2") +r1.Contains(v1) // true +r1.IsSatisfiedBy(v1) // false +``` + +Also check the [GoDocs](http://godoc.org/github.com/wmark/semver) +and [Gentoo Linux Ebuild File Format](http://devmanual.gentoo.org/ebuild-writing/file-format/), +[Gentoo's notation of dependencies](http://devmanual.gentoo.org/general-concepts/dependencies/). + +Please Note +----------- + +It is, ordered from lowest to highest: + + alpha < beta < pre < rc < (no release type/»common«) < p + +Therefore it is: + + Version("1.0.0-pre1") ≙ Version("1.0.0-1") < Version("1.0.0") < Version("1.0.0-p1") + +… because the SemVer specification says: + + 9. A pre-release version MAY be denoted by appending a hyphen and a series of + dot separated identifiers immediately following the patch version. […] + +Most *NodeJS* authors write **~1.2.3** where **>=1.2.3** would fit better. +*~1.2.3* is ```Range(">=1.2.3 <1.3.0")``` and excludes versions such as *1.4.0*, +which almost always work. + +Contribute +---------- + +Please open issues with minimal examples of what is going wrong. For example: + + Mark, this does not work but I feel that it should: + ```golang + v, err := semver.Version("17.1o0") + # yields an error + # I expected 17.100 + ``` + +Please write a test case with your expectations, if you ask for a new feature. + + I'd like **semver.Range** to support interface **expvar.Var**, like this: + ```golang + Convey("Range implements interface's expvar.Var…", t, func() { + r, _ := NewRange("1.2.3") + + Convey("String()", func() { + So(r.String(), ShouldEqual, "1.2.3") + }) + }) + ``` + +Pull requests are welcome. +Please add your name and email address to file *AUTHORS* and/or *CONTRIBUTORS*. +Thanks! diff --git a/vendor/src/github.com/wmark/semver/range.go b/vendor/src/github.com/wmark/semver/range.go new file mode 100644 index 0000000..001de65 --- /dev/null +++ b/vendor/src/github.com/wmark/semver/range.go @@ -0,0 +1,196 @@ +// Copyright 2014 The Semver Package Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package semver + +import ( + "errors" + "strings" +) + +type Range struct { + lower *Version + equalsLower bool + upper *Version + equalsUpper bool +} + +func NewRange(str string) (*Range, error) { + if str == "*" || str == "x" || str == "" { + // an empty Range contains everything + return new(Range), nil + } + if strings.HasSuffix(str, ".x") || strings.HasSuffix(str, ".*") { + str = strings.TrimRight(str, ".x*") + } + if str[0] == '^' || str[0] == '~' { + return newRangeByShortcut(str) + } + + isNaturalRange := strings.ContainsAny(str, " –") + if !isNaturalRange { + switch strings.Count(str, ".") { + case 1: + return newRangeByShortcut("~" + str) + case 0: + return newRangeByShortcut("^" + str) + } + } + + vr := new(Range) + if !isNaturalRange { + err := vr.setBound(str) + return vr, err + } + + for _, delimiter := range []string{" - ", " – ", "–", " "} { + if strings.Contains(str, delimiter) { + parts := strings.Split(str, delimiter) + if len(parts) == 2 { + if parts[0][0] == '>' { + vr.setBound(parts[0]) + } else { + vr.setBound(">=" + parts[0]) + } + if parts[1][0] == '<' { + vr.setBound(parts[1]) + } else { + vr.setBound("<=" + parts[1]) + } + return vr, nil + } else { + return nil, errors.New("Range contains more than two elements.") + } + } + } + + return nil, nil +} + +func (r *Range) setBound(str string) error { + t := strings.TrimLeft(str, "~=v<>^") + num, err := NewVersion(t) + if err != nil { + return err + } + + equalOk := strings.Contains(str, "=") + if !strings.Contains(str, ">") { + r.equalsUpper = equalOk + r.upper = num + } + if !strings.Contains(str, "<") { + r.equalsLower = equalOk + r.lower = num + } + + return nil +} + +func newRangeByShortcut(str string) (*Range, error) { + t := strings.TrimLeft(str, "~^") + num, err := NewVersion(t) + if err != nil { + return nil, err + } + if strings.HasPrefix(t, "0.0.") { + return NewRange(t) + } + + r := new(Range) + r.lower = num + r.equalsLower = true + r.upper = new(Version) + + switch { + case strings.HasPrefix(t, "0."): + r.upper.version[0] = r.lower.version[0] + r.upper.version[1] = r.lower.version[1] + 1 + case str[0] == '^' || !strings.ContainsAny(t, "."): + r.upper.version[0] = r.lower.version[0] + 1 + case str[0] == '~': + r.upper.version[0] = r.lower.version[0] + r.upper.version[1] = r.lower.version[1] + 1 + default: + return nil, errors.New("Unsupported shortcut notation for Range.") + } + + return r, nil +} + +// Checks if a Version falls into a Range. +func (r *Range) Contains(v *Version) bool { + if v == nil { + return false + } + + if r.upper == r.lower { + return r.lower.LimitedEqual(v) + } + + return r.satisfiesLowerBound(v) && r.satisfiesUpperBound(v) +} + +// Works like Contains, but rejects pre-releases if neither of the bounds is a pre-release. +// Use this in the context of pulling in packages because it follows the spirit of §9 SemVer. +// Also see https://github.com/npm/node-semver/issues/64 +func (r *Range) IsSatisfiedBy(v *Version) bool { + if !r.Contains(v) { + return false + } + if v.IsAPreRelease() { + if r.lower != nil && r.lower.IsAPreRelease() && r.lower.sharesPrefixWith(v) { + return true + } + if r.upper != nil && r.upper.IsAPreRelease() && r.upper.sharesPrefixWith(v) { + return true + } + return false + } + return true +} + +func (r *Range) satisfiesLowerBound(v *Version) bool { + if r.lower == nil { + return true + } + + equal := r.lower.LimitedEqual(v) + if r.equalsLower && equal { + return true + } + + return r.lower.limitedLess(v) && !equal +} + +func (r *Range) satisfiesUpperBound(v *Version) bool { + if r.upper == nil { + return true + } + + equal := r.upper.LimitedEqual(v) + if r.equalsUpper && equal { + return true + } + + if !r.equalsUpper && r.upper.version[idxReleaseType] == common { + equal = r.upper.sharesPrefixWith(v) + } + + return v.limitedLess(r.upper) && !equal +} + +// Convenience function for former NodeJS developers. +func Satisfies(aVersion, aRange string) (bool, error) { + v, err := NewVersion(aVersion) + if err != nil { + return false, err + } + r, err := NewRange(aRange) + if err != nil { + return false, err + } + + return r.IsSatisfiedBy(v), nil +} diff --git a/vendor/src/github.com/wmark/semver/range_test.go b/vendor/src/github.com/wmark/semver/range_test.go new file mode 100644 index 0000000..fb5fcfd --- /dev/null +++ b/vendor/src/github.com/wmark/semver/range_test.go @@ -0,0 +1,610 @@ +// Copyright 2014 The Semver Package Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package semver + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func hasLowerBound(aRange interface{}, aVersion ...interface{}) string { + a := aRange.(*Range) + if s := ShouldResemble(a.lower, aVersion[0]); s != "" { + return s + } + return ShouldBeTrue(a.equalsLower) +} + +func isLeftClosedBy(aRange interface{}, aVersion ...interface{}) string { + a := aRange.(*Range) + if s := ShouldResemble(a.lower, aVersion[0]); s != "" { + return s + } + return ShouldBeFalse(a.equalsLower) +} + +func hasUpperBound(aRange interface{}, aVersion ...interface{}) string { + a := aRange.(*Range) + if s := ShouldResemble(a.upper, aVersion[0]); s != "" { + return s + } + return ShouldBeTrue(a.equalsUpper) +} + +func isRightClosedBy(aRange interface{}, aVersion ...interface{}) string { + a := aRange.(*Range) + if s := ShouldResemble(a.upper, aVersion[0]); s != "" { + return s + } + return ShouldBeFalse(a.equalsUpper) +} + +func testIfResembles(actual, expected *Range) { + So(actual.lower, ShouldResemble, expected.lower) + So(actual.equalsLower, ShouldEqual, expected.equalsLower) + So(actual.upper, ShouldResemble, expected.upper) + So(actual.equalsUpper, ShouldEqual, expected.equalsUpper) +} + +func shouldContain(aRange interface{}, aVersion ...interface{}) string { + a := aRange.(*Range) + for _, version := range aVersion { + v := version.(string) + ver, err := NewVersion(v) + if err != nil { + return err.Error() + } + if s := ShouldBeTrue(a.Contains(ver)); s != "" { + return v + " is not in Range" + } + } + return "" +} + +func shouldNotContain(aRange interface{}, aVersion ...interface{}) string { + a := aRange.(*Range) + for _, version := range aVersion { + v := version.(string) + ver, err := NewVersion(v) + if err != nil { + return err.Error() + } + if s := ShouldBeFalse(a.Contains(ver)); s != "" { + return v + " is in Range" + } + } + return "" +} + +func TestRangeConstruction(t *testing.T) { + + Convey("version 1.2.3 should be part of…", t, func() { + ver, _ := NewVersion("1.2.3") + + Convey("specific Range 1.2.3", func() { + verRange, _ := NewRange("1.2.3") + So(verRange.lower, ShouldResemble, ver) + }) + }) + + v100, _ := NewVersion("1.0.0") + v120, _ := NewVersion("1.2.0") + v123, _ := NewVersion("1.2.3") + v130, _ := NewVersion("1.3.0") + v200, _ := NewVersion("2.0.0") + + Convey("Range >=1.2.3 <=1.3.0…", t, func() { + verRange, err := NewRange(">=1.2.3 <=1.3.0") + So(err, ShouldBeNil) + if err != nil { + return + } + + Convey("has lower bound >=1.2.3", func() { + So(verRange, hasLowerBound, v123) + }) + + Convey("has upper bound <=1.3.0", func() { + So(verRange, hasUpperBound, v130) + }) + }) + + Convey("Range >1.2.3 <1.3.0…", t, func() { + verRange, err := NewRange(">1.2.3 <1.3.0") + So(err, ShouldBeNil) + if err != nil { + return + } + + Convey("has lower bound >1.2.3", func() { + So(verRange, isLeftClosedBy, v123) + }) + + Convey("has upper bound <1.3.0", func() { + So(verRange, isRightClosedBy, v130) + }) + }) + + Convey("Range 1.2.3 - 1.3.0…", t, func() { + verRange, err := NewRange("1.2.3 - 1.3.0") + So(err, ShouldBeNil) + if err != nil { + return + } + + Convey("has lower bound >=1.2.3", func() { + So(verRange, hasLowerBound, v123) + }) + + Convey("has upper bound <=1.3.0", func() { + So(verRange, hasUpperBound, v130) + }) + }) + + Convey("Range ~1.2.3 equals: >=1.2.3 <1.3.0", t, func() { + verRange, err := NewRange("~1.2.3") + So(err, ShouldBeNil) + if err != nil { + return + } + + Convey("has lower bound >=1.2.3", func() { + So(verRange, hasLowerBound, v123) + }) + + Convey("has upper bound <1.3.0", func() { + So(verRange, isRightClosedBy, v130) + }) + }) + + Convey("Range ^1.2.3 equals: >=1.2.3 <2.0.0", t, func() { + verRange, err := NewRange("^1.2.3") + So(err, ShouldBeNil) + if err != nil { + return + } + + Convey("has lower bound >=1.2.3", func() { + So(verRange, hasLowerBound, v123) + }) + + Convey("has upper bound <2.0.0", func() { + So(verRange, isRightClosedBy, v200) + }) + }) + + Convey("Range ~1.2 equals: >=1.2.0 <1.3.0", t, func() { + verRange, err := NewRange("~1.2") + So(err, ShouldBeNil) + if err != nil { + return + } + + Convey("has lower bound >=1.2.0", func() { + So(verRange, hasLowerBound, v120) + }) + + Convey("has upper bound <1.3.0", func() { + So(verRange, isRightClosedBy, v130) + }) + }) + + Convey("Range ^1.2 equals: >=1.2.0 <2.0.0", t, func() { + verRange, err := NewRange("^1.2") + So(err, ShouldBeNil) + if err != nil { + return + } + + Convey("has lower bound >=1.2.0", func() { + So(verRange, hasLowerBound, v120) + }) + + Convey("has upper bound <2.0.0", func() { + So(verRange, isRightClosedBy, v200) + }) + }) + + Convey("Ranges ^1 and ~1 equal: >=1.0.0 <2.0.0", t, func() { + r1, err := NewRange("^1") + So(err, ShouldBeNil) + if err != nil { + return + } + r2, err := NewRange("^1") + So(err, ShouldBeNil) + if err != nil { + return + } + + Convey("has lower bound >=1.0.0", func() { + So(r1, hasLowerBound, v100) + So(r2, hasLowerBound, v100) + }) + + Convey("has upper bound <2.0.0", func() { + So(r1, isRightClosedBy, v200) + So(r2, isRightClosedBy, v200) + }) + }) + + Convey("Given .x and .* notations", t, func() { + refRange, _ := NewRange("^1") + + Convey("1.x equals ^1", func() { + r1, err := NewRange("1.x") + So(err, ShouldBeNil) + if err != nil { + return + } + testIfResembles(r1, refRange) + }) + + Convey("1.* equals ^1", func() { + r1, err := NewRange("1.*") + So(err, ShouldBeNil) + if err != nil { + return + } + testIfResembles(r1, refRange) + }) + + Convey("1 equals ^1", func() { + r1, err := NewRange("1") + So(err, ShouldBeNil) + if err != nil { + return + } + testIfResembles(r1, refRange) + }) + + smallRange, _ := NewRange("~1.2") + + Convey("1.2.x equals ~1.2", func() { + r1, err := NewRange("1.2.x") + So(err, ShouldBeNil) + if err != nil { + return + } + testIfResembles(r1, smallRange) + }) + + Convey("1.2.* equals ~1.2", func() { + r1, err := NewRange("1.2.*") + So(err, ShouldBeNil) + if err != nil { + return + } + testIfResembles(r1, smallRange) + }) + + Convey("1.2 equals ~1.2", func() { + r1, err := NewRange("1.2") + So(err, ShouldBeNil) + if err != nil { + return + } + testIfResembles(r1, smallRange) + }) + }) + + Convey("Notations for 'any'", t, func() { + refRange := new(Range) + + Convey("'x'", func() { + r1, err := NewRange("x") + So(err, ShouldBeNil) + if err != nil { + return + } + testIfResembles(r1, refRange) + }) + + Convey("'*'", func() { + r1, err := NewRange("*") + So(err, ShouldBeNil) + if err != nil { + return + } + testIfResembles(r1, refRange) + }) + + Convey("'' (empty string)", func() { + r1, err := NewRange("") + So(err, ShouldBeNil) + if err != nil { + return + } + testIfResembles(r1, refRange) + }) + }) + + // now come fringe cases + Convey("Range ^0.1.3 and ~0.1.3 equal: >=0.1.3 <0.2.0", t, func() { + r1, err := NewRange("^0.1.3") + So(err, ShouldBeNil) + if err != nil { + return + } + r2, err := NewRange("~0.1.3") + So(err, ShouldBeNil) + if err != nil { + return + } + v013, _ := NewVersion("0.1.3") + v020, _ := NewVersion("0.2.0") + + Convey("have lower bound >=0.1.3", func() { + So(r1, hasLowerBound, v013) + So(r2, hasLowerBound, v013) + }) + + Convey("have upper bound <0.2.0", func() { + So(r1, isRightClosedBy, v020) + So(r2, isRightClosedBy, v020) + }) + }) + + Convey("Range ^0.0.2 and ~0.0.2 are 0.2.0", t, func() { + r1, err := NewRange("^0.0.2") + So(err, ShouldBeNil) + if err != nil { + return + } + r2, err := NewRange("~0.0.2") + So(err, ShouldBeNil) + if err != nil { + return + } + r002, _ := NewRange("0.0.2") + + Convey("have the same bounds as 0.2.0", func() { + testIfResembles(r1, r002) + testIfResembles(r2, r002) + }) + }) +} + +func TestSingleBound(t *testing.T) { + + Convey("Given a specific version as Range…", t, func() { + Convey("1.2.3…", func() { + verRange, _ := NewRange("1.2.3") + + Convey("reject Version 1.2.4", func() { + So(verRange, shouldNotContain, "1.2.4") + }) + + Convey("accept Version 1.2.3", func() { + So(verRange, shouldContain, "1.2.3") + }) + + Convey("accept Version 1.2.3+build2014 (ignore build)", func() { + So(verRange, shouldContain, "1.2.3+build2014") + }) + + Convey("accept Version 1.2.3-p1 (patch levels are reasonably equal)", func() { + So(verRange, shouldContain, "1.2.3-p1") + }) + + Convey("reject pre-releases like 1.2.3-alpha", func() { + So(verRange, shouldNotContain, "1.2.3-alpha") + }) + }) + + Convey("1.2.3-alpha20…", func() { + verRange, _ := NewRange("1.2.3-alpha20") + + Convey("accept Version 1.2.3-alpha20", func() { + So(verRange, shouldContain, "1.2.3-alpha20") + }) + Convey("reject Version 1.2.3-alpha5", func() { + So(verRange, shouldNotContain, "1.2.3-alpha5") + }) + Convey("reject Version 1.2.3-beta", func() { + So(verRange, shouldNotContain, "1.2.3-beta") + }) + Convey("reject Version 1.2.3", func() { + So(verRange, shouldNotContain, "1.2.3") + }) + }) + }) + + Convey("Given the lower bound >1.2.3", t, func() { + verRange, _ := NewRange(">1.2.3") + + Convey("reject Version 1.2.3", func() { + So(verRange, shouldNotContain, "1.2.3") + }) + + Convey("reject Version 1.2.3-p1 (ignore release/pre-release)", func() { + So(verRange, shouldNotContain, "1.2.3-p1") + }) + + Convey("reject Version 1.2.3+build2014 (ignore build)", func() { + So(verRange, shouldNotContain, "1.2.3+build2014") + }) + + Convey("accept Versions…", func() { + Convey("1.2.4", func() { + So(verRange, shouldContain, "1.2.4") + }) + Convey("1.3.0", func() { + So(verRange, shouldContain, "1.3.0") + }) + Convey("2.0.0", func() { + So(verRange, shouldContain, "2.0.0") + }) + }) + }) + + Convey("A lower bound >=1.2.3…", t, func() { + verRange, _ := NewRange(">=1.2.3") + + Convey("should contain Version 1.2.3", func() { + So(verRange, shouldContain, "1.2.3") + }) + + Convey("should contain Version 1.2.3-p1", func() { + So(verRange, shouldContain, "1.2.3-p1") + }) + + Convey("but NOT contain 1.2.3-rc (exclude pre-release)", func() { + So(verRange, shouldNotContain, "1.2.3-rc") + }) + }) + + Convey("Over-specific lower bounds >=1.2.3-alpha4…", t, func() { + verRange, _ := NewRange(">=1.2.3-alpha4") + + Convey("should contain Version 1.2.4", func() { + So(verRange, shouldContain, "1.2.4") + }) + + Convey("should contain Version 1.2.3-alpha4", func() { + So(verRange, shouldContain, "1.2.3-alpha4") + }) + + Convey("but NOT contain 1.2.3-alpha", func() { + So(verRange, shouldNotContain, "1.2.3-alpha") + }) + }) + + Convey("Upper bounds such as <1.2.3…", t, func() { + verRange, _ := NewRange("<1.2.3") + + Convey("reject Version 1.2.3", func() { + So(verRange, shouldNotContain, "1.2.3") + }) + + Convey("reject Version 1.2.3-alpha3 (ignore release/pre-release)", func() { + So(verRange, shouldNotContain, "1.2.3-alpha3") + }) + + Convey("reject Version 1.2.3+build2014 (ignore build)", func() { + So(verRange, shouldNotContain, "1.2.3+build2014") + }) + + Convey("accept Versions…", func() { + Convey("1.2.0", func() { + So(verRange, shouldContain, "1.2.0") + }) + Convey("1.1.0", func() { + So(verRange, shouldContain, "1.1.0") + }) + Convey("1.0.0", func() { + So(verRange, shouldContain, "1.0.0") + }) + }) + }) + + Convey("An over-specific upper bound <1.2.3-beta20…", t, func() { + verRange, _ := NewRange("<1.2.3-beta20") + + Convey("reject Version 1.2.3-beta20", func() { + So(verRange, shouldNotContain, "1.2.3-beta20") + }) + + Convey("reject Version 1.2.3-beta20-alpha3 (ignore release specifier)", func() { + So(verRange, shouldNotContain, "1.2.3-beta20-alpha3") + }) + + Convey("reject Version 1.2.3-beta20+build2014 (ignore build)", func() { + So(verRange, shouldNotContain, "1.2.3-beta20+build2014") + }) + + Convey("accept Versions…", func() { + Convey("1.2.3-beta19", func() { + So(verRange, shouldContain, "1.2.3-beta19") + }) + Convey("1.2.3-alpha", func() { + So(verRange, shouldContain, "1.2.3-alpha") + }) + Convey("1.2.0", func() { + So(verRange, shouldContain, "1.2.0") + }) + }) + }) + + Convey("An upper bound with equality <=1.2.3…", t, func() { + verRange, _ := NewRange("<=1.2.3") + + Convey("should contain Version 1.2.3", func() { + So(verRange, shouldContain, "1.2.3") + }) + + Convey("should contain Version 1.2.3-p1", func() { + So(verRange, shouldContain, "1.2.3-p1") + }) + + Convey("and, that's new, contain 1.2.3-rc (INCLUDE pre-release)", func() { + So(verRange, shouldContain, "1.2.3-rc") + }) + }) + +} + +func TestSatisfies(t *testing.T) { + + Convey("Convenience function 'Satisfies'", t, func() { + + Convey("works with valid input", func() { + t, _ := Satisfies("1.2.3", "^1.2.2") + So(t, ShouldBeTrue) + t, _ = Satisfies("1.2.3-2", "^1.2.3-1") + So(t, ShouldBeTrue) + }) + + Convey("yields an error on invalid Version", func() { + t, err := Satisfies("1.2.3.4.5.6", "^1.2.2") + So(t, ShouldBeFalse) + So(err, ShouldNotBeNil) + }) + + Convey("yields an error on invalid Range", func() { + t, err := Satisfies("1.2.3", "^1.2.2/1.2.5") + So(t, ShouldBeFalse) + So(err, ShouldNotBeNil) + }) + }) + + Convey("Range.IsSatisfiedBy…", t, func() { + + Convey("rejects pre-releases", func() { + t, _ := Satisfies("1.2.3-1", "^1.2.2") + So(t, ShouldBeFalse) + t, _ = Satisfies("1.2.4-1", "^1.2.2-1") + So(t, ShouldBeFalse) + t, _ = Satisfies("1.2.3-1", "<1.2.3") + So(t, ShouldBeFalse) + }) + + Convey("accepts pre-releases for a pre-release upper bound with the same prefix", func() { + t, _ := Satisfies("1.2.3-1", "<1.2.3-1") + So(t, ShouldBeFalse) + t, _ = Satisfies("1.2.3-1", "<1.2.3-2") + So(t, ShouldBeTrue) + t, _ = Satisfies("1.2.3-1", "<=1.2.3-1") + So(t, ShouldBeTrue) + }) + + Convey("accepts pre-releases for a pre-release lower bound with the same prefix", func() { + t, _ := Satisfies("1.2.3-1", ">1.2.3-1") + So(t, ShouldBeFalse) + t, _ = Satisfies("1.2.3-2", ">1.2.3-1") + So(t, ShouldBeTrue) + t, _ = Satisfies("1.2.3-1", ">=1.2.3-1") + So(t, ShouldBeTrue) + }) + }) + + Convey("Test the examples found in README file.", t, func() { + v, _ := NewVersion("1.2.3-beta") + r, _ := NewRange("~1.2") + So(r.Contains(v), ShouldBeTrue) + So(r.IsSatisfiedBy(v), ShouldBeFalse) + }) +} diff --git a/vendor/src/github.com/wmark/semver/semver.go b/vendor/src/github.com/wmark/semver/semver.go new file mode 100644 index 0000000..0806843 --- /dev/null +++ b/vendor/src/github.com/wmark/semver/semver.go @@ -0,0 +1,173 @@ +// Copyright 2014 The Semver Package Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package semver + +import ( + "errors" + "regexp" + "strconv" + "strings" +) + +type dotDelimitedNumber []int + +func newDotDelimitedNumber(str string) (dotDelimitedNumber, error) { + strSequence := strings.Split(str, ".") + if len(strSequence) > 4 { + return nil, errors.New("too much columns on that number") + } + numSequence := make(dotDelimitedNumber, 0, len(strSequence)) + for _, s := range strSequence { + i, err := strconv.Atoi(s) + if err != nil { + return numSequence, err + } + numSequence = append(numSequence, i) + } + return numSequence, nil +} + +// alpha = -4, beta = -3, pre = -2, rc = -1, common = 0, patch = 1 +const ( + alpha = iota - 4 + beta + pre + rc + common + patch +) + +const ( + idxReleaseType = 4 + idxRelease = 5 + idxSpecifierType = 9 + idxSpecifier = 10 +) + +var releaseDesc = map[int]string{ + alpha: "alpha", + beta: "beta", + pre: "pre", + rc: "rc", + patch: "p", +} + +var releaseValue = map[string]int{ + "alpha": alpha, + "beta": beta, + "pre": pre, + "": pre, + "rc": rc, + "p": patch, +} + +var verRegexp = regexp.MustCompile(`^(\d+(?:\.\d+){0,3})(?:([-_]alpha|[-_]beta|[-_]pre|[-_]rc|[-_]p|-)(\d+(?:\.\d+){0,3})?)?(?:([-_]alpha|[-_]beta|[-_]pre|[-_]rc|[-_]p|-)(\d+(?:\.\d+){0,3})?)?(?:(\+build)(\d*))?$`) + +type Version struct { + // 0–3: version, 4: releaseType, 5–8: releaseVer, 9: releaseSpecifier, 10–14: specifier + version [14]int + build int +} + +func NewVersion(str string) (*Version, error) { + allMatches := verRegexp.FindAllStringSubmatch(str, -1) + if allMatches == nil { + return nil, errors.New("Given string does not resemble a Version.") + } + ver := new(Version) + m := allMatches[0] + + // version + n, err := newDotDelimitedNumber(m[1]) + if err != nil { + return nil, err + } + copy(ver.version[:], n) + + // release + if m[2] != "" { + ver.version[idxReleaseType] = releaseValue[strings.Trim(m[2], "-_")] + } + if m[3] != "" { + n, err := newDotDelimitedNumber(m[3]) + if err != nil { + return nil, err + } + copy(ver.version[idxRelease:], n) + } + + // release specifier + if m[4] != "" { + ver.version[idxSpecifierType] = releaseValue[strings.Trim(m[4], "-_")] + } + if m[5] != "" { + n, err := newDotDelimitedNumber(m[5]) + if err != nil { + return nil, err + } + copy(ver.version[idxSpecifier:], n) + } + + // build + if m[7] != "" { + i, err := strconv.Atoi(m[7]) + if err != nil { + return nil, err + } + ver.build = i + } + + return ver, nil +} + +// Returns sign(a - b). +func signDelta(a, b [14]int, cutoffIdx int) int8 { + for i := range a { + if i >= cutoffIdx { + return 0 + } + if a[i] < b[i] { + return -1 + } else if a[i] > b[i] { + return 1 + } + } + return 0 +} + +// Convenience function for sorting. +func (t *Version) Less(o *Version) bool { + sd := signDelta(t.version, o.version, 15) + return sd < 0 || (sd == 0 && t.build < o.build) +} + +// Limited to version, (pre-)release type and (pre-)release version. +// Commutative. +func (t *Version) limitedLess(o *Version) bool { + return signDelta(t.version, o.version, idxSpecifierType) < 0 +} + +// Equality limited to version, (pre-)release type and (pre-)release version. +// For example, 1.0.0-pre and 1.0.0-rc are not the same, but +// 1.0.0-beta-pre3 and 1.0.0-beta-pre5 are equal. +// Permits 'patch levels', regarding 1.0.0 equal to 1.0.0-p1. +// Non-commutative: 1.0.0-p1 does not equal 1.0.0! +func (t *Version) LimitedEqual(o *Version) bool { + if t.version[idxReleaseType] == common && o.version[idxReleaseType] > common { + return t.sharesPrefixWith(o) + } + return signDelta(t.version, o.version, idxSpecifierType) == 0 +} + +// Use this to exclude pre-releases. +func (v *Version) IsAPreRelease() bool { + return v.version[idxReleaseType] < common +} + +// A 'prefix' is the major, minor, patch and revision number. +// For example: 1.2.3.4… +func (t *Version) sharesPrefixWith(o *Version) bool { + return signDelta(t.version, o.version, idxReleaseType) == 0 +} diff --git a/vendor/src/github.com/wmark/semver/semver_test.go b/vendor/src/github.com/wmark/semver/semver_test.go new file mode 100644 index 0000000..382bf4e --- /dev/null +++ b/vendor/src/github.com/wmark/semver/semver_test.go @@ -0,0 +1,251 @@ +// Copyright 2014 The Semver Package Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package semver + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestDotDelimitedNumber(t *testing.T) { + goodVer := "1.3.8" + badVer := "a.b.c" + + Convey("Given an acceptable Version string", t, func() { + v, err := newDotDelimitedNumber(goodVer) + + Convey("convert it correctly", func() { + So(err, ShouldBeNil) + So(len(v), ShouldEqual, 3) + So(v, ShouldResemble, dotDelimitedNumber([]int{1, 3, 8})) + }) + }) + + Convey("Given a mal-formed Version string", t, func() { + v, err := newDotDelimitedNumber(badVer) + + Convey("yield an error", func() { + So(err, ShouldNotBeNil) + }) + + Convey("return an incomplete sequence", func() { + So(len(v), ShouldBeLessThan, 3) + }) + }) +} + +func TestVersion(t *testing.T) { + Convey("Version 1.3.8 should be part of Version…", t, func() { + v := []int{1, 3, 8, 0} + + Convey("1.3.8", func() { + refVer, err := NewVersion("1.3.8") + So(err, ShouldBeNil) + So(refVer.version[:4], ShouldResemble, v) + }) + + Convey("1.3.8+build20140722", func() { + refVer, err := NewVersion("1.3.8+build20140722") + So(err, ShouldBeNil) + So(refVer.version[:4], ShouldResemble, v) + }) + + Convey("1.3.8+build2014", func() { + refVer, err := NewVersion("1.3.8+build2014") + So(err, ShouldBeNil) + So(refVer.version[:4], ShouldResemble, v) + }) + + Convey("1.3.8-alpha", func() { + refVer, err := NewVersion("1.3.8-alpha") + So(err, ShouldBeNil) + So(refVer.version[:4], ShouldResemble, v) + }) + + Convey("1.3.8-beta", func() { + refVer, err := NewVersion("1.3.8-beta") + So(err, ShouldBeNil) + So(refVer.version[:4], ShouldResemble, v) + }) + + Convey("1.3.8-pre", func() { + refVer, err := NewVersion("1.3.8-pre") + So(err, ShouldBeNil) + So(refVer.version[:4], ShouldResemble, v) + }) + + Convey("1.3.8-r3", func() { + refVer, err := NewVersion("1.3.8-p3") + So(err, ShouldBeNil) + So(refVer.version[:4], ShouldResemble, v) + }) + + Convey("1.3.8-3", func() { + refVer, err := NewVersion("1.3.8-3") + So(err, ShouldBeNil) + So(refVer.version[:4], ShouldResemble, v) + }) + + }) + + Convey("Working order between Versions", t, func() { + + Convey("equality", func() { + v1, _ := NewVersion("1.3.8") + v2, _ := NewVersion("1.3.8") + So(v1, ShouldResemble, v2) + }) + + Convey("between different release types", func() { + Convey("1.0.0 < 2.0.0", func() { + v1, _ := NewVersion("1.0.0") + v2, _ := NewVersion("2.0.0") + So(v1.Less(v2), ShouldBeTrue) + So(v2.Less(v1), ShouldBeFalse) + So(v1, ShouldNotResemble, v2) + }) + + Convey("2.2.1 < 2.4.0-beta", func() { + v1, _ := NewVersion("2.2.1") + v2, _ := NewVersion("2.4.0-beta") + So(v1.Less(v2), ShouldBeTrue) + So(v2.Less(v1), ShouldBeFalse) + So(v1, ShouldNotResemble, v2) + }) + + Convey("1.0.0 < 1.0.0-p", func() { + v1, _ := NewVersion("1.0.0") + v2, _ := NewVersion("1.0.0-p") + So(v1.Less(v2), ShouldBeTrue) + So(v2.Less(v1), ShouldBeFalse) + So(v1, ShouldNotResemble, v2) + }) + + Convey("1.0.0-rc < 1.0.0", func() { + v1, _ := NewVersion("1.0.0-rc") + v2, _ := NewVersion("1.0.0") + So(v1.Less(v2), ShouldBeTrue) + So(v1, ShouldNotResemble, v2) + }) + + Convey("1.0.0-pre < 1.0.0-rc", func() { + v1, _ := NewVersion("1.0.0-pre") + v2, _ := NewVersion("1.0.0-rc") + So(v1.Less(v2), ShouldBeTrue) + So(v1, ShouldNotResemble, v2) + }) + + Convey("1.0.0-beta < 1.0.0-pre", func() { + v1, _ := NewVersion("1.0.0-beta") + v2, _ := NewVersion("1.0.0-pre") + So(v1.Less(v2), ShouldBeTrue) + So(v1, ShouldNotResemble, v2) + }) + + Convey("1.0.0-alpha < 1.0.0-beta", func() { + v1, _ := NewVersion("1.0.0-alpha") + v2, _ := NewVersion("1.0.0-beta") + So(v1.Less(v2), ShouldBeTrue) + So(v1, ShouldNotResemble, v2) + }) + }) + + Convey("between same release types", func() { + Convey("1.0.0-p0 < 1.0.0-p1", func() { + v1, _ := NewVersion("1.0.0-p0") + v2, _ := NewVersion("1.0.0-p1") + So(v1.Less(v2), ShouldBeTrue) + So(v1, ShouldNotResemble, v2) + }) + }) + + Convey("with release type specifier", func() { + Convey("1.0.0-rc4-alpha1 < 1.0.0-rc4", func() { + v1, _ := NewVersion("1.0.0-rc4-alpha1") + v2, _ := NewVersion("1.0.0-rc4") + So(v1.Less(v2), ShouldBeTrue) + So(v1, ShouldNotResemble, v2) + }) + }) + + Convey("with builds", func() { + Convey("1.0.0+build14 < 1.0.0+build15", func() { + v1, _ := NewVersion("1.0.0+build14") + v2, _ := NewVersion("1.0.0+build15") + So(v1.Less(v2), ShouldBeTrue) + So(v1, ShouldNotResemble, v2) + }) + + Convey("1.0.0_pre20140722+build14 < 1.0.0_pre20140722+build15", func() { + v1, _ := NewVersion("1.0.0_pre20140722+build14") + v2, _ := NewVersion("1.0.0_pre20140722+build15") + So(v1, ShouldNotResemble, v2) + So(v1.Less(v2), ShouldBeTrue) + }) + }) + + }) + + // see http://devmanual.gentoo.org/ebuild-writing/file-format/ + Convey("Gentoo's example of order works.", t, func() { + v1, _ := NewVersion("1.0.0_alpha_pre") + v2, _ := NewVersion("1.0.0_alpha_rc1") + v3, _ := NewVersion("1.0.0_beta_pre") + v4, _ := NewVersion("1.0.0_beta_p1") + So(v1.version, ShouldResemble, [...]int{1, 0, 0, 0, alpha, 0, 0, 0, 0, pre, 0, 0, 0, 0}) + So(v2.version, ShouldResemble, [...]int{1, 0, 0, 0, alpha, 0, 0, 0, 0, rc, 1, 0, 0, 0}) + So(v3.version, ShouldResemble, [...]int{1, 0, 0, 0, beta, 0, 0, 0, 0, pre, 0, 0, 0, 0}) + So(v4.version, ShouldResemble, [...]int{1, 0, 0, 0, beta, 0, 0, 0, 0, patch, 1, 0, 0, 0}) + + So(v1, ShouldNotResemble, v2) + So(v2, ShouldNotResemble, v3) + So(v3, ShouldNotResemble, v4) + So(v1.Less(v2), ShouldBeTrue) + So(v2.Less(v3), ShouldBeTrue) + So(v3.Less(v4), ShouldBeTrue) + }) + + Convey("Reject too long Versions.", t, func() { + Convey("with surplus digits", func() { + _, err := NewVersion("1.0.0.0.4") + So(err, ShouldNotBeNil) + }) + + Convey("with too long parts", func() { + _, err := NewVersion("100000000000007000000000000000070000000000000.0.0") + So(err, ShouldNotBeNil) + _, err = NewVersion("1.0.0_alpha444444444444444444444444444444444444444") + So(err, ShouldNotBeNil) + _, err = NewVersion("1.0.0_alpha-rc444444444444444444444444444444444444") + So(err, ShouldNotBeNil) + _, err = NewVersion("1.0.0_alpha-rc1+build44444444444444444444444444444") + So(err, ShouldNotBeNil) + }) + }) +} + +func TestVersionOrder(t *testing.T) { + + Convey("Version 1.2.3-alpha4 should be…", t, func() { + v1, _ := NewVersion("1.2.3-alpha4") + + Convey("reasonably less than Version 1.2.3", func() { + v2, _ := NewVersion("1.2.3") + So(v1.limitedLess(v2), ShouldBeTrue) + }) + + Convey("reasonably less than Version 1.2.3-alpha4.0.0.1", func() { + v2, _ := NewVersion("1.2.3-alpha4.0.0.1") + So(v1.limitedLess(v2), ShouldBeTrue) + }) + + Convey("not reasonably less than 1.2.3-alpha4-p5", func() { + v2, _ := NewVersion("1.2.3-alpha4-p5") + So(v1.limitedLess(v2), ShouldBeFalse) + }) + }) + +} From 42a21286f0f4ed03774da0d636dab8f7cce6a540 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Mon, 21 Sep 2015 17:35:25 +0300 Subject: [PATCH 03/34] #10 manual merge of semver-integration branch --- src/compose/client.go | 223 ++++++++++++------ src/compose/client_test.go | 8 +- src/compose/compose.go | 26 +- src/compose/config/compare_test.go | 2 +- src/compose/config/config.go | 5 +- src/compose/config/config_test.go | 10 - src/compose/config/reflect.go | 1 + src/compose/container.go | 39 ++- src/compose/container_test.go | 10 +- src/compose/diff_test.go | 4 +- src/compose/docker.go | 2 +- .../src/rocker/imagename/imagename_test.go | 2 +- 12 files changed, 208 insertions(+), 124 deletions(-) diff --git a/src/compose/client.go b/src/compose/client.go index 944cf60..96bcdbd 100644 --- a/src/compose/client.go +++ b/src/compose/client.go @@ -37,7 +37,7 @@ type Client interface { RunContainer(container *Container) error EnsureContainerExist(name *Container) error EnsureContainerState(name *Container) error - PullAll(config *config.Config) error + PullAll(containers []*Container) error Clean(config *config.Config) error AttachToContainers(container []*Container) error AttachToContainer(container *Container) error @@ -311,22 +311,11 @@ func (client *DockerClient) EnsureContainerState(container *Container) error { } // PullAll grabs all image names from containers in spec and pulls all of them -func (client *DockerClient) PullAll(config *config.Config) error { - // do not pull same image twice - pulledImages := map[string]struct{}{} - containers := GetContainersFromConfig(config) - - // do pull - for _, container := range containers { - imageName := container.Image.String() - if _, ok := pulledImages[imageName]; ok { - continue - } - if err := client.pullImageForContainer(container); err != nil { - return err - } - pulledImages[imageName] = struct{}{} +func (client *DockerClient) PullAll(containers []*Container) error { + if err := client.pullImageForContainers(true, containers...); err != nil { + return err } + return nil } @@ -362,11 +351,11 @@ func (client *DockerClient) Clean(config *config.Config) error { // collect tags for every image for _, image := range all { for _, repoTag := range image.RepoTags { - imageName := imagename.New(repoTag) + imageName := imagename.NewFromString(repoTag) for img := range images { if img.IsSameKind(*imageName) { images[img].Items = append(images[img].Items, &imagename.Tag{ - Id: image.ID, + ID: image.ID, Name: *imageName, Created: image.Created, }) @@ -505,64 +494,11 @@ func (client *DockerClient) WaitForContainer(container *Container) (err error) { // FetchImages fetches the missing images for all containers in the manifest func (client *DockerClient) FetchImages(containers []*Container) error { - type message struct { - container *Container - result chan error - } - - wg := util.NewErrorWaitGroup(len(containers)) - chPullImages := make(chan message) - done := make(chan struct{}, 1) - checkedImages := map[string]struct{}{} - - // Pull worker - // We do not want to pull images in parallel - // instead, we use an actor to pull images sequentially - // - // TODO: WAAAT? we can simplify it by going though a list of images sequentially - // and pull those images which are missing. - go func() { - for { - select { - case msg := <-chPullImages: - msg.result <- client.pullImageForContainer(msg.container) - case <-done: - return - } - } - }() - - defer func() { - done <- struct{}{} - }() - - for _, container := range containers { - if container.Image == nil { - return fmt.Errorf("Image is not specified for container %s", container.Name) - } - if _, ok := checkedImages[container.Image.String()]; ok { - wg.Done(nil) - continue - } - checkedImages[container.Image.String()] = struct{}{} - - go func(container *Container) { - image, err := client.Docker.InspectImage(container.Image.String()) - if err == docker.ErrNoSuchImage { - msg := message{container, make(chan error)} - chPullImages <- msg - wg.Done(<-msg.result) - return - } else if err != nil { - wg.Done(err) - return - } - container.ImageID = image.ID - wg.Done(nil) - }(container) + if err := client.pullImageForContainers(false, containers...); err != nil { + return err } - return wg.Wait() + return nil } // GetPulledImages returns the list of images pulled by a recent run @@ -593,6 +529,94 @@ func (client *DockerClient) pullImageForContainer(container *Container) error { return nil } +// pullImageForContainers pulls required images to container daemon storage +// forceUpdate: bool if true - will ensure that the most recent version of required image will be pulled +// containers: []Container - list of containers list of images should be pulled for +// +// force means that if we are using wildcard in image tag and force is false, we will +// choose already pulled appropriate image, otherwise we will find the most recent in +// docker hub of remote registry +func (client *DockerClient) pullImageForContainers(forceUpdate bool, containers ...*Container) (err error) { + pulled := map[string]struct{}{} + + // getting all images that we already have + var images []*imagename.ImageName + if images, err = client.getAllImageNames(); err != nil { + return + } + + // check images for each container + for _, container := range containers { + // error in configuration, fail fast + if container.Image == nil { + err = fmt.Errorf("Image is not specified for container: %s", container.Name) + return + } + + // already pulled it for other container, skip + if _, ok := pulled[container.Image.String()]; ok { + continue + } + + pulled[container.Image.String()] = struct{}{} + + // looking locally first + candidate := findMostRecentTag(container.Image, images) + if candidate != nil { + log.Debugf("Found %s at local repository", candidate) + } + + // force update if we don't find anything locally + forceUpdate = forceUpdate || candidate == nil + + // in case we want to include external images as well, pulling list of available + // images from repository or central docker hub + if forceUpdate { + hub := imagename.NewDockerHub() + var remote []*imagename.ImageName + if remote, err = hub.List(container.Image); err != nil { + err = fmt.Errorf("Failed to pull image %s for container %s from remote registry, error: %s", + container.Image, container.Name, err) + } else { + images = append(images, remote...) + } + + // getting the most applicable image + // it may be local or remote image, it depends of forceUpdate flag + candidate = findMostRecentTag(container.Image, images) + if candidate != nil { + log.Debugf("Found %s at remote repository", candidate) + } + } + + if candidate == nil { + err = fmt.Errorf("Image not found: %s make sure that it's exist", container.Image) + return + } + + if _, err = client.Docker.InspectImage(candidate.String()); err == docker.ErrNoSuchImage { + log.Infof("Pulling image: %s for %s", candidate, container.Name) + if err = PullDockerImage(client.Docker, candidate, client.Auth.ToDockerAPI()); err != nil { + err = fmt.Errorf("Failed to pull image %s for container %s, error: %s", container.Image, container.Name, err) + return + } + } + + if err != nil { + return + } + + log.Debugf("Image %s is the most recent image for container: %s", candidate, container.Name) + + // saving resolved image in case we want to run this container + container.ImageResolved = candidate + + client.pulledImages = append(client.pulledImages, container.Image) + } + + return +} + func (client *DockerClient) listenReAttach(containers []*Container) { // The code is partially borrowed from https://github.com/jwilder/docker-gen eventChan := make(chan *docker.APIEvents, 100) @@ -698,3 +722,54 @@ func (client *DockerClient) flushContainerLogs(container *Container) { log.Errorf("Failed to read logs of container %s, error: %s", container.Name, err2) } } + +func (client *DockerClient) getAllImageNames() (images []*imagename.ImageName, err error) { + var dockerImages []docker.APIImages + + // retrieving images currently available in docker + if dockerImages, err = client.Docker.ListImages(docker.ListImagesOptions{}); err != nil { + return + } + + for _, image := range dockerImages { + for _, repoTag := range image.RepoTags { + images = append(images, imagename.NewFromString(repoTag)) + } + } + + return +} + +// findMostRecentTag getting all applicable version from dockerhub and choose the most recent +func findMostRecentTag(image *imagename.ImageName, list []*imagename.ImageName) (img *imagename.ImageName) { + for _, candidate := range list { + if !image.Contains(candidate) { + //this image is from the same name/registry but tag is not applicable + // e.g. ~1.2.3 contains 1.2.5, but it's not true for 1.3.0 + continue + } + + if candidate.GetTag() == imagename.Latest { + // use latest if it's possible + img = candidate + return + } + + if img == nil { + img = candidate + continue + } + + // uncomparable candidate... skipping + if !candidate.HasVersion() { + continue + } + + // if both names has versions to compare, we cat safely compare them + if img.HasVersion() && candidate.HasVersion() && img.TagAsVersion().Less(candidate.TagAsVersion()) { + img = candidate + } + } + + return +} diff --git a/src/compose/client_test.go b/src/compose/client_test.go index 379d78d..4a3f5e4 100644 --- a/src/compose/client_test.go +++ b/src/compose/client_test.go @@ -192,7 +192,7 @@ containers: n := 0 for _, image := range all { for _, repoTag := range image.RepoTags { - imageName := imagename.New(repoTag) + imageName := imagename.NewFromString(repoTag) if imageName.Name == "rocker-compose-test-image-clean" { n++ } @@ -206,7 +206,7 @@ containers: removed := cli.GetRemovedImages() assert.Equal(t, 3, len(removed), "Expected to remove a particular number of images") - assert.EqualValues(t, &imagename.ImageName{"", "rocker-compose-test-image-clean", "3"}, removed[0], "removed wrong image") - assert.EqualValues(t, &imagename.ImageName{"", "rocker-compose-test-image-clean", "2"}, removed[1], "removed wrong image") - assert.EqualValues(t, &imagename.ImageName{"", "rocker-compose-test-image-clean", "1"}, removed[2], "removed wrong image") + assert.EqualValues(t, "rocker-compose-test-image-clean:3", removed[0].String(), "removed wrong image") + assert.EqualValues(t, "rocker-compose-test-image-clean:2", removed[1].String(), "removed wrong image") + assert.EqualValues(t, "rocker-compose-test-image-clean:1", removed[2].String(), "removed wrong image") } diff --git a/src/compose/compose.go b/src/compose/compose.go index 8901003..db2ead1 100644 --- a/src/compose/compose.go +++ b/src/compose/compose.go @@ -96,13 +96,6 @@ func New(config *Config) (*Compose, error) { // RunAction implements 'rocker-compose run' func (compose *Compose) RunAction() error { - // if --pull is specified - if compose.Pull { - if err := compose.PullAction(); err != nil { - return err - } - } - // get the actual list of existing containers from docker client actual, err := compose.client.GetContainers(compose.Manifest.HasExternalRefs()) if err != nil { @@ -116,12 +109,22 @@ func (compose *Compose) RunAction() error { expected = GetContainersFromConfig(compose.Manifest) } - // fetch missing images for containers needed to be started - if err := compose.client.FetchImages(expected); err != nil { + // if --pull is specified PullAll, otherwise Fetch required + if compose.Pull { + if err := compose.client.PullAll(expected); err != nil { + return err + } + } else if err := compose.client.FetchImages(expected); err != nil { return fmt.Errorf("Failed to fetch images of given containers, error: %s", err) } - // Aassign IDs of existing containers + // upgrading expected image to the most recent + // todo: it may be an option to skip this step in some cases + for _, container := range expected { + container.Image = container.ImageResolved + } + + // Assign IDs of existing containers for _, actualC := range actual { for _, expectedC := range expected { if expectedC.IsSameKind(actualC) { @@ -227,7 +230,8 @@ func (compose *Compose) RecoverAction() error { // PullAction implements 'rocker-compose pull' func (compose *Compose) PullAction() error { - if err := compose.client.PullAll(compose.Manifest); err != nil { + containers := GetContainersFromConfig(compose.Manifest) + if err := compose.client.PullAll(containers); err != nil { return fmt.Errorf("Failed to pull all images, error: %s", err) } diff --git a/src/compose/config/compare_test.go b/src/compose/config/compare_test.go index f727db8..451d36c 100644 --- a/src/compose/config/compare_test.go +++ b/src/compose/config/compare_test.go @@ -79,7 +79,7 @@ func TestConfigIsEqualTo(t *testing.T) { cases := tests{ // type: string fieldSpec{ - []string{"Image", "Pid", "Uts", "CpusetCpus", "Hostname", "Domainname", "User", "Workdir", "LogDriver"}, + []string{"Pid", "Uts", "CpusetCpus", "Hostname", "Domainname", "User", "Workdir", "LogDriver"}, []check{ check{shouldEqual, "KEY: foo", "KEY: foo"}, check{shouldEqual, "", ""}, diff --git a/src/compose/config/config.go b/src/compose/config/config.go index efc8dad..e1ee16f 100644 --- a/src/compose/config/config.go +++ b/src/compose/config/config.go @@ -346,7 +346,10 @@ func ReadConfig(configName string, reader io.Reader, vars template.Vars, funcs m if container.Image == nil { return nil, fmt.Errorf("Image should be specified for container: %s", name) } - if !imagename.New(*container.Image).HasTag() { + + img := imagename.NewFromString(*container.Image) + + if !img.IsStrict() && !img.HasVersionRange() && !img.All() { return nil, fmt.Errorf("Image `%s` for container `%s`: image without tag is not allowed", *container.Image, name) } diff --git a/src/compose/config/config_test.go b/src/compose/config/config_test.go index e117c19..0fd7f83 100644 --- a/src/compose/config/config_test.go +++ b/src/compose/config/config_test.go @@ -91,16 +91,6 @@ containers: assert.Equal(t, "Image should be specified for container: test", err.Error()) } -func TestConfigImageNoTag(t *testing.T) { - configStr := `namespace: test -containers: - test: - image: ubuntu` - - _, err := ReadConfig("test", strings.NewReader(configStr), configTestVars, map[string]interface{}{}, false) - assert.Equal(t, "Image `ubuntu` for container `test`: image without tag is not allowed", err.Error()) -} - func TestNewContainerNameFromString(t *testing.T) { type assertion struct { namespace string diff --git a/src/compose/config/reflect.go b/src/compose/config/reflect.go index be76be2..384ccd2 100644 --- a/src/compose/config/reflect.go +++ b/src/compose/config/reflect.go @@ -24,6 +24,7 @@ import ( // compareSkipFields defines which fields will not be compared var compareSkipFields = []string{ + "Image", "Extends", "KillTimeout", "NetworkDisabled", diff --git a/src/compose/container.go b/src/compose/container.go index d5ec19d..fd9578c 100644 --- a/src/compose/container.go +++ b/src/compose/container.go @@ -31,14 +31,15 @@ import ( // Container object represents a single container produced by a rocker-compose spec type Container struct { - ID string - Image *imagename.ImageName - ImageID string - Name *config.ContainerName - Created time.Time - State *ContainerState - Config *config.Container - Io *ContainerIo + ID string + Image *imagename.ImageName + ImageResolved *imagename.ImageName + ImageID string + Name *config.ContainerName + Created time.Time + State *ContainerState + Config *config.Container + Io *ContainerIo container *docker.Container } @@ -80,7 +81,7 @@ func NewContainerFromConfig(name *config.ContainerName, containerConfig *config. Config: containerConfig, } if containerConfig.Image != nil { - container.Image = imagename.New(*containerConfig.Image) + container.Image = imagename.NewFromString(*containerConfig.Image) } return container } @@ -96,7 +97,7 @@ func NewContainerFromDocker(dockerContainer *docker.Container) (*Container, erro } return &Container{ ID: dockerContainer.ID, - Image: imagename.New(dockerContainer.Config.Image), + Image: imagename.NewFromString(dockerContainer.Config.Image), ImageID: dockerContainer.Image, Name: config.NewContainerNameFromString(dockerContainer.Name), Created: dockerContainer.Created, @@ -151,7 +152,18 @@ func (a *Container) IsEqualTo(b *Container) bool { return false } - // check image + // check image version + if a.Image != nil && !a.Image.Contains(b.Image) { + log.Debugf("Comparing '%s' and '%s': image version '%s' is not satisfied (was %s should satisfy %s)", + a.Name.String(), + b.Name.String(), + a.Image, + b.Image, + a.Image) + return false + } + + // check image id if a.ImageID != "" && b.ImageID != "" && a.ImageID != b.ImageID { log.Debugf("Comparing '%s' and '%s': image '%s' updated (was %.12s became %.12s)", a.Name.String(), @@ -208,6 +220,11 @@ func (a *Container) CreateContainerOptions() (*docker.CreateContainerOptions, er apiConfig.Labels = labels + // Replace image with more specific one (config may contain image range or wildcards) + if a.ImageResolved != nil { + apiConfig.Image = a.ImageResolved.String() + } + return &docker.CreateContainerOptions{ Name: a.Name.String(), Config: apiConfig, diff --git a/src/compose/container_test.go b/src/compose/container_test.go index f864c65..e9f05c1 100644 --- a/src/compose/container_test.go +++ b/src/compose/container_test.go @@ -22,7 +22,6 @@ import ( "time" "github.com/fsouza/go-dockerclient" - "github.com/grammarly/rocker/src/rocker/imagename" "github.com/stretchr/testify/assert" ) @@ -86,11 +85,6 @@ func TestNewContainerFromDocker(t *testing.T) { t.Fatal(err) } - assertionImage := &imagename.ImageName{ - Registry: "quay.io", - Name: "myapp", - Tag: "1.9.2", - } assertionName := &config.ContainerName{ Namespace: "myapp", Name: "main", @@ -99,8 +93,8 @@ func TestNewContainerFromDocker(t *testing.T) { assert.Equal(t, id, container.ID) assert.Equal(t, &ContainerState{Running: true}, container.State) assert.Equal(t, createdTime, container.Created) - assert.Equal(t, assertionImage, container.Image) - assert.Equal(t, assertionImage.String(), *container.Config.Image) + assert.Equal(t, "quay.io/myapp:1.9.2", container.Image.String()) + assert.Equal(t, "quay.io/myapp:1.9.2", *container.Config.Image) assert.Equal(t, assertionName, container.Name) } diff --git a/src/compose/diff_test.go b/src/compose/diff_test.go index c3d89d5..39c0771 100644 --- a/src/compose/diff_test.go +++ b/src/compose/diff_test.go @@ -463,8 +463,8 @@ func (m *clientMock) PullImage(imageName *imagename.ImageName) error { return args.Error(0) } -func (m *clientMock) PullAll(cfg *config.Config) error { - args := m.Called(cfg) +func (m *clientMock) PullAll(containers []*Container) error { + args := m.Called(containers) return args.Error(0) } diff --git a/src/compose/docker.go b/src/compose/docker.go index 1566331..45f3cf0 100644 --- a/src/compose/docker.go +++ b/src/compose/docker.go @@ -48,7 +48,7 @@ func GetBridgeIP(client *docker.Client) (ip string, err error) { _, err = client.InspectImage(emptyImageName) if err != nil && err.Error() == "no such image" { log.Infof("Pulling image %s to obtain network bridge address", emptyImageName) - if err := PullDockerImage(client, imagename.New(emptyImageName), &docker.AuthConfiguration{}); err != nil { + if err := PullDockerImage(client, imagename.NewFromString(emptyImageName), &docker.AuthConfiguration{}); err != nil { return "", err } } else if err != nil { diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go index d959234..7f116a8 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go @@ -180,7 +180,7 @@ func TestImageLatest(t *testing.T) { } func TestImageIpRegistry(t *testing.T) { - img := NewFromString("10.0.31.117:5000/golang:1.4") + img := NewFromString("127.0.0.1:5000/golang:1.4") assert.Equal(t, "127.0.0.1:5000", img.Registry, "bag registry value") assert.Equal(t, "golang", img.Name, "bad image name") assert.Equal(t, "1.4", img.GetTag(), "bad image tag") From 84ae7d6f0f97fa79156e5ee7289dfba0c7fe3458 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 29 Sep 2015 13:55:45 +0300 Subject: [PATCH 04/34] vendor update rocker/template --- vendor/manifest | 2 +- .../rocker/src/rocker/template/template.go | 5 +- .../rocker/src/rocker/template/vars.go | 60 +++++++++++++++++-- .../rocker/src/rocker/template/vars_test.go | 53 ++++++++++++++++ 4 files changed, 112 insertions(+), 8 deletions(-) diff --git a/vendor/manifest b/vendor/manifest index 20987cf..dca6b38 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -108,7 +108,7 @@ { "importpath": "github.com/grammarly/rocker/src/rocker/template", "repository": "https://github.com/grammarly/rocker", - "revision": "b4e13649def6e2852db73477f6cf272b76d47e53", + "revision": "15024281d69f1e7a1814204e60b3ac8c771691ce", "branch": "master", "path": "/src/rocker/template" }, diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go b/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go index 69c2a78..a620b18 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go @@ -43,6 +43,9 @@ func Process(name string, reader io.Reader, vars Vars, funcs map[string]interfac return nil, fmt.Errorf("Error reading template %s, error: %s", name, err) } + // Copy the vars struct because we don't want to modify the original struct + vars = Vars{}.Merge(vars) + // merge OS environment variables with the given Vars map // todo: maybe, we need to make it configurable vars["Env"] = ParseKvPairs(os.Environ()) @@ -64,7 +67,7 @@ func Process(name string, reader io.Reader, vars Vars, funcs map[string]interfac "equalFold": strings.EqualFold, "hasPrefix": strings.HasPrefix, "hasSuffix": strings.HasSuffix, - "index": strings.Index, + "indexOf": strings.Index, "indexAny": strings.IndexAny, "join": strings.Join, "lastIndex": strings.LastIndex, diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars.go b/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars.go index 9146339..9231c47 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars.go @@ -26,6 +26,8 @@ import ( "regexp" "sort" "strings" + + "github.com/go-yaml/yaml" ) // Vars describes the data structure of the build variables @@ -113,6 +115,45 @@ func VarsFromStrings(pairs []string) (vars Vars, err error) { return vars, nil } +// VarsFromFile reads variables from either JSON or YAML file +func VarsFromFile(filename string) (vars Vars, err error) { + + if filename, err = resolveFileName(filename); err != nil { + return nil, err + } + + data, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + vars = Vars{} + + switch filepath.Ext(filename) { + case ".yaml", ".yml", ".": + if err := yaml.Unmarshal(data, &vars); err != nil { + return nil, err + } + case ".json": + if err := json.Unmarshal(data, &vars); err != nil { + return nil, err + } + } + + return vars, nil +} + +// VarsFromFileMulti reads multiple files and merge vars +func VarsFromFileMulti(files []string) (vars Vars, err error) { + varsList := make([]Vars, len(files)) + for i, f := range files { + if varsList[i], err = VarsFromFile(f); err != nil { + return nil, err + } + } + return Vars{}.Merge(varsList...), nil +} + // ParseKvPairs parses Vars from a slice of strings e.g. []string{"KEY=VALUE"} func ParseKvPairs(pairs []string) (vars Vars) { vars = make(Vars) @@ -123,7 +164,18 @@ func ParseKvPairs(pairs []string) (vars Vars) { return vars } -func loadFileContent(f string) (string, error) { +func loadFileContent(f string) (content string, err error) { + if f, err = resolveFileName(f); err != nil { + return "", err + } + data, err := ioutil.ReadFile(f) + if err != nil { + return "", err + } + return string(data), nil +} + +func resolveFileName(f string) (string, error) { if f == "~" || strings.HasPrefix(f, "~/") { f = strings.Replace(f, "~", os.Getenv("HOME"), 1) } @@ -134,11 +186,7 @@ func loadFileContent(f string) (string, error) { } f = path.Join(wd, f) } - data, err := ioutil.ReadFile(f) - if err != nil { - return "", err - } - return string(data), nil + return f, nil } // Code borrowed from https://github.com/docker/docker/blob/df0e0c76831bed08cf5e08ac9a1abebf6739da23/builder/support.go diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars_test.go b/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars_test.go index 7f77feb..6408cf3 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars_test.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars_test.go @@ -19,8 +19,10 @@ package template import ( "encoding/json" "fmt" + "io/ioutil" "os" "path" + "rocker/test" "testing" "github.com/stretchr/testify/assert" @@ -98,6 +100,41 @@ func TestVarsFromStrings(t *testing.T) { } } +func TestVarsFromFile_Yaml(t *testing.T) { + tempDir, rm := tplMkFiles(t, map[string]string{ + "vars.yml": ` +Foo: x +Bar: yes +`, + }) + defer rm() + + vars, err := VarsFromFile(tempDir + "/vars.yml") + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "x", vars["Foo"]) + assert.Equal(t, true, vars["Bar"]) +} + +func TestVarsFromFile_Json(t *testing.T) { + tempDir, rm := tplMkFiles(t, map[string]string{ + "vars.json": ` +{"Foo": "x", "Bar": true} +`, + }) + defer rm() + + vars, err := VarsFromFile(tempDir + "/vars.json") + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "x", vars["Foo"]) + assert.Equal(t, true, vars["Bar"]) +} + func TestVarsReplaceString(t *testing.T) { t.Parallel() @@ -227,3 +264,19 @@ func TestVarsFileContent(t *testing.T) { assert.Equal(t, "hello\n", result4["FOO"]) } + +func tplMkFiles(t *testing.T, files map[string]string) (string, func()) { + tempDir, err := ioutil.TempDir("", "rocker-vars-test") + if err != nil { + t.Fatal(err) + } + + if err = test.MakeFiles(tempDir, files); err != nil { + os.RemoveAll(tempDir) + t.Fatal(err) + } + + return tempDir, func() { + os.RemoveAll(tempDir) + } +} From 3a6d136666c0f9016f7a0df9dbdc2901c00ec385 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 29 Sep 2015 13:56:03 +0300 Subject: [PATCH 05/34] vendor update rocker/imagename --- vendor/manifest | 2 +- .../rocker/src/rocker/imagename/imagename.go | 16 ++++++++++++++++ .../src/rocker/imagename/imagename_test.go | 5 +++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/vendor/manifest b/vendor/manifest index dca6b38..8f780ea 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -115,7 +115,7 @@ { "importpath": "github.com/grammarly/rocker/src/rocker/imagename", "repository": "https://github.com/grammarly/rocker", - "revision": "b4e13649def6e2852db73477f6cf272b76d47e53", + "revision": "15024281d69f1e7a1814204e60b3ac8c771691ce", "branch": "master", "path": "/src/rocker/imagename" }, diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go index ab5894c..e72fcbd 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go @@ -24,6 +24,7 @@ import ( "sort" "strings" + "github.com/go-yaml/yaml" "github.com/wmark/semver" ) @@ -177,6 +178,21 @@ func (img ImageName) MarshalJSON() ([]byte, error) { return json.Marshal(img.String()) } +// UnmarshalYAML parses YAML string and returns ImageName +func (img *ImageName) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + *img = *NewFromString(s) + return nil +} + +// MarshalYAML serializes ImageName to YAML string +func (img ImageName) MarshalYAML() ([]byte, error) { + return yaml.Marshal(img.String()) +} + // Tags is a structure used for cleaning images // Sorts out old tags by creation date type Tags struct { diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go index 7f116a8..1a91f28 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go @@ -186,6 +186,11 @@ func TestImageIpRegistry(t *testing.T) { assert.Equal(t, "1.4", img.GetTag(), "bad image tag") } +func TestImageAll(t *testing.T) { + img := NewFromString("golang:1.*") + assert.False(t, img.All()) +} + func TestImageIsSameKind(t *testing.T) { assert.True(t, NewFromString("rocker-build").IsSameKind(*NewFromString("rocker-build"))) assert.True(t, NewFromString("rocker-build:latest").IsSameKind(*NewFromString("rocker-build:latest"))) From 40345e7ebe615b8e0086e4061d80fe475ab9a6fa Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 29 Sep 2015 13:56:51 +0300 Subject: [PATCH 06/34] pin experimental --- src/cmd/rocker-compose/main.go | 44 ++++++++++++++++- src/compose/client.go | 86 ++++++++++++++++++++++++++++++++-- src/compose/compose.go | 34 ++++++++++++++ src/compose/config/config.go | 4 ++ vendor/manifest | 12 ++--- 5 files changed, 170 insertions(+), 10 deletions(-) diff --git a/src/cmd/rocker-compose/main.go b/src/cmd/rocker-compose/main.go index a68f247..6d8283d 100644 --- a/src/cmd/rocker-compose/main.go +++ b/src/cmd/rocker-compose/main.go @@ -76,6 +76,11 @@ func main() { Value: &cli.StringSlice{}, Usage: "Set variables to pass to build tasks, value is like \"key=value\"", }, + cli.StringSliceFlag{ + Name: "vars-file", + Value: &cli.StringSlice{}, + Usage: "Load variables form a file, either JSON or YAML. Can pass multiple of this.", + }, cli.BoolFlag{ Name: "dry, d", Usage: "Don't execute any run/stop operations on target docker", @@ -165,6 +170,12 @@ func main() { }, }, composeFlags...), }, + { + Name: "pin", + Usage: "pin versions", + Action: pinCommand, + Flags: append([]cli.Flag{}, composeFlags...), + }, { Name: "recover", Usage: "recover containers from machine reboot or docker daemon restart", @@ -331,6 +342,29 @@ func cleanCommand(ctx *cli.Context) { } } +func pinCommand(ctx *cli.Context) { + initLogs(ctx) + + log.SetLevel(log.WarnLevel) + + dockerCli := initDockerClient(ctx) + config := initComposeConfig(ctx, dockerCli) + auth := initAuthConfig(ctx) + + compose, err := compose.New(&compose.Config{ + Manifest: config, + Docker: dockerCli, + Auth: auth, + }) + if err != nil { + log.Fatal(err) + } + + if err := compose.PinAction(); err != nil { + log.Fatal(err) + } +} + func recoverCommand(ctx *cli.Context) { initLogs(ctx) @@ -399,12 +433,20 @@ func initComposeConfig(ctx *cli.Context, dockerCli *docker.Client) *config.Confi print = ctx.Bool("print") ) - vars, err := template.VarsFromStrings(ctx.StringSlice("var")) + vars, err := template.VarsFromFileMulti(ctx.StringSlice("vars-file")) if err != nil { log.Fatal(err) os.Exit(1) } + cliVars, err := template.VarsFromStrings(ctx.StringSlice("var")) + if err != nil { + log.Fatal(err) + os.Exit(1) + } + + vars = vars.Merge(cliVars) + // TODO: find better place for providing this helper funcs := map[string]interface{}{ // lazy get bridge ip diff --git a/src/compose/client.go b/src/compose/client.go index 96bcdbd..7e6ffb8 100644 --- a/src/compose/client.go +++ b/src/compose/client.go @@ -45,6 +45,7 @@ type Client interface { WaitForContainer(container *Container) error GetPulledImages() []*imagename.ImageName GetRemovedImages() []*imagename.ImageName + Pin(containers []*Container) error } // DockerClient is an implementation of Client interface that do operations to a given docker client @@ -511,6 +512,10 @@ func (client *DockerClient) GetRemovedImages() []*imagename.ImageName { return client.removedImages } +func (client *DockerClient) Pin(containers []*Container) error { + return client.pullImageForContainers2(true, containers...) +} + // Internal func (client *DockerClient) pullImageForContainer(container *Container) error { @@ -529,6 +534,75 @@ func (client *DockerClient) pullImageForContainer(container *Container) error { return nil } +func (client *DockerClient) pullImageForContainers2(forceUpdate bool, containers ...*Container) (err error) { + pulled := map[string]*imagename.ImageName{} + + // getting all images that we already have + var images []*imagename.ImageName + if images, err = client.getAllImageNames(); err != nil { + return + } + + // check images for each container + for _, container := range containers { + // error in configuration, fail fast + if container.Image == nil { + err = fmt.Errorf("Image is not specified for container: %s", container.Name) + return + } + + // already pulled it for other container, skip + if _, ok := pulled[container.Image.String()]; ok { + // result = append(result, &res{ + // containerName: container.Name, + // image: pulled[container.Image.String()], + // }) + container.Image = pulled[container.Image.String()] + continue + } + + pulled[container.Image.String()] = nil + + // Override to not change the common images slice + images := images + + // in case we want to include external images as well, pulling list of available + // images from repository or central docker hub + if forceUpdate { + hub := imagename.NewDockerHub() + + log.Debugf("Getting list of tags for %s from the registry", container.Image) + + var remote []*imagename.ImageName + if remote, err = hub.List(container.Image); err != nil { + err = fmt.Errorf("Failed to pull image %s for container %s from remote registry, error: %s", + container.Image, container.Name, err) + } else { + images = append(images, remote...) + } + } + + // looking locally first + candidate := findMostRecentTag(container.Image, images) + if candidate == nil { + err = fmt.Errorf("Image not found: %s make sure that it's exist", container.Image) + return + } + + log.Infof("%s --> %s", container.Image, candidate.GetTag()) + + // result = append(result, &res{ + // containerName: container.Name, + // image: candidate, + // }) + + container.Image = candidate + pulled[container.Image.String()] = candidate + } + + return +} + // pullImageForContainers pulls required images to container daemon storage // forceUpdate: bool if true - will ensure that the most recent version of required image will be pulled // containers: []Container - list of containers list of images should be pulled for @@ -540,10 +614,12 @@ func (client *DockerClient) pullImageForContainers(forceUpdate bool, containers pulled := map[string]struct{}{} // getting all images that we already have + log.Debug("before getAllImageNames") var images []*imagename.ImageName if images, err = client.getAllImageNames(); err != nil { return } + log.Debug("after getAllImageNames %s", len(images)) // check images for each container for _, container := range containers { @@ -750,11 +826,15 @@ func findMostRecentTag(image *imagename.ImageName, list []*imagename.ImageName) } if candidate.GetTag() == imagename.Latest { - // use latest if it's possible - img = candidate - return + continue } + // if candidate.GetTag() == imagename.Latest { + // // use latest if it's possible + // img = candidate + // return + // } + if img == nil { img = candidate continue diff --git a/src/compose/compose.go b/src/compose/compose.go index db2ead1..69ced88 100644 --- a/src/compose/compose.go +++ b/src/compose/compose.go @@ -29,6 +29,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/fsouza/go-dockerclient" + "github.com/go-yaml/yaml" "github.com/kr/pretty" ) @@ -247,6 +248,39 @@ func (compose *Compose) CleanAction() error { return nil } +// PinAction implements 'rocker-compose pin' +func (compose *Compose) PinAction() error { + containers := GetContainersFromConfig(compose.Manifest) + if err := compose.client.Pin(containers); err != nil { + return fmt.Errorf("Failed to pin, error: %s", err) + } + + type versions struct { + Containers map[string]string + Images map[string]string + } + + // Populate versions to the variables + vars := compose.Manifest.Vars + vars["Versions"] = versions{ + Containers: map[string]string{}, + Images: map[string]string{}, + } + for _, c := range containers { + vars["Versions"].(versions).Containers[c.Name.Name] = c.Image.GetTag() + vars["Versions"].(versions).Images[c.Image.NameWithRegistry()] = c.Image.GetTag() + } + + data, err := yaml.Marshal(vars) + if err != nil { + return err + } + + fmt.Print(string(data)) + + return nil +} + // WritePlan saves various rocker-compose change information to the ansible.Response object // TODO: should compose know about ansible.Response at all? // maybe it should give some data struct back to main? diff --git a/src/compose/config/config.go b/src/compose/config/config.go index e1ee16f..bb6b8c3 100644 --- a/src/compose/config/config.go +++ b/src/compose/config/config.go @@ -44,6 +44,7 @@ import ( type Config struct { Namespace string // All containers names under current compose.yml will be prefixed with this namespace Containers map[string]*Container + Vars template.Vars } // Container represents a single container spec from compose.yml @@ -244,6 +245,9 @@ func ReadConfig(configName string, reader io.Reader, vars template.Vars, funcs m config.Namespace = regexp.MustCompile("[^a-z0-9\\-\\_]").ReplaceAllString(parentDir, "") } + // Save vars to config + config.Vars = vars + // Read extra data type ConfigExtra struct { Containers map[string]map[string]interface{} diff --git a/vendor/manifest b/vendor/manifest index 8f780ea..b88b16f 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -105,6 +105,12 @@ "revision": "c9ad0ce23f68428421adfc6ced9e6123f54788a5", "branch": "HEAD" }, + { + "importpath": "github.com/wmark/semver", + "repository": "https://github.com/wmark/semver", + "revision": "461c06b538be53cc0339815001a398e29ace025b", + "branch": "master" + }, { "importpath": "github.com/grammarly/rocker/src/rocker/template", "repository": "https://github.com/grammarly/rocker", @@ -118,12 +124,6 @@ "revision": "15024281d69f1e7a1814204e60b3ac8c771691ce", "branch": "master", "path": "/src/rocker/imagename" - }, - { - "importpath": "github.com/wmark/semver", - "repository": "https://github.com/wmark/semver", - "revision": "461c06b538be53cc0339815001a398e29ace025b", - "branch": "master" } ] } \ No newline at end of file From f897d03c6dd87b46fa5b01054c535ea5477d1df1 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 29 Sep 2015 13:58:59 +0300 Subject: [PATCH 07/34] satisfy linter and fix tests --- src/compose/client.go | 1 + src/compose/diff_test.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/compose/client.go b/src/compose/client.go index 7e6ffb8..f813cea 100644 --- a/src/compose/client.go +++ b/src/compose/client.go @@ -512,6 +512,7 @@ func (client *DockerClient) GetRemovedImages() []*imagename.ImageName { return client.removedImages } +// Pin resolves versions for given containers func (client *DockerClient) Pin(containers []*Container) error { return client.pullImageForContainers2(true, containers...) } diff --git a/src/compose/diff_test.go b/src/compose/diff_test.go index 39c0771..e1194ab 100644 --- a/src/compose/diff_test.go +++ b/src/compose/diff_test.go @@ -503,6 +503,11 @@ func (m *clientMock) GetRemovedImages() []*imagename.ImageName { return []*imagename.ImageName{} } +func (m *clientMock) Pin(container []*Container) error { + args := m.Called(container) + return args.Error(0) +} + type clientMock struct { mock.Mock } From 3deda6c3f43f38a32ad1e3e486a65c0661ffb8e5 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 29 Sep 2015 14:01:13 +0300 Subject: [PATCH 08/34] make local binary by default, add make install --- Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 9919a7e..8439cc7 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,15 @@ PKGS := $(foreach pkg, $(sort $(dir $(SRCS))), $(pkg)) TESTARGS ?= +binary: + GOPATH=$(shell pwd):$(shell pwd)/vendor go build \ + -ldflags "-X main.Version=$(VERSION) -X main.GitCommit=$(GITCOMMIT) -X main.GitBranch=$(GITBRANCH) -X main.BuildTime=$(BUILDTIME)" \ + -v -o bin/rocker-compose src/cmd/rocker-compose/main.go + +install: + cp bin/rocker-compose /usr/local/bin/rocker-compose + chmod +x /usr/local/bin/rocker-compose + all: $(ALL_BINARIES) $(foreach BIN, $(BINARIES), $(shell cp dist/$(VERSION)/$(shell go env GOOS)/amd64/$(BIN) dist/$(BIN))) @@ -70,11 +79,6 @@ $(ALL_BINARIES): build_image build_image: rocker build -f Rockerfile.build-cross -local-binary: - GOPATH=$(shell pwd):$(shell pwd)/vendor go build \ - -ldflags "-X main.Version=$(VERSION) -X main.GitCommit=$(GITCOMMIT) -X main.GitBranch=$(GITBRANCH) -X main.BuildTime=$(BUILDTIME)" \ - -v -o bin/rocker-compose src/cmd/rocker-compose/main.go - clean: rm -Rf dist From 051e807741c3c64b3d80b1cfce5cb8600e2004d9 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Wed, 30 Sep 2015 15:24:04 +0300 Subject: [PATCH 09/34] vendor update rocker/imagename --- vendor/manifest | 2 +- .../rocker/src/rocker/imagename/imagename.go | 75 ++++++- .../src/rocker/imagename/imagename_test.go | 199 ++++++++++++++++++ .../imagename/{dockerhub.go => registry.go} | 57 +++-- 4 files changed, 295 insertions(+), 38 deletions(-) rename vendor/src/github.com/grammarly/rocker/src/rocker/imagename/{dockerhub.go => registry.go} (62%) diff --git a/vendor/manifest b/vendor/manifest index b88b16f..8a18ed6 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -121,7 +121,7 @@ { "importpath": "github.com/grammarly/rocker/src/rocker/imagename", "repository": "https://github.com/grammarly/rocker", - "revision": "15024281d69f1e7a1814204e60b3ac8c771691ce", + "revision": "2ddda7b61203075ed35666dbaec9a7cf0fef4e08", "branch": "master", "path": "/src/rocker/imagename" } diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go index e72fcbd..ec681d9 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go @@ -67,10 +67,7 @@ func New(image string, tag string) *ImageName { dockerImage.Name = image } if tag != "" { - if rng, err := semver.NewRange(tag); err == nil && rng != nil { - dockerImage.Version = rng - } - dockerImage.Tag = tag + dockerImage.SetTag(tag) } return dockerImage } @@ -93,7 +90,23 @@ func (img ImageName) GetTag() string { return Latest } +// SetTag sets the new tag for the imagename +func (img *ImageName) SetTag(tag string) { + img.Version = nil + if rng, err := semver.NewRange(tag); err == nil && rng != nil { + img.Version = rng + } + img.Tag = tag +} + // IsStrict returns true if tag of the current image is specified and contains no fuzzy rules +// Example: +// golang:latest == true +// golang:stable == true +// golang:1.5.1 == true +// golang:1.5.* == false +// golang == false +// func (img ImageName) IsStrict() bool { if img.HasVersionRange() { return img.TagAsVersion() != nil @@ -102,16 +115,32 @@ func (img ImageName) IsStrict() bool { } // All returns true if tag of the current image refers to any version +// Example: +// golang:* == true +// golang == true +// golang:latest == false func (img ImageName) All() bool { return strings.Contains(Wildcards, img.Tag) } // HasVersion returns true if tag of the current image refers to a semver format +// Example: +// golang:1.5.1 == true +// golang:1.5.* == false +// golang:stable == false +// golang:latest == false func (img ImageName) HasVersion() bool { return img.TagAsVersion() != nil } // HasVersionRange returns true if tag of the current image refers to a semver format and is fuzzy +// Example: +// golang:1.5.1 == true +// golang:1.5.* == true +// golang:* == true +// golang:stable == false +// golang:latest == false +// golang == false func (img ImageName) HasVersionRange() bool { return img.Version != nil } @@ -163,6 +192,44 @@ func (img ImageName) Contains(b *ImageName) bool { return img.Tag == "" && !img.HasVersionRange() } +// ResolveVersion finds an applicable tag for current image among the list of available tags +func (img *ImageName) ResolveVersion(list []*ImageName) (result *ImageName) { + for _, candidate := range list { + // If we have a strict equality + if img.HasTag() && candidate.HasTag() && img.Tag == candidate.Tag { + return candidate + } + + // If image is without tag, latest will be fine + if !img.HasTag() && candidate.GetTag() == Latest { + return candidate + } + + if !img.Contains(candidate) { + //this image is from the same name/registry but tag is not applicable + // e.g. ~1.2.3 contains 1.2.5, but it's not true for 1.3.0 + continue + } + + if result == nil { + result = candidate + continue + } + + // uncomparable candidate... skipping + if !candidate.HasVersion() { + continue + } + + // if both names has versions to compare, we cat safely compare them + if result.HasVersion() && candidate.HasVersion() && result.TagAsVersion().Less(candidate.TagAsVersion()) { + result = candidate + } + } + + return +} + // UnmarshalJSON parses JSON string and returns ImageName func (img *ImageName) UnmarshalJSON(data []byte) error { var s string diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go index 1a91f28..2e6a084 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go @@ -17,9 +17,12 @@ package imagename import ( + "fmt" "testing" "time" + "github.com/kr/pretty" + "github.com/stretchr/testify/assert" "github.com/wmark/semver" ) @@ -191,6 +194,202 @@ func TestImageAll(t *testing.T) { assert.False(t, img.All()) } +func TestImageTest(t *testing.T) { + t.Skip() + names := []string{ + "golang:latest", + "golang:stable", + "golang:1.5.1", + "golang:1.5.*", + "golang:*", + "golang", + } + for _, n := range names { + img := NewFromString(n) + m := [][2]interface{}{ + {"IsStrict()", img.IsStrict()}, + {"HasVersion()", img.HasVersion()}, + {"HasVersionRange()", img.HasVersionRange()}, + {"All()", img.All()}, + {"GetTag()", img.GetTag()}, + } + fmt.Printf("%s\t%# v\n", n, pretty.Formatter(m)) + } +} + +func TestImageResolveVersion_Strict(t *testing.T) { + img := NewFromString("golang:1.5.2") + list := []*ImageName{ + NewFromString("golang:1.5.1"), + NewFromString("golang:1.5.2"), + NewFromString("golang:1.5.3"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:1.5.2", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_Wildcard(t *testing.T) { + img := NewFromString("golang:1.5.*") + list := []*ImageName{ + NewFromString("golang:1.5.1"), + NewFromString("golang:1.5.2"), + NewFromString("golang:1.5.3"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:1.5.3", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_WildcardMulti(t *testing.T) { + img := NewFromString("golang:1.4.*") + list := []*ImageName{ + NewFromString("golang:1.4.1"), + NewFromString("golang:1.4.2"), + NewFromString("golang:1.5.1"), + NewFromString("golang:1.5.2"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:1.4.2", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_WildcardMatchX(t *testing.T) { + img := NewFromString("golang:1.4.x") + list := []*ImageName{ + NewFromString("golang:1.4.1"), + NewFromString("golang:1.4.x"), + NewFromString("golang:1.4.2"), + NewFromString("golang:1.5.1"), + NewFromString("golang:1.5.2"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:1.4.x", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_WildcardMatchX2(t *testing.T) { + img := NewFromString("golang:1.x") + list := []*ImageName{ + NewFromString("golang:1.4.1"), + NewFromString("golang:1.4.x"), + NewFromString("golang:1.4.2"), + NewFromString("golang:1.5.1"), + NewFromString("golang:1.5.2"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:1.5.2", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_WildcardMatchX3(t *testing.T) { + img := NewFromString("golang:1.x") + list := []*ImageName{ + NewFromString("golang:1.x"), + NewFromString("golang:1.4.1"), + NewFromString("golang:1.4.x"), + NewFromString("golang:1.4.2"), + NewFromString("golang:1.5.1"), + NewFromString("golang:1.5.2"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:1.x", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_All(t *testing.T) { + img := NewFromString("golang:*") + list := []*ImageName{ + NewFromString("golang:1.4.1"), + NewFromString("golang:1.5.1"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:1.5.1", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_Latest(t *testing.T) { + img := NewFromString("golang:latest") + list := []*ImageName{ + NewFromString("golang:1.4.1"), + NewFromString("golang:1.5.1"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:latest", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_OtherTag(t *testing.T) { + img := NewFromString("golang:stable") + list := []*ImageName{ + NewFromString("golang:1.4.1"), + NewFromString("golang:1.5.1"), + NewFromString("golang:stable"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:stable", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_NoTag(t *testing.T) { + img := NewFromString("golang") + list := []*ImageName{ + NewFromString("golang:1.4.1"), + NewFromString("golang:1.5.1"), + NewFromString("golang:stable"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:latest", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_NoTagOnlyLatest(t *testing.T) { + img := NewFromString("golang") + list := []*ImageName{ + NewFromString("golang:stable"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:latest", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_PatchExact(t *testing.T) { + img := NewFromString("golang:1.4.1") + list := []*ImageName{ + NewFromString("golang:1.4.1"), + NewFromString("golang:1.4.1-p2"), + NewFromString("golang:1.4.1-p1"), + NewFromString("golang:1.5.1"), + NewFromString("golang:1.5.2"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:1.4.1", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_PatchMatch(t *testing.T) { + img := NewFromString("golang:1.4.1") + list := []*ImageName{ + NewFromString("golang:1.4.1-p1"), + NewFromString("golang:1.4.1-p2"), + NewFromString("golang:1.5.1"), + NewFromString("golang:1.5.2"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:1.4.1-p2", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_PatchStrict(t *testing.T) { + img := NewFromString("golang:1.4.1-p1") + list := []*ImageName{ + NewFromString("golang:1.4.1"), + NewFromString("golang:1.4.1-p2"), + NewFromString("golang:1.4.1-p1"), + NewFromString("golang:1.5.1"), + NewFromString("golang:1.5.2"), + NewFromString("golang:latest"), + } + assert.Equal(t, "golang:1.4.1-p1", img.ResolveVersion(list).String()) +} + +func TestImageResolveVersion_NotFound(t *testing.T) { + img := NewFromString("golang:1.5.1") + list := []*ImageName{ + NewFromString("golang:1.4.1"), + NewFromString("golang:stable"), + NewFromString("golang:latest"), + } + assert.Nil(t, img.ResolveVersion(list)) +} + func TestImageIsSameKind(t *testing.T) { assert.True(t, NewFromString("rocker-build").IsSameKind(*NewFromString("rocker-build"))) assert.True(t, NewFromString("rocker-build:latest").IsSameKind(*NewFromString("rocker-build:latest"))) diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/dockerhub.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/registry.go similarity index 62% rename from vendor/src/github.com/grammarly/rocker/src/rocker/imagename/dockerhub.go rename to vendor/src/github.com/grammarly/rocker/src/rocker/imagename/registry.go index 787acbd..a7bb17a 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/dockerhub.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/registry.go @@ -42,14 +42,14 @@ type manifests struct { SchemaVersion int `json:"schemaVersion,omitempty"` } -type hubTags struct { - Count int `json:"count,omitempty"` - Next string `json:"next,omitempty"` - Previous string `json:"previous,omitempty"` - Results []*hubTag `json:"results,omitempty"` +type registryTags struct { + Count int `json:"count,omitempty"` + Next string `json:"next,omitempty"` + Previous string `json:"previous,omitempty"` + Results []*registryTag `json:"results,omitempty"` } -type hubTag struct { +type registryTag struct { Name string `json:"name,omitempty"` FullSize int `json:"full_size,omitempty"` ID int `json:"id,omitempty"` @@ -60,17 +60,8 @@ type hubTag struct { V2 bool `json:"v2,omitempty"` } -// DockerHub is an facade for communicating with registries -// It is used for getting tag manifests and the list of image tags -type DockerHub struct{} - -// NewDockerHub returns new DockerHub instance -func NewDockerHub() *DockerHub { - return &DockerHub{} -} - -// Get returns docker.Image instance from the information stored in the registry -func (h *DockerHub) Get(image *ImageName) (img *docker.Image, err error) { +// RegistryGet returns docker.Image instance from the information stored in the registry +func RegistryGet(image *ImageName) (img *docker.Image, err error) { manifest := manifests{} img = &docker.Image{} @@ -79,7 +70,7 @@ func (h *DockerHub) Get(image *ImageName) (img *docker.Image, err error) { return } - if err = h.doGet(fmt.Sprintf("https://%s/v2/%s/manifests/%s", image.Registry, image.Name, image.Tag), &manifest); err != nil { + if err = registryGet(fmt.Sprintf("https://%s/v2/%s/manifests/%s", image.Registry, image.Name, image.Tag), &manifest); err != nil { return } @@ -93,49 +84,49 @@ func (h *DockerHub) Get(image *ImageName) (img *docker.Image, err error) { return } -// List returns the list of images instances obtained from all tags existing in the registry -func (h *DockerHub) List(image *ImageName) (images []*ImageName, err error) { +// RegistryListTags returns the list of images instances obtained from all tags existing in the registry +func RegistryListTags(image *ImageName) (images []*ImageName, err error) { if image.Registry != "" { - return h.listRegistry(image) + return registryListTags(image) } - return h.listHub(image) + return registryListTagsDockerHub(image) } -// listHub lists image tags from hub.docker.com -func (h *DockerHub) listHub(image *ImageName) (images []*ImageName, err error) { - tg := hubTags{} - if err = h.doGet(fmt.Sprintf("https://hub.docker.com/v2/repositories/library/%s/tags/?page_size=9999&page=1", image.Name), &tg); err != nil { +// registryListTagsDockerHub lists image tags from hub.docker.com +func registryListTagsDockerHub(image *ImageName) (images []*ImageName, err error) { + tg := registryTags{} + if err = registryGet(fmt.Sprintf("https://hub.docker.com/v2/repositories/library/%s/tags/?page_size=9999&page=1", image.Name), &tg); err != nil { return } for _, t := range tg.Results { candidate := New(image.NameWithRegistry(), t.Name) - if image.Contains(candidate) { + if image.Contains(candidate) || image.Tag == candidate.Tag { images = append(images, candidate) } } return } -// listRegistry lists image tags from a private docker registry -func (h *DockerHub) listRegistry(image *ImageName) (images []*ImageName, err error) { +// registryListTags lists image tags from a private docker registry +func registryListTags(image *ImageName) (images []*ImageName, err error) { tg := tags{} - if err = h.doGet(fmt.Sprintf("https://%s/v2/%s/tags/list", image.Registry, image.Name), &tg); err != nil { + if err = registryGet(fmt.Sprintf("https://%s/v2/%s/tags/list", image.Registry, image.Name), &tg); err != nil { return } for _, t := range tg.Tags { candidate := New(image.NameWithRegistry(), t) - if image.Contains(candidate) { + if image.Contains(candidate) || image.Tag == candidate.Tag { images = append(images, candidate) } } return } -// doGet executes HTTP get to a given registry -func (h *DockerHub) doGet(url string, obj interface{}) (err error) { +// registryGet executes HTTP get to a given registry +func registryGet(url string, obj interface{}) (err error) { var res *http.Response var body []byte From 0b555bd8dfd647094d78d9d18774939e21a52e79 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Wed, 30 Sep 2015 15:24:23 +0300 Subject: [PATCH 10/34] pin and override versions from vars --- src/cmd/rocker-compose/main.go | 72 ++++++- src/compose/client.go | 346 ++++++++++++--------------------- src/compose/client_test.go | 31 +++ src/compose/compose.go | 41 +--- src/compose/container.go | 5 - src/compose/diff_test.go | 13 +- 6 files changed, 239 insertions(+), 269 deletions(-) diff --git a/src/cmd/rocker-compose/main.go b/src/cmd/rocker-compose/main.go index 6d8283d..33b857e 100644 --- a/src/cmd/rocker-compose/main.go +++ b/src/cmd/rocker-compose/main.go @@ -19,18 +19,23 @@ package main import ( + "bytes" "compose" "compose/ansible" "compose/config" + "encoding/json" "fmt" + "io" "os" "path" + "path/filepath" "strings" "time" log "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" "github.com/fsouza/go-dockerclient" + "github.com/go-yaml/yaml" "github.com/grammarly/rocker/src/rocker/dockerclient" "github.com/grammarly/rocker/src/rocker/template" ) @@ -93,7 +98,7 @@ func main() { app.Flags = append([]cli.Flag{ cli.BoolFlag{ - Name: "verbose, vv", + Name: "verbose, vv, D", }, cli.StringFlag{ Name: "log, l", @@ -174,7 +179,26 @@ func main() { Name: "pin", Usage: "pin versions", Action: pinCommand, - Flags: append([]cli.Flag{}, composeFlags...), + Flags: append([]cli.Flag{ + cli.BoolTFlag{ + Name: "local, l", + Usage: "search across images available locally", + }, + cli.BoolTFlag{ + Name: "hub", + Usage: "search across images in the registry", + }, + cli.StringFlag{ + Name: "type, t", + Value: "yaml", + Usage: "output in specified format: json|yaml", + }, + cli.StringFlag{ + Name: "output, O", + Value: "-", + Usage: "write result in a file or stdout if the value is `-`", + }, + }, composeFlags...), }, { Name: "recover", @@ -345,7 +369,19 @@ func cleanCommand(ctx *cli.Context) { func pinCommand(ctx *cli.Context) { initLogs(ctx) - log.SetLevel(log.WarnLevel) + var ( + vars template.Vars + data []byte + output = ctx.String("output") + format = ctx.String("type") + local = ctx.BoolT("local") + hub = ctx.BoolT("hub") + fd = os.Stdout + ) + + if output == "-" && !ctx.GlobalIsSet("verbose") { + log.SetLevel(log.WarnLevel) + } dockerCli := initDockerClient(ctx) config := initComposeConfig(ctx, dockerCli) @@ -360,7 +396,35 @@ func pinCommand(ctx *cli.Context) { log.Fatal(err) } - if err := compose.PinAction(); err != nil { + if vars, err = compose.PinAction(local, hub); err != nil { + log.Fatal(err) + } + + if output != "-" { + if fd, err = os.Create(output); err != nil { + log.Fatal(err) + } + defer fd.Close() + + if ext := filepath.Ext(output); !ctx.IsSet("type") && ext == ".json" { + format = "json" + } + } + + switch format { + case "yaml": + if data, err = yaml.Marshal(vars); err != nil { + log.Fatal(err) + } + case "json": + if data, err = json.Marshal(vars); err != nil { + log.Fatal(err) + } + default: + log.Fatalf("Possible tyoes are `yaml` and `json`, unknown type `%s`", format) + } + + if _, err := io.Copy(fd, bytes.NewReader(data)); err != nil { log.Fatal(err) } } diff --git a/src/compose/client.go b/src/compose/client.go index f813cea..92a8d2a 100644 --- a/src/compose/client.go +++ b/src/compose/client.go @@ -23,6 +23,7 @@ import ( "util" "github.com/grammarly/rocker/src/rocker/imagename" + "github.com/grammarly/rocker/src/rocker/template" "github.com/kr/pretty" log "github.com/Sirupsen/logrus" @@ -37,15 +38,15 @@ type Client interface { RunContainer(container *Container) error EnsureContainerExist(name *Container) error EnsureContainerState(name *Container) error - PullAll(containers []*Container) error + PullAll(containers []*Container, vars template.Vars) error Clean(config *config.Config) error AttachToContainers(container []*Container) error AttachToContainer(container *Container) error - FetchImages(containers []*Container) error + FetchImages(containers []*Container, vars template.Vars) error WaitForContainer(container *Container) error GetPulledImages() []*imagename.ImageName GetRemovedImages() []*imagename.ImageName - Pin(containers []*Container) error + Pin(local, hub bool, vars template.Vars, containers []*Container) error } // DockerClient is an implementation of Client interface that do operations to a given docker client @@ -105,7 +106,7 @@ func (a *AuthConfig) ToDockerAPI() *docker.AuthConfiguration { // NewClient makes a new DockerClient object based on configuration params // that is given with input DockerClient object. -func NewClient(initialClient *DockerClient) (Client, error) { +func NewClient(initialClient *DockerClient) (*DockerClient, error) { client := &DockerClient{ Docker: initialClient.Docker, Attach: initialClient.Attach, @@ -312,12 +313,8 @@ func (client *DockerClient) EnsureContainerState(container *Container) error { } // PullAll grabs all image names from containers in spec and pulls all of them -func (client *DockerClient) PullAll(containers []*Container) error { - if err := client.pullImageForContainers(true, containers...); err != nil { - return err - } - - return nil +func (client *DockerClient) PullAll(containers []*Container, vars template.Vars) error { + return client.pullImageForContainers(true, vars, containers...) } // Clean finds the obsolete image tags from container specs that exist in docker daemon, @@ -494,12 +491,8 @@ func (client *DockerClient) WaitForContainer(container *Container) (err error) { } // FetchImages fetches the missing images for all containers in the manifest -func (client *DockerClient) FetchImages(containers []*Container) error { - if err := client.pullImageForContainers(false, containers...); err != nil { - return err - } - - return nil +func (client *DockerClient) FetchImages(containers []*Container, vars template.Vars) error { + return client.pullImageForContainers(false, vars, containers...) } // GetPulledImages returns the list of images pulled by a recent run @@ -513,187 +506,12 @@ func (client *DockerClient) GetRemovedImages() []*imagename.ImageName { } // Pin resolves versions for given containers -func (client *DockerClient) Pin(containers []*Container) error { - return client.pullImageForContainers2(true, containers...) +func (client *DockerClient) Pin(local, hub bool, vars template.Vars, containers []*Container) error { + return client.resolveVersions(local, hub, vars, containers) } // Internal -func (client *DockerClient) pullImageForContainer(container *Container) error { - if container.Image == nil { - return fmt.Errorf("Image is not specified for container: %s", container.Name) - } - - log.Infof("Pulling image: %s for %s", container.Image, container.Name) - - if err := PullDockerImage(client.Docker, container.Image, client.Auth.ToDockerAPI()); err != nil { - return fmt.Errorf("Failed to pull image %s for container %s, error: %s", container.Image, container.Name, err) - } - - client.pulledImages = append(client.pulledImages, container.Image) - - return nil -} - -func (client *DockerClient) pullImageForContainers2(forceUpdate bool, containers ...*Container) (err error) { - pulled := map[string]*imagename.ImageName{} - - // getting all images that we already have - var images []*imagename.ImageName - if images, err = client.getAllImageNames(); err != nil { - return - } - - // check images for each container - for _, container := range containers { - // error in configuration, fail fast - if container.Image == nil { - err = fmt.Errorf("Image is not specified for container: %s", container.Name) - return - } - - // already pulled it for other container, skip - if _, ok := pulled[container.Image.String()]; ok { - // result = append(result, &res{ - // containerName: container.Name, - // image: pulled[container.Image.String()], - // }) - container.Image = pulled[container.Image.String()] - continue - } - - pulled[container.Image.String()] = nil - - // Override to not change the common images slice - images := images - - // in case we want to include external images as well, pulling list of available - // images from repository or central docker hub - if forceUpdate { - hub := imagename.NewDockerHub() - - log.Debugf("Getting list of tags for %s from the registry", container.Image) - - var remote []*imagename.ImageName - if remote, err = hub.List(container.Image); err != nil { - err = fmt.Errorf("Failed to pull image %s for container %s from remote registry, error: %s", - container.Image, container.Name, err) - } else { - images = append(images, remote...) - } - } - - // looking locally first - candidate := findMostRecentTag(container.Image, images) - if candidate == nil { - err = fmt.Errorf("Image not found: %s make sure that it's exist", container.Image) - return - } - - log.Infof("%s --> %s", container.Image, candidate.GetTag()) - - // result = append(result, &res{ - // containerName: container.Name, - // image: candidate, - // }) - - container.Image = candidate - pulled[container.Image.String()] = candidate - } - - return -} - -// pullImageForContainers pulls required images to container daemon storage -// forceUpdate: bool if true - will ensure that the most recent version of required image will be pulled -// containers: []Container - list of containers list of images should be pulled for -// -// force means that if we are using wildcard in image tag and force is false, we will -// choose already pulled appropriate image, otherwise we will find the most recent in -// docker hub of remote registry -func (client *DockerClient) pullImageForContainers(forceUpdate bool, containers ...*Container) (err error) { - pulled := map[string]struct{}{} - - // getting all images that we already have - log.Debug("before getAllImageNames") - var images []*imagename.ImageName - if images, err = client.getAllImageNames(); err != nil { - return - } - log.Debug("after getAllImageNames %s", len(images)) - - // check images for each container - for _, container := range containers { - // error in configuration, fail fast - if container.Image == nil { - err = fmt.Errorf("Image is not specified for container: %s", container.Name) - return - } - - // already pulled it for other container, skip - if _, ok := pulled[container.Image.String()]; ok { - continue - } - - pulled[container.Image.String()] = struct{}{} - - // looking locally first - candidate := findMostRecentTag(container.Image, images) - if candidate != nil { - log.Debugf("Found %s at local repository", candidate) - } - - // force update if we don't find anything locally - forceUpdate = forceUpdate || candidate == nil - - // in case we want to include external images as well, pulling list of available - // images from repository or central docker hub - if forceUpdate { - hub := imagename.NewDockerHub() - var remote []*imagename.ImageName - if remote, err = hub.List(container.Image); err != nil { - err = fmt.Errorf("Failed to pull image %s for container %s from remote registry, error: %s", - container.Image, container.Name, err) - } else { - images = append(images, remote...) - } - - // getting the most applicable image - // it may be local or remote image, it depends of forceUpdate flag - candidate = findMostRecentTag(container.Image, images) - if candidate != nil { - log.Debugf("Found %s at remote repository", candidate) - } - } - - if candidate == nil { - err = fmt.Errorf("Image not found: %s make sure that it's exist", container.Image) - return - } - - if _, err = client.Docker.InspectImage(candidate.String()); err == docker.ErrNoSuchImage { - log.Infof("Pulling image: %s for %s", candidate, container.Name) - if err = PullDockerImage(client.Docker, candidate, client.Auth.ToDockerAPI()); err != nil { - err = fmt.Errorf("Failed to pull image %s for container %s, error: %s", container.Image, container.Name, err) - return - } - } - - if err != nil { - return - } - - log.Debugf("Image %s is the most recent image for container: %s", candidate, container.Name) - - // saving resolved image in case we want to run this container - container.ImageResolved = candidate - - client.pulledImages = append(client.pulledImages, container.Image) - } - - return -} - func (client *DockerClient) listenReAttach(containers []*Container) { // The code is partially borrowed from https://github.com/jwilder/docker-gen eventChan := make(chan *docker.APIEvents, 100) @@ -800,56 +618,140 @@ func (client *DockerClient) flushContainerLogs(container *Container) { } } -func (client *DockerClient) getAllImageNames() (images []*imagename.ImageName, err error) { - var dockerImages []docker.APIImages +// pullImageForContainers goes through all containers and inspects their images +// it pulls images if they cannot be found locally or forceUpdate flag is set to true +func (client *DockerClient) pullImageForContainers(forceUpdate bool, vars template.Vars, containers ...*Container) (err error) { - // retrieving images currently available in docker - if dockerImages, err = client.Docker.ListImages(docker.ListImagesOptions{}); err != nil { - return + if err := client.resolveVersions(true, forceUpdate, vars, containers); err != nil { + return err } - for _, image := range dockerImages { - for _, repoTag := range image.RepoTags { - images = append(images, imagename.NewFromString(repoTag)) + pulled := map[string]struct{}{} + + // check images for each container + for _, container := range containers { + if container.Image == nil { + err = fmt.Errorf("Cannot find image for container %s", container.Name) + return + } + // already pulled it for other container, skip + if _, ok := pulled[container.Image.String()]; ok { + continue + } + pulled[container.Image.String()] = struct{}{} + + if _, err = client.Docker.InspectImage(container.Image.String()); err == docker.ErrNoSuchImage || forceUpdate { + log.Infof("Pulling image: %s for %s", container.Image, container.Name) + if err = PullDockerImage(client.Docker, container.Image, client.Auth.ToDockerAPI()); err != nil { + err = fmt.Errorf("Failed to pull image %s for container %s, error: %s", container.Image, container.Name, err) + return + } + client.pulledImages = append(client.pulledImages, container.Image) + } + if err != nil { + return } } return } -// findMostRecentTag getting all applicable version from dockerhub and choose the most recent -func findMostRecentTag(image *imagename.ImageName, list []*imagename.ImageName) (img *imagename.ImageName) { - for _, candidate := range list { - if !image.Contains(candidate) { - //this image is from the same name/registry but tag is not applicable - // e.g. ~1.2.3 contains 1.2.5, but it's not true for 1.3.0 - continue +// resolveVersions walks through the list of images and resolves their tags in case they are not strict +func (client *DockerClient) resolveVersions(local, hub bool, vars template.Vars, containers []*Container) (err error) { + + // Provide function getter of all images to fetch only once + var available []*imagename.ImageName + getImages := func() ([]*imagename.ImageName, error) { + if available == nil { + available = []*imagename.ImageName{} + + if !local { + return available, nil + } + + // retrieving images currently available in docker + var dockerImages []docker.APIImages + if dockerImages, err = client.Docker.ListImages(docker.ListImagesOptions{}); err != nil { + return nil, err + } + + for _, image := range dockerImages { + for _, repoTag := range image.RepoTags { + available = append(available, imagename.NewFromString(repoTag)) + } + } } + return available, nil + } - if candidate.GetTag() == imagename.Latest { - continue + resolved := map[string]*imagename.ImageName{} + + // check images for each container + for _, container := range containers { + // error in configuration, fail fast + if container.Image == nil { + err = fmt.Errorf("Image is not specified for the container: %s", container.Name) + return } - // if candidate.GetTag() == imagename.Latest { - // // use latest if it's possible - // img = candidate - // return - // } + // Version specified in variables + var k string + k = fmt.Sprintf("v_image_%s", container.Image.NameWithRegistry()) + if tag, ok := vars[k]; ok { + log.Infof("Resolve %s --> %s (derived by variable %s)", container.Image, tag, k) + container.Image.SetTag(tag.(string)) + } + k = fmt.Sprintf("v_container_%s", container.Name.Name) + if tag, ok := vars[k]; ok { + log.Infof("Resolve %s --> %s (derived by variable %s)", container.Image, tag, k) + container.Image.SetTag(tag.(string)) + } - if img == nil { - img = candidate + // Do not resolve anything if the image is strict, e.g. "redis:2.8.11" or "redis:latest" + if container.Image.IsStrict() { continue } - // uncomparable candidate... skipping - if !candidate.HasVersion() { + // already resolved it for other container + if _, ok := resolved[container.Image.String()]; ok { + container.Image = resolved[container.Image.String()] continue } - // if both names has versions to compare, we cat safely compare them - if img.HasVersion() && candidate.HasVersion() && img.TagAsVersion().Less(candidate.TagAsVersion()) { - img = candidate + // Override to not change the common images slice + var images []*imagename.ImageName + if images, err = getImages(); err != nil { + return err + } + + // in case we want to include external images as well, pulling list of available + // images from repository or central docker hub + if hub { + log.Debugf("Getting list of tags for %s from the registry", container.Image) + + var remote []*imagename.ImageName + if remote, err = imagename.RegistryListTags(container.Image); err != nil { + err = fmt.Errorf("Failed to pull image %s for container %s from remote registry, error: %s", + container.Image, container.Name, err) + } else { + for _, img := range remote { + log.Debugf("remote tag for %s: %s", container.Image, img) + } + images = append(images, remote...) + } } + + // looking locally first + candidate := container.Image.ResolveVersion(images) + if candidate == nil { + err = fmt.Errorf("Image not found: %s", container.Image) + return + } + + log.Infof("Resolve %s --> %s", container.Image, candidate.GetTag()) + + container.Image = candidate + resolved[container.Image.String()] = candidate } return diff --git a/src/compose/client_test.go b/src/compose/client_test.go index 4a3f5e4..9b556b2 100644 --- a/src/compose/client_test.go +++ b/src/compose/client_test.go @@ -27,7 +27,9 @@ import ( "github.com/fsouza/go-dockerclient" "github.com/grammarly/rocker/src/rocker/dockerclient" "github.com/grammarly/rocker/src/rocker/imagename" + "github.com/grammarly/rocker/src/rocker/template" "github.com/grammarly/rocker/src/rocker/test" + "github.com/kr/pretty" "github.com/stretchr/testify/assert" ) @@ -210,3 +212,32 @@ containers: assert.EqualValues(t, "rocker-compose-test-image-clean:2", removed[1].String(), "removed wrong image") assert.EqualValues(t, "rocker-compose-test-image-clean:1", removed[2].String(), "removed wrong image") } + +func TestClientResolveVersions(t *testing.T) { + t.Skip() + + dockerCli, err := dockerclient.New() + if err != nil { + t.Fatal(err) + } + + client, err := NewClient(&DockerClient{ + Docker: dockerCli, + }) + if err != nil { + t.Fatal(err) + } + + containers := []*Container{ + &Container{ + Name: config.NewContainerName("test", "test"), + Image: imagename.NewFromString("golang:1.4.*"), + }, + } + + if err := client.resolveVersions(true, true, template.Vars{}, containers); err != nil { + t.Fatal(err) + } + + pretty.Println(containers) +} diff --git a/src/compose/compose.go b/src/compose/compose.go index 69ced88..020e81f 100644 --- a/src/compose/compose.go +++ b/src/compose/compose.go @@ -29,7 +29,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/fsouza/go-dockerclient" - "github.com/go-yaml/yaml" + "github.com/grammarly/rocker/src/rocker/template" "github.com/kr/pretty" ) @@ -112,19 +112,13 @@ func (compose *Compose) RunAction() error { // if --pull is specified PullAll, otherwise Fetch required if compose.Pull { - if err := compose.client.PullAll(expected); err != nil { + if err := compose.client.PullAll(expected, compose.Manifest.Vars); err != nil { return err } - } else if err := compose.client.FetchImages(expected); err != nil { + } else if err := compose.client.FetchImages(expected, compose.Manifest.Vars); err != nil { return fmt.Errorf("Failed to fetch images of given containers, error: %s", err) } - // upgrading expected image to the most recent - // todo: it may be an option to skip this step in some cases - for _, container := range expected { - container.Image = container.ImageResolved - } - // Assign IDs of existing containers for _, actualC := range actual { for _, expectedC := range expected { @@ -232,7 +226,7 @@ func (compose *Compose) RecoverAction() error { // PullAction implements 'rocker-compose pull' func (compose *Compose) PullAction() error { containers := GetContainersFromConfig(compose.Manifest) - if err := compose.client.PullAll(containers); err != nil { + if err := compose.client.PullAll(containers, compose.Manifest.Vars); err != nil { return fmt.Errorf("Failed to pull all images, error: %s", err) } @@ -249,36 +243,19 @@ func (compose *Compose) CleanAction() error { } // PinAction implements 'rocker-compose pin' -func (compose *Compose) PinAction() error { +func (compose *Compose) PinAction(local, hub bool) (template.Vars, error) { containers := GetContainersFromConfig(compose.Manifest) - if err := compose.client.Pin(containers); err != nil { - return fmt.Errorf("Failed to pin, error: %s", err) - } - - type versions struct { - Containers map[string]string - Images map[string]string + if err := compose.client.Pin(local, hub, compose.Manifest.Vars, containers); err != nil { + return nil, fmt.Errorf("Failed to pin, error: %s", err) } // Populate versions to the variables vars := compose.Manifest.Vars - vars["Versions"] = versions{ - Containers: map[string]string{}, - Images: map[string]string{}, - } for _, c := range containers { - vars["Versions"].(versions).Containers[c.Name.Name] = c.Image.GetTag() - vars["Versions"].(versions).Images[c.Image.NameWithRegistry()] = c.Image.GetTag() + vars[fmt.Sprintf("v_container_%s", c.Name.Name)] = c.Image.GetTag() } - data, err := yaml.Marshal(vars) - if err != nil { - return err - } - - fmt.Print(string(data)) - - return nil + return vars, nil } // WritePlan saves various rocker-compose change information to the ansible.Response object diff --git a/src/compose/container.go b/src/compose/container.go index fd9578c..9288378 100644 --- a/src/compose/container.go +++ b/src/compose/container.go @@ -220,11 +220,6 @@ func (a *Container) CreateContainerOptions() (*docker.CreateContainerOptions, er apiConfig.Labels = labels - // Replace image with more specific one (config may contain image range or wildcards) - if a.ImageResolved != nil { - apiConfig.Image = a.ImageResolved.String() - } - return &docker.CreateContainerOptions{ Name: a.Name.String(), Config: apiConfig, diff --git a/src/compose/diff_test.go b/src/compose/diff_test.go index e1194ab..96de4b7 100644 --- a/src/compose/diff_test.go +++ b/src/compose/diff_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/grammarly/rocker/src/rocker/imagename" + "github.com/grammarly/rocker/src/rocker/template" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -463,8 +464,8 @@ func (m *clientMock) PullImage(imageName *imagename.ImageName) error { return args.Error(0) } -func (m *clientMock) PullAll(containers []*Container) error { - args := m.Called(containers) +func (m *clientMock) PullAll(containers []*Container, vars template.Vars) error { + args := m.Called(containers, vars) return args.Error(0) } @@ -483,8 +484,8 @@ func (m *clientMock) AttachToContainers(container []*Container) error { return args.Error(0) } -func (m *clientMock) FetchImages(container []*Container) error { - args := m.Called(container) +func (m *clientMock) FetchImages(container []*Container, vars template.Vars) error { + args := m.Called(container, vars) return args.Error(0) } @@ -503,8 +504,8 @@ func (m *clientMock) GetRemovedImages() []*imagename.ImageName { return []*imagename.ImageName{} } -func (m *clientMock) Pin(container []*Container) error { - args := m.Called(container) +func (m *clientMock) Pin(local, hub bool, vars template.Vars, container []*Container) error { + args := m.Called(local, hub, vars, container) return args.Error(0) } From b77881347bcfa176d47e010f1b618a2bceaf92d8 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Wed, 30 Sep 2015 18:54:01 +0300 Subject: [PATCH 11/34] bump 0.1.3 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d917d3e..b1e80bb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.2 +0.1.3 From fa64e541046a0e6b9b70bebb1e935b24acf642c2 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Thu, 1 Oct 2015 13:59:55 +0300 Subject: [PATCH 12/34] fix passing resolved image tag to docker run --- src/compose/container.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/compose/container.go b/src/compose/container.go index 9288378..5c00df4 100644 --- a/src/compose/container.go +++ b/src/compose/container.go @@ -219,6 +219,7 @@ func (a *Container) CreateContainerOptions() (*docker.CreateContainerOptions, er labels["rocker-compose-config"] = string(yamlData) apiConfig.Labels = labels + apiConfig.Image = a.Image.String() return &docker.CreateContainerOptions{ Name: a.Name.String(), From 545930150e78d46cda65617e157b135db40af5f8 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Thu, 1 Oct 2015 14:46:00 +0300 Subject: [PATCH 13/34] vendor add rocker/textformatter --- vendor/manifest | 7 + .../src/rocker/textformatter/textformatter.go | 165 ++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 vendor/src/github.com/grammarly/rocker/src/rocker/textformatter/textformatter.go diff --git a/vendor/manifest b/vendor/manifest index 8a18ed6..abae95b 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -124,6 +124,13 @@ "revision": "2ddda7b61203075ed35666dbaec9a7cf0fef4e08", "branch": "master", "path": "/src/rocker/imagename" + }, + { + "importpath": "github.com/grammarly/rocker/src/rocker/textformatter", + "repository": "https://github.com/grammarly/rocker", + "revision": "4d0275e3fcdd6460c7fe467678b04721bc19c9c7", + "branch": "v1", + "path": "/src/rocker/textformatter" } ] } \ No newline at end of file diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/textformatter/textformatter.go b/vendor/src/github.com/grammarly/rocker/src/rocker/textformatter/textformatter.go new file mode 100644 index 0000000..18c74c7 --- /dev/null +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/textformatter/textformatter.go @@ -0,0 +1,165 @@ +// The MIT License (MIT) +// Copyright (c) 2014 Simon Eskildsen +// NOTE: modified to support no-color mode that is more human readable + +package textformatter + +import ( + "bytes" + "fmt" + "runtime" + "sort" + "strings" + "time" + + log "github.com/Sirupsen/logrus" +) + +const ( + nocolor = 0 + red = 31 + green = 32 + yellow = 33 + blue = 34 + gray = 37 +) + +var ( + baseTimestamp time.Time + isTerminal bool +) + +func init() { + baseTimestamp = time.Now() + isTerminal = log.IsTerminal() +} + +func miniTS() int { + return int(time.Since(baseTimestamp) / time.Second) +} + +// TextFormatter is a formatter for logrus that can print colored and uncolored human readable log messages +type TextFormatter struct { + // Set to true to bypass checking for a TTY before outputting colors. + ForceColors bool + + // Force disabling colors. + DisableColors bool + + // Disable timestamp logging. useful when output is redirected to logging + // system that already adds timestamps. + DisableTimestamp bool + + // Enable logging the full timestamp when a TTY is attached instead of just + // the time passed since beginning of execution. + FullTimestamp bool + + // TimestampFormat to use for display when a full timestamp is printed + TimestampFormat string + + // The fields are sorted by default for a consistent output. For applications + // that log extremely frequently and don't use the JSON formatter this may not + // be desired. + DisableSorting bool +} + +// Format formats log message string, it checks if the output should be colored +// and doest a particular formatting +func (f *TextFormatter) Format(entry *log.Entry) ([]byte, error) { + var keys = make([]string, 0, len(entry.Data)) + for k := range entry.Data { + keys = append(keys, k) + } + + if !f.DisableSorting { + sort.Strings(keys) + } + + b := &bytes.Buffer{} + + prefixFieldClashes(entry.Data) + + isColorTerminal := isTerminal && (runtime.GOOS != "windows") + isColored := (f.ForceColors || isColorTerminal) && !f.DisableColors + + if f.TimestampFormat == "" { + f.TimestampFormat = log.DefaultTimestampFormat + } + if isColored { + f.printColored(b, entry, keys) + } else { + f.printUncolored(b, entry, keys) + } + + b.WriteByte('\n') + return b.Bytes(), nil +} + +func (f *TextFormatter) printColored(b *bytes.Buffer, entry *log.Entry, keys []string) { + var levelColor int + switch entry.Level { + case log.DebugLevel: + levelColor = gray + case log.WarnLevel: + levelColor = yellow + case log.ErrorLevel, log.FatalLevel, log.PanicLevel: + levelColor = red + default: + levelColor = blue + } + + levelText := strings.ToUpper(entry.Level.String())[0:4] + + if !f.FullTimestamp { + fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%04d] %-44s ", levelColor, levelText, miniTS(), entry.Message) + } else { + fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %-44s ", levelColor, levelText, entry.Time.Format(f.TimestampFormat), entry.Message) + } + for _, k := range keys { + v := entry.Data[k] + fmt.Fprintf(b, " \x1b[%dm%s\x1b[0m=%+v", levelColor, k, v) + } +} + +func (f *TextFormatter) printUncolored(b *bytes.Buffer, entry *log.Entry, keys []string) { + levelText := strings.ToUpper(entry.Level.String())[0:4] + + if !f.FullTimestamp { + fmt.Fprintf(b, "%s[%04d] %-44s ", levelText, miniTS(), entry.Message) + } else { + fmt.Fprintf(b, "%s[%s] %-44s ", levelText, entry.Time.Format(f.TimestampFormat), entry.Message) + } + for _, k := range keys { + v := entry.Data[k] + fmt.Fprintf(b, " %s=%+v", k, v) + } +} + +// This is to not silently overwrite `time`, `msg` and `level` fields when +// dumping it. If this code wasn't there doing: +// +// logrus.WithField("level", 1).Info("hello") +// +// Would just silently drop the user provided level. Instead with this code +// it'll logged as: +// +// {"level": "info", "fields.level": 1, "msg": "hello", "time": "..."} +// +// It's not exported because it's still using Data in an opinionated way. It's to +// avoid code duplication between the two default formatters. +func prefixFieldClashes(data log.Fields) { + _, ok := data["time"] + if ok { + data["fields.time"] = data["time"] + } + + _, ok = data["msg"] + if ok { + data["fields.msg"] = data["msg"] + } + + _, ok = data["level"] + if ok { + data["fields.level"] = data["level"] + } +} From 8a0068bd64df9e54e38b27f327e46a0324d2961a Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Thu, 1 Oct 2015 14:46:11 +0300 Subject: [PATCH 14/34] make better non-colored logger --- src/cmd/rocker-compose/main.go | 49 +++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/src/cmd/rocker-compose/main.go b/src/cmd/rocker-compose/main.go index 33b857e..6f34218 100644 --- a/src/cmd/rocker-compose/main.go +++ b/src/cmd/rocker-compose/main.go @@ -38,6 +38,7 @@ import ( "github.com/go-yaml/yaml" "github.com/grammarly/rocker/src/rocker/dockerclient" "github.com/grammarly/rocker/src/rocker/template" + "github.com/grammarly/rocker/src/rocker/textformatter" ) var ( @@ -111,6 +112,9 @@ func main() { Value: "", Usage: "Docker auth, username and password in user:password format", }, + cli.BoolTFlag{ + Name: "colors", + }, }, dockerclient.GlobalCliParams()...) app.Commands = []cli.Command{ @@ -453,33 +457,42 @@ func recoverCommand(ctx *cli.Context) { } func initLogs(ctx *cli.Context) { + logger := log.StandardLogger() + if ctx.GlobalBool("verbose") { - log.SetLevel(log.DebugLevel) + logger.Level = log.DebugLevel } - if ctx.GlobalBool("json") { - log.SetFormatter(&log.JSONFormatter{}) - } + var ( + err error + isTerm = log.IsTerminal() + logFile = ctx.GlobalString("log") + logExt = path.Ext(logFile) + json = ctx.GlobalBool("json") || logExt == ".json" + useColors = isTerm && !json && logFile == "" + ) - logFilename, err := toAbsolutePath(ctx.GlobalString("log"), false) - if err != nil { - log.Debugf("Initializing log: Skipped, because Log %s", err) - return + if ctx.GlobalIsSet("colors") { + useColors = ctx.GlobalBool("colors") } - logFile, err := os.OpenFile(logFilename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) - if err != nil { - log.Warnf("Initializing log: Cannot initialize log file %s due to error %s", logFilename, err) - return + if logFile != "" { + if logFile, err = toAbsolutePath(logFile, false); err != nil { + log.Fatal(err) + } + if logger.Out, err = os.OpenFile(logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644); err != nil { + log.Fatalf("Initializing log: Cannot initialize log file %s due to error %s", logFile, err) + } + log.Debugf("Initializing log: Successfuly started loggin to '%s'", logFile) } - log.SetOutput(logFile) - - if path.Ext(logFilename) == ".json" { - log.SetFormatter(&log.JSONFormatter{}) + if json { + logger.Formatter = &log.JSONFormatter{} + } else { + formatter := &textformatter.TextFormatter{} + formatter.DisableColors = !useColors + logger.Formatter = formatter } - - log.Debugf("Initializing log: Successfuly started loggin to '%s'", logFilename) } func initComposeConfig(ctx *cli.Context, dockerCli *docker.Client) *config.Config { From 12c61360633e24c58cd7444ade125e599fd9b8a0 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Wed, 7 Oct 2015 15:57:36 +0300 Subject: [PATCH 15/34] improve error message --- src/compose/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compose/client.go b/src/compose/client.go index 92a8d2a..63f5a44 100644 --- a/src/compose/client.go +++ b/src/compose/client.go @@ -731,7 +731,7 @@ func (client *DockerClient) resolveVersions(local, hub bool, vars template.Vars, var remote []*imagename.ImageName if remote, err = imagename.RegistryListTags(container.Image); err != nil { - err = fmt.Errorf("Failed to pull image %s for container %s from remote registry, error: %s", + err = fmt.Errorf("Failed to list tags of image %s for container %s from the remote registry, error: %s", container.Image, container.Name, err) } else { for _, img := range remote { From 9371ae9e4ae1c0b9fa98a4adcccaec65180ed530 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Wed, 7 Oct 2015 16:07:08 +0300 Subject: [PATCH 16/34] Fix: update container if image id changed @hitthefrei --- src/compose/client.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/compose/client.go b/src/compose/client.go index 63f5a44..b3aecf1 100644 --- a/src/compose/client.go +++ b/src/compose/client.go @@ -626,7 +626,10 @@ func (client *DockerClient) pullImageForContainers(forceUpdate bool, vars templa return err } - pulled := map[string]struct{}{} + var ( + img *docker.Image + pulled = map[string]*docker.Image{} + ) // check images for each container for _, container := range containers { @@ -635,12 +638,12 @@ func (client *DockerClient) pullImageForContainers(forceUpdate bool, vars templa return } // already pulled it for other container, skip - if _, ok := pulled[container.Image.String()]; ok { + if img, ok := pulled[container.Image.String()]; ok { + container.ImageID = img.ID continue } - pulled[container.Image.String()] = struct{}{} - if _, err = client.Docker.InspectImage(container.Image.String()); err == docker.ErrNoSuchImage || forceUpdate { + if img, err = client.Docker.InspectImage(container.Image.String()); err == docker.ErrNoSuchImage || forceUpdate { log.Infof("Pulling image: %s for %s", container.Image, container.Name) if err = PullDockerImage(client.Docker, container.Image, client.Auth.ToDockerAPI()); err != nil { err = fmt.Errorf("Failed to pull image %s for container %s, error: %s", container.Image, container.Name, err) @@ -651,6 +654,9 @@ func (client *DockerClient) pullImageForContainers(forceUpdate bool, vars templa if err != nil { return } + + container.ImageID = img.ID + pulled[container.Image.String()] = img } return From eafb0efbc5ecb3f272d78e51c542b0f861bc2f02 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Wed, 7 Oct 2015 16:51:48 +0300 Subject: [PATCH 17/34] Ping docker server before running --- src/cmd/rocker-compose/main.go | 5 +++++ vendor/manifest | 14 ++++++------- .../src/rocker/dockerclient/dockerclient.go | 21 +++++++++++++++++++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/cmd/rocker-compose/main.go b/src/cmd/rocker-compose/main.go index 6f34218..d791eb1 100644 --- a/src/cmd/rocker-compose/main.go +++ b/src/cmd/rocker-compose/main.go @@ -555,6 +555,11 @@ func initComposeConfig(ctx *cli.Context, dockerCli *docker.Client) *config.Confi log.Fatal(err) } + // Check the docker connection before we actually run + if err := dockerclient.Ping(dockerCli, 5000); err != nil { + log.Fatal(err) + } + return manifest } diff --git a/vendor/manifest b/vendor/manifest index abae95b..840d680 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -85,13 +85,6 @@ "branch": "master", "path": "/src/rocker/test" }, - { - "importpath": "github.com/grammarly/rocker/src/rocker/dockerclient", - "repository": "https://github.com/grammarly/rocker", - "revision": "79bcadfcce121a8d9fa0f84e231314c6374bf8f3", - "branch": "master", - "path": "/src/rocker/dockerclient" - }, { "importpath": "github.com/grammarly/rocker/src/rocker/util", "repository": "https://github.com/grammarly/rocker", @@ -131,6 +124,13 @@ "revision": "4d0275e3fcdd6460c7fe467678b04721bc19c9c7", "branch": "v1", "path": "/src/rocker/textformatter" + }, + { + "importpath": "github.com/grammarly/rocker/src/rocker/dockerclient", + "repository": "https://github.com/grammarly/rocker", + "revision": "80510375c825db6731268f9f4f247a404879c966", + "branch": "v1", + "path": "/src/rocker/dockerclient" } ] } \ No newline at end of file diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/dockerclient/dockerclient.go b/vendor/src/github.com/grammarly/rocker/src/rocker/dockerclient/dockerclient.go index 20e8a20..7077c89 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/dockerclient/dockerclient.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/dockerclient/dockerclient.go @@ -25,6 +25,7 @@ import ( "os" "strconv" "strings" + "time" "github.com/codegangsta/cli" "github.com/fsouza/go-dockerclient" @@ -95,6 +96,26 @@ func NewFromCli(c *cli.Context) (*docker.Client, error) { return NewFromConfig(NewConfigFromCli(c)) } +// Ping pings docker client but with timeout +// The problem is that for some reason it's impossible to set the +// default timeout for the go-dockerclient Dialer, need to investigate +func Ping(client *docker.Client, timeoutMs int) error { + var ( + chErr = make(chan error) + timeout = time.Duration(timeoutMs) * time.Millisecond + ) + go func() { + chErr <- client.Ping() + }() + select { + case err := <-chErr: + return err + case <-time.After(timeout): + // TODO: can we kill the ping goroutine? + return fmt.Errorf("Failed to reach docker server, timeout %s", timeout) + } +} + // GlobalCliParams returns global params that configures docker client connection func GlobalCliParams() []cli.Flag { return []cli.Flag{ From 323a57b83a6ebcf75f0a0dbf46ab38d239ba813a Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Fri, 9 Oct 2015 08:35:21 +0300 Subject: [PATCH 18/34] fix panic when forcePull enabled --- src/compose/client.go | 2 +- src/compose/docker.go | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/compose/client.go b/src/compose/client.go index b3aecf1..96e6e28 100644 --- a/src/compose/client.go +++ b/src/compose/client.go @@ -645,7 +645,7 @@ func (client *DockerClient) pullImageForContainers(forceUpdate bool, vars templa if img, err = client.Docker.InspectImage(container.Image.String()); err == docker.ErrNoSuchImage || forceUpdate { log.Infof("Pulling image: %s for %s", container.Image, container.Name) - if err = PullDockerImage(client.Docker, container.Image, client.Auth.ToDockerAPI()); err != nil { + if img, err = PullDockerImage(client.Docker, container.Image, client.Auth.ToDockerAPI()); err != nil { err = fmt.Errorf("Failed to pull image %s for container %s, error: %s", container.Image, container.Name, err) return } diff --git a/src/compose/docker.go b/src/compose/docker.go index 45f3cf0..d932305 100644 --- a/src/compose/docker.go +++ b/src/compose/docker.go @@ -48,7 +48,7 @@ func GetBridgeIP(client *docker.Client) (ip string, err error) { _, err = client.InspectImage(emptyImageName) if err != nil && err.Error() == "no such image" { log.Infof("Pulling image %s to obtain network bridge address", emptyImageName) - if err := PullDockerImage(client, imagename.NewFromString(emptyImageName), &docker.AuthConfiguration{}); err != nil { + if _, err := PullDockerImage(client, imagename.NewFromString(emptyImageName), &docker.AuthConfiguration{}); err != nil { return "", err } } else if err != nil { @@ -89,7 +89,7 @@ func GetBridgeIP(client *docker.Client) (ip string, err error) { } // PullDockerImage pulls an image and streams to a logger respecting terminal features -func PullDockerImage(client *docker.Client, image *imagename.ImageName, auth *docker.AuthConfiguration) error { +func PullDockerImage(client *docker.Client, image *imagename.ImageName, auth *docker.AuthConfiguration) (*docker.Image, error) { pipeReader, pipeWriter := io.Pipe() pullOpts := docker.PullImageOptions{ @@ -121,12 +121,17 @@ func PullDockerImage(client *docker.Client, image *imagename.ImageName, auth *do } if err := jsonmessage.DisplayJSONMessagesStream(pipeReader, out, fd, isTerminal); err != nil { - return fmt.Errorf("Failed to process json stream for image: %s, error: %s", image, err) + return nil, fmt.Errorf("Failed to process json stream for image: %s, error: %s", image, err) } if err := <-errch; err != nil { - return fmt.Errorf("Failed to pull image %s, error: %s", image, err) + return nil, fmt.Errorf("Failed to pull image %s, error: %s", image, err) } - return nil + img, err := client.InspectImage(image.String()) + if err != nil { + return nil, fmt.Errorf("Failed to inspect image %s after pull, error: %s", image, err) + } + + return img, nil } From ccf7a45292fea51e203ac805c9574a46c305d15f Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 13 Oct 2015 09:46:54 +0300 Subject: [PATCH 19/34] vendor update rocker/imagename to support @sha256 tags --- vendor/manifest | 14 ++--- .../rocker/src/rocker/imagename/imagename.go | 57 ++++++++++++++----- .../src/rocker/imagename/imagename_test.go | 48 +++++++++++++++- 3 files changed, 97 insertions(+), 22 deletions(-) diff --git a/vendor/manifest b/vendor/manifest index 840d680..d237f56 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -111,13 +111,6 @@ "branch": "master", "path": "/src/rocker/template" }, - { - "importpath": "github.com/grammarly/rocker/src/rocker/imagename", - "repository": "https://github.com/grammarly/rocker", - "revision": "2ddda7b61203075ed35666dbaec9a7cf0fef4e08", - "branch": "master", - "path": "/src/rocker/imagename" - }, { "importpath": "github.com/grammarly/rocker/src/rocker/textformatter", "repository": "https://github.com/grammarly/rocker", @@ -131,6 +124,13 @@ "revision": "80510375c825db6731268f9f4f247a404879c966", "branch": "v1", "path": "/src/rocker/dockerclient" + }, + { + "importpath": "github.com/grammarly/rocker/src/rocker/imagename", + "repository": "https://github.com/grammarly/rocker", + "revision": "c9dd7ca1515b615c73507fb95555a6400ad59027", + "branch": "master", + "path": "/src/rocker/imagename" } ] } \ No newline at end of file diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go index ec681d9..3f6f2c7 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go @@ -46,34 +46,59 @@ type ImageName struct { // NewFromString parses a given string and returns ImageName func NewFromString(image string) *ImageName { - n := strings.LastIndex(image, ":") - if n < 0 { - return New(image, "") - } - if tag := image[n+1:]; !strings.Contains(tag, "/") { - return New(image[:n], tag) - } - return New(image, "") + name, tag := ParseRepositoryTag(image) + return New(name, tag) } // New parses a given 'image' and 'tag' strings and returns ImageName func New(image string, tag string) *ImageName { dockerImage := &ImageName{} - if strings.Contains(image, ".") || len(strings.SplitN(image, "/", 3)) > 2 { - registryAndName := strings.SplitN(image, "/", 2) - dockerImage.Registry = registryAndName[0] - dockerImage.Name = registryAndName[1] - } else { + nameParts := strings.SplitN(image, "/", 2) + + firstIsHost := strings.Contains(nameParts[0], ".") || + strings.Contains(nameParts[0], ":") || + nameParts[0] == "localhost" + + if len(nameParts) == 1 || !firstIsHost { dockerImage.Name = image + } else { + dockerImage.Registry = nameParts[0] + dockerImage.Name = nameParts[1] } + if tag != "" { dockerImage.SetTag(tag) } + return dockerImage } +// ParseRepositoryTag gets a repos name and returns the right reposName + tag|digest +// The tag can be confusing because of a port in a repository name. +// Ex: localhost.localdomain:5000/samalba/hipache:latest +// Digest ex: localhost:5000/foo/bar@sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb +// NOTE: borrowed from Docker under Apache 2.0, Copyright 2013-2015 Docker, Inc. +func ParseRepositoryTag(repos string) (string, string) { + n := strings.Index(repos, "@") + if n >= 0 { + parts := strings.Split(repos, "@") + return parts[0], parts[1] + } + n = strings.LastIndex(repos, ":") + if n < 0 { + return repos, "" + } + if tag := repos[n+1:]; !strings.Contains(tag, "/") { + return repos[:n], tag + } + return repos, "" +} + // String returns the string representation of the current image name func (img ImageName) String() string { + if img.TagIsSha() { + return img.NameWithRegistry() + "@" + img.GetTag() + } return img.NameWithRegistry() + ":" + img.GetTag() } @@ -82,6 +107,12 @@ func (img ImageName) HasTag() bool { return img.Tag != "" } +// TagIsSha returns true if the tag is content addressable sha256 +// e.g. golang@sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11 +func (img ImageName) TagIsSha() bool { + return strings.HasPrefix(img.Tag, "sha256:") +} + // GetTag returns the tag of the current image name func (img ImageName) GetTag() string { if img.HasTag() { diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go index 2e6a084..e3374e6 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go @@ -161,8 +161,8 @@ func TestImageRealLifeNamingExampleWithCapi(t *testing.T) { func TestImageParsingWithNamespace(t *testing.T) { img := NewFromString("hub/ns/name:1") - assert.Equal(t, "hub", img.Registry) - assert.Equal(t, "ns/name", img.Name) + assert.Equal(t, "", img.Registry) + assert.Equal(t, "hub/ns/name", img.Name) assert.Equal(t, "1", img.Tag) } @@ -175,6 +175,42 @@ func TestImageParsingWithoutTag(t *testing.T) { assert.Equal(t, "repo/name:latest", img.String()) } +func TestImageWithDotsWithoutTag(t *testing.T) { + img := NewFromString("a.b.c.d") + assert.Equal(t, "", img.Registry) + assert.Equal(t, "a.b.c.d", img.Name) + assert.Equal(t, "latest", img.GetTag()) + assert.Equal(t, false, img.HasTag()) + assert.Equal(t, "a.b.c.d:latest", img.String()) +} + +func TestImageWithDotsWithTag(t *testing.T) { + img := NewFromString("a.b.c.d:snapshot") + assert.Equal(t, "", img.Registry) + assert.Equal(t, "a.b.c.d", img.Name) + assert.Equal(t, "snapshot", img.GetTag()) + assert.Equal(t, true, img.HasTag()) + assert.Equal(t, "a.b.c.d:snapshot", img.String()) +} + +func TestImageWithRegistryAndDotsAndTag(t *testing.T) { + img := NewFromString("hub.com/a.b.c.d:snapshot") + assert.Equal(t, "hub.com", img.Registry) + assert.Equal(t, "a.b.c.d", img.Name) + assert.Equal(t, "snapshot", img.GetTag()) + assert.Equal(t, true, img.HasTag()) + assert.Equal(t, "hub.com/a.b.c.d:snapshot", img.String()) +} + +func TestImageWithRegistryAndSlashAndDotsAndTag(t *testing.T) { + img := NewFromString("hub.com/a.b/c.d:snapshot") + assert.Equal(t, "hub.com", img.Registry) + assert.Equal(t, "a.b/c.d", img.Name) + assert.Equal(t, "snapshot", img.GetTag()) + assert.Equal(t, true, img.HasTag()) + assert.Equal(t, "hub.com/a.b/c.d:snapshot", img.String()) +} + func TestImageLatest(t *testing.T) { img := NewFromString("rocker-build:latest") assert.Equal(t, "", img.Registry, "bag registry value") @@ -189,6 +225,14 @@ func TestImageIpRegistry(t *testing.T) { assert.Equal(t, "1.4", img.GetTag(), "bad image tag") } +func TestImageTagSha(t *testing.T) { + img := NewFromString("golang@sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11") + assert.Equal(t, "", img.Registry, "bag registry value") + assert.Equal(t, "golang", img.Name, "bad image name") + assert.Equal(t, "sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11", img.GetTag(), "bad image tag") + assert.Equal(t, "golang@sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11", img.String()) +} + func TestImageAll(t *testing.T) { img := NewFromString("golang:1.*") assert.False(t, img.All()) From dca4c4ab61a8b795bd15c2427a87030ef1318411 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 13 Oct 2015 16:08:13 +0300 Subject: [PATCH 20/34] vendor update rocker/template --- vendor/manifest | 14 +++---- .../rocker/src/rocker/template/README.md | 18 +++++++++ .../rocker/src/rocker/template/template.go | 39 +++++++++++++++++-- .../src/rocker/template/template_test.go | 4 ++ 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/vendor/manifest b/vendor/manifest index d237f56..329aeb8 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -104,13 +104,6 @@ "revision": "461c06b538be53cc0339815001a398e29ace025b", "branch": "master" }, - { - "importpath": "github.com/grammarly/rocker/src/rocker/template", - "repository": "https://github.com/grammarly/rocker", - "revision": "15024281d69f1e7a1814204e60b3ac8c771691ce", - "branch": "master", - "path": "/src/rocker/template" - }, { "importpath": "github.com/grammarly/rocker/src/rocker/textformatter", "repository": "https://github.com/grammarly/rocker", @@ -131,6 +124,13 @@ "revision": "c9dd7ca1515b615c73507fb95555a6400ad59027", "branch": "master", "path": "/src/rocker/imagename" + }, + { + "importpath": "github.com/grammarly/rocker/src/rocker/template", + "repository": "https://github.com/grammarly/rocker", + "revision": "794a751c93ae62c04c6ef4530a265e13f7318573", + "branch": "dev", + "path": "/src/rocker/template" } ] } \ No newline at end of file diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/README.md b/vendor/src/github.com/grammarly/rocker/src/rocker/template/README.md index b666f5e..6382429 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/README.md +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/README.md @@ -89,6 +89,24 @@ DOCKER_MACHINE_NAME: dev PATH: /usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin ``` +### {{ *anything* | yaml *N* }} where *N* is indentation level +Useful if you want to nest a yaml struct into another yaml file: + +```yaml +foo: + bar: + {{ .bar | yaml 2 }} +``` + +Will indent yaml-encoded `.bar` into two levels: + +```yaml +foo: + bar: + a: 1 + b: 2 +``` + ### {{ shell *string* }} or {{ *string* | shell }} Escapes given string so it can be substituted to a shell command. diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go b/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go index a620b18..1681b11 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go @@ -195,12 +195,45 @@ func jsonFn(v interface{}) (string, error) { return string(data), nil } -func yamlFn(v interface{}) (string, error) { - data, err := yaml.Marshal(v) +func yamlFn(args ...interface{}) (result string, err error) { + var ( + i = 0 + input interface{} + ) + + if len(args) == 1 { + input = args[0] + } else if len(args) == 2 { + if i, err = interfaceToInt(args[0]); err != nil { + return "", err + } + input = args[1] + } else { + return "", fmt.Errorf("yaml helper expects from 1 to 2 arguments, %d given", len(args)) + } + + data, err := yaml.Marshal(input) if err != nil { return "", err } - return string(data), nil + result = string(data) + + if i > 0 { + result = indent(strings.Repeat(" ", i), result) + } + + return result, nil +} + +func indent(prefix, s string) string { + var res []string + for _, line := range strings.Split(s, "\n") { + if line != "" { + line = prefix + line + } + res = append(res, line) + } + return strings.Join(res, "\n") } func interfaceToInt(v interface{}) (int, error) { diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/template_test.go b/vendor/src/github.com/grammarly/rocker/src/rocker/template/template_test.go index d038c1a..837dfc0 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/template_test.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/template_test.go @@ -116,6 +116,10 @@ func TestProcess_Yaml(t *testing.T) { assert.Equal(t, "key: |-\n hello\n world\n", processTemplate(t, "key: {{ \"hello\\nworld\" | yaml }}")) } +func TestProcess_YamlIndent(t *testing.T) { + assert.Equal(t, "key:\n foo: bar\n", processTemplate(t, "key:\n{{ .data | yaml 1 }}")) +} + func processTemplate(t *testing.T, tpl string) string { result, err := Process("test", strings.NewReader(tpl), configTemplateVars, map[string]interface{}{}) if err != nil { From 94b3a12f99d85895ddefe4a37694ea82a10f16ef Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 20 Oct 2015 18:02:50 +0300 Subject: [PATCH 21/34] #23 vendor update rocker/imagename --- vendor/manifest | 4 +- .../rocker/src/rocker/imagename/artifact.go | 62 +++++++++++++++++++ .../rocker/src/rocker/imagename/imagename.go | 5 +- .../src/rocker/imagename/imagename_test.go | 16 +++++ 4 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 vendor/src/github.com/grammarly/rocker/src/rocker/imagename/artifact.go diff --git a/vendor/manifest b/vendor/manifest index 329aeb8..46d90c8 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -121,8 +121,8 @@ { "importpath": "github.com/grammarly/rocker/src/rocker/imagename", "repository": "https://github.com/grammarly/rocker", - "revision": "c9dd7ca1515b615c73507fb95555a6400ad59027", - "branch": "master", + "revision": "51b6dce83b3f7b39f2a3843b00c57a5c614c0950", + "branch": "v1", "path": "/src/rocker/imagename" }, { diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/artifact.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/artifact.go new file mode 100644 index 0000000..7b1dd73 --- /dev/null +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/artifact.go @@ -0,0 +1,62 @@ +/*- + * Copyright 2015 Grammarly, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package imagename + +import ( + "fmt" + "strings" + + "time" +) + +// Artifact represents the artifact that is the result of image build +// It holds information about the pushed image and may be saved as a file +type Artifact struct { + Name *ImageName `yaml:"Name"` + Pushed bool `yaml:"Pushed"` + Tag string `yaml:"Tag"` + Digest string `yaml:"Digest"` + ImageID string `yaml:"ImageID"` + Addressable string `yaml:"Addressable"` + BuildTime time.Time `yaml:"BuildTime"` +} + +// Artifacts is a collection of Artifact entities +type Artifacts struct { + RockerArtifacts []Artifact `yaml:"RockerArtifacts"` +} + +// GetFileName constructs the base file name out of the image info +func (a *Artifact) GetFileName() string { + imageName := strings.Replace(a.Name.Name, "/", "_", -1) + return fmt.Sprintf("%s_%s.yml", imageName, a.Name.GetTag()) +} + +// Len returns the length of image tags +func (a *Artifacts) Len() int { + return len(a.RockerArtifacts) +} + +// Less returns true if item by index[i] is created after of item[j] +func (a *Artifacts) Less(i, j int) bool { + return a.RockerArtifacts[i].Name.Tag > a.RockerArtifacts[j].Name.Tag +} + +// Swap swaps items by indices [i] and [j] +func (a *Artifacts) Swap(i, j int) { + a.RockerArtifacts[i], a.RockerArtifacts[j] = a.RockerArtifacts[j], a.RockerArtifacts[i] +} diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go index 3f6f2c7..0e9d9cd 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go @@ -24,7 +24,6 @@ import ( "sort" "strings" - "github.com/go-yaml/yaml" "github.com/wmark/semver" ) @@ -287,8 +286,8 @@ func (img *ImageName) UnmarshalYAML(unmarshal func(interface{}) error) error { } // MarshalYAML serializes ImageName to YAML string -func (img ImageName) MarshalYAML() ([]byte, error) { - return yaml.Marshal(img.String()) +func (img ImageName) MarshalYAML() (interface{}, error) { + return img.String(), nil } // Tags is a structure used for cleaning images diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go index e3374e6..f9e8cf6 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "github.com/go-yaml/yaml" "github.com/kr/pretty" "github.com/stretchr/testify/assert" @@ -476,3 +477,18 @@ func TestTagsGetOld(t *testing.T) { assert.Equal(t, "hub/ns/name:2", old[1].String(), "bad old image 2") assert.Equal(t, "hub/ns/name:1", old[2].String(), "bad old image 3") } + +func TestImagename_ToYaml(t *testing.T) { + value := struct { + Name *ImageName + }{ + NewFromString("hub/ns/name:1"), + } + + data, err := yaml.Marshal(value) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "name: hub/ns/name:1\n", string(data)) +} From 5c5817dbfe20b369f174d9b69ce175268bd29cf8 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 20 Oct 2015 18:03:00 +0300 Subject: [PATCH 22/34] #23 vendor update rocker/template --- vendor/manifest | 4 +- .../rocker/src/rocker/template/README.md | 34 +++++++ .../rocker/src/rocker/template/template.go | 77 ++++++++++++++- .../src/rocker/template/template_test.go | 99 +++++++++++++++++++ .../rocker/src/rocker/template/vars.go | 81 +++++++++++++-- .../rocker/src/rocker/template/vars_test.go | 40 +++++++- 6 files changed, 324 insertions(+), 11 deletions(-) diff --git a/vendor/manifest b/vendor/manifest index 46d90c8..7d61a96 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -128,8 +128,8 @@ { "importpath": "github.com/grammarly/rocker/src/rocker/template", "repository": "https://github.com/grammarly/rocker", - "revision": "794a751c93ae62c04c6ef4530a265e13f7318573", - "branch": "dev", + "revision": "51b6dce83b3f7b39f2a3843b00c57a5c614c0950", + "branch": "v1", "path": "/src/rocker/template" } ] diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/README.md b/vendor/src/github.com/grammarly/rocker/src/rocker/template/README.md index 6382429..f7cf4bf 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/README.md +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/README.md @@ -153,6 +153,40 @@ If the `Version` variable is not given, then template processing will fail with Error executing template TEMPLATE_NAME, error: template: TEMPLATE_NAME:1:3: executing \"TEMPLATE_NAME\" at : error calling assert: Assertion failed ``` +### {{ image *docker_image_name_with_tag* }} or {{ image *docker_image_name* *tag* }} +Wrapper that is used to substitute images of particular versions derived by artifacts *(TODO: link to artifacts doc)*. + +Example: +```Dockerfile +FROM {{ image "ubuntu" }} +# OR +FROM {{ image "ubuntu:latest" }} +# OR +FROM {{ image "ubuntu" "latest" }} +``` + +Without any additional arguments it will resolve into this: +```Dockerfile +FROM ubuntu:latest +``` + +But if you have an artifact that is resulted by a previous rocker build, that can be fed back to rocker as variable, the artifact will be substituted: +```yaml +# shorten version of an artifact by rocker +RockerArtifacts: +- Name: ubuntu:latest + Digest: sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11 +``` + +```Dockerfile +# rocker build -vars artifacts/* +FROM ubuntu@sha256:ead434cd278824865d6e3b67e5d4579ded02eb2e8367fc165efa21138b225f11 +``` + +This feature is useful when you have a continuous integration pipeline and you want to build images on top of each other with guaranteed immutability. Also, this trick can be used with [rocker-compose](https://github.com/grammarly/rocker-compose) to run images of particular versions devired by the artifacts. + +*TODO: also describe semver matching behavior* + # Variables `rocker/template` automatically populates [os.Environ](https://golang.org/pkg/os/#Environ) to the template along with the variables that are passed from the outside. All environment variables are available under `.Env`. diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go b/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go index 1681b11..5e3e3a1 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go @@ -24,17 +24,25 @@ import ( "io/ioutil" "os" "reflect" + "sort" "strconv" "strings" "text/template" + "github.com/grammarly/rocker/src/rocker/imagename" + "github.com/go-yaml/yaml" "github.com/kr/pretty" + + log "github.com/Sirupsen/logrus" ) +// Funs is the list of additional helpers that may be given to the template +type Funs map[string]interface{} + // Process renders config through the template processor. // vars and additional functions are acceptable. -func Process(name string, reader io.Reader, vars Vars, funcs map[string]interface{}) (*bytes.Buffer, error) { +func Process(name string, reader io.Reader, vars Vars, funs Funs) (*bytes.Buffer, error) { var buf bytes.Buffer // read template @@ -58,6 +66,7 @@ func Process(name string, reader io.Reader, vars Vars, funcs map[string]interfac "json": jsonFn, "shell": EscapeShellarg, "yaml": yamlFn, + "image": makeImageHelper(vars), // `image` helper needs to make a closure on Vars // strings functions "compare": strings.Compare, @@ -89,7 +98,7 @@ func Process(name string, reader io.Reader, vars Vars, funcs map[string]interfac "trimSpace": strings.TrimSpace, "trimSuffix": strings.TrimSuffix, } - for k, f := range funcs { + for k, f := range funs { funcMap[k] = f } @@ -236,6 +245,70 @@ func indent(prefix, s string) string { return strings.Join(res, "\n") } +func makeImageHelper(vars Vars) func(string, ...string) (string, error) { + // Sort artifacts so we match semver on latest item + var ( + artifacts = &imagename.Artifacts{} + ok bool + ) + + if artifacts.RockerArtifacts, ok = vars["RockerArtifacts"].([]imagename.Artifact); !ok { + artifacts.RockerArtifacts = []imagename.Artifact{} + } + + sort.Sort(artifacts) + + log.Debugf("`image` helper got artifacts: %# v", pretty.Formatter(artifacts)) + + return func(img string, args ...string) (string, error) { + var ( + matched bool + ok bool + shouldMatch bool + image = imagename.NewFromString(img) + ) + + if len(args) > 0 { + image = imagename.New(img, args[0]) + } + + for _, a := range artifacts.RockerArtifacts { + if !image.IsSameKind(*a.Name) { + continue + } + + if image.HasVersionRange() { + if !image.Contains(a.Name) { + log.Debugf("Skipping artifact %s because it is not suitable for %s", a.Name, image) + continue + } + } else if image.GetTag() != a.Name.GetTag() { + log.Debugf("Skipping artifact %s because it is not suitable for %s", a.Name, image) + continue + } + + if a.Digest != "" { + log.Debugf("Apply digest %s for image %s", a.Digest, image) + image.SetTag(a.Digest) + matched = true + break + } + if a.Name.HasTag() { + log.Debugf("Apply tag %s for image %s", a.Name.GetTag(), image) + image.SetTag(a.Name.GetTag()) + matched = true + break + } + } + + if shouldMatch, ok = vars["DemandArtifacts"].(bool); ok && shouldMatch && !matched { + return "", fmt.Errorf("Cannot find suitable artifact for image %s", image) + } + + return image.String(), nil + } +} + func interfaceToInt(v interface{}) (int, error) { switch v.(type) { case int: diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/template_test.go b/vendor/src/github.com/grammarly/rocker/src/rocker/template/template_test.go index 837dfc0..654b4ff 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/template_test.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/template_test.go @@ -22,6 +22,8 @@ import ( "strings" "testing" + "github.com/grammarly/rocker/src/rocker/imagename" + "github.com/stretchr/testify/assert" ) @@ -32,6 +34,27 @@ var ( "data": map[string]string{ "foo": "bar", }, + "RockerArtifacts": []imagename.Artifact{ + imagename.Artifact{ + Name: imagename.NewFromString("alpine:3.2"), + Tag: "3.2", + }, + imagename.Artifact{ + Name: imagename.NewFromString("golang:1.5"), + Tag: "1.5", + Digest: "sha256:ead434", + }, + imagename.Artifact{ + Name: imagename.NewFromString("data:master"), + Tag: "master", + Digest: "sha256:fafe14", + }, + imagename.Artifact{ + Name: imagename.NewFromString("ssh:latest"), + Tag: "latest", + Digest: "sha256:ba41cd", + }, + }, } ) @@ -120,6 +143,77 @@ func TestProcess_YamlIndent(t *testing.T) { assert.Equal(t, "key:\n foo: bar\n", processTemplate(t, "key:\n{{ .data | yaml 1 }}")) } +func TestProcess_Image_Simple(t *testing.T) { + tests := []struct { + tpl string + result string + message string + }{ + {"{{ image `debian:7.7` }}", "debian:7.7", "should not alter the tag that is not in artifacts"}, + {"{{ image `debian` `7.7` }}", "debian:7.7", "should be possible to specify tag as a separate argument"}, + {"{{ image `debian` `sha256:afa` }}", "debian@sha256:afa", "should be possible to specify digest as a separate argument"}, + } + + for _, test := range tests { + assert.Equal(t, test.result, processTemplate(t, test.tpl), test.message) + } +} + +func TestProcess_Image_Advanced(t *testing.T) { + tests := []struct { + in string + result string + shouldMatch bool + message string + }{ + {"debian:7.7", "debian:7.7", false, "should not alter the tag that is not in artifacts"}, + {"debian:7.*", "debian:7.*", false, "should not alter the semver tag that is not in artifacts"}, + {"debian", "debian:latest", false, "should not match anything when no tag given (:latest) and no artifact"}, + {"alpine:3.1", "alpine:3.1", false, "should not match artifact with different version"}, + {"alpine:4.1", "alpine:4.1", false, "should not match artifact with different version"}, + {"alpine:3.*", "alpine:3.2", true, "should match artifact with version wildcard"}, + {"alpine", "alpine:latest", false, "should not match artifact when no tag given (:latest by default)"}, + {"alpine:latest", "alpine:latest", false, "should not match on a :latest tag"}, + {"alpine:snapshot", "alpine:snapshot", false, "should not match on a named tag"}, + {"golang:1.5", "golang@sha256:ead434", true, "should match semver tag and use digest"}, + {"golang:1.*", "golang@sha256:ead434", true, "should match on wildcard semver tag and use digest"}, + {"golang:1", "golang@sha256:ead434", true, "should match on prefix semver tag and use digest"}, + {"golang:1.4", "golang:1.4", false, "should not match on different semver tag"}, + {"golang:master", "golang:master", false, "should not match on a named tag"}, + {"data:1.2", "data:1.2", false, "should not match on a version tag against named artifact"}, + {"data:snapshot", "data:snapshot", false, "should not match on a different named tag against named artifact"}, + {"data:master", "data@sha256:fafe14", true, "should match on a same named tag against named artifact"}, + {"ssh:latest", "ssh@sha256:ba41cd", true, "should match on a :latest tag against :latest artifact"}, + {"ssh", "ssh@sha256:ba41cd", true, "should match on non-tagged tag against :latest artifact"}, + {"ssh:master", "ssh:master", false, "should match with other tag against :latest artifact"}, + {"ssh:1.2", "ssh:1.2", false, "should match with semver tag against :latest artifact"}, + } + + for _, test := range tests { + tpl := fmt.Sprintf("{{ image `%s` }}", test.in) + assert.Equal(t, test.result, processTemplate(t, tpl), test.message) + } + + // Now test the same but with DemandArtifact On + configTemplateVars["DemandArtifacts"] = true + defer func() { + configTemplateVars["DemandArtifacts"] = false + }() + + for _, test := range tests { + tpl := fmt.Sprintf("{{ image `%s` }}", test.in) + if test.shouldMatch { + assert.Equal(t, test.result, processTemplate(t, tpl), test.message) + } else { + err := processTemplateReturnError(t, tpl) + assert.Error(t, err, fmt.Sprintf("should give an error for test case: %s", test.message)) + if err != nil { + assert.Contains(t, err.Error(), fmt.Sprintf("Cannot find suitable artifact for image %s", test.in), test.message) + } + } + } +} + func processTemplate(t *testing.T, tpl string) string { result, err := Process("test", strings.NewReader(tpl), configTemplateVars, map[string]interface{}{}) if err != nil { @@ -127,3 +221,8 @@ func processTemplate(t *testing.T, tpl string) string { } return result.String() } + +func processTemplateReturnError(t *testing.T, tpl string) error { + _, err := Process("test", strings.NewReader(tpl), configTemplateVars, map[string]interface{}{}) + return err +} diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars.go b/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars.go index 9231c47..62a0e6d 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars.go @@ -23,11 +23,16 @@ import ( "os" "path" "path/filepath" + "reflect" "regexp" "sort" "strings" + "github.com/grammarly/rocker/src/rocker/imagename" + "github.com/go-yaml/yaml" + + log "github.com/Sirupsen/logrus" ) // Vars describes the data structure of the build variables @@ -37,7 +42,16 @@ type Vars map[string]interface{} func (vars Vars) Merge(varsList ...Vars) Vars { for _, mergeWith := range varsList { for k, v := range mergeWith { - vars[k] = v + // We want to merge slices of the same type by appending them to each other + // instead of overwriting + rv1 := reflect.ValueOf(vars[k]) + rv2 := reflect.ValueOf(v) + + if rv1.Kind() == reflect.Slice && rv2.Kind() == reflect.Slice && rv1.Type() == rv2.Type() { + vars[k] = reflect.AppendSlice(rv1, rv2).Interface() + } else { + vars[k] = v + } } } return vars @@ -91,6 +105,29 @@ func (vars *Vars) UnmarshalJSON(data []byte) (err error) { return nil } +// UnmarshalYAML parses YAML string and returns Vars +func (vars *Vars) UnmarshalYAML(unmarshal func(interface{}) error) (err error) { + // try unmarshal RockerArtifacts type + var artifacts imagename.Artifacts + if err = unmarshal(&artifacts); err != nil { + return err + } + + var value map[string]interface{} + if err = unmarshal(&value); err != nil { + return err + } + + // Fill artifacts if present + if len(artifacts.RockerArtifacts) > 0 { + value["RockerArtifacts"] = artifacts.RockerArtifacts + } + + *vars = value + + return nil +} + // VarsFromStrings parses Vars through ParseKvPairs and then loads content from files // for vars values with "@" prefix func VarsFromStrings(pairs []string) (vars Vars, err error) { @@ -117,6 +154,7 @@ func VarsFromStrings(pairs []string) (vars Vars, err error) { // VarsFromFile reads variables from either JSON or YAML file func VarsFromFile(filename string) (vars Vars, err error) { + log.Debugf("Load vars from file %s", filename) if filename, err = resolveFileName(filename); err != nil { return nil, err @@ -144,13 +182,32 @@ func VarsFromFile(filename string) (vars Vars, err error) { } // VarsFromFileMulti reads multiple files and merge vars -func VarsFromFileMulti(files []string) (vars Vars, err error) { - varsList := make([]Vars, len(files)) - for i, f := range files { - if varsList[i], err = VarsFromFile(f); err != nil { - return nil, err +func VarsFromFileMulti(files []string) (Vars, error) { + var ( + varsList = []Vars{} + matches []string + vars Vars + err error + ) + + for _, pat := range files { + // TODO: error if file not found (when not using wildcards) + matches = []string{pat} + + if containsWildcards(pat) { + if matches, err = filepath.Glob(pat); err != nil { + return nil, err + } + } + + for _, f := range matches { + if vars, err = VarsFromFile(f); err != nil { + return nil, err + } + varsList = append(varsList, vars) } } + return Vars{}.Merge(varsList...), nil } @@ -225,3 +282,15 @@ func (vars Vars) ReplaceString(str string) string { return str } + +func containsWildcards(name string) bool { + for i := 0; i < len(name); i++ { + ch := name[i] + if ch == '\\' { + i++ + } else if ch == '*' || ch == '?' || ch == '[' { + return true + } + } + return false +} diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars_test.go b/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars_test.go index 6408cf3..a942a99 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars_test.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars_test.go @@ -22,12 +22,26 @@ import ( "io/ioutil" "os" "path" - "rocker/test" "testing" + "github.com/grammarly/rocker/src/rocker/imagename" + "github.com/grammarly/rocker/src/rocker/test" + "github.com/stretchr/testify/assert" ) +func TestVars_MergeSlices(t *testing.T) { + v1 := Vars{ + "fruits": []string{"banana", "apple"}, + } + v2 := Vars{ + "fruits": []string{"pear", "orange"}, + } + v3 := v1.Merge(v2) + + assert.Equal(t, []string{"banana", "apple", "pear", "orange"}, v3["fruits"].([]string)) +} + func TestVarsToStrings(t *testing.T) { t.Parallel() @@ -100,6 +114,8 @@ func TestVarsFromStrings(t *testing.T) { } } +// TODO: test VarsFromFileMulti + func TestVarsFromFile_Yaml(t *testing.T) { tempDir, rm := tplMkFiles(t, map[string]string{ "vars.yml": ` @@ -118,6 +134,28 @@ Bar: yes assert.Equal(t, true, vars["Bar"]) } +func TestVarsFromFile_Yaml_Artifacts(t *testing.T) { + tempDir, rm := tplMkFiles(t, map[string]string{ + "vars.yml": ` +Foo: x +Bar: yes +RockerArtifacts: +- Name: golang:1.5 + Tag: "1.5" +`, + }) + defer rm() + + vars, err := VarsFromFile(tempDir + "/vars.yml") + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "x", vars["Foo"]) + assert.Equal(t, true, vars["Bar"]) + assert.IsType(t, []imagename.Artifact{}, vars["RockerArtifacts"]) +} + func TestVarsFromFile_Json(t *testing.T) { tempDir, rm := tplMkFiles(t, map[string]string{ "vars.json": ` From f1c1b7ec94b097d857c4fb2cf87e9df23ec97d7b Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 20 Oct 2015 18:06:45 +0300 Subject: [PATCH 23/34] #23 Use `image` helper to run with artifacts from rocker --- src/cmd/rocker-compose/main.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/cmd/rocker-compose/main.go b/src/cmd/rocker-compose/main.go index d791eb1..e59d34f 100644 --- a/src/cmd/rocker-compose/main.go +++ b/src/cmd/rocker-compose/main.go @@ -83,7 +83,7 @@ func main() { Usage: "Set variables to pass to build tasks, value is like \"key=value\"", }, cli.StringSliceFlag{ - Name: "vars-file", + Name: "vars", Value: &cli.StringSlice{}, Usage: "Load variables form a file, either JSON or YAML. Can pass multiple of this.", }, @@ -95,6 +95,10 @@ func main() { Name: "print", Usage: "just print the rendered compose config and exit", }, + cli.BoolFlag{ + Name: "demand-artifacts", + Usage: "fail if artifacts not found for {{ image }} helpers", + }, } app.Flags = append([]cli.Flag{ @@ -510,7 +514,7 @@ func initComposeConfig(ctx *cli.Context, dockerCli *docker.Client) *config.Confi print = ctx.Bool("print") ) - vars, err := template.VarsFromFileMulti(ctx.StringSlice("vars-file")) + vars, err := template.VarsFromFileMulti(ctx.StringSlice("vars")) if err != nil { log.Fatal(err) os.Exit(1) @@ -524,6 +528,10 @@ func initComposeConfig(ctx *cli.Context, dockerCli *docker.Client) *config.Confi vars = vars.Merge(cliVars) + if ctx.Bool("demand-artifacts") { + vars["DemandArtifacts"] = true + } + // TODO: find better place for providing this helper funcs := map[string]interface{}{ // lazy get bridge ip From 7d88590772ec69ca517df8929176eb4ab478bbc1 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Wed, 21 Oct 2015 17:49:52 +0300 Subject: [PATCH 24/34] update vendor rocker/imagename --- vendor/manifest | 10 +++++----- .../grammarly/rocker/src/rocker/imagename/imagename.go | 5 +++++ .../grammarly/rocker/src/rocker/imagename/registry.go | 8 +++++++- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/vendor/manifest b/vendor/manifest index 7d61a96..ea1bb3e 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -119,18 +119,18 @@ "path": "/src/rocker/dockerclient" }, { - "importpath": "github.com/grammarly/rocker/src/rocker/imagename", + "importpath": "github.com/grammarly/rocker/src/rocker/template", "repository": "https://github.com/grammarly/rocker", "revision": "51b6dce83b3f7b39f2a3843b00c57a5c614c0950", "branch": "v1", - "path": "/src/rocker/imagename" + "path": "/src/rocker/template" }, { - "importpath": "github.com/grammarly/rocker/src/rocker/template", + "importpath": "github.com/grammarly/rocker/src/rocker/imagename", "repository": "https://github.com/grammarly/rocker", - "revision": "51b6dce83b3f7b39f2a3843b00c57a5c614c0950", + "revision": "a86d7310cd246ddf2a4e3dc4d6cae71d2b2f2d02", "branch": "v1", - "path": "/src/rocker/template" + "path": "/src/rocker/imagename" } ] } \ No newline at end of file diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go index 0e9d9cd..e021d86 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/imagename.go @@ -225,6 +225,11 @@ func (img ImageName) Contains(b *ImageName) bool { // ResolveVersion finds an applicable tag for current image among the list of available tags func (img *ImageName) ResolveVersion(list []*ImageName) (result *ImageName) { for _, candidate := range list { + // If these are different images (different names/repos) + if !img.IsSameKind(*candidate) { + continue + } + // If we have a strict equality if img.HasTag() && candidate.HasTag() && img.Tag == candidate.Tag { return candidate diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/registry.go b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/registry.go index a7bb17a..f796742 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/registry.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/imagename/registry.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strings" "github.com/fsouza/go-dockerclient" ) @@ -95,8 +96,13 @@ func RegistryListTags(image *ImageName) (images []*ImageName, err error) { // registryListTagsDockerHub lists image tags from hub.docker.com func registryListTagsDockerHub(image *ImageName) (images []*ImageName, err error) { + name := image.Name + if !strings.Contains(name, "/") { + name = "library/" + name + } + tg := registryTags{} - if err = registryGet(fmt.Sprintf("https://hub.docker.com/v2/repositories/library/%s/tags/?page_size=9999&page=1", image.Name), &tg); err != nil { + if err = registryGet(fmt.Sprintf("https://hub.docker.com/v2/repositories/%s/tags/?page_size=9999&page=1", name), &tg); err != nil { return } From ffe95e36299074e2842b0001f914d0c9ac126da0 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Wed, 21 Oct 2015 18:15:17 +0300 Subject: [PATCH 25/34] vendor update rocker/template --- vendor/manifest | 12 ++++++------ .../grammarly/rocker/src/rocker/template/template.go | 4 ++-- .../grammarly/rocker/src/rocker/template/vars.go | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/vendor/manifest b/vendor/manifest index ea1bb3e..c466be3 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -119,18 +119,18 @@ "path": "/src/rocker/dockerclient" }, { - "importpath": "github.com/grammarly/rocker/src/rocker/template", + "importpath": "github.com/grammarly/rocker/src/rocker/imagename", "repository": "https://github.com/grammarly/rocker", - "revision": "51b6dce83b3f7b39f2a3843b00c57a5c614c0950", + "revision": "a86d7310cd246ddf2a4e3dc4d6cae71d2b2f2d02", "branch": "v1", - "path": "/src/rocker/template" + "path": "/src/rocker/imagename" }, { - "importpath": "github.com/grammarly/rocker/src/rocker/imagename", + "importpath": "github.com/grammarly/rocker/src/rocker/template", "repository": "https://github.com/grammarly/rocker", - "revision": "a86d7310cd246ddf2a4e3dc4d6cae71d2b2f2d02", + "revision": "3b5905678b83b1b5cc4f22557bed523ff4d2438e", "branch": "v1", - "path": "/src/rocker/imagename" + "path": "/src/rocker/template" } ] } \ No newline at end of file diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go b/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go index 5e3e3a1..6e51654 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/template.go @@ -288,13 +288,13 @@ func makeImageHelper(vars Vars) func(string, ...string) (string, error) { } if a.Digest != "" { - log.Debugf("Apply digest %s for image %s", a.Digest, image) + log.Infof("Apply artifact digest %s for image %s", a.Digest, image) image.SetTag(a.Digest) matched = true break } if a.Name.HasTag() { - log.Debugf("Apply tag %s for image %s", a.Name.GetTag(), image) + log.Infof("Apply artifact tag %s for image %s", a.Name.GetTag(), image) image.SetTag(a.Name.GetTag()) matched = true break diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars.go b/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars.go index 62a0e6d..52323ce 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/template/vars.go @@ -191,7 +191,6 @@ func VarsFromFileMulti(files []string) (Vars, error) { ) for _, pat := range files { - // TODO: error if file not found (when not using wildcards) matches = []string{pat} if containsWildcards(pat) { From 029b02b92da5c59ef390b1fcdd90b26b5acbb30c Mon Sep 17 00:00:00 2001 From: Seua Polyakov Date: Fri, 30 Oct 2015 09:56:09 +0200 Subject: [PATCH 26/34] Fix logging into stdin, when print option enable --- src/cmd/rocker-compose/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cmd/rocker-compose/main.go b/src/cmd/rocker-compose/main.go index e59d34f..cd5e1fe 100644 --- a/src/cmd/rocker-compose/main.go +++ b/src/cmd/rocker-compose/main.go @@ -465,6 +465,8 @@ func initLogs(ctx *cli.Context) { if ctx.GlobalBool("verbose") { logger.Level = log.DebugLevel + } else if ctx.Bool("print") && ctx.GlobalString("log") == "" { + logger.Level = log.ErrorLevel } var ( From 55f4339949879dff932e16e550822ae37704e5e0 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 3 Nov 2015 13:30:33 +0200 Subject: [PATCH 27/34] vendor update rocker/dockerclient and vendor do-homedir --- vendor/manifest | 20 ++- .../src/rocker/dockerclient/dockerclient.go | 6 + .../rocker/src/rocker/dockerclient/matrix.go | 3 +- .../github.com/mitchellh/go-homedir/LICENSE | 21 +++ .../github.com/mitchellh/go-homedir/README.md | 14 ++ .../mitchellh/go-homedir/homedir.go | 132 ++++++++++++++++++ .../mitchellh/go-homedir/homedir_test.go | 112 +++++++++++++++ 7 files changed, 300 insertions(+), 8 deletions(-) create mode 100644 vendor/src/github.com/mitchellh/go-homedir/LICENSE create mode 100644 vendor/src/github.com/mitchellh/go-homedir/README.md create mode 100644 vendor/src/github.com/mitchellh/go-homedir/homedir.go create mode 100644 vendor/src/github.com/mitchellh/go-homedir/homedir_test.go diff --git a/vendor/manifest b/vendor/manifest index c466be3..3bdd786 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -111,13 +111,6 @@ "branch": "v1", "path": "/src/rocker/textformatter" }, - { - "importpath": "github.com/grammarly/rocker/src/rocker/dockerclient", - "repository": "https://github.com/grammarly/rocker", - "revision": "80510375c825db6731268f9f4f247a404879c966", - "branch": "v1", - "path": "/src/rocker/dockerclient" - }, { "importpath": "github.com/grammarly/rocker/src/rocker/imagename", "repository": "https://github.com/grammarly/rocker", @@ -131,6 +124,19 @@ "revision": "3b5905678b83b1b5cc4f22557bed523ff4d2438e", "branch": "v1", "path": "/src/rocker/template" + }, + { + "importpath": "github.com/grammarly/rocker/src/rocker/dockerclient", + "repository": "https://github.com/grammarly/rocker", + "revision": "8632d6d097135c71dfff9b4f0b1bbe00eebf39b7", + "branch": "v1", + "path": "/src/rocker/dockerclient" + }, + { + "importpath": "github.com/mitchellh/go-homedir", + "repository": "https://github.com/mitchellh/go-homedir", + "revision": "d682a8f0cf139663a984ff12528da460ca963de9", + "branch": "master" } ] } \ No newline at end of file diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/dockerclient/dockerclient.go b/vendor/src/github.com/grammarly/rocker/src/rocker/dockerclient/dockerclient.go index 7077c89..d9994b0 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/dockerclient/dockerclient.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/dockerclient/dockerclient.go @@ -29,6 +29,7 @@ import ( "github.com/codegangsta/cli" "github.com/fsouza/go-dockerclient" + "github.com/mitchellh/go-homedir" ) var ( @@ -50,6 +51,11 @@ func NewConfig() *Config { certPath := os.Getenv("DOCKER_CERT_PATH") if certPath == "" { certPath = "~/.docker" + homePath, err := homedir.Dir() + if err != nil { + log.Fatal(err) + } + certPath = homePath + "/.docker" } host := os.Getenv("DOCKER_HOST") if host == "" { diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/dockerclient/matrix.go b/vendor/src/github.com/grammarly/rocker/src/rocker/dockerclient/matrix.go index 99a1d1e..2f49fb4 100644 --- a/vendor/src/github.com/grammarly/rocker/src/rocker/dockerclient/matrix.go +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/dockerclient/matrix.go @@ -23,8 +23,9 @@ import ( "path/filepath" "strings" - "github.com/fsouza/go-dockerclient" "github.com/grammarly/rocker/src/rocker/util" + + "github.com/fsouza/go-dockerclient" ) const ( diff --git a/vendor/src/github.com/mitchellh/go-homedir/LICENSE b/vendor/src/github.com/mitchellh/go-homedir/LICENSE new file mode 100644 index 0000000..f9c841a --- /dev/null +++ b/vendor/src/github.com/mitchellh/go-homedir/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/src/github.com/mitchellh/go-homedir/README.md b/vendor/src/github.com/mitchellh/go-homedir/README.md new file mode 100644 index 0000000..d70706d --- /dev/null +++ b/vendor/src/github.com/mitchellh/go-homedir/README.md @@ -0,0 +1,14 @@ +# go-homedir + +This is a Go library for detecting the user's home directory without +the use of cgo, so the library can be used in cross-compilation environments. + +Usage is incredibly simple, just call `homedir.Dir()` to get the home directory +for a user, and `homedir.Expand()` to expand the `~` in a path to the home +directory. + +**Why not just use `os/user`?** The built-in `os/user` package requires +cgo on Darwin systems. This means that any Go code that uses that package +cannot cross compile. But 99% of the time the use for `os/user` is just to +retrieve the home directory, which we can do for the current user without +cgo. This library does that, enabling cross-compilation. diff --git a/vendor/src/github.com/mitchellh/go-homedir/homedir.go b/vendor/src/github.com/mitchellh/go-homedir/homedir.go new file mode 100644 index 0000000..6944957 --- /dev/null +++ b/vendor/src/github.com/mitchellh/go-homedir/homedir.go @@ -0,0 +1,132 @@ +package homedir + +import ( + "bytes" + "errors" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" +) + +// DisableCache will disable caching of the home directory. Caching is enabled +// by default. +var DisableCache bool + +var homedirCache string +var cacheLock sync.RWMutex + +// Dir returns the home directory for the executing user. +// +// This uses an OS-specific method for discovering the home directory. +// An error is returned if a home directory cannot be detected. +func Dir() (string, error) { + if !DisableCache { + cacheLock.RLock() + cached := homedirCache + cacheLock.RUnlock() + if cached != "" { + return cached, nil + } + } + + cacheLock.Lock() + defer cacheLock.Unlock() + + var result string + var err error + if runtime.GOOS == "windows" { + result, err = dirWindows() + } else { + // Unix-like system, so just assume Unix + result, err = dirUnix() + } + + if err != nil { + return "", err + } + homedirCache = result + return result, nil +} + +// Expand expands the path to include the home directory if the path +// is prefixed with `~`. If it isn't prefixed with `~`, the path is +// returned as-is. +func Expand(path string) (string, error) { + if len(path) == 0 { + return path, nil + } + + if path[0] != '~' { + return path, nil + } + + if len(path) > 1 && path[1] != '/' && path[1] != '\\' { + return "", errors.New("cannot expand user-specific home dir") + } + + dir, err := Dir() + if err != nil { + return "", err + } + + return filepath.Join(dir, path[1:]), nil +} + +func dirUnix() (string, error) { + // First prefer the HOME environmental variable + if home := os.Getenv("HOME"); home != "" { + return home, nil + } + + // If that fails, try getent + var stdout bytes.Buffer + cmd := exec.Command("getent", "passwd", strconv.Itoa(os.Getuid())) + cmd.Stdout = &stdout + if err := cmd.Run(); err != nil { + // If "getent" is missing, ignore it + if err != exec.ErrNotFound { + return "", err + } + } else { + if passwd := strings.TrimSpace(stdout.String()); passwd != "" { + // username:password:uid:gid:gecos:home:shell + passwdParts := strings.SplitN(passwd, ":", 7) + if len(passwdParts) > 5 { + return passwdParts[5], nil + } + } + } + + // If all else fails, try the shell + stdout.Reset() + cmd = exec.Command("sh", "-c", "cd && pwd") + cmd.Stdout = &stdout + if err := cmd.Run(); err != nil { + return "", err + } + + result := strings.TrimSpace(stdout.String()) + if result == "" { + return "", errors.New("blank output when reading home directory") + } + + return result, nil +} + +func dirWindows() (string, error) { + drive := os.Getenv("HOMEDRIVE") + path := os.Getenv("HOMEPATH") + home := drive + path + if drive == "" || path == "" { + home = os.Getenv("USERPROFILE") + } + if home == "" { + return "", errors.New("HOMEDRIVE, HOMEPATH, and USERPROFILE are blank") + } + + return home, nil +} diff --git a/vendor/src/github.com/mitchellh/go-homedir/homedir_test.go b/vendor/src/github.com/mitchellh/go-homedir/homedir_test.go new file mode 100644 index 0000000..c34dbc7 --- /dev/null +++ b/vendor/src/github.com/mitchellh/go-homedir/homedir_test.go @@ -0,0 +1,112 @@ +package homedir + +import ( + "fmt" + "os" + "os/user" + "testing" +) + +func patchEnv(key, value string) func() { + bck := os.Getenv(key) + deferFunc := func() { + os.Setenv(key, bck) + } + + os.Setenv(key, value) + return deferFunc +} + +func BenchmarkDir(b *testing.B) { + // We do this for any "warmups" + for i := 0; i < 10; i++ { + Dir() + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + Dir() + } +} + +func TestDir(t *testing.T) { + u, err := user.Current() + if err != nil { + t.Fatalf("err: %s", err) + } + + dir, err := Dir() + if err != nil { + t.Fatalf("err: %s", err) + } + + if u.HomeDir != dir { + t.Fatalf("%#v != %#v", u.HomeDir, dir) + } +} + +func TestExpand(t *testing.T) { + u, err := user.Current() + if err != nil { + t.Fatalf("err: %s", err) + } + + cases := []struct { + Input string + Output string + Err bool + }{ + { + "/foo", + "/foo", + false, + }, + + { + "~/foo", + fmt.Sprintf("%s/foo", u.HomeDir), + false, + }, + + { + "", + "", + false, + }, + + { + "~", + u.HomeDir, + false, + }, + + { + "~foo/foo", + "", + true, + }, + } + + for _, tc := range cases { + actual, err := Expand(tc.Input) + if (err != nil) != tc.Err { + t.Fatalf("Input: %#v\n\nErr: %s", tc.Input, err) + } + + if actual != tc.Output { + t.Fatalf("Input: %#v\n\nOutput: %#v", tc.Input, actual) + } + } + + DisableCache = true + defer func() { DisableCache = false }() + defer patchEnv("HOME", "/custom/path/")() + expected := "/custom/path/foo/bar" + actual, err := Expand("~/foo/bar") + + if err != nil { + t.Errorf("No error is expected, got: %v", err) + } else if actual != "/custom/path/foo/bar" { + t.Errorf("Expected: %v; actual: %v", expected, actual) + } +} From f56ccf7085e5e3e298b0c5fb898e330f66a71a8b Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 3 Nov 2015 13:31:03 +0200 Subject: [PATCH 28/34] fixes #25 invalid HOME path resolving on Windows --- src/compose/config/config.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/compose/config/config.go b/src/compose/config/config.go index bb6b8c3..ff3be44 100644 --- a/src/compose/config/config.go +++ b/src/compose/config/config.go @@ -35,6 +35,7 @@ import ( "github.com/grammarly/rocker/src/rocker/imagename" "github.com/grammarly/rocker/src/rocker/template" + "github.com/mitchellh/go-homedir" "github.com/fsouza/go-dockerclient" "github.com/go-yaml/yaml" @@ -264,6 +265,17 @@ func ReadConfig(configName string, reader io.Reader, vars template.Vars, funcs m yamlFields[v] = true } + // Function that gets HOME (initialize only once) + homeMemo := "" + getHome := func() (h string, err error) { + if homeMemo == "" { + if homeMemo, err = homedir.Dir(); err != nil { + return "", err + } + } + return homeMemo, nil + } + // Process aliases on the first run, have to do it before extends // because Golang randomizes maps, sometimes inherited containers // process earlier then dependencies; also do initial validation @@ -386,7 +398,11 @@ func ReadConfig(configName string, reader io.Reader, vars template.Vars, funcs m continue } if strings.HasPrefix(split[0], "~") { - split[0] = strings.Replace(split[0], "~", os.Getenv("HOME"), 1) + home, err := getHome() + if err != nil { + return nil, fmt.Errorf("Failed to get HOME path, error: %s", err) + } + split[0] = strings.Replace(split[0], "~", home, 1) } if !path.IsAbs(split[0]) { split[0] = path.Join(basedir, split[0]) From f47284b7b6b0de7506e33700d067a83600d0490d Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 3 Nov 2015 13:35:50 +0200 Subject: [PATCH 29/34] fix cross-platform make --- Makefile | 2 +- Rockerfile.build-cross | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 8439cc7..50dbb8d 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ $(ALL_BINARIES): build_image docker run --rm -ti -v $(shell pwd)/dist:/src/dist \ -e GOOS=$(call os,$@) -e GOARCH=$(call arch,$@) -e GOPATH=/src:/src/vendor \ rocker-compose-build:latest go build \ - -ldflags "-X main.Version '$(VERSION)' -X main.GitCommit '$(GITCOMMIT)' -X main.GitBranch '$(GITBRANCH)' -X main.BuildTime '$(BUILDTIME)'" \ + -ldflags "-X main.Version=$(VERSION) -X main.GitCommit=$(GITCOMMIT) -X main.GitBranch=$(GITBRANCH) -X main.BuildTime=$(BUILDTIME)" \ -v -o $@ src/cmd/$(call bin,$@)/main.go build_image: diff --git a/Rockerfile.build-cross b/Rockerfile.build-cross index 5b3b4c4..4bd0415 100644 --- a/Rockerfile.build-cross +++ b/Rockerfile.build-cross @@ -1,10 +1,6 @@ -# Yeah, we have a tool called 'rocker' that is not open sourced yet. -# rocker is an extensible Dockerfile builder tool -FROM dockerhub.grammarly.io/golang-1.4.2-cross:v2 +FROM dockerhub.grammarly.io/golang-1.5.1-cross:v1 ADD . /src WORKDIR /src -TAG --no-alias rocker-compose-build:latest - -# ATTACH ["bash"] +TAG rocker-compose-build:latest From 78499db5cf80049b1ca045f70e3a269bc508a661 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Tue, 3 Nov 2015 19:33:36 +0200 Subject: [PATCH 30/34] fix semver resolution for non-pulled images --- src/compose/client.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/compose/client.go b/src/compose/client.go index 96e6e28..15270b4 100644 --- a/src/compose/client.go +++ b/src/compose/client.go @@ -730,25 +730,24 @@ func (client *DockerClient) resolveVersions(local, hub bool, vars template.Vars, return err } + // looking locally first + candidate := container.Image.ResolveVersion(images) + // in case we want to include external images as well, pulling list of available // images from repository or central docker hub - if hub { + if hub || candidate == nil { log.Debugf("Getting list of tags for %s from the registry", container.Image) var remote []*imagename.ImageName if remote, err = imagename.RegistryListTags(container.Image); err != nil { - err = fmt.Errorf("Failed to list tags of image %s for container %s from the remote registry, error: %s", + return fmt.Errorf("Failed to list tags of image %s for container %s from the remote registry, error: %s", container.Image, container.Name, err) - } else { - for _, img := range remote { - log.Debugf("remote tag for %s: %s", container.Image, img) - } - images = append(images, remote...) } + + // Re-Resolve having hub tags + candidate = container.Image.ResolveVersion(append(images, remote...)) } - // looking locally first - candidate := container.Image.ResolveVersion(images) if candidate == nil { err = fmt.Errorf("Image not found: %s", container.Image) return From 55c52d9bcdc4545259101cc6565eff312277b801 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Mon, 16 Nov 2015 17:46:26 +0200 Subject: [PATCH 31/34] vendor github.com/docker/docker/pkg/signal --- vendor/manifest | 7 ++ .../docker/docker/pkg/signal/README.md | 1 + .../docker/docker/pkg/signal/signal.go | 54 +++++++++++++ .../docker/docker/pkg/signal/signal_darwin.go | 41 ++++++++++ .../docker/pkg/signal/signal_freebsd.go | 43 ++++++++++ .../docker/docker/pkg/signal/signal_linux.go | 80 +++++++++++++++++++ .../docker/docker/pkg/signal/signal_unix.go | 19 +++++ .../docker/pkg/signal/signal_unsupported.go | 10 +++ .../docker/pkg/signal/signal_windows.go | 27 +++++++ .../docker/docker/pkg/signal/trap.go | 74 +++++++++++++++++ 10 files changed, 356 insertions(+) create mode 100644 vendor/src/github.com/docker/docker/pkg/signal/README.md create mode 100644 vendor/src/github.com/docker/docker/pkg/signal/signal.go create mode 100644 vendor/src/github.com/docker/docker/pkg/signal/signal_darwin.go create mode 100644 vendor/src/github.com/docker/docker/pkg/signal/signal_freebsd.go create mode 100644 vendor/src/github.com/docker/docker/pkg/signal/signal_linux.go create mode 100644 vendor/src/github.com/docker/docker/pkg/signal/signal_unix.go create mode 100644 vendor/src/github.com/docker/docker/pkg/signal/signal_unsupported.go create mode 100644 vendor/src/github.com/docker/docker/pkg/signal/signal_windows.go create mode 100644 vendor/src/github.com/docker/docker/pkg/signal/trap.go diff --git a/vendor/manifest b/vendor/manifest index 3bdd786..e234b0f 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -137,6 +137,13 @@ "repository": "https://github.com/mitchellh/go-homedir", "revision": "d682a8f0cf139663a984ff12528da460ca963de9", "branch": "master" + }, + { + "importpath": "github.com/docker/docker/pkg/signal", + "repository": "https://github.com/docker/docker", + "revision": "92487d7fb4963c0333c409d053ff694e619c538d", + "branch": "master", + "path": "/pkg/signal" } ] } \ No newline at end of file diff --git a/vendor/src/github.com/docker/docker/pkg/signal/README.md b/vendor/src/github.com/docker/docker/pkg/signal/README.md new file mode 100644 index 0000000..2b237a5 --- /dev/null +++ b/vendor/src/github.com/docker/docker/pkg/signal/README.md @@ -0,0 +1 @@ +This package provides helper functions for dealing with signals across various operating systems \ No newline at end of file diff --git a/vendor/src/github.com/docker/docker/pkg/signal/signal.go b/vendor/src/github.com/docker/docker/pkg/signal/signal.go new file mode 100644 index 0000000..68bb77c --- /dev/null +++ b/vendor/src/github.com/docker/docker/pkg/signal/signal.go @@ -0,0 +1,54 @@ +// Package signal provides helper functions for dealing with signals across +// various operating systems. +package signal + +import ( + "fmt" + "os" + "os/signal" + "strconv" + "strings" + "syscall" +) + +// CatchAll catches all signals and relays them to the specified channel. +func CatchAll(sigc chan os.Signal) { + handledSigs := []os.Signal{} + for _, s := range SignalMap { + handledSigs = append(handledSigs, s) + } + signal.Notify(sigc, handledSigs...) +} + +// StopCatch stops catching the signals and closes the specified channel. +func StopCatch(sigc chan os.Signal) { + signal.Stop(sigc) + close(sigc) +} + +// ParseSignal translates a string to a valid syscall signal. +// It returns an error if the signal map doesn't include the given signal. +func ParseSignal(rawSignal string) (syscall.Signal, error) { + s, err := strconv.Atoi(rawSignal) + if err == nil { + if s == 0 { + return -1, fmt.Errorf("Invalid signal: %s", rawSignal) + } + return syscall.Signal(s), nil + } + signal, ok := SignalMap[strings.TrimPrefix(strings.ToUpper(rawSignal), "SIG")] + if !ok { + return -1, fmt.Errorf("Invalid signal: %s", rawSignal) + } + return signal, nil +} + +// ValidSignalForPlatform returns true if a signal is valid on the platform +func ValidSignalForPlatform(sig syscall.Signal) bool { + for _, v := range SignalMap { + if v == sig { + return true + } + } + return false +} diff --git a/vendor/src/github.com/docker/docker/pkg/signal/signal_darwin.go b/vendor/src/github.com/docker/docker/pkg/signal/signal_darwin.go new file mode 100644 index 0000000..946de87 --- /dev/null +++ b/vendor/src/github.com/docker/docker/pkg/signal/signal_darwin.go @@ -0,0 +1,41 @@ +package signal + +import ( + "syscall" +) + +// SignalMap is a map of Darwin signals. +var SignalMap = map[string]syscall.Signal{ + "ABRT": syscall.SIGABRT, + "ALRM": syscall.SIGALRM, + "BUG": syscall.SIGBUS, + "CHLD": syscall.SIGCHLD, + "CONT": syscall.SIGCONT, + "EMT": syscall.SIGEMT, + "FPE": syscall.SIGFPE, + "HUP": syscall.SIGHUP, + "ILL": syscall.SIGILL, + "INFO": syscall.SIGINFO, + "INT": syscall.SIGINT, + "IO": syscall.SIGIO, + "IOT": syscall.SIGIOT, + "KILL": syscall.SIGKILL, + "PIPE": syscall.SIGPIPE, + "PROF": syscall.SIGPROF, + "QUIT": syscall.SIGQUIT, + "SEGV": syscall.SIGSEGV, + "STOP": syscall.SIGSTOP, + "SYS": syscall.SIGSYS, + "TERM": syscall.SIGTERM, + "TRAP": syscall.SIGTRAP, + "TSTP": syscall.SIGTSTP, + "TTIN": syscall.SIGTTIN, + "TTOU": syscall.SIGTTOU, + "URG": syscall.SIGURG, + "USR1": syscall.SIGUSR1, + "USR2": syscall.SIGUSR2, + "VTALRM": syscall.SIGVTALRM, + "WINCH": syscall.SIGWINCH, + "XCPU": syscall.SIGXCPU, + "XFSZ": syscall.SIGXFSZ, +} diff --git a/vendor/src/github.com/docker/docker/pkg/signal/signal_freebsd.go b/vendor/src/github.com/docker/docker/pkg/signal/signal_freebsd.go new file mode 100644 index 0000000..6b9569b --- /dev/null +++ b/vendor/src/github.com/docker/docker/pkg/signal/signal_freebsd.go @@ -0,0 +1,43 @@ +package signal + +import ( + "syscall" +) + +// SignalMap is a map of FreeBSD signals. +var SignalMap = map[string]syscall.Signal{ + "ABRT": syscall.SIGABRT, + "ALRM": syscall.SIGALRM, + "BUF": syscall.SIGBUS, + "CHLD": syscall.SIGCHLD, + "CONT": syscall.SIGCONT, + "EMT": syscall.SIGEMT, + "FPE": syscall.SIGFPE, + "HUP": syscall.SIGHUP, + "ILL": syscall.SIGILL, + "INFO": syscall.SIGINFO, + "INT": syscall.SIGINT, + "IO": syscall.SIGIO, + "IOT": syscall.SIGIOT, + "KILL": syscall.SIGKILL, + "LWP": syscall.SIGLWP, + "PIPE": syscall.SIGPIPE, + "PROF": syscall.SIGPROF, + "QUIT": syscall.SIGQUIT, + "SEGV": syscall.SIGSEGV, + "STOP": syscall.SIGSTOP, + "SYS": syscall.SIGSYS, + "TERM": syscall.SIGTERM, + "THR": syscall.SIGTHR, + "TRAP": syscall.SIGTRAP, + "TSTP": syscall.SIGTSTP, + "TTIN": syscall.SIGTTIN, + "TTOU": syscall.SIGTTOU, + "URG": syscall.SIGURG, + "USR1": syscall.SIGUSR1, + "USR2": syscall.SIGUSR2, + "VTALRM": syscall.SIGVTALRM, + "WINCH": syscall.SIGWINCH, + "XCPU": syscall.SIGXCPU, + "XFSZ": syscall.SIGXFSZ, +} diff --git a/vendor/src/github.com/docker/docker/pkg/signal/signal_linux.go b/vendor/src/github.com/docker/docker/pkg/signal/signal_linux.go new file mode 100644 index 0000000..d418cbe --- /dev/null +++ b/vendor/src/github.com/docker/docker/pkg/signal/signal_linux.go @@ -0,0 +1,80 @@ +package signal + +import ( + "syscall" +) + +const ( + sigrtmin = 34 + sigrtmax = 64 +) + +// SignalMap is a map of Linux signals. +var SignalMap = map[string]syscall.Signal{ + "ABRT": syscall.SIGABRT, + "ALRM": syscall.SIGALRM, + "BUS": syscall.SIGBUS, + "CHLD": syscall.SIGCHLD, + "CLD": syscall.SIGCLD, + "CONT": syscall.SIGCONT, + "FPE": syscall.SIGFPE, + "HUP": syscall.SIGHUP, + "ILL": syscall.SIGILL, + "INT": syscall.SIGINT, + "IO": syscall.SIGIO, + "IOT": syscall.SIGIOT, + "KILL": syscall.SIGKILL, + "PIPE": syscall.SIGPIPE, + "POLL": syscall.SIGPOLL, + "PROF": syscall.SIGPROF, + "PWR": syscall.SIGPWR, + "QUIT": syscall.SIGQUIT, + "SEGV": syscall.SIGSEGV, + "STKFLT": syscall.SIGSTKFLT, + "STOP": syscall.SIGSTOP, + "SYS": syscall.SIGSYS, + "TERM": syscall.SIGTERM, + "TRAP": syscall.SIGTRAP, + "TSTP": syscall.SIGTSTP, + "TTIN": syscall.SIGTTIN, + "TTOU": syscall.SIGTTOU, + "UNUSED": syscall.SIGUNUSED, + "URG": syscall.SIGURG, + "USR1": syscall.SIGUSR1, + "USR2": syscall.SIGUSR2, + "VTALRM": syscall.SIGVTALRM, + "WINCH": syscall.SIGWINCH, + "XCPU": syscall.SIGXCPU, + "XFSZ": syscall.SIGXFSZ, + "RTMIN": sigrtmin, + "RTMIN+1": sigrtmin + 1, + "RTMIN+2": sigrtmin + 2, + "RTMIN+3": sigrtmin + 3, + "RTMIN+4": sigrtmin + 4, + "RTMIN+5": sigrtmin + 5, + "RTMIN+6": sigrtmin + 6, + "RTMIN+7": sigrtmin + 7, + "RTMIN+8": sigrtmin + 8, + "RTMIN+9": sigrtmin + 9, + "RTMIN+10": sigrtmin + 10, + "RTMIN+11": sigrtmin + 11, + "RTMIN+12": sigrtmin + 12, + "RTMIN+13": sigrtmin + 13, + "RTMIN+14": sigrtmin + 14, + "RTMIN+15": sigrtmin + 15, + "RTMAX-14": sigrtmax - 14, + "RTMAX-13": sigrtmax - 13, + "RTMAX-12": sigrtmax - 12, + "RTMAX-11": sigrtmax - 11, + "RTMAX-10": sigrtmax - 10, + "RTMAX-9": sigrtmax - 9, + "RTMAX-8": sigrtmax - 8, + "RTMAX-7": sigrtmax - 7, + "RTMAX-6": sigrtmax - 6, + "RTMAX-5": sigrtmax - 5, + "RTMAX-4": sigrtmax - 4, + "RTMAX-3": sigrtmax - 3, + "RTMAX-2": sigrtmax - 2, + "RTMAX-1": sigrtmax - 1, + "RTMAX": sigrtmax, +} diff --git a/vendor/src/github.com/docker/docker/pkg/signal/signal_unix.go b/vendor/src/github.com/docker/docker/pkg/signal/signal_unix.go new file mode 100644 index 0000000..d4fea93 --- /dev/null +++ b/vendor/src/github.com/docker/docker/pkg/signal/signal_unix.go @@ -0,0 +1,19 @@ +// +build !windows + +package signal + +import ( + "syscall" +) + +// Signals used in api/client (no windows equivalent, use +// invalid signals so they don't get handled) + +const ( + // SIGCHLD is a signal sent to a process when a child process terminates, is interrupted, or resumes after being interrupted. + SIGCHLD = syscall.SIGCHLD + // SIGWINCH is a signal sent to a process when its controlling terminal changes its size + SIGWINCH = syscall.SIGWINCH + // DefaultStopSignal is the syscall signal used to stop a container in unix systems. + DefaultStopSignal = "SIGTERM" +) diff --git a/vendor/src/github.com/docker/docker/pkg/signal/signal_unsupported.go b/vendor/src/github.com/docker/docker/pkg/signal/signal_unsupported.go new file mode 100644 index 0000000..161ba27 --- /dev/null +++ b/vendor/src/github.com/docker/docker/pkg/signal/signal_unsupported.go @@ -0,0 +1,10 @@ +// +build !linux,!darwin,!freebsd,!windows + +package signal + +import ( + "syscall" +) + +// SignalMap is an empty map of signals for unsupported platform. +var SignalMap = map[string]syscall.Signal{} diff --git a/vendor/src/github.com/docker/docker/pkg/signal/signal_windows.go b/vendor/src/github.com/docker/docker/pkg/signal/signal_windows.go new file mode 100644 index 0000000..c80a951 --- /dev/null +++ b/vendor/src/github.com/docker/docker/pkg/signal/signal_windows.go @@ -0,0 +1,27 @@ +// +build windows + +package signal + +import ( + "syscall" +) + +// Signals used in api/client (no windows equivalent, use +// invalid signals so they don't get handled) +const ( + SIGCHLD = syscall.Signal(0xff) + SIGWINCH = syscall.Signal(0xff) + // DefaultStopSignal is the syscall signal used to stop a container in windows systems. + DefaultStopSignal = "15" +) + +// SignalMap is a map of "supported" signals. As per the comment in GOLang's +// ztypes_windows.go: "More invented values for signals". Windows doesn't +// really support signals in any way, shape or form that Unix does. +// +// We have these so that docker kill can be used to gracefully (TERM) and +// forcibly (KILL) terminate a container on Windows. +var SignalMap = map[string]syscall.Signal{ + "KILL": syscall.SIGKILL, + "TERM": syscall.SIGTERM, +} diff --git a/vendor/src/github.com/docker/docker/pkg/signal/trap.go b/vendor/src/github.com/docker/docker/pkg/signal/trap.go new file mode 100644 index 0000000..2cf5ccf --- /dev/null +++ b/vendor/src/github.com/docker/docker/pkg/signal/trap.go @@ -0,0 +1,74 @@ +package signal + +import ( + "os" + gosignal "os/signal" + "runtime" + "sync/atomic" + "syscall" + + "github.com/Sirupsen/logrus" +) + +// Trap sets up a simplified signal "trap", appropriate for common +// behavior expected from a vanilla unix command-line tool in general +// (and the Docker engine in particular). +// +// * If SIGINT or SIGTERM are received, `cleanup` is called, then the process is terminated. +// * If SIGINT or SIGTERM are received 3 times before cleanup is complete, then cleanup is +// skipped and the process is terminated immediately (allows force quit of stuck daemon) +// * A SIGQUIT always causes an exit without cleanup, with a goroutine dump preceding exit. +// +func Trap(cleanup func()) { + c := make(chan os.Signal, 1) + // we will handle INT, TERM, QUIT here + signals := []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT} + gosignal.Notify(c, signals...) + go func() { + interruptCount := uint32(0) + for sig := range c { + go func(sig os.Signal) { + logrus.Infof("Processing signal '%v'", sig) + switch sig { + case os.Interrupt, syscall.SIGTERM: + if atomic.LoadUint32(&interruptCount) < 3 { + // Initiate the cleanup only once + if atomic.AddUint32(&interruptCount, 1) == 1 { + // Call the provided cleanup handler + cleanup() + os.Exit(0) + } else { + return + } + } else { + // 3 SIGTERM/INT signals received; force exit without cleanup + logrus.Infof("Forcing docker daemon shutdown without cleanup; 3 interrupts received") + } + case syscall.SIGQUIT: + DumpStacks() + logrus.Infof("Forcing docker daemon shutdown without cleanup on SIGQUIT") + } + //for the SIGINT/TERM, and SIGQUIT non-clean shutdown case, exit with 128 + signal # + os.Exit(128 + int(sig.(syscall.Signal))) + }(sig) + } + }() +} + +// DumpStacks dumps the runtime stack. +func DumpStacks() { + var ( + buf []byte + stackSize int + ) + bufferLen := 16384 + for stackSize == len(buf) { + buf = make([]byte, bufferLen) + stackSize = runtime.Stack(buf, true) + bufferLen *= 2 + } + buf = buf[:stackSize] + // Note that if the daemon is started with a less-verbose log-level than "info" (the default), the goroutine + // traces won't show up in the log. + logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf) +} From dd6ce98b90000d26651c138a285ba52695361d20 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Mon, 16 Nov 2015 17:46:42 +0200 Subject: [PATCH 32/34] dump stack when receive SIGUSR1 --- src/cmd/rocker-compose/debugtrap_unix.go | 21 +++++++++++++++++++ .../rocker-compose/debugtrap_unsupported.go | 7 +++++++ src/cmd/rocker-compose/main.go | 1 + 3 files changed, 29 insertions(+) create mode 100644 src/cmd/rocker-compose/debugtrap_unix.go create mode 100644 src/cmd/rocker-compose/debugtrap_unsupported.go diff --git a/src/cmd/rocker-compose/debugtrap_unix.go b/src/cmd/rocker-compose/debugtrap_unix.go new file mode 100644 index 0000000..3747d88 --- /dev/null +++ b/src/cmd/rocker-compose/debugtrap_unix.go @@ -0,0 +1,21 @@ +// +build !windows + +package main + +import ( + "os" + "os/signal" + "syscall" + + psignal "github.com/docker/docker/pkg/signal" +) + +func setupDumpStackTrap() { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGUSR1) + go func() { + for range c { + psignal.DumpStacks() + } + }() +} diff --git a/src/cmd/rocker-compose/debugtrap_unsupported.go b/src/cmd/rocker-compose/debugtrap_unsupported.go new file mode 100644 index 0000000..659897f --- /dev/null +++ b/src/cmd/rocker-compose/debugtrap_unsupported.go @@ -0,0 +1,7 @@ +// +build !linux,!darwin,!freebsd + +package main + +func setupDumpStackTrap() { + return +} diff --git a/src/cmd/rocker-compose/main.go b/src/cmd/rocker-compose/main.go index cd5e1fe..8835742 100644 --- a/src/cmd/rocker-compose/main.go +++ b/src/cmd/rocker-compose/main.go @@ -58,6 +58,7 @@ var ( func init() { log.SetOutput(os.Stdout) log.SetLevel(log.InfoLevel) + setupDumpStackTrap() } func main() { From 473a03cc4ef580008ab9be1149e45b2288272bd5 Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Mon, 16 Nov 2015 19:40:40 +0200 Subject: [PATCH 33/34] use rocker/debugtrap to dump stacktrace on SIGUSR1 --- src/cmd/rocker-compose/main.go | 3 ++- vendor/manifest | 7 ++++++ .../src/rocker/debugtrap/debugtrap_unix.go | 23 +++++++++++++++++++ .../rocker/debugtrap/debugtrap_unsupported.go | 9 ++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 vendor/src/github.com/grammarly/rocker/src/rocker/debugtrap/debugtrap_unix.go create mode 100644 vendor/src/github.com/grammarly/rocker/src/rocker/debugtrap/debugtrap_unsupported.go diff --git a/src/cmd/rocker-compose/main.go b/src/cmd/rocker-compose/main.go index 8835742..29e68e2 100644 --- a/src/cmd/rocker-compose/main.go +++ b/src/cmd/rocker-compose/main.go @@ -36,6 +36,7 @@ import ( "github.com/codegangsta/cli" "github.com/fsouza/go-dockerclient" "github.com/go-yaml/yaml" + "github.com/grammarly/rocker/src/rocker/debugtrap" "github.com/grammarly/rocker/src/rocker/dockerclient" "github.com/grammarly/rocker/src/rocker/template" "github.com/grammarly/rocker/src/rocker/textformatter" @@ -58,7 +59,7 @@ var ( func init() { log.SetOutput(os.Stdout) log.SetLevel(log.InfoLevel) - setupDumpStackTrap() + debugtrap.SetupDumpStackTrap() } func main() { diff --git a/vendor/manifest b/vendor/manifest index e234b0f..8f1a693 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -144,6 +144,13 @@ "revision": "92487d7fb4963c0333c409d053ff694e619c538d", "branch": "master", "path": "/pkg/signal" + }, + { + "importpath": "github.com/grammarly/rocker/src/rocker/debugtrap", + "repository": "https://github.com/grammarly/rocker", + "revision": "d6a4bbfc503169324de3462a2ae2bb5a5dc2f041", + "branch": "v1", + "path": "/src/rocker/debugtrap" } ] } \ No newline at end of file diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/debugtrap/debugtrap_unix.go b/vendor/src/github.com/grammarly/rocker/src/rocker/debugtrap/debugtrap_unix.go new file mode 100644 index 0000000..097ae85 --- /dev/null +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/debugtrap/debugtrap_unix.go @@ -0,0 +1,23 @@ +// +build !windows + +package debugtrap + +import ( + "os" + "os/signal" + "syscall" + + psignal "github.com/docker/docker/pkg/signal" +) + +// SetupDumpStackTrap set up a handler for SIGUSR1 and dumps +// the goroutine stack trace to INFO log +func SetupDumpStackTrap() { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGUSR1) + go func() { + for range c { + psignal.DumpStacks() + } + }() +} diff --git a/vendor/src/github.com/grammarly/rocker/src/rocker/debugtrap/debugtrap_unsupported.go b/vendor/src/github.com/grammarly/rocker/src/rocker/debugtrap/debugtrap_unsupported.go new file mode 100644 index 0000000..c640640 --- /dev/null +++ b/vendor/src/github.com/grammarly/rocker/src/rocker/debugtrap/debugtrap_unsupported.go @@ -0,0 +1,9 @@ +// +build !linux,!darwin,!freebsd + +package debugtrap + +// SetupDumpStackTrap set up a handler for SIGUSR1 and dumps +// the goroutine stack trace to INFO log +func SetupDumpStackTrap() { + return +} From a791cfbef8f75383958543e161c35a0956e8818f Mon Sep 17 00:00:00 2001 From: Yuriy Bogdanov Date: Mon, 23 Nov 2015 19:47:36 +0200 Subject: [PATCH 34/34] fix dev version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b1e80bb..d917d3e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.3 +0.1.2