diff --git a/loader/include.go b/loader/include.go index ea784144..b81c4073 100644 --- a/loader/include.go +++ b/loader/include.go @@ -24,7 +24,6 @@ import ( "github.com/compose-spec/compose-go/dotenv" interp "github.com/compose-spec/compose-go/interpolation" "github.com/compose-spec/compose-go/types" - "github.com/pkg/errors" ) // LoadIncludeConfig parse the require config from raw yaml @@ -34,17 +33,6 @@ func LoadIncludeConfig(source []interface{}) ([]types.IncludeConfig, error) { return requires, err } -var transformIncludeConfig TransformerFunc = func(data interface{}) (interface{}, error) { - switch value := data.(type) { - case string: - return map[string]interface{}{"path": value}, nil - case map[string]interface{}: - return value, nil - default: - return data, errors.Errorf("invalid type %T for `include` configuration", value) - } -} - func loadInclude(ctx context.Context, filename string, configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, map[string][]types.IncludeConfig, error) { included := make(map[string][]types.IncludeConfig) for _, r := range model.Include { diff --git a/loader/interpolate.go b/loader/interpolate.go index aae6dc3a..8aa66416 100644 --- a/loader/interpolate.go +++ b/loader/interpolate.go @@ -22,6 +22,7 @@ import ( interp "github.com/compose-spec/compose-go/interpolation" "github.com/compose-spec/compose-go/tree" + "github.com/docker/go-units" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -60,7 +61,6 @@ var interpolateTypeCastMapping = map[tree.Path]interp.Cast{ servicePath("secrets", tree.PathMatchList, "mode"): toInt, servicePath("shm_size"): toUnitBytes, servicePath("stdin_open"): toBoolean, - servicePath("stop_grace_period"): toDuration, servicePath("tty"): toBoolean, servicePath("ulimits", tree.PathMatchAll): toInt, servicePath("ulimits", tree.PathMatchAll, "hard"): toInt, @@ -94,11 +94,7 @@ func toInt64(value string) (interface{}, error) { } func toUnitBytes(value string) (interface{}, error) { - return transformSize(value) -} - -func toDuration(value string) (interface{}, error) { - return transformStringToDuration(value) + return units.RAMInBytes(value) } func toFloat(value string) (interface{}, error) { diff --git a/loader/loader.go b/loader/loader.go index 19ca5346..d10bfd9e 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -26,7 +26,6 @@ import ( "path/filepath" "reflect" "regexp" - "strconv" "strings" "time" @@ -34,9 +33,8 @@ import ( interp "github.com/compose-spec/compose-go/interpolation" "github.com/compose-spec/compose-go/schema" "github.com/compose-spec/compose-go/template" + "github.com/compose-spec/compose-go/transform" "github.com/compose-spec/compose-go/types" - "github.com/docker/go-units" - "github.com/mattn/go-shellwords" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -268,6 +266,11 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options, } } + configDict, err := transform.ExpandShortSyntax(configDict) + if err != nil { + return errors.Wrapf(err, "failed to expand short syntax") + } + configDict = groupXFieldsIntoExtensions(configDict) cfg, err := loadSections(ctx, file.Filename, configDict, configDetails, opts) @@ -502,6 +505,14 @@ func groupXFieldsIntoExtensions(dict map[string]interface{}) map[string]interfac if d, ok := value.(map[string]interface{}); ok { dict[key] = groupXFieldsIntoExtensions(d) } + if s, ok := value.([]interface{}); ok { + for i, item := range s { + if d, ok := item.(map[string]interface{}); ok { + s[i] = groupXFieldsIntoExtensions(d) + } + } + } + } if len(extras) > 0 { dict[extensions] = extras @@ -521,6 +532,7 @@ func loadSections(ctx context.Context, filename string, config map[string]interf return nil, errors.New("project name must be a string") } } + cfg.Name = name cfg.Services, err = LoadServices(ctx, filename, getSection(config, "services"), configDetails.WorkingDir, configDetails.LookupEnv, opts) if err != nil { @@ -581,15 +593,15 @@ func (e *ForbiddenPropertiesError) Error() string { // Transform converts the source into the target struct with compose types transformer // and the specified transformers if any. -func Transform(source interface{}, target interface{}, additionalTransformers ...Transformer) error { +func Transform(source interface{}, target interface{}) error { data := mapstructure.Metadata{} config := &mapstructure.DecoderConfig{ - DecodeHook: mapstructure.ComposeDecodeHookFunc( - createTransformHook(additionalTransformers...), - mapstructure.StringToTimeDurationHookFunc()), Result: target, TagName: "yaml", Metadata: &data, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + decoderHook, + stringToDuration), } decoder, err := mapstructure.NewDecoder(config) if err != nil { @@ -598,55 +610,39 @@ func Transform(source interface{}, target interface{}, additionalTransformers .. return decoder.Decode(source) } -// TransformerFunc defines a function to perform the actual transformation -type TransformerFunc func(interface{}) (interface{}, error) +func stringToDuration(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(types.Duration(5)) { + return data, nil + } + // Convert it by parsing + return time.ParseDuration(data.(string)) +} -// Transformer defines a map to type transformer -type Transformer struct { - TypeOf reflect.Type - Func TransformerFunc +// see https://github.com/mitchellh/mapstructure/pull/294 +type decoder interface { + DecodeMapstructure(interface{}) error } -func createTransformHook(additionalTransformers ...Transformer) mapstructure.DecodeHookFuncType { - transforms := map[reflect.Type]func(interface{}) (interface{}, error){ - reflect.TypeOf(types.External{}): transformExternal, - reflect.TypeOf(types.HealthCheckTest{}): transformHealthCheckTest, - reflect.TypeOf(types.ShellCommand{}): transformShellCommand, - reflect.TypeOf(types.StringList{}): transformStringList, - reflect.TypeOf(map[string]string{}): transformMapStringString, - reflect.TypeOf(types.UlimitsConfig{}): transformUlimits, - reflect.TypeOf(types.UnitBytes(0)): transformSize, - reflect.TypeOf([]types.ServicePortConfig{}): transformServicePort, - reflect.TypeOf(types.ServiceSecretConfig{}): transformFileReferenceConfig, - reflect.TypeOf(types.ServiceConfigObjConfig{}): transformFileReferenceConfig, - reflect.TypeOf(types.StringOrNumberList{}): transformStringOrNumberList, - reflect.TypeOf(map[string]*types.ServiceNetworkConfig{}): transformServiceNetworkMap, - reflect.TypeOf(types.Mapping{}): transformMappingOrListFunc("=", false), - reflect.TypeOf(types.MappingWithEquals{}): transformMappingOrListFunc("=", true), - reflect.TypeOf(types.Labels{}): transformMappingOrListFunc("=", false), - reflect.TypeOf(types.MappingWithColon{}): transformMappingOrListFunc(":", false), - reflect.TypeOf(types.HostsList{}): transformMappingOrListFunc(":", false), - reflect.TypeOf(types.ServiceVolumeConfig{}): transformServiceVolumeConfig, - reflect.TypeOf(types.BuildConfig{}): transformBuildConfig, - reflect.TypeOf(types.Duration(0)): transformStringToDuration, - reflect.TypeOf(types.DependsOnConfig{}): transformDependsOnConfig, - reflect.TypeOf(types.ExtendsConfig{}): transformExtendsConfig, - reflect.TypeOf(types.DeviceRequest{}): transformServiceDeviceRequest, - reflect.TypeOf(types.SSHConfig{}): transformSSHConfig, - reflect.TypeOf(types.IncludeConfig{}): transformIncludeConfig, - } - - for _, transformer := range additionalTransformers { - transforms[transformer.TypeOf] = transformer.Func - } - - return func(_ reflect.Type, target reflect.Type, data interface{}) (interface{}, error) { - transform, ok := transforms[target] - if !ok { - return data, nil - } - return transform(data) +// see https://github.com/mitchellh/mapstructure/issues/115#issuecomment-735287466 +func decoderHook(from reflect.Value, to reflect.Value) (interface{}, error) { + // If the destination implements the decoder interface + u, ok := to.Interface().(decoder) + if !ok { + return from.Interface(), nil + } + // If it is nil and a pointer, create and assign the target value first + if to.Type().Kind() == reflect.Ptr && to.IsNil() { + to.Set(reflect.New(to.Type().Elem())) + u = to.Interface().(decoder) } + // Call the custom DecodeMapstructure method + if err := u.DecodeMapstructure(from.Interface()); err != nil { + return to.Interface(), err + } + return to.Interface(), nil } // keys need to be converted to strings for jsonschema @@ -794,6 +790,11 @@ func loadServiceWithExtends(ctx context.Context, filename, name string, services return nil, err } + baseFile, err = transform.ExpandShortSyntax(baseFile) + if err != nil { + return nil, errors.Wrapf(err, "failed to expand short syntax") + } + baseFileServices := getSection(baseFile, "services") baseService, err = loadServiceWithExtends(ctx, baseFilePath, baseServiceName, baseFileServices, filepath.Dir(baseFilePath), lookupEnv, opts, ct) if err != nil { @@ -1030,349 +1031,3 @@ func loadFileObjectConfig(name string, objType string, obj types.FileObjectConfi return obj, nil } - -var transformMapStringString TransformerFunc = func(data interface{}) (interface{}, error) { - switch value := data.(type) { - case map[string]interface{}: - return toMapStringString(value, false), nil - case map[string]string: - return value, nil - default: - return data, errors.Errorf("invalid type %T for map[string]string", value) - } -} - -var transformExternal TransformerFunc = func(data interface{}) (interface{}, error) { - switch value := data.(type) { - case bool: - return map[string]interface{}{"external": value}, nil - case map[string]interface{}: - return map[string]interface{}{"external": true, "name": value["name"]}, nil - default: - return data, errors.Errorf("invalid type %T for external", value) - } -} - -var transformServicePort TransformerFunc = func(data interface{}) (interface{}, error) { - switch entries := data.(type) { - case []interface{}: - // We process the list instead of individual items here. - // The reason is that one entry might be mapped to multiple ServicePortConfig. - // Therefore we take an input of a list and return an output of a list. - var ports []interface{} - for _, entry := range entries { - switch value := entry.(type) { - case int: - parsed, err := types.ParsePortConfig(fmt.Sprint(value)) - if err != nil { - return data, err - } - for _, v := range parsed { - ports = append(ports, v) - } - case string: - parsed, err := types.ParsePortConfig(value) - if err != nil { - return data, err - } - for _, v := range parsed { - ports = append(ports, v) - } - case map[string]interface{}: - published := value["published"] - if v, ok := published.(int); ok { - value["published"] = strconv.Itoa(v) - } - ports = append(ports, groupXFieldsIntoExtensions(value)) - default: - return data, errors.Errorf("invalid type %T for port", value) - } - } - return ports, nil - default: - return data, errors.Errorf("invalid type %T for port", entries) - } -} - -var transformServiceDeviceRequest TransformerFunc = func(data interface{}) (interface{}, error) { - switch value := data.(type) { - case map[string]interface{}: - count, ok := value["count"] - if ok { - switch val := count.(type) { - case int: - return value, nil - case string: - if strings.ToLower(val) == "all" { - value["count"] = -1 - return value, nil - } - i, err := strconv.ParseInt(val, 10, 64) - if err == nil { - value["count"] = i - return value, nil - } - return data, errors.Errorf("invalid string value for 'count' (the only value allowed is 'all' or a number)") - default: - return data, errors.Errorf("invalid type %T for device count", val) - } - } - return data, nil - default: - return data, errors.Errorf("invalid type %T for resource reservation", value) - } -} - -var transformFileReferenceConfig TransformerFunc = func(data interface{}) (interface{}, error) { - switch value := data.(type) { - case string: - return map[string]interface{}{"source": value}, nil - case map[string]interface{}: - if target, ok := value["target"]; ok { - value["target"] = cleanTarget(target.(string)) - } - return groupXFieldsIntoExtensions(value), nil - default: - return data, errors.Errorf("invalid type %T for secret", value) - } -} - -func cleanTarget(target string) string { - if target == "" { - return "" - } - return paths.Clean(target) -} - -var transformBuildConfig TransformerFunc = func(data interface{}) (interface{}, error) { - switch value := data.(type) { - case string: - return map[string]interface{}{"context": value}, nil - case map[string]interface{}: - return groupXFieldsIntoExtensions(data.(map[string]interface{})), nil - default: - return data, errors.Errorf("invalid type %T for service build", value) - } -} - -var transformDependsOnConfig TransformerFunc = func(data interface{}) (interface{}, error) { - switch value := data.(type) { - case []interface{}: - transformed := map[string]interface{}{} - for _, serviceIntf := range value { - service, ok := serviceIntf.(string) - if !ok { - return data, errors.Errorf("invalid type %T for service depends_on element, expected string", value) - } - transformed[service] = map[string]interface{}{"condition": types.ServiceConditionStarted, "required": true} - } - return transformed, nil - case map[string]interface{}: - transformed := map[string]interface{}{} - for service, val := range value { - dependsConfigIntf, ok := val.(map[string]interface{}) - if !ok { - return data, errors.Errorf("invalid type %T for service depends_on element", value) - } - if _, ok := dependsConfigIntf["required"]; !ok { - dependsConfigIntf["required"] = true - } - transformed[service] = dependsConfigIntf - } - return groupXFieldsIntoExtensions(transformed), nil - default: - return data, errors.Errorf("invalid type %T for service depends_on", value) - } -} - -var transformExtendsConfig TransformerFunc = func(value interface{}) (interface{}, error) { - switch value.(type) { - case string: - return map[string]interface{}{"service": value}, nil - case map[string]interface{}: - return value, nil - default: - return value, errors.Errorf("invalid type %T for extends", value) - } -} - -var transformServiceVolumeConfig TransformerFunc = func(data interface{}) (interface{}, error) { - switch value := data.(type) { - case string: - volume, err := ParseVolume(value) - volume.Target = cleanTarget(volume.Target) - return volume, err - case map[string]interface{}: - data := groupXFieldsIntoExtensions(data.(map[string]interface{})) - if target, ok := data["target"]; ok { - data["target"] = cleanTarget(target.(string)) - } - return data, nil - default: - return data, errors.Errorf("invalid type %T for service volume", value) - } -} - -var transformServiceNetworkMap TransformerFunc = func(value interface{}) (interface{}, error) { - if list, ok := value.([]interface{}); ok { - mapValue := map[interface{}]interface{}{} - for _, name := range list { - mapValue[name] = nil - } - return mapValue, nil - } - return value, nil -} - -var transformSSHConfig TransformerFunc = func(data interface{}) (interface{}, error) { - switch value := data.(type) { - case map[string]interface{}: - var result []types.SSHKey - for key, val := range value { - if val == nil { - val = "" - } - result = append(result, types.SSHKey{ID: key, Path: val.(string)}) - } - return result, nil - case []interface{}: - var result []types.SSHKey - for _, v := range value { - key, val := transformValueToMapEntry(v.(string), "=", false) - result = append(result, types.SSHKey{ID: key, Path: val.(string)}) - } - return result, nil - case string: - return ParseShortSSHSyntax(value) - } - return nil, errors.Errorf("expected a sting, map or a list, got %T: %#v", data, data) -} - -// ParseShortSSHSyntax parse short syntax for SSH authentications -func ParseShortSSHSyntax(value string) ([]types.SSHKey, error) { - if value == "" { - value = "default" - } - key, val := transformValueToMapEntry(value, "=", false) - result := []types.SSHKey{{ID: key, Path: val.(string)}} - return result, nil -} - -var transformStringOrNumberList TransformerFunc = func(value interface{}) (interface{}, error) { - list := value.([]interface{}) - result := make([]string, len(list)) - for i, item := range list { - result[i] = fmt.Sprint(item) - } - return result, nil -} - -var transformStringList TransformerFunc = func(data interface{}) (interface{}, error) { - switch value := data.(type) { - case string: - return []string{value}, nil - case []interface{}: - return value, nil - default: - return data, errors.Errorf("invalid type %T for string list", value) - } -} - -func transformMappingOrListFunc(sep string, allowNil bool) TransformerFunc { - return func(data interface{}) (interface{}, error) { - return transformMappingOrList(data, sep, allowNil) - } -} - -func transformMappingOrList(mappingOrList interface{}, sep string, allowNil bool) (interface{}, error) { - switch value := mappingOrList.(type) { - case map[string]interface{}: - return toMapStringString(value, allowNil), nil - case []interface{}: - result := make(map[string]interface{}) - for _, value := range value { - key, val := transformValueToMapEntry(value.(string), sep, allowNil) - result[key] = val - } - return result, nil - } - return nil, errors.Errorf("expected a map or a list, got %T: %#v", mappingOrList, mappingOrList) -} - -func transformValueToMapEntry(value string, separator string, allowNil bool) (string, interface{}) { - parts := strings.SplitN(value, separator, 2) - key := parts[0] - switch { - case len(parts) == 1 && allowNil: - return key, nil - case len(parts) == 1 && !allowNil: - return key, "" - default: - return key, parts[1] - } -} - -var transformShellCommand TransformerFunc = func(value interface{}) (interface{}, error) { - if str, ok := value.(string); ok { - return shellwords.Parse(str) - } - return value, nil -} - -var transformHealthCheckTest TransformerFunc = func(data interface{}) (interface{}, error) { - switch value := data.(type) { - case string: - return append([]string{"CMD-SHELL"}, value), nil - case []interface{}: - return value, nil - default: - return value, errors.Errorf("invalid type %T for healthcheck.test", value) - } -} - -var transformSize TransformerFunc = func(value interface{}) (interface{}, error) { - switch value := value.(type) { - case int: - return int64(value), nil - case int64, types.UnitBytes: - return value, nil - case string: - return units.RAMInBytes(value) - default: - return value, errors.Errorf("invalid type for size %T", value) - } -} - -var transformStringToDuration TransformerFunc = func(value interface{}) (interface{}, error) { - switch value := value.(type) { - case string: - d, err := time.ParseDuration(value) - if err != nil { - return value, err - } - return types.Duration(d), nil - case types.Duration: - return value, nil - default: - return value, errors.Errorf("invalid type %T for duration", value) - } -} - -func toMapStringString(value map[string]interface{}, allowNil bool) map[string]interface{} { - output := make(map[string]interface{}) - for key, value := range value { - output[key] = toString(value, allowNil) - } - return output -} - -func toString(value interface{}, allowNil bool) interface{} { - switch { - case value != nil: - return fmt.Sprint(value) - case allowNil: - return nil - default: - return "" - } -} diff --git a/loader/loader_test.go b/loader/loader_test.go index b40c447c..7a5e292a 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -1448,7 +1448,8 @@ func TestLoadVolumesWarnOnDeprecatedExternalNameVersion34(t *testing.T) { source := map[string]interface{}{ "foo": map[string]interface{}{ "external": map[string]interface{}{ - "name": "oops", + "external": true, + "name": "oops", }, }, } @@ -1462,7 +1463,6 @@ func TestLoadVolumesWarnOnDeprecatedExternalNameVersion34(t *testing.T) { } assert.Check(t, is.DeepEqual(expected, volumes)) assert.Check(t, is.Contains(buf.String(), "volume.external.name is deprecated")) - } func patchLogrus() (*bytes.Buffer, func()) { @@ -1479,7 +1479,8 @@ func TestLoadVolumesWarnOnDeprecatedExternalName(t *testing.T) { source := map[string]interface{}{ "foo": map[string]interface{}{ "external": map[string]interface{}{ - "name": "oops", + "external": true, + "name": "oops", }, }, } @@ -1533,7 +1534,8 @@ func TestLoadSecretsWarnOnDeprecatedExternalNameVersion35(t *testing.T) { source := map[string]interface{}{ "foo": map[string]interface{}{ "external": map[string]interface{}{ - "name": "oops", + "name": "oops", + "external": true, }, }, } @@ -1556,7 +1558,8 @@ func TestLoadNetworksWarnOnDeprecatedExternalNameVersion35(t *testing.T) { source := map[string]interface{}{ "foo": map[string]interface{}{ "external": map[string]interface{}{ - "name": "oops", + "name": "oops", + "external": true, }, }, } @@ -1580,7 +1583,8 @@ func TestLoadNetworksWarnOnDeprecatedExternalName(t *testing.T) { source := map[string]interface{}{ "foo": map[string]interface{}{ "external": map[string]interface{}{ - "name": "oops", + "name": "oops", + "external": true, }, }, } @@ -1809,30 +1813,6 @@ services: assert.Check(t, is.DeepEqual(expected, config.Services[0].Sysctls)) } -func TestTransform(t *testing.T) { - var source = []interface{}{ - "80-82:8080-8082", - "90-92:8090-8092/udp", - "85:8500", - 8600, - map[string]interface{}{ - "protocol": "udp", - "target": 53, - "published": 10053, - }, - map[string]interface{}{ - "mode": "host", - "target": 22, - "published": 10022, - }, - } - var ports []types.ServicePortConfig - err := Transform(source, &ports) - assert.NilError(t, err) - - assert.Check(t, is.DeepEqual(samplePortsConfig, ports)) -} - func TestLoadTemplateDriver(t *testing.T) { config, err := loadYAML(` name: load-template-driver @@ -2129,7 +2109,7 @@ services: devices: - driver: nvidia capabilities: [gpu] - count: somestring + count: some_string `) assert.ErrorContains(t, err, "invalid string value for 'count' (the only value allowed is 'all' or a number)") } @@ -2185,36 +2165,6 @@ func TestLoadServiceWithEnvFile(t *testing.T) { assert.Equal(t, "YES", *service.Environment["HALLO"]) } -func TestLoadServiceWithVolumes(t *testing.T) { - m := map[string]interface{}{ - "volumes": []interface{}{ - "source:/path 1/", - map[string]interface{}{ - "target": "/path 2/", - }, - }, - "configs": []interface{}{ - map[string]interface{}{ - "target": "/path 3/", - }, - }, - "secrets": []interface{}{ - map[string]interface{}{ - "target": "/path 4/", - }, - }, - } - s, err := LoadService("Test Name", m) - assert.NilError(t, err) - assert.Equal(t, len(s.Volumes), 2) - assert.Equal(t, "/path 1", s.Volumes[0].Target) - assert.Equal(t, "/path 2", s.Volumes[1].Target) - assert.Equal(t, len(s.Configs), 1) - assert.Equal(t, "/path 3", s.Configs[0].Target) - assert.Equal(t, len(s.Secrets), 1) - assert.Equal(t, "/path 4", s.Secrets[0].Target) -} - func TestLoadNoSSHInBuildConfig(t *testing.T) { actual, err := loadYAML(` name: load-no-ssh-in-build-config diff --git a/override/mergeYaml.go b/override/mergeYaml.go new file mode 100644 index 00000000..5eb3914e --- /dev/null +++ b/override/mergeYaml.go @@ -0,0 +1,141 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package override + +import ( + "fmt" + + "github.com/compose-spec/compose-go/tree" +) + +type merger func(interface{}, interface{}, tree.Path) (interface{}, error) + +// mergeSpecials defines the custom rules applied by compose when merging yaml trees +var mergeSpecials = map[tree.Path]merger{} + +func init() { + mergeSpecials["services.*.logging"] = mergeLogging + mergeSpecials["services.*.volumes"] = mergeVolumes + mergeSpecials["services.*.ports"] = mergePorts +} + +// MergeYaml merges map[string]interface{} yaml trees handling special rules +func MergeYaml(e interface{}, o interface{}, p tree.Path) (interface{}, error) { + for pattern, merger := range mergeSpecials { + if p.Matches(pattern) { + merged, err := merger(e, o, p) + if err != nil { + return nil, err + } + return merged, nil + } + } + switch value := e.(type) { + case map[string]interface{}: + other, ok := o.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("cannont override %s", p) + } + return mergeMappings(value, other, p) + case []interface{}: + other, ok := o.([]interface{}) + if !ok { + return nil, fmt.Errorf("cannont override %s", p) + } + return append(value, other...), nil + default: + return o, nil + } +} + +func mergeSlices(c []interface{}, o []interface{}, keyFn func(interface{}) interface{}, path tree.Path) (interface{}, error) { + merged := map[interface{}]interface{}{} + for _, v := range c { + merged[keyFn(v)] = v + } + for _, v := range o { + key := keyFn(v) + e, ok := merged[key] + if !ok { + merged[key] = v + continue + } + MergeYaml(e, v, path.Next("[]")) + } + sequence := make([]interface{}, 0, len(merged)) + for _, v := range merged { + sequence = append(sequence, v) + } + return sequence, nil +} + +func mergeMappings(mapping map[string]interface{}, other map[string]interface{}, p tree.Path) (map[string]interface{}, error) { + for k, v := range other { + next := p.Next(k) + e, ok := mapping[k] + if !ok { + mapping[k] = v + continue + } + merged, err := MergeYaml(e, v, next) + if err != nil { + return nil, err + } + mapping[k] = merged + } + return mapping, nil +} + +// ports is actually a map, indexed by ip:port:target/protocol +func mergePorts(v interface{}, o interface{}, path tree.Path) (interface{}, error) { + type port struct { + target interface{} + published interface{} + ip interface{} + protocol interface{} + } + return mergeSlices(v.([]interface{}), o.([]interface{}), func(i interface{}) interface{} { + m := i.(map[string]interface{}) + return port{ + target: m["target"], + published: m["published"], + ip: m["ip"], + protocol: m["protocol"], + } + }, path) +} + +// volumes is actually a map, indexed by mount path in container +func mergeVolumes(v interface{}, o interface{}, path tree.Path) (interface{}, error) { + return mergeSlices(v.([]interface{}), o.([]interface{}), func(i interface{}) interface{} { + m := i.(map[string]interface{}) + return m["target"] + }, path) +} + +// logging driver options are merged only when both compose file define the same driver +func mergeLogging(c interface{}, o interface{}, p tree.Path) (interface{}, error) { + config := c.(map[string]interface{}) + other := o.(map[string]interface{}) + // we override logging config if source and override have the same driver set, or none + d, ok1 := other["driver"] + o, ok2 := config["driver"] + if d == o || !ok1 || !ok2 { + return mergeMappings(config, other, p) + } + return other, nil +} diff --git a/override/mergeYaml_logging_test.go b/override/mergeYaml_logging_test.go new file mode 100644 index 00000000..c4c04a62 --- /dev/null +++ b/override/mergeYaml_logging_test.go @@ -0,0 +1,103 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package override + +import ( + "testing" +) + +// override using the same logging driver will override driver options +func Test_mergeYamlLoggingSameDriver(t *testing.T) { + assertMergeYaml(t, ` +services: + test: + image: foo + logging: + driver: syslog + options: + syslog-address: "tcp://192.168.0.42:123" +`, ` +services: + test: + logging: + driver: syslog + options: + syslog-address: "tcp://127.0.0.1:123" +`, ` +services: + test: + image: foo + logging: + driver: syslog + options: + syslog-address: "tcp://127.0.0.1:123" +`) +} + +// check override with a distinct logging driver fully overrides driver options +func Test_mergeYamlLoggingDistinctDriver(t *testing.T) { + assertMergeYaml(t, ` +services: + test: + image: foo + logging: + driver: local + options: + max-size: "10m" +`, ` +services: + test: + logging: + driver: syslog + options: + syslog-address: "tcp://127.0.0.1:123" +`, ` +services: + test: + image: foo + logging: + driver: syslog + options: + syslog-address: "tcp://127.0.0.1:123" +`) +} + +// check override without an explicit driver set (defaults to local driver) +func Test_mergeYamlLoggingImplicitDriver(t *testing.T) { + assertMergeYaml(t, ` +services: + test: + image: foo + logging: + options: + max-size: "10m" +`, ` +services: + test: + logging: + options: + max-file: 3 +`, ` +services: + test: + image: foo + logging: + options: + max-size: "10m" + max-file: 3 +`) +} diff --git a/override/mergeYaml_ports_test.go b/override/mergeYaml_ports_test.go new file mode 100644 index 00000000..63c3dbc6 --- /dev/null +++ b/override/mergeYaml_ports_test.go @@ -0,0 +1,54 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package override + +import ( + "testing" +) + +func Test_mergeYamlPortsLongSyntax(t *testing.T) { + assertMergeYaml(t, ` +services: + test: + image: foo + ports: + - target: 8080 + published: "80" + protocol: tcp +`, ` +services: + test: + ports: + - target: 8080 + published: "80" + protocol: tcp + - target: 8443 + published: "443" + protocol: tcp +`, ` +services: + test: + image: foo + ports: + - target: 8080 + published: "80" + protocol: tcp + - target: 8443 + published: "443" + protocol: tcp +`) +} diff --git a/override/mergeYaml_test.go b/override/mergeYaml_test.go new file mode 100644 index 00000000..f84e905a --- /dev/null +++ b/override/mergeYaml_test.go @@ -0,0 +1,59 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package override + +import ( + "testing" + + "github.com/compose-spec/compose-go/tree" + "gopkg.in/yaml.v3" + "gotest.tools/v3/assert" +) + +func Test_mergeYamlBase(t *testing.T) { + assertMergeYaml(t, ` +services: + test: + image: foo + container_name: foo + init: true +`, ` +services: + test: + image: bar + init: false +`, ` +services: + test: + container_name: foo + image: bar + init: false +`) +} + +func assertMergeYaml(t *testing.T, right string, left string, want string) { + got, err := MergeYaml(unmarshall(t, right), unmarshall(t, left), tree.NewPath()) + assert.NilError(t, err) + assert.DeepEqual(t, got, unmarshall(t, want)) +} + +func unmarshall(t *testing.T, s string) map[string]interface{} { + var val map[string]interface{} + err := yaml.Unmarshal([]byte(s), &val) + assert.NilError(t, err) + return val +} diff --git a/override/mergeYaml_volumes_test.go b/override/mergeYaml_volumes_test.go new file mode 100644 index 00000000..306f88ab --- /dev/null +++ b/override/mergeYaml_volumes_test.go @@ -0,0 +1,67 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package override + +import ( + "testing" +) + +func Test_mergeYamlVolumesLongSyntax(t *testing.T) { + assertMergeYaml(t, ` +services: + test: + image: foo + volumes: + # /src:/target + - type: bind + source: /src + target: /target + # /foo:/bar + - type: bind + source: /foo + target: /bar +`, ` +services: + test: + volumes: + # /src:/target + - type: bind + source: /src + target: /target + # /zot:/qix + - type: bind + source: /zot + target: /qix +`, ` +services: + test: + image: foo + volumes: + # /src:/target + - type: bind + source: /src + target: /target + # /foo:/bar + - type: bind + source: /foo + target: /bar + # /zot:/qix + - type: bind + source: /zot + target: /qix +`) +} diff --git a/transform/devices_test.go b/transform/devices_test.go new file mode 100644 index 00000000..f42bfd12 --- /dev/null +++ b/transform/devices_test.go @@ -0,0 +1,84 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import ( + "testing" + + "gotest.tools/v3/assert" +) + +func TestExpandDevicesAll(t *testing.T) { + assertExpand(t, ` +services: + test: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all +`, ` +services: + test: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: -1 + +`) +} + +func TestExpandDevicesCountAsString(t *testing.T) { + assertExpand(t, ` +services: + test: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: "1" +`, ` +services: + test: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + +`) +} + +func TestExpandDevicesCountInvalidString(t *testing.T) { + _, err := ExpandShortSyntax(unmarshall(t, ` +services: + test: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: some_string +`)) + assert.Error(t, err, "invalid string value for 'count' (the only value allowed is 'all' or a number)") + +} diff --git a/transform/expand.go b/transform/expand.go new file mode 100644 index 00000000..c1cf78b7 --- /dev/null +++ b/transform/expand.go @@ -0,0 +1,516 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import ( + "fmt" + "strconv" + "strings" + + "github.com/compose-spec/compose-go/tree" + "github.com/compose-spec/compose-go/types" + "github.com/docker/go-units" + "github.com/mattn/go-shellwords" + "github.com/pkg/errors" +) + +type transformFunc func(data interface{}) (interface{}, error) + +var ( + // TODO(ndeloof) Labels, Mapping, MappingWithEqual and HostsList could implement yaml.Unmarhsaller + transformLabels = transformMappingOrListFunc("=", false) + transformHostList = transformMappingOrListFunc(":", false) + transformMapping = transformMappingOrListFunc("=", false) + transformMappingWithEqual = transformMappingOrListFunc("=", true) +) + +// transformers list the places compose spec allows a short syntax we have to expand +var transformers = map[tree.Path]transformFunc{ + "services.*.annotations": transformMapping, + "services.*.blkio_config.device_read_bps.*.rate": transformSize, + "services.*.blkio_config.device_read_iops.*.rate": transformSize, + "services.*.blkio_config.device_write_bps.*.rate": transformSize, + "services.*.blkio_config.device_write_iops.*.rate": transformSize, + "services.*.build": transformBuildConfig, + "services.*.build.additional_contexts": transformMapping, + "services.*.build.args": transformMappingWithEqual, + "services.*.build.cache_from": transformStringList, + "services.*.build.cache_to": transformStringList, + "services.*.build.extra_hosts": transformHostList, + "services.*.build.labels": transformLabels, + "services.*.build.tags": transformStringList, + "services.*.build.platforms": transformStringList, + "services.*.build.secrets.*": transformFileReferenceConfig, + "services.*.build.ssh": transformSSHConfig, + "services.*.command": transformShellCommand, + "services.*.configs.*": transformFileReferenceConfig, + "services.*.depends_on": transformDependsOnConfig, + "services.*.deploy.labels": transformLabels, + "services.*.deploy.resources.limits.memory": transformSize, + "services.*.deploy.resources.reservations.memory": transformSize, + "services.*.deploy.resources.reservations.devices.*.count": transformServiceDeviceRequestCount, + "services.*.dns": transformStringList, + "services.*.dns_search": transformStringList, + "services.*.entrypoint": transformShellCommand, + "services.*.env_file": transformStringList, + "services.*.environment": transformMappingWithEqual, + "services.*.expose": transformStringOrNumberList, + "services.*.extends": transformExtends, + "services.*.extra_hosts": transformHostList, + "services.*.healthcheck.test": transformHealthCheckTest, + "services.*.labels": transformLabels, + "services.*.mem_limit": transformSize, + "services.*.mem_reservation": transformSize, + "services.*.mem_swap_limit": transformSize, + "services.*.mem_swapiness": transformSize, + "services.*.networks": transformServiceNetworks, + "services.*.ports": transformServicePorts, + "services.*.secrets.*": transformFileReferenceConfig, + "services.*.shm_size": transformSize, + "services.*.sysctls": transformMapping, + "services.*.tmpfs": transformStringList, + "services.*.ulimits.*": transformUlimits, + "services.*.volumes.*": transformServiceVolume, + "services.*.volumes.*.tmpfs.size": transformSize, + "include.*": transformIncludeConfig, + "include.*.path": transformStringList, + "include.*.env_file": transformStringList, + "configs.*.external": transformExternal, + "configs.*.labels": transformLabels, + "networks.*.driver_opts.*": transformDriverOpt, + "networks.*.external": transformExternal, + "networks.*.labels": transformLabels, + "volumes.*.driver_opts.*": transformDriverOpt, + "volumes.*.external": transformExternal, + "volumes.*.labels": transformLabels, + "secrets.*.external": transformExternal, + "secrets.*.labels": transformLabels, +} + +func ExpandShortSyntax(composefile map[string]interface{}) (map[string]interface{}, error) { + m, err := transform(composefile, tree.NewPath()) + if err != nil { + return nil, err + } + return m.(map[string]interface{}), nil +} + +func transform(e interface{}, p tree.Path) (interface{}, error) { + for pattern, transformer := range transformers { + if p.Matches(pattern) { + t, err := transformer(e) + if err != nil { + return nil, err + } + e = t + } + } + switch value := e.(type) { + case map[string]interface{}: + for k, v := range value { + t, err := transform(v, p.Next(k)) + if err != nil { + return nil, err + } + value[k] = t + } + return value, nil + case []interface{}: + for i, e := range value { + t, err := transform(e, p.Next("[]")) + if err != nil { + return nil, err + } + value[i] = t + } + return value, nil + default: + return e, nil + } +} + +func transformServiceVolume(data interface{}) (interface{}, error) { + if value, ok := data.(string); ok { + volume, err := parseVolume(value) + if err != nil { + return nil, err + } + vol := map[string]interface{}{ + "type": volume.Type, + "source": volume.Source, + "target": volume.Target, + "read_only": volume.ReadOnly, + } + if volume.Volume != nil { + vol["volume"] = map[string]interface{}{ + "nocopy": volume.Volume.NoCopy, + } + } + if volume.Bind != nil { + vol["bind"] = map[string]interface{}{ + "create_host_path": volume.Bind.CreateHostPath, + "propagation": volume.Bind.Propagation, + "selinux": volume.Bind.SELinux, + } + } + return omitEmpty(vol), nil + } + return data, nil +} + +// transformServicePorts process the list instead of individual ports as a port-range definition will result in multiple +// items, so we flatten this into a single sequence +func transformServicePorts(data interface{}) (interface{}, error) { + switch entries := data.(type) { + case []interface{}: + var ports []interface{} + for _, entry := range entries { + switch value := entry.(type) { + case int, string: + parsed, err := types.ParsePortConfig(fmt.Sprint(value)) + if err != nil { + return data, err + } + for _, v := range parsed { + ports = append(ports, map[string]interface{}{ + "mode": v.Mode, + "host_ip": v.HostIP, + "target": int(v.Target), + "published": v.Published, + "protocol": v.Protocol, + }) + } + case map[string]interface{}: + published := value["published"] + if v, ok := published.(int); ok { + value["published"] = strconv.Itoa(v) + } + ports = append(ports, value) + default: + return data, errors.Errorf("invalid type %T for port", value) + } + } + return ports, nil + default: + return data, errors.Errorf("invalid type %T for port", entries) + } +} + +func transformServiceNetworks(data interface{}) (interface{}, error) { + if list, ok := data.([]interface{}); ok { + mapValue := map[interface{}]interface{}{} + for _, name := range list { + mapValue[name] = nil + } + return mapValue, nil + } + return data, nil +} + +func transformExternal(data interface{}) (interface{}, error) { + switch value := data.(type) { + case bool: + return map[string]interface{}{"external": value}, nil + case map[string]interface{}: + return map[string]interface{}{"external": true, "name": value["name"]}, nil + default: + return data, errors.Errorf("invalid type %T for external", value) + } +} + +func transformHealthCheckTest(data interface{}) (interface{}, error) { + switch value := data.(type) { + case string: + return append([]string{"CMD-SHELL"}, value), nil + case []interface{}: + return value, nil + default: + return value, errors.Errorf("invalid type %T for healthcheck.test", value) + } +} + +func transformShellCommand(data interface{}) (interface{}, error) { + switch value := data.(type) { + case string: + args, err := shellwords.Parse(value) + res := make([]interface{}, len(args)) + for i, arg := range args { + res[i] = arg + } + return res, err + case []interface{}: + return value, nil + default: + // ShellCommand do NOT have omitempty tag, to distinguish unset vs empty + if data == nil { + return nil, nil + } + return data, errors.Errorf("invalid type %T for shell command", value) + } +} + +func transformDependsOnConfig(data interface{}) (interface{}, error) { + switch value := data.(type) { + case []interface{}: + transformed := map[string]interface{}{} + for _, serviceIntf := range value { + service, ok := serviceIntf.(string) + if !ok { + return data, errors.Errorf("invalid type %T for service depends_on element, expected string", value) + } + transformed[service] = map[string]interface{}{ + "condition": types.ServiceConditionStarted, + "required": true, + } + } + return transformed, nil + case map[string]interface{}: + transformed := map[string]interface{}{} + for service, val := range value { + dependsConfigIntf, ok := val.(map[string]interface{}) + if !ok { + return data, errors.Errorf("invalid type %T for service depends_on element", value) + } + if _, ok := dependsConfigIntf["required"]; !ok { + dependsConfigIntf["required"] = true + } + transformed[service] = dependsConfigIntf + } + return transformed, nil + default: + return data, errors.Errorf("invalid type %T for service depends_on", value) + } +} + +func transformFileReferenceConfig(data interface{}) (interface{}, error) { + switch value := data.(type) { + case string: + return map[string]interface{}{"source": value}, nil + case map[string]interface{}: + return value, nil + default: + return data, errors.Errorf("invalid type %T for secret", value) + } +} + +func transformBuildConfig(data interface{}) (interface{}, error) { + switch value := data.(type) { + case string: + return map[string]interface{}{"context": value}, nil + case map[string]interface{}: + return data, nil + default: + return data, errors.Errorf("invalid type %T for service build", value) + } +} + +func transformExtends(data interface{}) (interface{}, error) { + switch data.(type) { + case string: + return map[string]interface{}{"service": data}, nil + case map[string]interface{}: + return data, nil + default: + return data, errors.Errorf("invalid type %T for extends", data) + } +} + +func transformStringList(data interface{}) (interface{}, error) { + switch value := data.(type) { + case string: + return []string{value}, nil + case []interface{}: + return value, nil + default: + return data, errors.Errorf("invalid type %T for string list", value) + } +} + +// TODO(ndeloof) StringOrNumberList could implement yaml.Unmarhsaler +func transformStringOrNumberList(data interface{}) (interface{}, error) { + list := data.([]interface{}) + result := make([]string, len(list)) + for i, item := range list { + result[i] = fmt.Sprint(item) + } + return result, nil +} + +// TODO(ndeloof) UlimitsConfig could implement yaml.Unmarhsaler +func transformUlimits(data interface{}) (interface{}, error) { + switch value := data.(type) { + case int: + return map[string]interface{}{"single": value}, nil + case map[string]interface{}: + return data, nil + default: + return data, errors.Errorf("invalid type %T for ulimits", value) + } +} + +// TODO(ndeloof) UnitBytes could implement yaml.Unmarhsaler +func transformSize(data interface{}) (interface{}, error) { + switch value := data.(type) { + case int: + return value, nil + case int64, types.UnitBytes: + return value, nil + case string: + return units.RAMInBytes(value) + default: + return value, errors.Errorf("invalid type for size %T", value) + } +} + +// TODO(ndeloof) create a DriverOpts type and implement yaml.Unmarshaler +func transformDriverOpt(data interface{}) (interface{}, error) { + switch value := data.(type) { + case int: + return strconv.Itoa(value), nil + case string: + return value, nil + default: + return data, errors.Errorf("invalid type %T for driver_opts value", value) + } +} + +func transformServiceDeviceRequestCount(data interface{}) (interface{}, error) { + switch value := data.(type) { + case int: + return value, nil + case string: + if strings.ToLower(value) == "all" { + return -1, nil + } + i, err := strconv.Atoi(value) + if err == nil { + return i, nil + } + return data, errors.Errorf("invalid string value for 'count' (the only value allowed is 'all' or a number)") + default: + return data, errors.Errorf("invalid type %T for device count", value) + } +} + +// TODO(ndeloof) SSHConfig could implement yaml.Unmarshal +func transformSSHConfig(data interface{}) (interface{}, error) { + switch value := data.(type) { + case map[string]interface{}: + var result []interface{} + for key, val := range value { + if val == nil { + val = "" + } + result = append(result, map[string]interface{}{"id": key, "path": val.(string)}) + } + return result, nil + case []interface{}: + var result []interface{} + for _, v := range value { + key, val := transformValueToMapEntry(v.(string), "=", false) + result = append(result, map[string]interface{}{"id": key, "path": val.(string)}) + } + return result, nil + case string: + return ParseShortSSHSyntax(value) + } + return nil, errors.Errorf("expected a sting, map or a list, got %T: %#v", data, data) +} + +// ParseShortSSHSyntax parse short syntax for SSH authentications +func ParseShortSSHSyntax(value string) ([]types.SSHKey, error) { + if value == "" { + value = "default" + } + key, val := transformValueToMapEntry(value, "=", false) + result := []types.SSHKey{{ID: key, Path: val.(string)}} + return result, nil +} + +func transformValueToMapEntry(value string, separator string, allowNil bool) (string, interface{}) { + parts := strings.SplitN(value, separator, 2) + key := parts[0] + switch { + case len(parts) == 1 && allowNil: + return key, nil + case len(parts) == 1 && !allowNil: + return key, "" + default: + return key, parts[1] + } +} + +func transformIncludeConfig(data interface{}) (interface{}, error) { + switch value := data.(type) { + case string: + return map[string]interface{}{"path": value}, nil + case map[string]interface{}: + return value, nil + default: + return data, errors.Errorf("invalid type %T for `include` configuration", value) + } +} + +func transformMappingOrListFunc(sep string, allowNil bool) transformFunc { + return func(data interface{}) (interface{}, error) { + switch value := data.(type) { + case map[string]interface{}: + result := make(map[string]interface{}) + for key, v := range value { + result[key] = toString(v, allowNil) + } + return result, nil + case []interface{}: + result := make(map[string]interface{}) + for _, v := range value { + key, val := transformValueToMapEntry(v.(string), sep, allowNil) + result[key] = val + } + return result, nil + } + return nil, errors.Errorf("expected a map or a list, got %T: %#v", data, data) + } +} + +func toString(value interface{}, allowNil bool) interface{} { + switch { + case value != nil: + return fmt.Sprint(value) + case allowNil: + return nil + default: + return "" + } +} + +func omitEmpty(m map[string]interface{}) interface{} { + for k, v := range m { + switch e := v.(type) { + case string: + if e == "" { + delete(m, k) + } + case int, int32, int64: + if e == 0 { + delete(m, k) + } + case map[string]interface{}: + m[k] = omitEmpty(e) + } + } + return m +} diff --git a/transform/expand_test.go b/transform/expand_test.go new file mode 100644 index 00000000..3993d772 --- /dev/null +++ b/transform/expand_test.go @@ -0,0 +1,38 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import ( + "testing" + + "gopkg.in/yaml.v3" + "gotest.tools/v3/assert" +) + +func assertExpand(t *testing.T, input string, want string) { + got, err := ExpandShortSyntax(unmarshall(t, input)) + assert.NilError(t, err) + + assert.DeepEqual(t, unmarshall(t, want), got) +} + +func unmarshall(t *testing.T, s string) map[string]interface{} { + var val map[string]interface{} + err := yaml.Unmarshal([]byte(s), &val) + assert.NilError(t, err) + return val +} diff --git a/transform/external_test.go b/transform/external_test.go new file mode 100644 index 00000000..b50c2cea --- /dev/null +++ b/transform/external_test.go @@ -0,0 +1,53 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import "testing" + +func TestExpandExternal(t *testing.T) { + assertExpand(t, ` +volumes: + foo: + external: true +secrets: + foo: + external: true +configs: + foo: + external: true +networks: + foo: + external: true +`, ` +volumes: + foo: + external: + external: true +secrets: + foo: + external: + external: true +configs: + foo: + external: + external: true +networks: + foo: + external: + external: true +`) +} diff --git a/transform/extra_hosts_test.go b/transform/extra_hosts_test.go new file mode 100644 index 00000000..0de54db7 --- /dev/null +++ b/transform/extra_hosts_test.go @@ -0,0 +1,35 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import "testing" + +func TestExpandExtraHosts(t *testing.T) { + assertExpand(t, ` +services: + test: + extra_hosts: + - "alpha:50.31.209.229" + - "zulu:ff02::1" +`, ` +services: + test: + extra_hosts: + alpha: "50.31.209.229" + zulu: "ff02::1" +`) +} diff --git a/transform/ports_test.go b/transform/ports_test.go new file mode 100644 index 00000000..f43b57b5 --- /dev/null +++ b/transform/ports_test.go @@ -0,0 +1,85 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import "testing" + +func TestExpandServicePorts(t *testing.T) { + t.Parallel() + t.Run("simple int", func(t *testing.T) { + assertExpand(t, ` +services: + test: + ports: + - 9090 +`, ` +services: + test: + ports: + - mode: ingress + host_ip: "" + published: "" + target: 9090 + protocol: tcp +`) + }) + t.Run("simple port binding", func(t *testing.T) { + assertExpand(t, ` +services: + test: + ports: + - 127.0.0.1:8080:80/tcp +`, ` +services: + test: + ports: + - mode: ingress + host_ip: 127.0.0.1 + published: "8080" + target: 80 + protocol: tcp +`) + }) + t.Run("port range", func(t *testing.T) { + assertExpand(t, ` +services: + test: + ports: + - 127.0.0.1:8080-8082:80-82/tcp +`, ` +services: + test: + ports: + - mode: ingress + host_ip: 127.0.0.1 + published: "8080" + target: 80 + protocol: tcp + - mode: ingress + host_ip: 127.0.0.1 + published: "8081" + target: 81 + protocol: tcp + - mode: ingress + host_ip: 127.0.0.1 + published: "8082" + target: 82 + protocol: tcp +`) + }) + +} diff --git a/transform/shellCommands_test.go b/transform/shellCommands_test.go new file mode 100644 index 00000000..2b1d800e --- /dev/null +++ b/transform/shellCommands_test.go @@ -0,0 +1,35 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import ( + "testing" +) + +func TestExpandShellCommands(t *testing.T) { + assertExpand(t, ` +services: + test: + command: hello world + entrypoint: /bin/sh -c echo +`, ` +services: + test: + command: ["hello", "world"] + entrypoint: ["/bin/sh", "-c", "echo"] +`) +} diff --git a/transform/ulimits_test.go b/transform/ulimits_test.go new file mode 100644 index 00000000..747a2d7f --- /dev/null +++ b/transform/ulimits_test.go @@ -0,0 +1,41 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import ( + "testing" +) + +func TestUlimits(t *testing.T) { + assertExpand(t, ` +services: + test: + ulimits: + noproc: 65535 + nofile: + soft: 11111 + hard: 99999 +`, ` +services: + test: + ulimits: + noproc: + single: 65535 + nofile: + soft: 11111 + hard: 99999`) +} diff --git a/loader/volume.go b/transform/volume.go similarity index 95% rename from loader/volume.go rename to transform/volume.go index dd83414a..d0bfbbb2 100644 --- a/loader/volume.go +++ b/transform/volume.go @@ -14,7 +14,7 @@ limitations under the License. */ -package loader +package transform import ( "strings" @@ -27,8 +27,8 @@ import ( const endOfSpec = rune(0) -// ParseVolume parses a volume spec without any knowledge of the target platform -func ParseVolume(spec string) (types.ServiceVolumeConfig, error) { +// parseVolume parses a volume spec without interface{} knowledge of the target platform +func parseVolume(spec string) (types.ServiceVolumeConfig, error) { volume := types.ServiceVolumeConfig{} switch len(spec) { @@ -80,7 +80,7 @@ func populateFieldFromBuffer(char rune, buffer []rune, volume *types.ServiceVolu volume.Target = strBuffer return nil case char == ':': - return errors.New("too many colons") + return errors.New("too minterface{} colons") } for _, option := range strings.Split(strBuffer, ",") { switch option { diff --git a/loader/volume_test.go b/transform/volume_test.go similarity index 89% rename from loader/volume_test.go rename to transform/volume_test.go index 55be29e0..6f088088 100644 --- a/loader/volume_test.go +++ b/transform/volume_test.go @@ -14,7 +14,7 @@ limitations under the License. */ -package loader +package transform import ( "fmt" @@ -27,7 +27,7 @@ import ( func TestParseVolumeAnonymousVolume(t *testing.T) { for _, path := range []string{"/path", "/path/foo"} { - volume, err := ParseVolume(path) + volume, err := parseVolume(path) expected := types.ServiceVolumeConfig{Type: "volume", Target: path, Volume: &types.ServiceVolumeVolume{}} assert.NilError(t, err) assert.Check(t, is.DeepEqual(expected, volume)) @@ -36,7 +36,7 @@ func TestParseVolumeAnonymousVolume(t *testing.T) { func TestParseVolumeAnonymousVolumeWindows(t *testing.T) { for _, path := range []string{"C:\\path", "Z:\\path\\foo"} { - volume, err := ParseVolume(path) + volume, err := parseVolume(path) expected := types.ServiceVolumeConfig{Type: "volume", Target: path, Volume: &types.ServiceVolumeVolume{}} assert.NilError(t, err) assert.Check(t, is.DeepEqual(expected, volume)) @@ -44,13 +44,13 @@ func TestParseVolumeAnonymousVolumeWindows(t *testing.T) { } func TestParseVolumeTooManyColons(t *testing.T) { - _, err := ParseVolume("/foo:/foo:ro:foo") - assert.Error(t, err, "invalid spec: /foo:/foo:ro:foo: too many colons") + _, err := parseVolume("/foo:/foo:ro:foo") + assert.Error(t, err, "invalid spec: /foo:/foo:ro:foo: too minterface{} colons") } func TestParseVolumeShortVolumes(t *testing.T) { for _, path := range []string{".", "/a"} { - volume, err := ParseVolume(path) + volume, err := parseVolume(path) expected := types.ServiceVolumeConfig{Type: "volume", Target: path} assert.NilError(t, err) assert.Check(t, is.DeepEqual(expected, volume)) @@ -59,14 +59,14 @@ func TestParseVolumeShortVolumes(t *testing.T) { func TestParseVolumeMissingSource(t *testing.T) { for _, spec := range []string{":foo", "/foo::ro"} { - _, err := ParseVolume(spec) + _, err := parseVolume(spec) assert.ErrorContains(t, err, "empty section between colons") } } func TestParseVolumeBindMount(t *testing.T) { for _, path := range []string{"./foo", "~/thing", "../other", "/foo", "/home/user"} { - volume, err := ParseVolume(path + ":/target") + volume, err := parseVolume(path + ":/target") expected := types.ServiceVolumeConfig{ Type: "bind", Source: path, @@ -85,7 +85,7 @@ func TestParseVolumeRelativeBindMountWindows(t *testing.T) { "../other", "D:\\path", "/home/user", } { - volume, err := ParseVolume(path + ":d:\\target") + volume, err := parseVolume(path + ":d:\\target") expected := types.ServiceVolumeConfig{ Type: "bind", Source: path, @@ -98,7 +98,7 @@ func TestParseVolumeRelativeBindMountWindows(t *testing.T) { } func TestParseVolumeWithBindOptions(t *testing.T) { - volume, err := ParseVolume("/source:/target:slave") + volume, err := parseVolume("/source:/target:slave") expected := types.ServiceVolumeConfig{ Type: "bind", Source: "/source", @@ -113,7 +113,7 @@ func TestParseVolumeWithBindOptions(t *testing.T) { } func TestParseVolumeWithBindOptionsSELinuxShared(t *testing.T) { - volume, err := ParseVolume("/source:/target:ro,z") + volume, err := parseVolume("/source:/target:ro,z") expected := types.ServiceVolumeConfig{ Type: "bind", Source: "/source", @@ -129,7 +129,7 @@ func TestParseVolumeWithBindOptionsSELinuxShared(t *testing.T) { } func TestParseVolumeWithBindOptionsSELinuxPrivate(t *testing.T) { - volume, err := ParseVolume("/source:/target:ro,Z") + volume, err := parseVolume("/source:/target:ro,Z") expected := types.ServiceVolumeConfig{ Type: "bind", Source: "/source", @@ -145,7 +145,7 @@ func TestParseVolumeWithBindOptionsSELinuxPrivate(t *testing.T) { } func TestParseVolumeWithBindOptionsWindows(t *testing.T) { - volume, err := ParseVolume("C:\\source\\foo:D:\\target:ro,rprivate") + volume, err := parseVolume("C:\\source\\foo:D:\\target:ro,rprivate") expected := types.ServiceVolumeConfig{ Type: "bind", Source: "C:\\source\\foo", @@ -161,12 +161,12 @@ func TestParseVolumeWithBindOptionsWindows(t *testing.T) { } func TestParseVolumeWithInvalidVolumeOptions(t *testing.T) { - _, err := ParseVolume("name:/target:bogus") + _, err := parseVolume("name:/target:bogus") assert.NilError(t, err) } func TestParseVolumeWithVolumeOptions(t *testing.T) { - volume, err := ParseVolume("name:/target:nocopy") + volume, err := parseVolume("name:/target:nocopy") expected := types.ServiceVolumeConfig{ Type: "volume", Source: "name", @@ -179,7 +179,7 @@ func TestParseVolumeWithVolumeOptions(t *testing.T) { func TestParseVolumeWithReadOnly(t *testing.T) { for _, path := range []string{"./foo", "/home/user"} { - volume, err := ParseVolume(path + ":/target:ro") + volume, err := parseVolume(path + ":/target:ro") expected := types.ServiceVolumeConfig{ Type: "bind", Source: path, @@ -194,7 +194,7 @@ func TestParseVolumeWithReadOnly(t *testing.T) { func TestParseVolumeWithRW(t *testing.T) { for _, path := range []string{"./foo", "/home/user"} { - volume, err := ParseVolume(path + ":/target:rw") + volume, err := parseVolume(path + ":/target:rw") expected := types.ServiceVolumeConfig{ Type: "bind", Source: path, @@ -208,7 +208,7 @@ func TestParseVolumeWithRW(t *testing.T) { } func TestParseVolumeWindowsNamedPipe(t *testing.T) { - volume, err := ParseVolume(`\\.\pipe\docker_engine:\\.\pipe\inside`) + volume, err := parseVolume(`\\.\pipe\docker_engine:\\.\pipe\inside`) assert.NilError(t, err) expected := types.ServiceVolumeConfig{ Type: "bind", @@ -263,7 +263,7 @@ func TestParseVolumeSplitCases(t *testing.T) { // Cover directories with one-character name {`/tmp/x/y:/foo/x/y`, -1, []string{`/tmp/x/y`, `/foo/x/y`}}, } { - parsed, _ := ParseVolume(x.input) + parsed, _ := parseVolume(x.input) expected := len(x.expected) > 1 msg := fmt.Sprintf("Case %d: %s", casenumber, x.input) @@ -272,12 +272,12 @@ func TestParseVolumeSplitCases(t *testing.T) { } func TestParseVolumeInvalidEmptySpec(t *testing.T) { - _, err := ParseVolume("") + _, err := parseVolume("") assert.ErrorContains(t, err, "invalid empty volume spec") } func TestParseVolumeInvalidSections(t *testing.T) { - _, err := ParseVolume("/foo::rw") + _, err := parseVolume("/foo::rw") assert.ErrorContains(t, err, "invalid spec") } diff --git a/transform/volumes_test.go b/transform/volumes_test.go new file mode 100644 index 00000000..495ebb6e --- /dev/null +++ b/transform/volumes_test.go @@ -0,0 +1,38 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import "testing" + +func TestExpandServiceVolume(t *testing.T) { + assertExpand(t, ` +services: + test: + volumes: + - .:/src:ro +`, ` +services: + test: + volumes: + - type: bind + source: . + target: /src + read_only: true + bind: + create_host_path: true +`) +} diff --git a/types/project.go b/types/project.go index 713b2074..f3393060 100644 --- a/types/project.go +++ b/types/project.go @@ -504,6 +504,10 @@ func (p *Project) MarshalYAML() ([]byte, error) { // MarshalJSON makes Config implement json.Marshaler func (p *Project) MarshalJSON() ([]byte, error) { + return p.MarshalJSONIndent("", "") +} + +func (p *Project) MarshalJSONIndent(prefix, indent string) ([]byte, error) { m := map[string]interface{}{ "name": p.Name, "services": p.Services, @@ -524,7 +528,7 @@ func (p *Project) MarshalJSON() ([]byte, error) { for k, v := range p.Extensions { m[k] = v } - return json.Marshal(m) + return json.MarshalIndent(m, prefix, indent) } // ResolveServicesEnvironment parse env_files set for services to resolve the actual environment map for services