diff --git a/hack/docs/mutations.json b/hack/docs/mutations.json index 3f1d5b48b..1e04b58fc 100644 --- a/hack/docs/mutations.json +++ b/hack/docs/mutations.json @@ -219,7 +219,8 @@ "path": "properties.assets.properties.v1.items.properties.helm.properties.github.properties", "delete": [ "dest", - "mode" + "mode", + "strip_path" ] }, { @@ -352,15 +353,17 @@ "ref": "8fcaebe55af67fe6789fa678faaa76fa867fbc", "path": "k8s-yamls/", "dest": "./k8s/", - "source": "private" + "source": "private", + "strip_path": "" }, { "repo": "github.com/replicatedhq/ship", "ref": "master", "path": "hack/docs/", - "dest": "./docs/", + "dest": "./docs{{repl Add 1 1}}/", "source": "public", - "mode": 644 + "mode": 644, + "strip_path": "{{repl ParseBool \"true\"}}" } ] }, @@ -409,6 +412,12 @@ "description": "One of `public` or `private`, if `private`, access to the repo can be validated on release creation" } }, + { + "path": "properties.assets.properties.v1.items.properties.github.properties.strip_path", + "merge": { + "description": "If true, the github directory will not be included in the filepath of the generated files. For instance, when outputting all files within 'source/' in the repository to the 'dest/' directory, the file 'source/a/file.txt' would be placed at 'dest/source/a/file.txt' when this is false and 'dest/a/file.txt' when this is true." + } + }, { "path": "properties.assets.properties.v1.items.properties.github.properties.when", "merge": { diff --git a/hack/docs/schema.json b/hack/docs/schema.json index 301253740..78ad8f714 100644 --- a/hack/docs/schema.json +++ b/hack/docs/schema.json @@ -308,15 +308,17 @@ "ref": "8fcaebe55af67fe6789fa678faaa76fa867fbc", "path": "k8s-yamls/", "dest": "./k8s/", - "source": "private" + "source": "private", + "strip_path": "" }, { "repo": "github.com/replicatedhq/ship", "ref": "master", "path": "hack/docs/", - "dest": "./docs/", + "dest": "./docs{{repl Add 1 1}}/", "source": "public", - "mode": 644 + "mode": 644, + "strip_path": "{{repl ParseBool \"true\"}}" } ], "type": "object", @@ -348,6 +350,10 @@ "description": "One of `public` or `private`, if `private`, access to the repo can be validated on release creation", "type": "string" }, + "strip_path": { + "description": "If true, the github directory will not be included in the filepath of the generated files. For instance, when outputting all files within 'source/' in the repository to the 'dest/' directory, the file 'source/a/file.txt' would be placed at 'dest/source/a/file.txt' when this is false and 'dest/a/file.txt' when this is true.", + "type": "string" + }, "when": { "description": "This asset will be included when 'when' is omitted or true", "type": "string" diff --git a/integration/init_app/github-template-func/expected/.ship/release.yml b/integration/init_app/github-template-func/expected/.ship/release.yml index 3f7be4292..8228a47ff 100644 --- a/integration/init_app/github-template-func/expected/.ship/release.yml +++ b/integration/init_app/github-template-func/expected/.ship/release.yml @@ -20,6 +20,13 @@ assets: dest: ./github-noslash source: public ref: ad1e78d13c33fae7a7ce22ed19920945ceea23e9 + - github: + repo: replicatedhq/test-charts + path: template-functions + dest: ./github-stripped + strip_path: "{{repl ParseBool \"true\"}}" + source: public + ref: ad1e78d13c33fae7a7ce22ed19920945ceea23e9 config: v1: - name: option_group diff --git a/integration/init_app/github-template-func/expected/installer/github-stripped/config.md b/integration/init_app/github-template-func/expected/installer/github-stripped/config.md new file mode 100644 index 000000000..2065d896a --- /dev/null +++ b/integration/init_app/github-template-func/expected/installer/github-stripped/config.md @@ -0,0 +1,3 @@ +#This file tests a part of the Config suite of template functions in Ship + +Config option: abc123 diff --git a/integration/init_app/github-template-func/expected/installer/github-stripped/installation.md b/integration/init_app/github-template-func/expected/installer/github-stripped/installation.md new file mode 100644 index 000000000..bde862ed9 --- /dev/null +++ b/integration/init_app/github-template-func/expected/installer/github-stripped/installation.md @@ -0,0 +1,4 @@ +#This file tests a part of the Integration suite of template functions in Ship + +Release semver: 1.0.1-SNAPSHOT + diff --git a/integration/init_app/github-template-func/expected/installer/github-stripped/static.md b/integration/init_app/github-template-func/expected/installer/github-stripped/static.md new file mode 100644 index 000000000..e5d6ea7a6 --- /dev/null +++ b/integration/init_app/github-template-func/expected/installer/github-stripped/static.md @@ -0,0 +1,4 @@ +#This file tests a part of the Static suite of template functions in Ship + +TwoPlusTwo: 4 +UPPERCASE: UPPERCASE diff --git a/integration/init_app/github-template-func/input/.ship/release.yml b/integration/init_app/github-template-func/input/.ship/release.yml index 3f7be4292..8228a47ff 100644 --- a/integration/init_app/github-template-func/input/.ship/release.yml +++ b/integration/init_app/github-template-func/input/.ship/release.yml @@ -20,6 +20,13 @@ assets: dest: ./github-noslash source: public ref: ad1e78d13c33fae7a7ce22ed19920945ceea23e9 + - github: + repo: replicatedhq/test-charts + path: template-functions + dest: ./github-stripped + strip_path: "{{repl ParseBool \"true\"}}" + source: public + ref: ad1e78d13c33fae7a7ce22ed19920945ceea23e9 config: v1: - name: option_group diff --git a/pkg/api/asset.go b/pkg/api/asset.go index 1b71cf57f..9964cd9f5 100644 --- a/pkg/api/asset.go +++ b/pkg/api/asset.go @@ -62,6 +62,7 @@ type GitHubAsset struct { Ref string `json:"ref" yaml:"ref" hcl:"ref"` Path string `json:"path" yaml:"path" hcl:"path"` Source string `json:"source" yaml:"source" hcl:"source"` + StripPath string `json:"strip_path" yaml:"strip_path" hcl:"strip_path"` } // WebAsset is an asset whose contents are specified by the HTML at the corresponding URL diff --git a/pkg/lifecycle/render/amazoneks/render_test.go b/pkg/lifecycle/render/amazoneks/render_test.go index 5df9f084d..f836d8ad8 100644 --- a/pkg/lifecycle/render/amazoneks/render_test.go +++ b/pkg/lifecycle/render/amazoneks/render_test.go @@ -1003,7 +1003,7 @@ func TestBuildAsset(t *testing.T) { req.Error(err) } - req.Equal(got, tt.want) + req.Equal(tt.want, got) }) } } diff --git a/pkg/lifecycle/render/github/render.go b/pkg/lifecycle/render/github/render.go index 5f7a01369..8c5aadf7d 100644 --- a/pkg/lifecycle/render/github/render.go +++ b/pkg/lifecycle/render/github/render.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "os" "path/filepath" + "strings" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -100,7 +101,10 @@ func (r *LocalRenderer) Execute( return errors.Wrapf(err, "building %s", file.Path) } - filePath := filepath.Join(asset.Dest, file.Path) + filePath, err := getDestPath(file.Path, asset, builder) + if err != nil { + return errors.Wrapf(err, "determining destination for %s", file.Path) + } basePath := filepath.Dir(filePath) debug.Log("event", "mkdirall.attempt", "root", rootFs.RootPath, "dest", filePath, "basePath", basePath) @@ -123,3 +127,29 @@ func (r *LocalRenderer) Execute( return nil } } + +func getDestPath(githubPath string, asset api.GitHubAsset, builder *templates.Builder) (string, error) { + stripPath, err := builder.Bool(asset.StripPath, false) + if err != nil { + return "", errors.Wrapf(err, "parse boolean from %q", asset.StripPath) + } + + destDir, err := builder.String(asset.Dest) + if err != nil { + return "", errors.Wrapf(err, "get destination directory from %q", asset.Dest) + } + + if stripPath { + // remove asset.Path's directory from the beginning of githubPath + sourcePathDir := filepath.ToSlash(filepath.Dir(asset.Path)) + "/" + githubPath = strings.TrimPrefix(githubPath, sourcePathDir) + + // handle cases where the source path was a dir but a trailing slash was not included + if !strings.HasSuffix(asset.Path, "/") { + sourcePathBase := filepath.Base(asset.Path) + "/" + githubPath = strings.TrimPrefix(githubPath, sourcePathBase) + } + } + + return filepath.Join(destDir, githubPath), nil +} diff --git a/pkg/lifecycle/render/github/render_test.go b/pkg/lifecycle/render/github/render_test.go new file mode 100644 index 000000000..aa894272b --- /dev/null +++ b/pkg/lifecycle/render/github/render_test.go @@ -0,0 +1,228 @@ +package github + +import ( + "path/filepath" + "testing" + + "github.com/replicatedhq/libyaml" + "github.com/replicatedhq/ship/pkg/api" + "github.com/replicatedhq/ship/pkg/templates" + "github.com/replicatedhq/ship/pkg/testing/logger" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func Test_getDestPath(t *testing.T) { + type args struct { + githubPath string + asset api.GitHubAsset + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "basic file", + args: args{ + githubPath: "README.md", + asset: api.GitHubAsset{ + Path: "README.md", + StripPath: "", + AssetShared: api.AssetShared{ + Dest: "./", + }, + }, + }, + want: "README.md", + wantErr: false, + }, + { + name: "file in subdir", + args: args{ + githubPath: "subdir/README.md", + asset: api.GitHubAsset{ + Path: "subdir/", + StripPath: "", + AssetShared: api.AssetShared{ + Dest: "./", + }, + }, + }, + want: "subdir/README.md", + wantErr: false, + }, + { + name: "file in subdir with dest dir", + args: args{ + githubPath: "subdir/README.md", + asset: api.GitHubAsset{ + Path: "subdir/", + StripPath: "", + AssetShared: api.AssetShared{ + Dest: "./dest", + }, + }, + }, + want: "dest/subdir/README.md", + wantErr: false, + }, + { + name: "file in stripped subdir with dest dir", + args: args{ + githubPath: "subdir/README.md", + asset: api.GitHubAsset{ + Path: "subdir/", + StripPath: "true", + AssetShared: api.AssetShared{ + Dest: "./dest", + }, + }, + }, + want: "dest/README.md", + wantErr: false, + }, + { + name: "literal file in stripped subdir with dest dir", + args: args{ + githubPath: "dir/subdir/README.md", + asset: api.GitHubAsset{ + Path: "dir/subdir/README.md", + StripPath: "true", + AssetShared: api.AssetShared{ + Dest: "dest", + }, + }, + }, + want: "dest/README.md", + wantErr: false, + }, + { + name: "file in stripped subdir that lacks a trailing slash with dest dir", + args: args{ + githubPath: "dir/subdir/README.md", + asset: api.GitHubAsset{ + Path: "dir/subdir", + StripPath: "true", + AssetShared: api.AssetShared{ + Dest: "dest", + }, + }, + }, + want: "dest/README.md", + wantErr: false, + }, + { + name: "templated dest dir", + args: args{ + githubPath: "dir/subdir/README.md", + asset: api.GitHubAsset{ + Path: "dir/subdir", + StripPath: "false", + AssetShared: api.AssetShared{ + Dest: "dest{{repl Add 1 1}}", + }, + }, + }, + want: "dest2/dir/subdir/README.md", + wantErr: false, + }, + { + name: "templated stripPath (eval to true)", + args: args{ + githubPath: "dir/subdir/README.md", + asset: api.GitHubAsset{ + Path: "dir/subdir", + StripPath: `{{repl ParseBool "true"}}`, + AssetShared: api.AssetShared{ + Dest: "dest", + }, + }, + }, + want: "dest/README.md", + wantErr: false, + }, + { + name: "templated stripPath (eval to false)", + args: args{ + githubPath: "dir/subdir/README.md", + asset: api.GitHubAsset{ + Path: "dir/subdir", + StripPath: `{{repl ParseBool "false"}}`, + AssetShared: api.AssetShared{ + Dest: "dest", + }, + }, + }, + want: "dest/dir/subdir/README.md", + wantErr: false, + }, + { + name: "strip path of root dir file", + args: args{ + githubPath: "README.md", + asset: api.GitHubAsset{ + Path: "", + StripPath: "true", + AssetShared: api.AssetShared{ + Dest: "dest", + }, + }, + }, + want: "dest/README.md", + wantErr: false, + }, + { + name: "not a valid template function (dest)", + args: args{ + githubPath: "README.md", + asset: api.GitHubAsset{ + Path: "", + StripPath: "true", + AssetShared: api.AssetShared{ + Dest: "{{repl NotATemplateFunction }}", + }, + }, + }, + want: "", + wantErr: true, + }, + { + name: "not a valid template function (stripPath)", + args: args{ + githubPath: "README.md", + asset: api.GitHubAsset{ + Path: "", + StripPath: "{{repl NotATemplateFunction }}", + AssetShared: api.AssetShared{ + Dest: "dest", + }, + }, + }, + want: "", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := require.New(t) + + testLogger := &logger.TestLogger{T: t} + v := viper.New() + bb := templates.NewBuilderBuilder(testLogger, v) + builder, err := bb.FullBuilder(api.ReleaseMetadata{}, []libyaml.ConfigGroup{}, map[string]interface{}{}) + req.NoError(err) + + got, err := getDestPath(tt.args.githubPath, tt.args.asset, builder) + if !tt.wantErr { + req.NoErrorf(err, "getDestPath(%s, %+v, builder) error = %v", tt.args.githubPath, tt.args.asset, err) + } else { + req.Error(err) + } + + // convert the returned file to forwardslash format before testing - otherwise this test fails when the separator isn't '/' + req.Equal(tt.want, filepath.ToSlash(got)) + }) + } +} diff --git a/pkg/lifecycle/render/googlegke/render_test.go b/pkg/lifecycle/render/googlegke/render_test.go index 697cd43ba..c1e7e7970 100644 --- a/pkg/lifecycle/render/googlegke/render_test.go +++ b/pkg/lifecycle/render/googlegke/render_test.go @@ -246,7 +246,7 @@ func TestBuildAsset(t *testing.T) { req.Error(err) } - req.Equal(got, tt.want) + req.Equal(tt.want, got) }) } }