mechanism for setting github contents during local dev

What I Did
------------

Add a flag for overriding github repo contents when running `ship app`
with the `--runbook` flag for local dev.

We allow for the user to pass a flag `--set-github-contents`, and parse
a string of the form

    REPO:REPO_PATH:REF:LOCAL_PATH

e.g.

    replicatedhq/test-charts:/plain-k8s:master:$GOPATH/src/github.com/replicatedhq/test-charts

This will override an asset of the form

```yaml
assets:
  v1:
  - github:
      repo: replicatedhq/test-charts
      path: /plain-k8s
      ref: master
      # ...
```

with the contents of the local file system.

This allows for much faster iteration on ship-vendor apps that use the `github` asset type. Up to this point, such apps **had** to be built, pulled, and tested via console.replicated.com. Now you can "fake" the github assets by dropping in local paths to the checked-out repos.

I know `ship app` is going away eventually, but most of this can be reused to provide the same functionality for `ship init file::path/to/ship.yaml`.

How I Did it
------------

- Add a flag for `--set-github-contents`
- If `--runbook` is passed, loop over `set-github-contents` values, and override repository contents in the `api.Release#GithubContent` by walking the local filesystem.

Still TODO integration tests

How to verify it
------------

create at /tmp/ship/ship.yaml

```yaml
lifecycle:
  v1:
    - config: {}
    - render:
        root: ./
assets:
  v1:
  - github:
      repo: replicatedhq/test-charts
      path: /base
      ref: master
      dest: repo
```

create at /tmp/repo/base/deployment.yaml

```
fake
```

run ship with the flag

```
ship app --runbook /tmp/ship/ship.yaml --set-github-contents replicatedhq/test-charts:/base:master:/tmp/repo
```

Verify that one of the output files is

```
./base/deployment.yaml
```

with contents

```
fake
```

Description for the Changelog
------------

Add `--set-github-contents` flag for overriding [github assets](https://ship.replicated.com/reference/assets/github/) during local development of a `ship.yaml`.

integration/base/github/expected/.ship/release.yml

@@ -0,0 +1,10 @@
+assets:
+  v1:
+  - github:
+      repo: replicatedhq/test-charts
+      ref: master
+      path: /release.yml
+      dest: ./
+lifecycle:
+  v1:
+  - render: {}

integration/base/github/expected/.ship/state.json

@@ -0,0 +1,15 @@
+{
+  "v1": {
+    "config": {
+      "test_option": "abc123_test-option-value"
+    },
+    "contentSHA": "b9b0e107bb87dd3843d1d1178a6e9109d42d5e86ac8885a172f23c291b62d3b3",
+    "metadata": {
+      "applicationType": "replicated.app",
+      "customerID": "-Am-_6i5pw0u4AbspOwKN4lZUCn49u_G",
+      "installationID": "RULNveQLrLj4GDum1yQhvKAcABh0GcLY",
+      "releaseNotes": "",
+      "version": "0.0.4"
+    }
+  }
+}

integration/base/github/expected/installer/release.yml

@@ -0,0 +1,10 @@
+assets:
+  v1:
+  - github:
+      repo: replicatedhq/test-charts
+      ref: master
+      path: /release.yml
+      dest: ./
+lifecycle:
+  v1:
+  - render: {}

integration/base/github/input/.ship/release.yml

@@ -0,0 +1,10 @@
+assets:
+  v1:
+  - github:
+      repo: replicatedhq/test-charts
+      ref: master
+      path: /release.yml
+      dest: ./
+lifecycle:
+  v1:
+  - render: {}

integration/base/github/input/.ship/state.json

@@ -0,0 +1,7 @@
+{
+  "v1": {
+    "config": {
+      "test_option": "abc123_test-option-value"
+    }
+  }
+}

integration/base/github/metadata.yaml

@@ -0,0 +1,5 @@
+customer_id: "-Am-_6i5pw0u4AbspOwKN4lZUCn49u_G"
+installation_id: "RULNveQLrLj4GDum1yQhvKAcABh0GcLY"
+release_version: "0.0.4"
+set_github_contents: "replicatedhq/test-charts:/release.yml:master:.ship"
+disable_online: true

integration/base/integration_test.go

@@ -19,13 +19,13 @@ import (
 )
 
 type TestMetadata struct {
-	CustomerID     string `yaml:"customer_id"`
-	InstallationID string `yaml:"installation_id"`
-	ReleaseVersion string `yaml:"release_version"`
-	SetChannelName string `yaml:"set_channel_name"`
-	Flavor         string `yaml:"flavor"`
-	DisableOnline  bool   `yaml:"disable_online"`
-	NoStateFile    bool   `yaml:"no_state_file"` // used to denote that there is no input state.json
+	CustomerID        string `yaml:"customer_id"`
+	InstallationID    string `yaml:"installation_id"`
+	ReleaseVersion    string `yaml:"release_version"`
+	SetChannelName    string `yaml:"set_channel_name"`
+	SetGitHubContents string `yaml:"set_github_contents"`
+	DisableOnline     bool   `yaml:"disable_online"`
+	NoStateFile       bool   `yaml:"no_state_file"` // used to denote that there is no input state.json
 	//debugging
 	SkipCleanup bool `yaml:"skip_cleanup"`
 }
@@ -93,12 +93,14 @@ var _ = Describe("ship app", func() {
 						fmt.Sprintf("--installation-id=%s", testMetadata.InstallationID),
 						fmt.Sprintf("--set-channel-name=%s", testMetadata.SetChannelName),
 						fmt.Sprintf("--release-semver=%s", testMetadata.ReleaseVersion),
+						fmt.Sprintf("--set-github-contents=%s", testMetadata.SetGitHubContents),
 						"--log-level=off",
 						"--terraform-yes",
 					}
 					if !testMetadata.NoStateFile {
 						args = append(args, fmt.Sprintf("--state-file=%s", path.Join(testInputPath, ".ship/state.json")))
 					}
+
 					cmd.SetArgs(args)
 					err := cmd.Execute()
 					Expect(err).NotTo(HaveOccurred())

pkg/cli/app.go

@@ -2,6 +2,7 @@ package cli
 
 import (
 	"context"
+	"fmt"
 	"strings"
 
 	"github.com/replicatedhq/ship/pkg/ship"
@@ -43,6 +44,7 @@ func App() *cobra.Command {
 	cmd.Flags().String("runbook", "", developerFlagUsage)
 	cmd.Flags().String("set-channel-name", "", developerFlagUsage)
 	cmd.Flags().String("set-channel-icon", "", developerFlagUsage)
+	cmd.Flags().String("set-github-contents", "", fmt.Sprintf("Specify a REPO:REPO_PATH:REF:LOCAL_PATH to override github checkouts to use a local path on the filesystem. %s. ", developerFlagUsage))
 
 	// Deprecated developer flags
 	cmd.Flags().String("studio-file", "", developerFlagUsage)

pkg/lifecycle/render/github/render.go

@@ -82,7 +82,7 @@ func (r *LocalRenderer) Execute(
 
 		if len(files) == 0 {
 			level.Info(r.Logger).Log("msg", "no files for asset", "repo", asset.Repo, "path", asset.Path)
-			return nil
+			return errors.New("github asset returned no files")
 		}
 
 		builder, err := r.BuilderBuilder.FullBuilder(meta, configGroups, templateContext)

pkg/specs/replicatedapp/local.go

@@ -0,0 +1,117 @@
+// local.go has methods for resolving a local ship.yaml file, and patching in api.Release info
+// that would usually be returned by pg.replicated.com
+package replicatedapp
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"github.com/pkg/errors"
+	"github.com/replicatedhq/ship/pkg/constants"
+)
+
+func (r *resolver) resolveRunbookRelease() (*ShipRelease, error) {
+	debug := level.Debug(log.With(r.Logger, "method", "resolveRunbookRelease"))
+	debug.Log("phase", "load-specs", "from", "runbook", "file", r.Runbook)
+
+	specYAML, err := r.FS.ReadFile(r.Runbook)
+	if err != nil {
+		return nil, errors.Wrapf(err, "read specs from %s", r.Runbook)
+	}
+	debug.Log("phase", "load-specs", "from", "runbook", "file", r.Runbook, "spec", specYAML)
+
+	if err := r.persistSpec(specYAML); err != nil {
+		return nil, errors.Wrapf(err, "serialize last-used YAML to disk")
+	}
+	debug.Log("phase", "write-yaml", "from", r.Runbook, "write-location", constants.ReleasePath)
+
+	fakeGithubContents, err := r.loadLocalGitHubContents()
+	if err != nil {
+		return nil, errors.Wrapf(err, "load fake github contents")
+	}
+
+	return &ShipRelease{
+		Spec:           string(specYAML),
+		ChannelName:    r.SetChannelName,
+		ChannelIcon:    r.SetChannelIcon,
+		Semver:         r.RunbookReleaseSemver,
+		GithubContents: fakeGithubContents,
+	}, nil
+}
+
+func (r *resolver) loadLocalGitHubContents() ([]GithubContent, error) {
+	debug := level.Debug(log.With(r.Logger, "method", "loadLocalGitHubContents"))
+	var fakeGithubContents []GithubContent
+	for _, content := range r.SetGitHubContents {
+		debug.Log("event", "githubcontents.set", "received", content)
+		split := strings.Split(content, ":")
+		if len(split) != 4 {
+			return nil, errors.Errorf("set-github-contents %q invalid, expected a REPO:REPO_PATH:REF:LOCAL_PATH", content)
+		}
+		repo := split[0]
+		repoPath := split[1]
+		ref := split[2]
+		localpath := split[3]
+
+		debug.Log("event", "githubcontents.loadFiles", "localPath", localpath)
+		files, err := r.loadLocalGithubFiles(localpath, repoPath)
+		if err != nil {
+			return nil, errors.Wrapf(err, "set github files")
+		}
+
+		fakeGithubContents = append(fakeGithubContents, GithubContent{
+			Repo:  repo,
+			Path:  repoPath,
+			Ref:   ref,
+			Files: files,
+		})
+		debug.Log("event", "githubcontents.set.finished", "received", content)
+	}
+	return fakeGithubContents, nil
+}
+
+func (r *resolver) loadLocalGithubFiles(localpath string, repoPath string) ([]GithubFile, error) {
+	debug := level.Debug(log.With(r.Logger, "method", "loadLocalGitHubFiles"))
+	var files []GithubFile
+	err := r.FS.Walk(localpath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return errors.Wrapf(err, "walk %s from %s", info.Name(), path)
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		walkRepoPath := strings.TrimPrefix(path, localpath)
+		if !strings.HasPrefix(walkRepoPath, repoPath) {
+			return nil
+		}
+
+		contents, err := r.FS.ReadFile(path)
+		if err != nil {
+			return errors.Wrapf(err, "read %s from %s", info.Name(), path)
+		}
+		debug.Log("event", "githubcontents.loadFile.complete", "path", path, "name", info.Name())
+
+		encodedData := &bytes.Buffer{}
+		encoder := base64.NewEncoder(base64.StdEncoding, encodedData)
+		defer encoder.Close()
+		encoder.Write(contents)
+		sha := fmt.Sprintf("%x", sha256.Sum256(contents))
+		files = append(files, GithubFile{
+			Name: info.Name(),
+			Path: walkRepoPath,
+			Sha:  sha,
+			Size: info.Size(),
+			Data: encodedData.String(),
+		})
+		return nil
+	})
+	return files, err
+}

pkg/specs/replicatedapp/local_test.go

@@ -0,0 +1,126 @@
+package replicatedapp
+
+import (
+	"testing"
+
+	"github.com/replicatedhq/ship/pkg/testing/logger"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadLocalGitHubContents(t *testing.T) {
+	tests := []struct {
+		name           string
+		githubContent  []string
+		fs             map[string]string
+		expectContents []GithubContent
+	}{
+		{
+			name:           "none to set",
+			githubContent:  nil,
+			expectContents: nil,
+		},
+		{
+			name: "set one file",
+			fs: map[string]string{
+				"/foo/bar.txt": "some-contents",
+			},
+			githubContent: []string{"replicatedhq/test-stuff:/bar.txt:master:/foo"},
+			expectContents: []GithubContent{
+				{
+					Repo: "replicatedhq/test-stuff",
+					Path: "/bar.txt",
+					Ref:  "master",
+					Files: []GithubFile{
+						{
+							Path: "/bar.txt",
+							Name: "bar.txt",
+							Size: 13,
+							Sha:  "6e32ea34db1b3755d7dec972eb72c705338f0dd8e0be881d966963438fb2e800",
+							Data: "c29tZS1jb250ZW50",
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "set many files from one repo",
+			fs: map[string]string{
+				"/foo/bar.txt":     "some-contents",
+				"/foo/baz.txt":     "some-contents",
+				"/foo/bar/baz.txt": "some-contents",
+				"/spam/eggs.txt":   "some-other-contents",
+			},
+			githubContent: []string{
+				"replicatedhq/test-stuff:/:master:/foo",
+				"replicatedhq/other-tests:/eggs.txt:release:/spam",
+			},
+			expectContents: []GithubContent{
+				{
+					Repo: "replicatedhq/test-stuff",
+					Path: "/",
+					Ref:  "master",
+					Files: []GithubFile{
+						{
+							Path: "/bar/baz.txt",
+							Name: "baz.txt",
+							Size: 13,
+							Sha:  "6e32ea34db1b3755d7dec972eb72c705338f0dd8e0be881d966963438fb2e800",
+							Data: "c29tZS1jb250ZW50",
+						},
+						{
+							Path: "/bar.txt",
+							Name: "bar.txt",
+							Size: 13,
+							Sha:  "6e32ea34db1b3755d7dec972eb72c705338f0dd8e0be881d966963438fb2e800",
+							Data: "c29tZS1jb250ZW50",
+						},
+						{
+							Path: "/baz.txt",
+							Name: "baz.txt",
+							Size: 13,
+							Sha:  "6e32ea34db1b3755d7dec972eb72c705338f0dd8e0be881d966963438fb2e800",
+							Data: "c29tZS1jb250ZW50",
+						},
+					},
+				},
+				{
+					Repo: "replicatedhq/other-tests",
+					Path: "/eggs.txt",
+					Ref:  "release",
+					Files: []GithubFile{
+						{
+							Path: "/eggs.txt",
+							Name: "eggs.txt",
+							Size: 19,
+							Sha:  "a2c0a8c54d71e14e9533749c32716c12f92f61294dfdce4f3b4c07303c0119b0",
+							Data: "c29tZS1vdGhlci1jb250ZW50",
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			req := require.New(t)
+			mockFs := afero.Afero{Fs: afero.NewMemMapFs()}
+
+			for key, value := range test.fs {
+				err := mockFs.WriteFile(key, []byte(value), 0777)
+				req.NoError(err)
+			}
+
+			resolver := &resolver{
+				Logger:            &logger.TestLogger{T: t},
+				FS:                mockFs,
+				SetGitHubContents: test.githubContent,
+			}
+
+			result, err := resolver.loadLocalGitHubContents()
+
+			req.NoError(err)
+			req.Equal(test.expectContents, result)
+		})
+	}
+}