From 63b52513bbf5a555da38f9a302566140cc8bce03 Mon Sep 17 00:00:00 2001 From: Robert Oh Date: Wed, 17 Oct 2018 15:59:18 -0700 Subject: [PATCH] Allow ship to be pointed to git file paths (#646) --- .../git-single-file/expected/.ship/state.json | 8 ++++ .../expected/base/kustomization.yaml | 4 ++ .../base/plain-k8s/frontend-deployment.yaml | 33 ++++++++++++++ .../expected/overlays/ship/kustomization.yaml | 4 ++ .../git-single-file/expected/rendered.yaml | 28 ++++++++++++ .../init/git-single-file/metadata.yaml | 3 ++ pkg/specs/githubclient/client.go | 22 ++++------ pkg/specs/githubclient/client_test.go | 44 ++++++++++++++++++- 8 files changed, 131 insertions(+), 15 deletions(-) create mode 100644 integration/init/git-single-file/expected/.ship/state.json create mode 100644 integration/init/git-single-file/expected/base/kustomization.yaml create mode 100644 integration/init/git-single-file/expected/base/plain-k8s/frontend-deployment.yaml create mode 100644 integration/init/git-single-file/expected/overlays/ship/kustomization.yaml create mode 100644 integration/init/git-single-file/expected/rendered.yaml create mode 100644 integration/init/git-single-file/metadata.yaml diff --git a/integration/init/git-single-file/expected/.ship/state.json b/integration/init/git-single-file/expected/.ship/state.json new file mode 100644 index 000000000..c88cf2412 --- /dev/null +++ b/integration/init/git-single-file/expected/.ship/state.json @@ -0,0 +1,8 @@ +{ + "v1": { + "config": {}, + "upstream": "github.com/replicatedhq/test-charts/blob/3427d6997bd150c60caa00ba0298fdfe17e3ed04/plain-k8s/frontend-deployment.yaml", + "metadata": null, + "contentSHA": "9fa025c3fdbcc02c7de8e9c74c8058d16b2aada04917895685504b387563afda" + } +} \ No newline at end of file diff --git a/integration/init/git-single-file/expected/base/kustomization.yaml b/integration/init/git-single-file/expected/base/kustomization.yaml new file mode 100644 index 000000000..a0b121dc8 --- /dev/null +++ b/integration/init/git-single-file/expected/base/kustomization.yaml @@ -0,0 +1,4 @@ +kind: "" +apiversion: "" +resources: +- plain-k8s/frontend-deployment.yaml diff --git a/integration/init/git-single-file/expected/base/plain-k8s/frontend-deployment.yaml b/integration/init/git-single-file/expected/base/plain-k8s/frontend-deployment.yaml new file mode 100644 index 000000000..2b08cc9de --- /dev/null +++ b/integration/init/git-single-file/expected/base/plain-k8s/frontend-deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 # for k8s versions before 1.9.0 use apps/v1beta2 and before 1.8.0 use extensions/v1beta1 +kind: Deployment +metadata: + name: frontend +spec: + selector: + matchLabels: + app: guestbook + tier: frontend + replicas: 3 + template: + metadata: + labels: + app: guestbook + tier: frontend + spec: + containers: + - name: php-redis + image: gcr.io/google-samples/gb-frontend:v4 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access environment variables to find service host + # info, comment out the 'value: dns' line above, and uncomment the + # line below: + # value: env + ports: + - containerPort: 80 diff --git a/integration/init/git-single-file/expected/overlays/ship/kustomization.yaml b/integration/init/git-single-file/expected/overlays/ship/kustomization.yaml new file mode 100644 index 000000000..c80bb2245 --- /dev/null +++ b/integration/init/git-single-file/expected/overlays/ship/kustomization.yaml @@ -0,0 +1,4 @@ +kind: "" +apiversion: "" +bases: +- ../../base diff --git a/integration/init/git-single-file/expected/rendered.yaml b/integration/init/git-single-file/expected/rendered.yaml new file mode 100644 index 000000000..dfeee25e6 --- /dev/null +++ b/integration/init/git-single-file/expected/rendered.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: frontend +spec: + replicas: 3 + selector: + matchLabels: + app: guestbook + tier: frontend + template: + metadata: + labels: + app: guestbook + tier: frontend + spec: + containers: + - env: + - name: GET_HOSTS_FROM + value: dns + image: gcr.io/google-samples/gb-frontend:v4 + name: php-redis + ports: + - containerPort: 80 + resources: + requests: + cpu: 100m + memory: 100Mi diff --git a/integration/init/git-single-file/metadata.yaml b/integration/init/git-single-file/metadata.yaml new file mode 100644 index 000000000..0b65317c5 --- /dev/null +++ b/integration/init/git-single-file/metadata.yaml @@ -0,0 +1,3 @@ +upstream: "github.com/replicatedhq/test-charts/blob/3427d6997bd150c60caa00ba0298fdfe17e3ed04/plain-k8s/frontend-deployment.yaml" +args: [] +skip_cleanup: false diff --git a/pkg/specs/githubclient/client.go b/pkg/specs/githubclient/client.go index 3b58123f9..b22d299df 100644 --- a/pkg/specs/githubclient/client.go +++ b/pkg/specs/githubclient/client.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "path" "path/filepath" "strings" @@ -125,17 +126,6 @@ func (g *GithubClient) downloadAndExtractFiles( } switch header.Typeflag { - case tar.TypeDir: - dirName := strings.Join(strings.Split(header.Name, "/")[1:], "/") - if !strings.HasPrefix(dirName, basePath) { - continue - } - basePathFound = true - - dirName = strings.TrimPrefix(dirName, basePath) - if err := g.fs.MkdirAll(filepath.Join(filePath, dirName), 0755); err != nil { - return errors.Wrapf(err, "extract tar gz, mkdir") - } case tar.TypeReg: // need this in a func because defer in a loop was leaking handles err := func() error { @@ -145,7 +135,13 @@ func (g *GithubClient) downloadAndExtractFiles( } basePathFound = true - fileName = strings.TrimPrefix(fileName, basePath) + if fileName != basePath { + fileName = strings.TrimPrefix(fileName, basePath) + } + dirPath, _ := path.Split(fileName) + if err := g.fs.MkdirAll(filepath.Join(filePath, dirPath), 0755); err != nil { + return errors.Wrapf(err, "extract tar gz, mkdir") + } outFile, err := g.fs.Create(filepath.Join(filePath, fileName)) if err != nil { return errors.Wrapf(err, "extract tar gz, create") @@ -177,7 +173,7 @@ func decodeGitHubURL(chartPath string) (owner string, repo string, branch string branch = "" path = "" if len(splitPath) > 3 { - if splitPath[3] == "tree" { + if splitPath[3] == "tree" || splitPath[3] == "blob" { branch = splitPath[4] path = strings.Join(splitPath[5:], "/") } else { diff --git a/pkg/specs/githubclient/client_test.go b/pkg/specs/githubclient/client_test.go index 72d11aca0..6a710892d 100644 --- a/pkg/specs/githubclient/client_test.go +++ b/pkg/specs/githubclient/client_test.go @@ -42,10 +42,13 @@ func TestGithubClient(t *testing.T) { var _ = Describe("GithubClient", func() { client, mux, serverURL, teardown = setupGitClient() - mux.HandleFunc("/repos/o/r/tarball", func(w http.ResponseWriter, r *http.Request) { + redirectArchive := func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, serverURL+"/archive.tar.gz", http.StatusFound) return - }) + } + mux.HandleFunc("/repos/o/r/tarball/", redirectArchive) + mux.HandleFunc("/repos/o/r/tarball", redirectArchive) + mux.HandleFunc("/archive.tar.gz", func(w http.ResponseWriter, r *http.Request) { archiveData := `H4sIAJKjXFsAA+3WXW6CQBQFYJbCBmrv/D831ce+uIOpDtGEKQaoibt3qERbEmiNI6TxfC8TIwkXTg65lfW73D3ZcrXZ7t1zcg9EZJRKv059OonL09lKmRDcMM6k0SkxSYolqbrLNB2fVW3LMIoPr2DounBZlg383z7H+fwnqp/5v25sWc8O1ucR7xHeh5ZyKH9xzl+TDPkroylJKeIMvR48//fw8PC4Ov1fLl7mb4uZX8e8xzX9V4Y1/RdMof9jyIpi6hFgQp3+1y78tLWrYm6CV+1/oum/JqGx/42hN/+12+XFwbuPsA7euA3++v1n/LL/sZA/JyM4vv9juMQ89SQwhd7+V67cb1fu5vInf9n/zLf+y6b/nDP0fwxtzFOPAQAAAAAAAAAAAACRHQEZehxJACgAAA==` dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(archiveData)) @@ -127,6 +130,43 @@ var _ = Describe("GithubClient", func() { Expect(err.Error()).To(Equal("http://gitlab.com/o/r is not a Github URL")) }) }) + + Context("With a url path to a single file at the base of the repo", func() { + It("should fetch and persist the file", func() { + validGithubURLSingle := "github.com/o/r/blob/master/Chart.yaml" + mockFs := afero.Afero{Fs: afero.NewMemMapFs()} + gitClient := &GithubClient{ + client: client, + fs: mockFs, + logger: log.NewNopLogger(), + } + + err := gitClient.GetFiles(context.Background(), validGithubURLSingle, constants.HelmChartPath) + + chart, err := gitClient.fs.ReadFile(path.Join(constants.HelmChartPath, "Chart.yaml")) + Expect(err).NotTo(HaveOccurred()) + + Expect(string(chart)).To(Equal("bar")) + }) + }) + + Context("With a url path to a single nested file", func() { + It("should fetch and persist the file", func() { + validGithubURLSingle := "github.com/o/r/blob/master/templates/service.yml" + mockFs := afero.Afero{Fs: afero.NewMemMapFs()} + gitClient := &GithubClient{ + client: client, + fs: mockFs, + logger: log.NewNopLogger(), + } + + err := gitClient.GetFiles(context.Background(), validGithubURLSingle, constants.HelmChartPath) + chart, err := gitClient.fs.ReadFile(path.Join(constants.HelmChartPath, "templates", "service.yml")) + Expect(err).NotTo(HaveOccurred()) + + Expect(string(chart)).To(Equal("service")) + }) + }) }) Describe("decodeGitHubURL", func() {