From 19251cefc1743c7af640c18af6a36d41eea73aa5 Mon Sep 17 00:00:00 2001 From: Nick Santos Date: Tue, 16 Mar 2021 12:28:08 -0400 Subject: [PATCH] Add support for clusters and registries on remote docker (#99) Fixes https://github.com/tilt-dev/ctlptl/issues/92 This PR assumes that "non-empty DOCKER_HOST" means "ctlptl is talking to a remote docker and needs additional connections between localhost and the remote machine." I'm not entirely sure this is a safe assumption, but let's try it and see how it goes. --- .circleci/Dockerfile | 34 +++++++++ .circleci/config.yml | 16 +++- Makefile | 8 +- go.mod | 2 +- go.sum | 22 +++--- internal/socat/socat.go | 49 +++++++++++- pkg/cluster/cluster.go | 86 +++++++++++++++++++++- pkg/cluster/cluster_test.go | 4 + pkg/cluster/docker.go | 1 + pkg/registry/registry.go | 24 ++++++ pkg/registry/registry_test.go | 4 + test/kind-cluster-network/cluster.yaml | 2 +- test/kind-cluster-network/e2e.sh | 4 +- test/minikube-cluster-network/cluster.yaml | 2 +- test/minikube-cluster-network/e2e.sh | 4 +- 15 files changed, 238 insertions(+), 24 deletions(-) create mode 100644 .circleci/Dockerfile diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile new file mode 100644 index 0000000..4b48ca0 --- /dev/null +++ b/.circleci/Dockerfile @@ -0,0 +1,34 @@ +FROM golang:1.14-buster + +RUN apt update +RUN apt install -y curl ca-certificates liblz4-tool rsync socat + +# Install docker +# Adapted from https://github.com/circleci/circleci-images/blob/staging/shared/images/Dockerfile-basic.template +# Check https://download.docker.com/linux/static/stable/x86_64/ for latest versions +ENV DOCKER_VERSION=19.03.5 +RUN set -exu \ + && DOCKER_URL="https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" \ + && echo Docker URL: $DOCKER_URL \ + && curl --silent --show-error --location --fail --retry 3 --output /tmp/docker.tgz "${DOCKER_URL}" \ + && ls -lha /tmp/docker.tgz \ + && tar -xz -C /tmp -f /tmp/docker.tgz \ + && mv /tmp/docker/* /usr/bin \ + && rm -rf /tmp/docker /tmp/docker.tgz \ + && which docker \ + && (docker version || true) + +# Install kubectl client +RUN apt install -y apt-transport-https gnupg \ + && curl -fsS https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ + && touch /etc/apt/sources.list.d/kubernetes.list \ + && echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list \ + && apt update && apt install -y kubectl + +# install Kind +ENV KIND_VERSION=v0.10.0 +RUN set -exu \ + && curl -fLo ./kind-linux-amd64 "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" \ + && chmod +x ./kind-linux-amd64 \ + && mv ./kind-linux-amd64 /usr/local/bin/kind + diff --git a/.circleci/config.yml b/.circleci/config.yml index 596a49e..d6512bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,6 +15,17 @@ jobs: - slack/notify-on-failure: only_for_branches: master + e2e-remote-docker: + docker: + - image: gcr.io/windmill-public-containers/ctlptl-e2e-ci + working_directory: /go/src/github.com/tilt-dev/ctlptl + steps: + - checkout + - setup_remote_docker: + version: 19.03.12 + - run: make install + - run: test/kind-cluster-network/e2e.sh + e2e: machine: image: ubuntu-1604:201903-01 @@ -35,7 +46,7 @@ jobs: sudo mv ./minikube-linux-amd64 /usr/local/bin/minikube - run: | set -ex - export KIND_VERSION=v0.9.0 + export KIND_VERSION=v0.10.0 curl -fLo ./kind-linux-amd64 "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" chmod +x ./kind-linux-amd64 sudo mv ./kind-linux-amd64 /usr/local/bin/kind @@ -78,6 +89,9 @@ workflows: - e2e: requires: - build + - e2e-remote-docker: + requires: + - build - release-dry-run: requires: - build diff --git a/Makefile b/Makefile index 9765a79..df0874b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ GOPATH = $(shell go env GOPATH) -.PHONY: generate test vendor +.PHONY: generate test vendor publish-ci-image install: go install ./cmd/ctlptl @@ -22,4 +22,8 @@ golangci-lint: $(GOLANGCILINT) $(GOPATH)/bin/golangci-lint run --verbose $(GOLANGCILINT): - (cd /; GO111MODULE=on GOPROXY="direct" GOSUMDB=off go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.30.0) \ No newline at end of file + (cd /; GO111MODULE=on GOPROXY="direct" GOSUMDB=off go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.30.0) + +publish-ci-image: + docker build -t gcr.io/windmill-public-containers/ctlptl-e2e-ci -f .circleci/Dockerfile . + docker push gcr.io/windmill-public-containers/ctlptl-e2e-ci diff --git a/go.mod b/go.mod index e100c82..0c8089a 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/opencontainers/image-spec v1.0.1 // indirect github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pkg/errors v0.9.1 + github.com/shirou/gopsutil/v3 v3.21.2 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.6.1 @@ -42,7 +43,6 @@ require ( k8s.io/apimachinery v0.19.2 k8s.io/cli-runtime v0.19.2 k8s.io/client-go v0.19.2 - k8s.io/code-generator v0.20.2 // indirect k8s.io/klog/v2 v2.4.0 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd // indirect k8s.io/utils v0.0.0-20201015054608-420da100c033 // indirect diff --git a/go.sum b/go.sum index b2f34a9..5dc9f95 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,8 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alessio/shellescape v1.2.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= @@ -142,6 +144,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= @@ -363,6 +367,8 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shirou/gopsutil/v3 v3.21.2 h1:fIOk3hyqV1oGKogfGNjUZa0lUbtlkx3+ZT0IoJth2uM= +github.com/shirou/gopsutil/v3 v3.21.2/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= @@ -402,13 +408,16 @@ github.com/tilt-dev/localregistry-go v0.0.0-20201021185044-ffc4c827f097 h1:CiCHb github.com/tilt-dev/localregistry-go v0.0.0-20201021185044-ffc4c827f097/go.mod h1:SX7bKYACP+RsddxA+NBkfVzr5DOr5ranTirgT7xlxjA= github.com/tilt-dev/wmclient v0.0.0-20201109174454-1839d0355fbc h1:wGkAoZhrvnmq93B4W2v+agiPl7xzqUaxXkxmKrwJ6bc= github.com/tilt-dev/wmclient v0.0.0-20201109174454-1839d0355fbc/go.mod h1:n01fG3LbImzxBP3GGCTHkgXuPeJusWg6xv0QYGm9HtE= +github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M= +github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= +github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc= +github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -461,8 +470,6 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -549,6 +556,8 @@ golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201106081118-db71ae66460a h1:ALUFBKlIyeY7y5ZgPJmblk/vKz+zBQSnNiPkt41sgeg= golang.org/x/sys v0.0.0-20201106081118-db71ae66460a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210217105451-b926d437f341 h1:2/QtM1mL37YmcsT8HaDNHDgTqqFVw+zr8UzMiBVLzYU= +golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -602,9 +611,6 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 h1:HHeAlu5H9b71C+Fx0K+1dGgVFN1DM1/wz4aoGOA5qS8= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -718,12 +724,8 @@ k8s.io/cli-runtime v0.19.2/go.mod h1:CMynmJM4Yf02TlkbhKxoSzi4Zf518PukJ5xep/NaNeY k8s.io/client-go v0.18.4/go.mod h1:f5sXwL4yAZRkAtzOxRWUhA/N8XzGCb+nPZI8PfobZ9g= k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= -k8s.io/code-generator v0.20.2 h1:SQaysped4EtUDk3u1zphnUJiOAwFdhHx9xS3WKAE0x8= -k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded h1:JApXBKYyB7l9xx+DK7/+mFjC7A9Bt5A93FPvFD0HIFE= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= diff --git a/internal/socat/socat.go b/internal/socat/socat.go index 1a3f3fa..3612ea7 100644 --- a/internal/socat/socat.go +++ b/internal/socat/socat.go @@ -6,10 +6,12 @@ import ( "fmt" "net" "os/exec" + "strings" "time" "github.com/docker/docker/api/types" "github.com/docker/docker/client" + "github.com/shirou/gopsutil/v3/process" ) const serviceName = "ctlptl-portforward-service" @@ -70,12 +72,53 @@ func (c *Controller) StartRemotePortforwarder(ctx context.Context) error { return cmd.Run() } +// Returns the socat process listening on a port, plus its commandline. +func (c *Controller) socatProcessOnPort(port int) (*process.Process, string, error) { + processes, err := process.Processes() + if err != nil { + return nil, "", err + } + for _, p := range processes { + cmdline, err := p.Cmdline() + if err != nil { + continue + } + if strings.HasPrefix(cmdline, fmt.Sprintf("socat TCP-LISTEN:%d,", port)) { + return p, cmdline, nil + } + } + return nil, "", nil +} + // Create a port-forwarding server on the local machine, forwarding connections // to the same port on the remote Docker server. func (c *Controller) StartLocalPortforwarder(ctx context.Context, port int) error { - cmd := exec.Command("socat", fmt.Sprintf("TCP-LISTEN:%d,reuseaddr,fork", port), - fmt.Sprintf("EXEC:'docker exec -i %s socat STDIO TCP:localhost:%d'", serviceName, port)) - err := cmd.Start() + args := []string{ + fmt.Sprintf("TCP-LISTEN:%d,reuseaddr,fork", port), + fmt.Sprintf("EXEC:'docker exec -i %s socat STDIO TCP:localhost:%d'", serviceName, port), + } + + existing, cmdline, err := c.socatProcessOnPort(port) + if err != nil { + return fmt.Errorf("start portforwarder: %v", err) + } + + if existing != nil { + expectedCmdline := strings.Join(append([]string{"socat"}, args...), " ") + if expectedCmdline == cmdline { + // Already running. + return nil + } + + // Kill and restart. + err := existing.KillWithContext(ctx) + if err != nil { + return fmt.Errorf("start portforwarder: %v", err) + } + } + + cmd := exec.Command("socat", args...) + err = cmd.Start() if err != nil { return fmt.Errorf("creating local portforwarder: %v", err) } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index dd4f66c..965e6de 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -3,7 +3,9 @@ package cluster import ( "context" "fmt" + "os" "sort" + "strconv" "strings" "sync" "time" @@ -12,6 +14,7 @@ import ( "github.com/docker/docker/client" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" + "github.com/tilt-dev/ctlptl/internal/socat" "github.com/tilt-dev/ctlptl/pkg/api" "github.com/tilt-dev/ctlptl/pkg/registry" "github.com/tilt-dev/localregistry-go" @@ -54,6 +57,10 @@ type registryController interface { type clientLoader func(*rest.Config) (kubernetes.Interface, error) +type socatController interface { + ConnectRemoteDockerPort(ctx context.Context, port int) error +} + type Controller struct { iostreams genericclioptions.IOStreams config clientcmdapi.Config @@ -64,8 +71,16 @@ type Controller struct { configLoader configLoader configWriter configWriter registryCtl registryController - mu sync.Mutex clientLoader clientLoader + socat socatController + + // TODO(nick): I deeply regret making this struct use goroutines. It makes + // everything so much more complex. + // + // We should try to split this up into two structs - the part that needs + // concurrency for performance, and the part that is fine being + // single-threaded. + mu sync.Mutex } func DefaultController(iostreams genericclioptions.IOStreams) (*Controller, error) { @@ -100,6 +115,22 @@ func DefaultController(iostreams genericclioptions.IOStreams) (*Controller, erro }, nil } +func (c *Controller) getSocatController(ctx context.Context) (socatController, error) { + dcli, err := c.getDockerClient(ctx) + if err != nil { + return nil, err + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.socat == nil { + c.socat = socat.NewController(dcli) + } + + return c.socat, nil +} + func (c *Controller) getDockerClient(ctx context.Context) (dockerClient, error) { c.mu.Lock() defer c.mu.Unlock() @@ -210,6 +241,30 @@ func (c *Controller) configCopy() *clientcmdapi.Config { return c.config.DeepCopy() } +// Gets the port of the current API server. +func (c *Controller) currentAPIServerPort() int { + c.mu.Lock() + defer c.mu.Unlock() + + current := c.config.CurrentContext + context, ok := c.config.Contexts[current] + if !ok { + return 0 + } + + cluster, ok := c.config.Clusters[context.Cluster] + if !ok { + return 0 + } + + parts := strings.Split(cluster.Server, ":") + port, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + return 0 + } + return port +} + func (c *Controller) configCurrent() string { c.mu.Lock() defer c.mu.Unlock() @@ -607,6 +662,13 @@ func (c *Controller) Apply(ctx context.Context, desired *api.Cluster) (*api.Clus } if needsCreate { + // If the cluster apiserver is in a remote docker cluster, + // set up a portforwarder. + err := c.maybeCreateForwarderForCurrentCluster(ctx) + if err != nil { + return nil, err + } + if desired.Product == string(ProductMinikube) { err = c.waitForMinikubeInit(ctx, desired) if err != nil { @@ -828,3 +890,25 @@ func (c *Controller) List(ctx context.Context, options ListOptions) (*api.Cluste Items: result, }, nil } + +// If the current cluster is on a remote docker instance, +// we need a port-forwarder to connect it. +func (c *Controller) maybeCreateForwarderForCurrentCluster(ctx context.Context) error { + dockerHost := os.Getenv("DOCKER_HOST") + if dockerHost == "" { + return nil + } + + port := c.currentAPIServerPort() + if port == 0 { + return nil + } + + socat, err := c.getSocatController(ctx) + if err != nil { + return err + } + + _, _ = fmt.Fprintf(c.iostreams.ErrOut, " 🎮 Env DOCKER_HOST set. Assuming remote Docker and forwarding apiserver to localhost:%d\n", port) + return socat.ConnectRemoteDockerPort(ctx, port) +} diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 6a8eb33..5e5c8b4 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -408,6 +408,10 @@ func (c *fakeDockerClient) ContainerInspect(ctx context.Context, id string) (typ return types.ContainerJSON{}, nil } +func (d *fakeDockerClient) ContainerRemove(ctx context.Context, id string, options types.ContainerRemoveOptions) error { + return nil +} + type fakeD4MClient struct { lastSettings map[string]interface{} docker *fakeDockerClient diff --git a/pkg/cluster/docker.go b/pkg/cluster/docker.go index c6fb79b..96abfcc 100644 --- a/pkg/cluster/docker.go +++ b/pkg/cluster/docker.go @@ -10,4 +10,5 @@ type dockerClient interface { ServerVersion(ctx context.Context) (types.Version, error) Info(ctx context.Context) (types.Info, error) ContainerInspect(ctx context.Context, containerID string) (types.ContainerJSON, error) + ContainerRemove(ctx context.Context, id string, options types.ContainerRemoveOptions) error } diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index 331793c..d1c06ed 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -3,6 +3,7 @@ package registry import ( "context" "fmt" + "os" "sort" "strings" "time" @@ -12,6 +13,7 @@ import ( "github.com/docker/docker/client" "github.com/phayes/freeport" "github.com/tilt-dev/ctlptl/internal/exec" + "github.com/tilt-dev/ctlptl/internal/socat" "github.com/tilt-dev/ctlptl/pkg/api" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -44,14 +46,20 @@ func FillDefaults(registry *api.Registry) { } type ContainerClient interface { + ContainerInspect(ctx context.Context, containerID string) (types.ContainerJSON, error) ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error) ContainerRemove(ctx context.Context, id string, options types.ContainerRemoveOptions) error } +type socatController interface { + ConnectRemoteDockerPort(ctx context.Context, port int) error +} + type Controller struct { iostreams genericclioptions.IOStreams dockerClient ContainerClient runner exec.CmdRunner + socat socatController } func NewController(iostreams genericclioptions.IOStreams, dockerClient ContainerClient) (*Controller, error) { @@ -59,6 +67,7 @@ func NewController(iostreams genericclioptions.IOStreams, dockerClient Container iostreams: iostreams, dockerClient: dockerClient, runner: exec.RealCmdRunner{}, + socat: socat.NewController(dockerClient), }, nil } @@ -220,9 +229,24 @@ func (c *Controller) Apply(ctx context.Context, desired *api.Registry) (*api.Reg return nil, err } + err = c.maybeCreateForwarder(ctx, hostPort) + if err != nil { + return nil, err + } + return c.Get(ctx, desired.Name) } +func (c *Controller) maybeCreateForwarder(ctx context.Context, port int) error { + dockerHost := os.Getenv("DOCKER_HOST") + if dockerHost == "" { + return nil + } + + _, _ = fmt.Fprintf(c.iostreams.ErrOut, " 🎮 Env DOCKER_HOST set. Assuming remote Docker and forwarding registry to localhost:%d\n", port) + return c.socat.ConnectRemoteDockerPort(ctx, port) +} + // Delete the given registry. func (c *Controller) Delete(ctx context.Context, name string) error { registry, err := c.Get(ctx, name) diff --git a/pkg/registry/registry_test.go b/pkg/registry/registry_test.go index 74c82d1..1d919a5 100644 --- a/pkg/registry/registry_test.go +++ b/pkg/registry/registry_test.go @@ -126,6 +126,10 @@ type fakeDocker struct { lastRemovedContainer string } +func (d *fakeDocker) ContainerInspect(ctx context.Context, containerID string) (types.ContainerJSON, error) { + return types.ContainerJSON{}, nil +} + func (d *fakeDocker) ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error) { return d.containers, nil } diff --git a/test/kind-cluster-network/cluster.yaml b/test/kind-cluster-network/cluster.yaml index e04d491..215a9a0 100644 --- a/test/kind-cluster-network/cluster.yaml +++ b/test/kind-cluster-network/cluster.yaml @@ -3,4 +3,4 @@ kind: Cluster name: kind-ctlptl-test-cluster product: kind registry: ctlptl-test-registry -kubernetesVersion: v1.18.8 +kubernetesVersion: v1.18.15 diff --git a/test/kind-cluster-network/e2e.sh b/test/kind-cluster-network/e2e.sh index 115ba72..4c66fd8 100755 --- a/test/kind-cluster-network/e2e.sh +++ b/test/kind-cluster-network/e2e.sh @@ -25,8 +25,8 @@ k8sVersion=$(ctlptl get cluster kind-ctlptl-test-cluster -o go-template --templa ctlptl delete -f cluster.yaml -if [[ "$k8sVersion" != "v1.18.8" ]]; then - echo "Expected kubernetes version v1.18.8 but got $k8sVersion" +if [[ "$k8sVersion" != "v1.18.15" ]]; then + echo "Expected kubernetes version v1.18.15 but got $k8sVersion" exit 1 fi diff --git a/test/minikube-cluster-network/cluster.yaml b/test/minikube-cluster-network/cluster.yaml index e7f73b0..d3b13aa 100644 --- a/test/minikube-cluster-network/cluster.yaml +++ b/test/minikube-cluster-network/cluster.yaml @@ -3,4 +3,4 @@ kind: Cluster name: minikube-ctlptl-test-cluster product: minikube registry: ctlptl-test-registry -kubernetesVersion: v1.18.8 +kubernetesVersion: v1.18.15 diff --git a/test/minikube-cluster-network/e2e.sh b/test/minikube-cluster-network/e2e.sh index e4324eb..8704dab 100755 --- a/test/minikube-cluster-network/e2e.sh +++ b/test/minikube-cluster-network/e2e.sh @@ -25,8 +25,8 @@ k8sVersion=$(ctlptl get cluster minikube-ctlptl-test-cluster -o go-template --te ctlptl delete -f cluster.yaml -if [[ "$k8sVersion" != "v1.18.8" ]]; then - echo "Expected kubernetes version v1.18.8 but got $k8sVersion" +if [[ "$k8sVersion" != "v1.18.15" ]]; then + echo "Expected kubernetes version v1.18.15 but got $k8sVersion" exit 1 fi