From 9a06da88bed5c8c9000bc42f78e779909415446b Mon Sep 17 00:00:00 2001 From: Nikolaus Demmel Date: Fri, 5 Aug 2016 22:08:54 +0200 Subject: [PATCH 1/6] Never use proxy when communicating with the vault. Without this, building Dockerfiles with ONVAULT fails in the presence of an http proxy. --- ONVAULT | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ONVAULT b/ONVAULT index 54edcf4..30b8e2a 100755 --- a/ONVAULT +++ b/ONVAULT @@ -22,6 +22,10 @@ log () { echo -e "${GREEN}[Dockito Vault]${NC} $@" } +# don't go through proxy for accessing vault +no_proxy_old="$no_proxy" +export no_proxy="$VAULT_HOST" + if ! curl -s "${VAULT_URI}/_ping"; then COUNTER=0 echo 'Waiting 10s for dockito/vault to be ready...' @@ -54,6 +58,9 @@ if curl -s "${VAULT_URI}/_ping"; then echo -e "\nHost *\nIdentityFile ~/.ssh/$VAULT_SSH_KEY" >> ~/.ssh/config fi + # restore 'no_proxy' for executing the actual command + export no_proxy="$no_proxy_old" + log "Executing command: $@" eval $@ From 3cd68ccb17f4d4def2d97f74efff2b2a6c938393 Mon Sep 17 00:00:00 2001 From: Nikolaus Demmel Date: Fri, 5 Aug 2016 22:09:53 +0200 Subject: [PATCH 2/6] remove temporary tared archive of .ssh folder --- index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/index.js b/index.js index 6de406a..d28a6c5 100644 --- a/index.js +++ b/index.js @@ -33,6 +33,7 @@ app.get('/ssh.tgz', function (req, res) { var filestream = fs.createReadStream(file); filestream.pipe(res); + fs.unlink(file) }); }); }); From a57692e029485f5f1b2b43c43fb0e37a41f11a25 Mon Sep 17 00:00:00 2001 From: Nikolaus Demmel Date: Fri, 5 Aug 2016 22:12:00 +0200 Subject: [PATCH 3/6] make sure /vault/.ssh exists (e.g. in case we start the container with an empty volume) --- index.js | 4 +++- package.json | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index d28a6c5..af904c0 100644 --- a/index.js +++ b/index.js @@ -2,7 +2,8 @@ var express = require('express'), fs = require('fs'), exec = require('child_process').exec, mime = require('mime'), - path = require('path'); + path = require('path'), + mkdirp = require('mkdirp'); var host = process.env.HTTP_HOST || '0.0.0.0'; @@ -21,6 +22,7 @@ app.get('/_ping', function (req, res) { Bundle containing all the user's private keys and ssh configuration */ app.get('/ssh.tgz', function (req, res) { + mkdirp("/vault/.ssh"); exec('mktemp -q /tmp/ssh.XXXXXX', function (err, stdout) { var file = stdout.match(/(.+)/)[0]; diff --git a/package.json b/package.json index c8bfb35..0843fb6 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "license": "ISC", "dependencies": { "express": "^4.12.3", - "mime": "^1.3.4" + "mime": "^1.3.4", + "mkdirp": "^0.5.1" } } From b1f14381a3ced41579f239a2cc8e1846953377e7 Mon Sep 17 00:00:00 2001 From: Nikolaus Demmel Date: Fri, 5 Aug 2016 22:12:36 +0200 Subject: [PATCH 4/6] make sure ONVAULT works with empty .ssh folder --- ONVAULT | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ONVAULT b/ONVAULT index 30b8e2a..6ca92f5 100755 --- a/ONVAULT +++ b/ONVAULT @@ -49,8 +49,8 @@ if curl -s "${VAULT_URI}/_ping"; then log "Downloading private keys..." curl -s "${VAULT_URI}/ssh.tgz" | tar -C ~/.ssh/ -zxf - - chown `whoami` ~/.ssh/* - chmod 600 ~/.ssh/* + chown -f `whoami` ~/.ssh/* || true + chmod -f 600 ~/.ssh/* || true log "Using ssh key: $VAULT_SSH_KEY" if [[ "$VAULT_SSH_KEY" != "id_rsa" ]]; then From 743cf018b7a3f926e49e45069776a481f4e5a12e Mon Sep 17 00:00:00 2001 From: Nikolaus Demmel Date: Fri, 5 Aug 2016 22:13:13 +0200 Subject: [PATCH 5/6] Add .dockerignore for all except the actually needed files. --- .dockerignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..64b2baf --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +* +!index.js +!ONVAULT +!package.json From 8b7587afdb88e2ca5c7af842a4a2b3deef163742 Mon Sep 17 00:00:00 2001 From: Nikolaus Demmel Date: Sat, 6 Aug 2016 00:04:53 +0200 Subject: [PATCH 6/6] Add support for git https credentials This implements #21. It is a simple way to provide git credentials not only for ssh via keys, but also for https. In corporate settings with internet access through proxies, ssh is often not allowed and one is forced to use https. There are two ways to use this, either you provide a credential store and mount it when running `vault`. ``` docker run -p 172.17.0.1:14242:3000 -v $PWD/store:/vault/store vault ``` Alternatively you can also start the vault with an empty store and interactively add / remove credentials. ``` docker run -p 172.17.0.1:14242:3000 --name vault vault # the following prompts you for a username and password which are stored in the # running container docker exec -it vault credentials set github.com ``` The usage inside a `Dockerfile` is the same as for the ssh key. Simply prepend your `git clone` or other commands with `ONVAULT`. The changes are really fairly simple: - One route is added in `index.js` to download the file with the credentials. - ONVAULT is extended to download the additional file and configure git, as well as revert the changes after executing the passed command. - The `credential` helper script is added to the container. This is actually not essential, but more of a convenience such that you don't need to create the credential files manually. I did not yet update the README, but I can do that before merge if you are willing to include this addition. closes #21 --- .dockerignore | 1 + Dockerfile | 3 +- ONVAULT | 35 ++++++++++++-- credentials | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++ index.js | 18 +++++++ package.json | 1 + 6 files changed, 181 insertions(+), 4 deletions(-) create mode 100755 credentials diff --git a/.dockerignore b/.dockerignore index 64b2baf..8cc36b8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,4 @@ !index.js !ONVAULT !package.json +!credentials diff --git a/Dockerfile b/Dockerfile index 522213f..9553fb6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,12 @@ FROM mhart/alpine-node:0.10 -RUN mkdir -p /vault /usr/src/app +RUN mkdir -p /vault /vault/store /usr/src/app WORKDIR /usr/src/app COPY package.json /usr/src/app/ RUN npm install COPY . /usr/src/app/ +RUN ln -s /usr/src/app/credentials /usr/bin/credentials EXPOSE 3000 VOLUME /vault diff --git a/ONVAULT b/ONVAULT index 6ca92f5..d3961e1 100755 --- a/ONVAULT +++ b/ONVAULT @@ -47,24 +47,46 @@ if curl -s "${VAULT_URI}/_ping"; then cp -r ~/.ssh/* $tmp_ssh_vault fi - log "Downloading private keys..." + log "Downloading private keys and git credentials ..." curl -s "${VAULT_URI}/ssh.tgz" | tar -C ~/.ssh/ -zxf - chown -f `whoami` ~/.ssh/* || true chmod -f 600 ~/.ssh/* || true - log "Using ssh key: $VAULT_SSH_KEY" + if [ ! -f ~/.ssh/$VAULT_SSH_KEY ]; then + log "Did not find ssh key: $VAULT_SSH_KEY" + else + log "Using ssh key: $VAULT_SSH_KEY" + fi if [[ "$VAULT_SSH_KEY" != "id_rsa" ]]; then # configure the ssh to any host to use this ssh key echo -e "\nHost *\nIdentityFile ~/.ssh/$VAULT_SSH_KEY" >> ~/.ssh/config fi + # download git credential store to temporary location + tmp_git_credential_store=`mktemp ~/.git-credentials-XXXXXX` + curl -s "${VAULT_URI}/git-credentials" > "$tmp_git_credential_store" + + # check number of credentials (== non-blank lines) for seeing if something is wrong + credential_count=`cat "$tmp_git_credential_store" | sed '/^\s*$/d' | wc -l` + if [ $credential_count -eq 0 ]; then + log "Did not find any git https credentials" + else + log "Using $credential_count git https credential(s)" + fi + + # configure git + git_config_old_credential_helper=`git config --global credential.helper` || true + git_config_old_core_askpass=`git config --global core.askpass` || true + git config --global credential.helper "store --file=$tmp_git_credential_store" + git config --global core.askpass true + # restore 'no_proxy' for executing the actual command export no_proxy="$no_proxy_old" log "Executing command: $@" eval $@ - log "Removing private keys..." + log "Removing private keys and git credentials..." rm -rf ~/.ssh/* # copying backup to ssh directory @@ -72,6 +94,13 @@ if curl -s "${VAULT_URI}/_ping"; then cp -r $tmp_ssh_vault/* ~/.ssh rm -rf $tmp_ssh_vault fi + + rm $tmp_git_credential_store + + # restoring old setting + git config --global credential.helper "$git_config_old_credential_helper" + git config --global core.askpass "$git_config_old_core_askpass" + else log "ERROR: Start the dockito/vault container before using ONVAULT!" log "ex: docker run -d -p ${VAULT_HOST}:14242:3000 -v ~/.ssh:/vault/.ssh dockito/vault" diff --git a/credentials b/credentials new file mode 100755 index 0000000..7599e50 --- /dev/null +++ b/credentials @@ -0,0 +1,127 @@ +#!/bin/sh + +# allow overriding location of credential store +: ${CREDENTIALFILE:=/vault/store/git-credentials} + +usage () { + echo "Usage: + $(basename $0) set URL + $(basename $0) remove URL + $(basename $0) list + $(basename $0) clear + +Examples: + $(basename $0) set github.com + $(basename $0) set gitlab.com/foo/bar + $(basename $0) remove github.com +" +} + +# parse arguments +while [ $# -ge 1 ]; do + case "$1" in + set|remove) + if [ $# -eq 2 ]; then + CMD="credentials_$1 $2" + shift + fi + break + ;; + list|clear) + CMD="credentials_$1" + break + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac + + shift +done + +if [ $# -ne 1 ] || [ -z "$CMD" ]; then + usage + exit 1 +fi + +# helper to url-encode username and password using javascript +# proper escaping for username & password, see: https://gist.github.com/pierrevalade/6025241 +encodeurl () { + node </dev/null` + if [ -n "$matching" ]; then + tempfile=`mktemp -t git-credentials-XXXXXX` + grep -v "@$URL\$" "$CREDENTIALFILE" > "$tempfile" + mv "$tempfile" "$CREDENTIALFILE" + else + echo "No credentials for '$URL'" + exit 1 + fi +} + +credentials_set () { + # remove leading https:// + URL="$( echo "$1" | sed -e 's#^.*://##' )" + + # remove existing entry + matching=`grep "@$URL\$" "$CREDENTIALFILE" 2>/dev/null` + if [ -n "$matching" ]; then + echo "Removing existing entries:" + echo "$matching" | mask_passwords + read -p "Continue? [yN]: " CONFIRM + if ! [ "$CONFIRM" = "y" ]; then + exit 1 + fi + credentials_remove $URL + echo "" + fi + + echo "Adding credentials for 'https://$URL'." + read -p "Username: " USER + read -p "Password: " -s PASSWORD + echo "" + + USER=`encodeurl "$USER"` + PASSWORD=`encodeurl "$PASSWORD"` + + echo "https://$USER:$PASSWORD@$URL" >> "$CREDENTIALFILE" + chmod 600 "$CREDENTIALFILE" +} + +credentials_list () { + if [ -f "$CREDENTIALFILE" ]; then + # mask passwords and sort by URL (which starts after the "@") + cat "$CREDENTIALFILE" | mask_passwords | sort -k 2 -t "@" + fi +} + +credentials_clear () { + echo "" > "$CREDENTIALFILE" +} + + +$CMD diff --git a/index.js b/index.js index af904c0..dddb947 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ var express = require('express'), exec = require('child_process').exec, mime = require('mime'), path = require('path'), + touch = require("touch"), mkdirp = require('mkdirp'); @@ -41,6 +42,23 @@ app.get('/ssh.tgz', function (req, res) { }); +/** + Route to get the credenial store + */ +app.get('/git-credentials', function (req, res) { + var file = '/vault/store/git-credentials'; + touch(file); + var filename = path.basename(file); + var mimetype = mime.lookup(file); + + res.setHeader('Content-disposition', 'attachment; filename=' + filename); + res.setHeader('Content-type', mimetype); + + var filestream = fs.createReadStream(file); + filestream.pipe(res); +}); + + /** Route to get the ONVAULT utility to be used during build */ diff --git a/package.json b/package.json index 0843fb6..6ce1e1d 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "dependencies": { "express": "^4.12.3", "mime": "^1.3.4", + "touch": "^1.0.0", "mkdirp": "^0.5.1" } }