diff --git a/.env.example b/.env.example index 3508b6e7..37fe39eb 100644 --- a/.env.example +++ b/.env.example @@ -7,10 +7,20 @@ DATAPUSHER_CONTAINER_NAME=datapusher CKAN_CONTAINER_NAME=ckan WORKER_CONTAINER_NAME=ckan-worker -# Host Ports -CKAN_PORT_HOST=5000 -NGINX_PORT_HOST=81 -NGINX_SSLPORT_HOST=8443 +# Host hostname and ports +CKAN_DOMAIN=localhost +CKAN_PORT_HOST=5000 # Used only in the dev deployment +NGINX_PORT_HOST=80 +NGINX_SSLPORT_HOST=443 + +# SSL certificate details +CKAN_CERT_EMAIL=your_cert_email@example.com +CKAN_CERT_ORG=your_organisation +#CKAN_AUTO_CERT=true +#CKAN_CERT_OPTIONS= +#CKAN_CERT_RENEW_INTERVAL=12h +#LETSENCRYPT_DIR=/etc/letsencrypt +LETSENCRYPT_DRYRUN=--dry-run # CKAN databases POSTGRES_USER=postgres @@ -38,9 +48,9 @@ USE_HTTPS_FOR_DEV=false # CKAN core CKAN_VERSION=2.10.0 CKAN_SITE_ID=default -CKAN_SITE_URL=https://localhost:8443 CKAN_PORT=5000 CKAN_PORT_HOST=5000 +CKAN_SITE_URL=https://${CKAN_DOMAIN}:${NGINX_SSLPORT_HOST} CKAN___BEAKER__SESSION__SECRET=CHANGE_ME # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME diff --git a/README.md b/README.md index f5548a7c..70ef94c0 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ * [Datastore and Datapusher](#Datastore-and-datapusher) * [NGINX](#nginx) * [The ckanext-envvars extension](#envvars) -* [The CKAN_SITE_URL parameter](#CKAN_SITE_URL) +* [Connecting directly to CKAN](#connecting-directly) * [Changing the base image](#Changing-the-base-image) * [Replacing DataPusher with XLoader](#Replacing-DataPusher-with-XLoader) @@ -113,11 +113,15 @@ The new extension files and directories are created in the `/srv/app/src_extensi Sometimes is useful to run your local development instance under HTTPS, for instance if you are using authentication extensions like [ckanext-saml2auth](https://github.com/keitaroinc/ckanext-saml2auth). To enable it, set the following in your `.env` file: +``` USE_HTTPS_FOR_DEV=true +``` and update the site URL setting: - CKAN_SITE_URL=https://localhost:5000 +``` + CKAN_SITE_URL=https://${CKAN_DOMAIN}:${CKAN_PORT_HOST} +``` After recreating the `ckan-dev` container, you should be able to access CKAN at https://localhost:5000 @@ -167,7 +171,7 @@ ckan -c /srv/app/ckan.ini validation init-db And then in our `Dockerfile.dev` file we install the extension and copy the initialization scripts: ```Dockerfile -FROM ckan/ckan-base:2.9.7-dev +FROM ckan/ckan-base:2.10.1-dev RUN pip install -e git+https://github.com/frictionlessdata/ckanext-validation.git#egg=ckanext-validation && \ pip install -r https://raw.githubusercontent.com/frictionlessdata/ckanext-validation/master/requirements.txt @@ -175,7 +179,7 @@ RUN pip install -e git+https://github.com/frictionlessdata/ckanext-validation.gi COPY docker-entrypoint.d/* /docker-entrypoint.d/ ``` -NB: There are a number of extension examples commented out in the Dockerfile.dev file +> Note: There are a number of extension examples commented out in the Dockerfile.dev file ## 7. Applying patches @@ -218,11 +222,47 @@ running the latest version of Datapusher. ## 10. NGINX -The base Docker Compose configuration uses an NGINX image as the front-end (ie: reverse proxy). It includes HTTPS running on port number 8443. A "self-signed" SSL certificate is generated as part of the ENTRYPOINT. The NGINX `server_name` directive and the `CN` field in the SSL certificate have been both set to 'localhost'. This should obviously not be used for production. +The base Docker Compose configuration uses NGINX as the front-end (ie: reverse proxy) and +SSL terminator. It accepts HTTPS connections on ports 80 and 443 (port 80 gets redirected to 443). +A "self-signed" SSL certificate is generated when NGINX starts, unless you provide your own +certificate and key (see below). This should obviously not be used for production. + +NGINX can automatically obtain a [Let's Encrypt](https://letsencrypt.org/getting-started/) +SSL certificate and renew it periodically, if you configure the value `CKAN_AUTO_CERT` in +the `.env` file. The hostname and the email address to be used in the certificate request +must be defined in the `.env` file. + +> Note: If you are not using a self-signed certificate, you must define the domain name on +> which CKAN will be published in the variable `CKAN_DOMAIN` in the `.env` file. If you supply +> your own certificate, the configured domain name must match the domain name in the certificate. + +> Note: Even when you set `CKAN_AUTO_CERT` to `true` to use a Let's Encrypt SSL certificate, +> NGINX will still use a self-signed certificate and will only do dry runs for requesting a +> certificate to avoid the [rate limits](https://letsencrypt.org/docs/rate-limits/) of +> Let's Encrypt. Once you are satisfied that everything is running as expected, you can comment +> out the setting `LETSENCRYPT_DRYRUN` in the `.env` file. + +If you want to use your own SSL certificate with CKAN, you must supply the files +`fullchain.pem` and `privkey.pem`, respectively. Consider the following folder structure: + +``` +nginx +├── certificate +│ ├── my_cert.pem +│ └── my_key.pem +└── Dockerfile +``` + +Then modify the NGINX Dockerfile to use these files: + +``` +COPY certificate/my_cert.pem /usr/share/nginx/certificates/fullchain.pem +COPY certificate/my_key.pem /usr/share/nginx/certificates/privkey.pem +``` -Creating the SSL cert and key files as follows: -`openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -subj "/C=DE/ST=Berlin/L=Berlin/O=None/CN=localhost" -keyout ckan-local.key -out ckan-local.crt` -The `ckan-local.*` files will then need to be moved into the nginx/setup/ directory +> Note: In case you remove the CKAN containers, retain the volume `ssl_cert`, +> which contains the SSL certificate. This will avoid requesting a new one for the same domain, +> in case you redeploy CKAN (prevents exceeding Let's Encrypt rate limit). ## 11. envvars @@ -245,9 +285,12 @@ These parameters can be added to the `.env` file For more information please see [ckanext-envvars](https://github.com/okfn/ckanext-envvars) -## 12. CKAN_SITE_URL +## 12. Connecting directly -For convenience the CKAN_SITE_URL parameter should be set in the .env file. For development it can be set to http://localhost:5000 and non-development set to https://localhost:8443 +For convenience, in development deployments you can connect directly to the CKAN container +at http://localhost:5000. In non-development environments you can use https://localhost:443. +The hostname is defined in variable `CKAN_DOMAIN`, while the port is defined in +variable `CKAN_PORT_HOST` in the `.env` file. ## 13. Manage new users diff --git a/docker-compose.yml b/docker-compose.yml index 0f5330fb..5b1d262a 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,10 @@ version: "3" - volumes: ckan_storage: pg_data: solr_data: + ssl_cert: services: @@ -13,14 +13,33 @@ services: build: context: nginx/ dockerfile: Dockerfile + args: + - CKAN_URL=http://ckan:${CKAN_PORT}/ + - DOMAIN=${CKAN_DOMAIN} + - EMAIL=${CKAN_CERT_EMAIL} + - ORGANISATION=${CKAN_CERT_ORG} + - SSL_AUTO_CERT=${CKAN_AUTO_CERT:-true} + - SSL_CERT_OPTIONS=${CKAN_CERT_OPTIONS:-} + - SSL_CERT_RENEW=${CKAN_CERT_RENEW_INTERVAL:-12h} + - LETSENCRYPT_DIR=${LETSENCRYPT_DIR:-/etc/letsencrypt} + - LETSENCRYPT_DRYRUN=${LETSENCRYPT_DRYRUN:- } + volumes: + - ssl_cert:${LETSENCRYPT_DIR:-/etc/letsencrypt} networks: - webnet - ckannet + restart: unless-stopped + healthcheck: + test: ['CMD', '/opt/status.sh'] + start_period: 30s + interval: 1m + timeout: 5s depends_on: ckan: condition: service_healthy ports: - - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}" + - "0.0.0.0:${NGINX_PORT_HOST}:80" + - "0.0.0.0:${NGINX_SSLPORT_HOST}:443" ckan: container_name: ${CKAN_CONTAINER_NAME} diff --git a/nginx/Dockerfile b/nginx/Dockerfile index eda7994e..8d8d262b 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -1,23 +1,46 @@ FROM nginx:stable-alpine +ARG CKAN_URL +ARG DOMAIN +ARG EMAIL +ARG ORGANISATION +ARG SSL_AUTO_CERT +ARG SSL_CERT_OPTIONS +ARG SSL_CERT_RENEW +ARG LETSENCRYPT_DIR +ARG LETSENCRYPT_DRYRUN + +ENV DOMAIN=${DOMAIN} +ENV EMAIL=${EMAIL} +ENV ORGANISATION=${ORGANISATION} +ENV SSL_AUTO_CERT=${SSL_AUTO_CERT} +ENV SSL_CERT_OPTIONS=${SSL_CERT_OPTIONS} +ENV SSL_CERT_RENEW=${SSL_CERT_RENEW} +ENV LETSENCRYPT_DRYRUN=${LETSENCRYPT_DRYRUN} +ENV LETSENCRYPT_DIR=${LETSENCRYPT_DIR} ENV NGINX_DIR=/etc/nginx RUN apk update --no-cache && \ apk upgrade --no-cache && \ - apk add --no-cache openssl + apk add --no-cache openssl inotify-tools certbot -COPY setup/nginx.conf ${NGINX_DIR}/nginx.conf COPY setup/index.html /usr/share/nginx/html/index.html -COPY setup/default.conf ${NGINX_DIR}/conf.d/ - -RUN mkdir -p ${NGINX_DIR}/certs - -ENTRYPOINT \ - openssl req \ - -subj '/C=DE/ST=Berlin/L=Berlin/O=None/CN=localhost' \ - -x509 -newkey rsa:4096 \ - -nodes -keyout /etc/nginx/ssl/default_key.pem \ - -keyout ${NGINX_DIR}/certs/ckan-local.key \ - -out ${NGINX_DIR}/certs/ckan-local.crt \ - -days 365 && \ - nginx -g 'daemon off;' \ No newline at end of file +COPY setup/nginx.conf ${NGINX_DIR}/nginx.conf +COPY setup/default.conf ${NGINX_DIR}/conf.d/default.conf.template +RUN envsubst '${DOMAIN}${CKAN_URL}' < ${NGINX_DIR}/conf.d/default.conf.template > ${NGINX_DIR}/conf.d/default.conf && \ + rm -f ${NGINX_DIR}/conf.d/default.conf.template && \ + mkdir -p /var/cache/nginx/proxycache && \ + mkdir -p /var/cache/nginx/proxytemp + +WORKDIR /opt +COPY setup/cert-entrypoint.sh entrypoint.sh +COPY setup/cert-request.sh request.sh +COPY setup/health-check.sh status.sh + +RUN chmod a+x entrypoint.sh && \ + chmod a+x request.sh && \ + chmod a+x status.sh + +EXPOSE 80 443 + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/nginx/setup/cert-entrypoint.sh b/nginx/setup/cert-entrypoint.sh new file mode 100644 index 00000000..ee326d15 --- /dev/null +++ b/nginx/setup/cert-entrypoint.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +export DOMAIN=${DOMAIN} +export EMAIL=${EMAIL} +export SSL_CERT_OPTIONS=${SSL_CERT_OPTIONS} +export LETSENCRYPT_DIR=${LETSENCRYPT_DIR} +export LETSENCRYPT_DRYRUN=${LETSENCRYPT_DRYRUN} + +# Ensure we have a folder for the certificates +if [ ! -d /usr/share/nginx/certificates ]; then + echo "Creating certificate folder" + mkdir -p /usr/share/nginx/certificates +fi + +### If certificates do not exist yet, create self-signed ones before we start nginx +if [ ! -f /usr/share/nginx/certificates/fullchain.pem ]; then + echo "Generating self-signed certificate" + openssl genrsa -out /usr/share/nginx/certificates/privkey.pem 4096 + openssl req -new -key /usr/share/nginx/certificates/privkey.pem -out /usr/share/nginx/certificates/cert.csr -nodes -subj \ + "/C=PT/ST=World/L=World/O=$ORGANISATION/CN=$DOMAIN" + openssl x509 -req -days 365 -in /usr/share/nginx/certificates/cert.csr -signkey /usr/share/nginx/certificates/privkey.pem -out /usr/share/nginx/certificates/fullchain.pem +fi + +if [ -n "$DOMAIN" ] && [ "$DOMAIN" != "localhost" ] && [ -n "${SSL_AUTO_CERT}" ] && ${SSL_AUTO_CERT}; then + ### Send certbot emission/renewal to background + echo "Scheduling periodic check if certificate should be renewed" + $(while :; do /opt/request.sh; sleep "${SSL_CERT_RENEW}"; done;) & + + ### Check for changes in the certificate (i.e renewals or first start) in the background + $(while inotifywait -e close_write /usr/share/nginx/certificates; do echo "Reloading nginx with new certificate"; nginx -s reload; done) & +fi + +### Start nginx with daemon off as our main pid +echo "Starting nginx" +nginx -g 'daemon off;' diff --git a/nginx/setup/cert-request.sh b/nginx/setup/cert-request.sh new file mode 100644 index 00000000..13a5cef7 --- /dev/null +++ b/nginx/setup/cert-request.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +if [ ! -f /var/www/html ]; then + mkdir -p /var/www/html +fi + +if [ -n "$DOMAIN" ] && [ "$DOMAIN" != "localhost" ]; then + certbot certonly \ + --config-dir ${LETSENCRYPT_DIR} ${LETSENCRYPT_DRYRUN} \ + --agree-tos \ + --domains "$DOMAIN" \ + --email $EMAIL \ + --expand \ + --noninteractive \ + --webroot \ + --webroot-path /var/www/html \ + $SSL_CERT_OPTIONS || true + + if [ -f ${LETSENCRYPT_DIR}/live/$DOMAIN/privkey.pem ]; then + chmod +rx ${LETSENCRYPT_DIR}/live + chmod +rx ${LETSENCRYPT_DIR}/archive + chmod +r ${LETSENCRYPT_DIR}/archive/${DOMAIN}/fullchain*.pem + chmod +r ${LETSENCRYPT_DIR}/archive/${DOMAIN}/privkey*.pem + cp ${LETSENCRYPT_DIR}/live/$DOMAIN/privkey.pem /usr/share/nginx/certificates/privkey.pem + cp ${LETSENCRYPT_DIR}/live/$DOMAIN/fullchain.pem /usr/share/nginx/certificates/fullchain.pem + echo "Copied new certificate to /usr/share/nginx/certificates" + fi +fi diff --git a/nginx/setup/default.conf b/nginx/setup/default.conf index a628619f..66ad2dbc 100644 --- a/nginx/setup/default.conf +++ b/nginx/setup/default.conf @@ -1,11 +1,23 @@ server { - #listen 80; - #listen [::]:80; + listen 80 default; + listen [::]:80 default; + server_name $DOMAIN; + + location /.well-known/acme-challenge/ { + root /var/www/html; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { listen 443 ssl; listen [::]:443 ssl; - server_name localhost; - ssl_certificate /etc/nginx/certs/ckan-local.crt; - ssl_certificate_key /etc/nginx/certs/ckan-local.key; + server_name $DOMAIN; + ssl_certificate /usr/share/nginx/certificates/fullchain.pem; + ssl_certificate_key /usr/share/nginx/certificates/privkey.pem; # TLS 1.2 & 1.3 only ssl_protocols TLSv1.2 TLSv1.3; @@ -21,8 +33,12 @@ server { #access_log /var/log/nginx/host.access.log main; + location /.well-known/acme-challenge/ { + root /var/www/html; + } + location / { - proxy_pass http://ckan:5000/; + proxy_pass $CKAN_URL; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header Host $host; #proxy_cache cache; @@ -35,7 +51,6 @@ server { error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 421 422 423 424 425 426 428 429 431 451 500 501 502 503 504 505 506 507 508 510 511 /error.html; # redirect server error pages to the static page /error.html - # location = /error.html { ssi on; internal; diff --git a/nginx/setup/health-check.sh b/nginx/setup/health-check.sh new file mode 100644 index 00000000..4b7bc43e --- /dev/null +++ b/nginx/setup/health-check.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +FAIL_CODE=6 + +check_status() { + LRED="\033[1;31m" # Light Red + LGREEN="\033[1;32m" # Light Green + NC='\033[0m' # No Color + + curl -sfk "${1}" > /dev/null + + if [ ! $? = ${FAIL_CODE} ]; then + echo -e "${LGREEN}${1} is online${NC}" + exit 0 + else + echo -e "${LRED}${1} is down${NC}" + exit 1 + fi +} + +check_status "${1:-localhost}"