diff --git a/.ahoy.yml b/.ahoy.yml new file mode 100644 index 00000000..7d292e4b --- /dev/null +++ b/.ahoy.yml @@ -0,0 +1,152 @@ +--- +ahoyapi: v2 + +entrypoint: + - bash + - "-c" + - "-e" + - | + [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi + bash -e -c "$0" "$@" + - "{{cmd}}" + - "{{name}}" + +commands: + init: + usage: Initialise the codebase on first-time setup (ahoy init) + cmd : | + echo "Creating project variables." + cp .env.dbca .env + + echo "Install local development extensions." + sh src/dbca_install_extensions.sh + up: + usage: Build project. + cmd: | + docker compose -f $DOCKER_COMPOSE up -d "$@" + ahoy info; + + down: + usage: Delete project (CAUTION). + cmd: | + if [ "$1" == "y" ]; then + docker compose -f $DOCKER_COMPOSE down --volumes + else + ahoy confirm "Running this command will destroy your current site, database and build? Are you sure you didn't mean ahoy stop?" && + # Run this if confirm returns true + docker compose -f $DOCKER_COMPOSE down --volumes || + # Run this if confirm returns false + echo "OK, probably a wise choice..." + fi + + build: + usage: Build project. + cmd: | + if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then + CKAN_CONTAINER_NAME=ckan-dev + WORKER_CONTAINER_NAME=ckan-dev-worker + fi + docker compose -f $DOCKER_COMPOSE build "$@" + + cli: + usage: Start a shell inside container. + cmd: | + if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then + CKAN_CONTAINER_NAME=ckan-dev + fi + docker compose -f $DOCKER_COMPOSE exec ${1:-$CKAN_CONTAINER_NAME} sh + + run: + usage: Run command inside container. + cmd: | + if [[ $# -eq 2 ]]; then + SERVICE=$1 + COMMAND=$2 + else + if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then + CKAN_CONTAINER_NAME=ckan-dev + fi + SERVICE=$CKAN_CONTAINER_NAME + COMMAND=$1 + fi; + docker compose -f $DOCKER_COMPOSE exec -T $SERVICE sh -c "$COMMAND" + + logs: + usage: Show Docker logs. + cmd: | + TAIL=100 + if [[ $# -eq 2 ]]; then + SERVICE=$1 + TAIL=$2 + else + SERVICE=$1 + fi; + docker compose -f $DOCKER_COMPOSE logs -f --tail=$TAIL $SERVICE + + ps: + usage: List running Docker containers. + cmd: docker compose -f $DOCKER_COMPOSE ps + + restart: + usage: Restart Docker containers. + cmd: | + docker compose -f $DOCKER_COMPOSE restart "$@" + ahoy info; + + stop: + usage: Stop Docker containers. + cmd: docker compose -f $DOCKER_COMPOSE stop "$@" + + attach: + usage: Attach to a running container + cmd: docker attach $(docker compose -f $DOCKER_COMPOSE ps -q "$@") + + recreate: + usage: Recreate a local container | ahoy recreate ckan + cmd: | + docker compose -f $DOCKER_COMPOSE up -d --force-recreate --no-deps --build "${1}" + + info: + usage: Print information about this project. + cmd: | + if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then + SITE_URL="http://localhost:${CKAN_PORT_HOST}" + else + SITE_URL="https://localhost:${NGINX_SSLPORT_HOST}" + fi + echo "Site local URL : ${SITE_URL}" + if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then + echo "CKAN DB port on host : $(docker port $(docker compose -f $DOCKER_COMPOSE ps -q $POSTGRESQL_CONTAINER_NAME) 5432 | cut -d : -f 2)" + echo "SOLR port on host : $(docker port $(docker compose -f $DOCKER_COMPOSE ps -q $SOLR_CONTAINER_NAME) 8983 | cut -d : -f 2)" + fi + + db-import: + usage: Pipe in a postgres dump file. `ahoy db-import local.dump` + cmd: | + if [ -e "$@" ] ; then + docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_restore -U $CKAN_DB_USER -d $CKAN_DB --clean --if-exists -v --no-owner --no-privileges' < "$@" + else echo "Provided sql file" "$@" "does not exist" + fi + + db-dump: + usage: Dump data out into a file. `ahoy db-dump local.dump` + cmd: docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_dump -U $CKAN_DB_USER -d $CKAN_DB --format=custom -v' > "$@" + + confirm: + cmd: read -r -p "${@} [y/N] " response; [ ${response} = "y" ] + hide: true + + generate-extension: + usage: Generates a new CKAN extension into the src directory + cmd: ahoy run cli 'ckan generate extension -o ${SRC_DIR}' + + open: + usage: Open the site in your default browser + cmd: | + if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then + SITE_URL="http://localhost:${CKAN_PORT_HOST}" + else + SITE_URL="http://localhost:${NGINX_PORT_HOST}" + fi + open $SITE_URL + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..edb45725 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,82 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose +{ + "name": "CKAN DBCA 2.10", + + // Update the 'dockerComposeFile' list if you have more compose files or use different names. + // The .devcontainer/docker-compose.yml file contains any overrides you need/want to make. + "dockerComposeFile": [ + "../docker-compose.dev.yml", + "docker-compose.yml" + ], + + // The 'service' property is the name of the service for the container that VS Code should + // use. Update this value and .devcontainer/docker-compose.yml to the real service name. + "service": "ckan-dev", + + // The optional 'workspaceFolder' property is the path VS Code should open by default when + // connected. This is typically a file mount in .devcontainer/docker-compose.yml + "workspaceFolder": "/srv", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.autopep8", + "ms-python.flake8", + "VisualStudioExptTeam.vscodeintellicode", + "GitHub.vscode-pull-request-github", + "eamodio.gitlens", + "streetsidesoftware.code-spell-checker", + "wholroyd.jinja", + "GitHub.copilot", + "GitHub.copilot-chat" + ], + // Set *default* container specific settings.json values on container create. + "settings": { + "terminal.integrated.profiles.linux": { + "bash": { + "path": "/bin/bash" + } + }, + "python.languageServer": "Pylance", + "python.linting.enabled": true, + "python.linting.flake8Enabled": true, + "flake8.args": [ + "--max-line-length", + "120" + ], + "python.formatting.provider": "autopep8", + "autopep8.args": [ + "--max-line-length", + "120" + ], + "cSpell.language": "en-GB", + "python.analysis.extraPaths": [ + "./app/src" + ] + } + } + }, + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Uncomment the next line if you want start specific services in your Docker Compose config. + // "runServices": [], + + // Uncomment the next line if you want to keep your containers running after VS Code shuts down. + // "shutdownAction": "none", + + // Uncomment the next line to run commands after the container is created. + "postStartCommand": "${APP_DIR}/start_ckan_development.sh", + + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "devcontainer" +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 00000000..c9359625 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,25 @@ +version: '3' +services: + # Update this to the name of the service you want to work with in your docker-compose.yml file + ckan-dev: + # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer + # folder. Note that the path of the Dockerfile and context is relative to the *primary* + # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile" + # array). The sample below assumes your primary file is in the root of your project. + # + # build: + # context: . + # dockerfile: .devcontainer/Dockerfile + + volumes: + # Update this to wherever you want VS Code to mount the folder of your project + - .:/srv/workspace:cached + - .vscode:/srv/.vscode:cached + # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust. + # cap_add: + # - SYS_PTRACE + # security_opt: + # - seccomp:unconfined + + # Overrides default command so things don't shut down after the process ends. + command: /bin/sh -c "while sleep 1000; do :; done" diff --git a/.env.dbca b/.env.dbca new file mode 100644 index 00000000..28bb082b --- /dev/null +++ b/.env.dbca @@ -0,0 +1,82 @@ +# Container names +NGINX_CONTAINER_NAME=nginx +REDIS_CONTAINER_NAME=redis +POSTGRESQL_CONTAINER_NAME=db +SOLR_CONTAINER_NAME=solr +CKAN_CONTAINER_NAME=ckan +WORKER_CONTAINER_NAME=ckan-worker + +# Host Ports +CKAN_PORT_HOST=5000 +NGINX_PORT_HOST=81 +NGINX_SSLPORT_HOST=8443 + +# CKAN databases +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=postgres +POSTGRES_HOST=db +CKAN_DB_USER=ckandbuser +CKAN_DB_PASSWORD=ckandbpassword +CKAN_DB=ckandb +DATASTORE_READONLY_USER=datastore_ro +DATASTORE_READONLY_PASSWORD=datastore +DATASTORE_DB=datastore +CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb +CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore +CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore + +# Test database connections +TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test +TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test + +# Dev settings +USE_HTTPS_FOR_DEV=false + +# CKAN core +CKAN_VERSION=2.10.0 +CKAN_SITE_ID=default +CKAN_SITE_URL=http://localhost:5000 +CKAN_PORT=5000 + +CKAN___BEAKER__SESSION__SECRET=CHANGE_ME +# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings +CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME +CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME +CKAN_SYSADMIN_NAME=ckan_admin +CKAN_SYSADMIN_PASSWORD=test1234 +CKAN_SYSADMIN_EMAIL=your_email@example.com +CKAN_STORAGE_PATH=/var/lib/ckan +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost +TZ=UTC + +# Solr +SOLR_IMAGE_VERSION=2.10-solr9 +CKAN_SOLR_URL=http://solr:8983/solr/ckan +TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan + +# Redis +REDIS_VERSION=7 +CKAN_REDIS_URL=redis://redis:6379/1 +TEST_CKAN_REDIS_URL=redis://redis:6379/1 + +# Xloader +CKANEXT__XLOADER__JOBS_DB__URI=$CKAN_SQLALCHEMY_URL + +# NGINX +NGINX_PORT=80 +NGINX_SSLPORT=443 + +# Extensions + + +## WA DBCA Config ## +# Docker compose project name +COMPOSE_PROJECT_NAME=dbca +# The docker compose file to use. Options are docker-compose.dev.yml (The default for local development) or docker-compose.prod.yml (To test production builds) +DOCKER_COMPOSE=docker-compose.dev.yml diff --git a/.github/workflows/dbca_build.yml b/.github/workflows/dbca_build.yml new file mode 100644 index 00000000..72459033 --- /dev/null +++ b/.github/workflows/dbca_build.yml @@ -0,0 +1,88 @@ +name: Create and publish a Docker image + +on: + # Trigger the workflow on push on master, develop and tags + push: + branches: + - master + - develop + - tags/* + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker nginx image + id: meta_nginx + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-nginx + + - name: Extract metadata (tags, labels) for Docker postgres image + id: meta_postgres + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-postgres + + - name: Extract metadata (tags, labels) for Docker ckan image + id: meta_ckan + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ckan + + - name: Extract metadata (tags, labels) for Docker ckan worker image + id: meta_ckan_worker + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ckan-worker + + - name: NGINX Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: ./nginx + file: ./nginx/Dockerfile + push: true + tags: ${{ steps.meta_nginx.outputs.tags }} + + - name: PostgreSQL Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: ./postgresql + file: ./postgresql/Dockerfile + push: true + tags: ${{ steps.meta_postgres.outputs.tags }} + + - name: CKAN Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: ./ckan + file: ./ckan/Dockerfile + push: true + tags: ${{ steps.meta_ckan.outputs.tags }} + + - name: CKAN Worker Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: ./ckan + file: ./ckan/Dockerfile.worker + push: true + tags: ${{ steps.meta_ckan_worker.outputs.tags }} + build-args: CKAN_IMAGE=${{ steps.meta_ckan.outputs.tags }} + \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..9d24adba --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,55 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: CKAN Run & Debug", + "type": "python", + "request": "launch", + "module": "pdb", + "args": [ + "-c continue", + "/usr/bin/ckan", + "--config", + "/srv/app/config/dbca.ini", + "run", + "--host", + "0.0.0.0", + ], + "jinja": true, + "justMyCode": true + }, + { + "name": "Python: CKAN Jobs Worker Run & Debug", + "type": "python", + "request": "launch", + "module": "pdb", + "args": [ + "-c continue", + "/usr/bin/ckan", + "--config", + "/srv/app/config/dbca.ini", + "jobs", + "worker", + "priority" + ], + "justMyCode": true + }, + { + "name": "Python: CKAN CLI Command Run & Debug", + "type": "python", + "request": "launch", + "module": "pdb", + "args": [ + "-c continue", + "/srv/app/config/dbca.ini", + "--config", + "/srv/app/dbca.ini", + "" + ], + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/README_DBCA.md b/README_DBCA.md new file mode 100644 index 00000000..3e737d82 --- /dev/null +++ b/README_DBCA.md @@ -0,0 +1,60 @@ +# Docker Compose setup for DBCA CKAN + +## Install Dependencies +- docker +- git +- ahoy (https://github.com/ahoy-cli/ahoy) + +## Configure extensions that will be worked on + - Add any extensions you will be modifying in the file `src/dbca_install_extensions.sh` + +## Init and build local dev environment +- ahoy init + - This will clone the extensions from `dbca_install_extensions.sh` to the `src` folder which will be a mounted folder to the ckan docker containers + - Copy the `.env.dbca` to `.env` which is the env file uses for the CKAN docker containers + - Update the `.env` file to. The extension https://github.com/okfn/ckanext-envvars reads this file on CKAN startup and require to be in a certain format. Please read the readme file https://github.com/okfn/ckanext-envvars#ckanext-envvars + - Enable plugins via `CKAN__PLUGINS` + - Add/Update CKAN core configuration values + - Add/Update CKAN extensions configuration values + - Any updates to this file will recreate the container service to use the updates values when `ahoy up` is used +- ahoy build (Build the projects docker images) +- ahoy up (Starts the projects container services) + - The first time the CKAN Dev containers are created the mapped volume will look in the `src:/srv/app/src_extensions` folder to install any extensions cloned from the `ahoy init` step and will pip install the extension and any any requirements file if they exists +- To see the list of available commands with short descriptions run ahoy +``` +ahoy +NAME: + ahoy - Creates a configurable cli app for running commands. +USAGE: + ahoy [global options] command [command options] [arguments...] + +COMMANDS: + attach Attach to a running container + build Build project. + cli Start a shell inside container. + db-dump Dump data out into a file. `ahoy db-dump local.dump` + db-import Pipe in a postgres dump file. `ahoy db-import local.dump` + down Delete project (CAUTION). + generate-extension Generates a new CKAN extension into the src directory + info Print information about this project. + init Initialise the codebase on first-time setup (ahoy init) + logs Show Docker logs. + open Open the site in your default browser + ps List running Docker containers. + recreate Recreate a local container | ahoy recreate ckan + restart Restart Docker containers. + run Run command inside container. + stop Stop Docker containers. + up Build project. + +GLOBAL OPTIONS: + --verbose, -v Output extra details like the commands to be run. [$AHOY_VERBOSE] + --file value, -f value Use a specific ahoy file. + --help, -h show help + --version print the version + --generate-bash-completion + +VERSION: + 2.0.2-homebrew + +[fatal] Missing flag or argument.``` \ No newline at end of file diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 5a1ee544..937c3f2a 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -2,6 +2,10 @@ FROM ckan/ckan-base:2.10.3 # Install any extensions needed by your CKAN instance # See Dockerfile.dev for more details and examples +COPY setup/dbca_requirements.sh ${APP_DIR} +RUN pip3 install pip --upgrade && \ + chmod +x ${APP_DIR}/dbca_requirements.sh && \ + ${APP_DIR}/dbca_requirements.sh # Copy custom initialization scripts COPY docker-entrypoint.d/* /docker-entrypoint.d/ @@ -17,3 +21,15 @@ RUN for d in $APP_DIR/patches/*; do \ done ; \ fi ; \ done + +## DBCA specific configuration ## + +# Override the default start_ckan.sh script +COPY setup/dbca_start_ckan.sh.override ${APP_DIR}/start_ckan.sh +RUN chmod +x ${APP_DIR}/start_ckan.sh + +# Copy config files +COPY config/*.ini ${APP_DIR}/config/ + +# Override default CKAN config file to use dbca.ini +ENV CKAN_INI=${APP_DIR}/config/dbca.ini \ No newline at end of file diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index 52ed9753..bb877c04 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -31,6 +31,13 @@ FROM ckan/ckan-dev:2.10.3 #RUN pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \ # pip3 install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt + +# Install any extensions needed by your CKAN instance +COPY setup/dbca_requirements.sh ${APP_DIR} +RUN pip3 install pip --upgrade && \ + chmod +x ${APP_DIR}/dbca_requirements.sh && \ + ${APP_DIR}/dbca_requirements.sh + # Clone the extension(s) your are writing for your own project in the `src` folder # to get them mounted in this image at runtime @@ -48,3 +55,15 @@ RUN for d in $APP_DIR/patches/*; do \ done ; \ fi ; \ done + +## DBCA specific configuration ## + +# Override the default dbca_start_ckan_development.sh script +COPY setup/dbca_start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh +RUN chmod +x ${APP_DIR}/start_ckan_development.sh + +# Copy config files +COPY config/*.ini ${APP_DIR}/config/ + +# Override default CKAN config file to use dbca.ini +ENV CKAN_INI=${APP_DIR}/config/dbca.ini \ No newline at end of file diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker new file mode 100644 index 00000000..4c8a4657 --- /dev/null +++ b/ckan/Dockerfile.worker @@ -0,0 +1,16 @@ +ARG CKAN_IMAGE +FROM ${CKAN_IMAGE} + +## Create logs folder +RUN mkdir -p $APP_DIR/logs && \ + touch $APP_DIR/logs/ckan-cron-jobs.log && \ + chown -R ckan:ckan $APP_DIR/logs && \ + chmod -R 755 $APP_DIR/logs + +## Supervisor config +COPY supervisor/*.conf /etc/supervisord.d + +## Cron jobs config +COPY setup/dbca_ckan_cron_jobs $APP_DIR +RUN chmod -x $APP_DIR/dbca_ckan_cron_jobs +RUN crontab -u ckan $APP_DIR/dbca_ckan_cron_jobs diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini new file mode 100644 index 00000000..ecc26b86 --- /dev/null +++ b/ckan/config/dbca.ini @@ -0,0 +1,64 @@ +# +# DBCA configuration +# + +[app:main] + +## General settings ############################################################ +# This will include the config file settings from the default ckan config file +use = config:/srv/app/ckan.ini +ckan.devserver.watch_patterns = /srv/app/ckan.ini + +## Plugins Settings ############################################################ +ckan.plugins = image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat envvars + +## Resource Views Settings ##################################################### +ckan.views.default_views = text_view datatables_view pdf_view + +## Internationalisation Settings ############################################### +ckan.locale_default = en_AU +ckan.display_timezone = Australia/West + +## CKAN Extensions configuration ############################################### + + +## Logging configuration +[loggers] +keys = root, ckan, ckanext, werkzeug + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_werkzeug] +level = WARNING +handlers = console +qualname = werkzeug +propagate = 0 + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/ckan/config/uwsgi.ini b/ckan/config/uwsgi.ini new file mode 100644 index 00000000..b60f71e8 --- /dev/null +++ b/ckan/config/uwsgi.ini @@ -0,0 +1,16 @@ +[uwsgi] + +plugins= http,python +socket=/tmp/uwsgi.sock +wsgi-file=/srv/app/wsgi.py +module=wsgi:application +uid=92 +gid=92 +http=0.0.0.0:5000 +master=enable-threads +lazy-apps=true +processes=5 +disable-logging=true +buffer-size=32768 +vacuum=true +harakiri=50 \ No newline at end of file diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh new file mode 100644 index 00000000..eb5cb320 --- /dev/null +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +if [[ $CKAN__PLUGINS == *"xloader"* ]]; then + # Add ckan.xloader.api_token to the CKAN config file (updated with corrected value later) + echo "Setting a temporary value for ckanext.xloader.api_token" + ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add $CKAN_SYSADMIN_NAME xloader | tail -n 1 | tr -d '\t')" +fi + +## Examples of how to initialise DB for the extensions +# if [[ $CKAN__PLUGINS == *"archiver"* ]]; then +# ckan -c $CKAN_INI archiver init +# fi + +# if [[ $CKAN__PLUGINS == *"report"* ]]; then +# ckan -c $CKAN_INI report initdb +# fi + +# if [[ $CKAN__PLUGINS == *"harvest"* ]]; then +# ckan -c $CKAN_INI db upgrade -p harvest +# fi + +if [[ $CKAN__PLUGINS == *"pages"* ]]; then + ckan -c $CKAN_INI pages initdb +fi \ No newline at end of file diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs new file mode 100644 index 00000000..a4bec897 --- /dev/null +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -0,0 +1,3 @@ +# Crontab for CKAN cron jobs +# Example cron job runs the harvester run command every 15 mins +#*/15 * * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester run >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 \ No newline at end of file diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh new file mode 100644 index 00000000..9ffcbe63 --- /dev/null +++ b/ckan/setup/dbca_requirements.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +## Must Have ## +# Archiver +pip3 install -e 'git+https://github.com/ckan/ckanext-archiver.git@master#egg=ckanext-archiver' +pip3 install -r ${SRC_DIR}/ckanext-archiver/requirements.txt + +# DCAT +pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v1.5.1#egg=ckanext-dcat +pip3 install -r ${SRC_DIR}/ckanext-dcat/requirements.txt + +# Harvester +pip3 install -e 'git+https://github.com/ckan/ckanext-harvest.git@v1.5.6#egg=ckanext-harvest' +pip3 install -r ${SRC_DIR}/ckanext-harvest/requirements.txt + +# Hierarchy +pip3 install -e git+https://github.com/ckan/ckanext-hierarchy.git@v1.2.1#egg=ckanext-hierarchy +pip3 install -r ${SRC_DIR}/ckanext-hierarchy/requirements.txt + +# Pages +pip3 install -e git+https://github.com/ckan/ckanext-pages.git@v0.5.2#egg=ckanext-pages + +# PDF View +pip3 install -e git+https://github.com/ckan/ckanext-pdfview.git@0.0.8#egg=ckanext-pdfview + +# Report +pip3 install -e git+http://github.com/ckan/ckanext-report.git@master#egg=ckanext-report +pip3 install -r ${SRC_DIR}/ckanext-report/requirements.txt + +# Showcase +pip3 install -e git+https://github.com/ckan/ckanext-showcase.git@v1.6.1#egg=ckanext-showcase +pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt + +# Scheming +pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming' + +# Spatial +pip3 install -e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext-spatial +pip3 install -r ${SRC_DIR}/ckanext-spatial/requirements.txt + +# XLoader +pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@1.0.1#egg=ckanext-xloader' +pip3 install -r ${SRC_DIR}/ckanext-xloader/requirements.txt + +# 3rd Party # +# DOI +pip3 install -e git+https://github.com/NaturalHistoryMuseum/ckanext-doi@v3.1.9#egg=ckanext-doi \ No newline at end of file diff --git a/ckan/setup/dbca_start_ckan.sh.override b/ckan/setup/dbca_start_ckan.sh.override new file mode 100755 index 00000000..174cd06d --- /dev/null +++ b/ckan/setup/dbca_start_ckan.sh.override @@ -0,0 +1,46 @@ +#!/bin/bash + +# Update the default ckan config file +export CKAN_INI=$APP_DIR/ckan.ini + +# Set up the Secret key used by Beaker and Flask +# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var +if grep -qE "beaker.session.secret ?= ?$" ckan.ini +then + echo "Setting beaker.session.secret in ini file" + ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') + ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" + ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" +fi + +# Run the prerun script to init CKAN and create the default admin user +python3 prerun.py + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; bash "$f" ;; + *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + done +fi + +# Revert back touse DBCA config file +export CKAN_INI=$APP_DIR/config/dbca.ini +# Unset CKAN__PLUGINS to stop the extension envvars overriding the plugins value set in the dbca config file +unset CKAN__PLUGINS + +if [ $? -eq 0 ] +then + # Start supervisord + supervisord --configuration /etc/supervisord.conf & + # Start uwsgi + uwsgi -i $APP_DIR/config/uwsgi.ini +else + echo "[prerun] failed...not starting CKAN." +fi \ No newline at end of file diff --git a/ckan/setup/dbca_start_ckan_development.sh.override b/ckan/setup/dbca_start_ckan_development.sh.override new file mode 100644 index 00000000..75cce24e --- /dev/null +++ b/ckan/setup/dbca_start_ckan_development.sh.override @@ -0,0 +1,109 @@ +#!/bin/sh + +# Only run these start up scripts the first time the container is created +if [ ! -f /tmp/container_ready ]; then + # Install any local extensions in the src_extensions volume + echo "Looking for local extensions to install..." + echo "Extension dir contents:" + ls -la $SRC_EXTENSIONS_DIR + for i in $SRC_EXTENSIONS_DIR/* + do + if [ -d $i ]; + then + + if [ -f $i/pip-requirements.txt ]; + then + pip install -r $i/pip-requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/requirements.txt ]; + then + pip install -r $i/requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/dev-requirements.txt ]; + then + pip install -r $i/dev-requirements.txt + echo "Found dev-requirements file in $i" + fi + if [ -f $i/setup.py ]; + then + cd $i + python3 $i/setup.py develop + echo "Found setup.py file in $i" + cd $APP_DIR + fi + + # Point `use` in test.ini to location of `test-core.ini` + if [ -f $i/test.ini ]; + then + echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i" + ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" + fi + fi + done + # Update the default ckan config file + export CKAN_INI=$APP_DIR/ckan.ini + # Set debug to true + echo "Enabling debug mode" + ckan config-tool $CKAN_INI -s DEFAULT "debug = true" + + # Set up the Secret key used by Beaker and Flask + # This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var + if grep -E "beaker.session.secret ?= ?$" ckan.ini + then + echo "Setting beaker.session.secret in ini file" + ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') + ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" + ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" + fi + + # Update the plugins setting in the ini file with the values defined in the env var + echo "Loading the following plugins: $CKAN__PLUGINS" + ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" + + # Update test-core.ini DB, SOLR & Redis settings + echo "Loading test settings into test-core.ini" + ckan config-tool $SRC_DIR/ckan/test-core.ini \ + "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \ + "ckan.datastore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \ + "ckan.datastore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \ + "solr_url = $TEST_CKAN_SOLR_URL" \ + "ckan.redis.url = $TEST_CKAN_REDIS_URL" + + # Run the prerun script to init CKAN and create the default admin user + python3 prerun.py + + # Run any startup scripts provided by images extending this one + if [[ -d "/docker-entrypoint.d" ]] + then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; bash "$f" ;; + *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done + fi + + # Revert back touse DBCA config file + export CKAN_INI=$APP_DIR/config/dbca.ini + # Unset CKAN__PLUGINS to stop the extension envvars overriding the plugins value set in the dbca config file + unset CKAN__PLUGINS + # Set the container as ready so the startup scripts are not run again + touch /tmp/container_ready +fi + +# Start supervisord +supervisord --configuration /etc/supervisord.conf & + +# Start the development server as the ckan user with automatic reload +if [ "$USE_HTTPS_FOR_DEV" = true ] ; then + su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0 -C unsafe.cert -K unsafe.key" +else + su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0" +fi + diff --git a/ckan/supervisor/ckan_cron_jobs.conf b/ckan/supervisor/ckan_cron_jobs.conf new file mode 100644 index 00000000..d3268a40 --- /dev/null +++ b/ckan/supervisor/ckan_cron_jobs.conf @@ -0,0 +1,42 @@ +; ======================================================= +; Supervisor configuration for CKAN cron jobs +; ======================================================= + +; 1. Copy this file to /etc/supervisor/conf.d +; 2. Make sure the paths below match your setup + + +[program:ckan-cron-jobs] + +; Use the full paths to the virtualenv and your configuration file here. +command=/usr/sbin/crond -f -L /srv/app/logs/ckan-cron-jobs.log + + +; User the worker runs as. +user=root + + +; Start just a single worker. Increase this number if you have many or +; particularly long running background jobs. +numprocs=1 +process_name=%(program_name)s-%(process_num)02d + + +; Log files. +stdout_logfile=/srv/app/logs/ckan-crons-jobs.stdout.log +stderr_logfile=/srv/app/logs/ckan-crons-jobs.stderr.log + + +; Make sure that the worker is started on system start and automatically +; restarted if it crashes unexpectedly. +autostart=true +autorestart=true + + +; Number of seconds the process has to run before it is considered to have +; started successfully. +startsecs=10 + +; Need to wait for currently executing tasks to finish at shutdown. +; Increase this if you have very long running tasks. +stopwaitsecs = 600 diff --git a/ckan/supervisor/ckan_worker_default.conf b/ckan/supervisor/ckan_worker_default.conf new file mode 100644 index 00000000..f30bf16d --- /dev/null +++ b/ckan/supervisor/ckan_worker_default.conf @@ -0,0 +1,42 @@ +; ======================================================= +; Supervisor configuration for CKAN background job worker +; ======================================================= + +; 1. Copy this file to /etc/supervisor/conf.d +; 2. Make sure the paths below match your setup + + +[program:ckan-worker-default] + +; Use the full paths to the virtualenv and your configuration file here. +command=/usr/bin/ckan -c /srv/app/ckan.ini jobs worker + + +; User the worker runs as. +user=ckan + + +; Start just a single worker. Increase this number if you have many or +; particularly long running background jobs. +numprocs=1 +process_name=%(program_name)s-%(process_num)02d + + +; Log files. +stdout_logfile=/srv/app/logs/ckan-worker.stdout.log +stderr_logfile=/srv/app/logs/ckan-worker.stderr.log + + +; Make sure that the worker is started on system start and automatically +; restarted if it crashes unexpectedly. +autostart=true +autorestart=true + + +; Number of seconds the process has to run before it is considered to have +; started successfully. +startsecs=10 + +; Need to wait for currently executing tasks to finish at shutdown. +; Increase this if you have very long running tasks. +stopwaitsecs = 600 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index dc2ecdf9..99a3736a 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -13,6 +13,10 @@ services: dockerfile: Dockerfile.dev args: - TZ=${TZ} + platforms: + - linux/amd64 + image: ${COMPOSE_PROJECT_NAME}-ckan-dev + platform: linux/amd64 env_file: - .env depends_on: @@ -27,15 +31,42 @@ services: volumes: - ckan_storage:/var/lib/ckan - ./src:/srv/app/src_extensions + - ./ckan/config:/srv/app/config restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] + stdin_open: true + tty: true - datapusher: - image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION} + ckan-dev-worker: + build: + context: ckan/ + dockerfile: Dockerfile.worker + args: + - TZ=${TZ} + - CKAN_IMAGE=${COMPOSE_PROJECT_NAME}-ckan-dev + platforms: + - linux/amd64 + platform: linux/amd64 + env_file: + - .env + depends_on: + db: + condition: service_healthy + solr: + condition: service_healthy + redis: + condition: service_healthy + ckan-dev: + condition: service_started + volumes: + - ckan_storage:/var/lib/ckan + - ./src:/srv/app/src_extensions restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"] + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] + stdin_open: true + tty: true db: build: @@ -55,14 +86,19 @@ services: restart: unless-stopped healthcheck: test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] + ports: + - :5432 solr: image: ckan/ckan-solr:${SOLR_IMAGE_VERSION} + platform: linux/amd64 volumes: - solr_data:/var/solr restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"] + ports: + - :8983 redis: image: redis:${REDIS_VERSION} diff --git a/docker-compose.yml b/docker-compose.yml index 0f5330fb..0eaad072 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,8 @@ services: networks: - webnet - ckannet + env_file: + - .env depends_on: ckan: condition: service_healthy @@ -24,11 +26,15 @@ services: ckan: container_name: ${CKAN_CONTAINER_NAME} + image: ${COMPOSE_PROJECT_NAME}-${CKAN_CONTAINER_NAME} + platform: linux/amd64 build: context: ckan/ dockerfile: Dockerfile args: - TZ=${TZ} + platforms: + - linux/amd64 networks: - ckannet - dbnet @@ -49,15 +55,39 @@ services: healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] - datapusher: - container_name: ${DATAPUSHER_CONTAINER_NAME} + ckan-worker: + container_name: ${WORKER_CONTAINER_NAME} + image: ${COMPOSE_PROJECT_NAME}-${WORKER_CONTAINER_NAME} + platform: linux/amd64 + build: + context: ckan/ + dockerfile: Dockerfile.worker + args: + - TZ=${TZ} + - CKAN_IMAGE=${COMPOSE_PROJECT_NAME}-${CKAN_CONTAINER_NAME} + platforms: + - linux/amd64 networks: - ckannet - dbnet - image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION} + - solrnet + - redisnet + env_file: + - .env + depends_on: + db: + condition: service_healthy + solr: + condition: service_healthy + redis: + condition: service_healthy + ckan: + condition: service_started + volumes: + - ckan_storage:/var/lib/ckan restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"] + test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] db: container_name: ${POSTGRESQL_CONTAINER_NAME} @@ -86,6 +116,7 @@ services: networks: - solrnet image: ckan/ckan-solr:${SOLR_IMAGE_VERSION} + platform: linux/amd64 volumes: - solr_data:/var/solr restart: unless-stopped diff --git a/nginx/Dockerfile b/nginx/Dockerfile index eda7994e..05aeb380 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -9,6 +9,7 @@ RUN apk update --no-cache && \ COPY setup/nginx.conf ${NGINX_DIR}/nginx.conf COPY setup/index.html /usr/share/nginx/html/index.html COPY setup/default.conf ${NGINX_DIR}/conf.d/ +COPY setup/default.conf.template ${NGINX_DIR}/templates/ RUN mkdir -p ${NGINX_DIR}/certs diff --git a/nginx/setup/default.conf.template b/nginx/setup/default.conf.template new file mode 100644 index 00000000..fb23b91a --- /dev/null +++ b/nginx/setup/default.conf.template @@ -0,0 +1,46 @@ +server { + #listen 80; + #listen [::]:80; + listen 443 ssl; + listen [::]:443 ssl; + server_name localhost; + ssl_certificate /etc/nginx/certs/ckan-local.crt; + ssl_certificate_key /etc/nginx/certs/ckan-local.key; + + # TLS 1.2 & 1.3 only + ssl_protocols TLSv1.2 TLSv1.3; + + # Disable weak ciphers + ssl_prefer_server_ciphers on; + ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; + + # SSL sessions + ssl_session_timeout 1d; + # ssl_session_cache dfine in stream and http + ssl_session_tickets off; + + #access_log /var/log/nginx/host.access.log main; + + location / { + proxy_pass http://${CKAN_CONTAINER_NAME:-ckan}:${CKAN_PORT:-5000}/; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Host $host; + #proxy_cache cache; + proxy_cache_bypass $cookie_auth_tkt; + proxy_no_cache $cookie_auth_tkt; + proxy_cache_valid 30m; + proxy_cache_key $host$scheme$proxy_host$request_uri; + } + + error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 421 422 423 424 425 426 428 429 431 451 500 501 502 503 504 505 506 507 508 510 511 /error.html; + + # redirect server error pages to the static page /error.html + # + location = /error.html { + ssi on; + internal; + auth_basic off; + root /usr/share/nginx/html; + } + +} \ No newline at end of file diff --git a/postgresql/Dockerfile b/postgresql/Dockerfile index 4caf7765..64f5e262 100755 --- a/postgresql/Dockerfile +++ b/postgresql/Dockerfile @@ -1,4 +1,4 @@ -FROM postgres:12-alpine +FROM postgres:16-alpine # Include extra setup scripts (eg datastore) COPY --chown=postgres:postgres docker-entrypoint-initdb.d /docker-entrypoint-initdb.d diff --git a/src/dbca_install_extensions.sh b/src/dbca_install_extensions.sh new file mode 100644 index 00000000..3538b477 --- /dev/null +++ b/src/dbca_install_extensions.sh @@ -0,0 +1,27 @@ + +#!/bin/sh + +### Extensions that need upgrading to be compatiable with CKAN 2.10 ### +# Uncomment the following lines to install these extension you are working on to upgrade to CKAN 2.10 + +cd src/ + +## Must Have ## +# QA +# git clone https://github.com/dbca-wa/ckanext-qa.git +# These extensions will be installed by default, but we don't want them +# sed -i".$(date +%Y%m%d_%H%M%S).bak" -e '/ckanext-report/d' -e '/ckanext-archiver/d' ckanext-qa/dev-requirements.txt +# Office Docs +# git clone https://github.com/dbca-wa/ckanext-officedocs + +## Should have ## +# Geopusher +# git clone https://github.com/dbca-wa/ckanext-geopusher.git + +## Could have ## +# Cesium Preview +# git clone https://github.com/dbca-wa/ckanext-cesiumpreview.git +# Featured Views +# git clone https://github.com/dbca-wa/ckanext-featuredviews + +echo "Ready to build project: ahoy build"