From b1660438b7b85e5f144731916083fedeb967a4c0 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 14 May 2024 10:23:48 +0200 Subject: [PATCH 1/2] Remove *.override scripts and update README.md --- README.md | 4 +- ckan/setup/README.md | 5 + ckan/setup/prerun.py.override | 212 ------------------ ckan/setup/start_ckan.sh.override | 57 ----- ckan/setup/start_ckan_development.sh.override | 99 -------- 5 files changed, 8 insertions(+), 369 deletions(-) create mode 100644 ckan/setup/README.md delete mode 100644 ckan/setup/prerun.py.override delete mode 100755 ckan/setup/start_ckan.sh.override delete mode 100755 ckan/setup/start_ckan_development.sh.override diff --git a/README.md b/README.md index 7e4fde87..6a2651b2 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,9 @@ The Docker image config files used to build your CKAN project are located in the * CKAN is started running this: `/usr/bin/ckan -c /srv/app/ckan.ini run -H 0.0.0.0`. * Make sure to add the local plugins to the `CKAN__PLUGINS` env var in the `.env` file. -* Any custom changes to the scripts run during container start up can be made to scripts in the `setup/` directory. For instance if you wanted to change the port on which CKAN runs you would need to make changes to the Docker Compose yaml file, and the `start_ckan.sh.override` file. Then you would need to add the following line to the Dockerfile ie: `COPY setup/start_ckan.sh.override ${APP_DIR}/start_ckan.sh`. The `start_ckan.sh` file in the locally built image would override the `start_ckan.sh` file included in the base image +### Overriding the setup/* files derived from the CKAN base image + +Any deviations or custom changes to the scripts run during container startup can be made by overriding those scripts in the `ckan/◊setup/` directory. For instance if you wanted to change the port on which CKAN runs you would need to make changes to the Docker Compose yaml file, plus the `start_ckan.sh` file. The `start_ckan.sh` file in the locally built image will override the `start_ckan.sh` file included in the base image. You would also need to include a `COPY setup/start_ckan.sh ${APP_DIR}` command in the relevant Dockerfile to copy the (override) script to the newly-built local image ### Extending the base images diff --git a/ckan/setup/README.md b/ckan/setup/README.md new file mode 100644 index 00000000..8ef69da7 --- /dev/null +++ b/ckan/setup/README.md @@ -0,0 +1,5 @@ +Create scripts in this folder to override the scripts inherited from the CKAN base images. +These scripts would most likely be one of: +1. `start_ckan.sh` +2. `start_ckan_development.sh` +3. `prerun.py` diff --git a/ckan/setup/prerun.py.override b/ckan/setup/prerun.py.override deleted file mode 100644 index 3d686969..00000000 --- a/ckan/setup/prerun.py.override +++ /dev/null @@ -1,212 +0,0 @@ -import os -import sys -import subprocess -import psycopg2 -try: - from urllib.request import urlopen - from urllib.error import URLError -except ImportError: - from urllib2 import urlopen - from urllib2 import URLError - -import time -import re -import json - -ckan_ini = os.environ.get("CKAN_INI", "/srv/app/ckan.ini") - -RETRY = 5 - - -def update_plugins(): - - plugins = os.environ.get("CKAN__PLUGINS", "") - print(("[prerun] Setting the following plugins in {}:".format(ckan_ini))) - print(plugins) - cmd = ["ckan", "config-tool", ckan_ini, "ckan.plugins = {}".format(plugins)] - subprocess.check_output(cmd, stderr=subprocess.STDOUT) - print("[prerun] Plugins set.") - - -def check_main_db_connection(retry=None): - - conn_str = os.environ.get("CKAN_SQLALCHEMY_URL") - if not conn_str: - print("[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db") - return check_db_connection(conn_str, retry) - - -def check_datastore_db_connection(retry=None): - - conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") - if not conn_str: - print("[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db") - return check_db_connection(conn_str, retry) - - -def check_db_connection(conn_str, retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print("[prerun] Giving up after 5 tries...") - sys.exit(1) - - try: - connection = psycopg2.connect(conn_str) - - except psycopg2.Error as e: - print(str(e)) - print("[prerun] Unable to connect to the database, waiting...") - time.sleep(10) - check_db_connection(conn_str, retry=retry - 1) - else: - connection.close() - - -def check_solr_connection(retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print("[prerun] Giving up after 5 tries...") - sys.exit(1) - - url = os.environ.get("CKAN_SOLR_URL", "") - search_url = '{url}/schema/name?wt=json'.format(url=url) - - try: - connection = urlopen(search_url) - except URLError as e: - print(str(e)) - print("[prerun] Unable to connect to solr, waiting...") - time.sleep(10) - check_solr_connection(retry=retry - 1) - else: - import re - conn_info = connection.read() - schema_name = json.loads(conn_info) - if 'ckan' in schema_name['name']: - print('[prerun] Succesfully connected to solr and CKAN schema loaded') - else: - print('[prerun] Succesfully connected to solr, but CKAN schema not found') - - -def init_db(): - - db_command = ["ckan", "-c", ckan_ini, "db", "init"] - print("[prerun] Initializing or upgrading db - start") - try: - subprocess.check_output(db_command, stderr=subprocess.STDOUT) - print("[prerun] Initializing or upgrading db - end") - except subprocess.CalledProcessError as e: - if "OperationalError" in e.output: - print(e.output) - print("[prerun] Database not ready, waiting a bit before exit...") - time.sleep(5) - sys.exit(1) - else: - print(e.output) - raise e - - -def init_datastore_db(): - - conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") - if not conn_str: - print("[prerun] Skipping datastore initialization") - return - - datastore_perms_command = ["ckan", "-c", ckan_ini, "datastore", "set-permissions"] - - connection = psycopg2.connect(conn_str) - cursor = connection.cursor() - - print("[prerun] Initializing datastore db - start") - try: - datastore_perms = subprocess.Popen( - datastore_perms_command, stdout=subprocess.PIPE - ) - - perms_sql = datastore_perms.stdout.read() - # Remove internal pg command as psycopg2 does not like it - perms_sql = re.sub(b'\\\\connect "(.*)"', b"", perms_sql) - cursor.execute(perms_sql) - for notice in connection.notices: - print(notice) - - connection.commit() - - print("[prerun] Initializing datastore db - end") - print(datastore_perms.stdout.read()) - except psycopg2.Error as e: - print("[prerun] Could not initialize datastore") - print(str(e)) - - except subprocess.CalledProcessError as e: - if "OperationalError" in e.output: - print(e.output) - print("[prerun] Database not ready, waiting a bit before exit...") - time.sleep(5) - sys.exit(1) - else: - print(e.output) - raise e - finally: - cursor.close() - connection.close() - - -def create_sysadmin(): - - name = os.environ.get("CKAN_SYSADMIN_NAME") - password = os.environ.get("CKAN_SYSADMIN_PASSWORD") - email = os.environ.get("CKAN_SYSADMIN_EMAIL") - - if name and password and email: - - # Check if user exists - command = ["ckan", "-c", ckan_ini, "user", "show", name] - - out = subprocess.check_output(command) - if b"User:None" not in re.sub(b"\s", b"", out): - print("[prerun] Sysadmin user exists, skipping creation") - return - - # Create user - command = [ - "ckan", - "-c", - ckan_ini, - "user", - "add", - name, - "password=" + password, - "email=" + email, - ] - - subprocess.call(command) - print("[prerun] Created user {0}".format(name)) - - # Make it sysadmin - command = ["ckan", "-c", ckan_ini, "sysadmin", "add", name] - - subprocess.call(command) - print("[prerun] Made user {0} a sysadmin".format(name)) - - -if __name__ == "__main__": - - maintenance = os.environ.get("MAINTENANCE_MODE", "").lower() == "true" - - if maintenance: - print("[prerun] Maintenance mode, skipping setup...") - else: - check_main_db_connection() - init_db() - update_plugins() - check_datastore_db_connection() - init_datastore_db() - check_solr_connection() - create_sysadmin() - \ No newline at end of file diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override deleted file mode 100755 index 0c8409c9..00000000 --- a/ckan/setup/start_ckan.sh.override +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh - -# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later) -ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx - -# Set up the Secret key used by Beaker and Flask -# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var -if grep -E "beaker.session.secret ?= ?$" ckan.ini -then - echo "Setting beaker.session.secret in ini file" - ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" - ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" - JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') - ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" - ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" -fi - -# Run the prerun script to init CKAN and create the default admin user -sudo -u ckan -EH python3 prerun.py - -echo "Set up ckan.datapusher.api_token in the CKAN config file" -ckan config-tool $CKAN_INI "ckan.datapusher.api_token=$(ckan -c $CKAN_INI user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')" - -# Run any startup scripts provided by images extending this one -if [[ -d "/docker-entrypoint.d" ]] -then - for f in /docker-entrypoint.d/*; do - case "$f" in - *.sh) echo "$0: Running init file $f"; . "$f" ;; - *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; - *) echo "$0: Ignoring $f (not an sh or py file)" ;; - esac - echo - done -fi - -# Set the common uwsgi options -UWSGI_OPTS="--plugins http,python \ - --socket /tmp/uwsgi.sock \ - --wsgi-file /srv/app/wsgi.py \ - --module wsgi:application \ - --uid 92 --gid 92 \ - --http 0.0.0.0:5000 \ - --master --enable-threads \ - --lazy-apps \ - -p 2 -L -b 32768 --vacuum \ - --harakiri $UWSGI_HARAKIRI" - -if [ $? -eq 0 ] -then - # Start supervisord - supervisord --configuration /etc/supervisord.conf & - # Start uwsgi - sudo -u ckan -EH uwsgi $UWSGI_OPTS -else - echo "[prerun] failed...not starting CKAN." -fi diff --git a/ckan/setup/start_ckan_development.sh.override b/ckan/setup/start_ckan_development.sh.override deleted file mode 100755 index 93ce8141..00000000 --- a/ckan/setup/start_ckan_development.sh.override +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/sh - -# Install any local extensions in the src_extensions volume -echo "Looking for local extensions to install..." -echo "Extension dir contents:" -ls -la $SRC_EXTENSIONS_DIR -for i in $SRC_EXTENSIONS_DIR/* -do - if [ -d $i ]; - then - - if [ -f $i/pip-requirements.txt ]; - then - pip install -r $i/pip-requirements.txt - echo "Found requirements file in $i" - fi - if [ -f $i/requirements.txt ]; - then - pip install -r $i/requirements.txt - echo "Found requirements file in $i" - fi - if [ -f $i/dev-requirements.txt ]; - then - pip install -r $i/dev-requirements.txt - echo "Found dev-requirements file in $i" - fi - if [ -f $i/setup.py ]; - then - cd $i - python3 $i/setup.py develop - echo "Found setup.py file in $i" - cd $APP_DIR - fi - - # Point `use` in test.ini to location of `test-core.ini` - if [ -f $i/test.ini ]; - then - echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i" - ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" - fi - fi -done - -# Set debug to true -echo "Enabling debug mode" -ckan config-tool $CKAN_INI -s DEFAULT "debug = true" - -# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later) -ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx - -# Set up the Secret key used by Beaker and Flask -# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var -if grep -E "beaker.session.secret ?= ?$" ckan.ini -then - echo "Setting beaker.session.secret in ini file" - ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" - ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" - JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') - ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" - ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" -fi - -# Update the plugins setting in the ini file with the values defined in the env var -echo "Loading the following plugins: $CKAN__PLUGINS" -ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" - -# Update test-core.ini DB, SOLR & Redis settings -echo "Loading test settings into test-core.ini" -ckan config-tool $SRC_DIR/ckan/test-core.ini \ - "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \ - "ckan.datastore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \ - "ckan.datastore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \ - "solr_url = $TEST_CKAN_SOLR_URL" \ - "ckan.redis.url = $TEST_CKAN_REDIS_URL" - -# Run the prerun script to init CKAN and create the default admin user -sudo -u ckan -EH python3 prerun.py - -echo "Set up ckan.datapusher.api_token in the CKAN config file" -ckan config-tool $CKAN_INI "ckan.datapusher.api_token=$(ckan -c $CKAN_INI user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')" - -# Run any startup scripts provided by images extending this one -if [[ -d "/docker-entrypoint.d" ]] -then - for f in /docker-entrypoint.d/*; do - case "$f" in - *.sh) echo "$0: Running init file $f"; . "$f" ;; - *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; - *) echo "$0: Ignoring $f (not an sh or py file)" ;; - esac - echo - done -fi - -# Start supervisord -supervisord --configuration /etc/supervisord.conf & - -# Start the development server with automatic reload -sudo -u ckan -EH ckan -c $CKAN_INI run -H 0.0.0.0 \ No newline at end of file From 329f24045fa1b3ceee474de5e5383ee3478e8a0d Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 14 May 2024 10:29:58 +0200 Subject: [PATCH 2/2] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6a2651b2..4fe715f0 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ The Docker image config files used to build your CKAN project are located in the ### Overriding the setup/* files derived from the CKAN base image -Any deviations or custom changes to the scripts run during container startup can be made by overriding those scripts in the `ckan/◊setup/` directory. For instance if you wanted to change the port on which CKAN runs you would need to make changes to the Docker Compose yaml file, plus the `start_ckan.sh` file. The `start_ckan.sh` file in the locally built image will override the `start_ckan.sh` file included in the base image. You would also need to include a `COPY setup/start_ckan.sh ${APP_DIR}` command in the relevant Dockerfile to copy the (override) script to the newly-built local image +Any deviations or custom changes to the scripts run during container startup can be made by overriding those scripts in the `ckan/setup/` directory. For instance if you wanted to change the port on which CKAN runs you would need to make changes to the Docker Compose yaml file, plus the `start_ckan.sh` file. The `start_ckan.sh` file in the locally built image will override the `start_ckan.sh` file included in the base image. You would also need to include a `COPY setup/start_ckan.sh ${APP_DIR}` command in the relevant Dockerfile to copy the (override) script to the newly-built local image ### Extending the base images