From ec6f797bdcaa9845d1011abfb7b0da95ce4a1a8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Thu, 21 Nov 2024 09:46:45 +0100 Subject: [PATCH 1/7] update to 24.2 --- galaxy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 0ff33291..bbf258ab 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -15,7 +15,7 @@ ENV GALAXY_ROOT_DIR=/galaxy \ GALAXY_VIRTUAL_ENV=/galaxy_venv \ GALAXY_HOME=/home/galaxy \ GALAXY_CONDA_PREFIX=/tool_deps/_conda \ - MINIFORGE_VERSION=24.3.0-0 + MINIFORGE_VERSION=24.9.2-0 # Install miniforge and then virtualenv from conda @@ -39,7 +39,7 @@ RUN curl -s -L https://github.com/conda-forge/miniforge/releases/download/$MINIF FROM build_base AS build_galaxy -ARG GALAXY_RELEASE=release_24.1 +ARG GALAXY_RELEASE=release_24.2 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy ENV NODE_OPTIONS=--max-old-space-size=4096 From 7929327ee489c2d29b6f0602457eed1879d1a99d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 24 Nov 2024 22:13:16 +0100 Subject: [PATCH 2/7] add file-source templates, add object-store templates, activate vault --- .../ansible/galaxy_file_source_templates.yml | 11 ++++++++ .../ansible/galaxy_object_store_templates.yml | 6 +++++ galaxy/ansible/galaxy_vault_config.yml | 12 +++++++++ galaxy/ansible/group_vars/all.yml | 14 +++++++--- galaxy/ansible/provision.yml | 13 +++++++++ .../templates/file_source_templates.yml.j2 | 10 +++++++ .../templates/object_store_templates.yml.j2 | 8 ++++++ .../ansible/templates/production_b2drop.yml | 27 +++++++++++++++++++ galaxy/ansible/templates/vault_conf.yml.j2 | 11 ++++++++ 9 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 galaxy/ansible/galaxy_file_source_templates.yml create mode 100644 galaxy/ansible/galaxy_object_store_templates.yml create mode 100644 galaxy/ansible/galaxy_vault_config.yml create mode 100644 galaxy/ansible/templates/file_source_templates.yml.j2 create mode 100644 galaxy/ansible/templates/object_store_templates.yml.j2 create mode 100644 galaxy/ansible/templates/production_b2drop.yml create mode 100644 galaxy/ansible/templates/vault_conf.yml.j2 diff --git a/galaxy/ansible/galaxy_file_source_templates.yml b/galaxy/ansible/galaxy_file_source_templates.yml new file mode 100644 index 00000000..be322121 --- /dev/null +++ b/galaxy/ansible/galaxy_file_source_templates.yml @@ -0,0 +1,11 @@ +- hosts: localhost + connection: local + remote_user: root + tasks: + - name: "Setup user configurable file source templates, also called BYOD" + template: src=file_source_templates.yml.j2 dest={{ galaxy_file_source_templates_config_file }} owner={{ galaxy_user_name }} group={{ galaxy_user_name }} + + - name: "Copy B2Drop file source template" + template: src=production_b2drop.yml dest=/galaxy/lib/galaxy/files/templates/examples/production_b2drop.yml owner={{ galaxy_user_name }} group={{ galaxy_user_name }} + + diff --git a/galaxy/ansible/galaxy_object_store_templates.yml b/galaxy/ansible/galaxy_object_store_templates.yml new file mode 100644 index 00000000..e616aca0 --- /dev/null +++ b/galaxy/ansible/galaxy_object_store_templates.yml @@ -0,0 +1,6 @@ +- hosts: localhost + connection: local + remote_user: root + tasks: + - name: "Setup user configurable object store templates, also called BYOS" + template: src=object_store_templates.yml.j2 dest={{ galaxy_object_store_templates_config_file }} owner={{ galaxy_user_name }} group={{ galaxy_user_name }} diff --git a/galaxy/ansible/galaxy_vault_config.yml b/galaxy/ansible/galaxy_vault_config.yml new file mode 100644 index 00000000..d5c26c82 --- /dev/null +++ b/galaxy/ansible/galaxy_vault_config.yml @@ -0,0 +1,12 @@ +- hosts: localhost + connection: local + remote_user: root +# You should change this key in production. You can generate Fernet keys with: +#from cryptography.fernet import Fernet +#Fernet.generate_key().decode('utf-8') + vars: + galaxy_vault_encryption_keys: + - pwiL08wXlpkBm-_Dr75aw1_uOPVA3HET1y7xrpynhKU= + tasks: + - name: "Configure Galaxy vault" + template: src=vault_conf.yml.j2 dest={{ galaxy_vault_config_file }} owner={{ galaxy_user_name }} group={{ galaxy_user_name }} diff --git a/galaxy/ansible/group_vars/all.yml b/galaxy/ansible/group_vars/all.yml index 436c08de..7c50e703 100644 --- a/galaxy/ansible/group_vars/all.yml +++ b/galaxy/ansible/group_vars/all.yml @@ -16,6 +16,9 @@ galaxy_pbs: false galaxy_k8s_jobs: false galaxy_supervisor: true galaxy_job_metrics: true +galaxy_file_source_templates: true +galaxy_object_store_templates: true +galaxy_vault_config: true galaxy_scripts: true galaxy_domain: "localhost" # This is used by letsencrypt and Interactive Tools, set it to the domain name under which galaxy can be reached galaxy_startup: true @@ -59,9 +62,13 @@ galaxy_k8s_jobs_pull_policy: IfNotPresent # Point at the existing Galaxy configuration. galaxy_server_dir: "/galaxy" -galaxy_job_conf_path: "{{ galaxy_server_dir }}/config/job_conf.xml" -galaxy_container_resolvers_conf_path: "{{ galaxy_server_dir }}/config/container_resolvers_conf.yml" -galaxy_job_metrics_conf_path: "{{ galaxy_server_dir }}/config/job_metrics_conf.yml" +galaxy_config_dir: "{{ galaxy_server_dir }}/config" +galaxy_job_conf_path: "{{ galaxy_config_dir }}/job_conf.xml" +galaxy_container_resolvers_conf_path: "{{ galaxy_config_dir }}/container_resolvers_conf.yml" +galaxy_job_metrics_conf_path: "{{ galaxy_config_dir }}/job_metrics_conf.yml" +galaxy_file_source_templates_config_file: "{{ galaxy_config_dir }}/file_source_templates.yml" +galaxy_object_store_templates_config_file: "{{ galaxy_config_dir }}/object_store_templates.yml" +galaxy_vault_config_file: "{{ galaxy_config_dir }}/vault_conf.yml" galaxy_user_name: "galaxy" galaxy_home_dir: "/home/{{ galaxy_user_name }}" galaxy_source_shellrc: false @@ -69,7 +76,6 @@ galaxy_user_shellrc: "{{ galaxy_home_dir }}/.bashrc" galaxy_logs_dir: "{{ galaxy_home_dir }}/" galaxy_venv_dir: "{{ galaxy_home_dir }}/venv" -galaxy_config_dir: "{{ galaxy_server_dir }}/config" galaxy_config_file: "{{ galaxy_config_dir }}/galaxy.yml" galaxy_toolshed_config_file: "{{ galaxy_config_dir }}/tool_shed.yml" galaxy_tool_data_table_config_file: "{{ galaxy_config_dir }}/tool_data_table_conf.xml" diff --git a/galaxy/ansible/provision.yml b/galaxy/ansible/provision.yml index e4ba63e2..ebd71983 100644 --- a/galaxy/ansible/provision.yml +++ b/galaxy/ansible/provision.yml @@ -70,3 +70,16 @@ - import_playbook: galaxy_job_metrics.yml when: galaxy_job_metrics | bool tags: galaxy_job_metrics + +- import_playbook: galaxy_file_source_templates.yml + when: galaxy_file_source_templates | bool + tags: galaxy_file_source_templates + +- import_playbook: galaxy_object_store_templates.yml + when: galaxy_object_store_templates | bool + tags: galaxy_object_store_templates + +- import_playbook: galaxy_vault_config.yml + when: galaxy_vault_config | bool + tags: galaxy_vault_config + diff --git a/galaxy/ansible/templates/file_source_templates.yml.j2 b/galaxy/ansible/templates/file_source_templates.yml.j2 new file mode 100644 index 00000000..3e6b4607 --- /dev/null +++ b/galaxy/ansible/templates/file_source_templates.yml.j2 @@ -0,0 +1,10 @@ +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_azure.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_ftp.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_s3fs.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_aws_private_bucket.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_aws_public_bucket.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_azure.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_dropbox.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_google_drive.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_webdav.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_b2drop.yml" diff --git a/galaxy/ansible/templates/object_store_templates.yml.j2 b/galaxy/ansible/templates/object_store_templates.yml.j2 new file mode 100644 index 00000000..70732cbc --- /dev/null +++ b/galaxy/ansible/templates/object_store_templates.yml.j2 @@ -0,0 +1,8 @@ +# This is a catalog file for all the user object store templates +- include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/production_azure_blob.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/production_aws_s3.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/production_generic_s3.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/production_gcp_s3.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/cloudflare.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/onedata.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/minio_just_buckets.yml" diff --git a/galaxy/ansible/templates/production_b2drop.yml b/galaxy/ansible/templates/production_b2drop.yml new file mode 100644 index 00000000..dadd2c2a --- /dev/null +++ b/galaxy/ansible/templates/production_b2drop.yml @@ -0,0 +1,27 @@ +- id: webdav + version: 0 + name: B2DROP + description: | + B2DROP is a Nextcloud to sync and share your research data. + variables: + username: + label: Username / Account Name + type: string + help: | + The username or account name to use to connect to B2DROP. This is not your email address but the name show in the URL of your profile page. + writable: + label: Writable? + type: boolean + default: false + help: Allow Galaxy to write data to B2DROP. + secrets: + password: + label: Password + help: | + The password to use to connect to B2DROP. + configuration: + type: webdav + url: 'https://b2drop.bsc.es' + root: '/remote.php/dav/files/{{ variables.username }}' + writable: '{{ variables.writable }}' + password: '{{ secrets.password }}' diff --git a/galaxy/ansible/templates/vault_conf.yml.j2 b/galaxy/ansible/templates/vault_conf.yml.j2 new file mode 100644 index 00000000..0972837d --- /dev/null +++ b/galaxy/ansible/templates/vault_conf.yml.j2 @@ -0,0 +1,11 @@ +type: database +path_prefix: /galaxy +# Encryption keys must be valid fernet keys +# To generate a valid key: +# +# Use the ascii string value as a key +# For more details, see: https://cryptography.io/en/latest/fernet/# +encryption_keys: +{% for encryption_key in galaxy_vault_encryption_keys %} + - {{ encryption_key }} +{% endfor %} From 0c3c648fbc84b91b172224a30edfd3991c04a7ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 24 Nov 2024 22:14:01 +0100 Subject: [PATCH 3/7] add file-source templates, add object-store templates, activate vault --- galaxy/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index bbf258ab..fc215c5d 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -191,6 +191,8 @@ ENV GALAXY_CONFIG_MANAGED_CONFIG_DIR=$EXPORT_DIR/${GALAXY_ROOT_DIR#/}/database/c GALAXY_CONFIG_BOOTSTRAP_ADMIN_API_KEY=HSNiugRFvgT574F43jZ7N9F3 \ GALAXY_CONFIG_BRAND="Galaxy Docker Build" \ GALAXY_CONFIG_STATIC_ENABLED=False \ + GALAXY_CONFIG_FILE_SOURCE_TEMPPLATES=$GALAXY_CONFIG_DIR/file_source_templates.yml \ + GALAXY_CONFIG_VAULT_CONFIG_FILE=$GALAXY_CONFIG_DIR/vault_conf.yml \ GALAXY_INTERACTIVE_TOOLS_CONFIG_FILE=$GALAXY_CONFIG_DIR/tool_conf_interactive.xml \ # The following ENV var can be used to set the number of gunicorn workers GUNICORN_WORKERS=2 \ From add982e6436d90fcd8b16bcc3565427ffd569950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Mon, 25 Nov 2024 13:26:47 +0100 Subject: [PATCH 4/7] add secret --- galaxy/ansible/templates/production_b2drop.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy/ansible/templates/production_b2drop.yml b/galaxy/ansible/templates/production_b2drop.yml index dadd2c2a..9297acd8 100644 --- a/galaxy/ansible/templates/production_b2drop.yml +++ b/galaxy/ansible/templates/production_b2drop.yml @@ -23,5 +23,6 @@ type: webdav url: 'https://b2drop.bsc.es' root: '/remote.php/dav/files/{{ variables.username }}' + login: '{{ variables.username }}' writable: '{{ variables.writable }}' password: '{{ secrets.password }}' From 5333efae642447c648be349f6c95d66614eb0939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 26 Nov 2024 22:55:24 +0100 Subject: [PATCH 5/7] move around files --- .../production_b2drop.yml | 2 +- .../ansible/galaxy_file_source_templates.yml | 20 ++++++++++++++++--- galaxy/ansible/rabbitmq.yml | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) rename galaxy/ansible/{templates => files}/production_b2drop.yml (94%) diff --git a/galaxy/ansible/templates/production_b2drop.yml b/galaxy/ansible/files/production_b2drop.yml similarity index 94% rename from galaxy/ansible/templates/production_b2drop.yml rename to galaxy/ansible/files/production_b2drop.yml index 9297acd8..1165bfc0 100644 --- a/galaxy/ansible/templates/production_b2drop.yml +++ b/galaxy/ansible/files/production_b2drop.yml @@ -18,7 +18,7 @@ password: label: Password help: | - The password to use to connect to B2DROP. + The password to use to connect to B2DROP. configuration: type: webdav url: 'https://b2drop.bsc.es' diff --git a/galaxy/ansible/galaxy_file_source_templates.yml b/galaxy/ansible/galaxy_file_source_templates.yml index be322121..5a022e4a 100644 --- a/galaxy/ansible/galaxy_file_source_templates.yml +++ b/galaxy/ansible/galaxy_file_source_templates.yml @@ -2,10 +2,24 @@ connection: local remote_user: root tasks: + - name: Install fs.webdavfs for Galaxy's file source plugins + pip: + name: "fs.webdavfs" + extra_args: "--index-url https://wheels.galaxyproject.org/simple/ --extra-index-url https://pypi.python.org/simple" + virtualenv: "{{ galaxy_venv_dir }}" + environment: + PYTHOPATH: null + VIRTUAL_ENV: "{{ galaxy_venv_dir }}" + become_user: "{{ galaxy_user_name }}" + - name: "Setup user configurable file source templates, also called BYOD" template: src=file_source_templates.yml.j2 dest={{ galaxy_file_source_templates_config_file }} owner={{ galaxy_user_name }} group={{ galaxy_user_name }} - name: "Copy B2Drop file source template" - template: src=production_b2drop.yml dest=/galaxy/lib/galaxy/files/templates/examples/production_b2drop.yml owner={{ galaxy_user_name }} group={{ galaxy_user_name }} - - + ansible.builtin.copy: + src: "./ansible/files/production_b2drop.yml" + dest: "/galaxy/lib/galaxy/files/templates/examples/production_b2drop.yml" + owner: "{{ galaxy_user_name }}" + group: "{{ galaxy_user_name }}" + mode: '0644' + diff --git a/galaxy/ansible/rabbitmq.yml b/galaxy/ansible/rabbitmq.yml index 93a003a5..5e556e89 100644 --- a/galaxy/ansible/rabbitmq.yml +++ b/galaxy/ansible/rabbitmq.yml @@ -2,7 +2,7 @@ connection: local remote_user: root vars: - rabbitmq_version: 4.0.3 + rabbitmq_version: 4.0.4 rabbitmq_apt_repository: "deb [signed-by=/etc/apt/trusted.gpg.d/rabbitmq-9F4587F226208342.gpg] https://ppa1.rabbitmq.com/rabbitmq/rabbitmq-server/deb/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} main" rabbitmq_apt_gpg_url: https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-server.9F4587F226208342.key erlang_apt_repository: "deb [signed-by=/etc/apt/trusted.gpg.d/erlang-E495BB49CC4BBE5B.gpg] https://ppa1.rabbitmq.com/rabbitmq/rabbitmq-erlang/deb/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} main" From 239a991d7e0ff760c7b782605c8a9c941101081f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 26 Nov 2024 23:30:49 +0100 Subject: [PATCH 6/7] fix path --- galaxy/ansible/galaxy_file_source_templates.yml | 4 ++-- galaxy/ansible/templates/file_source_templates.yml.j2 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/galaxy/ansible/galaxy_file_source_templates.yml b/galaxy/ansible/galaxy_file_source_templates.yml index 5a022e4a..4675b130 100644 --- a/galaxy/ansible/galaxy_file_source_templates.yml +++ b/galaxy/ansible/galaxy_file_source_templates.yml @@ -17,8 +17,8 @@ - name: "Copy B2Drop file source template" ansible.builtin.copy: - src: "./ansible/files/production_b2drop.yml" - dest: "/galaxy/lib/galaxy/files/templates/examples/production_b2drop.yml" + src: "./files/production_b2drop.yml" + dest: "{{ galaxy_config_dir }}/production_b2drop.yml" owner: "{{ galaxy_user_name }}" group: "{{ galaxy_user_name }}" mode: '0644' diff --git a/galaxy/ansible/templates/file_source_templates.yml.j2 b/galaxy/ansible/templates/file_source_templates.yml.j2 index 3e6b4607..0878cf6a 100644 --- a/galaxy/ansible/templates/file_source_templates.yml.j2 +++ b/galaxy/ansible/templates/file_source_templates.yml.j2 @@ -7,4 +7,4 @@ - include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_dropbox.yml" - include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_google_drive.yml" - include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_webdav.yml" -- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_b2drop.yml" +- include: "{{ galaxy_config_dir }}/production_b2drop.yml" From a8141ca0ef87fc2df24f04f1b58342a8b18b7648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 26 Nov 2024 23:35:56 +0100 Subject: [PATCH 7/7] change ID --- galaxy/ansible/files/production_b2drop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/ansible/files/production_b2drop.yml b/galaxy/ansible/files/production_b2drop.yml index 1165bfc0..15aa97f3 100644 --- a/galaxy/ansible/files/production_b2drop.yml +++ b/galaxy/ansible/files/production_b2drop.yml @@ -1,4 +1,4 @@ -- id: webdav +- id: b2drop version: 0 name: B2DROP description: |