added multi-stage build in Dockerfile

README.md

@@ -251,7 +251,7 @@ With this method, you keep a backup in case you decide to downgrade, but require
 
   ```
   $ sudo rsync -var /data/galaxy-data-old/tool_deps/* /data/galaxy-data/tool_deps/
-  $ sudo rsync -var /data/galaxy-data-old/shed_tools/* /data/galaxy-data/shed_tools/
+  $ sudo rsync -var /data/galaxy-data-old/galaxy-central/database/shed_tools/* /data/galaxy-data/galaxy-central/database/shed_tools/
   ```
 10. Copy the welcome page and all its files.
 

galaxy/ansible/group_vars/all.yml

@@ -15,7 +15,6 @@ galaxy_condor: true
 galaxy_pbs: false
 galaxy_k8s_jobs: false
 galaxy_supervisor: true
-galaxy_root: true
 galaxy_job_metrics: true
 galaxy_scripts: true
 galaxy_domain: "localhost"  # This is used by letsencrypt and Interactive Tools, set it to the domain name under which galaxy can be reached
@@ -202,7 +201,6 @@ nginx_proxy_flower: true
 nginx_proxy_interactive_tools: true
 
 # Certbot Configuration.
-certbot_dir: "{{ galaxy_venv_dir }}"
 certbot_auto_renew_hour: "{{ 23 |random(seed=inventory_hostname)  }}"
 certbot_auto_renew_minute: "{{ 59 |random(seed=inventory_hostname)  }}"
 certbot_auth_method: --webroot

galaxy/ansible/pbs.yml

@@ -20,6 +20,7 @@
       environment:
         PYTHOPATH: null
         VIRTUAL_ENV: "{{ galaxy_venv_dir }}"
+      become_user: "{{ galaxy_user_name }}"
 
     - name: "Set PBS/torque server name"
       lineinfile: dest=/etc/torque/server_name line={{ pbs_server_name }} state=present create=yes

galaxy/ansible/provision.yml

@@ -1,12 +1,4 @@
 ---
-- import_playbook: galaxy_root.yml
-  when: galaxy_root | bool
-  tags: galaxy_root
-
-- import_playbook: galaxy_job_metrics.yml
-  when: galaxy_job_metrics | bool
-  tags: galaxy_job_metrics
-
 - import_playbook: gravity.yml
   when: galaxy_gravity | bool
   tags: galaxy_gravity
@@ -74,3 +66,7 @@
 - import_playbook: galaxy_job_conf.yml
   when: galaxy_job_conf | bool
   tags: galaxy_job_conf
+
+- import_playbook: galaxy_job_metrics.yml
+  when: galaxy_job_metrics | bool
+  tags: galaxy_job_metrics

galaxy/ansible/slurm.yml

@@ -11,9 +11,23 @@
       StateSaveLocation: /tmp/slurm
       ReturnToService: 1
   roles:
-    - role: galaxyproject.repos
     - role: galaxyproject.slurm
+    # - role: galaxyproject.repos
   tasks:
+    # The ppa repository addition task from galaxyproject.repos role is not working
+    # after setting "APT::Install-Recommends" to false by default. Hence, added the task manually.
+    # FAILED! => {"changed": false, "msg": "Failed to update apt cache: E:The repository 'http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu sid Release' does not have a Release file."}
+    - name: Import the GPG signing key for the PPA
+      apt_key:
+        url: "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x8DE68488997C5C6BA19021136F2CC56412788738"
+        state: present
+
+    - name: Add PPA repository for Slurm-DRMAA
+      apt_repository:
+        repo: "deb https://ppa.launchpadcontent.net/natefoo/slurm-drmaa/ubuntu {{ ansible_distribution_release }} main"
+        state: present
+        update_cache: yes
+
     - name: Install slurm-drmaa package
       package:
         name: slurm-drmaa1
@@ -49,3 +63,4 @@
       environment:
         PYTHOPATH: null
         VIRTUAL_ENV: "{{ galaxy_venv_dir }}"
+      become_user: "{{ galaxy_user_name }}"

galaxy/ansible/templates/export_user_files.py.j2

@@ -142,7 +142,7 @@ if __name__ == "__main__":
     change_path( os.path.join('/tool_deps') )
     change_path( os.path.join(galaxy_root_dir, 'tool-data') )
     change_path( os.path.join(galaxy_root_dir, 'database') )
-    change_path( '/shed_tools/' )
+    change_path( '/tus_upload_store/' )
 
     if os.path.exists('/export/common_htpasswd'):
         shutil.copy('/export/common_htpasswd', '/etc/nginx/htpasswd')

galaxy/common_cleanup.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -x
+
+# This usually drastically reduced the container size
+# at the cost of the startup time of your application
+find / -name '*.pyc' -delete
+
+find / -name '*.log' -delete
+find / -name '.cache' -delete
+find / -name '.npm' -delete
+rm -rf /var/lib/apt/lists/*
+rm -rf /var/cache/*
+rm -rf /tmp/*
+rm -rf /var/tmp/*
+
+# https://askubuntu.com/questions/266738/how-to-truncate-all-logfiles
+truncate -s 0 /var/log/*log || true
+truncate -s 0 /var/log/**/*log || true

galaxy/reports.yml.sample

@@ -11,10 +11,10 @@ reports:
   #database_connection: postgresql://galaxy:galaxy@localhost:5432/galaxy?client_encoding=utf8
 
   # Where dataset files are stored.
-  #file_path: database/files
+  file_path: database/files
 
   # Where temporary files are stored.
-  #new_file_path: database/tmp
+  new_file_path: database/files
 
   # Mako templates are compiled as needed and cached for reuse, this
   # directory is used for the cache

galaxy/startup.sh

@@ -37,7 +37,9 @@ if [[ ! -z $PROXY_PREFIX ]]
 then
     echo "Configuring with proxy prefix: $PROXY_PREFIX"
     export GALAXY_CONFIG_GALAXY_URL_PREFIX="$PROXY_PREFIX"
-    export GALAXY_CONFIG_INTERACTIVETOOLS_BASE_PATH="$PROXY_PREFIX"
+
+    # TODO: Set this using GALAXY_CONFIG_INTERACTIVETOOLS_BASE_PATH after gravity config manager is updated to handle env vars properly
+    ansible localhost -m replace -a "path=${GALAXY_CONFIG_FILE} regexp='^  #interactivetools_base_path:.*' replace='  interactivetools_base_path: ${PROXY_PREFIX}'" &> /dev/null
 
     python3 /usr/local/bin/update_yaml_value "${GRAVITY_CONFIG_FILE}" "gravity.reports.url_prefix" "$PROXY_PREFIX/reports" &> /dev/null
 
@@ -177,19 +179,25 @@ fi
 if [[ ! -z $LOAD_GALAXY_CONDITIONAL_DEPENDENCIES ]]
     then
         echo "Installing optional dependencies in galaxy virtual environment..."
-        : ${GALAXY_WHEELS_INDEX_URL:="https://wheels.galaxyproject.org/simple"}
-        : ${PYPI_INDEX_URL:="https://pypi.python.org/simple"}
-        GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "import galaxy.dependencies; print('\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE')))")
-        [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -q -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
+        sudo -E -u $GALAXY_USER bash -c '
+            . $GALAXY_VIRTUAL_ENV/bin/activate
+            : ${GALAXY_WHEELS_INDEX_URL:="https://wheels.galaxyproject.org/simple"}
+            : ${PYPI_INDEX_URL:="https://pypi.python.org/simple"}
+            GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "import galaxy.dependencies; print(\"\\n\".join(galaxy.dependencies.optional(\"$GALAXY_CONFIG_FILE\")))")
+            [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -q -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
+        '
 fi
 
 if [[ ! -z $LOAD_GALAXY_CONDITIONAL_DEPENDENCIES ]] && [[ ! -z $LOAD_PYTHON_DEV_DEPENDENCIES ]]
     then
         echo "Installing development requirements in galaxy virtual environment..."
-        : ${GALAXY_WHEELS_INDEX_URL:="https://wheels.galaxyproject.org/simple"}
-        : ${PYPI_INDEX_URL:="https://pypi.python.org/simple"}
-        dev_requirements='./lib/galaxy/dependencies/dev-requirements.txt'
-        [ -f $dev_requirements ] && pip install -q -r $dev_requirements --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
+        sudo -E -u $GALAXY_USER bash -c '
+            . $GALAXY_VIRTUAL_ENV/bin/activate
+            : ${GALAXY_WHEELS_INDEX_URL:="https://wheels.galaxyproject.org/simple"}
+            : ${PYPI_INDEX_URL:="https://pypi.python.org/simple"}
+            dev_requirements="./lib/galaxy/dependencies/dev-requirements.txt"
+            [ -f $dev_requirements ] && pip install -q -r $dev_requirements --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
+        '
 fi
 
 # Enable Test Tool Shed