From ce7958e6fbaca9606554f78b72bc7c346cdbd345 Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Tue, 8 Oct 2024 13:27:01 +0200 Subject: [PATCH 01/25] Remove vagrant --- Vagrantfile | 58 ----------------------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 Vagrantfile diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 5513b4fec..000000000 --- a/Vagrantfile +++ /dev/null @@ -1,58 +0,0 @@ -# -*- ruby -*- - -if ENV['ENV'] == 'dev' - development = true -end - -if development - # Define custom error with non-translated message - class EngineBlockError < Vagrant::Errors::VagrantError - def initialize(dir); - @dir = dir - super() - end - - def error_message; "Directory " + @dir + " must exist before being able to mount OpenConext-engineblock" end - end - - # Check if required directory exists - _engineblock_dir = File.dirname(__FILE__) + "/../OpenConext-engineblock" - unless Dir.exists?(_engineblock_dir) - raise EngineBlockError.new(_engineblock_dir) - end -end - -Vagrant.configure("2") do |config| - config.vm.box = "OpenConext-CentOS-7.0" - config.vm.box_url = "https://build.openconext.org/vagrant_boxes/openconext.json" - config.vm.define "lb_centos7" do |lb_centos7| - lb_centos7.vm.network :private_network, ip: "192.168.66.98" - lb_centos7.vm.hostname = "lb.vm.openconext.org" - lb_centos7.vm.provider :virtualbox do |vb| - vb.name = "OpenConext Engineblock Loadbalancer" - vb.customize ["modifyvm", :id, "--memory", "512"] - vb.customize ["modifyvm", :id, "--cpus", "1"] - end - end - config.vm.define "apps_centos7", primary: true do |apps_centos7| - apps_centos7.vm.network :private_network, ip: "192.168.66.99" - apps_centos7.vm.hostname = "apps.vm.openconext.org" - apps_centos7.vm.provider :virtualbox do |vb| - vb.name = "OpenConext Engineblock Apps" - vb.customize ["modifyvm", :id, "--memory", "4096"] - vb.customize ["modifyvm", :id, "--cpus", "2"] - end - - if development - apps_centos7.vm.synced_folder "../OpenConext-engineblock", "/opt/openconext/OpenConext-engineblock", - type: "nfs", - nfs_version: 4, - nfs_udp: false - apps_centos7.vm.synced_folder ".", "/vagrant", - type: "nfs", - nfs_version: 4, - nfs_udp: false - end - - end -end From 9b1544b41108e75a66d6f9574c8687a4e29a2634 Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Tue, 8 Oct 2024 15:46:44 +0200 Subject: [PATCH 02/25] Remove docker build. Dockers are now in OpenConext-devconf --- .github/workflows/docker-build-image.yml | 63 ----------------- docker-compose.yml | 11 --- tests/Dockerfile.centos-7 | 25 ------- tests/Dockerfile.centos-7-ga | 23 ------ tests/github.yml | 13 ---- tests/githubactions-build.sh | 90 ------------------------ 6 files changed, 225 deletions(-) delete mode 100644 .github/workflows/docker-build-image.yml delete mode 100644 docker-compose.yml delete mode 100644 tests/Dockerfile.centos-7 delete mode 100644 tests/Dockerfile.centos-7-ga delete mode 100644 tests/github.yml delete mode 100644 tests/githubactions-build.sh diff --git a/.github/workflows/docker-build-image.yml b/.github/workflows/docker-build-image.yml deleted file mode 100644 index 4a90007f5..000000000 --- a/.github/workflows/docker-build-image.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Github action to build and publish a Docker image -# -name: build and publish docker -on: - pull_request: - branches: - - 'master' - push: - branches: - - 'master' - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - name: Disable apparmor - run: | - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - - - name: Build the docker image - run: docker build --rm -t surfnet/centos7-openconext-ga -f tests/Dockerfile.centos-7-ga . - - - name: Run Ansible on the Docker container to install OpenConext - run: sh tests/githubactions-build.sh - - - name: Check other issues - if: ${{ failure() }} - run: | - docker exec ansible-test-ga cat /var/log/messages - docker exec ansible-test-ga cat /var/log/manage/manage.log - docker exec systemctl status haproxy - - - name: Set branch name - if: github.event_name != 'pull_request' - shell: bash - run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - - - name: Set branch name on pull_request - if: github.event_name == 'pull_request' - shell: bash - run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | sed 's/\//_/g')" >> $GITHUB_ENV - - - name: Commit and tag the Docker image - run: | - DOCKER_COMMIT_SHA=$(docker commit ansible-test-ga) - docker tag $DOCKER_COMMIT_SHA ghcr.io/openconext/openconext-deploy/openconext-core:${{ env.BRANCH_NAME }} - - - name: Show the docker image - run: docker images - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GH_CONTAINER_REGISTRY_PAT }} - - - name: Push the image - run: docker push ghcr.io/openconext/openconext-deploy/openconext-core:${{ env.BRANCH_NAME }} diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index e3fbe6de5..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- - -version: "3.8" - -services: - web: - image: ghcr.io/openconext/openconext-deploy/openconext-core - privileged: true - ports: - - "443:443" - diff --git a/tests/Dockerfile.centos-7 b/tests/Dockerfile.centos-7 deleted file mode 100644 index 3e0d48d79..000000000 --- a/tests/Dockerfile.centos-7 +++ /dev/null @@ -1,25 +0,0 @@ -FROM centos:7 - -MAINTAINER "Thijs Kinkhorst" - -ENV container docker - -RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ -rm -f /lib/systemd/system/multi-user.target.wants/*;\ -rm -f /etc/systemd/system/*.wants/*;\ -rm -f /lib/systemd/system/local-fs.target.wants/*; \ -rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ -rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ -rm -f /lib/systemd/system/basic.target.wants/*;\ -rm -f /lib/systemd/system/anaconda.target.wants/*; - -# install ansible -RUN yum clean all && \ - yum -y update && \ - yum -y install epel-release && \ - yum -y install PyYAML python-jinja2 python-httplib2 python3 python-paramiko python-setuptools git python-pip ansible iproute -RUN echo -e '[local]\nlocalhost' > /etc/ansible/hosts - -VOLUME [ "/sys/fs/cgroup" ] - -CMD ["/usr/sbin/init"] diff --git a/tests/Dockerfile.centos-7-ga b/tests/Dockerfile.centos-7-ga deleted file mode 100644 index 5189e2892..000000000 --- a/tests/Dockerfile.centos-7-ga +++ /dev/null @@ -1,23 +0,0 @@ -FROM centos:7 - -MAINTAINER "Bart Geesink" -LABEL org.opencontainers.image.source https://github.com/OpenConext/OpenConext-deploy - -ENV container docker - -RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ -rm -f /lib/systemd/system/multi-user.target.wants/*;\ -rm -f /etc/systemd/system/*.wants/*;\ -rm -f /lib/systemd/system/local-fs.target.wants/*; \ -rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ -rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ -rm -f /lib/systemd/system/basic.target.wants/*;\ -rm -f /lib/systemd/system/anaconda.target.wants/*; - -RUN yum clean all && \ - yum -y update && \ - yum -y install python3 - -VOLUME [ "/sys/fs/cgroup" ] - -CMD ["/usr/sbin/init"] diff --git a/tests/github.yml b/tests/github.yml deleted file mode 100644 index 4a3e08b15..000000000 --- a/tests/github.yml +++ /dev/null @@ -1,13 +0,0 @@ -haproxy_redirects: - - name: "redirecttest" - url: "redirect.vm.openconext.org" - redirecturl: "https://engine.vm.openconext.org" - -listen_address_ip4: 0.0.0.0 -postfix_interfaces: ipv4 -dashboard_install: False -update_hosts_file: False -manage_show_oidc_rp_tab: true -manage_exclude_oidc_rp_imports_in_push: true -manage_exclude_sram_imports_in_push: true -mongo_tls_host_altname_dnsorip: DNS diff --git a/tests/githubactions-build.sh b/tests/githubactions-build.sh deleted file mode 100644 index 80a2e3327..000000000 --- a/tests/githubactions-build.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -set -e - -# keep exit status -status=0 - -ANSIBLE_PLAYBOOK=./provision.yml -ANSIBLE_INVENTORY=./environments-external/github/inventory -ANSIBLE_SECRETS=./environments-external/github/secrets/vm.yml -ANSIBLE_PLAYBOOK_WRAPPER=./provision -ANSIBLE_USER=root - -# start docker container -docker run --detach \ - -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ - -t \ - --privileged \ - --publish 443:443 \ - --name ansible-test-ga \ - --add-host static.vm.openconext.org:127.0.0.1 \ - --add-host metadata.vm.openconext.org:127.0.0.1 \ - --add-host engine.vm.openconext.org:127.0.0.1 \ - --add-host profile.vm.openconext.org:127.0.0.1 \ - --add-host mujina-sp.vm.openconext.org:127.0.0.1 \ - --add-host mujina-idp.vm.openconext.org:127.0.0.1 \ - --add-host teams.vm.openconext.org:127.0.0.1 \ - --add-host voot.vm.openconext.org:127.0.0.1 \ - --add-host db.vm.openconext.org:127.0.0.1 \ - --add-host pdp.vm.openconext.org:127.0.0.1 \ - --add-host engine-api.vm.openconext.org:127.0.0.1 \ - --add-host aa.vm.openconext.org:127.0.0.1 \ - --add-host link.vm.openconext.org:127.0.0.1 \ - --add-host connect.vm.openconext.org:127.0.0.1 \ - --add-host oidc-playground.vm.openconext.org:127.0.0.1 \ - --add-host manage.vm.openconext.org:127.0.0.1 \ - --add-host redirect.vm.openconext.org:127.0.0.1 \ - --add-host localhost:127.0.0.1 \ - --add-host ansible-test-ga:127.0.0.1 \ - --hostname test.openconext.org \ - -e TERM=xterm \ - surfnet/centos7-openconext-ga - -# initialize ansible.cfg -cat <<-'EOF' > ansible.cfg - [defaults] - callback_whitelist=profile_tasks - [ssh_connection] - ssh_args=-o ControlMaster=auto -o ControlPersist=60m - pipelining=True -EOF - -# Prepare the environment -echo "Prepping the environment" -mkdir -p environments-external -/bin/cp -r environments/vm/ environments-external/github -/bin/mv environments-external/github/group_vars/vm.yml environments-external/github/group_vars/github.yml -sed -i 's/192.168.66.98/0.0.0.0/g' environments-external/github/group_vars/github.yml -sed -i 's/192.168.66.99/127.0.0.1/g' environments-external/github/group_vars/github.yml -sed -i 's/oidc_push_enabled: true/oidc_push_enabled: false/g' environments-external/github/group_vars/github.yml -sed -i 's/pdp_push_enabled: true/pdp_push_enabled: false/g' environments-external/github/group_vars/github.yml -# Change the hostname in the inventory -/bin/cp environments/template/inventory environments-external/github/ -sed -i 's/%env%/github/g' environments-external/github/inventory -sed -i 's/%target_host%/ansible-test-ga ansible_connection=docker/g' environments-external/github/inventory - -# Create the proper host_vars file -/bin/cp environments/template/host_vars/template.yml environments-external/github/host_vars/ansible-test-ga.yml - -# Remove ipv6 listening address in Haproxy -sed -i '/haproxy_sni_ip\.ipv6/d' roles/haproxy/templates/haproxy_frontend.cfg.j2 - -echo -echo "=================================================================" -echo "=================================================================" -echo "== STARTING MAIN PLAYBOOK RUN ===================================" -echo "=================================================================" -echo "=================================================================" -echo - -./provision github $ANSIBLE_USER $ANSIBLE_SECRETS -e springboot_service_to_deploy=manage,mujina-sp,mujina-idp -e @tests/github.yml -t core - -# Make the image a bit smaller -docker exec ansible-test-ga systemctl stop mysql mongod -docker exec ansible-test-ga yum -y remove mongodb-org-mongos mongodb-org-tools -docker exec ansible-test-ga rm -rf /var/lib/mongo/journal/* -docker exec ansible-test-ga rm -rf /var/lib/mysql/ib_logfile* -docker stop ansible-test-ga ansible-test-ga - -exit $status From e165f7f20a8fb4fdf21a769f8fb8435b71f21fd7 Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Tue, 8 Oct 2024 15:49:13 +0200 Subject: [PATCH 03/25] Remove obsolete molecule workflows --- .github/workflows/molecule-base.yml | 38 -------- .github/workflows/molecule-frontend.yml | 35 -------- .github/workflows/molecule-java.yml | 42 --------- .github/workflows/molecule-mysql.yml | 41 --------- .github/workflows/molecule-php.yml | 50 ----------- molecule/Dockerfile.j2 | 25 ------ molecule/base/converge.yml | 16 ---- molecule/base/molecule.yml | 40 --------- molecule/base/tests/test_default.py | 34 ------- molecule/frontend/converge.yml | 23 ----- molecule/frontend/molecule.yml | 30 ------- molecule/frontend/prepare.yml | 72 --------------- molecule/frontend/tests/test_default.py | 56 ------------ molecule/java/converge.yml | 42 --------- molecule/java/molecule.yml | 39 -------- molecule/java/prepare.yml | 68 -------------- molecule/java/tests/test_default.py | 58 ------------ molecule/mysql/converge.yml | 42 --------- molecule/mysql/molecule.yml | 43 --------- molecule/mysql/prepare.yml | 115 ------------------------ molecule/mysql/tests/test_default.py | 69 -------------- molecule/php/converge.yml | 21 ----- molecule/php/molecule.yml | 30 ------- molecule/php/prepare.yml | 28 ------ molecule/php/tests/test_default.py | 50 ----------- 25 files changed, 1107 deletions(-) delete mode 100644 .github/workflows/molecule-base.yml delete mode 100644 .github/workflows/molecule-frontend.yml delete mode 100644 .github/workflows/molecule-java.yml delete mode 100644 .github/workflows/molecule-mysql.yml delete mode 100644 .github/workflows/molecule-php.yml delete mode 100644 molecule/Dockerfile.j2 delete mode 100644 molecule/base/converge.yml delete mode 100644 molecule/base/molecule.yml delete mode 100644 molecule/base/tests/test_default.py delete mode 100644 molecule/frontend/converge.yml delete mode 100644 molecule/frontend/molecule.yml delete mode 100644 molecule/frontend/prepare.yml delete mode 100644 molecule/frontend/tests/test_default.py delete mode 100644 molecule/java/converge.yml delete mode 100644 molecule/java/molecule.yml delete mode 100644 molecule/java/prepare.yml delete mode 100644 molecule/java/tests/test_default.py delete mode 100644 molecule/mysql/converge.yml delete mode 100644 molecule/mysql/molecule.yml delete mode 100644 molecule/mysql/prepare.yml delete mode 100644 molecule/mysql/tests/test_default.py delete mode 100644 molecule/php/converge.yml delete mode 100644 molecule/php/molecule.yml delete mode 100644 molecule/php/prepare.yml delete mode 100644 molecule/php/tests/test_default.py diff --git a/.github/workflows/molecule-base.yml b/.github/workflows/molecule-base.yml deleted file mode 100644 index b8d7a69e1..000000000 --- a/.github/workflows/molecule-base.yml +++ /dev/null @@ -1,38 +0,0 @@ -# Github action to run Molecule for "base" - -name: base -on: - push: - paths: - - 'roles/common/**' - - 'roles/rsyslog/**' - - 'roles/selfsigned_certs/**' - - 'molecule/base/**' - - '.github/workflows/molecule-base.yml' - pull_request: - paths: - - 'roles/common/**' - - 'roles/rsyslog/**' - - 'roles/selfsigned_certs/**' - - 'molecule/base/**' - - '.github/workflows/molecule-base.yml' - -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Installing dependencies - run: pip install jinja2 ansible molecule molecule-docker pytest-testinfra pytest setuptools - - - name: Remove chrony apparmor - run: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.chronyd - - - name: Run role tests - run: molecule test -s base diff --git a/.github/workflows/molecule-frontend.yml b/.github/workflows/molecule-frontend.yml deleted file mode 100644 index ff94f3617..000000000 --- a/.github/workflows/molecule-frontend.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Github action to run Molecule for "frontend" - -name: frontend -on: - push: - paths: - - 'roles/hosts/**' - - 'roles/httpd/**' - - 'roles/welcome/**' - - 'molecule/frontend/**' - - '.github/workflows/molecule-frontend.yml' - pull_request: - paths: - - 'roles/hosts/**' - - 'roles/httpd/**' - - 'roles/welcome/**' - - 'molecule/frontend/**' - - '.github/workflows/molecule-frontend.yml' - -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Installing dependencies - run: pip install jinja2 ansible molecule molecule-docker pytest-testinfra pytest setuptools - - - name: Run role tests - run: molecule test -s frontend diff --git a/.github/workflows/molecule-java.yml b/.github/workflows/molecule-java.yml deleted file mode 100644 index 2496b101c..000000000 --- a/.github/workflows/molecule-java.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Github action to run Molecule for "java" - -name: java -on: - push: - paths: - - 'roles/java/**' - - 'roles/shibboleth/**' - - 'roles/springboot/**' - - 'molecule/java/**' - - '.github/workflows/molecule-java.yml' - pull_request: - paths: - - 'roles/java/**' - - 'roles/shibboleth/**' - - 'roles/springboot/**' - - 'molecule/java/**' - - '.github/workflows/molecule-java.yml' - - -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Installing dependencies - run: pip install jinja2 ansible molecule molecule-docker pytest-testinfra pytest setuptools - - - name: disable apparmor for mysql - run: sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - - - name: parse apparmor for mysql - run: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - - - name: Run role tests - run: molecule test -s java diff --git a/.github/workflows/molecule-mysql.yml b/.github/workflows/molecule-mysql.yml deleted file mode 100644 index 59b93cd61..000000000 --- a/.github/workflows/molecule-mysql.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Github action to run Molecule for "base" - -name: mysql -on: - push: - paths: - - 'roles/mysql/**' - - 'roles/galera/**' - - 'roles/galera_create_users/**' - - 'molecule/mysql/**' - - '.github/workflows/molecule-mysql.yml' - pull_request: - paths: - - 'roles/mysql/**' - - 'roles/galera/**' - - 'roles/galera_create_users/**' - - 'molecule/mysql/**' - - '.github/workflows/molecule-mysql.yml' - -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Installing dependencies - run: pip install jinja2 ansible molecule molecule-docker pytest-testinfra pytest setuptools - - - name: disable apparmor for mysql - run: sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - - - name: parse apparmor for mysql - run: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - - - name: Run role tests - run: molecule test -s mysql diff --git a/.github/workflows/molecule-php.yml b/.github/workflows/molecule-php.yml deleted file mode 100644 index a62375bf2..000000000 --- a/.github/workflows/molecule-php.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Github action to run Molecule for "php" - -name: php -on: - push: - paths: - - 'roles/httpd/**' - - 'roles/php/**' - - 'roles/static/**' - - 'roles/metadata/**' - - 'roles/openconext-common/**' - - 'roles/engineblock/**' - - 'roles/profile/**' - - 'molecule/php/**' - - '.github/workflows/molecule-php.yml' - pull_request: - paths: - - 'roles/httpd/**' - - 'roles/php/**' - - 'roles/static/**' - - 'roles/metadata/**' - - 'roles/openconext-common/**' - - 'roles/engineblock/**' - - 'roles/profile/**' - - 'molecule/php/**' - - '.github/workflows/molecule-php.yml' - - -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Installing dependencies - run: pip install jinja2 ansible molecule molecule-docker pytest-testinfra pytest setuptools - - - name: disable apparmor for mysql - run: sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - - - name: parse apparmor for mysql - run: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - - - name: Run role tests - run: molecule test -s php diff --git a/molecule/Dockerfile.j2 b/molecule/Dockerfile.j2 deleted file mode 100644 index 85b767deb..000000000 --- a/molecule/Dockerfile.j2 +++ /dev/null @@ -1,25 +0,0 @@ -FROM centos:7 - -ENV container docker - -RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ -rm -f /lib/systemd/system/multi-user.target.wants/*;\ -rm -f /etc/systemd/system/*.wants/*;\ -rm -f /lib/systemd/system/local-fs.target.wants/*; \ -rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ -rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ -rm -f /lib/systemd/system/basic.target.wants/*;\ -rm -f /lib/systemd/system/anaconda.target.wants/*; - -RUN yum makecache fast && yum install -y systemd python sudo yum-plugin-ovl bash iproute && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all - -# install ansible -RUN yum clean all && \ - yum -y update && \ - yum -y install epel-release && \ - yum -y install PyYAML python-jinja2 python-httplib2 python3 python-paramiko python-setuptools git python-pip ansible iproute cronie -RUN echo -e '[local]\nlocalhost' > /etc/ansible/hosts - -VOLUME [ "/sys/fs/cgroup" ] - -CMD ["/usr/sbin/init"] diff --git a/molecule/base/converge.yml b/molecule/base/converge.yml deleted file mode 100644 index 70c7e778b..000000000 --- a/molecule/base/converge.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Converge - hosts: all - pre_tasks: - - name: Read vars from secrets file - include_vars: "../../environments/vm/secrets/vm.yml" - - name: Read vars from group_vars file - include_vars: "../../environments/vm/group_vars/vm.yml" - - name: Set fact - set_fact: - postfix_interfaces: ipv4 - - roles: - - role: common - - role: rsyslog - - role: selfsigned_certs diff --git a/molecule/base/molecule.yml b/molecule/base/molecule.yml deleted file mode 100644 index 99c93156c..000000000 --- a/molecule/base/molecule.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker - -platforms: - - name: openconext-centos7-base - image: centos7-ansible - dockerfile: ../Dockerfile.j2 - command: /usr/sbin/init - privileged: True - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - groups: - - "loadbalancer-vm" - - loadbalancer - - name: openconext-rocky8-base - image: rocky8-ansible - dockerfile: ../Dockerfile-Rocky8.j2 - command: /usr/sbin/init - privileged: True - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - groups: - - "loadbalancer-vm" - - loadbalancer - -provisioner: - name: ansible - env: - ANSIBLE_REMOTE_TMP: /tmp/ - ANSIBLE_ROLES_PATH: ../../roles - ANSIBLE_FILTER_PLUGINS: ../../filter_plugins - inventory: - links: - group_vars: ../../group_vars - -verifier: - name: testinfra diff --git a/molecule/base/tests/test_default.py b/molecule/base/tests/test_default.py deleted file mode 100644 index d7702e314..000000000 --- a/molecule/base/tests/test_default.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import pytest - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - -@pytest.mark.parametrize("removed_packages", [ - ("exim"), - ("sendmail"), - ("sendmail-cf"), -]) -def test_packages_removed(host, removed_packages): - rpackage = host.package(removed_packages) - assert not rpackage.is_installed - - -@pytest.mark.parametrize("files, owner, group, mode", [ - ("/etc/pki/tls/private/star.vm.openconext.org.key", "root", "root", 0o600), - ("/etc/pki/tls/certs/star.vm.openconext.org.pem", "root", "root", 0o644), -]) -def test_openconext_star_files(host, files, owner, group, mode): - openconext_star = host.file(files) - assert openconext_star.user == owner - assert openconext_star.group == group - assert openconext_star.mode == mode - - -def test_tls_bundle(host): - tls_bundle = host.file("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem") - assert tls_bundle.mode == 0o444 - - assert tls_bundle.contains('*.vm.openconext.org') diff --git a/molecule/frontend/converge.yml b/molecule/frontend/converge.yml deleted file mode 100644 index d0299a882..000000000 --- a/molecule/frontend/converge.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- name: Converge - hosts: all - - vars: - inventory_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/../../environments/vm" - - pre_tasks: - - name: Read vars from secrets file - include_vars: "../../environments/vm/secrets/vm.yml" - - name: Read vars from group_vars file - include_vars: "../../environments/vm/group_vars/vm.yml" - - - name: "Override some defaults" - set_fact: - base_domain: molecule.openconext.org - haproxy_backend_tls: true - backend_tls_key: "{{lookup('file', inventory_dir + '/files/certs/backend.molecule.openconext.org.key') }}" - - roles: - - role: hosts - - role: httpd - - role: welcome diff --git a/molecule/frontend/molecule.yml b/molecule/frontend/molecule.yml deleted file mode 100644 index 11248ecfb..000000000 --- a/molecule/frontend/molecule.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker - -platforms: - - name: openconext-centos7-frontend - image: centos7-ansible - dockerfile: ../Dockerfile.j2 - command: /usr/sbin/init - privileged: True - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - groups: - - "loadbalancer-vm" - - loadbalancer - -provisioner: - name: ansible - env: - ANSIBLE_REMOTE_TMP: /tmp/ - ANSIBLE_ROLES_PATH: ../../roles - ANSIBLE_FILTER_PLUGINS: ../../filter_plugins - inventory: - links: - group_vars: ../../group_vars - -verifier: - name: testinfra diff --git a/molecule/frontend/prepare.yml b/molecule/frontend/prepare.yml deleted file mode 100644 index c4aeffd55..000000000 --- a/molecule/frontend/prepare.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -- name: Prepare - hosts: all - vars: - inventory_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/../../environments/vm" - - tasks: - - name: Read vars from secrets file - include_vars: "../../environments/vm/secrets/vm.yml" - - name: Read vars from group_vars file - include_vars: "../../environments/vm/group_vars/vm.yml" - - - name: "Set some facts" - set_fact: - base_domain: molecule.openconext.org - haproxy_backend_tls: true - - - name: Install packages - yum: - name: - - python2-cryptography - - unzip - state: "installed" - register: prepare_packages_installed - until: prepare_packages_installed is succeeded - - - name: create CA key - openssl_privatekey: - path: /root/CA_key.key - - - name: create the CA CSR - openssl_csr: - path: /root/CA.csr - privatekey_path: /root/CA_key.key - common_name: "my-ca" - - - name: sign the CA CSR - openssl_certificate: - path: /root/CA.crt - csr_path: /root/CA.csr - privatekey_path: /root/CA_key.key - provider: selfsigned - - - name: create host key - openssl_privatekey: - path: /root/example_com_host_key.key - - - name: create the CSR for the http server - openssl_csr: - path: /root/example_com.csr - privatekey_path: /root/example_com_host_key.key - common_name: "{{ base_domain }}" - subject_alt_name: 'DNS:{{ inventory_hostname }}' - - - name: sign the CSR for the http server - openssl_certificate: - path: /root/example_com.crt - csr_path: /root/example_com.csr - provider: ownca - ownca_path: /root/CA.crt - ownca_privatekey_path: /root/CA_key.key - - - name: "Fetch files" - fetch: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - flat: true - with_items: - - src: /root/example_com.crt - dest: "{{ inventory_dir }}/files/certs/backend.{{ base_domain }}.pem" - - src: /root/example_com_host_key.key - dest: "{{ inventory_dir }}/files/certs/backend.{{ base_domain }}.key" diff --git a/molecule/frontend/tests/test_default.py b/molecule/frontend/tests/test_default.py deleted file mode 100644 index 0bf66941c..000000000 --- a/molecule/frontend/tests/test_default.py +++ /dev/null @@ -1,56 +0,0 @@ -import os -import pytest - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -@pytest.mark.parametrize("installed_packages", [ - ("httpd"), - ("mod_ssl"), -]) -def test_packages_installed(host, installed_packages): - rpackage = host.package(installed_packages) - assert rpackage.is_installed - - -@pytest.mark.parametrize("services", [ - ("httpd"), -]) -def test_services_running_and_enabled(host, services): - service = host.service(services) - assert service.is_enabled - assert service.is_running - - -@pytest.mark.parametrize("files", [ - ("/etc/httpd/conf.d/welcome-vm.conf"), -]) -def test_welcome(host, files): - welcome = host.file(files) - assert welcome.user == "root" - assert welcome.group == "root" - assert welcome.mode == 0o644 - - -def test_http_ssl_conf(host): - http_ssl_conf = host.file("/etc/httpd/conf.d/ssl.conf") - assert not http_ssl_conf.contains('Listen 443') - - -@pytest.mark.parametrize("files", [ - ("/etc/httpd/conf.d/welcome.conf"), - ("/etc/httpd/conf.d/userdir.conf"), - ("/etc/httpd/conf.d/autoindex.conf"), -]) -def test_empty_config(host, files): - test_empty_config = host.file(files) - assert test_empty_config.size == 0 - - -def test_subject_ssll_key(host): - cmd = host.run("openssl x509 -in /etc/pki/tls/certs/backend.molecule.openconext.org.pem -noout -subject") - assert 'subject= /CN=molecule.openconext.org' in cmd.stdout - assert cmd.rc == 0 diff --git a/molecule/java/converge.yml b/molecule/java/converge.yml deleted file mode 100644 index 379ef65da..000000000 --- a/molecule/java/converge.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -- name: Converge - hosts: all - - vars: - inventory_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/../../environments/vm" - - pre_tasks: - - name: Read vars from secrets file - include_vars: "../../environments/vm/secrets/vm.yml" - - name: Read vars from group_vars file - include_vars: "../../environments/vm/group_vars/vm.yml" - - name: Read vars from hosts_vars file from template - include_vars: "../../environments/template/host_vars/template.yml" - - - name: "Override some defaults" - set_fact: - base_domain: molecule.openconext.org - springboot_services_state: - manage: true - teams: false - pdp: false - attribute_aggregation: false - oidc_playground: false - myconext: false - account: false - eduid: false - oidcng: true - voot: false - dashboard: false - mujina_sp: true - mujina_idp: true - mongo_replication_role: primary - mongo_tls_host_altname_dnsorip: DNS - - - roles: - - role: java - - role: shibboleth - - role: springboot - handlers: - - include: ../../roles/httpd/handlers/main.yml diff --git a/molecule/java/molecule.yml b/molecule/java/molecule.yml deleted file mode 100644 index 2c5b74f09..000000000 --- a/molecule/java/molecule.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker - -platforms: - - name: openconext-centos7-java - image: centos7-ansible - dockerfile: ../Dockerfile.j2 - command: /usr/sbin/init - privileged: True - sysctls: - net.ipv6.conf.all.disable_ipv6: 1 - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - groups: - - loadbalancer - - mongo_servers - - selfsigned_certs - etc_hosts: - engine.molecule.openconext.org: 127.0.0.1 - engine-api.molecule.openconext.org: 127.0.0.2 - pdp.molecule.openconext.org: 127.0.0.1 - manage.molecule.openconext.org: 127.0.0.1 - connect.molecule.openconext.org: 127.0.0.1 - -provisioner: - name: ansible - env: - ANSIBLE_REMOTE_TMP: /tmp/ - ANSIBLE_ROLES_PATH: ../../roles - ANSIBLE_FILTER_PLUGINS: ../../filter_plugins - inventory: - links: - group_vars: ../../group_vars - -verifier: - name: testinfra diff --git a/molecule/java/prepare.yml b/molecule/java/prepare.yml deleted file mode 100644 index b71a3c072..000000000 --- a/molecule/java/prepare.yml +++ /dev/null @@ -1,68 +0,0 @@ ---- -- name: Prepare - hosts: all - vars: - inventory_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/../../environments/vm" - - pre_tasks: - - name: Read vars from secrets file - include_vars: "../../environments/vm/secrets/vm.yml" - - name: Read vars from group_vars file - include_vars: "../../environments/vm/group_vars/vm.yml" - - name: Read vars from hosts_vars file from template - include_vars: "../../environments/template/host_vars/template.yml" - - - name: "Set some facts" - set_fact: - base_domain: molecule.openconext.org - use_selfsigned_certs: True - postfix_interfaces: ipv4 - - - name: Install packages - yum: - name: - - unzip - - crontabs - state: "installed" - register: prepare_packages_installed - until: prepare_packages_installed is succeeded - - roles: - - role: selfsigned_certs - -- name: Prepare - hosts: all - - vars: - inventory_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/../../environments/vm" - - pre_tasks: - - name: Read vars from secrets file - include_vars: "../../environments/vm/secrets/vm.yml" - - name: Read vars from group_vars file - include_vars: "../../environments/template/group_vars/template.yml" - - name: Read vars from hosts_vars file from template - include_vars: "../../environments/template/host_vars/template.yml" - - - name: "Override some defaults" - set_fact: - base_domain: molecule.openconext.org - myconext_base_domain: molecule.openconext.org - use_selfsigned_certs: True - engine_trusted_proxy_ips: - - 192.168.1.1 - - 10.0.0.1 - - "{{ ansible_all_ipv4_addresses[0] }}" - mongo_replication_role: primary - mongo_tls_host_altname_dnsorip: DNS - - roles: - - role: rsyslog - - role: common - - role: haproxy - - role: httpd - - role: mysql - - role: mongo - - role: php - - role: openconext-common - - role: engineblock diff --git a/molecule/java/tests/test_default.py b/molecule/java/tests/test_default.py deleted file mode 100644 index 7f3c4da36..000000000 --- a/molecule/java/tests/test_default.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import pytest - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - - -def test_java_binary(host): - java_binary = host.file("/usr/bin/java") - command = host.run('/usr/bin/java -version 2>&1 | grep openjdk') - assert java_binary.exists - assert java_binary.is_file - - assert command.rc == 0 - assert 'version "11.' in command.stdout - - -@pytest.mark.parametrize("components, dir_owner, file_owner, group, httpd_listen, spring_listen", [ - ("manage", "root", "manage", "root", "617", "9393"), - ("mujina-idp", "mujina-idp", "mujina-idp", "mujina-idp", "608", "9390"), - ("mujina-sp", "mujina-sp", "mujina-sp", "mujina-sp", "607", "9391"), -]) -def test_components(host, components, dir_owner, file_owner, group, httpd_listen, spring_listen): - user = host.user(components) - service = host.service(components) - socket_httpd = host.socket("tcp://127.0.0.1:" + httpd_listen) - socket_springboot = host.socket("tcp://127.0.0.1:" + spring_listen) - opt_dir = host.file("/opt/" + components) - logback = host.file("/opt/" + components + "/logback.xml") - application = host.file("/opt/" + components + "/application.yml") - http_file = host.file("/etc/httpd/conf.d/" + components.replace("-", "_") + '.conf') - # manage contains a version in symlink, so lets skip that for now. - if components != "manage": - jar_file = host.file("/opt/" + components + "/" + components + '.jar') - assert jar_file.is_symlink - - assert user.exists - - assert service.is_enabled - assert service.is_running - - assert opt_dir.is_directory - assert opt_dir.user == dir_owner - assert opt_dir.group == group - - assert logback.exists - assert logback.user == file_owner - assert application.exists - assert application.user == file_owner - - assert http_file.exists - assert http_file.is_file - - assert socket_httpd.is_listening - assert socket_springboot.is_listening diff --git a/molecule/mysql/converge.yml b/molecule/mysql/converge.yml deleted file mode 100644 index 8e27fa2cc..000000000 --- a/molecule/mysql/converge.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -- name: Converge - hosts: all - - vars: - inventory_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/../../environments/vm" - - pre_tasks: - - name: Read vars from secrets file - include_vars: "../../environments/vm/secrets/vm.yml" - - name: Read vars from group_vars file - include_vars: "../../environments/vm/group_vars/vm.yml" - - - name: "Set fact for backend_ipv4" - set_fact: - backend_ipv4: "{{ ansible_all_ipv4_addresses[0] }}" - mariadb_cluster_name: molecule - mariadb_cluster_password: secret - mariadb_root_password: secret - mariadb_backup_password: secret - galera_bootstrap_node: openconext-centos7-mysql - galera_server_key: "{{lookup('file', inventory_dir + '/files/certs/galera/galera_server.key') }}" - galera_client_key: "{{lookup('file', inventory_dir + '/files/certs/galera/galera_server.key') }}" - galera_client_crt_name: "galera_server.pem" - - roles: - - role: mysql -# - role: galera -# innodb_buffer_pool_size: 32M -# galera_handler_restart: False -# galera_root_users: -# - name: molecule -# password: secret -# privs: -# - '*.*:ALL' -# hosts: -# - '%' - - role: galera_create_users - databases: - users: - - { name: amolecule, db_name: amolecule, password: secret, privilege: ALL } - diff --git a/molecule/mysql/molecule.yml b/molecule/mysql/molecule.yml deleted file mode 100644 index 7e14e7ce2..000000000 --- a/molecule/mysql/molecule.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker - -platforms: - - name: openconext-centos7-mysql - image: centos7-ansible - dockerfile: ../Dockerfile.j2 - command: /usr/sbin/init - privileged: true - networks: - - name: mariadb - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - groups: - - storage - - dbcluster - - dbcluster_nodes - - name: openconext-centos7-mysql-2 - image: centos7-ansible - dockerfile: ../Dockerfile.j2 - command: /usr/sbin/init - privileged: true - networks: - - name: mariadb - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - groups: - - storage - - dbcluster - - dbcluster_nodes - -provisioner: - name: ansible - env: - ANSIBLE_REMOTE_TMP: /tmp/ - ANSIBLE_ROLES_PATH: ../../roles - ANSIBLE_FILTER_PLUGINS: ../../filter_plugins - inventory: - links: - group_vars: ../../group_vars diff --git a/molecule/mysql/prepare.yml b/molecule/mysql/prepare.yml deleted file mode 100644 index 157eddb1d..000000000 --- a/molecule/mysql/prepare.yml +++ /dev/null @@ -1,115 +0,0 @@ ---- -- name: Prepare - hosts: all - - vars: - inventory_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/../../environments/vm" - copy_tls_ca: - - src: /root/CA.crt - dest: "{{ inventory_dir }}/files/certs/galera/galera_ca.pem" - - src: /root/CA_key.pem - dest: "{{ inventory_dir }}/files/certs/galera/galera_ca.key" - - pre_tasks: - - name: Read vars from secrets file - include_vars: "../../environments/vm/secrets/vm.yml" - - name: Read vars from group_vars file - include_vars: "../../environments/vm/group_vars/vm.yml" - - - name: "Install package(s)" - yum: - name: - - crontabs - - python2-cryptography - state: present - register: prepare_packages_installed - until: prepare_packages_installed is succeeded - - - name: "Create mysqlusers group" - group: - name: mysqlusers - state: present - - - name: create CA key - openssl_privatekey: - path: /root/CA_key.pem - register: ca_key - run_once: true - - - name: create the CA CSR - openssl_csr: - path: /root/CA.csr - privatekey_path: "{{ ca_key.filename }}" - common_name: "my-ca" - register: ca_csr - run_once: true - - - name: sign the CA CSR - openssl_certificate: - path: /root/CA.crt - csr_path: "{{ ca_csr.filename }}" - privatekey_path: "{{ ca_key.filename }}" - provider: selfsigned - register: ca_crt - run_once: true - - - name: "Fetch CA crt" - fetch: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - flat: true - with_items: "{{ copy_tls_ca }}" - run_once: true - - - name: "Copy CA crt file" - copy: - src: "{{ item.dest }}" - dest: "{{ item.src }}" - with_items: "{{ copy_tls_ca }}" - - - name: create host CSR signing key - openssl_privatekey: - path: /root/example_com_host_key.pem - register: example_com_key - - - name: create the CSR for the Galera server - openssl_csr: - path: /root/example_com.csr - privatekey_path: "{{ example_com_key.filename }}" - common_name: example.com - subject_alt_name: 'DNS:{{ inventory_hostname }}' - register: example_com_csr - - - name: sign the CSR for the Galera server - openssl_certificate: - path: /root/example_com.crt - csr_path: "{{ example_com_csr.filename }}" - provider: ownca - ownca_path: "{{ hostvars['openconext-centos7-mysql'].ca_crt.filename }}" - ownca_privatekey_path: "{{ hostvars['openconext-centos7-mysql'].ca_key.filename }}" - register: example_com_crt - - - name: "Fetch files" - fetch: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - flat: true - with_items: - - src: /root/CA.crt - dest: "{{ inventory_dir }}/files/certs/galera/galera_ca.pem" - - src: /root/example_com_host_key.pem - dest: "{{ inventory_dir }}/files/certs/galera/galera_server.key" - - src: /root/example_com.crt - dest: "{{ inventory_dir }}/files/certs/galera/galera_server.pem" - - src: /root/example_com.crt - dest: "{{ inventory_dir }}/files/certs/galera/{{ inventory_hostname }}-galera_client.pem" - run_once: true - - roles: - - role: keepalived - keepalived: - state_master: MASTER - state_backup: BACKUP - masterprio: 101 - backupprio: 100 - keepalived_loadbalancer_vrrp_password: mysecret diff --git a/molecule/mysql/tests/test_default.py b/molecule/mysql/tests/test_default.py deleted file mode 100644 index 491deae3e..000000000 --- a/molecule/mysql/tests/test_default.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -import pytest - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -@pytest.mark.parametrize("installed_packages", [ - ("MariaDB-server"), - ("MariaDB-client"), - ("python2-PyMySQL"), - ("galera"), -]) -def test_packages_installed(host, installed_packages): - rpackage = host.package(installed_packages) - assert rpackage.is_installed - - -@pytest.mark.parametrize("services", [ - ("mariadb"), -]) -def test_services_running_and_enabled(host, services): - service = host.service(services) - assert service.is_enabled - assert service.is_running - - -@pytest.mark.parametrize("files", [ - ("/etc/pki/mysql/galera_client.key"), - ("/etc/pki/mysql/galera_server.key"), - ("/etc/pki/mysql/galera_server.pem"), - ("/etc/pki/mysql/galera_sst.pem"), -]) -def test_galera_tls(host, files): - file = host.file(files) - assert file.user == "mysql" - assert file.group == "root" - assert file.mode == 0o400 - - -def test_create_test_database(host): - ansible_vars = host.ansible.get_variables() - current_hostname = ansible_vars['inventory_hostname'] - if current_hostname == 'openconext-centos7-mysql': - host.command('mysql -e "drop database if exists moleculetest"') - command = host.command('mysql -e "create database moleculetest;"') - assert command.rc == 0 - - -def test_show_databases(host): - command = host.command('mysql -e "show databases;" | grep -q moleculetest | wc -l') - assert command.rc == 0 - assert int(command.stdout) == 1 - - -def test_create_test_database(host): - ansible_vars = host.ansible.get_variables() - current_hostname = ansible_vars['inventory_hostname'] - if current_hostname == 'openconext-centos7-mysql': - command = host.command('mysql -e "drop database if exists moleculetest"') - assert command.rc == 0 - - -def test_show_databases(host): - command = host.command('mysql -e "show databases;" | grep -q moleculetest | wc -l') - assert command.rc == 0 - assert int(command.stdout) == 0 diff --git a/molecule/php/converge.yml b/molecule/php/converge.yml deleted file mode 100644 index 703d5d598..000000000 --- a/molecule/php/converge.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -- name: Converge - hosts: all - - vars: - inventory_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/../../environments/vm" - - pre_tasks: - - name: Read vars from secrets file - include_vars: "../../environments/vm/secrets/vm.yml" - - name: Read vars from group_vars file - include_vars: "../../environments/vm/group_vars/vm.yml" - - roles: - - role: php - - role: static - - role: metadata - - role: openconext-common - - role: engineblock - handlers: - - include: ../../roles/httpd/handlers/main.yml diff --git a/molecule/php/molecule.yml b/molecule/php/molecule.yml deleted file mode 100644 index 56c8ed2a7..000000000 --- a/molecule/php/molecule.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker - -platforms: - - name: openconext-centos7-php - image: centos7-ansible - dockerfile: ../Dockerfile.j2 - command: /usr/sbin/init - privileged: true - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - groups: - - "loadbalancer-vm" - - loadbalancer - -provisioner: - name: ansible - env: - ANSIBLE_REMOTE_TMP: /tmp/ - ANSIBLE_ROLES_PATH: ../../roles - ANSIBLE_FILTER_PLUGINS: ../../filter_plugins - inventory: - links: - group_vars: ../../group_vars - -verifier: - name: testinfra diff --git a/molecule/php/prepare.yml b/molecule/php/prepare.yml deleted file mode 100644 index a9aeed4bc..000000000 --- a/molecule/php/prepare.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- name: Prepare - hosts: all - vars: - inventory_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/../../environments/vm" - - pre_tasks: - - name: Read vars from secrets file - include_vars: "../../environments/vm/secrets/vm.yml" - - name: Read vars from group_vars file - include_vars: "../../environments/vm/group_vars/vm.yml" - - - name: "Set some facts" - set_fact: - base_domain: molecule.openconext.org - - - name: Install packages - yum: - name: - - unzip - state: "installed" - register: prepare_packages_installed - until: prepare_packages_installed is succeeded - - roles: - - role: selfsigned_certs - - role: httpd - - role: mysql diff --git a/molecule/php/tests/test_default.py b/molecule/php/tests/test_default.py deleted file mode 100644 index dc4ce0258..000000000 --- a/molecule/php/tests/test_default.py +++ /dev/null @@ -1,50 +0,0 @@ -import os -import pytest - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -@pytest.mark.parametrize("installed_packages", [ - ("httpd"), - ("php72-php-fpm"), - ("php72-php-mysqlnd"), -]) -def test_packages_installed(host, installed_packages): - rpackage = host.package(installed_packages) - assert rpackage.is_installed - - -@pytest.mark.parametrize("services", [ - ("httpd"), - ("php72-php-fpm"), -]) -def test_services_running_and_enabled(host, services): - service = host.service(services) - assert service.is_enabled - assert service.is_running - - -@pytest.mark.parametrize("files", [ - ("/etc/opt/remi/php72/php.d/40-apcu.ini"), - ("/etc/opt/remi/php72/php.d/openconext.ini"), - ("/etc/opt/remi/php72/php-fpm.conf"), - ("/etc/opt/remi/php72/php-fpm.d/www.conf"), - ("/etc/httpd/conf.d/metadata.conf"), - ("/etc/httpd/conf.d/static.conf"), -]) -def test_php_files(host, files): - php_file = host.file(files) - assert php_file.user == "root" - assert php_file.group == "root" - assert php_file.mode == 0o644 - - -@pytest.mark.parametrize("components", [ - ("engine"), -]) -def test_components(host, components): - component = host.user(components) - assert component.exists From 3cbcc8a9c4c400728fc7fa24aa72dc3cd75b375f Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Wed, 9 Oct 2024 11:13:41 +0200 Subject: [PATCH 04/25] Update the documentation. There is no more Vagrant support, and all applications run from Docker. The docs have been updated to reflect this --- DEVELOPMENT.md | 82 -------------------- README.md | 203 ++++++++++++++++--------------------------------- 2 files changed, 65 insertions(+), 220 deletions(-) delete mode 100644 DEVELOPMENT.md diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md deleted file mode 100644 index b53775b14..000000000 --- a/DEVELOPMENT.md +++ /dev/null @@ -1,82 +0,0 @@ -Development Environment -============================== - -The development environment differs to the other environments in three important ways, each will be highlighted below. - -## Mounting EngineBlock -Unlike other provisioning scripts, it will not download and symlink EngineBlock in a folder on the -VM as it will mount a (shared) directory, expected to be present at `../OpenConext-engineblock/` and -accessible on the `apps` VM at `/opt/openconext/OpenConext-engineblock`. - -This way, one can easily manage OpenConext-engineblock code from outside the VM. - -## Xdebug -For development purposes, provisioning the `apps` VM includes Xdebug. To circumvent the `loadbalancer` VM, it is -configured with `192.168.66.1` as IP of the host machine. Should there be any issues, make sure this IP address is -correct for your setup. The IDEkey used is `PHPSTORM`. It is not configured to automatically connect to the IDE, as this -gives issues with the requests made by haproxy as well as being unable to map all PHP applications to code in -EngineBlock. Simplest is to use a browser addon to enable xdebug for that request, such as [this Chrome addon][1] or -[this Firefox addon][2]. - -## Database Access - -In the development environment an additional database user is created that has full access to all databases that are -present in the VM. The credentials are username: `development` and password `development`. This user can connect from -outside the VM, allowing you to set up your favorite Database Administration Tool for usage with this VM. - -# How to set up the development environment - -## Create the required Directory Structure - -Due to the mounting requirements a specific directory structure is required. Easiest is to create a directory -`OpenConext` somewhere (e.g. `/opt/OpenConext`) and use that as root for all OpenConext projects. In this directory you -can checkout the OpenConext-deploy project without specifying a directory -(`git clone git@github.com:OpenConext/OpenConext-deploy.git`) Then repeat this for the OpenConect-engineblock project -(`git clone git@github.com:OpenConext/OpenConext-engineblock.git`). This creates the following directory structure: - -``` -/opt/OpenConext - ├── OpenConext-deploy - │ └── (project contents) - └── OpenConect-engineblock - └── (project contents) -``` - -In order to be able to run OpenConext EngineBlock, all dependencies must be installed by using [Composer][3] ( -installation instructions can be found [here][4]). This is done by navigating to the OpenConext-engineblock project -and running `composer install`. - -## Using Vagrant - -The development environment can be created using [Vagrant][5]. In order to be use the additional functionality -such as provisioning specifically for development, all vagrant commands must be prefixed with `ENV=dev`. -In order to start using the development environment, navigate to the OpenConext-deploy project and run -`$ ./provision devvm ` to start the VMs and start the provisioning. - -Alternatively you can start/restart, and perform other `vagrant` actions on the virtual machines as such: - -``` -$ ENV=dev vagrant reload -$ ENV=dev vagrant suspend lb_centos7 -``` - -## Provisioning - -The OpenConext suite is changing constantly, and updating your development evnironment with all the latest changes -should be done regularly. The best way to do this is using Ansible, we provided an utility for this. Some examples: - -``` -# Run all playbooks: -$ ./provision devvm - - -# Run a single tag, engineblock in this example. See ./playbook.yml for tagnames associated with specific roles -./provision devvm --tags eb - -``` - -[1]: https://chrome.google.com/webstore/detail/xdebug-helper/eadndfjplgieldjbigjakmdgkmoaaaoc -[2]: https://addons.mozilla.org/en-us/firefox/addon/the-easiest-xdebug/ -[3]: https://getcomposer.org/ -[4]: https://getcomposer.org/download/ -[5]: https://www.vagrantup.com/ diff --git a/README.md b/README.md index 7ebf46b42..76fb8da81 100644 --- a/README.md +++ b/README.md @@ -1,157 +1,98 @@ Ansible-driven provisioning of the OpenConext platform. ============================== -# Getting started +# Introduction -# Deploy to a remote machine +This repository contains everything you need if you want to use Ansible for deployment of OpenConext applications, including the Stepup suite. It does currently not provide you with a step by step manual to get the whole OpenConext suite installed and working. With some Ansible experience and some work, you will be able to use this repository to deploy the OpenConext applications however. The document will provide information on how to do that. -A manual to run the deploy to a single target machine (e.g. a hosted VM) is in the wiki: -[Installation steps to deploy OpenConext on a single system](https://github.com/OpenConext/OpenConext-deploy/wiki/Installation-steps-to-deploy-OpenConext-on-a-single-system-other-than-the-Vagrant-VM-centOS7). +If you want to get started with OpenConext, or with OpenConext development you can use our Docker compose based environment to get up and running quickly on a VM or your local laptop. Please refer to the devconf project that can be found here: https://github.com/OpenConext/OpenConext-devconf -# Deploy with Vagrant +# Contents of this repository -To run a development instance on your local machine with Vagrant and VirtualBox, follow these steps. -They are based on Mac OS X and the Open Source [Homebrew](http://brew.sh) package manager. +## Application roles +Every application has a seperate role to install it. The following roles can be found: -It is also possible to deploy using Vagrant and libvirt/qemu (on Linux). -Instructions are provided below. +| name | function | +| --- | --- | +| engine | Engineblock, the SAML proxy | +| oidcng | OpenID connect proxy | +| myconext | eduID | +| profile | Profile page | +| manage | Entity registration | +| teams | Group membership app | +| mujina | Mujina IdP | +| voot | Voot membership API | +| pdp | Policy Decicions API | +| attribute-aggregation | Attribute aggregation API | +| invite | Invite based groups | +| welcome | Invite UI | +| dashboard | IdP dashboard | +| lifecycle | User lifecycle | +| stats | Statistics | +| monitoring-tests | end2end monitoring app | +| diyidp | A SimpleSAMLphp based test IdP | +| stepupazuremfa | Stepup AzureMFA GSSP | +| stepuptiqr | Stepup TIQR GSSP | +| stepupwebauthn | Stepup Webauthn GSSP | +| stepupgateway | Stepup SAML gateway | +| stepupmiddleware | Stepup middleware | +| stepupra | Stepup ra interface | +| stepupselfservice | Stepup selfservice interface | -## Install Vagrant and VirtualBox +All these applications run in Docker. You can use the "docker" role to install docker and Traefik. The result is a Docker application server, with port 443 open. Applications are served by Traefik and recognized on basis of a Host: header. If you run a small installation, you can add a https certificate to Traefik and run a single node application server. -VirtualBox is a powerful x86 and AMD64/Intel64 virtualization product, downloads and user manual can be found on the [VirtualBox website](https://www.virtualbox.org/wiki/Downloads). -> Vagrant provides easy to configure, reproducible, and portable work environments built on top of industry-standard technology and controlled by a single consistent workflow to help maximize the productivity and flexibility of you and your team. +For a fully functioning environment you also need a MariaDB database server and a Mongo database server. -For installation instructions see [the website](https://docs.vagrantup.com/v2/installation/index.html). +## Infra roles +This repository is used for deployment of SURFconext, and several roles that the SURFconext teams uses to provision our infrastructure are provided here as well. You can use them for your own infrastructure or use them as inspiration. +| name | remarks | +| --- | --- | +| bind | DNS server for high availability. Very specific for SURFconext | +| docker | To deploy Docker and Traefik application servers | +| elk | Not maintained Elasticsearch, Logstash and Kibana role. For inspiration only | +| haproxy | Loadbalancer configuration. The role has its' own README | +| haproxy_mgnt | For red / blue deployments using haproxy | +| iptables | Manage your iptables based firewall | +| keepalived | VRRP config for HA between loadbalancers and database nodes | +| rsyslog | For central logging and parsing login statistics for stats | +| galera | Install multi master MariaDB database with galera. Runs on Rocky 9 | +| mongo | Install a mongo cluster (has its own README) | +| manage_provision_entities|Provision entities to Manage | -You will need at least Vagrant 1.7. Do not use Vagrant 1.8.5, which contains a bug that makes that the provisioning fails with the message "Warning: Authentication failure. Retrying...". Also, more recent versions (around 1.9.1) have problems detecting the network devices inside the VM, causing vagrant to fail to connect using ssh. +# Environment specific variables +Many variables can be overridden to create a setup suitable for your needs. The environment should be placed in the directory environments_external. -To install both with Homebrew: +A script is available to provision a new environment. It will create a new environment directory under environments-external/ and it will create all necessary passwords and (self-signed) certificates. Replace with the name of the target. Replace with the domain of the target. - brew cask install vagrant - brew cask install virtualbox - -With the above commands you get the latest versions. There might be incompatibilities. Vagrant will tell you and if you need a different version install cask versions and install the correct version of virtualbox and / or vagrant: - - brew tap homebrew/cask-versions - brew cask install virtualbox4330101610 - -## Install Ansible - -Ansible is the configuration tool we use to describe our servers. -Installation instruction can be found on the [Ansible website](http://docs.ansible.com/intro_installation.html). -The minimum required version of Ansible is 2.4. -To install for development with Homebrew: - - brew install python - pip install --upgrade setuptools - pip install --upgrade pip - brew install ansible - -## Run playbooks - -The VM will install everything on a two boxes for demo purposes. - -To provision the VM please run: - -```bash -Clone the repository: -git clone https://github.com/OpenConext/OpenConext-deploy.git -cd OpenConext-deploy -./provision vm -``` - -When the script is done, wait a little while to let all services come up and initialize themselves. Then point your browser to [https://welcome.vm.openconext.org](https://welcome.vm.openconext.org) - -These are the steps the above script performs: - -1. Setup a Vagrant VM and will make sure the HOSTS file is able to handle the defined base_domain -2. Setup a MariaDB server. -3. Inserts entities and metadata in Manage and initial load of engineblock to bootstrap. -4. Install all Java apps for the openconext platform. -5. Install all PHP apps for the openconext platform. -6. Install Haproxy for loadbalacing and SSL termination on the loadbalancer machine -7. Install [mujina](https://github.com/OpenConext/Mujina) as IDP and SP for the VM environment. - -## Add hostname entries to your own /etc/hosts file - -We need pseudo-DNS entries so that your browser can reach the VM-platform we just installed. So, add this very long line to your `/etc/hosts` file: ``` -192.168.66.98 welcome.vm.openconext.org static.vm.openconext.org metadata.vm.openconext.org db.vm.openconext.org engine.vm.openconext.org profile.vm.openconext.org mujina-sp.vm.openconext.org mujina-idp.vm.openconext.org teams.vm.openconext.org voot.vm.openconext.org pdp.vm.openconext.org engine-api.vm.openconext.org aa.vm.openconext.org link.vm.openconext.org manage.vm.openconext.org connect.vm.openconext.org +/prep-env ``` - -Here, the ip-address `192.168.66.98` refers to the address that is mentioned in ./Vagrantfile. - -## Enjoy your new VM! - -Go to [https://welcome.vm.openconext.org](https://welcome.vm.openconext.org). To ssh to the machines use the following: - +Then run ``` -vagrant ssh lb_centos7 -vagrant ssh apps_centos7 +cp environments-external//host_vars/template.yml environments-external//host_vars/.yml ``` +(where is the ip address or hostname of your target machine, whatever is set in your inventory file) -(using `vagrant ssh` without a VM specified leads to the Apps VM) - -The lb vm contains haproxy. The apps vm contains all the applications, apache and database. +Change in environments-external//inventory: +Change all references from %target_host% to -## Deploy using libvirt/qemu - -Instead of using Virtualbox as described above, it is also possible to use libvirt/qemu on Linux -machines. This requires a number of additional steps. - -1. Make sure you have a recent version of vagrant, and that libvirt/qemu is - working as expected for normal VMs (e.g., check if virt-manager works - correctly to create a new VM). -2. Install the `vagrant-libvirt` and `vagrant-mutate` plugins: - -``` -╰─▶ vagrant plugin install vagrant-libvirt -╰─▶ vagrant plugin install vagrant-mutate -``` - (or use the version provided by your distribution). -3. Download the Openconext base CentOS7 image. This is a Virtualbox-image, so - it needs to be converted to a libvirt-image using `vagrant mutate`: ``` -╰─▶ vagrant box add https://build.openconext.org/vagrant_boxes/virtualbox-centos7.box --name CentOS-7.0 -╰─▶ vagrant mutate CentOS-7.0 libvirt --force-virtio +Please note that this has not been tested in quite a while. You will need a lot of manual work to get this environment working ``` -4. Vagrant should now have two variants of the CentOS-7.0 image: -``` -╰─▶ vagrant box list -CentOS-7.0 (libvirt, 0) -CentOS-7.0 (virtualbox, 0) -``` -5. From a checked-out version of the OpenConext-deploy repository, run the - following command to check if the boxes come up: -``` -╰─▶ vagrant up --provider libvirt lb_centos7 -╰─▶ vagrant up --provider libvirt apps_centos7 -``` -(set the environment variable `VAGRANT_LOG=debug` to increase verbosity of -anything goes wrong. -6. You should be set to run the `./provision vm` command. - -# Releases to vm, test, acc, prod +# Playbooks, tags and the provision wrapper script -To update single applications - e.g. release - use tags: +Two playbooks exist in this repository: provision.yml and playbook_haproxy.yml. The latter can be used to do red/blue deployments if you also use our haproxy role. +The main playbook is provision.yml. It contains series of plays to install every role on the right node. All roles are tagged, so you can use the [Ansible tag mechanism](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_tags.html) to deploy a specific role. +If you would like to deploy manage to your test environment, you would run: ``` -./provision $env $remote_user path/to/your/secrets.yml --tags eb +ansible-playbook -i environments-external/test/inventory --tags manage -u THE_REMOTE_SSH_USER_WITH_SUDO_PERMISSIONS ``` -Where: -$env: Your environment. The vm is located in environments/vm. If you use your own repository you'll have to place it in environments_external -$remote_user: The remote user with sudo permissions -path/to/your/secrets.yml: The secrets used by Ansible are externalized. For the VM the secrets are in this GitHub repo. For other environments (your installation) they can be located in a separate repository. -# Making changes - -When making changes, please consider that people are continuously deploying -vm's from master. Therefore, please do your best to keep HEAD in a working -state, and make any invasive changes like adding new components or refactoring -on a separate branch. +A wrapper script which enables you to use your own roles can be used as well. That is documented here: https://github.com/OpenConext/OpenConext-deploy/wiki/Add-your-own-roles-and-playbooks # License @@ -162,17 +103,3 @@ These files are licensed under version 2.0 of the Apache License, as described i * You can ask questions on the [OpenConext mailing list](https://openconext.org/get-involved/mailing-lists/) * Or you can join our [Slack Workspace](https://edu.nl/ocslk) -# VM - -To provision the VM use the following (password is vagrant and sudo password is - -``` -ansible-playbook -u vagrant -i ./environments/vm/inventory -K --e secrets_file=./environments/vm/secrets/vm.yml provision-vm.yml -``` -To provision a certain role use tags: -``` -ansible-playbook -u vagrant -i ./environments/vm/inventory -K --e secrets_file=./environments/vm/secrets/vm.yml provision-vm.yml --tags vm_only_provision_manage_eb -``` - -Setting up a development environment is described in the file [DEVELOPMENT](DEVELOPMENT.md). - From c19587f6a49c8cc7e62fbf99d222e429a460e200 Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Wed, 9 Oct 2024 11:38:23 +0200 Subject: [PATCH 05/25] Simplify the provision script - Remove Vagrant VM stuff - Remove remove secrets_file location and use a default value - Remove the user argument for the provision script - enviromment_dir is no longer needed. The ansible magic var inventory_dir is reintroduced. You need to add localhost to the inventory for this to work --- environments/template/inventory | 3 + group_vars/all.yml | 2 +- provision | 119 +++++++------------------------- provision.yml | 10 ++- 4 files changed, 32 insertions(+), 102 deletions(-) diff --git a/environments/template/inventory b/environments/template/inventory index 49d109d5a..3a070aed8 100644 --- a/environments/template/inventory +++ b/environments/template/inventory @@ -87,3 +87,6 @@ java_apps_vm [app_oidc:children] oidc + +[local] +localhost ansible_connection=local diff --git a/group_vars/all.yml b/group_vars/all.yml index 0fc312cc4..2ceaa798e 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -2,7 +2,7 @@ # The develop flag is used for development-specific tasks/roles and is set to true via --extra-vars # By default, these should not be run develop: false - +secrets_file: "{{ inventory_dir }}/secrets/secrets.yml" # To enable a minimal installation the following variable should be enabled: minimal_install: false diff --git a/provision b/provision index f698f287b..616d86d77 100755 --- a/provision +++ b/provision @@ -2,22 +2,12 @@ set -e export ANSIBLE_ROLES_PATH="roles:roles-external" - -# helper function: check if a specified value is present in a space-separated list -listcontains() { - match=$1 - shift - for word in $@ - do - [[ $word = $match ]] && return 0 - done - return 1 -} - # parse parameters -help="Usage: $0 vm [ANSIBLE_OPT]\n $0 [ANSIBLE_OPT]" -if [ "$#" -eq 0 ] -then +help="Usage: $0 [ANSIBLE_OPT] +eg $0 acc --tags eb +will provision the tag eb on acc" + +if [ "$#" -eq 0 ]; then echo -e "$help" exit 1 fi @@ -25,92 +15,31 @@ fi env=$1 shift -if [ "$env" = "vm" ] || [ $env = "devvm" ] -then - user="vagrant" - secret="environments/vm/secrets/vm.yml" - environment_dir="environments/vm/" -elif [ $# -lt 2 ] -then - echo -e "$help" - exit 1 -else - user=$1 - secret=$2 - environment_dir="environments-external/$env/" - shift; shift -fi - -# Download extra roles when requirements.yml is present -if [ -f "$environment_dir"/requirements.yml ] -then - echo "Downloading roles to roles-external" - ansible-galaxy install -r "$environment_dir"/requirements.yml -f -p . -else - echo "No extra roles found to be downloaded" -fi - - -# find matching playbook +# Set some variables +environment_dir="environments-external/$env/" +inventory="environments-external/$env/inventory" playbook="provision.yml" -#listcontains "$env" "vm" && playbook="provision-vm.yml" -# find inventory -if [ $env = "vm" ] || [ $env = "template" ] -then - inventory="environments/$env/inventory" -elif [ $env = "devvm" ] -then - inventory="environments/vm/inventory" -else - inventory="environments-external/$env/inventory" -fi - -if ! [ -e $inventory ] -then - echo "Inventory file '$inventory' for environment '$env' not found." - exit 1 +if ! [ -e "$inventory" ]; then + echo "Inventory file '$inventory' for environment '$env' not found." + exit 1 fi -# run vm -if [ "$env" = "vm" ] -then - vagrant up -fi -if [ "$env" = "devvm" ] -then - ENV=dev vagrant up +# Download extra roles when requirements.yml is present +if [ -f "$environment_dir"/requirements.yml ]; then + echo "Downloading roles to roles-external" + ansible-galaxy install -r "$environment_dir"/requirements.yml -f -p . +else + echo "No extra roles found to be downloaded" fi -# run ansible -if [ $env = "vm" ] || [ $env = "template" ] -then - cmd=$(cat <<-EOF - ansible-playbook -i "$inventory" -u "$user" "$playbook" - --extra-vars="secrets_file=$secret" - --extra-vars="environment_dir=$environment_dir" - $@ - EOF +cmd=$( + cat <<-EOF +ansible-playbook -i $inventory $playbook $@ +EOF ) -elif [ $env = "devvm" ] -then - cmd=$(cat <<-EOF - ansible-playbook -i "$inventory" -u "$user" "$playbook" - --extra-vars="secrets_file=$secret" - --extra-vars="environment_dir=$environment_dir" - --extra-vars="@environments/vm/group_vars/dev.yml" - $@ - EOF - ) -else - cmd=$(cat <<-EOF - ansible-playbook -i $inventory -u $user $playbook - --extra-vars=secrets_file=$secret - --extra-vars=environment_dir=$environment_dir - $@ - EOF - ) -fi echo "executing $cmd" | tr -d "\n" | tr -s ' ' -$cmd +echo $cmd + +$cmd diff --git a/provision.yml b/provision.yml index 5a21697c3..fa3b0db24 100644 --- a/provision.yml +++ b/provision.yml @@ -102,10 +102,6 @@ when: - inventory_hostname not in groups['lifecycle'] tags: ['core', 'app_php', 'eb'] - - role: lifecycle - when: - - inventory_hostname in groups['lifecycle'] - tags: ['app_php', 'lifecycle'] handlers: - import_tasks: roles/httpd/handlers/main.yml @@ -189,5 +185,7 @@ - { role: lifecycle, tags: ["lifecycle"] } - { role: stepuptiqr, tags: ['stepuptiqr' , 'stepup'] } -- import_playbook: "{{ environment_dir }}/playbook.yml" - +- hosts: localhost + become: false + tasks: + - import_playbook: "{{ inventory_dir }}/playbook.yml" From f83e2e586ede47d78745b0e66a339fa69867976d Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Wed, 9 Oct 2024 11:55:39 +0200 Subject: [PATCH 06/25] Syntax check workflow: Use the latest versions of the actions --- .github/workflows/syntax.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/syntax.yml b/.github/workflows/syntax.yml index 80e9b4c4c..c4a1e63aa 100644 --- a/.github/workflows/syntax.yml +++ b/.github/workflows/syntax.yml @@ -17,12 +17,12 @@ on: jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.8 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.8 From e26e50553ba8aa698468e11e5abbde6fe66d9bb9 Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Thu, 14 Nov 2024 14:38:17 +0100 Subject: [PATCH 07/25] Add small Mongo docker role, for running on a single node --- provision.yml | 6 ++ roles/mongodbdocker/defaults/main.yml | 2 + roles/mongodbdocker/tasks/main.yml | 87 +++++++++++++++++++ .../templates/backup_mongo.pl.j2 | 37 ++++++++ 4 files changed, 132 insertions(+) create mode 100644 roles/mongodbdocker/defaults/main.yml create mode 100644 roles/mongodbdocker/tasks/main.yml create mode 100644 roles/mongodbdocker/templates/backup_mongo.pl.j2 diff --git a/provision.yml b/provision.yml index 5a21697c3..4d292a291 100644 --- a/provision.yml +++ b/provision.yml @@ -189,5 +189,11 @@ - { role: lifecycle, tags: ["lifecycle"] } - { role: stepuptiqr, tags: ['stepuptiqr' , 'stepup'] } +- hosts: docker_mariadb + become: true + roles: + - { role: mariadbdocker, tags: ['mariadbdocker']} + - { role: mongodbdocker, tags: ['mongodbdocker']} + - import_playbook: "{{ environment_dir }}/playbook.yml" diff --git a/roles/mongodbdocker/defaults/main.yml b/roles/mongodbdocker/defaults/main.yml new file mode 100644 index 000000000..c0095f38b --- /dev/null +++ b/roles/mongodbdocker/defaults/main.yml @@ -0,0 +1,2 @@ +replica_set_name: "{{ instance_name }}" +docker_mongodb_network_range: "172.21.22.0/24" diff --git a/roles/mongodbdocker/tasks/main.yml b/roles/mongodbdocker/tasks/main.yml new file mode 100644 index 000000000..fcc667d54 --- /dev/null +++ b/roles/mongodbdocker/tasks/main.yml @@ -0,0 +1,87 @@ +--- +- name: Install required packages + ansible.builtin.apt: + name: "python3-pymongo" + state: present + +- name: Create MongoDB volume + community.docker.docker_volume: + name: openconext_mongodb + state: present + +- name: Create MongoDB network + community.docker.docker_network: + name: openconext_mongodb + state: present + internal: false + ipam_config: + - subnet: "{{ docker_mongodb_network_range }}" + +- name: Create the MongoDB container + community.docker.docker_container: + name: openconext_mongodb + image: bitnami/mongodb:7.0 + state: started + pull: true + restart_policy: "always" + ports: "127.0.0.1:27017:27017" + networks: + - name: "openconext_mongodb" + mounts: + - type: volume + source: openconext_mongodb + target: /var/lib/mysql + - type: bind + source: /home/backup/mongo/ + target: /home/backup + env: + MONGODB_ROOT_USER: admin + MONGODB_ROOT_PASSWORD: "{{ mongo_admin_password }}" + MONGODB_REPLICA_SET_NAME: "{{ replica_set_name }}" + MONGODB_REPLICA_SET_MODE: primary + MONGODB_REPLICA_SET_KEY: "{{ mongodb_replicateset_key }}" + MONGODB_ADVERTISED_HOSTNAME: openconext_mongodb + volumes: + - openconext_mongodb:/bitnami/mongodb + hostname: openconext_mongodb + +- name: Create mongo database users + community.mongodb.mongodb_user: + login_database: admin + database: "{{ item.db_name }}" + login_user: admin + login_password: "{{ mongo_admin_password }}" + login_host: 127.0.0.1 + name: "{{ item.name }}" + password: "{{ item.password }}" + roles: readWrite + replica_set: "{{ replica_set_name }}" + strict_compatibility: false + no_log: false + run_once: true + with_items: "{{ mongo.users }}" + changed_when: false + tags: mongo_users + +- name: Create the backupdir + ansible.builtin.file: + path: /home/backup/mongo + owner: 1001 + group: 1001 + mode: "0700" + +- name: Install the backup script + ansible.builtin.template: + src: "backup_mongo.pl.j2" + dest: "/usr/local/sbin/backup_mongo.pl" + mode: "0700" + owner: root + group: root + +- name: Create cron symlink for backup script + ansible.builtin.file: + src: "/usr/local/sbin/backup_mongo.pl" + dest: "/etc/cron.daily/mongodb_backup" + state: link + mode: "0700" + owner: root diff --git a/roles/mongodbdocker/templates/backup_mongo.pl.j2 b/roles/mongodbdocker/templates/backup_mongo.pl.j2 new file mode 100644 index 000000000..c8e014742 --- /dev/null +++ b/roles/mongodbdocker/templates/backup_mongo.pl.j2 @@ -0,0 +1,37 @@ +#!/usr/bin/perl +# Variables + +$backupdir = "/home/backup"; +$username = "admin"; +$password = "{{ mongo_admin_password }}"; + +umask 0077; + +# Determine current day +$day = `/bin/date +'%a'`; +chomp($day); + +# Remove old backups if exists +if ( -e "$backupdir/mongo-dump-$day/") { +`rm -rf $backupdir/mongo-dump-$day/`; +} + +# Dump databases +`docker exec openconext_mongodb mongodump --username $username --password $password --authenticationDatabase admin --out $backupdir/mongo-dump-$day`; + +# Gzip dumps +opendir(BDIR, "$backupdir/mongo-dump-$day/"); +my @files = readdir(BDIR); +closedir(BDIR); +chdir("$backupdir/mongo-dump-$day/"); +foreach $dir (@files) { +if ($dir !~ /^\.+$/) { +if ($dir !~ /\.\./g) { +if ( -d "$backupdir/mongo-dump-$day/$dir") { +`tar -cvzf $backupdir/mongo-dump-$day/$dir.tar.gz $dir/`; +`rm -rf $backupdir/mongo-dump-$day/$dir/`; +} +} +} +} +umask 0022; From 268fc153390a206f79afd3869cf5f1cb41dce9ef Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Tue, 26 Nov 2024 21:54:54 +0100 Subject: [PATCH 08/25] Manage: make it possible to connect to a locally available mongo in docker --- roles/manage/defaults/main.yml | 7 +++++-- roles/manage/tasks/main.yml | 10 ++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/roles/manage/defaults/main.yml b/roles/manage/defaults/main.yml index ef5c4271d..bc41cd266 100644 --- a/roles/manage/defaults/main.yml +++ b/roles/manage/defaults/main.yml @@ -1,8 +1,8 @@ --- manage_dir: /config/ -manage_snapshot_timestamp: '' +manage_snapshot_timestamp: "" manage_jar: manage-current.jar -manage_random_source: 'file:///dev/urandom' +manage_random_source: "file:///dev/urandom" manage_cronjobmaster: false manage_disclaimer_background_color: "{{ environment_ribbon_colour }}" manage_disclaimer_content: "{{ environment_shortname }}" @@ -30,3 +30,6 @@ manage_tabs_enabled: - single_tenant_template - provisioning - sram + +manage_docker_networks: + - name: loadbalancer diff --git a/roles/manage/tasks/main.yml b/roles/manage/tasks/main.yml index 9a4132c42..3ab287de4 100644 --- a/roles/manage/tasks/main.yml +++ b/roles/manage/tasks/main.yml @@ -72,6 +72,13 @@ notify: - "restart manageserver" +- name: Add the mongodb docker network to the list of networks when MongoDB runs in Docker + ansible.builtin.set_fact: + manage_docker_networks: + - name: loadbalancer + - name: openconext_mongodb + when: mongodb_in_docker | default(false) | bool + - name: Create and start the server container community.docker.docker_container: name: manageserver @@ -80,8 +87,7 @@ pull: true restart_policy: "always" state: started - networks: - - name: "loadbalancer" + networks: "{{ manage_docker_networks }}" mounts: - source: /opt/openconext/manage/ target: /config/ From 053df90b31706cfbc536307c43b28550fc1883fb Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Tue, 26 Nov 2024 21:56:38 +0100 Subject: [PATCH 09/25] Myconext: make it possible to connect to a locally available mongo in docker --- roles/myconext/defaults/main.yml | 2 ++ roles/myconext/tasks/main.yml | 14 ++++++++++---- roles/myconext/templates/application.yml.j2 | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/roles/myconext/defaults/main.yml b/roles/myconext/defaults/main.yml index 2fa088e32..4082fa28d 100644 --- a/roles/myconext/defaults/main.yml +++ b/roles/myconext/defaults/main.yml @@ -1,2 +1,4 @@ --- myconext_cronjobmaster: true +myconext_docker_networks: + - name: loadbalancer diff --git a/roles/myconext/tasks/main.yml b/roles/myconext/tasks/main.yml index c5d424713..167b35523 100644 --- a/roles/myconext/tasks/main.yml +++ b/roles/myconext/tasks/main.yml @@ -56,7 +56,7 @@ notify: - "restart myconextserver" -- name: copy / create private key +- name: Copy / create private key ansible.builtin.copy: content: "{{ myconext_private_key }}" dest: "/opt/openconext/myconext/myconext_saml.key" @@ -66,7 +66,7 @@ notify: - "restart myconextserver" -- name: copy / create certificate +- name: Copy / create certificate ansible.builtin.copy: src: "{{ inventory_dir }}/files/certs/myconext/myconext_saml.crt" dest: "/opt/openconext/myconext/myconext_saml.crt" @@ -92,6 +92,13 @@ group: "root" mode: "0755" +- name: Add the mongodb docker network to the list of networks when MongoDB runs in Docker + ansible.builtin.set_fact: + myconext_docker_networks: + - name: loadbalancer + - name: openconext_mongodb + when: mongodb_in_docker | default(false) | bool + - name: Create and start the server container community.docker.docker_container: name: myconextserver @@ -102,8 +109,7 @@ env: USE_SYSTEM_CA_CERTS: "1" TZ: "{{ timezone }}" - networks: - - name: "loadbalancer" + networks: "{{ myconext_docker_networks }}" mounts: - source: /opt/openconext/myconext/ target: /config/ diff --git a/roles/myconext/templates/application.yml.j2 b/roles/myconext/templates/application.yml.j2 index 2502621b8..081a8196e 100644 --- a/roles/myconext/templates/application.yml.j2 +++ b/roles/myconext/templates/application.yml.j2 @@ -215,7 +215,7 @@ verify: spring: data: mongodb: - uri: mongodb://{{ myconext.mongo_user }}:{{ myconext.mongo_password }}@{% for host in groups['mongo_servers'] %}{{ hostvars[host]['inventory_hostname'] }}:{{ myconext.mongo_port }}{% if not loop.last %},{% endif %}{% endfor %}/{{ myconext.mongo_database }}?ssl=true + uri: mongodb://{{ myconext.mongo_user }}:{{ myconext.mongo_password }}@{% for host in groups['mongo_servers'] %}{{ hostvars[host]['inventory_hostname'] }}:{{ myconext.mongo_port }}{% if not loop.last %},{% endif %}{% endfor %}/{{ myconext.mongo_database }}?ssl={{ mongodb_ssl | default('true') }} mail: host: {{ smtp_server }} From 6a0d3e1765a7cd41d470283d61e70e8dd32ba561 Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Tue, 26 Nov 2024 21:57:12 +0100 Subject: [PATCH 10/25] OIDCNG: make it possible to connect to a locally available mongo in docker --- roles/oidcng/defaults/main.yml | 7 ++++--- roles/oidcng/tasks/main.yml | 13 ++++++++----- roles/oidcng/templates/application.yml.j2 | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/roles/oidcng/defaults/main.yml b/roles/oidcng/defaults/main.yml index cc444fba4..44641c6c1 100644 --- a/roles/oidcng/defaults/main.yml +++ b/roles/oidcng/defaults/main.yml @@ -1,8 +1,8 @@ --- oidcng_dir: /opt/openconext/oidcng oidcng_config_dir: /config -oidcng_version: '' -oidcng_snapshot_timestamp: '' +oidcng_version: "" +oidcng_snapshot_timestamp: "" oidcng_cronjobmaster: true oidcng_saml_sp_entityid: https://connect.{{ base_domain }} oidcng_idp_metadata_url: https://engine.{{ base_domain }}/authentication/idp/metadata @@ -19,4 +19,5 @@ oidcng_manage_provision_samlsp_metadata_url: "https://connect.{{ base_domain }}/ oidcng_manage_provision_samlsp_sp_cert: "{{ lookup('file', '{{ inventory_dir }}/files/certs/oidc/oidcsaml.crt') | depem }}" oidcng_manage_provision_samlsp_sign: "True" oidcng_manage_provision_samlsp_trusted_proxy: "True" - +oidcng_docker_networks: + - name: loadbalancer diff --git a/roles/oidcng/tasks/main.yml b/roles/oidcng/tasks/main.yml index fdd8834e8..fa35fac7c 100644 --- a/roles/oidcng/tasks/main.yml +++ b/roles/oidcng/tasks/main.yml @@ -88,6 +88,13 @@ group: "root" mode: "0755" +- name: Add the mongodb docker network to the list of networks when MongoDB runs in Docker + ansible.builtin.set_fact: + oidcng_docker_networks: + - name: loadbalancer + - name: openconext_mongodb + when: mongodb_in_docker | default(false) | bool + - name: Create and start the server container community.docker.docker_container: name: oidcngserver @@ -96,8 +103,7 @@ pull: true restart_policy: "always" state: started - networks: - - name: "loadbalancer" + networks: "{{ oidcng_docker_networks }}" mounts: - source: "{{ oidcng_dir }}" target: /config/ @@ -137,9 +143,6 @@ traefik.http.middlewares.oidcngmw.replacepathregex.regex: "^/.well-known/openid-configuration" traefik.http.middlewares.oidcngmw.replacepathregex.replacement: "/oidc/.well-known/openid-configuration" register: oidcngservercontainer - - - #- name: Include the role manage_provision_entities to provision oidncg to Manage # ansible.builtin.include_role: # name: manage_provision_entities diff --git a/roles/oidcng/templates/application.yml.j2 b/roles/oidcng/templates/application.yml.j2 index f4595497c..c4f4d58d6 100644 --- a/roles/oidcng/templates/application.yml.j2 +++ b/roles/oidcng/templates/application.yml.j2 @@ -76,7 +76,7 @@ idp: spring: data: mongodb: - uri: "mongodb://{{ oidcng.mongo_user }}:{{ oidcng.mongo_password }}@{% for host in groups['mongo_servers'] %}{{ hostvars[host]['inventory_hostname'] }}:{{ oidcng.mongo_port }}{% if not loop.last %},{% endif %}{%endfor %}/{{ oidcng.mongo_database }}?ssl=true" + uri: "mongodb://{{ oidcng.mongo_user }}:{{ oidcng.mongo_password }}@{% for host in groups['mongo_servers'] %}{{ hostvars[host]['inventory_hostname'] }}:{{ oidcng.mongo_port }}{% if not loop.last %},{% endif %}{%endfor %}/{{ oidcng.mongo_database }}?ssl={{ mongodb_ssl | default('true') }}" thymeleaf: cache: true From ddf04a368344f2495ab9c1aee61d62b8dd49cb15 Mon Sep 17 00:00:00 2001 From: Pieter van der Meulen Date: Tue, 10 Dec 2024 10:18:20 +0100 Subject: [PATCH 11/25] Add activation flow sellection using entitlement attribute for selfservice 5.0.5 --- roles/stepupselfservice/templates/parameters.yml.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/stepupselfservice/templates/parameters.yml.j2 b/roles/stepupselfservice/templates/parameters.yml.j2 index cb2f25759..54c6e1bb3 100644 --- a/roles/stepupselfservice/templates/parameters.yml.j2 +++ b/roles/stepupselfservice/templates/parameters.yml.j2 @@ -89,6 +89,11 @@ parameters: preferred_activation_flow_name: activate preferred_activation_flow_options: [ra, self] + activation_flow_attribute_name: urn:mace:dir:attribute-def:eduPersonEntitlement + activation_flow_attributes: + ra: urn:mace:surf.nl:surfsecureid:activation:ra + self: urn:mace:surf.nl:surfsecureid:activation:self + # Self-asserted tokens: enable/disable recovery methods # # One of the two options should be enabled to have a fully functioning From 5f77f8ec83e47f311cf52f8b4eaead74a962051e Mon Sep 17 00:00:00 2001 From: Okke Harsta Date: Wed, 11 Dec 2024 10:15:41 +0100 Subject: [PATCH 12/25] https://github.com/OpenConext/OpenConext-myconext/issues/606 --- environments/template/group_vars/template.yml | 1 + environments/vm/group_vars/vm.yml | 1 + roles/myconext/templates/application.yml.j2 | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/environments/template/group_vars/template.yml b/environments/template/group_vars/template.yml index 0512e06fc..5093e0f92 100644 --- a/environments/template/group_vars/template.yml +++ b/environments/template/group_vars/template.yml @@ -420,6 +420,7 @@ myconext: feature_id_verify: true feature_remote_creation_api: true feature_mail_institution_mail_usage: true + feature_mail_inactivity_mails: true feature_nudge_app_mail: true feature_send_js_exceptions: true feature_deny_disposable_email_providers: true diff --git a/environments/vm/group_vars/vm.yml b/environments/vm/group_vars/vm.yml index ad0cb3160..1ed11a8a1 100644 --- a/environments/vm/group_vars/vm.yml +++ b/environments/vm/group_vars/vm.yml @@ -311,6 +311,7 @@ myconext: feature_id_verify: true feature_remote_creation_api: true feature_mail_institution_mail_usage: true + feature_mail_inactivity_mails: true feature_nudge_app_mail: true feature_send_js_exceptions: true feature_deny_disposable_email_providers: true diff --git a/roles/myconext/templates/application.yml.j2 b/roles/myconext/templates/application.yml.j2 index 4b180a428..66ac68c53 100644 --- a/roles/myconext/templates/application.yml.j2 +++ b/roles/myconext/templates/application.yml.j2 @@ -61,6 +61,8 @@ cron: nudge-app-mail-expression: "0 30 6 * * ?" # Number of days after creation of the eduID account which the nudge mail is send nudge-app-mail-days-after-creation: 14 + # Every day at 7:30AM + inactivity-users-expression: "0 30 7 * * ?" manage: username: myconext @@ -111,6 +113,8 @@ feature: send_js_exceptions: {{ myconext.feature_send_js_exceptions }} # Do we periodically mail users who have used their institution account mail_institution_mail_usage: {{ myconext.feature_mail_institution_mail_usage }} + # Do we periodically mail users who are inactive and might have their account deleted + mail_inactivity_mails: {{ myconext.feature_mail_inactivity_mails }} # Do we mail users who have not installed the eduID app nudge_app_mail: {{ myconext.feature_nudge_app_mail }} From cdb57e463b9e4777765d509612709718d5d42b75 Mon Sep 17 00:00:00 2001 From: Bas Zoetekouw Date: Fri, 13 Dec 2024 14:36:30 +0100 Subject: [PATCH 13/25] MANAGE: add `coin:disable_scoping` by defualt for new IdPs (closes OpenConext/OpenConext-Manage#458) --- roles/manage/files/metadata_templates/saml20_idp.template.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/manage/files/metadata_templates/saml20_idp.template.json b/roles/manage/files/metadata_templates/saml20_idp.template.json index 8698863f2..3c29ee713 100644 --- a/roles/manage/files/metadata_templates/saml20_idp.template.json +++ b/roles/manage/files/metadata_templates/saml20_idp.template.json @@ -9,7 +9,8 @@ "name:en": "", "OrganizationName:en": "", "SingleSignOnService:0:Binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect", - "SingleSignOnService:0:Location": "" + "SingleSignOnService:0:Location": "", + "coin:disable_scoping": true }, "autoRefresh": { "enabled": true, From e9ec11dd8ad3f62ea55c663f97b802a0768bccc7 Mon Sep 17 00:00:00 2001 From: Bas Zoetekouw Date: Mon, 16 Dec 2024 17:32:03 +0100 Subject: [PATCH 14/25] add dependabot got github-actions --- .github/dependabot.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..23c4cb3b5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +--- +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" From 5d82362487958511d91b8d72333d6b0f48f66f44 Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Tue, 17 Dec 2024 10:33:22 +0100 Subject: [PATCH 15/25] Manage: Make the SSL connection to Mongo optional, usefull in docker scenarios --- roles/manage/templates/application.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/manage/templates/application.yml.j2 b/roles/manage/templates/application.yml.j2 index be4189448..89be30d50 100644 --- a/roles/manage/templates/application.yml.j2 +++ b/roles/manage/templates/application.yml.j2 @@ -79,7 +79,7 @@ spring: port: 25 data: mongodb: - uri: mongodb://{{ manage.mongo_user }}:{{ manage.mongo_password }}@{% for host in groups['mongo_servers'] %}{{ hostvars[host]['inventory_hostname'] }}:{{ manage.mongo_port }}{% if not loop.last %},{% endif %}{% endfor %}/{{ manage.mongo_database }}?ssl=true + uri: mongodb://{{ manage.mongo_user }}:{{ manage.mongo_password }}@{% for host in groups['mongo_servers'] %}{{ hostvars[host]['inventory_hostname'] }}:{{ manage.mongo_port }}{% if not loop.last %},{% endif %}{% endfor %}/{{ manage.mongo_database }}?ssl={{ mongodb_ssl | default('true') }} datasource: url: jdbc:mysql://{{ pdp.db_host }}/{{ pdp.db_name }}?permitMysqlScheme username: {{ pdp.db_user }} From eb478b09c61721663b19d78b484ea2818d9425a6 Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Tue, 17 Dec 2024 10:34:00 +0100 Subject: [PATCH 16/25] Manage: fix the docker networks for manage. Soon the mariadb network will be obsolete for manage --- roles/manage/tasks/main.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/roles/manage/tasks/main.yml b/roles/manage/tasks/main.yml index 6818f3a0d..8fc4666a7 100644 --- a/roles/manage/tasks/main.yml +++ b/roles/manage/tasks/main.yml @@ -72,18 +72,13 @@ notify: - "restart manageserver" -- name: Add the mongodb docker network to the list of networks when MongoDB runs in Docker +- name: Add the mongodb and mariadb docker network to the list of networks when MongoDB runs in Docker ansible.builtin.set_fact: manage_docker_networks: - name: loadbalancer - name: openconext_mongodb - when: mongodb_in_docker | default(false) | bool -- name: Add the MariaDB docker network to the list of networks when MariaDB runs in Docker - ansible.builtin.set_fact: - manage_docker_networks: - - name: loadbalancer - name: openconext_mariadb - when: mariadb_in_docker | default(false) | bool + when: mongodb_in_docker | default(false) | bool - name: Create and start the server container community.docker.docker_container: From 466ab08144ab1b1136217201e0b6b5d2f6f2cdf3 Mon Sep 17 00:00:00 2001 From: Okke Harsta Date: Tue, 17 Dec 2024 10:35:09 +0100 Subject: [PATCH 17/25] https://github.com/OpenConext/OpenConext-dashboard/issues/591 --- .../saml20_idp.schema.json.j2 | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/roles/manage/templates/metadata_configuration/saml20_idp.schema.json.j2 b/roles/manage/templates/metadata_configuration/saml20_idp.schema.json.j2 index 3f32dc622..edaf89541 100644 --- a/roles/manage/templates/metadata_configuration/saml20_idp.schema.json.j2 +++ b/roles/manage/templates/metadata_configuration/saml20_idp.schema.json.j2 @@ -371,20 +371,6 @@ "default": false, "info": "Must be set for this Identity Provider to be excluded in the EB push." }, - "coin:allow_maintainers_to_manage_authz_rules": { - "type": "boolean", - "info": "Set this to allow maintainers of this Identity Provider to maintain authorization rules. Used in Dashboard." - }, - "coin:display_admin_emails_in_dashboard": { - "type": "boolean", - "default": false, - "info": "Set this to allow regular users of this Identity Provider to see the SCB and SCV emails. Used in Dashboard." - }, - "coin:display_stats_in_dashboard": { - "type": "boolean", - "default": false, - "info": "Set this to allow regular users of this Identity Provider to see the statistics. Used in Dashboard." - } }, "patternProperties": { "^name:({{ supported_language_codes | replace(',','|') }})$": { From 23834fe2e03af9c5fd0b53f3670cf7c9a199177c Mon Sep 17 00:00:00 2001 From: Thijs Kinkhorst Date: Tue, 17 Dec 2024 11:37:09 +0100 Subject: [PATCH 18/25] Phased out `sab` in favour of `sabrest` --- .../templates/metadata_configuration/oidc10_rp.schema.json.j2 | 1 - .../templates/metadata_configuration/saml20_sp.schema.json.j2 | 1 - .../metadata_configuration/single_tenant_template.schema.json.j2 | 1 - .../manage/templates/metadata_configuration/sram.schema.json.j2 | 1 - 4 files changed, 4 deletions(-) diff --git a/roles/manage/templates/metadata_configuration/oidc10_rp.schema.json.j2 b/roles/manage/templates/metadata_configuration/oidc10_rp.schema.json.j2 index 31386095d..ff5159f1e 100644 --- a/roles/manage/templates/metadata_configuration/oidc10_rp.schema.json.j2 +++ b/roles/manage/templates/metadata_configuration/oidc10_rp.schema.json.j2 @@ -130,7 +130,6 @@ "invite", "manage", "orcid", - "sab", "sabrest", "voot", ], diff --git a/roles/manage/templates/metadata_configuration/saml20_sp.schema.json.j2 b/roles/manage/templates/metadata_configuration/saml20_sp.schema.json.j2 index a102ec194..df9a4526a 100644 --- a/roles/manage/templates/metadata_configuration/saml20_sp.schema.json.j2 +++ b/roles/manage/templates/metadata_configuration/saml20_sp.schema.json.j2 @@ -173,7 +173,6 @@ "invite", "manage", "orcid", - "sab", "sabrest", "voot", ], diff --git a/roles/manage/templates/metadata_configuration/single_tenant_template.schema.json.j2 b/roles/manage/templates/metadata_configuration/single_tenant_template.schema.json.j2 index 7ec1e09c8..3f359cbde 100644 --- a/roles/manage/templates/metadata_configuration/single_tenant_template.schema.json.j2 +++ b/roles/manage/templates/metadata_configuration/single_tenant_template.schema.json.j2 @@ -122,7 +122,6 @@ "invite", "manage", "orcid", - "sab", "sabrest", "voot", ], diff --git a/roles/manage/templates/metadata_configuration/sram.schema.json.j2 b/roles/manage/templates/metadata_configuration/sram.schema.json.j2 index 95d413efa..aeb7f4409 100644 --- a/roles/manage/templates/metadata_configuration/sram.schema.json.j2 +++ b/roles/manage/templates/metadata_configuration/sram.schema.json.j2 @@ -153,7 +153,6 @@ "manage", "orcid", "pseudo_email", - "sab", "sabrest", "voot" ], From e77dd33202f7585b4f5b397b3b604c755a40e08f Mon Sep 17 00:00:00 2001 From: Tim van Dijen Date: Tue, 17 Dec 2024 12:10:00 +0100 Subject: [PATCH 19/25] Update url Old one doesn't exist anymore --- roles/oidcng/templates/application.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/oidcng/templates/application.yml.j2 b/roles/oidcng/templates/application.yml.j2 index c4f4d58d6..268a3aa68 100644 --- a/roles/oidcng/templates/application.yml.j2 +++ b/roles/oidcng/templates/application.yml.j2 @@ -10,7 +10,7 @@ cron: node-cron-job-responsible: {{ oidcng_cronjobmaster }} # Every 15 minutes token-cleaner-expression: "0 0/15 * * * *" - # Differs per env: see https://docs.spring.io/spring/docs/current/javadoc-api/org/springframework/scheduling/support/CronSequenceGenerator.html + # Differs per env: see https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/scheduling/support/CronExpression.html key-rollover-expression: {{ oidcng.key_rollover_cron_expression }} consent-expiry-duration-days: 90 From fc79c175d1894ad7914100d7d513607807732fcd Mon Sep 17 00:00:00 2001 From: Okke Harsta Date: Tue, 17 Dec 2024 14:46:38 +0100 Subject: [PATCH 20/25] Part of https://github.com/OpenConext/OpenConext-pdp/issues/233 --- roles/pdp/templates/serverapplication.properties.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/pdp/templates/serverapplication.properties.j2 b/roles/pdp/templates/serverapplication.properties.j2 index 27fd45f51..47a0eca3d 100644 --- a/roles/pdp/templates/serverapplication.properties.j2 +++ b/roles/pdp/templates/serverapplication.properties.j2 @@ -74,9 +74,9 @@ voot.clientSecret={{ pdp_client_secret }} voot.scopes ={{ pdp_manage_provision_oidcrp_scopes }} # SAB configuration -sab.userName={{ pdp.sab_username }} -sab.password={{ pdp.sab_password }} -sab.endpoint={{ pdp.sab_endpoint }} +sab.userName="{{ aa.sab_rest_username }}" +sab.password="{{ aa.sab_rest_password }}" +sab.endpoint="{{ aa.sab_rest_endpoint }}" # Do not expose any of the Spring BOOT endpoints except the health and info for loadbalancer health check endpoints.enabled=false From 5357fc795a23fcddfdabab05a2631c93c9a23640 Mon Sep 17 00:00:00 2001 From: Okke Harsta Date: Tue, 17 Dec 2024 15:17:09 +0100 Subject: [PATCH 21/25] Do not encode sab variables --- roles/pdp/defaults/main.yml | 2 +- roles/pdp/templates/serverapplication.properties.j2 | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/pdp/defaults/main.yml b/roles/pdp/defaults/main.yml index 687f5f5ee..782635029 100644 --- a/roles/pdp/defaults/main.yml +++ b/roles/pdp/defaults/main.yml @@ -22,4 +22,4 @@ pdp_manage_provision_samlsp_sign: false pdp_spring_flyway_enabled: true pdp_manage_push_testmode: true pdp_docker_networks: - -name: loadbalancer + - name: loadbalancer diff --git a/roles/pdp/templates/serverapplication.properties.j2 b/roles/pdp/templates/serverapplication.properties.j2 index 47a0eca3d..2afe79c37 100644 --- a/roles/pdp/templates/serverapplication.properties.j2 +++ b/roles/pdp/templates/serverapplication.properties.j2 @@ -74,9 +74,9 @@ voot.clientSecret={{ pdp_client_secret }} voot.scopes ={{ pdp_manage_provision_oidcrp_scopes }} # SAB configuration -sab.userName="{{ aa.sab_rest_username }}" -sab.password="{{ aa.sab_rest_password }}" -sab.endpoint="{{ aa.sab_rest_endpoint }}" +sab.userName={{ aa.sab_rest_username }} +sab.password={{ aa.sab_rest_password }} +sab.endpoint={{ aa.sab_rest_endpoint }} # Do not expose any of the Spring BOOT endpoints except the health and info for loadbalancer health check endpoints.enabled=false From f6904c543f3bde0e26fc3a5c48c19dd14bb92bda Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Thu, 19 Dec 2024 21:03:03 +0100 Subject: [PATCH 22/25] Main playbook: Fix import playbook --- provision | 2 +- provision.yml | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/provision b/provision index 616d86d77..f62b5f0db 100755 --- a/provision +++ b/provision @@ -35,7 +35,7 @@ fi cmd=$( cat <<-EOF -ansible-playbook -i $inventory $playbook $@ +ansible-playbook -i $inventory $playbook -e environment_dir=$environment_dir $@ EOF ) echo "executing $cmd" | tr -d "\n" | tr -s ' ' diff --git a/provision.yml b/provision.yml index 7aec2358a..d4304c82c 100644 --- a/provision.yml +++ b/provision.yml @@ -177,7 +177,4 @@ - { role: mariadbdocker, tags: ['mariadbdocker']} - { role: mongodbdocker, tags: ['mongodbdocker']} -- hosts: localhost - become: false - tasks: - - import_playbook: "{{ inventory_dir }}/playbook.yml" +- import_playbook: "{{ environment_dir }}/playbook.yml" From 461d9a6a9cc6e2b742baf45b6137e8b4093d45e6 Mon Sep 17 00:00:00 2001 From: Okke Harsta Date: Sat, 21 Dec 2024 08:22:59 +0100 Subject: [PATCH 23/25] INFO logging idp-dashboard --- roles/dashboard/templates/logback.xml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/dashboard/templates/logback.xml.j2 b/roles/dashboard/templates/logback.xml.j2 index a73a85e17..f31a69c85 100644 --- a/roles/dashboard/templates/logback.xml.j2 +++ b/roles/dashboard/templates/logback.xml.j2 @@ -20,7 +20,7 @@ - + From 0ae6887787df37f41122f0756f86bc30eccfdba0 Mon Sep 17 00:00:00 2001 From: Okke Harsta Date: Mon, 23 Dec 2024 13:39:48 +0100 Subject: [PATCH 24/25] https://github.com/OpenConext/OpenConext-dashboard/issues/654 --- roles/dashboard/templates/serverapplication.yml.j2 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/roles/dashboard/templates/serverapplication.yml.j2 b/roles/dashboard/templates/serverapplication.yml.j2 index 11576ef72..410d97f70 100644 --- a/roles/dashboard/templates/serverapplication.yml.j2 +++ b/roles/dashboard/templates/serverapplication.yml.j2 @@ -26,10 +26,6 @@ spDashboard.username={{ dashboard.sp_dashboard_user }} spDashboard.password={{ dashboard_sp_dashboard_password }} # SAB connection details -sab.endpoint={{ dashboard.sab_endpoint }} -sab.username=cdk -sab.password={{ dashboard_sab_password }} - sab-rest.endpoint={{ dashboard.sab_rest_endpoint }} sab-rest.username=cdk sab-rest.password={{ dashboard_sab_rest_password }} From ddd5528037d273622df8aeb9e8ecb55f807c08d7 Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Sun, 5 Jan 2025 20:13:11 +0100 Subject: [PATCH 25/25] SPD: Fix base_domain parameter name --- roles/spdashboard/templates/env.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/spdashboard/templates/env.j2 b/roles/spdashboard/templates/env.j2 index 6fa3dba68..c8fe05e91 100644 --- a/roles/spdashboard/templates/env.j2 +++ b/roles/spdashboard/templates/env.j2 @@ -13,7 +13,7 @@ mail_receiver={{ spdashboard_support_email }} secret=ThisTokenIsNotSoSecretChangeIt session_max_absolute_lifetime=32400 session_max_relative_lifetime=14400 -logout_redirect_url=https://engine.{{ spashboard_base_domain }}/logout +logout_redirect_url=https://engine.{{ spdashboard_base_domain }}/logout administrator_teams={% for team in spdashboard.adminteams %}{{ team }}{% if not loop.last %},{% endif %}{% endfor %} saml_sp_publickey={{ spdashboard_saml_sp_publickey }}