From c9870bb918ac74efbbcf1ab8b4ee67f30f771faf Mon Sep 17 00:00:00 2001 From: Algorhythm and Funktion Date: Thu, 22 Apr 2021 12:22:29 +0200 Subject: [PATCH 1/7] Open Justice Playbook --- apps/open-justice/README.md | 48 +++++++++++++++++ apps/open-justice/aleph.env | 103 ++++++++++++++++++++++++++++++++++++ apps/open-justice/aleph.yml | 59 +++++++++++++++++++++ inventory/prod.yml | 4 +- 4 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 apps/open-justice/README.md create mode 100644 apps/open-justice/aleph.env create mode 100644 apps/open-justice/aleph.yml diff --git a/apps/open-justice/README.md b/apps/open-justice/README.md new file mode 100644 index 0000000..a889bdb --- /dev/null +++ b/apps/open-justice/README.md @@ -0,0 +1,48 @@ +Open Justice Aleph +================== + +Install `ansible-role-docker` +---------------------------- + +``` +ansible-galaxy install geerlingguy.docker +``` + +Post playbook +------------- +Start aleph in detached mode and set up tables + +``` +docker-compose up -d +docker-compose run --rm shell upgrade +``` + +To set up a user +---------------- +Enter the Aleph shell: + +``` +make shell +``` + +Then: + +``` +aleph createuser --name="Alice" \ + --admin \ + --password=123abc \ + user@example.com +``` + +Load sample data +--------------- + +``` +aleph crawldir /aleph/contrib/testdata +``` + + + + + + diff --git a/apps/open-justice/aleph.env b/apps/open-justice/aleph.env new file mode 100644 index 0000000..54e3b36 --- /dev/null +++ b/apps/open-justice/aleph.env @@ -0,0 +1,103 @@ +# Aleph environment configuration +# +# This file is loaded by docker-compose and transformed into a set of +# environment variables inside the containers. These are, in turn, parsed +# by aleph and used to configure the system. + +# Random string: +#ALEPH_SECRET_KEY= + +# Visible instance name in the UI +ALEPH_APP_TITLE=Open Justice +# Name needs to be a slug, as it is used e.g. for the ES index, SQS queue name: +ALEPH_APP_NAME=open-justice +ALEPH_UI_URL=http://openjustice.openup.org.za:8080/ + +# ALEPH_URL_SCHEME=https +# ALEPH_FAVICON=https://investigativedashboard.org/static/favicon.ico +# ALEPH_LOGO=http://assets.pudo.org/img/logo_bigger.png + +# Other customisations +ALEPH_SAMPLE_SEARCHES=Vladimir Putin:TeliaSonera + +# Set email addresses, separated by colons, that will be made admin. +# ALEPH_ADMINS=friedrich@pudo.org:demo@pudo.org + +# Single-user mode disables authentication and authorization autologging in any user as admin +ALEPH_SINGLE_USER=false + +# Login modalities +ALEPH_PASSWORD_LOGIN=true + +# OAuth configuration +# Currently supported providers are Google, Facebook, Cognito and Azure AD OAuth +# Note that you do not need to fill out all fields in order to use it +ALEPH_OAUTH=false +ALEPH_OAUTH_KEY= +ALEPH_OAUTH_SECRET= + +# You also need to provide an OpenID Connect (OIDC) configuration URL. Examples: +# +# Google: +# ALEPH_OAUTH_METADATA_URL=https://accounts.google.com/.well-known/openid-configuration +# +# KeyCloak/OCCRP: +# ALEPH_OAUTH_METADATA_URL=https://secure.occrp.org/auth/realms/general/.well-known/openid-configuration +# +# Azure: +# ALEPH_OAUTH_METADATA_URL=https://login.microsoftonline.com/common/v2.0/.well-known/openid-configuration +# +# Amazon Cognito: +# ALEPH_OAUTH_METADATA_URL=https://cognito-idp.{region}.amazonaws.com/{userPoolId}/.well-known/openid-configuration + +# Where and how to store the underlying files: +# ARCHIVE_TYPE=file +# ARCHIVE_PATH=/data + +# Or, if 'ALEPH_ARCHIVE_TYPE' configuration is 's3': +# ARCHIVE_TYPE=s3 +# ARCHIVE_BUCKET= +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= +# AWS_REGION= + +# To use an external ElasticSearch service: +# ALEPH_ELASTICSEARCH_URI= + +# To use an external Postgres service: +# ALEPH_DATABASE_URI=postgresql://:@/ + +# Define a different ftm entity store: +# FTM_STORE_URI=postgresql://:@/ + +# To run the Convert-Document service externally +# INGESTORS_CONVERT_DOCUMENT_URL=http://:3000/convert + +# Queue mechanism +# REDIS_URL=redis://redis:6379/0 + +# Content options +ALEPH_OCR_DEFAULTS=eng +# ALEPH_LANGUAGES=en:de:fr:es:tr:ar ... + +# Provide a valid email to send alerts from: +ALEPH_MAIL_FROM= +ALEPH_MAIL_HOST= +ALEPH_MAIL_USERNAME= +ALEPH_MAIL_PASSWORD= +ALEPH_MAIL_PORT=25 +ALEPH_MAIL_TLS=true +ALEPH_MAIL_SSL=false +ALEPH_MAIL_DEBUG=false + +# Debug mode (insecure) +ALEPH_DEBUG=false + +# Read-only mode: +# ALEPH_MAINTENANCE=true + +# Enable HTTP caching +# ALEPH_CACHE=true + +# Configure logging +LOG_FORMAT=TEXT # TEXT or JSON diff --git a/apps/open-justice/aleph.yml b/apps/open-justice/aleph.yml new file mode 100644 index 0000000..067b307 --- /dev/null +++ b/apps/open-justice/aleph.yml @@ -0,0 +1,59 @@ +--- +- hosts: + - open-justice + become: yes + + roles: + - geerlingguy.docker + + vars: + postgres_password: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/POSTGRES')}}" + postgres_hostname: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/POSTGRES subkey=hostname')}}" + postgres_username: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/POSTGRES subkey=username')}}" + postgres_database: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/POSTGRES subkey=database')}}" + aleph_secret_key: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/ALEPH_SECRET_KEY')}}" + + tasks: + - name: apt-get Update + apt: + update_cache: yes + + - name: Install make + apt: + name: make + state: present + + - name: Install git + apt: + name: git + state: present + + - name: Map ElasticSearch Memory + become: yes + sysctl: + name: vm.max_map_count + value: 262144 + state: present + + - name: Clone Aleph repo + become: yes + git: + repo: https://github.com/alephdata/aleph.git + dest: /var/open-justice/aleph + + - name: Copy env file + ansible.builtin.copy: + src: aleph.env + dest: /var/open-justice/aleph + + - name: Add database details to env file + ansible.builtin.lineinfile: + path: /var/open-justice/aleph/aleph.env + regexp: "^ALEPH_DATABASE_URI=" + line: "ALEPH_DATABASE_URI=postgresql://{{postgres_username}}:{{postgres_password}}@{{postgres_hostname}}/{{postgres_database}}" + + - name: Add ALEPH_SECRET_KEY + ansible.builtin.lineinfile: + path: /var/open-justice/aleph/aleph.env + regexp: "^ALEPH_SECRET_KEY=" + line: "ALEPH_SECRET_KEY={{aleph_secret_key}}" \ No newline at end of file diff --git a/inventory/prod.yml b/inventory/prod.yml index c1e232a..0a20b7b 100644 --- a/inventory/prod.yml +++ b/inventory/prod.yml @@ -38,7 +38,6 @@ all: idp1-aws.openup.org.za: pmg1-aws.pmg.org.za: elasticsearch1-aws.pmg.org.za: - justice1-hetzner.openup.org.za: ## App groups @@ -93,3 +92,6 @@ all: hetzner1.openup.org.za: vars: app_domain: search.opengazettes.org.za + open-justice: + hosts: + justice1-hetzner.openup.org.za: From d8ef735ed2e7fb7283534fe2536bd40f6d8f79bf Mon Sep 17 00:00:00 2001 From: Algorhythm and Funktion Date: Thu, 22 Apr 2021 12:27:52 +0200 Subject: [PATCH 2/7] Change justice1-hetzner.openup.org.za to openjustice.openup.org.za --- inventory/prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/prod.yml b/inventory/prod.yml index 0a20b7b..add88ab 100644 --- a/inventory/prod.yml +++ b/inventory/prod.yml @@ -94,4 +94,4 @@ all: app_domain: search.opengazettes.org.za open-justice: hosts: - justice1-hetzner.openup.org.za: + openjustice.openup.org.za: From 786899038ed94c601be4b19cd38961d270dcab21 Mon Sep 17 00:00:00 2001 From: Algorhythm and Funktion Date: Thu, 29 Apr 2021 11:53:35 +0200 Subject: [PATCH 3/7] Aleph.env templating --- apps/open-justice/aleph.env | 14 +++++++------- apps/open-justice/aleph.yml | 20 +++++--------------- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/apps/open-justice/aleph.env b/apps/open-justice/aleph.env index 54e3b36..31d791e 100644 --- a/apps/open-justice/aleph.env +++ b/apps/open-justice/aleph.env @@ -5,7 +5,7 @@ # by aleph and used to configure the system. # Random string: -#ALEPH_SECRET_KEY= +ALEPH_SECRET_KEY={{aleph_secret_key}} # Visible instance name in the UI ALEPH_APP_TITLE=Open Justice @@ -55,17 +55,17 @@ ALEPH_OAUTH_SECRET= # ARCHIVE_PATH=/data # Or, if 'ALEPH_ARCHIVE_TYPE' configuration is 's3': -# ARCHIVE_TYPE=s3 -# ARCHIVE_BUCKET= -# AWS_ACCESS_KEY_ID= -# AWS_SECRET_ACCESS_KEY= -# AWS_REGION= +ARCHIVE_TYPE=s3 +ARCHIVE_BUCKET=open-justice +AWS_ACCESS_KEY_ID={{aws_access_key_id}} +AWS_SECRET_ACCESS_KEY={{aws_secret_access_key}} +AWS_REGION=eu-central-1 # To use an external ElasticSearch service: # ALEPH_ELASTICSEARCH_URI= # To use an external Postgres service: -# ALEPH_DATABASE_URI=postgresql://:@/ +ALEPH_DATABASE_URI=postgresql://{{postgres_username}}:{{postgres_password}}@{{postgres_hostname}}/{{postgres_database}} # Define a different ftm entity store: # FTM_STORE_URI=postgresql://:@/ diff --git a/apps/open-justice/aleph.yml b/apps/open-justice/aleph.yml index 067b307..afe1e7e 100644 --- a/apps/open-justice/aleph.yml +++ b/apps/open-justice/aleph.yml @@ -12,6 +12,8 @@ postgres_username: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/POSTGRES subkey=username')}}" postgres_database: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/POSTGRES subkey=database')}}" aleph_secret_key: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/ALEPH_SECRET_KEY')}}" + aws_access_key_id: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/AWS subkey=AWS_ACCESS_KEY_ID')}}" + aws_secret_access_key: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/AWS')}}" tasks: - name: apt-get Update @@ -41,19 +43,7 @@ repo: https://github.com/alephdata/aleph.git dest: /var/open-justice/aleph - - name: Copy env file - ansible.builtin.copy: + - name: Set up env file + ansible.builtin.template: src: aleph.env - dest: /var/open-justice/aleph - - - name: Add database details to env file - ansible.builtin.lineinfile: - path: /var/open-justice/aleph/aleph.env - regexp: "^ALEPH_DATABASE_URI=" - line: "ALEPH_DATABASE_URI=postgresql://{{postgres_username}}:{{postgres_password}}@{{postgres_hostname}}/{{postgres_database}}" - - - name: Add ALEPH_SECRET_KEY - ansible.builtin.lineinfile: - path: /var/open-justice/aleph/aleph.env - regexp: "^ALEPH_SECRET_KEY=" - line: "ALEPH_SECRET_KEY={{aleph_secret_key}}" \ No newline at end of file + dest: /var/open-justice/aleph \ No newline at end of file From 68fdbae1bbb2db5e1033b64f9a7e5e7758069b20 Mon Sep 17 00:00:00 2001 From: Algorhythm and Funktion Date: Fri, 7 May 2021 12:30:05 +0200 Subject: [PATCH 4/7] Changed to new S3 Bucket and updated the README --- apps/open-justice/README.md | 1 + apps/open-justice/aleph.env | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/open-justice/README.md b/apps/open-justice/README.md index a889bdb..25f866e 100644 --- a/apps/open-justice/README.md +++ b/apps/open-justice/README.md @@ -13,6 +13,7 @@ Post playbook Start aleph in detached mode and set up tables ``` +cd /var/open-justice/aleph docker-compose up -d docker-compose run --rm shell upgrade ``` diff --git a/apps/open-justice/aleph.env b/apps/open-justice/aleph.env index 31d791e..c59e660 100644 --- a/apps/open-justice/aleph.env +++ b/apps/open-justice/aleph.env @@ -56,10 +56,10 @@ ALEPH_OAUTH_SECRET= # Or, if 'ALEPH_ARCHIVE_TYPE' configuration is 's3': ARCHIVE_TYPE=s3 -ARCHIVE_BUCKET=open-justice +ARCHIVE_BUCKET=open-justice-openup AWS_ACCESS_KEY_ID={{aws_access_key_id}} AWS_SECRET_ACCESS_KEY={{aws_secret_access_key}} -AWS_REGION=eu-central-1 +AWS_REGION=eu-west-1 # To use an external ElasticSearch service: # ALEPH_ELASTICSEARCH_URI= From 64b41c57af5385d7486c8f7f3a64f69e534330d7 Mon Sep 17 00:00:00 2001 From: Algorhythm and Funktion Date: Thu, 20 May 2021 11:49:46 +0200 Subject: [PATCH 5/7] Added mail settings Changed S3 bucket --- apps/open-justice/README.md | 1 + apps/open-justice/aleph.env | 10 +++++----- apps/open-justice/aleph.yml | 2 ++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/open-justice/README.md b/apps/open-justice/README.md index 25f866e..bca113a 100644 --- a/apps/open-justice/README.md +++ b/apps/open-justice/README.md @@ -16,6 +16,7 @@ Start aleph in detached mode and set up tables cd /var/open-justice/aleph docker-compose up -d docker-compose run --rm shell upgrade +sysctl -w vm.max_map_count=262144 ``` To set up a user diff --git a/apps/open-justice/aleph.env b/apps/open-justice/aleph.env index c59e660..cff6538 100644 --- a/apps/open-justice/aleph.env +++ b/apps/open-justice/aleph.env @@ -81,11 +81,11 @@ ALEPH_OCR_DEFAULTS=eng # ALEPH_LANGUAGES=en:de:fr:es:tr:ar ... # Provide a valid email to send alerts from: -ALEPH_MAIL_FROM= -ALEPH_MAIL_HOST= -ALEPH_MAIL_USERNAME= -ALEPH_MAIL_PASSWORD= -ALEPH_MAIL_PORT=25 +ALEPH_MAIL_FROM=info@openup.org.za +ALEPH_MAIL_HOST=smtp.sengrid.net +ALEPH_MAIL_USERNAME={{aleph_mail_username}} +ALEPH_MAIL_PASSWORD={{aleph_mail_password}} +ALEPH_MAIL_PORT=587 ALEPH_MAIL_TLS=true ALEPH_MAIL_SSL=false ALEPH_MAIL_DEBUG=false diff --git a/apps/open-justice/aleph.yml b/apps/open-justice/aleph.yml index afe1e7e..f7b508f 100644 --- a/apps/open-justice/aleph.yml +++ b/apps/open-justice/aleph.yml @@ -14,6 +14,8 @@ aleph_secret_key: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/ALEPH_SECRET_KEY')}}" aws_access_key_id: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/AWS subkey=AWS_ACCESS_KEY_ID')}}" aws_secret_access_key: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/AWS')}}" + aleph_mail_username: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/MAIL subkey=ALEPH_MAIL_USERNAME')}}" + aleph_mail_password: "{{ lookup('passwordstore', 'apps/open-justice/{{ env_name }}/MAIL')}}" tasks: - name: apt-get Update From 570200acc0b1a8493036b4e1909eda1de737940b Mon Sep 17 00:00:00 2001 From: Algorhythm and Funktion Date: Thu, 20 May 2021 11:51:28 +0200 Subject: [PATCH 6/7] Changed hostname --- inventory/prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/prod.yml b/inventory/prod.yml index add88ab..0a20b7b 100644 --- a/inventory/prod.yml +++ b/inventory/prod.yml @@ -94,4 +94,4 @@ all: app_domain: search.opengazettes.org.za open-justice: hosts: - openjustice.openup.org.za: + justice1-hetzner.openup.org.za: From 2934b122e4957fa13d8258405d05610a17168075 Mon Sep 17 00:00:00 2001 From: Algorhythm and Funktion Date: Wed, 26 May 2021 10:11:44 +0200 Subject: [PATCH 7/7] Removed port from open-justice aleph_ui_url --- apps/open-justice/README.md | 1 - apps/open-justice/aleph.env | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/open-justice/README.md b/apps/open-justice/README.md index bca113a..25f866e 100644 --- a/apps/open-justice/README.md +++ b/apps/open-justice/README.md @@ -16,7 +16,6 @@ Start aleph in detached mode and set up tables cd /var/open-justice/aleph docker-compose up -d docker-compose run --rm shell upgrade -sysctl -w vm.max_map_count=262144 ``` To set up a user diff --git a/apps/open-justice/aleph.env b/apps/open-justice/aleph.env index cff6538..b6bed58 100644 --- a/apps/open-justice/aleph.env +++ b/apps/open-justice/aleph.env @@ -11,7 +11,7 @@ ALEPH_SECRET_KEY={{aleph_secret_key}} ALEPH_APP_TITLE=Open Justice # Name needs to be a slug, as it is used e.g. for the ES index, SQS queue name: ALEPH_APP_NAME=open-justice -ALEPH_UI_URL=http://openjustice.openup.org.za:8080/ +ALEPH_UI_URL=http://openjustice.openup.org.za/ # ALEPH_URL_SCHEME=https # ALEPH_FAVICON=https://investigativedashboard.org/static/favicon.ico