Merge branch 'develop' into feature/rabbitmq-again

alephdata · Mar 18, 2024 · 91e00e0 · 91e00e0
2 parents 6d17a79 + 1f42d28
commit 91e00e0
Show file tree

Hide file tree

Showing 14 changed files with 31 additions and 21 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -49,4 +49,4 @@ ENV ALEPH_ELASTICSEARCH_URI=http://elasticsearch:9200/ \
 RUN mkdir /run/prometheus
 
 # Run the green unicorn
-CMD gunicorn --config /aleph/gunicorn.conf.py --workers 5 --log-level info --log-file - aleph.manage:app
+CMD gunicorn --config /aleph/gunicorn.conf.py --workers 6 --log-level debug --log-file -
diff --git a/aleph/logic/html.py b/aleph/logic/html.py
@@ -57,7 +57,7 @@ def sanitize_html(html_text, base_url, encoding=None):
             if el.tag == "a":
                 el.set("target", "_blank")
                 el.set("rel", "nofollow noreferrer external noopener")
-        return tostring(doc)
+        return tostring(doc, method="html", encoding="unicode")
     except Exception as exc:
         log.warning("HTML sanitizer failure [%s]: %s", type(exc), exc)
         return gettext("[HTML removed: could not be sanitized]")

diff --git a/aleph/manage.py b/aleph/manage.py
@@ -66,13 +66,16 @@ def get_collection(foreign_id):
 
 
 def ensure_collection(foreign_id, label):
-    authz = Authz.from_role(Role.load_cli_user())
-    config = {
-        "foreign_id": foreign_id,
-        "label": label,
-    }
-    create_collection(config, authz)
-    return Collection.by_foreign_id(foreign_id)
+    collection = Collection.by_foreign_id(foreign_id, deleted=True)
+    if collection is None:
+        authz = Authz.from_role(Role.load_cli_user())
+        config = {
+            "foreign_id": foreign_id,
+            "label": label,
+        }
+        create_collection(config, authz)
+        return Collection.by_foreign_id(foreign_id)
+    return collection
 
 
 @click.group(cls=FlaskGroup, create_app=create_app)

diff --git a/aleph/tests/test_view_util.py b/aleph/tests/test_view_util.py
@@ -36,6 +36,11 @@ def test_sanitize_html(self):
         assert html.find(".//a").get("target") == "_blank", html
         assert "nofollow" in html.find(".//a").get("rel"), html
 
+    def test_sanitize_html_no_self_closing_tags(self):
+        html_str = '<html><body><noscript><script src="https://example.org></script></noscript></body></html>'
+        processed = sanitize_html(html_str, "https://example.org")
+        assert processed == "<html><body><div><noscript></noscript></div></body></html>"
+
     def test_validate_returns_errors_for_paths(self):
         # given
         schema = "RoleCreate"  # name min length 4, password min length 6

diff --git a/contrib/aleph-traefik-minio-keycloak/docker-compose.yml b/contrib/aleph-traefik-minio-keycloak/docker-compose.yml
@@ -36,7 +36,7 @@ services:
       - "traefik.enable=false"
 
   ingest-file:
-    image: ghcr.io/alephdata/ingest-file:3.19.3
+    image: ghcr.io/alephdata/ingest-file:3.20.0
     tmpfs:
       - /tmp:mode=777
     volumes:

diff --git a/contrib/keycloak/docker-compose.dev-keycloak.yml b/contrib/keycloak/docker-compose.dev-keycloak.yml
@@ -35,7 +35,7 @@ services:
   ingest-file:
     build:
       context: services/ingest-file
-    image: ghcr.io/alephdata/ingest-file:3.19.3
+    image: ghcr.io/alephdata/ingest-file:3.20.0
     hostname: ingest
     tmpfs: /tmp
     volumes:

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -7,6 +7,8 @@ services:
       - "127.0.0.1:5432:5432"
     volumes:
       - postgres-data:/var/lib/postgresql/data
+    # enable this to log all SQL statements
+    # command: ["postgres", "-c", "log_statement=all"]
     environment:
       POSTGRES_USER: aleph
       POSTGRES_PASSWORD: aleph

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -33,7 +33,7 @@ services:
       - rabbitmq-data:/var/lib/rabbitmq/
 
   ingest-file:
-    image: ghcr.io/alephdata/ingest-file:3.19.3
+    image: ghcr.io/alephdata/ingest-file:3.20.0
     tmpfs:
       - /tmp:mode=777
     volumes:

diff --git a/docs/src/pages/developers/alephclient.mdx b/docs/src/pages/developers/alephclient.mdx
@@ -54,7 +54,7 @@ Using the `write-entities` command, users can load JSON-formatted entities forma
 A typical use might look this:
 
 ```bash
-curl -o md_companies.yml https://raw.githubusercontent.com/alephdata/aleph/master/mappings/md_companies.yml
+curl -o md_companies.yml https://raw.githubusercontent.com/alephdata/aleph/main/mappings/md_companies.yml
 ftm map md_companies.yml | ftm aggregate | alephclient write-entities -f md_companies
 ```
 
@@ -81,7 +81,7 @@ alephclient stream-entities -f us_ofac | ftm validate | ftm export-excel -o OFAC
 The `bulkload` command executes an [entity mapping](/developers/mappings) within the Aleph system. Its only argument is a YAML mapping file:
 
 ```
-curl -o md_companies.yml https://raw.githubusercontent.com/alephdata/aleph/master/mappings/md_companies.yml
+curl -o md_companies.yml https://raw.githubusercontent.com/alephdata/aleph/main/mappings/md_companies.yml
 alephclient bulkload md_companies.yml
 ```
 

diff --git a/docs/src/pages/developers/followthemoney/ftm.mdx b/docs/src/pages/developers/followthemoney/ftm.mdx
@@ -51,7 +51,7 @@ pip install pyicu
 Probably the most common task for ftm is to generate FtM entities from some structured data source. This is done using a YAML-formatted mapping file, [described here](/developers/mappings). With such a YAML file in hand, you can generate FtM entities like this:
 
 ```bash
-curl -o md_companies.yml https://raw.githubusercontent.com/alephdata/aleph/master/mappings/md_companies.yml
+curl -o md_companies.yml https://raw.githubusercontent.com/alephdata/aleph/main/mappings/md_companies.yml
 ftm map md_companies.yml
 ```
 
@@ -66,7 +66,7 @@ This will yield a line-based JSON stream of every company in Moldova, their dire
 You might note, however, that this actually generates multiple entity fragments for each company (i.e. multiple entities with the same ID). This is due to the way the md_companies mapping is written: each query section generates a partial company record. In order to mitigate this, you will need to perform entity aggregation:
 
 ```bash
-curl -o md_companies.yml https://raw.githubusercontent.com/alephdata/aleph/master/mappings/md_companies.yml
+curl -o md_companies.yml https://raw.githubusercontent.com/alephdata/aleph/main/mappings/md_companies.yml
 ftm map md_companies.yml | ftm aggregate >moldova.ijson
 ```
 

diff --git a/docs/src/pages/developers/installation.mdx b/docs/src/pages/developers/installation.mdx
@@ -193,7 +193,7 @@ Aleph is distributed as a set of Docker containers, which can be run on any serv
 
 To begin a production deployment:
 
-- Obtain a copy of Aleph's [docker-compose](https://github.com/alephdata/aleph/blob/master/docker-compose.yml) file and [base configurations](https://github.com/alephdata/aleph/blob/master/aleph.env.tmpl) (named `aleph.env.tmpl`).
+- Obtain a copy of Aleph's [docker-compose](https://github.com/alephdata/aleph/blob/main/docker-compose.yml) file and [base configurations](https://github.com/alephdata/aleph/blob/main/aleph.env.tmpl) (named `aleph.env.tmpl`).
 - Make a copy of the configurations file named `aleph.env` and define settings for your production instance. Check the [section on configuration](/developers/installation#configuration) for more information regarding the available options.
 
 <Callout>

diff --git a/docs/src/pages/users/faq/account/index.mdx b/docs/src/pages/users/faq/account/index.mdx
@@ -73,7 +73,7 @@ Your Aleph account’s username is the e-mail address you used to register. If y
 
 ## I removed my 2FA app/lost my phone and I cannot log in to Aleph. What should I do?
 
-In either of the above situations, you will have to reset the password to your Aleph account. Please follow the instructions to [reset your password](http://localhost:3000/users/getting-started/account#reset-your-password).
+In either of the above situations, you will have to reset the password to your Aleph account. Please follow the instructions to [reset your password](/users/getting-started/account#reset-your-password).
 
 ## I tried everything and I can’t log in to Aleph? How do I get in touch with OCCRP to ask for assistance?
 

diff --git a/helm/charts/aleph/values.yaml b/helm/charts/aleph/values.yaml
@@ -125,7 +125,7 @@ ingestfile:
 
   image:
     repository: ghcr.io/alephdata/ingest-file
-    tag: "3.19.2"
+    tag: "3.20.0"
     pullPolicy: Always
 
   containerSecurityContext:

diff --git a/helm/examples/dev/README.md b/helm/examples/dev/README.md
@@ -113,7 +113,7 @@ helm upgrade --set "aleph.enabled=true" --set "global.prometheus.enabled=true" -
 In order to access the Prometheus dashboard forward the respective port…
 
 ```
-kubectl port-forward svc/aleph-api 9100
+kubectl port-forward svc/kube-prometheus-stack-prometheus 9090
 ```
 
-… and open `http://kubernetes.docker.internal:9100` in your browser.
+… and open `http://localhost:9090` in your browser.