From 20fc31d5ea199914b547760a76f7ddbb4e883ffd Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Fri, 1 Mar 2024 12:49:38 -0500 Subject: [PATCH] Add unit test for concept_trees GET --- .github/workflows/main.yml | 71 +++++++++++++++ arches_rdm/const.py | 12 +++ arches_rdm/test_settings.py | 34 +++++++ arches_rdm/tests.py | 166 +++++++++++++++++++++++++++++++++++ arches_rdm/urls.py | 3 +- arches_rdm/views/__init__.py | 38 ++++---- manage.py | 28 ++++++ 7 files changed, 333 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/main.yml create mode 100644 arches_rdm/const.py create mode 100644 arches_rdm/test_settings.py create mode 100644 arches_rdm/tests.py create mode 100644 manage.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..8a3b88ef --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,71 @@ +name: CI + +on: + # push: -- just run on PRs for now + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + services: + postgres: + image: postgis/postgis:13-3.0 + env: + POSTGRES_PASSWORD: postgis + POSTGRES_DB: arches + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + steps: + - name: Check out arches + uses: actions/checkout@v3 + with: + repository: archesproject/arches + ref: dev/7.6.x + path: arches + + - name: Check out arches_rdm + uses: actions/checkout@v2 + with: + path: arches_rdm + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install arches_rdm + working-directory: arches_rdm + run: | + python -m pip install --upgrade pip + pip install . + echo Project installed + + - name: Install arches dev version + working-directory: arches + run: | + python -m pip uninstall arches -y + python -m pip install . + echo Arches dev version installed + + - name: Install Java, GDAL, and other system dependencies + run: | + sudo apt update + sudo apt-get install libxml2-dev libpq-dev openjdk-8-jdk libgdal-dev + echo Postgres and ES dependencies installed + + - uses: ankane/setup-elasticsearch@v1 + with: + elasticsearch-version: 8 + + - name: Check for missing migrations + working-directory: arches_rdm + run: | + PYTHONPATH="../arches" python manage.py makemigrations --check --settings=arches_rdm.test_settings + + - name: Run arches_rdm unit tests + working-directory: arches_rdm + run: | + PYTHONPATH="../arches" python -W default::DeprecationWarning manage.py test --settings=arches_rdm.test_settings diff --git a/arches_rdm/const.py b/arches_rdm/const.py new file mode 100644 index 00000000..8ca9eef0 --- /dev/null +++ b/arches_rdm/const.py @@ -0,0 +1,12 @@ +CONCEPTS_GRAPH_ID = "bf73e576-4888-11ee-8a8d-11afefc4bff7" +SCHEMES_GRAPH_ID = "56788995-423b-11ee-8a8d-11afefc4bff7" + +TOP_CONCEPT_OF_NODE_AND_NODEGROUP = "bf73e5b9-4888-11ee-8a8d-11afefc4bff7" + +BROADER_NODE_AND_NODEGROUP = "bf73e5f5-4888-11ee-8a8d-11afefc4bff7" + +CONCEPT_LABEL_NODEGROUP = "bf73e616-4888-11ee-8a8d-11afefc4bff7" +CONCEPT_LABEL_NODE = "bf73e695-4888-11ee-8a8d-11afefc4bff7" + +SCHEME_LABEL_NODEGROUP = "749a27cf-423c-11ee-8a8d-11afefc4bff7" +SCHEME_LABEL_NODE = "749a27d5-423c-11ee-8a8d-11afefc4bff7" diff --git a/arches_rdm/test_settings.py b/arches_rdm/test_settings.py new file mode 100644 index 00000000..0e49bb4a --- /dev/null +++ b/arches_rdm/test_settings.py @@ -0,0 +1,34 @@ +import os + +# Depends on /arches being on the python path, as arches.tests is not importable +from tests.test_settings import * + +APP_NAME = "arches_rdm" +APP_ROOT = os.path.dirname(__file__) +INSTALLED_APPS = ( + "webpack_loader", + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django.contrib.gis", + "arches", + "arches.app.models", + "arches.management", + "guardian", + "captcha", + "revproxy", + "corsheaders", + "oauth2_provider", + "django_celery_results", + "arches_rdm", +) +ROOT_URLCONF = "arches_rdm.urls" + +LOCALE_PATHS = [os.path.join(APP_ROOT, "locale")] + +# Further settings may need to be added from project, just don't +# want to clobber anything from core test settings for now. +# Also, settings can be overridden directly. See @override_settings diff --git a/arches_rdm/tests.py b/arches_rdm/tests.py new file mode 100644 index 00000000..49c3a1c9 --- /dev/null +++ b/arches_rdm/tests.py @@ -0,0 +1,166 @@ +import json + +from django.contrib.auth.models import User +from django.test import TestCase +from django.test.client import Client +from django.urls import reverse + +from arches.app.models.models import ( + GraphModel, + Node, + NodeGroup, + ResourceInstance, + TileModel, +) + +from arches_rdm.const import ( + CONCEPTS_GRAPH_ID, + SCHEMES_GRAPH_ID, + TOP_CONCEPT_OF_NODE_AND_NODEGROUP, + BROADER_NODE_AND_NODEGROUP, + CONCEPT_LABEL_NODEGROUP, + CONCEPT_LABEL_NODE, + SCHEME_LABEL_NODEGROUP, + SCHEME_LABEL_NODE, +) + + +def setUpModule(): + """Bootstrap just a few nodes as an alternative to loading the entire package.""" + if not GraphModel.objects.filter(pk=SCHEMES_GRAPH_ID).exists(): + GraphModel.objects.create(pk=SCHEMES_GRAPH_ID, isresource=True) + GraphModel.objects.create(pk=CONCEPTS_GRAPH_ID, isresource=True) + + for nodegroup_id, node_id, datatype in zip( + [ + TOP_CONCEPT_OF_NODE_AND_NODEGROUP, + BROADER_NODE_AND_NODEGROUP, + SCHEME_LABEL_NODEGROUP, + CONCEPT_LABEL_NODEGROUP, + ], + [ + TOP_CONCEPT_OF_NODE_AND_NODEGROUP, + BROADER_NODE_AND_NODEGROUP, + SCHEME_LABEL_NODE, + CONCEPT_LABEL_NODE, + ], + [ + "concept-list", + "concept-list", + "string", + "string", + ], + ): + NodeGroup.objects.create(pk=nodegroup_id) + Node.objects.create( + pk=node_id, + graph_id=CONCEPTS_GRAPH_ID, + nodegroup_id=nodegroup_id, + istopnode=False, + datatype=datatype, + isrequired=datatype == "string", + ) + + +def localized_string(text, language="en", direction="ltr"): + return {language: {"value": text, "direction": direction}} + + +class ConceptTreeViewTests(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.client = Client() + cls.admin = User.objects.get(username="admin") + + @classmethod + def setUpTestData(cls): + # Create a scheme with five concepts, each one narrower than the last, + # and each concept after the top concept also narrower than the top. + cls.scheme = ResourceInstance.objects.create(graph_id=SCHEMES_GRAPH_ID) + TileModel.objects.create( + resourceinstance=cls.scheme, + nodegroup_id=SCHEME_LABEL_NODEGROUP, + data={SCHEME_LABEL_NODE: localized_string("Test Scheme")}, + ) + + MAX_DEPTH = 5 + CONCEPT_COUNT = 5 + cls.concepts = [ + ResourceInstance(graph_id=SCHEMES_GRAPH_ID) for _ in range(CONCEPT_COUNT) + ] + ResourceInstance.objects.bulk_create(cls.concepts) + + for i, concept in enumerate(cls.concepts): + # Create label tile + TileModel.objects.create( + resourceinstance=concept, + nodegroup_id=CONCEPT_LABEL_NODEGROUP, + data={CONCEPT_LABEL_NODE: localized_string(f"Concept {i + 1}")}, + ) + # Create top concept/narrower tile + if i == 0: + TileModel.objects.create( + resourceinstance=concept, + nodegroup_id=TOP_CONCEPT_OF_NODE_AND_NODEGROUP, + data={ + TOP_CONCEPT_OF_NODE_AND_NODEGROUP: [ + {"resourceId": str(cls.scheme.pk)}, + ], + }, + ) + elif i < MAX_DEPTH: + TileModel.objects.create( + resourceinstance=concept, + nodegroup_id=BROADER_NODE_AND_NODEGROUP, + data={ + BROADER_NODE_AND_NODEGROUP: [ + # Previous concept + {"resourceId": str(cls.concepts[i - 1].pk)}, + # Also add top concept + {"resourceId": str(cls.concepts[0].pk)}, + ], + }, + ) + else: + TileModel.objects.create( + resourceinstance=concept, + nodegroup_id=BROADER_NODE_AND_NODEGROUP, + data={ + BROADER_NODE_AND_NODEGROUP: [ + # Top concept only + {"resourceId": str(cls.concepts[0].pk)}, + ], + }, + ) + + def test_get_concept_trees(self): + self.client.force_login(self.admin) + with self.assertNumQueries(5): + # 1: session + # 2: auth + # 3: select broader tiles, subquery for labels + # 4: select top concept tiles, subquery for labels + # 5: select schemes, subquery for labels + response = self.client.get(reverse("concept_trees")) + + self.assertEqual(response.status_code, 200) + result = json.loads(response.content) + scheme = result["schemes"][0] + + self.assertEqual(scheme["labels"][0]["value"], "Test Scheme") + self.assertEqual(len(scheme["top_concepts"]), 1) + top = scheme["top_concepts"][0] + self.assertEqual(top["labels"][0]["value"], "Concept 1") + self.assertEqual(len(top["narrower"]), 4) + self.assertEqual( + {n["labels"][0]["value"] for n in top["narrower"]}, + {"Concept 2", "Concept 3", "Concept 4", "Concept 5"}, + ) + concept_2 = [ + c for c in top["narrower"] if c["labels"][0]["value"] == "Concept 2" + ][0] + self.assertEqual( + {n["labels"][0]["value"] for n in concept_2["narrower"]}, + {"Concept 3"}, + ) diff --git a/arches_rdm/urls.py b/arches_rdm/urls.py index cb6425e1..d819f5c8 100644 --- a/arches_rdm/urls.py +++ b/arches_rdm/urls.py @@ -13,10 +13,11 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ -from django.urls import path +from django.urls import include, path from arches_rdm.views import ConceptTreeView urlpatterns = [ + path("", include("arches.urls")), path("concept_trees/", ConceptTreeView.as_view(), name="concept_trees"), ] diff --git a/arches_rdm/views/__init__.py b/arches_rdm/views/__init__.py index daefc58d..1b733fc3 100644 --- a/arches_rdm/views/__init__.py +++ b/arches_rdm/views/__init__.py @@ -11,25 +11,24 @@ from arches.app.utils.response import JSONResponse -TOP_CONCEPT_OF_NODE_AND_NODEGROUP = "bf73e5b9-4888-11ee-8a8d-11afefc4bff7" -TOP_CONCEPT_OF_LOOKUP = f"data__{TOP_CONCEPT_OF_NODE_AND_NODEGROUP}" +from arches_rdm.const import ( + SCHEMES_GRAPH_ID, + TOP_CONCEPT_OF_NODE_AND_NODEGROUP, + BROADER_NODE_AND_NODEGROUP, + CONCEPT_LABEL_NODEGROUP, + CONCEPT_LABEL_NODE, + SCHEME_LABEL_NODEGROUP, + SCHEME_LABEL_NODE, +) -BROADER_NODE_AND_NODEGROUP = "bf73e5f5-4888-11ee-8a8d-11afefc4bff7" +TOP_CONCEPT_OF_LOOKUP = f"data__{TOP_CONCEPT_OF_NODE_AND_NODEGROUP}" BROADER_LOOKUP = f"data__{BROADER_NODE_AND_NODEGROUP}" - -CONCEPT_LABEL_NODEGROUP = "bf73e616-4888-11ee-8a8d-11afefc4bff7" -CONCEPT_LABEL_NODE = "bf73e695-4888-11ee-8a8d-11afefc4bff7" CONCEPT_LABEL_LOOKUP = f"data__{CONCEPT_LABEL_NODE}" - -SCHEME_LABEL_NODE_AND_NODEGROUP = "749a27cf-423c-11ee-8a8d-11afefc4bff7" -SCHEME_LABEL_NODE = "749a27d5-423c-11ee-8a8d-11afefc4bff7" SCHEME_LABEL_LOOKUP = f"data__{SCHEME_LABEL_NODE}" -CONCEPTS_GRAPH_ID = "bf73e576-4888-11ee-8a8d-11afefc4bff7" -SCHEMES_GRAPH_ID = "56788995-423b-11ee-8a8d-11afefc4bff7" - -class JsonArrayElements(Func): +class JsonbArrayElements(Func): + """https://forum.djangoproject.com/t/django-4-2-behavior-change-when-using-arrayagg-on-unnested-arrayfield-postgresql-specific/21547/5""" contains_subquery = True function = "JSONB_ARRAY_ELEMENTS" @@ -40,14 +39,17 @@ class JsonArrayElements(Func): class ConceptTreeView(View): def __init__(self): self.schemes = ResourceInstance.objects.none() - self.top_concepts = defaultdict(set) - self.narrower_concepts = defaultdict(set) # str: str - self.labels = defaultdict(set) + # key=scheme resourceid (str) val=set of concept resourceids (str) + self.top_concepts: dict[str: set[str]] = defaultdict(set) + # key=concept resourceid (str) val=set of concept resourceids (str) + self.narrower_concepts: dict[str: set[str]] = defaultdict(set) + # key=resourceid (str) val=list of label dicts + self.labels: dict[str: list[dict]] = defaultdict(set) @staticmethod def resources_from_tiles(lookup_expression: str): return CombinedExpression( - JsonArrayElements(F(lookup_expression)), + JsonbArrayElements(F(lookup_expression)), "->>", Value("resourceId"), output_field=CharField(), @@ -58,7 +60,7 @@ def labels_subquery(label_lookup): if label_lookup == SCHEME_LABEL_LOOKUP: # Annotating a ResourceInstance outer = OuterRef("resourceinstanceid") - nodegroup_id = SCHEME_LABEL_NODE_AND_NODEGROUP + nodegroup_id = SCHEME_LABEL_NODEGROUP else: # Annotating a Tile outer = OuterRef("resourceinstance_id") diff --git a/manage.py b/manage.py new file mode 100644 index 00000000..7a45d2c8 --- /dev/null +++ b/manage.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python + +''' +ARCHES - a program developed to inventory and manage immovable cultural heritage. +Copyright (C) 2013 J. Paul Getty Trust and World Monuments Fund + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +''' + + +import os +import sys + +if __name__ == "__main__": + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv)