Skip to content

Commit

Permalink
Add unit test for concept_trees GET
Browse files Browse the repository at this point in the history
  • Loading branch information
jacobtylerwalls committed Mar 8, 2024
1 parent fe78669 commit 20fc31d
Show file tree
Hide file tree
Showing 7 changed files with 333 additions and 19 deletions.
71 changes: 71 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
name: CI

on:
# push: -- just run on PRs for now
pull_request:
workflow_dispatch:

jobs:
build:
runs-on: ubuntu-latest
services:
postgres:
image: postgis/postgis:13-3.0
env:
POSTGRES_PASSWORD: postgis
POSTGRES_DB: arches
ports:
- 5432:5432
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5

steps:
- name: Check out arches
uses: actions/checkout@v3
with:
repository: archesproject/arches
ref: dev/7.6.x
path: arches

- name: Check out arches_rdm
uses: actions/checkout@v2
with:
path: arches_rdm

- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: '3.11'

- name: Install arches_rdm
working-directory: arches_rdm
run: |
python -m pip install --upgrade pip
pip install .
echo Project installed
- name: Install arches dev version
working-directory: arches
run: |
python -m pip uninstall arches -y
python -m pip install .
echo Arches dev version installed
- name: Install Java, GDAL, and other system dependencies
run: |
sudo apt update
sudo apt-get install libxml2-dev libpq-dev openjdk-8-jdk libgdal-dev
echo Postgres and ES dependencies installed
- uses: ankane/setup-elasticsearch@v1
with:
elasticsearch-version: 8

- name: Check for missing migrations
working-directory: arches_rdm
run: |
PYTHONPATH="../arches" python manage.py makemigrations --check --settings=arches_rdm.test_settings
- name: Run arches_rdm unit tests
working-directory: arches_rdm
run: |
PYTHONPATH="../arches" python -W default::DeprecationWarning manage.py test --settings=arches_rdm.test_settings
12 changes: 12 additions & 0 deletions arches_rdm/const.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
CONCEPTS_GRAPH_ID = "bf73e576-4888-11ee-8a8d-11afefc4bff7"
SCHEMES_GRAPH_ID = "56788995-423b-11ee-8a8d-11afefc4bff7"

TOP_CONCEPT_OF_NODE_AND_NODEGROUP = "bf73e5b9-4888-11ee-8a8d-11afefc4bff7"

BROADER_NODE_AND_NODEGROUP = "bf73e5f5-4888-11ee-8a8d-11afefc4bff7"

CONCEPT_LABEL_NODEGROUP = "bf73e616-4888-11ee-8a8d-11afefc4bff7"
CONCEPT_LABEL_NODE = "bf73e695-4888-11ee-8a8d-11afefc4bff7"

SCHEME_LABEL_NODEGROUP = "749a27cf-423c-11ee-8a8d-11afefc4bff7"
SCHEME_LABEL_NODE = "749a27d5-423c-11ee-8a8d-11afefc4bff7"
34 changes: 34 additions & 0 deletions arches_rdm/test_settings.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import os

# Depends on /arches being on the python path, as arches.tests is not importable
from tests.test_settings import *

APP_NAME = "arches_rdm"
APP_ROOT = os.path.dirname(__file__)
INSTALLED_APPS = (
"webpack_loader",
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"django.contrib.gis",
"arches",
"arches.app.models",
"arches.management",
"guardian",
"captcha",
"revproxy",
"corsheaders",
"oauth2_provider",
"django_celery_results",
"arches_rdm",
)
ROOT_URLCONF = "arches_rdm.urls"

LOCALE_PATHS = [os.path.join(APP_ROOT, "locale")]

# Further settings may need to be added from project, just don't
# want to clobber anything from core test settings for now.
# Also, settings can be overridden directly. See @override_settings
166 changes: 166 additions & 0 deletions arches_rdm/tests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
import json

from django.contrib.auth.models import User
from django.test import TestCase
from django.test.client import Client
from django.urls import reverse

from arches.app.models.models import (
GraphModel,
Node,
NodeGroup,
ResourceInstance,
TileModel,
)

from arches_rdm.const import (
CONCEPTS_GRAPH_ID,
SCHEMES_GRAPH_ID,
TOP_CONCEPT_OF_NODE_AND_NODEGROUP,
BROADER_NODE_AND_NODEGROUP,
CONCEPT_LABEL_NODEGROUP,
CONCEPT_LABEL_NODE,
SCHEME_LABEL_NODEGROUP,
SCHEME_LABEL_NODE,
)


def setUpModule():
"""Bootstrap just a few nodes as an alternative to loading the entire package."""
if not GraphModel.objects.filter(pk=SCHEMES_GRAPH_ID).exists():
GraphModel.objects.create(pk=SCHEMES_GRAPH_ID, isresource=True)
GraphModel.objects.create(pk=CONCEPTS_GRAPH_ID, isresource=True)

for nodegroup_id, node_id, datatype in zip(
[
TOP_CONCEPT_OF_NODE_AND_NODEGROUP,
BROADER_NODE_AND_NODEGROUP,
SCHEME_LABEL_NODEGROUP,
CONCEPT_LABEL_NODEGROUP,
],
[
TOP_CONCEPT_OF_NODE_AND_NODEGROUP,
BROADER_NODE_AND_NODEGROUP,
SCHEME_LABEL_NODE,
CONCEPT_LABEL_NODE,
],
[
"concept-list",
"concept-list",
"string",
"string",
],
):
NodeGroup.objects.create(pk=nodegroup_id)
Node.objects.create(
pk=node_id,
graph_id=CONCEPTS_GRAPH_ID,
nodegroup_id=nodegroup_id,
istopnode=False,
datatype=datatype,
isrequired=datatype == "string",
)


def localized_string(text, language="en", direction="ltr"):
return {language: {"value": text, "direction": direction}}


class ConceptTreeViewTests(TestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.client = Client()
cls.admin = User.objects.get(username="admin")

@classmethod
def setUpTestData(cls):
# Create a scheme with five concepts, each one narrower than the last,
# and each concept after the top concept also narrower than the top.
cls.scheme = ResourceInstance.objects.create(graph_id=SCHEMES_GRAPH_ID)
TileModel.objects.create(
resourceinstance=cls.scheme,
nodegroup_id=SCHEME_LABEL_NODEGROUP,
data={SCHEME_LABEL_NODE: localized_string("Test Scheme")},
)

MAX_DEPTH = 5
CONCEPT_COUNT = 5
cls.concepts = [
ResourceInstance(graph_id=SCHEMES_GRAPH_ID) for _ in range(CONCEPT_COUNT)
]
ResourceInstance.objects.bulk_create(cls.concepts)

for i, concept in enumerate(cls.concepts):
# Create label tile
TileModel.objects.create(
resourceinstance=concept,
nodegroup_id=CONCEPT_LABEL_NODEGROUP,
data={CONCEPT_LABEL_NODE: localized_string(f"Concept {i + 1}")},
)
# Create top concept/narrower tile
if i == 0:
TileModel.objects.create(
resourceinstance=concept,
nodegroup_id=TOP_CONCEPT_OF_NODE_AND_NODEGROUP,
data={
TOP_CONCEPT_OF_NODE_AND_NODEGROUP: [
{"resourceId": str(cls.scheme.pk)},
],
},
)
elif i < MAX_DEPTH:
TileModel.objects.create(
resourceinstance=concept,
nodegroup_id=BROADER_NODE_AND_NODEGROUP,
data={
BROADER_NODE_AND_NODEGROUP: [
# Previous concept
{"resourceId": str(cls.concepts[i - 1].pk)},
# Also add top concept
{"resourceId": str(cls.concepts[0].pk)},
],
},
)
else:
TileModel.objects.create(
resourceinstance=concept,
nodegroup_id=BROADER_NODE_AND_NODEGROUP,
data={
BROADER_NODE_AND_NODEGROUP: [
# Top concept only
{"resourceId": str(cls.concepts[0].pk)},
],
},
)

def test_get_concept_trees(self):
self.client.force_login(self.admin)
with self.assertNumQueries(5):
# 1: session
# 2: auth
# 3: select broader tiles, subquery for labels
# 4: select top concept tiles, subquery for labels
# 5: select schemes, subquery for labels
response = self.client.get(reverse("concept_trees"))

self.assertEqual(response.status_code, 200)
result = json.loads(response.content)
scheme = result["schemes"][0]

self.assertEqual(scheme["labels"][0]["value"], "Test Scheme")
self.assertEqual(len(scheme["top_concepts"]), 1)
top = scheme["top_concepts"][0]
self.assertEqual(top["labels"][0]["value"], "Concept 1")
self.assertEqual(len(top["narrower"]), 4)
self.assertEqual(
{n["labels"][0]["value"] for n in top["narrower"]},
{"Concept 2", "Concept 3", "Concept 4", "Concept 5"},
)
concept_2 = [
c for c in top["narrower"] if c["labels"][0]["value"] == "Concept 2"
][0]
self.assertEqual(
{n["labels"][0]["value"] for n in concept_2["narrower"]},
{"Concept 3"},
)
3 changes: 2 additions & 1 deletion arches_rdm/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,11 @@
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.urls import path
from django.urls import include, path

from arches_rdm.views import ConceptTreeView

urlpatterns = [
path("", include("arches.urls")),
path("concept_trees/", ConceptTreeView.as_view(), name="concept_trees"),
]
38 changes: 20 additions & 18 deletions arches_rdm/views/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,25 +11,24 @@

from arches.app.utils.response import JSONResponse

TOP_CONCEPT_OF_NODE_AND_NODEGROUP = "bf73e5b9-4888-11ee-8a8d-11afefc4bff7"
TOP_CONCEPT_OF_LOOKUP = f"data__{TOP_CONCEPT_OF_NODE_AND_NODEGROUP}"
from arches_rdm.const import (
SCHEMES_GRAPH_ID,
TOP_CONCEPT_OF_NODE_AND_NODEGROUP,
BROADER_NODE_AND_NODEGROUP,
CONCEPT_LABEL_NODEGROUP,
CONCEPT_LABEL_NODE,
SCHEME_LABEL_NODEGROUP,
SCHEME_LABEL_NODE,
)

BROADER_NODE_AND_NODEGROUP = "bf73e5f5-4888-11ee-8a8d-11afefc4bff7"
TOP_CONCEPT_OF_LOOKUP = f"data__{TOP_CONCEPT_OF_NODE_AND_NODEGROUP}"
BROADER_LOOKUP = f"data__{BROADER_NODE_AND_NODEGROUP}"

CONCEPT_LABEL_NODEGROUP = "bf73e616-4888-11ee-8a8d-11afefc4bff7"
CONCEPT_LABEL_NODE = "bf73e695-4888-11ee-8a8d-11afefc4bff7"
CONCEPT_LABEL_LOOKUP = f"data__{CONCEPT_LABEL_NODE}"

SCHEME_LABEL_NODE_AND_NODEGROUP = "749a27cf-423c-11ee-8a8d-11afefc4bff7"
SCHEME_LABEL_NODE = "749a27d5-423c-11ee-8a8d-11afefc4bff7"
SCHEME_LABEL_LOOKUP = f"data__{SCHEME_LABEL_NODE}"

CONCEPTS_GRAPH_ID = "bf73e576-4888-11ee-8a8d-11afefc4bff7"
SCHEMES_GRAPH_ID = "56788995-423b-11ee-8a8d-11afefc4bff7"


class JsonArrayElements(Func):
class JsonbArrayElements(Func):
"""https://forum.djangoproject.com/t/django-4-2-behavior-change-when-using-arrayagg-on-unnested-arrayfield-postgresql-specific/21547/5"""
contains_subquery = True
function = "JSONB_ARRAY_ELEMENTS"

Expand All @@ -40,14 +39,17 @@ class JsonArrayElements(Func):
class ConceptTreeView(View):
def __init__(self):
self.schemes = ResourceInstance.objects.none()
self.top_concepts = defaultdict(set)
self.narrower_concepts = defaultdict(set) # str: str
self.labels = defaultdict(set)
# key=scheme resourceid (str) val=set of concept resourceids (str)
self.top_concepts: dict[str: set[str]] = defaultdict(set)
# key=concept resourceid (str) val=set of concept resourceids (str)
self.narrower_concepts: dict[str: set[str]] = defaultdict(set)
# key=resourceid (str) val=list of label dicts
self.labels: dict[str: list[dict]] = defaultdict(set)

@staticmethod
def resources_from_tiles(lookup_expression: str):
return CombinedExpression(
JsonArrayElements(F(lookup_expression)),
JsonbArrayElements(F(lookup_expression)),
"->>",
Value("resourceId"),
output_field=CharField(),
Expand All @@ -58,7 +60,7 @@ def labels_subquery(label_lookup):
if label_lookup == SCHEME_LABEL_LOOKUP:
# Annotating a ResourceInstance
outer = OuterRef("resourceinstanceid")
nodegroup_id = SCHEME_LABEL_NODE_AND_NODEGROUP
nodegroup_id = SCHEME_LABEL_NODEGROUP
else:
# Annotating a Tile
outer = OuterRef("resourceinstance_id")
Expand Down
28 changes: 28 additions & 0 deletions manage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/usr/bin/env python

'''
ARCHES - a program developed to inventory and manage immovable cultural heritage.
Copyright (C) 2013 J. Paul Getty Trust and World Monuments Fund
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
'''


import os
import sys

if __name__ == "__main__":
from django.core.management import execute_from_command_line

execute_from_command_line(sys.argv)

0 comments on commit 20fc31d

Please sign in to comment.