From 17874c8066acc6380d382e7707cccab4774c110c Mon Sep 17 00:00:00 2001 From: Johnathan Clementi Date: Mon, 1 Jul 2024 11:39:22 -0400 Subject: [PATCH 1/4] Update resource descriptors #29 --- .../pkg/graphs/resource_models/Concept.json | 16 ++++++++-------- .../pkg/graphs/resource_models/Scheme.json | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/arches_lingo/pkg/graphs/resource_models/Concept.json b/arches_lingo/pkg/graphs/resource_models/Concept.json index 866c6bcb..b002697a 100644 --- a/arches_lingo/pkg/graphs/resource_models/Concept.json +++ b/arches_lingo/pkg/graphs/resource_models/Concept.json @@ -3470,16 +3470,16 @@ "config": { "descriptor_types": { "description": { - "nodegroup_id": "802bc768-19a6-11ee-8f04-cd21e680a247", - "string_template": "" + "nodegroup_id": "bf73e5d7-4888-11ee-8a8d-11afefc4bff7", + "string_template": "" }, "map_popup": { "nodegroup_id": "", "string_template": "" }, "name": { - "nodegroup_id": "bf73e616-4888-11ee-8a8d-11afefc4bff7", - "string_template": "" + "nodegroup_id": "ab9fee9c-0eb6-11ef-93db-0a58a9feac02", + "string_template": "" } }, "triggering_nodegroups": [] @@ -6337,9 +6337,9 @@ } ], "metadata": { - "db": "PostgreSQL 14.10 on aarch64-unknown-linux-gnu, compiled by gcc (GCC) 7.3.1 20180712 (Red Hat 7.3.1-6), 64-bit", - "git hash": "/bin/sh: 1: git: not found", - "os": "Linux", - "os version": "5.10.215-203.850.amzn2.x86_64" + "db": "PostgreSQL 14.5 on x86_64-apple-darwin20.6.0, compiled by Apple clang version 12.0.0 (clang-1200.0.32.29), 64-bit", + "git hash": "728621149 2024-06-27 09:32:13 -0700", + "os": "Darwin", + "os version": "23.5.0" } } \ No newline at end of file diff --git a/arches_lingo/pkg/graphs/resource_models/Scheme.json b/arches_lingo/pkg/graphs/resource_models/Scheme.json index 9f740322..a2c48fdd 100644 --- a/arches_lingo/pkg/graphs/resource_models/Scheme.json +++ b/arches_lingo/pkg/graphs/resource_models/Scheme.json @@ -2069,16 +2069,16 @@ "config": { "descriptor_types": { "description": { - "nodegroup_id": "", - "string_template": "" + "nodegroup_id": "7131bc72-11e0-11ef-9493-0a58a9feac02", + "string_template": "" }, "map_popup": { "nodegroup_id": "", "string_template": "" }, "name": { - "nodegroup_id": "749a27cf-423c-11ee-8a8d-11afefc4bff7", - "string_template": "" + "nodegroup_id": "ef87ac28-11de-11ef-9493-0a58a9feac02", + "string_template": "" } }, "triggering_nodegroups": [] From 1882fadf034ef073573f6acc484022684fb74bc3 Mon Sep 17 00:00:00 2001 From: Johnathan Clementi Date: Mon, 1 Jul 2024 11:40:24 -0400 Subject: [PATCH 2/4] Add languages and identifiers to ref-datatype concept placeholders #29 --- .../collections/collections.xml | 237 ++++++++++-------- 1 file changed, 134 insertions(+), 103 deletions(-) diff --git a/arches_lingo/pkg/reference_data/collections/collections.xml b/arches_lingo/pkg/reference_data/collections/collections.xml index 36c46d86..988a322c 100644 --- a/arches_lingo/pkg/reference_data/collections/collections.xml +++ b/arches_lingo/pkg/reference_data/collections/collections.xml @@ -3,201 +3,232 @@ xmlns:skos="http://www.w3.org/2004/02/skos/core#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" > - + + {"id": "32817961-1a23-48ab-ad0e-c0e58f017f00", "value": "label"} - + - {"id": "26b5a068-df40-441e-920a-02f6d6c6478b", "value": "Statuteses"} - + - - - + - {"id": "a3b5bf31-1646-44ef-854c-aa445dc52240", "value": "Languages"} + + - + - + - + - + - - - {"id": "db749170-299d-4c08-b9fe-c41d86af3e4c", "value": "Name Types"} - + - + - - - {"id": "3b01965f-786c-400b-a7f2-a6c6c5157c78", "value": "Right Types"} - - - {"id": "621ac89a-98c1-48e9-9b4b-205d7bef676c", "value": "Scheme Identifier Types"} - - - + - {"id": "57c8a2b7-9e8a-4621-9b7a-6a64e390ee36", "value": "Metatypes"} - + - + - - - {"id": "40879c17-cb58-4957-927f-a00ffc5bb635", "value": "Scheme Types"} - + - - - {"id": "7ab97324-867e-4be9-b705-f35c90df8442", "value": "Event Types"} - + - - - + + {"id": "a3b5bf31-1646-44ef-854c-aa445dc52240", "value": "Languages"} - + - + + + + {"id": "1d648e40-e9e2-4fa5-8b7b-1c331496c8ce", "value": "Identifier Types - General"} - + - {"id": "9f9dfd1a-b7f6-4976-8915-e671d9fcb4ee", "value": "Term Types"} - + - + - + - + - + {"id": "3f58f3d2-d3d4-4ace-98ef-7825d309544e", "value": "E55 Type"} - + - + - {"id": "32817961-1a23-48ab-ad0e-c0e58f017f00", "value": "label"} + {"id": "d8c622f6-e786-11e6-905a-475a5eee86f5", "value": "Resource To Resource Relationship Types"} + + - + + {"id": "621ac89a-98c1-48e9-9b4b-205d7bef676c", "value": "Scheme Identifier Types"} + + - + - - - + - + - {"id": "7fc9d400-c1d9-4c7f-bbde-d2cd36c0ea6f", "value": "Identifier Types"} + {"id": "57c8a2b7-9e8a-4621-9b7a-6a64e390ee36", "value": "Metatypes"} + + + + + + {"id": "26b5a068-df40-441e-920a-02f6d6c6478b", "value": "Statuteses"} - + - - {"id": "1d648e40-e9e2-4fa5-8b7b-1c331496c8ce", "value": "Identifier Types - General"} + + {"id": "40879c17-cb58-4957-927f-a00ffc5bb635", "value": "Scheme Types"} - + - + - - - - - - - + + - {"id": "d505fc19-e27c-4623-9228-539e00bf2684", "value": "note"} + {"id": "faf6c183-0e9b-4ccc-b6ab-87bf3f59ff9a", "value": "related"} - + - - - + + {"id": "9f9dfd1a-b7f6-4976-8915-e671d9fcb4ee", "value": "Term Types"} - + - + - {"id": "6eb4125d-1651-4852-ada2-3b93e97865d1", "value": "mappingRelation"} - + - + - + - {"id": "bd8822bd-6115-4587-8d4c-6c5918fea7ad", "value": "Subject Types"} - + + {"id": "db749170-299d-4c08-b9fe-c41d86af3e4c", "value": "Name Types"} - - {"id": "faf6c183-0e9b-4ccc-b6ab-87bf3f59ff9a", "value": "related"} + + {"id": "6eb4125d-1651-4852-ada2-3b93e97865d1", "value": "mappingRelation"} - - + + + + + + + + + + + - - {"id": "d8c622f6-e786-11e6-905a-475a5eee86f5", "value": "Resource To Resource Relationship Types"} + + + + + + + + + + + + {"id": "d505fc19-e27c-4623-9228-539e00bf2684", "value": "note"} + + + {"id": "5f390dd0-4818-4d10-9ae9-f23de571de9a", "value": "Label Type"} + + + + + + + {"id": "7fc9d400-c1d9-4c7f-bbde-d2cd36c0ea6f", "value": "Identifier Types"} - + + + + - + - - + + {"id": "bc7a90c8-979c-45b7-ab3a-8b0d374ba430", "value": "broaderTransitive"} - - {"id": "5f390dd0-4818-4d10-9ae9-f23de571de9a", "value": "Label Type"} + + {"id": "3b01965f-786c-400b-a7f2-a6c6c5157c78", "value": "Right Types"} + + + {"id": "7ab97324-867e-4be9-b705-f35c90df8442", "value": "Event Types"} + + + + + + + + + {"id": "bd8822bd-6115-4587-8d4c-6c5918fea7ad", "value": "Subject Types"} + + + - - - - - - - + + + + + + + From 9fbd8dac821b55069f047ef74ed6bae1f35e4037 Mon Sep 17 00:00:00 2001 From: Johnathan Clementi Date: Mon, 1 Jul 2024 11:44:32 -0400 Subject: [PATCH 3/4] Remap RDM to Lingo ETL #29 --- arches_lingo/etl_modules/migrate_to_lingo.py | 39 ++++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/arches_lingo/etl_modules/migrate_to_lingo.py b/arches_lingo/etl_modules/migrate_to_lingo.py index c938f213..cddbcfd4 100644 --- a/arches_lingo/etl_modules/migrate_to_lingo.py +++ b/arches_lingo/etl_modules/migrate_to_lingo.py @@ -23,7 +23,7 @@ SCHEMES_GRAPH_ID = uuid.UUID("56788995-423b-11ee-8a8d-11afefc4bff7") CONCEPTS_GRAPH_ID = uuid.UUID("bf73e576-4888-11ee-8a8d-11afefc4bff7") CONCEPTS_TOP_CONCEPT_OF_NODEGROUP_ID = uuid.UUID("bf73e5b9-4888-11ee-8a8d-11afefc4bff7") -CONCEPTS_BROADER_NODEGROUP_ID = uuid.UUID("bf73e5f5-4888-11ee-8a8d-11afefc4bff7") +CONCEPTS_BROADER_NODEGROUP_ID = uuid.UUID("f3f7bbea-0eb9-11ef-93db-0a58a9feac02") CONCEPTS_PART_OF_SCHEME_NODEGROUP_ID = uuid.UUID("bf73e60a-4888-11ee-8a8d-11afefc4bff7") details = { @@ -62,16 +62,25 @@ def etl_schemes(self, cursor, nodegroup_lookup, node_lookup): concept.pk ) # use old conceptid as new resourceinstanceid - name = {} + appellative_status = {} identifier = {} if ( value.valuetype_id == "prefLabel" or value.valuetype_id == "altLabel" ): - name["name_content"] = value.value - name["name_language"] = value.language_id - name["name_type"] = value.valuetype_id - scheme_to_load["tile_data"].append({"name": name}) + appellative_status["appellative_status_ascribed_name_content"] = ( + value.value + ) + + appellative_status["appellative_status_ascribed_name_language"] = ( + value.language_id + ) + appellative_status["appellative_status_ascribed_relation"] = ( + value.valuetype_id + ) + scheme_to_load["tile_data"].append( + {"appellative_status": appellative_status} + ) elif value.valuetype_id == "identifier": identifier["identifier_content"] = value.value identifier["identifier_type"] = value.valuetype_id @@ -90,16 +99,24 @@ def etl_concepts(self, cursor, nodegroup_lookup, node_lookup): concept.pk ) # use old conceptid as new resourceinstanceid - name = {} + appellative_status = {} identifier = {} if ( value.valuetype_id == "prefLabel" or value.valuetype_id == "altLabel" ): - name["name_content"] = value.value - name["name_language"] = value.language_id - name["name_type"] = value.valuetype_id - concept_to_load["tile_data"].append({"name": name}) + appellative_status["appellative_status_ascribed_name_content"] = ( + value.value + ) + appellative_status["appellative_status_ascribed_name_language"] = ( + value.language_id + ) + appellative_status["appellative_status_ascribed_relation"] = ( + value.valuetype_id + ) + concept_to_load["tile_data"].append( + {"appellative_status": appellative_status} + ) elif value.valuetype_id == "identifier": identifier["identifier_content"] = value.value identifier["identifier_type"] = value.valuetype_id From 7d9aec1068c281d6a21d430f692347c93d80257e Mon Sep 17 00:00:00 2001 From: Johnathan Clementi Date: Mon, 1 Jul 2024 22:32:06 -0400 Subject: [PATCH 4/4] Remap consts to updated graphs #29 --- arches_lingo/const.py | 31 +++++++++++++++----- arches_lingo/etl_modules/migrate_to_lingo.py | 20 ++++++------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/arches_lingo/const.py b/arches_lingo/const.py index db15544c..75bb6eab 100644 --- a/arches_lingo/const.py +++ b/arches_lingo/const.py @@ -1,19 +1,34 @@ +### Concepts Model Nodes & Nodegroups ### CONCEPTS_GRAPH_ID = "bf73e576-4888-11ee-8a8d-11afefc4bff7" -SCHEMES_GRAPH_ID = "56788995-423b-11ee-8a8d-11afefc4bff7" TOP_CONCEPT_OF_NODE_AND_NODEGROUP = "bf73e5b9-4888-11ee-8a8d-11afefc4bff7" -BROADER_NODE_AND_NODEGROUP = "bf73e5f5-4888-11ee-8a8d-11afefc4bff7" +BROADER_NODE_AND_NODEGROUP = "f3f7bbea-0eb9-11ef-93db-0a58a9feac02" # classification_status_ascribed_classification -CONCEPT_NAME_NODEGROUP = "bf73e616-4888-11ee-8a8d-11afefc4bff7" +# appellative_status +CONCEPT_NAME_NODEGROUP = "ab9fee9c-0eb6-11ef-93db-0a58a9feac02" +# appellative_status_ascribed_name_content CONCEPT_NAME_CONTENT_NODE = "bf73e695-4888-11ee-8a8d-11afefc4bff7" -CONCEPT_NAME_TYPE_NODE = "b08eebb4-d44c-11ee-a986-0242ac130005" -CONCEPT_NAME_LANGUAGE_NODE = "444b7de6-d44c-11ee-8fe3-0242ac130005" +# appellative_status_ascribed_name_language +CONCEPT_NAME_LANGUAGE_NODE = "a8ecaf54-0eb7-11ef-93db-0a58a9feac02" +# appellative_status_ascribed_relation +CONCEPT_NAME_TYPE_NODE = "1ddffab4-0eb8-11ef-93db-0a58a9feac02" + +CONCEPTS_PART_OF_SCHEME_NODEGROUP_ID = "bf73e60a-4888-11ee-8a8d-11afefc4bff7" + -SCHEME_NAME_NODEGROUP = "749a27cf-423c-11ee-8a8d-11afefc4bff7" -SCHEME_NAME_CONTENT_NODE = "749a27d5-423c-11ee-8a8d-11afefc4bff7" -SCHEME_NAME_TYPE_NODE = "1330cc4c-d44d-11ee-9261-0242ac130005" +### Scheme Model Nodes & Nodegroups ### +SCHEMES_GRAPH_ID = "56788995-423b-11ee-8a8d-11afefc4bff7" + +# appellative_status +SCHEME_NAME_NODEGROUP = "ab9fee9c-0eb6-11ef-93db-0a58a9feac02" +# appellative_status_ascribed_name_content +SCHEME_NAME_CONTENT_NODE = "a8ecaf54-0eb7-11ef-93db-0a58a9feac02" +# appellative_status_ascribed_name_language SCHEME_NAME_LANGUAGE_NODE = "2deaf45e-d44d-11ee-b78d-0242ac130005" +# appellative_status_ascribed_relation +SCHEME_NAME_TYPE_NODE = "1ddffab4-0eb8-11ef-93db-0a58a9feac02" + PREF_LABEL_VALUE_ID = "3b8a03f1-9047-48e4-9ca0-b3fe887f6f9d" ALT_LABEL_VALUE_ID = "c02f97c5-da16-4ff0-864a-92c34da84e38" diff --git a/arches_lingo/etl_modules/migrate_to_lingo.py b/arches_lingo/etl_modules/migrate_to_lingo.py index cddbcfd4..8ce6c375 100644 --- a/arches_lingo/etl_modules/migrate_to_lingo.py +++ b/arches_lingo/etl_modules/migrate_to_lingo.py @@ -16,16 +16,16 @@ from arches.app.models.models import LoadStaging, NodeGroup, LoadEvent from arches.app.utils.betterJSONSerializer import JSONSerializer import arches_lingo.tasks as tasks +from arches_lingo.const import ( + SCHEMES_GRAPH_ID, + CONCEPTS_GRAPH_ID, + TOP_CONCEPT_OF_NODE_AND_NODEGROUP, + BROADER_NODE_AND_NODEGROUP, + CONCEPTS_PART_OF_SCHEME_NODEGROUP_ID, +) logger = logging.getLogger(__name__) -#### Constants #### -SCHEMES_GRAPH_ID = uuid.UUID("56788995-423b-11ee-8a8d-11afefc4bff7") -CONCEPTS_GRAPH_ID = uuid.UUID("bf73e576-4888-11ee-8a8d-11afefc4bff7") -CONCEPTS_TOP_CONCEPT_OF_NODEGROUP_ID = uuid.UUID("bf73e5b9-4888-11ee-8a8d-11afefc4bff7") -CONCEPTS_BROADER_NODEGROUP_ID = uuid.UUID("f3f7bbea-0eb9-11ef-93db-0a58a9feac02") -CONCEPTS_PART_OF_SCHEME_NODEGROUP_ID = uuid.UUID("bf73e60a-4888-11ee-8a8d-11afefc4bff7") - details = { "etlmoduleid": "11cad3ca-e155-44b1-9910-c50b3def47f6", "name": "Migrate to Lingo", @@ -259,9 +259,9 @@ def init_relationships(self, cursor, loadid): where relationtype = 'hasTopConcept'; """, ( - CONCEPTS_TOP_CONCEPT_OF_NODEGROUP_ID, + TOP_CONCEPT_OF_NODE_AND_NODEGROUP, loadid, - CONCEPTS_TOP_CONCEPT_OF_NODEGROUP_ID, + TOP_CONCEPT_OF_NODE_AND_NODEGROUP, ), ) @@ -300,7 +300,7 @@ def init_relationships(self, cursor, loadid): from relations where relationtype = 'narrower'; """, - (CONCEPTS_BROADER_NODEGROUP_ID, loadid, CONCEPTS_BROADER_NODEGROUP_ID), + (BROADER_NODE_AND_NODEGROUP, loadid, BROADER_NODE_AND_NODEGROUP), ) # Create Part of Scheme relationships - derived by recursively generating concept hierarchy & associating