From 962a0e8dcdc57a34fd5a8c765d033f11565d7879 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Wed, 26 Jun 2024 22:39:09 +0200 Subject: [PATCH 1/8] setup: move to python3.12 only --- setup.cfg | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index ddfb7dd..ce2fc8f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2020-2022 Graz University of Technology. +# Copyright (C) 2020-2024 Graz University of Technology. # # invenio-records-lom is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -20,9 +20,7 @@ author_email = info@tugraz.at platforms = any url = https://github.com/tu-graz-library/invenio-records-lom classifiers = - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Development Status :: 5 - Production/Stable License :: OSI Approved :: MIT License Operating System :: OS Independent From a7ba0c02bf6205d274580325456d49f6b5bf4687 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Fri, 21 Jun 2024 15:10:15 +0200 Subject: [PATCH 2/8] services: move add identifier to components * the components can handle that more naturally --- invenio_records_lom/services/components.py | 11 ++++++++++- invenio_records_lom/services/services.py | 23 +--------------------- tests/test_service.py | 2 ++ 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/invenio_records_lom/services/components.py b/invenio_records_lom/services/components.py index 08d5d82..eae5456 100644 --- a/invenio_records_lom/services/components.py +++ b/invenio_records_lom/services/components.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2021 Graz University of Technology. +# Copyright (C) 2021-2024 Graz University of Technology. # # invenio-records-lom is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -24,6 +24,7 @@ ) from invenio_records_resources.services.uow import TaskOp +from ..utils import LOMMetadata from .tasks import register_or_update_pid @@ -91,6 +92,14 @@ def new_version( class LOMPIDsComponent(PIDsComponent): """LOM Sevice component for PIDs.""" + def create(self, identity, data=None, record=None, errors=None): + """This method is called on draft creation.""" + super().create(identity, data, record, errors) + + metadata = LOMMetadata(data["metadata"]) + metadata.append_identifier(record.pid.pid_value, catalog="repo-pid") + record.metadata = metadata.json + # overwrite `publish`` to use the celery-task from this package # this was copied from its parent class, except for its last line def publish(self, identity, draft=None, record=None): diff --git a/invenio_records_lom/services/services.py b/invenio_records_lom/services/services.py index 61cdc8e..73f0b55 100644 --- a/invenio_records_lom/services/services.py +++ b/invenio_records_lom/services/services.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2021 Graz University of Technology. +# Copyright (C) 2021-2024 Graz University of Technology. # # invenio-records-lom is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -8,29 +8,8 @@ """Record services configured for LOM-use.""" from invenio_rdm_records.services import RDMRecordService -from invenio_records_resources.services.uow import unit_of_work - -from ..utils import LOMMetadata # pylint: disable-next=abstract-method class LOMRecordService(RDMRecordService): """RecordService configured for LOM-use.""" - - @unit_of_work() - def create(self, identity, data, uow=None, expand=False): - """Create.""" - draft_item = super().create(identity, data, uow=uow, expand=expand) - - # add repo-pid to the record's identifiers - metadata = LOMMetadata(data["metadata"]) - metadata.append_identifier(draft_item.id, catalog="repo-pid") - data["metadata"] = metadata.json - - return super().update_draft( - identity=identity, - id_=draft_item.id, - data=data, - uow=uow, - expand=expand, - ) diff --git a/tests/test_service.py b/tests/test_service.py index c2448ef..4ba369e 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -107,6 +107,7 @@ def test_create_draft(service, db, identity, access): # pylint: disable=too-man # TODO: check the necessity of this del data["metadata"]["type"] + del json["metadata"]["general"]["identifier"] assert json["metadata"] == data["metadata"] assert "access" in json assert json["access"]["files"] == access["files"] @@ -165,6 +166,7 @@ def test_publish(service, db, identity, access): # pylint: disable=too-many-loc json_pid = general["identifier"][0]["entry"]["langstring"]["#text"] assert json_pid == record_pid.pid_value + del json["metadata"]["general"]["identifier"] assert json["metadata"] == data["metadata"] assert "access" in json assert json["access"]["files"] == access["files"] From a644f605af17cf81245efb37ad09f94cab87a32c Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Fri, 21 Jun 2024 15:10:57 +0200 Subject: [PATCH 3/8] utils: add create_record and update_record func --- invenio_records_lom/utils/__init__.py | 4 ++ invenio_records_lom/utils/util.py | 86 ++++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/invenio_records_lom/utils/__init__.py b/invenio_records_lom/utils/__init__.py index 32b176c..1781217 100644 --- a/invenio_records_lom/utils/__init__.py +++ b/invenio_records_lom/utils/__init__.py @@ -13,8 +13,10 @@ DotAccessWrapper, LOMDuplicateRecordError, check_about_duplicate, + create_record, get_learningresourcetypedict, get_oefosdict, + update_record, ) from .vcard import make_lom_vcard @@ -27,5 +29,7 @@ "make_lom_vcard", "build_record_unique_id", "check_about_duplicate", + "create_record", + "update_record", "LOMDuplicateRecordError", ) diff --git a/invenio_records_lom/utils/util.py b/invenio_records_lom/utils/util.py index 1e30de0..2d6e0a1 100644 --- a/invenio_records_lom/utils/util.py +++ b/invenio_records_lom/utils/util.py @@ -7,16 +7,21 @@ """Utilities for creation of LOM-compliant metadata.""" - -import re from collections.abc import MutableMapping from csv import reader from importlib import resources from json import load +from pathlib import Path +from re import compile as re_compile +from re import sub +from time import sleep from typing import Any, Iterator, Optional, Union +from flask_principal import Identity +from invenio_records_resources.services.base import Service from invenio_search import RecordsSearch from invenio_search.engine import dsl +from marshmallow.exceptions import ValidationError class DotAccessWrapper(MutableMapping): @@ -223,7 +228,7 @@ def standardize_url(url: str) -> str: If `url`'s scheme is "http" or "https", make it "https". Also ensure existence of a trailing "/". """ - pattern = re.compile("^https?://(.*?)/?$") + pattern = re_compile("^https?://(.*?)/?$") if m := pattern.match(url): middle = m.group(1) # excludes initial "https://", excludes trailing "/" return f"https://{middle}/" @@ -265,3 +270,78 @@ def check_about_duplicate(identifier: str, catalog: str): catalog=catalog, id_=results[0]["id"], ) + + +def add_file_to_record( + lomid, + file_path, + file_service, + identity, +): + """Add the file to the record.""" + file_ = Path(file_path) + filename = sub(r"-([^-]*?)\.", r".", file_.name) + data = [{"key": filename}] + + with file_.open(mode="rb") as file_pointer: + file_service.init_files(id_=lomid, identity=identity, data=data) + file_service.set_file_content( + id_=lomid, file_key=filename, identity=identity, stream=file_pointer + ) + file_service.commit_file(id_=lomid, file_key=filename, identity=identity) + + +def create_record( + service: Service, # services.LOMRecordService + data: dict, + file_paths: list, + identity: Identity, + *, + do_publish: bool = True, +): + """Create record.""" + data["files"] = {"enabled": len(file_paths) > 0} + data["access"] = { + "access": { + "record": "public", + "files": "public", + }, + } + + draft = service.create(data=data, identity=identity) + + try: + for file_path in file_paths: + add_file_to_record( + lomid=draft.id, + file_path=file_path, + file_service=service.draft_files, + identity=identity, + ) + + if do_publish: + # to prevent the race condition bug. + # see https://github.com/inveniosoftware/invenio-rdm-records/issues/809 + sleep(0.5) + + return service.publish(id_=draft.id, identity=identity) + except (FileNotFoundError, ValidationError) as error: + service.delete_draft(id_=draft.id, identity=identity) + raise error + + return draft + + +def update_record( + pid: str, + service: Service, # services.LOMRecordService + data: dict, + identity: Identity, + *, + do_publish: bool = True, +): + """Update record.""" + service.update_draft(id_=pid, data=data, identity=identity) + + if do_publish: + service.publish(id_=pid, identity=identity) From 5e2be671be7904e52146148e7860c1d5a0ebdd42 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Thu, 4 Jul 2024 23:22:09 +0200 Subject: [PATCH 4/8] utils: add LOMRecordData class * this class should be used as programmatic api for data --- invenio_records_lom/utils/__init__.py | 3 ++- invenio_records_lom/utils/metadata.py | 30 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/invenio_records_lom/utils/__init__.py b/invenio_records_lom/utils/__init__.py index 1781217..4f69fd6 100644 --- a/invenio_records_lom/utils/__init__.py +++ b/invenio_records_lom/utils/__init__.py @@ -7,7 +7,7 @@ """Utilities for creation of LOM-compliant metadata.""" -from .metadata import LOMCourseMetadata, LOMMetadata +from .metadata import LOMCourseMetadata, LOMMetadata, LOMRecordData from .statistics import build_record_unique_id from .util import ( DotAccessWrapper, @@ -26,6 +26,7 @@ "get_oefosdict", "LOMMetadata", "LOMCourseMetadata", + "LOMRecordData", "make_lom_vcard", "build_record_unique_id", "check_about_duplicate", diff --git a/invenio_records_lom/utils/metadata.py b/invenio_records_lom/utils/metadata.py index dea1979..e440869 100644 --- a/invenio_records_lom/utils/metadata.py +++ b/invenio_records_lom/utils/metadata.py @@ -24,6 +24,36 @@ ) +class LOMRecordData(dict): + """LOM record data.""" + + def __init__( + self, + resource_type=None, + pids=None, + metadata=None, + **kwargs: dict, + ) -> None: + """Construct.""" + self.update(**kwargs) + self.resource_type = resource_type + self.pids = pids + self.metadata = ( + metadata + if isinstance(metadata, LOMMetadata) + else LOMMetadata(metadata, overwritable=True) + ) + + @property + def json(self) -> dict: + """Json.""" + return { + "pids": self.pids, + "metadata": self.metadata.json, + "resource_type": self.resource_type, + } + + class BaseLOMMetadata: """Base LOM Metadata.""" From d68c9d4c146155710dc63fcc3bb3aae2784961af Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Thu, 4 Jul 2024 23:23:47 +0200 Subject: [PATCH 5/8] fix: missed removing metadata attribute usage --- invenio_records_lom/utils/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invenio_records_lom/utils/metadata.py b/invenio_records_lom/utils/metadata.py index e440869..66eae7b 100644 --- a/invenio_records_lom/utils/metadata.py +++ b/invenio_records_lom/utils/metadata.py @@ -224,7 +224,7 @@ def create( def append_course(self, course: LOMCourseMetadata) -> None: """Append course.""" # pylint: disable-next=unsupported-membership-test - if "courses" not in self.record["metadata"]: + if "courses" not in self.record: self.record["courses"] = [] courses = self.record["courses"] From eb5c66fbbb0dec76bb3b409c878067f36e4bd9f5 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Wed, 26 Jun 2024 21:20:28 +0200 Subject: [PATCH 6/8] fix: metadata rights not exists use param * without that, on dublin core a {} would be used which is then wrong on the global search --- invenio_records_lom/utils/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invenio_records_lom/utils/metadata.py b/invenio_records_lom/utils/metadata.py index 66eae7b..5ce1be8 100644 --- a/invenio_records_lom/utils/metadata.py +++ b/invenio_records_lom/utils/metadata.py @@ -553,7 +553,7 @@ def set_rights_url(self, url: str) -> None: def get_rights(self, url_only=False) -> dict | str: """Get rights.""" if "rights" not in self.record: - return {} + return "" if url_only else {} if url_only: try: From 118a56f46a092da278d9724fa3e93302861a81ae Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Tue, 2 Jul 2024 23:43:50 +0200 Subject: [PATCH 7/8] fix: deduped assume parent is a list * this enforces if parent is not a list to be treated as a list --- invenio_records_lom/utils/metadata.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/invenio_records_lom/utils/metadata.py b/invenio_records_lom/utils/metadata.py index 5ce1be8..9b6d908 100644 --- a/invenio_records_lom/utils/metadata.py +++ b/invenio_records_lom/utils/metadata.py @@ -82,6 +82,10 @@ def deduped_append(self, parent_key: str, value: Any): """Append `value` to `self.record[key]` if not already appended.""" self.record.setdefault(parent_key, []) parent = self.record[parent_key] + + if not isinstance(parent, list): + parent = [parent] + if value not in parent: parent.append(value) From 0160b2bdbfc79a000fb2a610deba089e2aee636a Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Tue, 2 Jul 2024 23:44:28 +0200 Subject: [PATCH 8/8] metadata: add method get_courses --- invenio_records_lom/utils/metadata.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/invenio_records_lom/utils/metadata.py b/invenio_records_lom/utils/metadata.py index 9b6d908..04b318a 100644 --- a/invenio_records_lom/utils/metadata.py +++ b/invenio_records_lom/utils/metadata.py @@ -239,6 +239,13 @@ def append_course(self, course: LOMCourseMetadata) -> None: if course.record["course.version"] not in versions: self.record["courses"].append(course.record.data) + def get_courses(self) -> list: + """Get courses.""" + try: + return self.record["courses"] + except KeyError: + return [] + ############### # # methods for manipulating LOM's `general` (1) category