From 670bc3e9f0d9a2ffff67d336737eacab87974006 Mon Sep 17 00:00:00 2001 From: Micha Moskovic Date: Mon, 15 May 2017 18:26:43 +0200 Subject: [PATCH] schemas: promote acquisition_source to array * INCOMPATIBLE The field `acquisition_source` is replaced by the `acquisition_sources` array, allowing to capture all sources of metadata in the record instead of only one. This is useful as a record typically aggregates information from several sources. Signed-off-by: Micha Moskovic --- inspire_schemas/builders.py | 20 ++++++++++++---- inspire_schemas/records/authors.yml | 12 ++++++++-- .../records/elements/acquisition_source.yml | 6 ----- inspire_schemas/records/hep.yml | 12 ++++++++-- .../integration/fixtures/authors_example.json | 20 ++++++++-------- .../fixtures/expected_data_hep.yaml | 23 ++++++++++++------- tests/integration/fixtures/hep_example.json | 20 ++++++++-------- .../integration/fixtures/input_data_hep.yaml | 2 ++ tests/integration/test_builders.py | 4 ++++ 9 files changed, 79 insertions(+), 40 deletions(-) diff --git a/inspire_schemas/builders.py b/inspire_schemas/builders.py index 96cbe757..632fa952 100644 --- a/inspire_schemas/builders.py +++ b/inspire_schemas/builders.py @@ -108,7 +108,7 @@ def __init__(self, source, record=None): def _append_to(self, field, element): """Append the ``element`` to the ``field`` of the record. - This method is smart: it does nothing if ``field`` is empty and + This method is smart: it does nothing if ``element`` is empty and creates ``field`` if it does not exit yet. :param field: the name of the field of the record to append to @@ -119,6 +119,20 @@ def _append_to(self, field, element): self.record.setdefault(field, []) self.record.get(field).append(element) + def _prepend_to(self, field, element): + """Prepend the ``element`` to the ``field`` of the record. + + This method is smart: it does nothing if ``element`` is empty and + creates ``field`` if it does not exit yet. + + :param field: the name of the field of the record to prepend to + :type field: string + :param element: the element to prepend + """ + if element not in EMPTIES: + self.record.setdefault(field, []) + self.record.get(field).insert(0, element) + def __str__(self): """Print the current record.""" return str(self.record) @@ -651,8 +665,6 @@ def add_acquisition_source( warnings.warn("Use 'datetime', not 'date'", DeprecationWarning) datetime = date - self.record.setdefault('acquisition_source', {}) - acquisition_source = {} acquisition_source['submission_number'] = str(submission_number) @@ -661,7 +673,7 @@ def add_acquisition_source( if locals()[key] is not None: acquisition_source[key] = locals()[key] - self.record['acquisition_source'] = acquisition_source + self._prepend_to('acquisition_sources', acquisition_source) @filter_empty_parameters def add_document_type(self, document_type): diff --git a/inspire_schemas/records/authors.yml b/inspire_schemas/records/authors.yml index 519bbab4..975df4eb 100644 --- a/inspire_schemas/records/authors.yml +++ b/inspire_schemas/records/authors.yml @@ -13,8 +13,16 @@ properties: $ref: elements/sourced_value.json type: array uniqueItems: true - acquisition_source: - $ref: elements/acquisition_source.json + acquisition_sources: + description: |- + :MARC: ``541`` (only the earliest source was stored) + + Sources of the metadata in the record, in inverse chronological + order (i.e. the first element is the latest one). + items: + $ref: elements/acquisition_source.json + type: array + uniqueItems: true advisors: items: additionalProperties: false diff --git a/inspire_schemas/records/elements/acquisition_source.yml b/inspire_schemas/records/elements/acquisition_source.yml index 8ccdcb38..673ba212 100644 --- a/inspire_schemas/records/elements/acquisition_source.yml +++ b/inspire_schemas/records/elements/acquisition_source.yml @@ -1,11 +1,5 @@ $schema: http://json-schema.org/schema# additionalProperties: false -description: |- - :MARC: ``541`` - - Only the first source is stored: if the record later gets enriched with - metadata coming from a second source, the `acquisition_source` is not - updated. properties: datetime: description: |- diff --git a/inspire_schemas/records/hep.yml b/inspire_schemas/records/hep.yml index 6f9de2c7..6362d313 100644 --- a/inspire_schemas/records/hep.yml +++ b/inspire_schemas/records/hep.yml @@ -207,8 +207,16 @@ properties: title: List of related accelerators/experiments type: array uniqueItems: true - acquisition_source: - $ref: elements/acquisition_source.json + acquisition_sources: + description: |- + :MARC: ``541`` (only the earliest source was stored) + + Sources of the metadata in the record, in inverse chronological + order (i.e. the first element is the latest one). + items: + $ref: elements/acquisition_source.json + type: array + uniqueItems: true arxiv_eprints: items: additionalProperties: false diff --git a/tests/integration/fixtures/authors_example.json b/tests/integration/fixtures/authors_example.json index 7165269f..559fc729 100644 --- a/tests/integration/fixtures/authors_example.json +++ b/tests/integration/fixtures/authors_example.json @@ -24,15 +24,17 @@ "value": "nulla com" } ], - "acquisition_source": { - "datetime": "2234-06-08T05:46:05.263Z", - "email": "rLmoeWrC4fI6iZ@IBfNPyIlR.vwp", - "internal_uid": 76120453, - "method": "submitter", - "orcid": "8351-4483-7418-2367", - "source": "voluptate qui fugiat nulla", - "submission_number": "voluptate reprehenderit nisi nostrud elit" - }, + "acquisition_sources": [ + { + "datetime": "2234-06-08T05:46:05.263Z", + "email": "rLmoeWrC4fI6iZ@IBfNPyIlR.vwp", + "internal_uid": 76120453, + "method": "submitter", + "orcid": "8351-4483-7418-2367", + "source": "voluptate qui fugiat nulla", + "submission_number": "voluptate reprehenderit nisi nostrud elit" + } + ], "advisors": [ { "curated_relation": false, diff --git a/tests/integration/fixtures/expected_data_hep.yaml b/tests/integration/fixtures/expected_data_hep.yaml index 078cde05..0c06c296 100644 --- a/tests/integration/fixtures/expected_data_hep.yaml +++ b/tests/integration/fixtures/expected_data_hep.yaml @@ -116,14 +116,21 @@ "value":"7892" } ], - "acquisition_source":{ - "email":"albert.einstein@hep.edu", - "source":"submitter", - "internal_uid":1, - "submission_number":"12", - "orcid":"0000-0001-8528-2091", - "method":"batchuploader" - }, + "acquisition_sources":[ + { + "submission_number":"13", + "source":"submitter", + "method":"hepcrawl" + }, + { + "email":"albert.einstein@hep.edu", + "source":"submitter", + "internal_uid":1, + "submission_number":"12", + "orcid":"0000-0001-8528-2091", + "method":"batchuploader" + } + ], "license":[ { "url":"https://opensource.org/licenses/MIT", diff --git a/tests/integration/fixtures/hep_example.json b/tests/integration/fixtures/hep_example.json index 4a7e67ae..979b8a55 100644 --- a/tests/integration/fixtures/hep_example.json +++ b/tests/integration/fixtures/hep_example.json @@ -142,15 +142,17 @@ } } ], - "acquisition_source": { - "datetime": "4655-08-10T13:24:39.377Z", - "email": "YMD@FNozTaLqsdeiHRpHEe.gmlg", - "internal_uid": 33119180, - "method": "submitter", - "orcid": "7517-2459-3361-8685", - "source": "do occaecat voluptate elit veniam", - "submission_number": "Lorem ad reprehenderit sed esse" - }, + "acquisition_sources": [ + { + "datetime": "4655-08-10T13:24:39.377Z", + "email": "YMD@FNozTaLqsdeiHRpHEe.gmlg", + "internal_uid": 33119180, + "method": "submitter", + "orcid": "7517-2459-3361-8685", + "source": "do occaecat voluptate elit veniam", + "submission_number": "Lorem ad reprehenderit sed esse" + } + ], "arxiv_eprints": [ { "categories": [ diff --git a/tests/integration/fixtures/input_data_hep.yaml b/tests/integration/fixtures/input_data_hep.yaml index 10690f00..7a3f6c39 100644 --- a/tests/integration/fixtures/input_data_hep.yaml +++ b/tests/integration/fixtures/input_data_hep.yaml @@ -40,6 +40,8 @@ "internal_uid":1, "email":"albert.einstein@hep.edu", "orcid":"0000-0001-8528-2091", + "method2":"hepcrawl", + "submission_number2":"13", "document_type":"report", "material":"visual production", "holder":"cern", diff --git a/tests/integration/test_builders.py b/tests/integration/test_builders.py index 3fccea92..a407ba74 100644 --- a/tests/integration/test_builders.py +++ b/tests/integration/test_builders.py @@ -132,6 +132,10 @@ def test_literature_builder_valid_record(input_data_hep, expected_data_hep): email=input_data_hep['email'], orcid=input_data_hep['orcid'] ) + builder.add_acquisition_source( + method=input_data_hep['method2'], + submission_number=input_data_hep['submission_number2'], + ) builder.add_document_type(document_type=input_data_hep['document_type']) builder.add_copyright( material=input_data_hep['material'],