From e6dc9deb56bd21ccd03b3ad80db50a9071d996a3 Mon Sep 17 00:00:00 2001 From: Jan van Mansum Date: Mon, 9 Dec 2024 14:58:51 +0100 Subject: [PATCH] WIP (#3) * FilesInDatasetCache * Extended examples * rename * Changed copy-examples.sh so that you can run it from a different directory and it will still use the dd-dataverse-ingest project directory as the reference point. * Transplanted moveFile algortithm from old code * Copied old classes to new package before pruning * Pruning and simplifying old code * More pruning * More pruning * Simplified DepositFile/FileInfo * Renamed Deposit in legacy code to DansBagDeposit * Removed originalFilePathMapping as we are not going to use it in the Data Stations * Removed domain package * Removed more unnecessary classes and interfaces * Done simplifying for now * First transplant of update logic from dd-ingest-flow. Now, to test! * Refactored EditFilesComposer for easier unit testing. * Some unit tests for EditFilesComposer. * Finishing testing EditFilesComposer for now. * Started testing EditFilesComposerForUpdate. * Added 'addUnrestrictedFiles' property. It is no longer the default, but must be declared explicitly. 'ignoredFiles' can now be removed. * Made ignoring files the default instead of adding unrestricted. * Started implementing tests for the sword examples. * Removed redundant helper. * Added saving the NBN to deposit.properties onSuccess + fixes some regression introduced after changing to explicit addUnrestrictedFiles. * Some fixes in update editor. * movements example from sword-examples working correctly. --- copy-examples.sh | 9 +- docs/description.md | 11 +- pom.xml | 22 +- .../DdDataverseIngestApplication.java | 2 +- .../client/ValidateDansBagService.java | 5 +- .../config/DansDepositConversionConfig.java | 1 - .../dans/dvingest/config/IngestConfig.java | 2 +- .../config/ValidateDansBagConfig.java | 1 - .../dvingest/core/DataverseIngestDeposit.java | 7 +- .../nl/knaw/dans/dvingest/core/Deposit.java | 2 + .../knaw/dans/dvingest/core/DepositTask.java | 8 +- .../knaw/dans/dvingest/core/IngestArea.java | 3 - .../core/bagprocessor/BagProcessor.java | 4 +- .../core/bagprocessor/DataversePath.java | 2 +- .../FileUploadInclusionPredicate.java | 14 +- .../core/bagprocessor/FilesEditor.java | 115 ++--- .../bagprocessor/FilesInDatasetCache.java | 125 +++++ .../core/bagprocessor/package-info.java} | 22 +- .../core/dansbag/DansBagMappingService.java | 18 +- .../dansbag/DansBagMappingServiceImpl.java | 271 +++++------ .../core/dansbag/DansDepositConverter.java | 9 +- .../core/dansbag/DansDepositProperties.java | 54 +++ .../core/dansbag/DansDepositSupport.java | 40 +- .../core/dansbag/EditFilesComposer.java | 161 +++++++ .../dansbag/EditFilesComposerForUpdate.java | 207 +++++++++ .../core/dansbag/LightweightBagInfo.java} | 29 +- .../core/dansbag/ManifestUtil.java} | 9 +- .../dans/dvingest/core/dansbag/SetUtils.java | 38 ++ .../core/dansbag/SupportedLicenses.java | 13 +- .../core/dansbag/deposit/DansBagDeposit.java} | 21 +- .../deposit/DansBagDepositReader.java} | 26 +- .../deposit/DansBagDepositReaderImpl.java | 207 +++++++++ .../core/dansbag/deposit/DepositFile.java} | 21 +- .../core/dansbag/deposit}/DepositState.java | 2 +- .../core/dansbag/deposit}/FileInfo.java | 16 +- .../core/dansbag/deposit}/VaultMetadata.java | 18 +- .../core/dansbag/deposit/package-info.java} | 11 +- .../exception/InvalidDepositException.java | 2 +- .../MissingRequiredFieldException.java | 2 +- .../exception/RejectedDepositException.java | 8 +- .../mapper}/DepositDatasetFieldNames.java | 2 +- .../DepositToDvDatasetMetadataMapper.java | 86 ++-- .../builder/ArchaeologyFieldBuilder.java | 18 +- .../mapper/builder/CitationFieldBuilder.java | 42 +- .../builder/CompoundFieldGenerator.java | 2 +- .../mapper/builder/DataVaultFieldBuilder.java | 14 +- .../dansbag}/mapper/builder/FieldBuilder.java | 2 +- .../mapper/builder/PrimitiveFieldBuilder.java | 2 +- .../mapper/builder/RelationFieldBuilder.java | 8 +- .../mapper/builder/RightsFieldBuilder.java | 8 +- .../builder/TemporalSpatialFieldBuilder.java | 12 +- .../mapper/mapping/AbrAcquisitionMethod.java | 6 +- .../dansbag}/mapper/mapping/AbrReport.java | 6 +- .../dansbag}/mapper/mapping/AccessRights.java | 4 +- .../core/dansbag}/mapper/mapping/Amd.java | 8 +- .../dansbag}/mapper/mapping/Audience.java | 2 +- .../core/dansbag}/mapper/mapping/Author.java | 4 +- .../core/dansbag}/mapper/mapping/Base.java | 6 +- .../core/dansbag}/mapper/mapping/Contact.java | 10 +- .../dansbag}/mapper/mapping/Contributor.java | 6 +- .../core/dansbag}/mapper/mapping/Creator.java | 6 +- .../mapper/mapping/DatesOfCollection.java | 8 +- .../dansbag}/mapper/mapping/DcxDaiAuthor.java | 41 +- .../mapper/mapping/DcxDaiOrganization.java | 35 +- .../mapping/DepositPropertiesOtherDoi.java | 8 +- .../DepositPropertiesVaultMetadata.java | 8 +- .../dansbag}/mapper/mapping/Description.java | 8 +- .../dansbag}/mapper/mapping/FileElement.java | 19 +- .../core/dansbag}/mapper/mapping/Funder.java | 10 +- .../mapping/HasOrganizationalIdentifier.java | 8 +- .../dansbag}/mapper/mapping/IdUriHelper.java | 2 +- .../dansbag}/mapper/mapping/Identifier.java | 28 +- .../dansbag}/mapper/mapping/InCollection.java | 2 +- .../dansbag}/mapper/mapping/Language.java | 10 +- .../dansbag}/mapper/mapping/LicenseElem.java | 4 +- .../dansbag}/mapper/mapping/PersonalData.java | 2 +- .../dansbag}/mapper/mapping/Publisher.java | 14 +- .../dansbag}/mapper/mapping/Relation.java | 10 +- .../core/dansbag}/mapper/mapping/Spatial.java | 2 +- .../dansbag}/mapper/mapping/SpatialBox.java | 14 +- .../mapper/mapping/SpatialCoverage.java | 2 +- .../dansbag}/mapper/mapping/SpatialPoint.java | 10 +- .../core/dansbag}/mapper/mapping/Subject.java | 10 +- .../dansbag}/mapper/mapping/SubjectAbr.java | 16 +- .../dansbag}/mapper/mapping/TemporalAbr.java | 10 +- .../core/dansbag/package-info.java} | 11 +- .../core/dansbag/xml}/XPathConstants.java | 2 +- .../core/dansbag/xml}/XPathEvaluator.java | 2 +- .../core/dansbag/xml}/XmlNamespaces.java | 2 +- .../core/dansbag/xml}/XmlReader.java | 4 +- .../core/dansbag/xml}/XmlReaderImpl.java | 13 +- .../core/service/DataverseService.java | 6 +- .../core/service/DataverseServiceImpl.java | 31 ++ .../core/service/YamlServiceImpl.java | 1 - .../dans/dvingest/core/yaml/AddEmbargo.java | 1 - .../dans/dvingest/core/yaml/EditFiles.java | 5 +- .../dvingest/core/yaml/EditFilesRoot.java | 1 - .../core/yaml/package-info.java} | 11 +- .../ingest/core/dataverse/DatasetService.java | 35 -- .../ingest/core/deposit/BagDirResolver.java | 41 -- .../core/deposit/BagDirResolverImpl.java | 66 --- .../core/deposit/DepositFileLister.java | 27 -- .../core/deposit/DepositFileListerImpl.java | 85 ---- .../core/deposit/DepositLocationReader.java | 27 -- .../deposit/DepositLocationReaderImpl.java | 97 ---- .../ingest/core/deposit/DepositManager.java | 77 --- .../core/deposit/DepositManagerImpl.java | 61 --- .../ingest/core/deposit/DepositReader.java | 30 -- .../core/deposit/DepositReaderImpl.java | 139 ------ .../core/deposit/DepositWriterImpl.java | 65 --- .../ingest/core/domain/DatasetAuthor.java | 33 -- .../core/domain/DatasetOrganization.java | 30 -- .../dans/ingest/core/domain/DepositFile.java | 39 -- .../ingest/core/domain/DepositLocation.java | 37 -- .../core/domain/OriginalFilePathMapping.java | 51 -- .../CannotUpdateDraftDatasetException.java | 25 - .../DepositorValidatorException.java | 26 -- .../exception/FailedDepositException.java | 28 -- .../InvalidDatasetStateException.java | 26 -- .../exception/MissingTargetException.java | 22 - .../dans/ingest/core/io/BagDataManager.java | 82 ---- .../ingest/core/io/BagDataManagerImpl.java | 122 ----- .../ingest/core/service/DatasetCreator.java | 169 ------- .../ingest/core/service/DatasetEditor.java | 252 ---------- .../ingest/core/service/DatasetUpdater.java | 439 ------------------ .../ingest/core/service/ManifestHelper.java | 26 -- .../ingest/core/service/ZipFileHandler.java | 101 ---- .../FileUploadInclusionPredicateTest.java | 33 +- .../core/bagprocessor/FilesEditorTest.java | 4 +- .../FilesInDatasetCacheNonNullTest.java | 84 ++++ .../bagprocessor/FilesInDatasetCacheTest.java | 155 +++++++ .../dansbag/DansBagMappingServiceTest.java | 55 +++ .../core/dansbag/DansConversionFixture.java | 171 +++++++ .../dansbag/DansDepositConverterTest.java | 105 +---- .../dansbag/EditFilesComposerFixture.java | 131 ++++++ .../EditFilesComposerForUpdateTest.java | 186 ++++++++ .../core/dansbag/EditFilesComposerTest.java | 214 +++++++++ .../core/dansbag/LightweightBagInfoTest.java | 62 +++ .../1/data/probl\303\250me/.txt" | 2 + .../1/dataset.yml | 5 + .../1/edit-files.yml | 12 +- .../1/manifest-sha512.txt | 3 +- .../2/data/ignored-file.txt | 1 + .../2/edit-files.yml | 3 + .../deposit.properties | 11 + .../revision01/README.md | 28 ++ .../revision01/bag-info.txt | 3 + .../revision01/bagit.txt | 2 + .../revision01/data/file1.txt | 1 + .../revision01/data/file2.txt | 1 + .../revision01/data/file3.txt | 1 + .../revision01/data/file4.txt | 1 + .../revision01/data/subdir/fileA.txt | 1 + .../revision01/data/subdir/fileB.txt | 1 + .../revision01/data/subdir/fileC.txt | 1 + .../revision01/data/subdir/fileD.txt | 1 + .../revision01/manifest-sha1.txt | 8 + .../revision01/metadata/dataset.xml | 39 ++ .../revision01/metadata/files.xml | 29 ++ .../revision01/tagmanifest-sha1.txt | 4 + .../deposit.properties | 11 + .../revision02/bag-info.txt | 4 + .../revision02/bagit.txt | 2 + .../revision02/data/file1.txt | 1 + .../revision02/data/file2-renamed.txt | 1 + .../revision02/data/file4.txt | 1 + .../revision02/data/from-root-to-subdir.txt | 1 + .../revision02/data/subdir/fileA.txt | 1 + .../revision02/data/subdir/fileB-renamed.txt | 1 + .../revision02/data/subdir/fileD.txt | 1 + .../data/subdir/from-subdir-to-root.txt | 1 + .../revision02/manifest-sha1.txt | 8 + .../revision02/metadata/dataset.xml | 34 ++ .../revision02/metadata/files.xml | 33 ++ .../revision02/tagmanifest-sha1.txt | 4 + .../deposit.properties | 11 + .../revision03/bag-info.txt | 4 + .../revision03/bagit.txt | 2 + .../revision03/data/file1.txt | 1 + .../revision03/data/file2-renamed.txt | 1 + .../revision03/data/file3.txt | 1 + .../revision03/data/file4.txt | 1 + .../revision03/data/from-subdir-to-root.txt | 1 + .../revision03/data/subdir/fileA.txt | 1 + .../revision03/data/subdir/fileB-renamed.txt | 1 + .../revision03/data/subdir/fileC.txt | 1 + .../revision03/data/subdir/fileD.txt | 1 + .../data/subdir/from-root-to-subdir.txt | 1 + .../revision03/manifest-sha1.txt | 10 + .../revision03/metadata/dataset.xml | 34 ++ .../revision03/metadata/files.xml | 39 ++ .../revision03/tagmanifest-sha1.txt | 4 + 192 files changed, 2927 insertions(+), 3070 deletions(-) create mode 100644 src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesInDatasetCache.java rename src/main/java/nl/knaw/dans/{ingest/core/domain/OutboxSubDir.java => dvingest/core/bagprocessor/package-info.java} (66%) create mode 100644 src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositProperties.java create mode 100644 src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java create mode 100644 src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java rename src/main/java/nl/knaw/dans/{ingest/core/io/FileServiceImpl.java => dvingest/core/dansbag/LightweightBagInfo.java} (53%) rename src/main/java/nl/knaw/dans/{ingest/core/service/ManifestHelperImpl.java => dvingest/core/dansbag/ManifestUtil.java} (93%) create mode 100644 src/main/java/nl/knaw/dans/dvingest/core/dansbag/SetUtils.java rename src/main/java/nl/knaw/dans/{ingest/core/domain/Deposit.java => dvingest/core/dansbag/deposit/DansBagDeposit.java} (89%) rename src/main/java/nl/knaw/dans/{ingest/core/deposit/DepositWriter.java => dvingest/core/dansbag/deposit/DansBagDepositReader.java} (52%) create mode 100644 src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DansBagDepositReaderImpl.java rename src/main/java/nl/knaw/dans/{ingest/core/io/FileService.java => dvingest/core/dansbag/deposit/DepositFile.java} (71%) rename src/main/java/nl/knaw/dans/{ingest/core/domain => dvingest/core/dansbag/deposit}/DepositState.java (93%) rename src/main/java/nl/knaw/dans/{ingest/core/domain => dvingest/core/dansbag/deposit}/FileInfo.java (79%) rename src/main/java/nl/knaw/dans/{ingest/core/domain => dvingest/core/dansbag/deposit}/VaultMetadata.java (75%) rename src/main/java/nl/knaw/dans/{ingest/core/exception/TargetBlockedException.java => dvingest/core/dansbag/deposit/package-info.java} (78%) rename src/main/java/nl/knaw/dans/{ingest/core => dvingest/core/dansbag}/exception/InvalidDepositException.java (94%) rename src/main/java/nl/knaw/dans/{ingest/core => dvingest/core/dansbag}/exception/MissingRequiredFieldException.java (94%) rename src/main/java/nl/knaw/dans/{ingest/core => dvingest/core/dansbag}/exception/RejectedDepositException.java (75%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag/mapper}/DepositDatasetFieldNames.java (99%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/DepositToDvDatasetMetadataMapper.java (88%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/builder/ArchaeologyFieldBuilder.java (67%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/builder/CitationFieldBuilder.java (69%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/builder/CompoundFieldGenerator.java (93%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/builder/DataVaultFieldBuilder.java (69%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/builder/FieldBuilder.java (98%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/builder/PrimitiveFieldBuilder.java (97%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/builder/RelationFieldBuilder.java (77%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/builder/RightsFieldBuilder.java (77%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/builder/TemporalSpatialFieldBuilder.java (72%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/AbrAcquisitionMethod.java (80%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/AbrReport.java (82%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/AccessRights.java (93%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Amd.java (87%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Audience.java (97%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Author.java (91%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Base.java (95%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Contact.java (72%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Contributor.java (95%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Creator.java (79%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/DatesOfCollection.java (81%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/DcxDaiAuthor.java (78%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/DcxDaiOrganization.java (75%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/DepositPropertiesOtherDoi.java (74%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/DepositPropertiesVaultMetadata.java (82%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Description.java (90%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/FileElement.java (92%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Funder.java (84%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/HasOrganizationalIdentifier.java (81%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/IdUriHelper.java (96%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Identifier.java (77%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/InCollection.java (96%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Language.java (84%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/LicenseElem.java (94%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/PersonalData.java (94%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Publisher.java (69%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Relation.java (83%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Spatial.java (97%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/SpatialBox.java (79%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/SpatialCoverage.java (96%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/SpatialPoint.java (82%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/Subject.java (87%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/SubjectAbr.java (74%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag}/mapper/mapping/TemporalAbr.java (77%) rename src/main/java/nl/knaw/dans/{ingest/core/exception/TargetNotFoundException.java => dvingest/core/dansbag/package-info.java} (74%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag/xml}/XPathConstants.java (94%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag/xml}/XPathEvaluator.java (98%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag/xml}/XmlNamespaces.java (97%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag/xml}/XmlReader.java (87%) rename src/main/java/nl/knaw/dans/{ingest/core/service => dvingest/core/dansbag/xml}/XmlReaderImpl.java (87%) rename src/main/java/nl/knaw/dans/{ingest/core/exception/InvalidDepositorRoleException.java => dvingest/core/yaml/package-info.java} (76%) delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/dataverse/DatasetService.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/deposit/BagDirResolver.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/deposit/BagDirResolverImpl.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/deposit/DepositFileLister.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/deposit/DepositFileListerImpl.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/deposit/DepositLocationReader.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/deposit/DepositLocationReaderImpl.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/deposit/DepositManager.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/deposit/DepositManagerImpl.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/deposit/DepositReader.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/deposit/DepositReaderImpl.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/deposit/DepositWriterImpl.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/domain/DatasetAuthor.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/domain/DatasetOrganization.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/domain/DepositFile.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/domain/DepositLocation.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/domain/OriginalFilePathMapping.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/exception/CannotUpdateDraftDatasetException.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/exception/DepositorValidatorException.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/exception/FailedDepositException.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/exception/InvalidDatasetStateException.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/exception/MissingTargetException.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/io/BagDataManager.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/io/BagDataManagerImpl.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/service/DatasetCreator.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/service/DatasetEditor.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/service/DatasetUpdater.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/service/ManifestHelper.java delete mode 100644 src/main/java/nl/knaw/dans/ingest/core/service/ZipFileHandler.java create mode 100644 src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesInDatasetCacheNonNullTest.java create mode 100644 src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesInDatasetCacheTest.java create mode 100644 src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceTest.java create mode 100644 src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansConversionFixture.java create mode 100644 src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerFixture.java create mode 100644 src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdateTest.java create mode 100644 src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerTest.java create mode 100644 src/test/java/nl/knaw/dans/dvingest/core/dansbag/LightweightBagInfoTest.java create mode 100644 "src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/data/probl\303\250me/.txt" create mode 100644 src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/2/data/ignored-file.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/deposit.properties create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/README.md create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/bag-info.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/bagit.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file1.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file2.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file3.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file4.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileA.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileB.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileC.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileD.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/manifest-sha1.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/metadata/dataset.xml create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/metadata/files.xml create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/tagmanifest-sha1.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/deposit.properties create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/bag-info.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/bagit.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/file1.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/file2-renamed.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/file4.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/from-root-to-subdir.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/fileA.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/fileB-renamed.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/fileD.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/from-subdir-to-root.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/manifest-sha1.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/metadata/dataset.xml create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/metadata/files.xml create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/tagmanifest-sha1.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/deposit.properties create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/bag-info.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/bagit.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file1.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file2-renamed.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file3.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file4.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/from-subdir-to-root.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileA.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileB-renamed.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileC.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileD.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/from-root-to-subdir.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/manifest-sha1.txt create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/metadata/dataset.xml create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/metadata/files.xml create mode 100644 src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/tagmanifest-sha1.txt diff --git a/copy-examples.sh b/copy-examples.sh index bee3278..768c381 100755 --- a/copy-examples.sh +++ b/copy-examples.sh @@ -15,10 +15,9 @@ # limitations under the License. # - +SCRIPT_DIR=$(cd $(dirname "$0") && pwd) BATCH_NAME=${1:-"test-deposits"} -rm -rf data/import/inbox/$BATCH_NAME -rm -fr data/import/outbox/$BATCH_NAME -cp -r src/test/resources/test-deposits data/import/inbox/$BATCH_NAME - +rm -rf "$SCRIPT_DIR/data/import/inbox/$BATCH_NAME" +rm -rf "$SCRIPT_DIR/data/import/outbox/$BATCH_NAME" +cp -r "$SCRIPT_DIR/src/test/resources/test-deposits" "$SCRIPT_DIR/data/import/inbox/$BATCH_NAME" diff --git a/docs/description.md b/docs/description.md index 3bacbe7..195366e 100644 --- a/docs/description.md +++ b/docs/description.md @@ -82,7 +82,8 @@ editFiles: addRestrictedFiles: - 'file4.txt' - 'subdirectory/file5.txt' - # Unrestricted files are added by default at this point + addUnrestrictedFiles: + - 'file6.txt' moveFiles: - from: 'file6.txt' # Old location in the dataset to: 'subdirectory/file6.txt' # New location in the dataset @@ -92,11 +93,9 @@ editFiles: directoryLabel: "subdirectory" restricted: false categories: [ 'Testlabel' ] - ignoreFiles: - - 'file7.txt' # This file will NOT be added to the dataset - renameAtUploadFiles: - - from: 'file8.txt' # Local file name - to: 'file9.txt' # The file name assigned in the dataset + autoRenameFiles: + - from: "Unsanitize'd/file?" # Local file name + to: "Sanitize_d/file_" # The file name assigned in the dataset addEmbargoes: - filePaths: [ 'file1.txt' ] # All other files will NOT be embargoed dateAvailable: '2030-01-01' diff --git a/pom.xml b/pom.xml index 8303407..b80ce15 100644 --- a/pom.xml +++ b/pom.xml @@ -37,10 +37,11 @@ nl.knaw.dans.dvingest.DdDataverseIngestApplication 0.2.0 1.0.0 + 1.0.0 - scm:git:ssh://github.com/DANS-KNAW/dd-dataverse-ingest + scm:git:ssh://github.com/DANS-KNAW/${project.artifactId} HEAD @@ -216,6 +217,7 @@ maven-dependency-plugin + compile-dependencies initialize unpack @@ -237,6 +239,24 @@ + + test-dependencies + generate-test-resources + + unpack + + + + + nl.knaw.dans + dd-dans-sword2-examples + ${dd-dans-sword2-examples.version} + ${project.build.directory}/test + example-bags/** + + + + diff --git a/src/main/java/nl/knaw/dans/dvingest/DdDataverseIngestApplication.java b/src/main/java/nl/knaw/dans/dvingest/DdDataverseIngestApplication.java index 13d6784..f2716d6 100644 --- a/src/main/java/nl/knaw/dans/dvingest/DdDataverseIngestApplication.java +++ b/src/main/java/nl/knaw/dans/dvingest/DdDataverseIngestApplication.java @@ -28,6 +28,7 @@ import nl.knaw.dans.dvingest.core.dansbag.DansBagMappingService; import nl.knaw.dans.dvingest.core.dansbag.DansBagMappingServiceImpl; import nl.knaw.dans.dvingest.core.dansbag.SupportedLicenses; +import nl.knaw.dans.dvingest.core.dansbag.mapper.DepositToDvDatasetMetadataMapper; import nl.knaw.dans.dvingest.core.service.DataverseService; import nl.knaw.dans.dvingest.core.service.DataverseServiceImpl; import nl.knaw.dans.dvingest.core.service.UtilityServicesImpl; @@ -35,7 +36,6 @@ import nl.knaw.dans.dvingest.resources.DefaultApiResource; import nl.knaw.dans.dvingest.resources.IllegalArgumentExceptionMapper; import nl.knaw.dans.dvingest.resources.IngestApiResource; -import nl.knaw.dans.ingest.core.service.mapper.DepositToDvDatasetMetadataMapper; import nl.knaw.dans.lib.dataverse.DataverseException; import nl.knaw.dans.lib.util.MappingLoader; import org.apache.commons.io.FileUtils; diff --git a/src/main/java/nl/knaw/dans/dvingest/client/ValidateDansBagService.java b/src/main/java/nl/knaw/dans/dvingest/client/ValidateDansBagService.java index 02c0cb4..b189d13 100644 --- a/src/main/java/nl/knaw/dans/dvingest/client/ValidateDansBagService.java +++ b/src/main/java/nl/knaw/dans/dvingest/client/ValidateDansBagService.java @@ -15,12 +15,13 @@ */ package nl.knaw.dans.dvingest.client; -import nl.knaw.dans.ingest.core.exception.RejectedDepositException; import nl.knaw.dans.validatedansbag.client.api.ValidateOkDto; -import nl.knaw.dans.validatedansbag.client.api.ValidateOkDto.InformationPackageTypeEnum; import java.nio.file.Path; +/** + * Proxy for dd-validate-dans-bag service. + */ public interface ValidateDansBagService { ValidateOkDto validate(Path bag); diff --git a/src/main/java/nl/knaw/dans/dvingest/config/DansDepositConversionConfig.java b/src/main/java/nl/knaw/dans/dvingest/config/DansDepositConversionConfig.java index 343529c..52a0f37 100644 --- a/src/main/java/nl/knaw/dans/dvingest/config/DansDepositConversionConfig.java +++ b/src/main/java/nl/knaw/dans/dvingest/config/DansDepositConversionConfig.java @@ -16,7 +16,6 @@ package nl.knaw.dans.dvingest.config; import lombok.Data; -import lombok.NonNull; import javax.validation.Valid; import javax.validation.constraints.NotNull; diff --git a/src/main/java/nl/knaw/dans/dvingest/config/IngestConfig.java b/src/main/java/nl/knaw/dans/dvingest/config/IngestConfig.java index aa5d5e5..bffce7b 100644 --- a/src/main/java/nl/knaw/dans/dvingest/config/IngestConfig.java +++ b/src/main/java/nl/knaw/dans/dvingest/config/IngestConfig.java @@ -16,8 +16,8 @@ package nl.knaw.dans.dvingest.config; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; import io.dropwizard.util.DataSize; +import lombok.Data; import javax.validation.Valid; import javax.validation.constraints.NotNull; diff --git a/src/main/java/nl/knaw/dans/dvingest/config/ValidateDansBagConfig.java b/src/main/java/nl/knaw/dans/dvingest/config/ValidateDansBagConfig.java index 226d55e..f45f16c 100644 --- a/src/main/java/nl/knaw/dans/dvingest/config/ValidateDansBagConfig.java +++ b/src/main/java/nl/knaw/dans/dvingest/config/ValidateDansBagConfig.java @@ -15,7 +15,6 @@ */ package nl.knaw.dans.dvingest.config; -import io.dropwizard.client.HttpClientConfiguration; import io.dropwizard.client.JerseyClientConfiguration; import lombok.Data; diff --git a/src/main/java/nl/knaw/dans/dvingest/core/DataverseIngestDeposit.java b/src/main/java/nl/knaw/dans/dvingest/core/DataverseIngestDeposit.java index 282afd6..235a846 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/DataverseIngestDeposit.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/DataverseIngestDeposit.java @@ -43,7 +43,6 @@ public class DataverseIngestDeposit implements Comparable { @Override public boolean evaluate(File file) { - if (restrictedFiles) { - return editFiles != null && isRestricted(file) && notReplaced(file) && notIgnored(file); - } - else { - return editFiles == null || notRestricted(file) && notReplaced(file) && notIgnored(file); - } + return editFiles != null && (restrictedFiles ? isRestricted(file) : notRestricted(file)) + && notReplaced(file); } private boolean notReplaced(File file) { @@ -47,10 +43,6 @@ private boolean isRestricted(File file) { } private boolean notRestricted(File file) { - return !editFiles.getAddRestrictedFiles().contains(dataDir.relativize(file.toPath()).toString()); - } - - private boolean notIgnored(File file) { - return !editFiles.getIgnoreFiles().contains(dataDir.relativize(file.toPath()).toString()); + return editFiles.getAddUnrestrictedFiles().contains(dataDir.relativize(file.toPath()).toString()); } } \ No newline at end of file diff --git a/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditor.java b/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditor.java index af2ad80..e693839 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditor.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditor.java @@ -18,11 +18,11 @@ import lombok.AccessLevel; import lombok.Getter; import lombok.NonNull; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import nl.knaw.dans.dvingest.core.service.DataverseService; import nl.knaw.dans.dvingest.core.service.UtilityServices; import nl.knaw.dans.dvingest.core.yaml.EditFiles; +import nl.knaw.dans.dvingest.core.yaml.FromTo; import nl.knaw.dans.lib.dataverse.DataverseException; import nl.knaw.dans.lib.dataverse.model.dataset.Embargo; import nl.knaw.dans.lib.dataverse.model.file.FileMeta; @@ -35,32 +35,42 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; @Slf4j -@RequiredArgsConstructor + +/** + * Edits files in a dataset in Dataverse, based on the edit-files.yml file, which has been read and parsed into an EditFiles object. + */ public class FilesEditor { - @NonNull private final UUID depositId; - @NonNull private final Path dataDir; private final EditFiles editFiles; - - @NonNull private final DataverseService dataverseService; - - @NonNull private final UtilityServices utilityServices; + @Getter(AccessLevel.PACKAGE) // for testing + private final FilesInDatasetCache filesInDatasetCache; private String pid; - @Getter(AccessLevel.PACKAGE) // Getter for unit testing - private final Map filesInDataset = new java.util.HashMap<>(); - private boolean filesRetrieved = false; + public FilesEditor(@NonNull UUID depositId, @NonNull Path dataDir, @NonNull EditFiles editFiles, @NonNull DataverseService dataverseService, + @NonNull UtilityServices utilityServices) { + this.depositId = depositId; + this.dataDir = dataDir; + this.editFiles = editFiles; + this.dataverseService = dataverseService; + this.utilityServices = utilityServices; + this.filesInDatasetCache = new FilesInDatasetCache(dataverseService, getRenameMap(editFiles.getAutoRenameFiles())); + } - private Map renameMap; + private static Map getRenameMap(List renames) { + return renames.stream() + .map(rename -> Map.entry(rename.getFrom(), rename.getTo())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } public void editFiles(String pid) throws IOException, DataverseException { /* @@ -79,14 +89,12 @@ public void editFiles(String pid) throws IOException, DataverseException { log.debug("Start editing files for deposit {}", depositId); this.pid = pid; - this.renameMap = getRenameMap(); + filesInDatasetCache.downloadFromDataset(pid); if (editFiles != null) { deleteFiles(); replaceFiles(); addRestrictedFiles(); - } - addUnrestrictedFiles(); - if (editFiles != null) { + addUnrestrictedFiles(); moveFiles(); updateFileMetas(); addEmbargoes(); @@ -102,24 +110,24 @@ private boolean isEmptyDir(Path dir) throws IOException { private void deleteFiles() throws IOException, DataverseException { log.debug("Start deleting {} files for deposit {}", depositId, editFiles.getDeleteFiles().size()); - for (var file : editFiles.getDeleteFiles()) { - log.debug("Deleting file: {}", file); - var fileToDelete = filesInDataset().get(file); + for (var filepath : editFiles.getDeleteFiles()) { + log.debug("Deleting file: {}", filepath); + var fileToDelete = filesInDatasetCache.get(filepath); if (fileToDelete == null) { - throw new IllegalArgumentException("File to delete not found in dataset: " + file); + throw new IllegalArgumentException("File to delete not found in dataset: " + filepath); } dataverseService.deleteFile(fileToDelete.getDataFile().getId()); - filesInDataset.remove(file); + filesInDatasetCache.remove(filepath); } log.debug("End deleting files for deposit {}", depositId); } private void replaceFiles() throws IOException, DataverseException { log.debug("Start replacing {} files for deposit {}", depositId, editFiles.getReplaceFiles().size()); - for (var file : editFiles.getReplaceFiles()) { - log.debug("Replacing file: {}", file); - var fileMeta = filesInDataset().get(file); - dataverseService.replaceFile(pid, fileMeta, dataDir.resolve(file)); + for (var filepath : editFiles.getReplaceFiles()) { + log.debug("Replacing file: {}", filepath); + var fileMeta = filesInDatasetCache.get(filepath); + dataverseService.replaceFile(pid, fileMeta, dataDir.resolve(filepath)); } log.debug("End replacing files for deposit {}", depositId); } @@ -154,16 +162,10 @@ private Iterator getRestrictedFilesToUpload() { new FileUploadInclusionPredicate(editFiles, dataDir, true)); } - private Map getRenameMap() { - return editFiles.getRenameAtUploadFiles().stream() - .map(rename -> Map.entry(rename.getFrom(), rename.getTo())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - private void uploadFileBatch(PathIterator iterator, boolean restrict) throws IOException, DataverseException { var tempZipFile = utilityServices.createTempZipFile(); try { - var zipFile = utilityServices.createPathIteratorZipperBuilder(renameMap) + var zipFile = utilityServices.createPathIteratorZipperBuilder(filesInDatasetCache.getAutoRenamedFiles()) .rootDir(dataDir) .sourceIterator(iterator) .targetZipFile(tempZipFile) @@ -172,10 +174,10 @@ private void uploadFileBatch(PathIterator iterator, boolean restrict) throws IOE var fileMeta = new FileMeta(); fileMeta.setRestricted(restrict); log.debug("Start uploading zip file at {} for deposit {}", zipFile, depositId); - var fileList = dataverseService.addFile(pid, zipFile, fileMeta); - log.debug("Uploaded {} files, {} cumulative)", fileList.getFiles().size(), iterator.getIteratedCount()); - for (var file : fileList.getFiles()) { - filesInDataset.put(getPath(file), file); + var addedFileMetaList = dataverseService.addFile(pid, zipFile, fileMeta); + log.debug("Uploaded {} files, {} cumulative)", addedFileMetaList.getFiles().size(), iterator.getIteratedCount()); + for (var fm : addedFileMetaList.getFiles()) { + filesInDatasetCache.put(fm); // auto-rename is done by PathIteratorZipper } } finally { @@ -186,11 +188,11 @@ private void uploadFileBatch(PathIterator iterator, boolean restrict) throws IOE private void moveFiles() throws IOException, DataverseException { log.debug("Start moving files {} for deposit {}", editFiles.getMoveFiles().size(), depositId); for (var move : editFiles.getMoveFiles()) { - var fileMeta = filesInDataset().get(move.getFrom()); - var dvToPath = new DataversePath(move.getTo()); - fileMeta.setDirectoryLabel(dvToPath.getDirectoryLabel()); - fileMeta.setLabel(dvToPath.getLabel()); + var fileMeta = filesInDatasetCache.get(move.getFrom()); + fileMeta = filesInDatasetCache.createFileMetaForMovedFile(move.getTo(), fileMeta); dataverseService.updateFileMetadata(fileMeta.getDataFile().getId(), fileMeta); + filesInDatasetCache.remove(move.getFrom()); + filesInDatasetCache.put(fileMeta); // auto-rename is done by getMovedFile } log.debug("End moving files for deposit {}", depositId); } @@ -198,28 +200,12 @@ private void moveFiles() throws IOException, DataverseException { private void updateFileMetas() throws IOException, DataverseException { log.debug("Start updating {} file metas for deposit {}", editFiles.getUpdateFileMetas().size(), depositId); for (var fileMeta : editFiles.getUpdateFileMetas()) { - var id = filesInDataset().get(getPath(fileMeta)).getDataFile().getId(); + var id = filesInDatasetCache.get(getPath(fileMeta)).getDataFile().getId(); dataverseService.updateFileMetadata(id, fileMeta); } log.debug("End updating file metadata for deposit {}", depositId); } - private Map filesInDataset() throws IOException, DataverseException { - if (!filesRetrieved) { - log.debug("Start getting files in dataset for deposit {}", depositId); - var files = dataverseService.getFiles(pid); - for (var file : files) { - filesInDataset.put(getPath(file), file); - } - filesRetrieved = true; - log.debug("End getting files in dataset for deposit {}", depositId); - } - else { - log.debug("Files in dataset already retrieved for deposit {}", depositId); - } - return filesInDataset; - } - private String getPath(FileMeta file) { var dataversePath = new DataversePath(file.getDirectoryLabel(), file.getLabel()); return dataversePath.toString(); @@ -233,24 +219,11 @@ private void addEmbargoes() throws IOException, DataverseException { embargo.setReason(addEmbargo.getReason()); var fileIds = addEmbargo.getFilePaths() .stream() - .map(this::renameFile) - .map(this::throwIfFileNotInDataset) - .map(filesInDataset::get) + .map(filesInDatasetCache::get) .mapToInt(file -> file.getDataFile().getId()).toArray(); embargo.setFileIds(fileIds); dataverseService.addEmbargo(pid, embargo); } log.debug("End adding embargoes for deposit {}", depositId); } - - private String renameFile(String file) { - return renameMap.getOrDefault(file, file); - } - - private String throwIfFileNotInDataset(String file) { - if (!filesInDataset.containsKey(file)) { - throw new IllegalArgumentException("File not found in dataset: " + file); - } - return file; - } } diff --git a/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesInDatasetCache.java b/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesInDatasetCache.java new file mode 100644 index 0000000..f3b4cfd --- /dev/null +++ b/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesInDatasetCache.java @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.bagprocessor; + +import lombok.Getter; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import nl.knaw.dans.dvingest.core.service.DataverseService; +import nl.knaw.dans.lib.dataverse.DataverseException; +import nl.knaw.dans.lib.dataverse.model.file.FileMeta; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +/** + *

+ * Keeps track of the FileMeta objects of files in a dataset. The cache is initialized by downloading the files from the dataset. + *

+ */ +@Slf4j +public class FilesInDatasetCache { + private final DataverseService dataverseService; + /* + * Key: filepath after auto-rename / Value: FileMeta object + */ + @Getter + private final Map filesInDataset = new java.util.HashMap<>(); + @Getter + private final Map autoRenamedFiles; + private boolean initialized = false; + + public FilesInDatasetCache(@NonNull DataverseService dataverseService, @NonNull Map autoRenamedFiles) { + this.dataverseService = dataverseService; + this.autoRenamedFiles = Collections.unmodifiableMap(autoRenamedFiles); + } + + /** + * Returns the cached FileMeta object for the given filepath. The filepath will be auto-renamed if it is in the renamedFiles map, so the local path from the bag is used. + * + * @param filepath before auto-rename + * @return the FileMeta object for the file in the dataset + */ + public FileMeta get(@NonNull String filepath) { + return filesInDataset.get(autoRenamePath(filepath)); + } + + private String autoRenamePath(@NonNull String filepath) { + return autoRenamedFiles.getOrDefault(filepath, filepath); + } + + /** + * Adds or updates the FileMeta object for the given filepath. No auto-rename is done. It is assumed that the FileMeta object was returned by the Dataverse API or that the filepath was already + * auto-renamed by the client. + * + * @param fileMeta the FileMeta object for the file in the dataset + */ + public void put(@NonNull FileMeta fileMeta) { + filesInDataset.put(getPath(fileMeta), fileMeta); + } + + /** + * A move operation is in fact a file metadata update operation in which the directory label and label are updated. This method allows to calculate the file metadata for the moved file in the + * dataset. The filepath will be auto-renamed if it is in the renamedFiles map, so the local path from the bag is used. + * + * @param toPath new filepath before auto-rename + * @param fileMeta the FileMeta object for the file in the dataset after the move + * @return the FileMeta object for the moved file in the dataset + */ + public FileMeta createFileMetaForMovedFile(@NonNull String toPath, @NonNull FileMeta fileMeta) { + var newPath = autoRenamePath(toPath); + var dataversePath = new DataversePath(newPath); + fileMeta.setDirectoryLabel(dataversePath.getDirectoryLabel()); + fileMeta.setLabel(dataversePath.getLabel()); + return fileMeta; + } + + /** + * Removes the FileMeta object for the given filepath. The filepath will be auto-renamed if it is in the renamedFiles map, so the local path from the bag is used. + * + * @param filepath before auto-rename + */ + public void remove(@NonNull String filepath) { + filesInDataset.remove(autoRenamePath(filepath)); + } + + /** + * Download the file metadata from the dataset with the given persistent identifier, initializing the cache. This method can only be called once. Subsequent calls will throw an exception. + * + * @param pid the persistent identifier of the dataset + * @throws IOException if an I/O error occurs + * @throws DataverseException if the Dataverse API returns an error + * @throws IllegalStateException if the cache is already initialized + */ + public void downloadFromDataset(@NonNull String pid) throws IOException, DataverseException { + if (initialized) { + throw new IllegalStateException("Cache already initialized"); + } + + var files = dataverseService.getFiles(pid); + for (var file : files) { + filesInDataset.put(getPath(file), file); + } + initialized = true; + } + + private String getPath(@NonNull FileMeta file) { + var dataversePath = new DataversePath(file.getDirectoryLabel(), file.getLabel()); + return dataversePath.toString(); + } + +} diff --git a/src/main/java/nl/knaw/dans/ingest/core/domain/OutboxSubDir.java b/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/package-info.java similarity index 66% rename from src/main/java/nl/knaw/dans/ingest/core/domain/OutboxSubDir.java rename to src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/package-info.java index 053fcda..a9b57eb 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/domain/OutboxSubDir.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/package-info.java @@ -13,20 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.domain; - -public enum OutboxSubDir { - PROCESSED("processed"), - REJECTED("rejected"), - FAILED("failed"); - - private final String value; - - OutboxSubDir(String value) { - this.value = value; - } - - public String getValue() { - return value; - } -} +/** + * This package contains the classes that are responsible for processing Dataverse Ingest Bags. The main class is the {@link nl.knaw.dans.dvingest.core.bagprocessor.BagProcessor BagProcessor}. For + * clarity, it delegates parts of its work to other classes in this package. + */ +package nl.knaw.dans.dvingest.core.bagprocessor; \ No newline at end of file diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingService.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingService.java index da9d286..f5d3f7b 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingService.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingService.java @@ -15,26 +15,26 @@ */ package nl.knaw.dans.dvingest.core.dansbag; +import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDeposit; +import nl.knaw.dans.dvingest.core.dansbag.exception.InvalidDepositException; import nl.knaw.dans.dvingest.core.yaml.EditFiles; import nl.knaw.dans.dvingest.core.yaml.EditPermissions; -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.domain.DepositState; -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; +import nl.knaw.dans.lib.dataverse.DataverseException; import nl.knaw.dans.lib.dataverse.model.dataset.Dataset; import java.io.IOException; import java.nio.file.Path; public interface DansBagMappingService { - Deposit readDansDeposit(Path depositDir) throws InvalidDepositException; + String getUpdatesDataset(Path depositDir) throws IOException, DataverseException; - Dataset getDatasetMetadataFromDansDeposit(Deposit dansDeposit); + DansBagDeposit readDansDeposit(Path depositDir) throws InvalidDepositException; - EditFiles getEditFilesFromDansDeposit(Deposit dansDeposit); + Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit); - EditPermissions getEditPermissionsFromDansDeposit(Deposit dansDeposit); + EditFiles getEditFilesFromDansDeposit(DansBagDeposit dansDeposit, String updatesDataset); - void updateDepositStatus(Deposit deposit, DepositState state, String pid); + EditPermissions getEditPermissionsFromDansDeposit(DansBagDeposit dansDeposit, String updatesDataset); - String packageOriginalMetadata(Deposit dansDeposit) throws IOException; + String packageOriginalMetadata(DansBagDeposit dansDeposit) throws IOException; } diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java index 389a967..6bba7cd 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java @@ -17,38 +17,22 @@ import gov.loc.repository.bagit.reader.BagReader; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.dvingest.core.bagprocessor.DataversePath; +import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDeposit; +import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDepositReader; +import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDepositReaderImpl; +import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo; +import nl.knaw.dans.dvingest.core.dansbag.exception.InvalidDepositException; +import nl.knaw.dans.dvingest.core.dansbag.mapper.DepositToDvDatasetMetadataMapper; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.FileElement; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; +import nl.knaw.dans.dvingest.core.dansbag.xml.XmlReader; +import nl.knaw.dans.dvingest.core.dansbag.xml.XmlReaderImpl; import nl.knaw.dans.dvingest.core.service.DataverseService; -import nl.knaw.dans.dvingest.core.yaml.AddEmbargo; import nl.knaw.dans.dvingest.core.yaml.EditFiles; import nl.knaw.dans.dvingest.core.yaml.EditPermissions; -import nl.knaw.dans.dvingest.core.yaml.FromTo; -import nl.knaw.dans.ingest.core.deposit.BagDirResolver; -import nl.knaw.dans.ingest.core.deposit.BagDirResolverImpl; -import nl.knaw.dans.ingest.core.deposit.DepositFileLister; -import nl.knaw.dans.ingest.core.deposit.DepositFileListerImpl; -import nl.knaw.dans.ingest.core.deposit.DepositReader; -import nl.knaw.dans.ingest.core.deposit.DepositReaderImpl; -import nl.knaw.dans.ingest.core.deposit.DepositWriter; -import nl.knaw.dans.ingest.core.deposit.DepositWriterImpl; -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.domain.DepositState; -import nl.knaw.dans.ingest.core.domain.FileInfo; -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; -import nl.knaw.dans.ingest.core.io.BagDataManager; -import nl.knaw.dans.ingest.core.io.BagDataManagerImpl; -import nl.knaw.dans.ingest.core.io.FileService; -import nl.knaw.dans.ingest.core.io.FileServiceImpl; -import nl.knaw.dans.ingest.core.service.ManifestHelper; -import nl.knaw.dans.ingest.core.service.ManifestHelperImpl; -import nl.knaw.dans.ingest.core.service.XPathEvaluator; -import nl.knaw.dans.ingest.core.service.XmlReader; -import nl.knaw.dans.ingest.core.service.XmlReaderImpl; -import nl.knaw.dans.ingest.core.service.mapper.DepositToDvDatasetMetadataMapper; -import nl.knaw.dans.ingest.core.service.mapper.mapping.FileElement; +import nl.knaw.dans.lib.dataverse.DataverseException; import nl.knaw.dans.lib.dataverse.model.RoleAssignment; import nl.knaw.dans.lib.dataverse.model.dataset.Dataset; -import nl.knaw.dans.lib.dataverse.model.file.FileMeta; import nl.knaw.dans.lib.dataverse.model.user.AuthenticatedUser; import nl.knaw.dans.lib.util.ZipUtil; import org.apache.commons.lang3.StringUtils; @@ -57,16 +41,15 @@ import org.joda.time.format.DateTimeFormatter; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; -import java.text.SimpleDateFormat; +import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; -import java.util.ArrayList; -import java.util.Date; import java.util.List; import java.util.Map; import java.util.Optional; @@ -76,12 +59,10 @@ @Slf4j public class DansBagMappingServiceImpl implements DansBagMappingService { private static final DateTimeFormatter yyyymmddPattern = DateTimeFormat.forPattern("YYYY-MM-dd"); - private static final SimpleDateFormat yyyymmddFormat = new SimpleDateFormat("YYYY-MM-dd"); private final DepositToDvDatasetMetadataMapper depositToDvDatasetMetadataMapper; - private final DataverseService datasetService; - private final DepositReader depositReader; - private final DepositWriter depositWriter; + private final DataverseService dataverseService; + private final DansBagDepositReader dansBagDepositReader; private final SupportedLicenses supportedLicenses; private final Pattern fileExclusionPattern; private final List embargoExclusions; @@ -89,24 +70,55 @@ public class DansBagMappingServiceImpl implements DansBagMappingService { public DansBagMappingServiceImpl(DepositToDvDatasetMetadataMapper depositToDvDatasetMetadataMapper, DataverseService dataverseService, SupportedLicenses supportedLicenses, Pattern fileExclusionPattern, List embargoExclusions) { this.depositToDvDatasetMetadataMapper = depositToDvDatasetMetadataMapper; - this.datasetService = dataverseService; + this.dataverseService = dataverseService; BagReader bagReader = new BagReader(); - ManifestHelper manifestHelper = new ManifestHelperImpl(); - DepositFileLister depositFileLister = new DepositFileListerImpl(); - BagDataManager bagDataManager = new BagDataManagerImpl(bagReader); XmlReader xmlReader = new XmlReaderImpl(); - FileService fileService = new FileServiceImpl(); - BagDirResolver bagDirResolver = new BagDirResolverImpl(fileService); - depositReader = new DepositReaderImpl(xmlReader, bagDirResolver, fileService, bagDataManager, depositFileLister, manifestHelper); - depositWriter = new DepositWriterImpl(bagDataManager); + dansBagDepositReader = new DansBagDepositReaderImpl(xmlReader, bagReader); this.supportedLicenses = supportedLicenses; this.fileExclusionPattern = fileExclusionPattern; this.embargoExclusions = embargoExclusions; } @Override - public Dataset getDatasetMetadataFromDansDeposit(Deposit dansDeposit) { + public String getUpdatesDataset(Path depositDir) throws IOException, DataverseException { + var dansDepositProperties = new DansDepositProperties(depositDir.resolve("deposit.properties")); + try (var stream = Files.list(depositDir)) { + var bag = stream.filter(Files::isDirectory) + .findFirst().orElseThrow(() -> new IllegalArgumentException("No bag found in deposit")); + var bagInfo = new LightweightBagInfo(bag.resolve("bag-info.txt")); + var isVersionOf = bagInfo.get("Is-Version-Of"); + if (isVersionOf != null) { + log.debug("Found Is-Version-Of in bag-info.txt, so this is an update-deposit: {}", isVersionOf); + List results; + if (dansDepositProperties.getSwordToken() != null) { + log.debug("Found sword token in deposit.properties, looking for target dataset by sword token"); + results = dataverseService.findDoiByMetadataField("dansSwordToken", dansDepositProperties.getSwordToken()); + } + else if (depositToDvDatasetMetadataMapper.isMigration()) { + log.debug("This is a migration deposit, looking for target dataset by dansBagIt. Note that this will only work for two versions of the same dataset"); + results = dataverseService.findDoiByMetadataField("dansBagId", isVersionOf); + + } + else { + throw new IllegalArgumentException("Update deposit should have either a sword token or be a migration deposit"); + } + if (results.size() == 1) { + return results.get(0); + } + else { + throw new IllegalArgumentException("Update deposit should update exactly one dataset, found " + results.size()); + } + } + else { + log.debug("No Is-Version-Of found in bag-info.txt, so this is a deposit of a new dataset"); + return null; + } + } + } + + @Override + public Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit) { var dataset = depositToDvDatasetMetadataMapper.toDataverseDataset( dansDeposit.getDdm(), dansDeposit.getOtherDoiId(), @@ -129,85 +141,67 @@ public Dataset getDatasetMetadataFromDansDeposit(Deposit dansDeposit) { } @Override - public EditFiles getEditFilesFromDansDeposit(Deposit dansDeposit) { - var editFiles = new EditFiles(); - - var pathFileInfoMap = getFileInfo(dansDeposit); - - // TODO: in update also ignore any files that have not changed (content or metadata) - var ignoredFiles = getIgnoredFiles(pathFileInfoMap).stream().map(Path::toString).toList(); - editFiles.setIgnoreFiles(ignoredFiles); - - pathFileInfoMap = removeIgnoredFiles(pathFileInfoMap, ignoredFiles); - - editFiles.setRenameAtUploadFiles(getRenameAtUpload(pathFileInfoMap)); - - editFiles.setAddRestrictedFiles(pathFileInfoMap.entrySet().stream() - .filter(entry -> entry.getValue().getMetadata().getRestricted()) - .map(Map.Entry::getKey) - .map(Path::toString).toList()); - - editFiles.setUpdateFileMetas(pathFileInfoMap.values().stream() - .map(FileInfo::getMetadata) - .filter(this::hasAttributes) - .toList()); - + public EditFiles getEditFilesFromDansDeposit(DansBagDeposit dansDeposit, String updatesDataset) { + var files = getFileInfo(dansDeposit); var dateAvailable = getDateAvailable(dansDeposit); - var filePathsToEmbargo = getEmbargoedFiles(pathFileInfoMap, dateAvailable); - if (!filePathsToEmbargo.isEmpty()) { - var addEmbargo = new AddEmbargo(); - addEmbargo.setDateAvailable(yyyymmddFormat.format(Date.from(dateAvailable))); - addEmbargo.setFilePaths(filePathsToEmbargo.stream().map(Path::toString).toList()); - editFiles.setAddEmbargoes(List.of(addEmbargo)); + if (updatesDataset == null) { + return new EditFilesComposer(files, dateAvailable, fileExclusionPattern, embargoExclusions).composeEditFiles(); + } + else { + return new EditFilesComposerForUpdate(files, dateAvailable, updatesDataset, fileExclusionPattern, embargoExclusions, dataverseService).composeEditFiles(); } - return editFiles; } - private List getRenameAtUpload(Map files) { - ArrayList fromTos = new ArrayList<>(); - for (var entry : files.entrySet()) { - if (entry.getValue().isSanitized()) { - var from = entry.getKey().toString(); - var to = new DataversePath(entry.getValue().getMetadata().getDirectoryLabel(), entry.getValue().getMetadata().getLabel()).toString(); - fromTos.add(new FromTo(from, to)); - } + @Override + public EditPermissions getEditPermissionsFromDansDeposit(DansBagDeposit dansDeposit, String updatesDataset) { + if (updatesDataset == null) { + var userId = dansDeposit.getDepositorUserId(); + var editPermissions = new EditPermissions(); + var roleAssignment = new RoleAssignment(); + roleAssignment.setAssignee("@" + userId); + roleAssignment.setRole("contributorplus"); // TODO: make this configurable + editPermissions.setAddRoleAssignments(List.of(roleAssignment)); + return editPermissions; + } + else { + return new EditPermissions(); } - return fromTos; } - private List getIgnoredFiles(Map files) { - if (fileExclusionPattern == null) { - return List.of(); - } - return files.keySet().stream() - .filter(f -> fileExclusionPattern.matcher(f.toString()).matches()).toList(); + @Override + public String packageOriginalMetadata(DansBagDeposit dansDeposit) throws IOException { + // Zip the contents of the metadata directory of the bag + var metadataDir = dansDeposit.getBagDir().resolve("metadata"); + var zipFile = dansDeposit.getBagDir().resolve("data/original-metadata.zip"); + ZipUtil.zipDirectory(metadataDir, zipFile, false); + return zipFile.toString(); } - private Map removeIgnoredFiles(Map files, List ignoredFiles) { + // todo: move to mapping package + private Map getFileInfo(DansBagDeposit dansDeposit) { + var files = FileElement.pathToFileInfo(dansDeposit, false); // TODO: handle migration case + return files.entrySet().stream() - .filter(entry -> !ignoredFiles.contains(entry.getKey().toString())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } + .map(entry -> { + // relativize the path + var bagPath = entry.getKey(); + var fileInfo = entry.getValue(); + var newKey = Path.of("data").relativize(bagPath); - private List getEmbargoedFiles(Map files, Instant dateAvailable) { - var now = Instant.now(); - if (dateAvailable.isAfter(now)) { - return files.keySet().stream() - .filter(f -> !embargoExclusions.contains(f.toString())).toList(); - } - else { - log.debug("Date available in the past, no embargo: {}", dateAvailable); - return List.of(); - } + return Map.entry(newKey, fileInfo); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private Instant getDateAvailable(Deposit deposit) { - return XPathEvaluator.strings(deposit.getDdm(), "/ddm:DDM/ddm:profile/ddm:available") + // todo: move to mapping package + private Instant getDateAvailable(DansBagDeposit dansBagDeposit) { + return XPathEvaluator.strings(dansBagDeposit.getDdm(), "/ddm:DDM/ddm:profile/ddm:available") .map(DansBagMappingServiceImpl::parseDate) .findFirst() .orElseThrow(() -> new IllegalArgumentException("Deposit without a ddm:available element")); } + // todo: move to util class private static Instant parseDate(String value) { try { log.debug("Trying to parse {} as LocalDate", value); @@ -226,59 +220,7 @@ private static Instant parseDate(String value) { } } - @Override - public EditPermissions getEditPermissionsFromDansDeposit(Deposit dansDeposit) { - var userId = dansDeposit.getDepositorUserId(); - var editPermissions = new EditPermissions(); - var roleAssignment = new RoleAssignment(); - roleAssignment.setAssignee("@" + userId); - roleAssignment.setRole("contributorplus"); // TODO: make this configurable - editPermissions.setAddRoleAssignments(List.of(roleAssignment)); - return editPermissions; - } - - @Override - public void updateDepositStatus(Deposit deposit, DepositState state, String pid) { - try { - deposit.setState(state); - deposit.setDoi(pid); - depositWriter.saveDeposit(deposit); - } - catch (InvalidDepositException e) { - throw new RuntimeException("Failed to update deposit status", e); - } - } - - @Override - public String packageOriginalMetadata(Deposit dansDeposit) throws IOException { - // Zip the contents of the metadata directory of the bag - var metadataDir = dansDeposit.getBagDir().resolve("metadata"); - var zipFile = dansDeposit.getBagDir().resolve("data/original-metadata.zip"); - ZipUtil.zipDirectory(metadataDir, zipFile, false); - return zipFile.toString(); - } - - private boolean hasAttributes(FileMeta fileMeta) { - return (fileMeta.getCategories() != null && !fileMeta.getCategories().isEmpty()) || - (fileMeta.getDescription() != null && !fileMeta.getDescription().isBlank()); - } - - Map getFileInfo(Deposit dansDeposit) { - var files = FileElement.pathToFileInfo(dansDeposit, false); // TODO: handle migration case - - return files.entrySet().stream() - .map(entry -> { - // relativize the path - var bagPath = entry.getKey(); - var fileInfo = entry.getValue(); - var newKey = Path.of("data").relativize(bagPath); - - return Map.entry(newKey, fileInfo); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - Optional getDateOfDeposit(Deposit dansDeposit) { + Optional getDateOfDeposit(DansBagDeposit dansDeposit) { if (dansDeposit.isUpdate()) { return Optional.empty(); // See for implementation CIT025B in DatasetUpdater } @@ -287,15 +229,22 @@ Optional getDateOfDeposit(Deposit dansDeposit) { } } - Optional getDatasetContact(Deposit dansDeposit) { + Optional getDatasetContact(DansBagDeposit dansDeposit) { return Optional.ofNullable(dansDeposit.getDepositorUserId()) .filter(StringUtils::isNotBlank) - .map(userId -> datasetService.getUserById(userId) + .map(userId -> dataverseService.getUserById(userId) .orElseThrow(() -> new RuntimeException("Unable to fetch user with id " + userId))); } @Override - public Deposit readDansDeposit(Path depositDir) throws InvalidDepositException { - return depositReader.readDeposit(depositDir); + public DansBagDeposit readDansDeposit(Path depositDir) throws InvalidDepositException { + var deposit = dansBagDepositReader.readDeposit(depositDir); + try { + ManifestUtil.ensureSha1ManifestPresent(deposit.getBag()); + return deposit; + } + catch (IOException | NoSuchAlgorithmException e) { + throw new IllegalStateException("Error reading deposit", e); + } } } diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverter.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverter.java index d6d5859..7f0d83f 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverter.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverter.java @@ -16,17 +16,18 @@ package nl.knaw.dans.dvingest.core.dansbag; import lombok.AllArgsConstructor; +import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDeposit; import nl.knaw.dans.dvingest.core.service.YamlService; import nl.knaw.dans.dvingest.core.yaml.EditFilesRoot; import nl.knaw.dans.dvingest.core.yaml.EditPermissionsRoot; import nl.knaw.dans.dvingest.core.yaml.UpdateState; -import nl.knaw.dans.ingest.core.domain.Deposit; import java.io.IOException; @AllArgsConstructor public class DansDepositConverter { - private final Deposit dansDeposit; + private final DansBagDeposit dansDeposit; + private final String updatesDataset; private final DansBagMappingService mappingService; private final YamlService yamlService; @@ -37,10 +38,10 @@ public void run() throws IOException { var dataset = mappingService.getDatasetMetadataFromDansDeposit(dansDeposit); yamlService.writeYaml(dataset, dansDeposit.getBagDir().resolve("dataset.yml")); - var editFiles = mappingService.getEditFilesFromDansDeposit(dansDeposit); + var editFiles = mappingService.getEditFilesFromDansDeposit(dansDeposit, updatesDataset); yamlService.writeYaml(new EditFilesRoot(editFiles), dansDeposit.getBagDir().resolve("edit-files.yml")); - var editPermissions = mappingService.getEditPermissionsFromDansDeposit(dansDeposit); + var editPermissions = mappingService.getEditPermissionsFromDansDeposit(dansDeposit, updatesDataset); yamlService.writeYaml(new EditPermissionsRoot(editPermissions), dansDeposit.getBagDir().resolve("edit-permissions.yml")); var updateState = new UpdateState(); diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositProperties.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositProperties.java new file mode 100644 index 0000000..c183178 --- /dev/null +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositProperties.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Properties; + +/** + * Represents the properties of a DANS bag deposit. + */ +@Slf4j +public class DansDepositProperties { + private final Properties properties; + private final String depositId; + + public DansDepositProperties(Path depositPropertiesFile) throws IOException { + // Load the properties from the file + properties = new Properties(); + properties.load(depositPropertiesFile.toUri().toURL().openStream()); + depositId = depositPropertiesFile.getParent().getFileName().toString(); + } + + public String getDepositorUserId() { + return properties.getProperty("depositor.userId"); + } + + public String getSwordToken() { + return properties.getProperty("dataverse.sword-token"); + } + + public String getBagId() { + return properties.getProperty("dataverse.bag-id"); + } + + public String getDepositId() { + return depositId; + } +} diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositSupport.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositSupport.java index 3f0d378..94c1873 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositSupport.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositSupport.java @@ -23,13 +23,15 @@ import nl.knaw.dans.dvingest.core.DataverseIngestBag; import nl.knaw.dans.dvingest.core.DataverseIngestDeposit; import nl.knaw.dans.dvingest.core.Deposit; +import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDeposit; +import nl.knaw.dans.dvingest.core.dansbag.exception.InvalidDepositException; +import nl.knaw.dans.dvingest.core.dansbag.exception.RejectedDepositException; +import nl.knaw.dans.dvingest.core.service.DataverseService; import nl.knaw.dans.dvingest.core.service.YamlService; -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; -import nl.knaw.dans.ingest.core.exception.RejectedDepositException; +import nl.knaw.dans.lib.dataverse.DataverseException; import java.io.IOException; import java.nio.file.Path; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -40,16 +42,18 @@ public class DansDepositSupport implements Deposit { private final ValidateDansBagService validateDansBagService; private final DansBagMappingService dansBagMappingService; + private final DataverseService dataverseService; private final YamlService yamlService; private final DataverseIngestDeposit ingestDataverseIngestDeposit; private final boolean isDansDeposit; - private nl.knaw.dans.ingest.core.domain.Deposit dansDeposit; + private DansBagDeposit dansDeposit; - public DansDepositSupport(ValidateDansBagService validateDansBagService, DataverseIngestDeposit dataverseIngestDeposit, DansBagMappingService dansBagMappingService, YamlService yamlService) { + public DansDepositSupport(DataverseIngestDeposit dataverseIngestDeposit, ValidateDansBagService validateDansBagService, DansBagMappingService dansBagMappingService, DataverseService dataverseService, YamlService yamlService) { this.validateDansBagService = validateDansBagService; this.ingestDataverseIngestDeposit = dataverseIngestDeposit; this.dansBagMappingService = dansBagMappingService; + this.dataverseService = dataverseService; this.yamlService = yamlService; try { this.isDansDeposit = dataverseIngestDeposit.getBags().get(0).looksLikeDansBag(); @@ -64,12 +68,16 @@ public boolean convertDansDepositIfNeeded() { if (isDansDeposit && dansDeposit == null) { log.info("Converting deposit to Dataverse ingest metadata"); try { + var updatesDataset = dansBagMappingService.getUpdatesDataset(ingestDataverseIngestDeposit.getLocation()); + if (updatesDataset != null) { + ingestDataverseIngestDeposit.updateProperties(Map.of(UPDATES_DATASET_KEY, updatesDataset)); + } dansDeposit = dansBagMappingService.readDansDeposit(ingestDataverseIngestDeposit.getLocation()); - new DansDepositConverter(dansDeposit, dansBagMappingService, yamlService).run(); + new DansDepositConverter(dansDeposit, updatesDataset, dansBagMappingService, yamlService).run(); log.info("Conversion successful"); return true; } - catch (IOException | InvalidDepositException e) { + catch (IOException | InvalidDepositException | DataverseException e) { throw new RuntimeException("Error converting deposit to Dataverse ingest metadata", e); } } @@ -102,12 +110,18 @@ public void onSuccess(@NonNull String pid, String message) { var bag = ingestDataverseIngestDeposit.getBags().get(0); var action = bag.getUpdateState().getAction(); if (action.startsWith("publish")) { - ingestDataverseIngestDeposit.updateProperties(Map.of( - "state.label", "PUBLISHED", - "state.description", "The dataset is published", - "identifier.doi", pid - ) - ); + try { + var nbn = dataverseService.getDatasetUrnNbn(pid); + ingestDataverseIngestDeposit.updateProperties(Map.of( + "state.label", "PUBLISHED", + "state.description", "The dataset is published", + "identifier.doi", pid, + "identifier.urn", nbn + ) + ); + } catch (IOException | DataverseException e) { + throw new RuntimeException("Error getting URN:NBN", e); // Cancelling the "success" + } } else if (action.equals("submit-for-review")) { ingestDataverseIngestDeposit.updateProperties(Map.of( diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java new file mode 100644 index 0000000..8421ecb --- /dev/null +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java @@ -0,0 +1,161 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag; + +import lombok.AllArgsConstructor; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import nl.knaw.dans.dvingest.core.bagprocessor.DataversePath; +import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo; +import nl.knaw.dans.dvingest.core.yaml.AddEmbargo; +import nl.knaw.dans.dvingest.core.yaml.EditFiles; +import nl.knaw.dans.dvingest.core.yaml.FromTo; +import nl.knaw.dans.lib.dataverse.model.file.FileMeta; + +import java.nio.file.Path; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Composes the EditFiles object from the information found in the deposit. This object handles the case of a deposit for a new dataset. Several methods are abstract, so that they can be overridden by + * a subclass that handles the case of an update to an existing dataset. + */ +@Slf4j +@AllArgsConstructor +public class EditFilesComposer { + protected static final SimpleDateFormat yyyymmddFormat = new SimpleDateFormat("yyyy-MM-dd"); + + @NonNull + protected final Map files; + @NonNull + protected final Instant dateAvailable; + + protected final Pattern fileExclusionPattern; + @NonNull + protected final List embargoExclusions; + + public EditFiles composeEditFiles() { + var pathFileInfoMap = files; + var renamedFiles = getAutoRenameMap(pathFileInfoMap); + var ignoredFiles = getFilesToIgnore(pathFileInfoMap); + + var editFiles = new EditFiles(); + pathFileInfoMap = removeIgnoredFiles(pathFileInfoMap, ignoredFiles); + + editFiles.setAutoRenameFiles(getAutoRenamedFiles(renamedFiles)); + editFiles.setAddRestrictedFiles(getRestrictedFilesToAdd(pathFileInfoMap)); + editFiles.setAddUnrestrictedFiles(getUnrestrictedFilesToAdd(pathFileInfoMap)); + editFiles.setUpdateFileMetas(getUpdatedFileMetas(pathFileInfoMap)); + + var filePathsToEmbargo = getEmbargoedFiles(pathFileInfoMap, dateAvailable); + if (!filePathsToEmbargo.isEmpty()) { + var addEmbargo = new AddEmbargo(); + addEmbargo.setDateAvailable(yyyymmddFormat.format(Date.from(dateAvailable))); + addEmbargo.setFilePaths(filePathsToEmbargo.stream().map(Path::toString).toList()); + editFiles.setAddEmbargoes(List.of(addEmbargo)); + } + return editFiles; + } + + /** + * Get the files that should not be processed by the ingest service. + * + * @param files the file infos found inf files.xml + * @return a list of file paths that should be ignored + */ + protected List getFilesToIgnore(Map files) { + if (fileExclusionPattern == null) { + return List.of(); + } + return files.keySet().stream() + .map(Path::toString) + .filter(string -> fileExclusionPattern.matcher(string).matches()).toList(); + } + + /** + * Get the files that should be added as restricted files. + * + * @param files the file infos found in files.xml + * @return a list of file paths that should be added as restricted files + */ + private List getRestrictedFilesToAdd(Map files) { + return files.entrySet().stream() + .filter(entry -> entry.getValue().getMetadata().getRestricted()) + .map(entry -> entry.getKey().toString()) + .toList(); + } + + private List getUnrestrictedFilesToAdd(Map files) { + return files.entrySet().stream() + .filter(entry -> !entry.getValue().getMetadata().getRestricted()) + .map(entry -> entry.getKey().toString()) + .toList(); + } + + /** + * Get the files that have metadata that should be updated. + * + * @param files the file infos found in files.xml + * @return a list of FileMetas that should be updated + */ + private List getUpdatedFileMetas(Map files) { + return files.values().stream() + .map(FileInfo::getMetadata) + .filter(this::hasAttributes) + .toList(); + } + + private List getEmbargoedFiles(Map files, Instant dateAvailable) { + var now = Instant.now(); + if (dateAvailable.isAfter(now)) { + return files.keySet().stream() + .filter(f -> !embargoExclusions.contains(f.toString())).toList(); + } + else { + log.debug("Date available in the past, no embargo: {}", dateAvailable); + return List.of(); + } + } + + private boolean hasAttributes(FileMeta fileMeta) { + return (fileMeta.getCategories() != null && !fileMeta.getCategories().isEmpty()) || + (fileMeta.getDescription() != null && !fileMeta.getDescription().isBlank()); + } + + private Map removeIgnoredFiles(Map files, List ignoredFiles) { + return files.entrySet().stream() + .filter(entry -> !ignoredFiles.contains(entry.getKey().toString())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private List getAutoRenamedFiles(Map renamedFiles) { + return renamedFiles.entrySet().stream() + .map(entry -> new FromTo(entry.getKey(), entry.getValue())) + .toList(); + } + + protected Map getAutoRenameMap(Map files) { + return files.entrySet().stream() + .filter(entry -> entry.getValue().isSanitized()) + .collect(Collectors.toMap(entry -> entry.getKey().toString(), + entry -> new DataversePath(entry.getValue().getMetadata().getDirectoryLabel(), entry.getValue().getMetadata().getLabel()).toString())); + } +} diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java new file mode 100644 index 0000000..c5ce787 --- /dev/null +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java @@ -0,0 +1,207 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag; + +import lombok.extern.slf4j.Slf4j; +import nl.knaw.dans.dvingest.core.bagprocessor.FilesInDatasetCache; +import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo; +import nl.knaw.dans.dvingest.core.service.DataverseService; +import nl.knaw.dans.dvingest.core.yaml.EditFiles; +import nl.knaw.dans.dvingest.core.yaml.FromTo; +import nl.knaw.dans.lib.dataverse.DataverseException; +import nl.knaw.dans.lib.dataverse.model.file.FileMeta; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Instant; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static nl.knaw.dans.dvingest.core.dansbag.SetUtils.diff; +import static nl.knaw.dans.dvingest.core.dansbag.SetUtils.union; + +@Slf4j +public class EditFilesComposerForUpdate extends EditFilesComposer { + private final String updatesDatasetPid; + private final DataverseService dataverseService; + + public EditFilesComposerForUpdate(Map files, Instant dateAvailable, String updatesDatasetPid, Pattern fileExclusionPattern, List embargoExclusions, + DataverseService dataverseService) { + super(files, dateAvailable, fileExclusionPattern, embargoExclusions); + this.updatesDatasetPid = updatesDatasetPid; + this.dataverseService = dataverseService; + } + + @Override + public EditFiles composeEditFiles() { + var pathFileInfoMap = files; + var renamedFiles = getAutoRenameMap(pathFileInfoMap); + // TODO: this should be a read-only variant of the cache + FilesInDatasetCache filesInDatasetCache = new FilesInDatasetCache(dataverseService, renamedFiles); + try { + filesInDatasetCache.downloadFromDataset(updatesDatasetPid); + } + catch (IOException | DataverseException e) { + log.error("Could not download files from dataset with pid {}", updatesDatasetPid, e); + } + var editFiles = new EditFiles(); + + // move old paths to new paths + // Convert from Path to String + var filesInDataset = filesInDatasetCache.getFilesInDataset().entrySet().stream() + .collect(Collectors.toMap(e -> Path.of(e.getKey()), Map.Entry::getValue)); + var oldToNewPathMovedFiles = getOldToNewPathOfFilesToMove(filesInDataset, pathFileInfoMap); + var fileMovementFromTos = oldToNewPathMovedFiles.entrySet().stream() + .map(e -> new FromTo(e.getKey().toString(), e.getValue().toString())) + .collect(Collectors.toList()); + log.debug("fileMovements = {}", fileMovementFromTos); + editFiles.setMoveFiles(fileMovementFromTos); + + /* + * File replacement can only happen on files with paths that are not also involved in a rename/move action. Otherwise, we end up with: + * + * - trying to update the file metadata by a database ID that is not the "HEAD" of a file version history (which Dataverse doesn't allow anyway, it + * fails with "You cannot edit metadata on a dataFile that has been replaced"). This happens when a file A is renamed to B, but a different file A + * is also added in the same update. Schematically, (1) means unique file 1, A is a file name: + * V1 -> V2 + * + * (1)A -> (1)B (move) + * .. -> (2)A (add) + * + * - trying to add a file with a name that already exists. This happens when a file A is renamed to B, while B is also part of the latest version (V1) + * + * V1 -> V2 + * (1)A -> (1)B (move) + * (2)B -> .. (delete) + * + */ + var fileReplacementCandidates = filesInDataset.entrySet().stream() + .filter(pathToFileInfoEntry -> !oldToNewPathMovedFiles.containsKey(pathToFileInfoEntry.getKey())) + .filter(pathToFileInfoEntry -> !oldToNewPathMovedFiles.containsValue(pathToFileInfoEntry.getKey())) // TODO in the OG version, this was a Set; check what performance is like + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + var filesToReplace = getFilesToReplace(pathFileInfoMap, fileReplacementCandidates); + log.debug("filesToReplace = {}", filesToReplace); + editFiles.setReplaceFiles(filesToReplace.stream().map(Path::toString).collect(Collectors.toList())); + + /* + * To find the files to delete we start from the paths in the deposit payload. In principle, these paths are remaining, so should NOT be deleted. + * However, if a file is moved/renamed to a path that was also present in the latest version, then the old file at that path must first be deleted + * (and must therefore NOT be included in candidateRemainingFiles). Otherwise, we'll end up trying to use an existing (directoryLabel, label) pair. + */ + var newPathsOfMovedFiles = new HashSet<>(oldToNewPathMovedFiles.values()); + var candidateRemainingFiles = diff(pathFileInfoMap.keySet(), newPathsOfMovedFiles); + + /* + * The paths to delete, now, are the paths in the latest version minus the remaining files. We further subtract the old paths of the moved files. + * This may be a bit confusing, but the goal is to make sure that the underlying FILE remains present (after all, it is to be renamed/moved). The + * path itself WILL be "removed" from the latest version by the move. (It MAY be filled again by a file addition in the same update, though.) + */ + var oldPathsOfMovedFiles = new HashSet<>(oldToNewPathMovedFiles.keySet()); + var pathsToDelete = diff(diff(filesInDataset.keySet(), candidateRemainingFiles), oldPathsOfMovedFiles); + log.debug("pathsToDelete = {}", pathsToDelete); + editFiles.setDeleteFiles(pathsToDelete.stream().map(Path::toString).collect(Collectors.toList())); + + /* + * After the movements have been performed, which paths are occupied? We start from the paths of the latest version (pathToFileMetaInLatestVersion.keySet) + * + * The old paths of the moved files (oldToNewPathMovedFiles.keySet) are no longer occupied, so they must be subtracted. This is important in the case where + * a deposit renames/moves a file (leaving the checksum unchanged) but provides a new file for the vacated path. + * + * The paths of the deleted files (pathsToDelete) are no longer occupied, so must be subtracted. (It is not strictly necessary for the calculation + * of pathsToAdd, but done anyway to keep the logic consistent.) + * + * The new paths of the moved files (oldToNewPathMovedFiles.values.toSet) *are* now occupied, so the must be added. This is important to + * avoid those files from being marked as "new" files, i.e. files to be added. + * + * All paths in the deposit that are not occupied, are new files to be added. + */ + var diffed = diff(diff(filesInDataset.keySet(), newPathsOfMovedFiles), pathsToDelete); + var occupiedPaths = union(diffed, oldToNewPathMovedFiles.values()); + + log.debug("occupiedPaths = {}", occupiedPaths); + var pathsToAdd = diff(pathFileInfoMap.keySet(), occupiedPaths); + editFiles.setAddRestrictedFiles(pathsToAdd.stream() + .filter(p -> pathFileInfoMap.get(p).getMetadata().getRestricted()).toList().stream().map(Path::toString).toList()); + + editFiles.setAddUnrestrictedFiles(pathsToAdd.stream() + .filter(p -> !pathFileInfoMap.get(p).getMetadata().getRestricted()).toList().stream().map(Path::toString).toList()); + + // todo: embargoes + + return editFiles; + } + + private Set getFilesToReplace(Map pathToFileInfo, Map fileReplacementCandidates) { + + var intersection = SetUtils.intersection(pathToFileInfo.keySet(), fileReplacementCandidates.keySet()); + + log.debug("Intersection paths for replacing = {}", intersection); + + return intersection.stream() + .filter(p -> !pathToFileInfo.get(p).getChecksum().equals(fileReplacementCandidates.get(p).getDataFile().getChecksum().getValue())) + .collect(Collectors.toSet()); + + } + + /** + * Creating a mapping for moving files to a new location. To determine this, the file needs to be unique in the old and the new version, because its checksum is used to locate it. Files that occur + * multiple times in either the old or the new version cannot be moved in this way. They will appear to have been deleted in the old version and added in the new. This has the same net result, + * except that the "Changes" overview in Dataverse does not record that the file was effectively moved. + * + * @param pathToFileMetaInLatestVersion map from path to file metadata in the old version + * @param pathToFileInfo map from path to file info in the new version (i.e. the deposit). + * @return a map from old path to new path + */ + Map getOldToNewPathOfFilesToMove(Map pathToFileMetaInLatestVersion, Map pathToFileInfo) { + + var depositChecksums = pathToFileInfo.entrySet().stream() + .map(e -> Map.entry(e.getKey(), e.getValue().getChecksum())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + var latestFileChecksums = pathToFileMetaInLatestVersion.entrySet().stream() + .map(e -> Map.entry(e.getKey(), e.getValue().getDataFile().getChecksum().getValue())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + var checksumsToPathNonDuplicatedFilesInDeposit = getChecksumsToPathOfNonDuplicateFiles(depositChecksums); + var checksumsToPathNonDuplicatedFilesInLatestVersion = getChecksumsToPathOfNonDuplicateFiles(latestFileChecksums); + + var intersects = checksumsToPathNonDuplicatedFilesInDeposit.keySet().stream() + .filter(checksumsToPathNonDuplicatedFilesInLatestVersion::containsKey) + .collect(Collectors.toSet()); + + return intersects.stream() + .map(c -> Map.entry(checksumsToPathNonDuplicatedFilesInLatestVersion.get(c), checksumsToPathNonDuplicatedFilesInDeposit.get(c))) + .filter(entry -> !entry.getKey().equals(entry.getValue())) // filter out files that are not moved (this was not present in the old code) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + } + + private Map getChecksumsToPathOfNonDuplicateFiles(Map pathToChecksum) { + // inverse map first + var inverse = pathToChecksum.entrySet().stream() + .collect(Collectors.groupingBy(Map.Entry::getValue, Collectors.mapping(Map.Entry::getKey, Collectors.toList()))); + + // filter out items with 0 or more than 1 item + return inverse.entrySet().stream() + .filter(item -> item.getValue().size() == 1) + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().get(0))); + } +} diff --git a/src/main/java/nl/knaw/dans/ingest/core/io/FileServiceImpl.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/LightweightBagInfo.java similarity index 53% rename from src/main/java/nl/knaw/dans/ingest/core/io/FileServiceImpl.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/LightweightBagInfo.java index 7ab76a8..d6eb48c 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/io/FileServiceImpl.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/LightweightBagInfo.java @@ -13,26 +13,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.io; +package nl.knaw.dans.dvingest.core.dansbag; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.stream.Stream; +import java.util.Map; +import java.util.stream.Collectors; -public class FileServiceImpl implements FileService { - @Override - public boolean isDirectory(Path path) { - return Files.isDirectory(path); - } +/** + * Lightweight bag-info.txt parser, which can be used without reading the whole bag. + */ +public class LightweightBagInfo { + private final Map keyValues; - @Override - public Stream listDirectories(Path path) throws IOException { - return Files.list(path).filter(Files::isDirectory); + public LightweightBagInfo(Path file) throws IOException { + try (var linesStream = Files.lines(file)) { + keyValues = linesStream + .map(line -> line.split(":", 2)) + .collect(Collectors.toMap(parts -> parts[0].trim(), parts -> parts[1].trim())); + } } - @Override - public boolean fileExists(Path path) { - return Files.exists(path); + public String get(String key) { + return keyValues.get(key); } } diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/ManifestHelperImpl.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/ManifestUtil.java similarity index 93% rename from src/main/java/nl/knaw/dans/ingest/core/service/ManifestHelperImpl.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/ManifestUtil.java index e1f062b..7729b24 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/ManifestHelperImpl.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/ManifestUtil.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service; +package nl.knaw.dans.dvingest.core.dansbag; import gov.loc.repository.bagit.creator.CreatePayloadManifestsVistor; import gov.loc.repository.bagit.creator.CreateTagManifestsVistor; @@ -37,10 +37,9 @@ import static gov.loc.repository.bagit.hash.StandardSupportedAlgorithms.SHA1; -public class ManifestHelperImpl implements ManifestHelper { +public class ManifestUtil { - @Override - public void ensureSha1ManifestPresent(Bag bag) throws NoSuchAlgorithmException, IOException { + public static void ensureSha1ManifestPresent(Bag bag) throws NoSuchAlgorithmException, IOException { var manifests = bag.getPayLoadManifests(); var algorithms = manifests.stream().map(Manifest::getAlgorithm); @@ -70,7 +69,7 @@ public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IO /* * Fix for EASY-1306: a tag manifest must not contain an entry for itself, as this is practically * impossible to calculate. It could in theory contain entries for other tag manifests. However, - * the CreateTagManifestsVistor, once it finds an entry for a tag file in ONE of the tag manifests, + * the CreateTagManifestsVisitor, once it finds an entry for a tag file in ONE of the tag manifests, * will add an entry in ALL tag manifests. * * Therefore, we adopt the strategy NOT to calculate any checksums for the tag manifests themselves. diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/SetUtils.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/SetUtils.java new file mode 100644 index 0000000..f2b9eeb --- /dev/null +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/SetUtils.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag; + +import java.util.Collection; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +// TODO: move to dans-java-utils or find opensource alternative +public class SetUtils { + + public static Set diff(Collection a, Collection b) { + return a.stream().filter(k -> !b.contains(k)).collect(Collectors.toSet()); + } + + public static Set intersection(Collection a, Collection b) { + return a.stream().filter(b::contains).collect(Collectors.toSet()); + } + + public static Set union(Collection a, Collection b) { + return Stream.of(a.stream(), b.stream()).flatMap(i -> i).collect(Collectors.toSet()); + // a.stream().filter(b::contains).collect(Collectors.toSet()); + } +} diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/SupportedLicenses.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/SupportedLicenses.java index f64368a..9660d3c 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/SupportedLicenses.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/SupportedLicenses.java @@ -15,12 +15,11 @@ */ package nl.knaw.dans.dvingest.core.dansbag; -import lombok.AllArgsConstructor; +import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDeposit; +import nl.knaw.dans.dvingest.core.dansbag.exception.RejectedDepositException; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.LicenseElem; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; import nl.knaw.dans.dvingest.core.service.DataverseService; -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.exception.RejectedDepositException; -import nl.knaw.dans.ingest.core.service.XPathEvaluator; -import nl.knaw.dans.ingest.core.service.mapper.mapping.LicenseElem; import nl.knaw.dans.lib.dataverse.DataverseException; import nl.knaw.dans.lib.dataverse.model.dataset.License; @@ -29,7 +28,7 @@ import java.util.HashMap; import java.util.Map; -import static nl.knaw.dans.ingest.core.service.XPathConstants.DDM_DCMI_METADATA; +import static nl.knaw.dans.dvingest.core.dansbag.xml.XPathConstants.DDM_DCMI_METADATA; public class SupportedLicenses { private final Map supportedLicenses; @@ -47,7 +46,7 @@ public SupportedLicenses(DataverseService dataverseService) throws IOException, } - public License getLicenseFromDansDeposit(Deposit dansDeposit) { + public License getLicenseFromDansDeposit(DansBagDeposit dansDeposit) { var optLicenseUri = XPathEvaluator.nodes(dansDeposit.getDdm(), DDM_DCMI_METADATA + "/dcterms:license") .filter(LicenseElem::isLicenseUri) .findFirst() diff --git a/src/main/java/nl/knaw/dans/ingest/core/domain/Deposit.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DansBagDeposit.java similarity index 89% rename from src/main/java/nl/knaw/dans/ingest/core/domain/Deposit.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DansBagDeposit.java index 871b493..dde2ec7 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/domain/Deposit.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DansBagDeposit.java @@ -13,31 +13,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.domain; +package nl.knaw.dans.dvingest.core.dansbag.deposit; import gov.loc.repository.bagit.domain.Bag; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import nl.knaw.dans.ingest.core.service.XPathEvaluator; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; import org.apache.commons.lang3.StringUtils; import org.w3c.dom.Document; -import org.w3c.dom.Node; import java.nio.file.Path; import java.time.Instant; import java.time.OffsetDateTime; import java.util.List; -import java.util.stream.Collectors; -import static nl.knaw.dans.ingest.core.service.XPathConstants.DDM_PROFILE; -import static nl.knaw.dans.ingest.core.service.XPathConstants.FILES_FILE; -import static nl.knaw.dans.ingest.core.service.XmlNamespaces.NAMESPACE_XSI; +import static nl.knaw.dans.dvingest.core.dansbag.xml.XPathConstants.DDM_PROFILE; +import static nl.knaw.dans.dvingest.core.dansbag.xml.XPathConstants.FILES_FILE; +/** + * Represents a DANS bag deposit, i.e. a deposit containing a bag conforming to the DANS BagIt Profile. + */ @Data @NoArgsConstructor @AllArgsConstructor -public class Deposit { +public class DansBagDeposit { private Path dir; private Path bagDir; @@ -77,11 +77,6 @@ public VaultMetadata getVaultMetadata() { return new VaultMetadata(getDataversePid(), getDataverseBagId(), getDataverseNbn(), getDataverseOtherId(), getDataverseSwordToken()); } - private static boolean hasTypeDoi(Node n) { - Node type = n.getAttributes().getNamedItemNS(NAMESPACE_XSI, "type"); - return type != null && type.getTextContent().contains("DOI"); - } - public String getDataversePid() { return String.format("%s:%s/%s", dataverseIdProtocol, dataverseIdAuthority, dataverseId); } diff --git a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositWriter.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DansBagDepositReader.java similarity index 52% rename from src/main/java/nl/knaw/dans/ingest/core/deposit/DepositWriter.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DansBagDepositReader.java index ea86c54..cc51172 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositWriter.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DansBagDepositReader.java @@ -13,21 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.deposit; +package nl.knaw.dans.dvingest.core.dansbag.deposit; -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; +import nl.knaw.dans.dvingest.core.dansbag.exception.InvalidDepositException; -import java.io.IOException; import java.nio.file.Path; -public interface DepositWriter { - - void saveDeposit(Deposit deposit) throws InvalidDepositException; - - void moveDeposit(Deposit deposit, Path outbox) throws IOException; - - void moveDeposit(Path source, Path outbox) throws IOException; +/** + * Reads a DANS bag deposit from a deposit directory into a {@link DansBagDeposit} object. + */ +public interface DansBagDepositReader { - void saveBagInfo(Deposit deposit) throws IOException; + /** + * Reads a DANS bag deposit from a deposit directory into a {@link DansBagDeposit} object. + * + * @param depositDir the deposit directory + * @return the deposit object + * @throws InvalidDepositException if the deposit is invalid + */ + DansBagDeposit readDeposit(Path depositDir) throws InvalidDepositException; } diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DansBagDepositReaderImpl.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DansBagDepositReaderImpl.java new file mode 100644 index 0000000..168bc14 --- /dev/null +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DansBagDepositReaderImpl.java @@ -0,0 +1,207 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag.deposit; + +import gov.loc.repository.bagit.domain.Bag; +import gov.loc.repository.bagit.reader.BagReader; +import nl.knaw.dans.dvingest.core.dansbag.exception.InvalidDepositException; +import nl.knaw.dans.dvingest.core.dansbag.ManifestUtil; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; +import nl.knaw.dans.dvingest.core.dansbag.xml.XmlReader; +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.FileBasedConfiguration; +import org.apache.commons.configuration2.PropertiesConfiguration; +import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder; +import org.apache.commons.configuration2.builder.fluent.Parameters; +import org.apache.commons.configuration2.ex.ConfigurationException; +import org.apache.commons.lang3.StringUtils; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static nl.knaw.dans.dvingest.core.dansbag.xml.XPathConstants.FILES_FILE; + +public class DansBagDepositReaderImpl implements DansBagDepositReader { + private static final String DEPOSIT_PROPERTIES_FILENAME = "deposit.properties"; + + private final XmlReader xmlReader; + private final BagReader bagReader; + + public DansBagDepositReaderImpl(XmlReader xmlReader, BagReader bagReader) { + this.xmlReader = xmlReader; + this.bagReader = bagReader; + } + + @Override + public DansBagDeposit readDeposit(Path depositDir) throws InvalidDepositException { + try { + var bagDir = getBagDir(depositDir); + + var depositProperties = readDepositProperties(depositDir); + var bag = bagReader.read(bagDir); + + var deposit = mapToDeposit(bag, depositProperties); + + deposit.setBag(bag); + deposit.setDdm(readRequiredXmlFile(deposit.getDdmPath())); + deposit.setFilesXml(readRequiredXmlFile(deposit.getFilesXmlPath())); + deposit.setAmd(readOptionalXmlFile(deposit.getAmdPath())); + + deposit.setFiles(getDepositFiles(deposit)); + + return deposit; + } + catch (Exception cex) { + throw new InvalidDepositException(cex.getMessage(), cex); + } + } + + private Path getBagDir(Path depositDir) throws InvalidDepositException, IOException { + if (!Files.isDirectory(depositDir)) { + throw new InvalidDepositException(String.format("%s is not a directory", depositDir)); + } + + try (var substream = Files.list(depositDir).filter(Files::isDirectory)) { + var directories = substream.toList(); + + // only 1 directory allowed, not 0 or more than 1 + if (directories.size() != 1) { + throw new InvalidDepositException(String.format( + "%s has more or fewer than one subdirectory", depositDir + )); + } + + // check for the presence of deposit.properties and bagit.txt + if (!Files.exists(depositDir.resolve("deposit.properties"))) { + throw new InvalidDepositException(String.format( + "%s does not contain a deposit.properties file", depositDir + )); + } + + var bagDir = directories.get(0); + + if (!Files.exists(bagDir.resolve("bagit.txt"))) { + throw new InvalidDepositException(String.format( + "%s does not contain a bag", depositDir + )); + } + + return bagDir; + } + } + + + private Configuration readDepositProperties(Path depositDir) throws ConfigurationException { + var propertiesFile = depositDir.resolve(DEPOSIT_PROPERTIES_FILENAME); + var params = new Parameters(); + var paramConfig = params.properties() + .setFileName(propertiesFile.toString()); + + var builder = new FileBasedConfigurationBuilder + (PropertiesConfiguration.class, null, true) + .configure(paramConfig); + + return builder.getConfiguration(); + } + + private List getDepositFiles(DansBagDeposit dansBagDeposit) throws IOException { + var bag = dansBagDeposit.getBag(); + var filePathToSha1 = ManifestUtil.getFilePathToSha1(bag); + + return XPathEvaluator.nodes(dansBagDeposit.getFilesXml(), FILES_FILE) + .map(node -> { + var filePath = Optional.ofNullable(node.getAttributes().getNamedItem("filepath")) + .map(Node::getTextContent) + .map(Path::of) + .orElseThrow(() -> new IllegalArgumentException("File element without filepath attribute")); + + var sha1 = filePathToSha1.get(filePath); + + return new DepositFile(filePath, sha1, node); + }) + .collect(Collectors.toList()); + } + + private Document readRequiredXmlFile(Path path) throws ParserConfigurationException, IOException, SAXException { + if (!Files.exists(path)) { + throw new IllegalArgumentException("Required file not found: " + path); + } + return xmlReader.readXmlFile(path); + } + + private Document readOptionalXmlFile(Path path) throws ParserConfigurationException, IOException, SAXException { + if (Files.exists(path)) { + return xmlReader.readXmlFile(path); + } + + return null; + } + + private DansBagDeposit mapToDeposit(Bag bag, Configuration depositProperties) { + var deposit = new DansBagDeposit(); + deposit.setBagDir(bag.getRootDir()); + deposit.setDir(bag.getRootDir().getParent()); + deposit.setDoi(depositProperties.getString("identifier.doi", "")); + deposit.setUrn(depositProperties.getString("identifier.urn")); + deposit.setCreated(Optional.ofNullable(depositProperties.getString("creation.timestamp")).map(OffsetDateTime::parse).orElse(null)); + deposit.setDepositorUserId(depositProperties.getString("depositor.userId")); + + deposit.setDataverseIdProtocol(depositProperties.getString("dataverse.id-protocol", "")); + deposit.setDataverseIdAuthority(depositProperties.getString("dataverse.id-authority", "")); + deposit.setDataverseId(depositProperties.getString("dataverse.id-identifier", "")); + deposit.setDataverseBagId(depositProperties.getString("dataverse.bag-id", "")); + deposit.setDataverseNbn(depositProperties.getString("dataverse.nbn", "")); + deposit.setDataverseOtherId(depositProperties.getString("dataverse.other-id", "")); + deposit.setDataverseOtherIdVersion(depositProperties.getString("dataverse.other-id-version", "")); + deposit.setDataverseSwordToken(depositProperties.getString("dataverse.sword-token", "")); + deposit.setHasOrganizationalIdentifier(getFirstValue(bag.getMetadata().get("Has-Organizational-Identifier"))); + + var isVersionOf = bag.getMetadata().get("Is-Version-Of"); + + if (isVersionOf != null) { + isVersionOf.stream() + .filter(StringUtils::isNotBlank) + .findFirst() + .ifPresent(item -> { + deposit.setUpdate(true); + deposit.setIsVersionOf(item); + }); + } + + return deposit; + } + + private String getFirstValue(List value) { + if (value == null) { + return null; + } + + return value.stream() + .filter(StringUtils::isNotBlank) + .findFirst() + .orElse(null); + } + +} diff --git a/src/main/java/nl/knaw/dans/ingest/core/io/FileService.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DepositFile.java similarity index 71% rename from src/main/java/nl/knaw/dans/ingest/core/io/FileService.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DepositFile.java index 23116a4..ea740d4 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/io/FileService.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DepositFile.java @@ -13,17 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.io; +package nl.knaw.dans.dvingest.core.dansbag.deposit; -import java.io.IOException; -import java.nio.file.Path; -import java.util.stream.Stream; - -public interface FileService { +import lombok.AllArgsConstructor; +import lombok.Value; +import org.w3c.dom.Node; - boolean isDirectory(Path path); - - Stream listDirectories(Path path) throws IOException; +import java.nio.file.Path; - boolean fileExists(Path path); +@Value +@AllArgsConstructor +public class DepositFile { + Path path; + String checksum; + Node xmlNode; } diff --git a/src/main/java/nl/knaw/dans/ingest/core/domain/DepositState.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DepositState.java similarity index 93% rename from src/main/java/nl/knaw/dans/ingest/core/domain/DepositState.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DepositState.java index 0eeb3f1..154abc2 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/domain/DepositState.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/DepositState.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.domain; +package nl.knaw.dans.dvingest.core.dansbag.deposit; public enum DepositState { ARCHIVED, DRAFT, FAILED, FINALIZING, INVALID, REJECTED, SUBMITTED, UPLOADED, PUBLISHED, ACCEPTED diff --git a/src/main/java/nl/knaw/dans/ingest/core/domain/FileInfo.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/FileInfo.java similarity index 79% rename from src/main/java/nl/knaw/dans/ingest/core/domain/FileInfo.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/FileInfo.java index 1163d99..452eb7f 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/domain/FileInfo.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/FileInfo.java @@ -13,23 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.domain; +package nl.knaw.dans.dvingest.core.dansbag.deposit; import lombok.AllArgsConstructor; -import lombok.Data; import lombok.ToString; +import lombok.Value; import nl.knaw.dans.lib.dataverse.model.file.FileMeta; import java.nio.file.Path; -@Data +@Value @ToString @AllArgsConstructor public class FileInfo { - private Path path; - private Path physicalPath; - private String checksum; - private boolean sanitized; - private FileMeta metadata; - + Path path; + String checksum; + boolean sanitized; + FileMeta metadata; } diff --git a/src/main/java/nl/knaw/dans/ingest/core/domain/VaultMetadata.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/VaultMetadata.java similarity index 75% rename from src/main/java/nl/knaw/dans/ingest/core/domain/VaultMetadata.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/VaultMetadata.java index 5187d4f..f6868bb 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/domain/VaultMetadata.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/VaultMetadata.java @@ -13,21 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.domain; +package nl.knaw.dans.dvingest.core.dansbag.deposit; import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import lombok.ToString; +import lombok.Value; -@Data -@NoArgsConstructor +@Value @AllArgsConstructor @ToString public class VaultMetadata { - private String pid; - private String bagId; - private String nbn; - private String otherId; - private String swordToken; + String pid; + String bagId; + String nbn; + String otherId; + String swordToken; } diff --git a/src/main/java/nl/knaw/dans/ingest/core/exception/TargetBlockedException.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/package-info.java similarity index 78% rename from src/main/java/nl/knaw/dans/ingest/core/exception/TargetBlockedException.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/package-info.java index 6d906b8..d265175 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/exception/TargetBlockedException.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/deposit/package-info.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.exception; +/** + * Classes for reading and writing a DansBagDeposit. + * + */ +package nl.knaw.dans.dvingest.core.dansbag.deposit; -public class TargetBlockedException extends Exception { - public TargetBlockedException(String msg) { - super(msg); - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/exception/InvalidDepositException.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/exception/InvalidDepositException.java similarity index 94% rename from src/main/java/nl/knaw/dans/ingest/core/exception/InvalidDepositException.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/exception/InvalidDepositException.java index 0816276..6a68511 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/exception/InvalidDepositException.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/exception/InvalidDepositException.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.exception; +package nl.knaw.dans.dvingest.core.dansbag.exception; public class InvalidDepositException extends Exception { public InvalidDepositException(String msg) { diff --git a/src/main/java/nl/knaw/dans/ingest/core/exception/MissingRequiredFieldException.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/exception/MissingRequiredFieldException.java similarity index 94% rename from src/main/java/nl/knaw/dans/ingest/core/exception/MissingRequiredFieldException.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/exception/MissingRequiredFieldException.java index 4ea6b63..e612e66 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/exception/MissingRequiredFieldException.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/exception/MissingRequiredFieldException.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.exception; +package nl.knaw.dans.dvingest.core.dansbag.exception; public class MissingRequiredFieldException extends RuntimeException { private final String title; diff --git a/src/main/java/nl/knaw/dans/ingest/core/exception/RejectedDepositException.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/exception/RejectedDepositException.java similarity index 75% rename from src/main/java/nl/knaw/dans/ingest/core/exception/RejectedDepositException.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/exception/RejectedDepositException.java index edeb26f..537e7b5 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/exception/RejectedDepositException.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/exception/RejectedDepositException.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.exception; +package nl.knaw.dans.dvingest.core.dansbag.exception; -import nl.knaw.dans.ingest.core.domain.Deposit; +import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDeposit; public class RejectedDepositException extends RuntimeException { - public RejectedDepositException(Deposit deposit, String message) { - super(String.format("Rejected %s: %s", deposit.getDir(), message)); + public RejectedDepositException(DansBagDeposit dansBagDeposit, String message) { + super(String.format("Rejected %s: %s", dansBagDeposit.getDir(), message)); } public RejectedDepositException(nl.knaw.dans.dvingest.core.Deposit deposit, String message) { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/DepositDatasetFieldNames.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/DepositDatasetFieldNames.java similarity index 99% rename from src/main/java/nl/knaw/dans/ingest/core/service/DepositDatasetFieldNames.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/DepositDatasetFieldNames.java index bdf82e8..486e532 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/DepositDatasetFieldNames.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/DepositDatasetFieldNames.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service; +package nl.knaw.dans.dvingest.core.dansbag.mapper; public interface DepositDatasetFieldNames { String TITLE = "title"; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/DepositToDvDatasetMetadataMapper.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/DepositToDvDatasetMetadataMapper.java similarity index 88% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/DepositToDvDatasetMetadataMapper.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/DepositToDvDatasetMetadataMapper.java index a877c3d..7a29730 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/DepositToDvDatasetMetadataMapper.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/DepositToDvDatasetMetadataMapper.java @@ -13,48 +13,49 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper; +package nl.knaw.dans.dvingest.core.dansbag.mapper; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.domain.VaultMetadata; -import nl.knaw.dans.ingest.core.exception.MissingRequiredFieldException; -import nl.knaw.dans.ingest.core.service.XPathEvaluator; -import nl.knaw.dans.ingest.core.service.mapper.builder.ArchaeologyFieldBuilder; -import nl.knaw.dans.ingest.core.service.mapper.builder.CitationFieldBuilder; -import nl.knaw.dans.ingest.core.service.mapper.builder.DataVaultFieldBuilder; -import nl.knaw.dans.ingest.core.service.mapper.builder.FieldBuilder; -import nl.knaw.dans.ingest.core.service.mapper.builder.RelationFieldBuilder; -import nl.knaw.dans.ingest.core.service.mapper.builder.RightsFieldBuilder; -import nl.knaw.dans.ingest.core.service.mapper.builder.TemporalSpatialFieldBuilder; -import nl.knaw.dans.ingest.core.service.mapper.mapping.AbrAcquisitionMethod; -import nl.knaw.dans.ingest.core.service.mapper.mapping.AbrReport; -import nl.knaw.dans.ingest.core.service.mapper.mapping.Audience; -import nl.knaw.dans.ingest.core.service.mapper.mapping.Author; -import nl.knaw.dans.ingest.core.service.mapper.mapping.Base; -import nl.knaw.dans.ingest.core.service.mapper.mapping.Contact; -import nl.knaw.dans.ingest.core.service.mapper.mapping.Contributor; -import nl.knaw.dans.ingest.core.service.mapper.mapping.DatesOfCollection; -import nl.knaw.dans.ingest.core.service.mapper.mapping.DcxDaiAuthor; -import nl.knaw.dans.ingest.core.service.mapper.mapping.DcxDaiOrganization; -import nl.knaw.dans.ingest.core.service.mapper.mapping.DepositPropertiesOtherDoi; -import nl.knaw.dans.ingest.core.service.mapper.mapping.Description; -import nl.knaw.dans.ingest.core.service.mapper.mapping.Funder; -import nl.knaw.dans.ingest.core.service.mapper.mapping.HasOrganizationalIdentifier; -import nl.knaw.dans.ingest.core.service.mapper.mapping.Identifier; -import nl.knaw.dans.ingest.core.service.mapper.mapping.InCollection; -import nl.knaw.dans.ingest.core.service.mapper.mapping.Language; -import nl.knaw.dans.ingest.core.service.mapper.mapping.PersonalData; -import nl.knaw.dans.ingest.core.service.mapper.mapping.Publisher; -import nl.knaw.dans.ingest.core.service.mapper.mapping.Relation; -import nl.knaw.dans.ingest.core.service.mapper.mapping.SpatialBox; -import nl.knaw.dans.ingest.core.service.mapper.mapping.SpatialCoverage; -import nl.knaw.dans.ingest.core.service.mapper.mapping.SpatialPoint; -import nl.knaw.dans.ingest.core.service.mapper.mapping.Subject; -import nl.knaw.dans.ingest.core.service.mapper.mapping.SubjectAbr; -import nl.knaw.dans.ingest.core.service.mapper.mapping.TemporalAbr; +import nl.knaw.dans.dvingest.core.dansbag.deposit.VaultMetadata; +import nl.knaw.dans.dvingest.core.dansbag.exception.MissingRequiredFieldException; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.ArchaeologyFieldBuilder; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CitationFieldBuilder; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.DataVaultFieldBuilder; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.FieldBuilder; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.RelationFieldBuilder; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.RightsFieldBuilder; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.TemporalSpatialFieldBuilder; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.AbrAcquisitionMethod; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.AbrReport; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Audience; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Author; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Base; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Contact; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Contributor; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.DatesOfCollection; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.DcxDaiAuthor; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.DcxDaiOrganization; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.DepositPropertiesOtherDoi; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Description; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Funder; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.HasOrganizationalIdentifier; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Identifier; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.InCollection; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Language; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.PersonalData; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Publisher; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Relation; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.SpatialBox; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.SpatialCoverage; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.SpatialPoint; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Subject; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.SubjectAbr; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.TemporalAbr; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; import nl.knaw.dans.lib.dataverse.CompoundFieldBuilder; import nl.knaw.dans.lib.dataverse.model.dataset.Dataset; import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion; @@ -76,15 +77,16 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.RIGHTS_HOLDER; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SUBJECT; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.TITLE; -import static nl.knaw.dans.ingest.core.service.XPathConstants.DDM_DCMI_METADATA; -import static nl.knaw.dans.ingest.core.service.XPathConstants.DDM_PROFILE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.RIGHTS_HOLDER; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SUBJECT; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.TITLE; +import static nl.knaw.dans.dvingest.core.dansbag.xml.XPathConstants.DDM_DCMI_METADATA; +import static nl.knaw.dans.dvingest.core.dansbag.xml.XPathConstants.DDM_PROFILE; @Slf4j @RequiredArgsConstructor public class DepositToDvDatasetMetadataMapper { + @Getter private final boolean isMigration; private final boolean deduplicate; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/ArchaeologyFieldBuilder.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/ArchaeologyFieldBuilder.java similarity index 67% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/ArchaeologyFieldBuilder.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/ArchaeologyFieldBuilder.java index f154887..83521f5 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/ArchaeologyFieldBuilder.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/ArchaeologyFieldBuilder.java @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.builder; +package nl.knaw.dans.dvingest.core.dansbag.mapper.builder; import org.w3c.dom.Node; import java.util.stream.Stream; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.ABR_ARTIFACT; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.ABR_COMPLEX; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.ABR_PERIOD; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.ABR_RAPPORT_NUMMER; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.ABR_RAPPORT_TYPE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.ABR_VERWERVINGSWIJZE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.ARCHIS_NUMBER; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.ARCHIS_ZAAK_ID; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.ABR_ARTIFACT; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.ABR_COMPLEX; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.ABR_PERIOD; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.ABR_RAPPORT_NUMMER; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.ABR_RAPPORT_TYPE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.ABR_VERWERVINGSWIJZE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.ARCHIS_NUMBER; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.ARCHIS_ZAAK_ID; public class ArchaeologyFieldBuilder extends FieldBuilder { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/CitationFieldBuilder.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/CitationFieldBuilder.java similarity index 69% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/CitationFieldBuilder.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/CitationFieldBuilder.java index d80587d..cf23778 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/CitationFieldBuilder.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/CitationFieldBuilder.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.builder; +package nl.knaw.dans.dvingest.core.dansbag.mapper.builder; import nl.knaw.dans.lib.dataverse.model.user.AuthenticatedUser; import org.w3c.dom.Node; @@ -22,26 +22,26 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.ALTERNATIVE_TITLE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.AUTHOR; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.CONTRIBUTOR; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DATASET_CONTACT; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DATA_SOURCES; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DATE_OF_COLLECTION; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DATE_OF_DEPOSIT; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DESCRIPTION; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DISTRIBUTION_DATE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DISTRIBUTOR; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.GRANT_NUMBER; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.KEYWORD; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.LANGUAGE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.NOTES_TEXT; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.OTHER_ID; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.PRODUCTION_DATE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.PUBLICATION; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SERIES; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SUBJECT; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.TITLE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.ALTERNATIVE_TITLE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.AUTHOR; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.CONTRIBUTOR; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DATASET_CONTACT; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DATA_SOURCES; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DATE_OF_COLLECTION; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DATE_OF_DEPOSIT; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DESCRIPTION; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DISTRIBUTION_DATE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DISTRIBUTOR; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.GRANT_NUMBER; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.KEYWORD; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.LANGUAGE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.NOTES_TEXT; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.OTHER_ID; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.PRODUCTION_DATE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.PUBLICATION; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SERIES; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SUBJECT; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.TITLE; public class CitationFieldBuilder extends FieldBuilder { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/CompoundFieldGenerator.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/CompoundFieldGenerator.java similarity index 93% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/CompoundFieldGenerator.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/CompoundFieldGenerator.java index 5f04e42..67b525d 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/CompoundFieldGenerator.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/CompoundFieldGenerator.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.builder; +package nl.knaw.dans.dvingest.core.dansbag.mapper.builder; import nl.knaw.dans.lib.dataverse.CompoundFieldBuilder; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/DataVaultFieldBuilder.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/DataVaultFieldBuilder.java similarity index 69% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/DataVaultFieldBuilder.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/DataVaultFieldBuilder.java index 8b64b1f..a823d3a 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/DataVaultFieldBuilder.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/DataVaultFieldBuilder.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.builder; +package nl.knaw.dans.dvingest.core.dansbag.mapper.builder; import java.util.stream.Stream; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.BAG_ID; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DANS_OTHER_ID; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DANS_OTHER_ID_VERSION; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DATA_SUPPLIER; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.NBN; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SWORD_TOKEN; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.BAG_ID; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DANS_OTHER_ID; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DANS_OTHER_ID_VERSION; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DATA_SUPPLIER; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.NBN; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SWORD_TOKEN; public class DataVaultFieldBuilder extends FieldBuilder { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/FieldBuilder.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/FieldBuilder.java similarity index 98% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/FieldBuilder.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/FieldBuilder.java index 50bb7f0..af79c68 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/FieldBuilder.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/FieldBuilder.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.builder; +package nl.knaw.dans.dvingest.core.dansbag.mapper.builder; import nl.knaw.dans.lib.dataverse.CompoundFieldBuilder; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/PrimitiveFieldBuilder.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/PrimitiveFieldBuilder.java similarity index 97% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/PrimitiveFieldBuilder.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/PrimitiveFieldBuilder.java index 8a378c6..929638a 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/PrimitiveFieldBuilder.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/PrimitiveFieldBuilder.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.builder; +package nl.knaw.dans.dvingest.core.dansbag.mapper.builder; import nl.knaw.dans.lib.dataverse.model.dataset.ControlledMultiValueField; import nl.knaw.dans.lib.dataverse.model.dataset.ControlledSingleValueField; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/RelationFieldBuilder.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/RelationFieldBuilder.java similarity index 77% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/RelationFieldBuilder.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/RelationFieldBuilder.java index 430730d..e6a4d96 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/RelationFieldBuilder.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/RelationFieldBuilder.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.builder; +package nl.knaw.dans.dvingest.core.dansbag.mapper.builder; import org.w3c.dom.Node; import java.util.stream.Stream; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.AUDIENCE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.COLLECTION; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.RELATION; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.AUDIENCE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.COLLECTION; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.RELATION; public class RelationFieldBuilder extends FieldBuilder { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/RightsFieldBuilder.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/RightsFieldBuilder.java similarity index 77% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/RightsFieldBuilder.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/RightsFieldBuilder.java index 9e71a3b..7c3035b 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/RightsFieldBuilder.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/RightsFieldBuilder.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.builder; +package nl.knaw.dans.dvingest.core.dansbag.mapper.builder; import java.util.Objects; import java.util.stream.Stream; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.LANGUAGE_OF_METADATA; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.PERSONAL_DATA_PRESENT; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.RIGHTS_HOLDER; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.LANGUAGE_OF_METADATA; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.PERSONAL_DATA_PRESENT; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.RIGHTS_HOLDER; public class RightsFieldBuilder extends FieldBuilder { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/TemporalSpatialFieldBuilder.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/TemporalSpatialFieldBuilder.java similarity index 72% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/TemporalSpatialFieldBuilder.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/TemporalSpatialFieldBuilder.java index c26bc2d..38bc05b 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/builder/TemporalSpatialFieldBuilder.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/builder/TemporalSpatialFieldBuilder.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.builder; +package nl.knaw.dans.dvingest.core.dansbag.mapper.builder; import org.w3c.dom.Node; import java.util.stream.Stream; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SPATIAL_BOX; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SPATIAL_COVERAGE_CONTROLLED; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SPATIAL_COVERAGE_UNCONTROLLED; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SPATIAL_POINT; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.TEMPORAL_COVERAGE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SPATIAL_BOX; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SPATIAL_COVERAGE_CONTROLLED; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SPATIAL_COVERAGE_UNCONTROLLED; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SPATIAL_POINT; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.TEMPORAL_COVERAGE; public class TemporalSpatialFieldBuilder extends FieldBuilder { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/AbrAcquisitionMethod.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/AbrAcquisitionMethod.java similarity index 80% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/AbrAcquisitionMethod.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/AbrAcquisitionMethod.java index 5b4a012..9cc93d4 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/AbrAcquisitionMethod.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/AbrAcquisitionMethod.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; import org.w3c.dom.Node; import java.util.Map; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_ABR_VERWERVINGSWIJZE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_URI_ABR_VERWERVINGSWIJZE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_ABR_VERWERVINGSWIJZE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_URI_ABR_VERWERVINGSWIJZE; @Slf4j public class AbrAcquisitionMethod extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/AbrReport.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/AbrReport.java similarity index 82% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/AbrReport.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/AbrReport.java index d68f6e9..9ba6475 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/AbrReport.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/AbrReport.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; import org.w3c.dom.Node; import java.util.Map; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_ABR_RAPPORT_TYPE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_URI_ABR_RAPPORT_TYPE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_ABR_RAPPORT_TYPE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_URI_ABR_RAPPORT_TYPE; @Slf4j public class AbrReport extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/AccessRights.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/AccessRights.java similarity index 93% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/AccessRights.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/AccessRights.java index db48b78..7946665 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/AccessRights.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/AccessRights.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.service.XPathEvaluator; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; import org.w3c.dom.Node; import java.util.Map; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Amd.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Amd.java similarity index 87% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Amd.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Amd.java index 4b1ed45..e8d929a 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Amd.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Amd.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.service.XPathEvaluator; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; import org.apache.commons.lang3.StringUtils; import org.w3c.dom.Node; import java.util.Optional; -import static nl.knaw.dans.ingest.core.domain.DepositState.PUBLISHED; -import static nl.knaw.dans.ingest.core.domain.DepositState.SUBMITTED; +import static nl.knaw.dans.dvingest.core.dansbag.deposit.DepositState.PUBLISHED; +import static nl.knaw.dans.dvingest.core.dansbag.deposit.DepositState.SUBMITTED; @Slf4j public class Amd extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Audience.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Audience.java similarity index 97% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Audience.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Audience.java index 4e52b93..f8eb67d 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Audience.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Audience.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import com.google.common.collect.Comparators; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Author.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Author.java similarity index 91% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Author.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Author.java index 6451c2a..389857c 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Author.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Author.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.w3c.dom.Node; public class Author extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Base.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Base.java similarity index 95% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Base.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Base.java index f3358ac..ad5d3ad 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Base.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Base.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.service.XPathEvaluator; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; import org.apache.commons.lang3.StringUtils; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; @@ -29,7 +29,7 @@ import java.util.Optional; import java.util.stream.Stream; -import static nl.knaw.dans.ingest.core.service.XmlReader.NAMESPACE_XSI; +import static nl.knaw.dans.dvingest.core.dansbag.xml.XmlReader.NAMESPACE_XSI; @Slf4j public class Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Contact.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Contact.java similarity index 72% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Contact.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Contact.java index 5db0eec..2360865 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Contact.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Contact.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import nl.knaw.dans.lib.dataverse.model.user.AuthenticatedUser; import org.apache.commons.lang3.StringUtils; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DATASET_CONTACT_AFFILIATION; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DATASET_CONTACT_EMAIL; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DATASET_CONTACT_NAME; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DATASET_CONTACT_AFFILIATION; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DATASET_CONTACT_EMAIL; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DATASET_CONTACT_NAME; public class Contact { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Contributor.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Contributor.java similarity index 95% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Contributor.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Contributor.java index e2726ca..3d96017 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Contributor.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Contributor.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.w3c.dom.Node; import java.util.HashMap; import java.util.Map; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.CONTRIBUTOR_NAME; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.CONTRIBUTOR_NAME; @Slf4j public final class Contributor extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Creator.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Creator.java similarity index 79% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Creator.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Creator.java index a18a6a4..7fbb5d1 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Creator.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Creator.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.w3c.dom.Node; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.AUTHOR_NAME; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.AUTHOR_NAME; @Slf4j public final class Creator extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DatesOfCollection.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DatesOfCollection.java similarity index 81% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DatesOfCollection.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DatesOfCollection.java index e1c1db4..f2d524c 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DatesOfCollection.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DatesOfCollection.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.w3c.dom.Node; import java.util.regex.Pattern; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DATE_OF_COLLECTION_END; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DATE_OF_COLLECTION_START; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DATE_OF_COLLECTION_END; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DATE_OF_COLLECTION_START; @Slf4j public class DatesOfCollection extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DcxDaiAuthor.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DcxDaiAuthor.java similarity index 78% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DcxDaiAuthor.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DcxDaiAuthor.java index 97af05f..5bf5678 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DcxDaiAuthor.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DcxDaiAuthor.java @@ -13,30 +13,45 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; +import lombok.Builder; +import lombok.Data; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.domain.DatasetAuthor; -import nl.knaw.dans.ingest.core.service.XPathEvaluator; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; import org.apache.commons.lang3.StringUtils; import org.w3c.dom.Node; import java.util.List; import java.util.Optional; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.AUTHOR_AFFILIATION; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.AUTHOR_IDENTIFIER; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.AUTHOR_IDENTIFIER_SCHEME; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.AUTHOR_NAME; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.CONTRIBUTOR_NAME; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.CONTRIBUTOR_TYPE; -import static nl.knaw.dans.ingest.core.service.mapper.mapping.Contributor.contributorRoleToContributorType; -import static nl.knaw.dans.ingest.core.service.mapper.mapping.IdUriHelper.reduceUriToId; -import static nl.knaw.dans.ingest.core.service.mapper.mapping.IdUriHelper.reduceUriToOrcidId; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.Contributor.contributorRoleToContributorType; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.IdUriHelper.reduceUriToId; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.IdUriHelper.reduceUriToOrcidId; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.AUTHOR_AFFILIATION; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.AUTHOR_IDENTIFIER; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.AUTHOR_IDENTIFIER_SCHEME; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.AUTHOR_NAME; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.CONTRIBUTOR_NAME; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.CONTRIBUTOR_TYPE; @Slf4j public final class DcxDaiAuthor extends Base { + @Data + @Builder + public static class DatasetAuthor { + private String titles; + private String initials; + private String insertions; + private String surname; + private String dai; + private String isni; + private String orcid; + private String role; + private String organization; + } + public static CompoundFieldGenerator toAuthorValueObject = (builder, node) -> { var author = parseAuthor(node); var name = formatName(author); diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DcxDaiOrganization.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DcxDaiOrganization.java similarity index 75% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DcxDaiOrganization.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DcxDaiOrganization.java index 1314d24..a3d2d51 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DcxDaiOrganization.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DcxDaiOrganization.java @@ -13,28 +13,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; +import lombok.Builder; +import lombok.Data; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.domain.DatasetOrganization; -import nl.knaw.dans.ingest.core.service.XPathEvaluator; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; import org.apache.commons.lang3.StringUtils; import org.w3c.dom.Node; import java.util.Set; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.AUTHOR_IDENTIFIER; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.AUTHOR_IDENTIFIER_SCHEME; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.AUTHOR_NAME; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.CONTRIBUTOR_NAME; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.CONTRIBUTOR_TYPE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.GRANT_NUMBER_AGENCY; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.GRANT_NUMBER_VALUE; -import static nl.knaw.dans.ingest.core.service.mapper.mapping.IdUriHelper.reduceUriToId; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.IdUriHelper.reduceUriToId; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.AUTHOR_IDENTIFIER; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.AUTHOR_IDENTIFIER_SCHEME; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.AUTHOR_NAME; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.CONTRIBUTOR_NAME; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.CONTRIBUTOR_TYPE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.GRANT_NUMBER_AGENCY; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.GRANT_NUMBER_VALUE; @Slf4j public final class DcxDaiOrganization { + @Data + @Builder + @ToString + public static class DatasetOrganization { + private String name; + private String role; + private String isni; + private String viaf; + } public static CompoundFieldGenerator toAuthorValueObject = (builder, node) -> { var organization = parseOrganization(node); diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DepositPropertiesOtherDoi.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DepositPropertiesOtherDoi.java similarity index 74% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DepositPropertiesOtherDoi.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DepositPropertiesOtherDoi.java index d1cf434..20ad5a9 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DepositPropertiesOtherDoi.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DepositPropertiesOtherDoi.java @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.OTHER_ID_AGENCY; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.OTHER_ID_VALUE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.OTHER_ID_AGENCY; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.OTHER_ID_VALUE; public class DepositPropertiesOtherDoi extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DepositPropertiesVaultMetadata.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DepositPropertiesVaultMetadata.java similarity index 82% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DepositPropertiesVaultMetadata.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DepositPropertiesVaultMetadata.java index 1e8908b..95e5c06 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/DepositPropertiesVaultMetadata.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/DepositPropertiesVaultMetadata.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.apache.commons.lang3.StringUtils; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.OTHER_ID_AGENCY; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.OTHER_ID_VALUE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.OTHER_ID_AGENCY; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.OTHER_ID_VALUE; public class DepositPropertiesVaultMetadata extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Description.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Description.java similarity index 90% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Description.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Description.java index 1a5a811..63aeb72 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Description.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Description.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.apache.commons.lang3.StringUtils; import org.w3c.dom.Node; @@ -24,8 +24,8 @@ import java.util.Map; import java.util.stream.Collectors; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DESCRIPTION_VALUE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SERIES_INFORMATION; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DESCRIPTION_VALUE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SERIES_INFORMATION; public class Description extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/FileElement.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/FileElement.java similarity index 92% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/FileElement.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/FileElement.java index df97a70..8134b70 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/FileElement.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/FileElement.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.Value; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.domain.FileInfo; -import nl.knaw.dans.ingest.core.service.XPathEvaluator; +import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDeposit; +import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; import nl.knaw.dans.lib.dataverse.model.file.FileMeta; import org.apache.commons.lang3.StringUtils; import org.w3c.dom.Node; @@ -45,7 +45,7 @@ public class FileElement extends Base { ); @Value - private static class FileMetaResult { + public static class FileMetaResult { boolean sanitized; FileMeta fileMeta; } @@ -212,21 +212,20 @@ private static String replaceForbiddenCharactersInFilename(String filename) { return filenameForbidden.matcher(filename).replaceAll("_"); } - public static Map pathToFileInfo(Deposit deposit, boolean isMigration) { + public static Map pathToFileInfo(DansBagDeposit dansBagDeposit, boolean isMigration) { // FIL006 - var defaultRestrict = XPathEvaluator.nodes(deposit.getDdm(), "/ddm:DDM/ddm:profile/ddm:accessRights") + var defaultRestrict = XPathEvaluator.nodes(dansBagDeposit.getDdm(), "/ddm:DDM/ddm:profile/ddm:accessRights") .map(AccessRights::toDefaultRestrict) .findFirst() .orElse(true); var result = new HashMap(); - var bagDir = deposit.getBagDir(); + var bagDir = dansBagDeposit.getBagDir(); - deposit.getFiles().forEach(depositFile -> { + dansBagDeposit.getFiles().forEach(depositFile -> { var fileMetaResult = toFileMeta(depositFile.getXmlNode(), defaultRestrict, isMigration); result.put(depositFile.getPath(), new FileInfo( bagDir.resolve(depositFile.getPath()), - bagDir.resolve(depositFile.getPhysicalPath()), depositFile.getChecksum(), fileMetaResult.isSanitized(), fileMetaResult.getFileMeta() diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Funder.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Funder.java similarity index 84% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Funder.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Funder.java index 0c394b9..d32d8f9 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Funder.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Funder.java @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.Builder; import lombok.Data; -import nl.knaw.dans.ingest.core.service.XPathEvaluator; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; import org.apache.commons.lang3.StringUtils; import org.w3c.dom.Node; import java.util.stream.Collectors; import java.util.stream.Stream; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.GRANT_NUMBER_AGENCY; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.GRANT_NUMBER_VALUE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.GRANT_NUMBER_AGENCY; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.GRANT_NUMBER_VALUE; public class Funder { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/HasOrganizationalIdentifier.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/HasOrganizationalIdentifier.java similarity index 81% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/HasOrganizationalIdentifier.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/HasOrganizationalIdentifier.java index 1f1215f..242d9ea 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/HasOrganizationalIdentifier.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/HasOrganizationalIdentifier.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.apache.commons.lang3.StringUtils; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.OTHER_ID_AGENCY; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.OTHER_ID_VALUE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.OTHER_ID_AGENCY; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.OTHER_ID_VALUE; public class HasOrganizationalIdentifier { public static CompoundFieldGenerator toOtherIdValue = (builder, value) -> { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/IdUriHelper.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/IdUriHelper.java similarity index 96% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/IdUriHelper.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/IdUriHelper.java index 4bea9bf..b74aa7e 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/IdUriHelper.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/IdUriHelper.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Identifier.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Identifier.java similarity index 77% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Identifier.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Identifier.java index 4342090..a4fd63a 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Identifier.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Identifier.java @@ -13,26 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; -import nl.knaw.dans.ingest.core.service.XmlReader; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.xml.XmlReader; import org.w3c.dom.Node; import java.util.Map; import java.util.Optional; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.ARCHIS_NUMBER_ID; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.ARCHIS_NUMBER_TYPE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.GRANT_NUMBER_AGENCY; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.GRANT_NUMBER_VALUE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.OTHER_ID_AGENCY; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.OTHER_ID_VALUE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.PUBLICATION_CITATION; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.PUBLICATION_ID_NUMBER; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.PUBLICATION_ID_TYPE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.PUBLICATION_URL; -import static nl.knaw.dans.ingest.core.service.XmlReader.NAMESPACE_XSI; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.ARCHIS_NUMBER_ID; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.ARCHIS_NUMBER_TYPE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.GRANT_NUMBER_AGENCY; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.GRANT_NUMBER_VALUE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.OTHER_ID_AGENCY; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.OTHER_ID_VALUE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.PUBLICATION_CITATION; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.PUBLICATION_ID_NUMBER; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.PUBLICATION_ID_TYPE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.PUBLICATION_URL; +import static nl.knaw.dans.dvingest.core.dansbag.xml.XmlReader.NAMESPACE_XSI; public class Identifier extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/InCollection.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/InCollection.java similarity index 96% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/InCollection.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/InCollection.java index cd6171f..dc7fb51 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/InCollection.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/InCollection.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; import org.w3c.dom.Node; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Language.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Language.java similarity index 84% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Language.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Language.java index 845db9e..1ecd551 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Language.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Language.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.w3c.dom.Node; import java.util.Map; import java.util.Optional; import java.util.Set; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.KEYWORD_VALUE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.KEYWORD_VOCABULARY; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.KEYWORD_VOCABULARY_URI; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.KEYWORD_VALUE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.KEYWORD_VOCABULARY; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.KEYWORD_VOCABULARY_URI; public class Language extends Base { public static CompoundFieldGenerator toKeywordValue = (builder, value) -> { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/LicenseElem.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/LicenseElem.java similarity index 94% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/LicenseElem.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/LicenseElem.java index 0f43a59..787b703 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/LicenseElem.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/LicenseElem.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.service.XmlReader; +import nl.knaw.dans.dvingest.core.dansbag.xml.XmlReader; import org.w3c.dom.Node; import java.net.URI; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/PersonalData.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/PersonalData.java similarity index 94% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/PersonalData.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/PersonalData.java index c77fd44..fa8c7e9 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/PersonalData.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/PersonalData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; import org.w3c.dom.Node; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Publisher.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Publisher.java similarity index 69% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Publisher.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Publisher.java index 23c2e74..c8c9b46 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Publisher.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Publisher.java @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.w3c.dom.Node; import java.util.Set; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DISTRIBUTOR_ABBREVIATION; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DISTRIBUTOR_AFFILIATION; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DISTRIBUTOR_LOGO_URL; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DISTRIBUTOR_NAME; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.DISTRIBUTOR_URL; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DISTRIBUTOR_ABBREVIATION; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DISTRIBUTOR_AFFILIATION; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DISTRIBUTOR_LOGO_URL; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DISTRIBUTOR_NAME; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.DISTRIBUTOR_URL; @Slf4j public class Publisher extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Relation.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Relation.java similarity index 83% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Relation.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Relation.java index bcb65ac..862f466 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Relation.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Relation.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.w3c.dom.Node; import java.util.HashMap; import java.util.Map; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.RELATION_TEXT; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.RELATION_TYPE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.RELATION_URI; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.RELATION_TEXT; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.RELATION_TYPE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.RELATION_URI; public class Relation extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Spatial.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Spatial.java similarity index 97% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Spatial.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Spatial.java index 836ef48..98ab1bd 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Spatial.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Spatial.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/SpatialBox.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/SpatialBox.java similarity index 79% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/SpatialBox.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/SpatialBox.java index b54891f..5a60582 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/SpatialBox.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/SpatialBox.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.apache.commons.lang3.StringUtils; import org.w3c.dom.Node; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SPATIAL_BOX_EAST; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SPATIAL_BOX_NORTH; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SPATIAL_BOX_SCHEME; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SPATIAL_BOX_SOUTH; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SPATIAL_BOX_WEST; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SPATIAL_BOX_EAST; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SPATIAL_BOX_NORTH; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SPATIAL_BOX_SCHEME; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SPATIAL_BOX_SOUTH; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SPATIAL_BOX_WEST; @Slf4j public class SpatialBox extends Spatial { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/SpatialCoverage.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/SpatialCoverage.java similarity index 96% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/SpatialCoverage.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/SpatialCoverage.java index a77d4b6..731c04c 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/SpatialCoverage.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/SpatialCoverage.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import org.w3c.dom.Node; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/SpatialPoint.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/SpatialPoint.java similarity index 82% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/SpatialPoint.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/SpatialPoint.java index 7867d25..1a1affb 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/SpatialPoint.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/SpatialPoint.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.apache.commons.lang3.StringUtils; import org.w3c.dom.Node; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SPATIAL_POINT_SCHEME; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SPATIAL_POINT_X; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SPATIAL_POINT_Y; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SPATIAL_POINT_SCHEME; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SPATIAL_POINT_X; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SPATIAL_POINT_Y; @Slf4j public class SpatialPoint extends Spatial { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Subject.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Subject.java similarity index 87% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Subject.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Subject.java index f0be74e..7dc0c01 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/Subject.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/Subject.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.service.mapper.builder.CompoundFieldGenerator; +import nl.knaw.dans.dvingest.core.dansbag.mapper.builder.CompoundFieldGenerator; import org.w3c.dom.Node; import java.util.regex.Pattern; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.KEYWORD_VALUE; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.KEYWORD_VOCABULARY; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.KEYWORD_VOCABULARY_URI; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.KEYWORD_VALUE; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.KEYWORD_VOCABULARY; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.KEYWORD_VOCABULARY_URI; @Slf4j public class Subject extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/SubjectAbr.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/SubjectAbr.java similarity index 74% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/SubjectAbr.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/SubjectAbr.java index 9adfb34..a8fc508 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/SubjectAbr.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/SubjectAbr.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; import org.w3c.dom.Node; @@ -23,13 +23,13 @@ import java.nio.file.Paths; import java.util.Map; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.ABR_BASE_URL; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_ABR_ARTIFACT; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_ABR_COMPLEX; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_ABR_OLD; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_URI_ABR_ARTIFACT; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_URI_ABR_COMPLEX; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_URI_ABR_OLD; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.ABR_BASE_URL; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_ABR_ARTIFACT; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_ABR_COMPLEX; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_ABR_OLD; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_URI_ABR_ARTIFACT; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_URI_ABR_COMPLEX; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_URI_ABR_OLD; @Slf4j public class SubjectAbr extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/TemporalAbr.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/TemporalAbr.java similarity index 77% rename from src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/TemporalAbr.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/TemporalAbr.java index 31269bd..25b10a9 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/mapper/mapping/TemporalAbr.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/mapper/mapping/TemporalAbr.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service.mapper.mapping; +package nl.knaw.dans.dvingest.core.dansbag.mapper.mapping; import lombok.extern.slf4j.Slf4j; import org.w3c.dom.Node; import java.util.Map; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_ABR_PERIOD; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_ABR_PLUS; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_URI_ABR_PERIOD; -import static nl.knaw.dans.ingest.core.service.DepositDatasetFieldNames.SCHEME_URI_ABR_PLUS; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_ABR_PERIOD; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_ABR_PLUS; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_URI_ABR_PERIOD; +import static nl.knaw.dans.dvingest.core.dansbag.mapper.DepositDatasetFieldNames.SCHEME_URI_ABR_PLUS; @Slf4j public class TemporalAbr extends Base { diff --git a/src/main/java/nl/knaw/dans/ingest/core/exception/TargetNotFoundException.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/package-info.java similarity index 74% rename from src/main/java/nl/knaw/dans/ingest/core/exception/TargetNotFoundException.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/package-info.java index 80ed562..90d1ffb 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/exception/TargetNotFoundException.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/package-info.java @@ -13,10 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.exception; - -public class TargetNotFoundException extends Exception { - public TargetNotFoundException(String msg) { - super(msg); - } -} +/** + * Classes responsible for converting a DANS bag deposit to a Dataverse ingest compatible deposit. Note, that this is the only package that deals with DANS bag specific logic. + */ +package nl.knaw.dans.dvingest.core.dansbag; \ No newline at end of file diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/XPathConstants.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XPathConstants.java similarity index 94% rename from src/main/java/nl/knaw/dans/ingest/core/service/XPathConstants.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XPathConstants.java index 0fed66a..5478817 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/XPathConstants.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XPathConstants.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service; +package nl.knaw.dans.dvingest.core.dansbag.xml; public interface XPathConstants { String DDM_PROFILE = "/ddm:DDM/ddm:profile"; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/XPathEvaluator.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XPathEvaluator.java similarity index 98% rename from src/main/java/nl/knaw/dans/ingest/core/service/XPathEvaluator.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XPathEvaluator.java index bf3677c..71aa3b6 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/XPathEvaluator.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XPathEvaluator.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service; +package nl.knaw.dans.dvingest.core.dansbag.xml; import org.w3c.dom.Node; import org.w3c.dom.NodeList; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/XmlNamespaces.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XmlNamespaces.java similarity index 97% rename from src/main/java/nl/knaw/dans/ingest/core/service/XmlNamespaces.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XmlNamespaces.java index e94f064..4552caa 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/XmlNamespaces.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XmlNamespaces.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service; +package nl.knaw.dans.dvingest.core.dansbag.xml; public interface XmlNamespaces { String NAMESPACE_XML = "http://www.w3.org/XML/1998/namespace"; diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/XmlReader.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XmlReader.java similarity index 87% rename from src/main/java/nl/knaw/dans/ingest/core/service/XmlReader.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XmlReader.java index d55ffff..2c37217 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/XmlReader.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XmlReader.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service; +package nl.knaw.dans.dvingest.core.dansbag.xml; import org.w3c.dom.Document; import org.xml.sax.SAXException; @@ -26,6 +26,4 @@ public interface XmlReader extends XmlNamespaces { Document readXmlFile(Path path) throws ParserConfigurationException, IOException, SAXException; - Document readXmlString(String str) throws ParserConfigurationException, IOException, SAXException; - } diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/XmlReaderImpl.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XmlReaderImpl.java similarity index 87% rename from src/main/java/nl/knaw/dans/ingest/core/service/XmlReaderImpl.java rename to src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XmlReaderImpl.java index ac532b2..b9a1dfd 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/service/XmlReaderImpl.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/xml/XmlReaderImpl.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.service; +package nl.knaw.dans.dvingest.core.dansbag.xml; import org.w3c.dom.Document; -import org.xml.sax.InputSource; import org.xml.sax.SAXException; import javax.xml.XMLConstants; @@ -26,7 +25,6 @@ import javax.xml.xpath.XPath; import javax.xml.xpath.XPathFactory; import java.io.IOException; -import java.io.StringReader; import java.nio.file.Path; import java.util.Iterator; import java.util.Map; @@ -82,15 +80,6 @@ public Document readXmlFile(Path path) throws ParserConfigurationException, IOEx .parse(path.toFile()); } - @Override - public Document readXmlString(String str) throws ParserConfigurationException, IOException, SAXException { - var factory = getFactory(); - - return factory - .newDocumentBuilder() - .parse(new InputSource(new StringReader(str))); - } - public DocumentBuilderFactory getFactory() throws ParserConfigurationException { var factory = DocumentBuilderFactory.newInstance(); factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); diff --git a/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseService.java b/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseService.java index b774c3d..9f5d43c 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseService.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseService.java @@ -26,10 +26,8 @@ import nl.knaw.dans.lib.dataverse.model.dataset.UpdateType; import nl.knaw.dans.lib.dataverse.model.file.FileMeta; import nl.knaw.dans.lib.dataverse.model.user.AuthenticatedUser; -import org.checkerframework.checker.nullness.qual.NonNull; import java.io.IOException; -import java.net.URI; import java.nio.file.Path; import java.util.List; import java.util.Optional; @@ -47,6 +45,8 @@ public interface DataverseService { void deleteFile(int id) throws DataverseException, IOException; + String getDatasetUrnNbn(String datasetId) throws IOException, DataverseException; + void waitForState(String persistentId, String state) throws DataverseException; void updateMetadata(String targetDatasetPid, DatasetVersion datasetMetadata) throws DataverseException, IOException; @@ -70,4 +70,6 @@ public interface DataverseService { Set getActiveMetadataBlockNames() throws IOException, DataverseException; void addEmbargo(String pid, Embargo embargo) throws IOException, DataverseException; + + List findDoiByMetadataField(String fieldName, String value) throws IOException, DataverseException; } diff --git a/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseServiceImpl.java b/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseServiceImpl.java index b3d1490..c4e3cf9 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseServiceImpl.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseServiceImpl.java @@ -31,8 +31,10 @@ import nl.knaw.dans.lib.dataverse.model.dataset.License; import nl.knaw.dans.lib.dataverse.model.dataset.MetadataBlockSummary; import nl.knaw.dans.lib.dataverse.model.dataset.MetadataField; +import nl.knaw.dans.lib.dataverse.model.dataset.PrimitiveSingleValueField; import nl.knaw.dans.lib.dataverse.model.dataset.UpdateType; import nl.knaw.dans.lib.dataverse.model.file.FileMeta; +import nl.knaw.dans.lib.dataverse.model.search.DatasetResultItem; import nl.knaw.dans.lib.dataverse.model.user.AuthenticatedUser; import java.io.IOException; @@ -175,6 +177,35 @@ public void addEmbargo(String pid, Embargo embargo) throws IOException, Datavers log.debug(result.getEnvelopeAsString()); } + @Override + public List findDoiByMetadataField(String key, String value) throws IOException, DataverseException { + var query = String.format("%s:\"%s\"", key, value); + + log.trace("Searching datasets with query '{}'", query); + var results = dataverseClient.search().find(query); + var items = results.getData().getItems(); + + return items.stream() + .filter(r -> r instanceof DatasetResultItem) + .map(r -> (DatasetResultItem) r) + .map(DatasetResultItem::getGlobalId) + .collect(Collectors.toList()); + } + + @Override + public String getDatasetUrnNbn(String pid) throws IOException, DataverseException { + var dataset = dataverseClient.dataset(pid); + var version = dataset.getVersion(); + var data = version.getData(); + var metadata = data.getMetadataBlocks().get("dansDataVaultMetadata"); + + return metadata.getFields().stream() + .filter(f -> f.getTypeName().equals("dansNbn")) + .map(f -> (PrimitiveSingleValueField) f) + .map(PrimitiveSingleValueField::getValue) + .findFirst().orElseThrow(() -> new IllegalStateException("No URN:NBN found in dataset")); + } + // TODO: move this to dans-dataverse-client-lib; it is similar to awaitLockState. public void waitForState(String datasetId, String expectedState) { var numberOfTimesTried = 0; diff --git a/src/main/java/nl/knaw/dans/dvingest/core/service/YamlServiceImpl.java b/src/main/java/nl/knaw/dans/dvingest/core/service/YamlServiceImpl.java index bc188dd..dfaed55 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/service/YamlServiceImpl.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/service/YamlServiceImpl.java @@ -31,7 +31,6 @@ import nl.knaw.dans.dvingest.core.yaml.UpdateState; import nl.knaw.dans.lib.dataverse.MetadataFieldDeserializer; import nl.knaw.dans.lib.dataverse.model.dataset.Dataset; -import nl.knaw.dans.lib.dataverse.model.dataset.MetadataBlockSummary; import nl.knaw.dans.lib.dataverse.model.dataset.MetadataField; import nl.knaw.dans.lib.dataverse.model.file.FileMeta; diff --git a/src/main/java/nl/knaw/dans/dvingest/core/yaml/AddEmbargo.java b/src/main/java/nl/knaw/dans/dvingest/core/yaml/AddEmbargo.java index 81c8cb0..9590e8a 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/yaml/AddEmbargo.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/yaml/AddEmbargo.java @@ -16,7 +16,6 @@ package nl.knaw.dans.dvingest.core.yaml; import lombok.Data; -import nl.knaw.dans.lib.dataverse.model.dataset.Embargo; import java.util.List; diff --git a/src/main/java/nl/knaw/dans/dvingest/core/yaml/EditFiles.java b/src/main/java/nl/knaw/dans/dvingest/core/yaml/EditFiles.java index e09364a..0aa3961 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/yaml/EditFiles.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/yaml/EditFiles.java @@ -24,10 +24,11 @@ public class EditFiles { private List deleteFiles = List.of(); private List replaceFiles = List.of(); + private List addUnrestrictedFiles = List.of(); private List addRestrictedFiles = List.of(); private List moveFiles = List.of(); - private List ignoreFiles = List.of(); - private List renameAtUploadFiles = List.of(); +// private List ignoreFiles = List.of(); + private List autoRenameFiles = List.of(); private List updateFileMetas = List.of(); private List addEmbargoes = List.of(); } diff --git a/src/main/java/nl/knaw/dans/dvingest/core/yaml/EditFilesRoot.java b/src/main/java/nl/knaw/dans/dvingest/core/yaml/EditFilesRoot.java index 0bf2c00..68e8704 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/yaml/EditFilesRoot.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/yaml/EditFilesRoot.java @@ -18,7 +18,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.checkerframework.checker.units.qual.A; @Data @AllArgsConstructor diff --git a/src/main/java/nl/knaw/dans/ingest/core/exception/InvalidDepositorRoleException.java b/src/main/java/nl/knaw/dans/dvingest/core/yaml/package-info.java similarity index 76% rename from src/main/java/nl/knaw/dans/ingest/core/exception/InvalidDepositorRoleException.java rename to src/main/java/nl/knaw/dans/dvingest/core/yaml/package-info.java index 37d5320..89b9403 100644 --- a/src/main/java/nl/knaw/dans/ingest/core/exception/InvalidDepositorRoleException.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/yaml/package-info.java @@ -13,10 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package nl.knaw.dans.ingest.core.exception; - -public class InvalidDepositorRoleException extends Exception { - public InvalidDepositorRoleException(String msg) { - super(msg); - } -} +/** + * Classes that model the YAML configuration for the ingest process. Note, that parts of the YAML configuration is taken from dans-dataverse-client-lib. + */ +package nl.knaw.dans.dvingest.core.yaml; \ No newline at end of file diff --git a/src/main/java/nl/knaw/dans/ingest/core/dataverse/DatasetService.java b/src/main/java/nl/knaw/dans/ingest/core/dataverse/DatasetService.java deleted file mode 100644 index 34f299b..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/dataverse/DatasetService.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.dataverse; - -import nl.knaw.dans.ingest.core.exception.InvalidDatasetStateException; -import nl.knaw.dans.lib.dataverse.DataverseClient; -import nl.knaw.dans.lib.dataverse.DataverseException; -import nl.knaw.dans.lib.dataverse.model.search.DatasetResultItem; -import nl.knaw.dans.lib.dataverse.model.user.AuthenticatedUser; - -import java.io.IOException; -import java.net.URI; -import java.time.Instant; -import java.util.Collection; -import java.util.List; -import java.util.Optional; - -public interface DatasetService { - void setEmbargo(String datasetId, Instant dateAvailable, Collection fileIds) throws IOException, DataverseException; - DataverseClient _getClient(); - List getLicenses() throws IOException, DataverseException; -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/deposit/BagDirResolver.java b/src/main/java/nl/knaw/dans/ingest/core/deposit/BagDirResolver.java deleted file mode 100644 index fc56852..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/deposit/BagDirResolver.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.deposit; - -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; - -import java.io.IOException; -import java.nio.file.Path; - -public interface BagDirResolver { - - /** - * Returns the first subdirectory of the provided path if it looks like a bag. It will not check if the bag is valid and only do a very superficial check. It will throw an exception if - *
    - *
  • The path is not a directory
    • - *
  • There is no deposit.properties file inside the path
    • - *
  • There are no subdirectories in the path
    • - *
  • There are more than 1 subdirectories in the path
    • - *
  • The subdirectory does not contain a bagit.txt file
    • - *
- * - * @param depositDir The deposit directory - * @return The path to the first subdirectory - * @throws InvalidDepositException If any of the checks above fail - * @throws IOException Propagated from underlying systems to indicate an IO error - */ - Path getBagDir(Path depositDir) throws InvalidDepositException, IOException; -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/deposit/BagDirResolverImpl.java b/src/main/java/nl/knaw/dans/ingest/core/deposit/BagDirResolverImpl.java deleted file mode 100644 index 6725e53..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/deposit/BagDirResolverImpl.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.deposit; - -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; -import nl.knaw.dans.ingest.core.io.FileService; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.stream.Collectors; - -public class BagDirResolverImpl implements BagDirResolver { - private final FileService fileService; - - public BagDirResolverImpl(FileService fileService) { - this.fileService = fileService; - } - - @Override - public Path getBagDir(Path depositDir) throws InvalidDepositException, IOException { - if (!fileService.isDirectory(depositDir)) { - throw new InvalidDepositException(String.format("%s is not a directory", depositDir)); - } - - try (var substream = fileService.listDirectories(depositDir)) { - var directories = substream.collect(Collectors.toList()); - - // only 1 directory allowed, not 0 or more than 1 - if (directories.size() != 1) { - throw new InvalidDepositException(String.format( - "%s has more or fewer than one subdirectory", depositDir - )); - } - - // check for the presence of deposit.properties and bagit.txt - if (!fileService.fileExists(depositDir.resolve("deposit.properties"))) { - throw new InvalidDepositException(String.format( - "%s does not contain a deposit.properties file", depositDir - )); - } - - var bagDir = directories.get(0); - - if (!fileService.fileExists(bagDir.resolve("bagit.txt"))) { - throw new InvalidDepositException(String.format( - "%s does not contain a bag", depositDir - )); - } - - return bagDir; - } - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositFileLister.java b/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositFileLister.java deleted file mode 100644 index c41da5d..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositFileLister.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.deposit; - -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.domain.DepositFile; - -import java.io.IOException; -import java.util.List; - -public interface DepositFileLister { - List getDepositFiles(Deposit deposit) throws IOException; - -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositFileListerImpl.java b/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositFileListerImpl.java deleted file mode 100644 index ce6c44f..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositFileListerImpl.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.deposit; - -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.domain.DepositFile; -import nl.knaw.dans.ingest.core.domain.OriginalFilePathMapping; -import nl.knaw.dans.ingest.core.service.ManifestHelperImpl; -import nl.knaw.dans.ingest.core.service.XPathEvaluator; -import org.w3c.dom.Node; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static nl.knaw.dans.ingest.core.service.XPathConstants.FILES_FILE; - -public class DepositFileListerImpl implements DepositFileLister { - @Override - public List getDepositFiles(Deposit deposit) throws IOException { - var bag = deposit.getBag(); - var bagDir = bag.getRootDir(); - var filePathToSha1 = ManifestHelperImpl.getFilePathToSha1(bag); - var originalFilePathMappings = getOriginalFilePathMapping(bagDir); - - return XPathEvaluator.nodes(deposit.getFilesXml(), FILES_FILE) - .map(node -> { - var filePath = Optional.ofNullable(node.getAttributes().getNamedItem("filepath")) - .map(Node::getTextContent) - .map(Path::of) - .orElseThrow(() -> new IllegalArgumentException("File element without filepath attribute")); - - var physicalFile = originalFilePathMappings.getPhysicalPath(filePath); - var sha1 = filePathToSha1.get(physicalFile); - - return new DepositFile(filePath, physicalFile, sha1, node); - }) - .collect(Collectors.toList()); - } - - private OriginalFilePathMapping getOriginalFilePathMapping(Path bagDir) throws IOException { - var originalFilepathsFile = bagDir.resolve("original-filepaths.txt"); - - if (Files.exists(originalFilepathsFile)) { - var lines = Files.readAllLines(originalFilepathsFile); - var mappings = lines.stream().map(line -> { - var parts = line.split("\\s+", 2); - - if (parts.length == 2) { - return new OriginalFilePathMapping.Mapping( - Path.of(parts[0].trim()), - Path.of(parts[1].trim()) - ); - } - - return null; - }) - .filter(Objects::nonNull) - .collect(Collectors.toSet()); - - return new OriginalFilePathMapping(mappings); - } - else { - return new OriginalFilePathMapping(Set.of()); - } - } -} \ No newline at end of file diff --git a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositLocationReader.java b/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositLocationReader.java deleted file mode 100644 index d8a77be..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositLocationReader.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.deposit; - -import nl.knaw.dans.ingest.core.domain.DepositLocation; -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; - -import java.io.IOException; -import java.nio.file.Path; - -public interface DepositLocationReader { - - DepositLocation readDepositLocation(Path depositDir) throws InvalidDepositException, IOException; -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositLocationReaderImpl.java b/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositLocationReaderImpl.java deleted file mode 100644 index bb4256f..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositLocationReaderImpl.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.deposit; - -import lombok.AllArgsConstructor; -import nl.knaw.dans.ingest.core.domain.DepositLocation; -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; -import nl.knaw.dans.ingest.core.exception.MissingTargetException; -import nl.knaw.dans.ingest.core.io.BagDataManager; -import org.apache.commons.configuration2.Configuration; -import org.apache.commons.configuration2.ex.ConfigurationException; -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.nio.file.Path; -import java.time.OffsetDateTime; -import java.time.format.DateTimeParseException; -import java.util.UUID; - -@AllArgsConstructor -public class DepositLocationReaderImpl implements DepositLocationReader { - private final BagDataManager bagDataManager; - - @Override - public DepositLocation readDepositLocation(Path depositDir) throws InvalidDepositException, IOException { - try { - var properties = bagDataManager.readDepositProperties(depositDir); - var depositId = getDepositId(depositDir); - var target = getTarget(properties); - var created = getCreated(properties); - - return new DepositLocation(depositDir, target, depositId.toString(), created); - } - catch (ConfigurationException e) { - throw new InvalidDepositException("Deposit.properties file could not be read", e); - } - catch (MissingTargetException e) { - throw new InvalidDepositException(e.getMessage(), e); - } - } - - String getTarget(Configuration properties) throws MissingTargetException { - // the logic for the target should be - // 1. if there is a dataverse.sword-token, use that - // 2. otherwise, use identifier.doi - var target = properties.getString("dataverse.sword-token"); - - if (StringUtils.isBlank(target)) { - target = properties.getString("identifier.doi"); - } - - if (StringUtils.isBlank(target)) { - throw new MissingTargetException("No viable target found in deposit"); - } - - return target; - } - - UUID getDepositId(Path path) throws InvalidDepositException { - try { - return UUID.fromString(path.getFileName().toString()); - } - catch (IllegalArgumentException e) { - throw new InvalidDepositException(String.format( - "Deposit dir must be an UUID; found '%s'", - path.getFileName() - ), e); - } - } - - OffsetDateTime getCreated(Configuration properties) throws InvalidDepositException { - try { - var created = properties.getString("creation.timestamp"); - if (StringUtils.isBlank(created)) { - throw new InvalidDepositException("No creation timestamp found in deposit"); - } - - return OffsetDateTime.parse(created); - } - catch (DateTimeParseException e) { - throw new InvalidDepositException("Error while parsing date", e); - } - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositManager.java b/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositManager.java deleted file mode 100644 index 67a06db..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositManager.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.deposit; - -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.domain.DepositLocation; -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; - -import java.io.IOException; -import java.nio.file.Path; - -// an adapter around DepositReader and DepositWriter - -/** - * An adapter class around several functionalities regarding deposit reading, writing and moving. - */ -public interface DepositManager { - void saveBagInfo(Deposit deposit) throws IOException; - - /** - * Reads a Deposit object given the data found in a DepositLocation's dir property. - * - * @param location The DepositLocation that refers to a deposit directory - * @return A fully read Deposit - * @throws InvalidDepositException - */ - Deposit readDeposit(DepositLocation location) throws InvalidDepositException; - - /** - * Reads a DepositLocation from the provided path. This is different from a regular Deposit in that it only contains the minimal amount of data required to sort and group tasks. This data is: - * 
-     *     - A path
-     *     - The created date
-     *     - The target, either a sword token or a DOI
-     *     - The ID of the deposit
-     *
- * - * @param path - * @return A DepositLocation object - * @throws InvalidDepositException - * @throws IOException - */ - DepositLocation readDepositLocation(Path path) throws InvalidDepositException, IOException; - - /** - * Update the deposit.properties file with the data found in this Deposit, and move it to the target path. The path should NOT include the deposit ID. - * - * @param deposit The Deposit that should be written. - * @param outbox The path that this deposit should move to after saving the properties. This path should NOT include the deposit ID. - * @throws IOException - * @throws InvalidDepositException - */ - void updateAndMoveDeposit(Deposit deposit, Path outbox) throws IOException, InvalidDepositException; - - /** - * Move the first path to the target path. The target path should NOT include the deposit ID. This method should only be used if an exception occurred while reading the deposit, making it - * impossible to read/write to the deposit.properties file. - * - * @param depositDir The deposit dir that was attempted to be read - * @param outbox The path to which the deposit dir should be moved - * @throws IOException - */ - void moveDeposit(Path depositDir, Path outbox) throws IOException; -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositManagerImpl.java b/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositManagerImpl.java deleted file mode 100644 index f688d64..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositManagerImpl.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.deposit; - -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.domain.DepositLocation; -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; - -import java.io.IOException; -import java.nio.file.Path; - -public class DepositManagerImpl implements DepositManager { - private final DepositReader depositReader; - private final DepositLocationReader depositLocationReader; - private final DepositWriter depositWriter; - - public DepositManagerImpl(DepositReader depositReader, DepositLocationReader depositLocationReader, DepositWriter depositWriter) { - this.depositReader = depositReader; - this.depositLocationReader = depositLocationReader; - this.depositWriter = depositWriter; - } - - @Override - public void saveBagInfo(Deposit deposit) throws IOException { - depositWriter.saveBagInfo(deposit); - } - - @Override - public Deposit readDeposit(DepositLocation location) throws InvalidDepositException { - return depositReader.readDeposit(location); - } - - @Override - public DepositLocation readDepositLocation(Path path) throws InvalidDepositException, IOException { - return depositLocationReader.readDepositLocation(path); - } - - @Override - public void updateAndMoveDeposit(Deposit deposit, Path outbox) throws IOException, InvalidDepositException { - depositWriter.saveDeposit(deposit); - depositWriter.moveDeposit(deposit, outbox); - } - - @Override - public void moveDeposit(Path depositDir, Path outbox) throws IOException { - depositWriter.moveDeposit(depositDir, outbox); - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositReader.java b/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositReader.java deleted file mode 100644 index 0ba9c06..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositReader.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.deposit; - -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.domain.DepositLocation; -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; - -import java.nio.file.Path; - -public interface DepositReader { - - Deposit readDeposit(DepositLocation location) throws InvalidDepositException; - - Deposit readDeposit(Path depositDir) throws InvalidDepositException; - -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositReaderImpl.java b/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositReaderImpl.java deleted file mode 100644 index d9df69c..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositReaderImpl.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.deposit; - -import gov.loc.repository.bagit.domain.Bag; -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.domain.DepositLocation; -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; -import nl.knaw.dans.ingest.core.io.BagDataManager; -import nl.knaw.dans.ingest.core.io.FileService; -import nl.knaw.dans.ingest.core.service.ManifestHelper; -import nl.knaw.dans.ingest.core.service.XmlReader; -import org.apache.commons.configuration2.Configuration; -import org.apache.commons.lang3.StringUtils; -import org.w3c.dom.Document; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.IOException; -import java.nio.file.Path; -import java.time.OffsetDateTime; -import java.util.List; -import java.util.Optional; - -public class DepositReaderImpl implements DepositReader { - private final XmlReader xmlReader; - private final BagDirResolver bagDirResolver; - private final FileService fileService; - private final BagDataManager bagDataManager; - private final DepositFileLister depositFileLister; - - private final ManifestHelper manifestHelper; - - public DepositReaderImpl(XmlReader xmlReader, BagDirResolver bagDirResolver, FileService fileService, BagDataManager bagDataManager, DepositFileLister depositFileLister, ManifestHelper manifestHelper) { - this.xmlReader = xmlReader; - this.bagDirResolver = bagDirResolver; - this.fileService = fileService; - this.bagDataManager = bagDataManager; - this.depositFileLister = depositFileLister; - this.manifestHelper = manifestHelper; - } - - @Override - public synchronized Deposit readDeposit(DepositLocation location) throws InvalidDepositException { - return readDeposit(location.getDir()); - } - - @Override - public Deposit readDeposit(Path depositDir) throws InvalidDepositException { - try { - var bagDir = bagDirResolver.getBagDir(depositDir); - - var config = bagDataManager.readDepositProperties(depositDir); - var bag = bagDataManager.readBag(bagDir); - manifestHelper.ensureSha1ManifestPresent(bag); - - var deposit = mapToDeposit(depositDir, bagDir, config, bag); - - deposit.setBag(bag); - deposit.setDdm(readOptionalXmlFile(deposit.getDdmPath())); - deposit.setFilesXml(readOptionalXmlFile(deposit.getFilesXmlPath())); - deposit.setAmd(readOptionalXmlFile(deposit.getAmdPath())); - - deposit.setFiles(depositFileLister.getDepositFiles(deposit)); - - return deposit; - } - catch (Throwable cex) { - throw new InvalidDepositException(cex.getMessage(), cex); - } - } - - Document readOptionalXmlFile(Path path) throws ParserConfigurationException, IOException, SAXException { - if (fileService.fileExists(path)) { - return xmlReader.readXmlFile(path); - } - - return null; - } - - Deposit mapToDeposit(Path path, Path bagDir, Configuration config, Bag bag) { - var deposit = new Deposit(); - deposit.setBagDir(bagDir); - deposit.setDir(path); - deposit.setDoi(config.getString("identifier.doi", "")); - deposit.setUrn(config.getString("identifier.urn")); - deposit.setCreated(Optional.ofNullable(config.getString("creation.timestamp")).map(OffsetDateTime::parse).orElse(null)); - deposit.setDepositorUserId(config.getString("depositor.userId")); - - deposit.setDataverseIdProtocol(config.getString("dataverse.id-protocol", "")); - deposit.setDataverseIdAuthority(config.getString("dataverse.id-authority", "")); - deposit.setDataverseId(config.getString("dataverse.id-identifier", "")); - deposit.setDataverseBagId(config.getString("dataverse.bag-id", "")); - deposit.setDataverseNbn(config.getString("dataverse.nbn", "")); - deposit.setDataverseOtherId(config.getString("dataverse.other-id", "")); - deposit.setDataverseOtherIdVersion(config.getString("dataverse.other-id-version", "")); - deposit.setDataverseSwordToken(config.getString("dataverse.sword-token", "")); - deposit.setHasOrganizationalIdentifier(getFirstValue(bag.getMetadata().get("Has-Organizational-Identifier"))); - - var isVersionOf = bag.getMetadata().get("Is-Version-Of"); - - if (isVersionOf != null) { - isVersionOf.stream() - .filter(StringUtils::isNotBlank) - .findFirst() - .ifPresent(item -> { - deposit.setUpdate(true); - deposit.setIsVersionOf(item); - }); - } - - return deposit; - } - - private String getFirstValue(List value) { - if (value == null) { - return null; - } - - return value.stream() - .filter(StringUtils::isNotBlank) - .findFirst() - .orElse(null); - } - -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositWriterImpl.java b/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositWriterImpl.java deleted file mode 100644 index 5ef1706..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/deposit/DepositWriterImpl.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.deposit; - -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.exception.InvalidDepositException; -import nl.knaw.dans.ingest.core.io.BagDataManager; -import org.apache.commons.configuration2.ex.ConfigurationException; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; - -public class DepositWriterImpl implements DepositWriter { - private final BagDataManager bagDataManager; - - public DepositWriterImpl(BagDataManager bagDataManager) { - this.bagDataManager = bagDataManager; - } - - @Override - public void saveDeposit(Deposit deposit) throws InvalidDepositException { - var config = new HashMap(); - config.put("state.label", deposit.getState().toString()); - config.put("state.description", deposit.getStateDescription()); - config.put("identifier.doi", deposit.getDoi()); - config.put("identifier.urn", deposit.getUrn()); - - try { - bagDataManager.saveDepositProperties(deposit.getDir(), config); - } - catch (ConfigurationException cex) { - throw new InvalidDepositException("Unable to save deposit properties", cex); - } - } - - @Override - public void moveDeposit(Deposit deposit, Path outbox) throws IOException { - moveDeposit(deposit.getDir(), outbox); - } - - @Override - public void moveDeposit(Path source, Path outbox) throws IOException { - Files.move(source, outbox.resolve(source.getFileName())); - } - - @Override - public void saveBagInfo(Deposit deposit) throws IOException { - bagDataManager.writeBagMetadata(deposit.getBag()); - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/domain/DatasetAuthor.java b/src/main/java/nl/knaw/dans/ingest/core/domain/DatasetAuthor.java deleted file mode 100644 index 22c8845..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/domain/DatasetAuthor.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.domain; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder -public class DatasetAuthor { - private String titles; - private String initials; - private String insertions; - private String surname; - private String dai; - private String isni; - private String orcid; - private String role; - private String organization; -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/domain/DatasetOrganization.java b/src/main/java/nl/knaw/dans/ingest/core/domain/DatasetOrganization.java deleted file mode 100644 index 131d15f..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/domain/DatasetOrganization.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.domain; - -import lombok.Builder; -import lombok.Data; -import lombok.ToString; - -@Data -@Builder -@ToString -public class DatasetOrganization { - private String name; - private String role; - private String isni; - private String viaf; -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/domain/DepositFile.java b/src/main/java/nl/knaw/dans/ingest/core/domain/DepositFile.java deleted file mode 100644 index f0439fd..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/domain/DepositFile.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.domain; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.w3c.dom.Node; - -import java.nio.file.Path; - -@Data -@AllArgsConstructor -public class DepositFile { - private Path path; - private Path physicalPath; - private String checksum; - private Node xmlNode; - - public Path getPhysicalPath() { - if (this.physicalPath != null) { - return physicalPath; - } - - return path; - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/domain/DepositLocation.java b/src/main/java/nl/knaw/dans/ingest/core/domain/DepositLocation.java deleted file mode 100644 index c07fb13..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/domain/DepositLocation.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.domain; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.nio.file.Path; -import java.time.OffsetDateTime; - -/** - * This class represents the location of a deposit that was done in one of the inboxes. It is intended to be a lightweight pointer to the deposit to be used for enqueuing a large number of deposits - * without incurring the overhead of loading all the deposit metadata into memory. - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class DepositLocation { - private Path dir; - private String target; - private String depositId; - private OffsetDateTime created; -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/domain/OriginalFilePathMapping.java b/src/main/java/nl/knaw/dans/ingest/core/domain/OriginalFilePathMapping.java deleted file mode 100644 index 6ff29f3..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/domain/OriginalFilePathMapping.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.domain; - -import lombok.AllArgsConstructor; -import lombok.Data; - -import java.nio.file.Path; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -public class OriginalFilePathMapping { - // the first entry is the logical (original) name, as provided by the deposit - // the second entry is the physical path, renamed to solve filesystem limitations - private final Map mappings = new HashMap<>(); - - public OriginalFilePathMapping(Collection mappings) { - for (var mapping: mappings) { - this.mappings.put(mapping.getOriginalPath(), mapping.getPhysicalPath()); - } - } - - public boolean hasMapping(Path path) { - return this.mappings.containsKey(path); - } - - public Path getPhysicalPath(Path path) { - return this.mappings.getOrDefault(path, path); - } - - @Data - @AllArgsConstructor - public static class Mapping { - private Path physicalPath; - private Path originalPath; - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/exception/CannotUpdateDraftDatasetException.java b/src/main/java/nl/knaw/dans/ingest/core/exception/CannotUpdateDraftDatasetException.java deleted file mode 100644 index c4c3c24..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/exception/CannotUpdateDraftDatasetException.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.exception; - -import nl.knaw.dans.ingest.core.domain.Deposit; - -public class CannotUpdateDraftDatasetException extends RuntimeException { - - public CannotUpdateDraftDatasetException(Deposit deposit) { - super("Latest version must be published before update-deposit can be processed"); - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/exception/DepositorValidatorException.java b/src/main/java/nl/knaw/dans/ingest/core/exception/DepositorValidatorException.java deleted file mode 100644 index 92b5a6b..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/exception/DepositorValidatorException.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.exception; - -public class DepositorValidatorException extends Exception { - public DepositorValidatorException(Throwable e) { - super(e); - } - - public DepositorValidatorException(String message) { - super(message); - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/exception/FailedDepositException.java b/src/main/java/nl/knaw/dans/ingest/core/exception/FailedDepositException.java deleted file mode 100644 index dabf4d7..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/exception/FailedDepositException.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.exception; - -import nl.knaw.dans.ingest.core.domain.Deposit; - -public class FailedDepositException extends RuntimeException { - public FailedDepositException(Deposit deposit, String message) { - super(String.format("Failed %s: %s", deposit.getDir(), message)); - } - - public FailedDepositException(Deposit deposit, String message, Throwable e) { - super(String.format("Failed %s: %s", deposit.getDir(), message), e); - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/exception/InvalidDatasetStateException.java b/src/main/java/nl/knaw/dans/ingest/core/exception/InvalidDatasetStateException.java deleted file mode 100644 index b1b569b..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/exception/InvalidDatasetStateException.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.exception; - -public class InvalidDatasetStateException extends Throwable { - public InvalidDatasetStateException(String msg) { - super(msg); - } - - public InvalidDatasetStateException(String msg, Throwable cause) { - super(msg, cause); - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/exception/MissingTargetException.java b/src/main/java/nl/knaw/dans/ingest/core/exception/MissingTargetException.java deleted file mode 100644 index 6c4571f..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/exception/MissingTargetException.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.exception; - -public class MissingTargetException extends Exception { - public MissingTargetException(String message) { - super(message); - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/io/BagDataManager.java b/src/main/java/nl/knaw/dans/ingest/core/io/BagDataManager.java deleted file mode 100644 index 3652e78..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/io/BagDataManager.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.io; - -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Metadata; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; -import gov.loc.repository.bagit.exceptions.UnparsableVersionException; -import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; -import org.apache.commons.configuration2.Configuration; -import org.apache.commons.configuration2.ex.ConfigurationException; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.Map; - -public interface BagDataManager { - - /** - * Reads the bag metadata on the path that contains bag-info.txt and bagit.txt, among other files - * - * @param bagDir the path that contains bag-info.txt and bagit.txt - * @return The Metadata object - * @throws UnparsableVersionException - * @throws InvalidBagitFileFormatException - * @throws IOException - */ - Metadata readBagMetadata(Path bagDir) throws UnparsableVersionException, InvalidBagitFileFormatException, IOException; - - /** - * Writes the current bag metadata to disk and updates the entry for bag-info.txt in all existing tag manifests, so that the bag stays valid. - * - * @param bag the bag object - * @throws IOException - */ - void writeBagMetadata(Bag bag) throws IOException; - - /** - * Reads the bag on the path that contains bag-info.txt and bagit.txt, among other files - * - * @param bagDir the path that contains bag-info.txt and bagit.txt - * @return The bag object - * @throws UnparsableVersionException - * @throws InvalidBagitFileFormatException - * @throws IOException - * @throws MaliciousPathException - * @throws UnsupportedAlgorithmException - */ - Bag readBag(Path bagDir) throws UnparsableVersionException, InvalidBagitFileFormatException, IOException, MaliciousPathException, UnsupportedAlgorithmException; - - /** - * Reads the deposit.properties file found inside the folder provided. The path should NOT reference deposit.properties directly - * - * @param depositDir The directory that contains a deposit.properties file - * @return - * @throws ConfigurationException - */ - Configuration readDepositProperties(Path depositDir) throws ConfigurationException; - - /** - * Saves the provided configuration to the deposit.properties file in the provided path. The path should NOT reference deposit.properties directly - * - * @param depositDir The directory that should contain the deposit.properties file - * @param configuration - * @throws ConfigurationException - */ - void saveDepositProperties(Path depositDir, Map configuration) throws ConfigurationException; -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/io/BagDataManagerImpl.java b/src/main/java/nl/knaw/dans/ingest/core/io/BagDataManagerImpl.java deleted file mode 100644 index 85b0efc..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/io/BagDataManagerImpl.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.io; - -import gov.loc.repository.bagit.domain.Bag; -import gov.loc.repository.bagit.domain.Metadata; -import gov.loc.repository.bagit.domain.Version; -import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; -import gov.loc.repository.bagit.exceptions.MaliciousPathException; -import gov.loc.repository.bagit.exceptions.UnparsableVersionException; -import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; -import gov.loc.repository.bagit.reader.BagReader; -import gov.loc.repository.bagit.reader.BagitTextFileReader; -import gov.loc.repository.bagit.reader.KeyValueReader; -import gov.loc.repository.bagit.writer.ManifestWriter; -import gov.loc.repository.bagit.writer.MetadataWriter; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.configuration2.Configuration; -import org.apache.commons.configuration2.FileBasedConfiguration; -import org.apache.commons.configuration2.PropertiesConfiguration; -import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder; -import org.apache.commons.configuration2.builder.fluent.Parameters; -import org.apache.commons.configuration2.ex.ConfigurationException; -import org.apache.commons.io.FileUtils; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Map; - -public class BagDataManagerImpl implements BagDataManager { - private final String DEPOSIT_PROPERTIES_FILENAME = "deposit.properties"; - private final BagReader bagReader; - - public BagDataManagerImpl(BagReader bagReader) { - this.bagReader = bagReader; - } - - @Override - public Metadata readBagMetadata(Path bagDir) throws UnparsableVersionException, InvalidBagitFileFormatException, IOException { - var bagitInfo = BagitTextFileReader.readBagitTextFile(bagDir.resolve("bagit.txt")); - var encoding = bagitInfo.getValue(); - - var values = KeyValueReader.readKeyValuesFromFile(bagDir.resolve("bag-info.txt"), ":", encoding); - var metadata = new Metadata(); - metadata.addAll(values); - - return metadata; - } - - public void writeBagMetadata(Bag bag) throws IOException { - MetadataWriter.writeBagMetadata(bag.getMetadata(), Version.LATEST_BAGIT_VERSION(), bag.getRootDir(), StandardCharsets.UTF_8); - try { - for (var m : bag.getTagManifests()) { - var messageDigest = MessageDigest.getInstance(m.getAlgorithm().getMessageDigestName()); - try (var is = FileUtils.openInputStream(bag.getRootDir().resolve("bag-info.txt").toFile())) { - var hash = Hex.encodeHexString(DigestUtils.digest(messageDigest, is)); - m.getFileToChecksumMap().put(bag.getRootDir().resolve("bag-info.txt"), hash); - } - } - ManifestWriter.writeTagManifests(bag.getTagManifests(), bag.getRootDir(), bag.getRootDir(), StandardCharsets.UTF_8); - } - catch (NoSuchAlgorithmException e) { - throw new IllegalStateException("Bag declared to have a tagmanifest with an algorithm but the algorithm does not exist ???", e); - } - } - - @Override - public Bag readBag(Path bagDir) throws UnparsableVersionException, InvalidBagitFileFormatException, IOException, MaliciousPathException, UnsupportedAlgorithmException { - return bagReader.read(bagDir); - } - - @Override - public Configuration readDepositProperties(Path depositDir) throws ConfigurationException { - var propertiesFile = depositDir.resolve(DEPOSIT_PROPERTIES_FILENAME); - var params = new Parameters(); - var paramConfig = params.properties() - .setFileName(propertiesFile.toString()); - - var builder = new FileBasedConfigurationBuilder - (PropertiesConfiguration.class, null, true) - .configure(paramConfig); - - return builder.getConfiguration(); - } - - @Override - public void saveDepositProperties(Path depositDir, Map configuration) throws ConfigurationException { - var params = new Parameters(); - var paramConfig = params.properties() - .setFileName(depositDir.resolve(DEPOSIT_PROPERTIES_FILENAME).toString()); - - var builder = new FileBasedConfigurationBuilder( - PropertiesConfiguration.class, null, true - ).configure(paramConfig); - - var config = builder.getConfiguration(); - - for (var entry : configuration.entrySet()) { - // setProperty overwrites existing values, so no clearing is needed - config.setProperty(entry.getKey(), entry.getValue()); - } - - builder.save(); - } -} \ No newline at end of file diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/DatasetCreator.java b/src/main/java/nl/knaw/dans/ingest/core/service/DatasetCreator.java deleted file mode 100644 index 030770f..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/service/DatasetCreator.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.dataverse.DatasetService; -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.domain.FileInfo; -import nl.knaw.dans.ingest.core.exception.FailedDepositException; -import nl.knaw.dans.lib.dataverse.DataverseApi; -import nl.knaw.dans.lib.dataverse.DataverseException; -import nl.knaw.dans.lib.dataverse.model.RoleAssignment; -import nl.knaw.dans.lib.dataverse.model.dataset.Dataset; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.net.URI; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - -import static java.util.Collections.singletonMap; - -@Slf4j -public class DatasetCreator extends DatasetEditor { - - private final String depositorRole; - - public DatasetCreator( - boolean isMigration, - Dataset dataset, - Deposit deposit, - ObjectMapper objectMapper, - List supportedLicenses, - Pattern fileExclusionPattern, - ZipFileHandler zipFileHandler, - String depositorRole, - DatasetService datasetService, - String vaultMetadataKey, - boolean deleteDraftOnFailure) { - super( - isMigration, - dataset, - deposit, - supportedLicenses, - fileExclusionPattern, - zipFileHandler, objectMapper, datasetService, - vaultMetadataKey, - deleteDraftOnFailure); - - this.depositorRole = depositorRole; - } - - @Override - public String performEdit() { - var api = dataverseClient.dataverse("root"); - - log.debug("Creating new dataset"); - String persistentId = null; - - try { - persistentId = importDataset(api); - log.debug("New persistent ID: {}", persistentId); - modifyDataset(persistentId); - return persistentId; - } - catch (Exception e) { - try { - if (persistentId != null) { - deleteDraftIfExists(persistentId); - } - } - catch (IOException | DataverseException ex) { - log.error("Error deleting draft dataset", ex); - } - throw new FailedDepositException(deposit, "Error creating dataset" + (this.deleteDraftOnFailure ? ". Deleting draft" : "."), e); - } - } - - private void modifyDataset(String persistentId) throws IOException, DataverseException { - var api = dataverseClient.dataset(persistentId); - - // This will set fileAccessRequest and termsOfAccess - var version = dataset.getDatasetVersion(); - version.setFileAccessRequest(deposit.allowAccessRequests()); - if (!deposit.allowAccessRequests() && StringUtils.isBlank(version.getTermsOfAccess())) { - version.setTermsOfAccess("N/a"); - } - var keyMap = new HashMap(singletonMap("dansDataVaultMetadata", vaultMetadataKey)); - api.updateMetadata(version, keyMap); - api.awaitUnlock(); - - // license stuff - var license = toJson(Map.of("http://schema.org/license", getLicense(deposit.getDdm()))); - log.debug("Setting license to {}", license); - api.updateMetadataFromJsonLd(license, true); - api.awaitUnlock(); - - // add files to dataset - var pathToFileInfo = getFileInfo(); - - FileInfo originalMetadata = null; - if (!isMigration) { - originalMetadata = createOriginalMetadataFileInfo(); - pathToFileInfo.put(Paths.get(ORIGINAL_METADATA_ZIP), originalMetadata); - } - - log.debug("File info: {}", pathToFileInfo); - var databaseIdToFileInfo = addFiles(persistentId, pathToFileInfo.values()); - - if (originalMetadata != null) { - FileUtils.deleteQuietly(originalMetadata.getPath().toFile()); - } - - log.debug("Database ID -> FileInfo: {}", databaseIdToFileInfo); - // update individual files metadata - updateFileMetadata(databaseIdToFileInfo); - api.awaitUnlock(); - - api.assignRole(getRoleAssignment()); - api.awaitUnlock(); - - var dateAvailable = getDateAvailable(deposit); //deposit.getDateAvailable().get(); - embargoFiles(persistentId, dateAvailable); - } - - private RoleAssignment getRoleAssignment() { - var result = new RoleAssignment(); - result.setRole(depositorRole); - result.setAssignee(String.format("@%s", deposit.getDepositorUserId())); - - return result; - } - - private void updateFileMetadata(Map databaseIds) throws IOException, DataverseException { - for (var entry : databaseIds.entrySet()) { - var id = entry.getKey(); - var result = dataverseClient.file(id).updateMetadata(entry.getValue().getMetadata()); - log.debug("Called updateFileMetadata for id = {}; result = {} {}", id, result.getHttpResponse().getCode(), result.getHttpResponse().getReasonPhrase()); - } - } - - String importDataset(DataverseApi api) throws IOException, DataverseException { - var keyMap = new HashMap(singletonMap("dansDataVaultMetadata", vaultMetadataKey)); - - var response = isMigration - ? api.importDataset(dataset, String.format("doi:%s", deposit.getDoi()), false, keyMap) - : api.createDataset(dataset, keyMap); - - return response.getData().getPersistentId(); - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/DatasetEditor.java b/src/main/java/nl/knaw/dans/ingest/core/service/DatasetEditor.java deleted file mode 100644 index 8190787..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/service/DatasetEditor.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.service; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.dataverse.DatasetService; -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.domain.FileInfo; -import nl.knaw.dans.ingest.core.service.mapper.mapping.FileElement; -import nl.knaw.dans.lib.dataverse.DataverseClient; -import nl.knaw.dans.lib.dataverse.DataverseException; -import nl.knaw.dans.lib.dataverse.Version; -import nl.knaw.dans.lib.dataverse.model.dataset.Dataset; -import nl.knaw.dans.lib.dataverse.model.file.DataFile; -import nl.knaw.dans.lib.dataverse.model.file.FileMeta; -import org.apache.commons.codec.digest.DigestUtils; -import org.w3c.dom.Node; - -import java.io.FileInputStream; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeParseException; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -@Slf4j -public abstract class DatasetEditor { - - public static final String ORIGINAL_METADATA_ZIP = "original-metadata.zip"; - protected static final List embargoExclusions = Arrays.asList("easy-migration.zip", ORIGINAL_METADATA_ZIP); - - protected final DataverseClient dataverseClient; - protected final boolean isMigration; - protected final Dataset dataset; - protected final Deposit deposit; - protected final List supportedLicenses; - - protected final Pattern fileExclusionPattern; - protected final ZipFileHandler zipFileHandler; - - protected final ObjectMapper objectMapper; - protected final DatasetService datasetService; - - protected final String vaultMetadataKey; - - protected boolean deleteDraftOnFailure; - - protected DatasetEditor(boolean isMigration, - Dataset dataset, - Deposit deposit, - List supportedLicenses, - Pattern fileExclusionPattern, - ZipFileHandler zipFileHandler, - ObjectMapper objectMapper, - DatasetService datasetService, - String vaultMetadataKey, - boolean deleteDraftOnFailure) { - this.dataverseClient = datasetService._getClient(); - this.isMigration = isMigration; - this.dataset = dataset; - this.deposit = deposit; - this.supportedLicenses = supportedLicenses; - this.fileExclusionPattern = fileExclusionPattern; - this.zipFileHandler = zipFileHandler; - this.objectMapper = objectMapper; - this.datasetService = datasetService; - this.vaultMetadataKey = vaultMetadataKey; - this.deleteDraftOnFailure = deleteDraftOnFailure; - } - - private static Instant parseDate(String value) { - try { - log.debug("Trying to parse {} as LocalDate", value); - return LocalDate.parse(value).atStartOfDay(ZoneId.systemDefault()).toInstant(); - } - catch (DateTimeParseException e) { - try { - log.debug("Trying to parse {} as ZonedDateTime", value); - return ZonedDateTime.parse(value).toInstant(); - } - catch (DateTimeParseException ee) { - log.debug("Trying to parse {} as LocalDateTime", value); - var id = ZoneId.systemDefault().getRules().getOffset(Instant.now()); - return LocalDateTime.parse(value).toInstant(id); - } - } - } - - public abstract String performEdit() throws IOException, DataverseException, InterruptedException; - - Map addFiles(String persistentId, Collection fileInfos) throws IOException, DataverseException { - var result = new HashMap(fileInfos.size()); - - for (var fileInfo : fileInfos) { - log.debug("Adding file, directoryLabel = {}, label = {}", - fileInfo.getMetadata().getDirectoryLabel(), fileInfo.getMetadata().getLabel()); - - var id = addFile(persistentId, fileInfo); - result.put(id, fileInfo); - } - - return result; - } - - protected FileInfo createOriginalMetadataFileInfo() throws IOException { - var path = zipFileHandler.zipOriginalMetadata(deposit.getDdmPath(), deposit.getFilesXmlPath()); - var checksum = DigestUtils.sha1Hex(new FileInputStream(path.toFile())); - var fileMeta = new FileMeta(); - fileMeta.setLabel(ORIGINAL_METADATA_ZIP); - return new FileInfo(path, path, checksum, false, fileMeta); - } - - private Integer addFile(String persistentId, FileInfo fileInfo) throws IOException, DataverseException { - var dataset = dataverseClient.dataset(persistentId); - var wrappedZip = zipFileHandler.wrapIfZipFile(fileInfo.getPhysicalPath()); - - var file = wrappedZip.orElse(fileInfo.getPhysicalPath()); - if (log.isDebugEnabled()) { - var metadata = objectMapper.writeValueAsString(fileInfo.getMetadata()); - log.debug("Adding file {} with metadata {}", file, metadata); - } - var result = dataset.addFile(file, fileInfo.getMetadata()); - log.debug("Called addFile for {}; result: {}", file, result); - - if (wrappedZip.isPresent()) { - try { - Files.deleteIfExists(wrappedZip.get()); - } - catch (IOException e) { - log.error("Unable to delete zipfile {}", wrappedZip.get(), e); - } - } - - log.debug("Result: {}", result); - return result.getData().getFiles().get(0).getDataFile().getId(); - } - - protected String getLicense(Node node) { -// return XPathEvaluator.nodes(node, DDM_DCMI_METADATA + "/dcterms:license") -// .filter(Licenses::isLicenseUri) -// .findFirst() -// .map(n -> Licenses.getLicenseUri(supportedLicenses, n)) -// .map(URI::toASCIIString) -// .orElseThrow(() -> new RejectedDepositException(deposit, "no license specified")); - return null; - } - - protected String toJson(Map input) throws JsonProcessingException { - return objectMapper.writeValueAsString(input); - } - - Map getFileInfo() { - var files = FileElement.pathToFileInfo(deposit, isMigration); - - return files.entrySet().stream() - .map(entry -> { - // relativize the path - var bagPath = entry.getKey(); - var fileInfo = entry.getValue(); - var newKey = Path.of("data").relativize(bagPath); - - return Map.entry(newKey, fileInfo); - }) - .filter(entry -> { - // remove entries that match the file exclusion pattern - var path = entry.getKey().toString(); - - return (fileExclusionPattern == null || !fileExclusionPattern.matcher(path).matches()); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - // FIL008, FIL009 - void embargoFiles(String persistentId, Instant dateAvailable) throws IOException, DataverseException { - var now = Instant.now(); - - if (!dateAvailable.isAfter(now)) { - log.debug("Date available in the past, no embargo: {}", dateAvailable); - } - else { - var api = dataverseClient.dataset(persistentId); - var files = api.getFiles(Version.LATEST.toString()).getData(); - - var items = files.stream() - .filter(f -> !embargoExclusions.contains(f.getLabel())) - .map(FileMeta::getDataFile) - .map(DataFile::getId) - .collect(Collectors.toList()); - - embargoFiles(persistentId, dateAvailable, items); - } - } - - void embargoFiles(String persistentId, Instant dateAvailable, Collection fileIds) throws IOException, DataverseException { - var now = Instant.now(); - - if (!dateAvailable.isAfter(now)) { - log.debug("Date available in the past, no embargo: {}", dateAvailable); - } - else if (fileIds.size() == 0) { - log.debug("No files to embargo"); - } - else { - datasetService.setEmbargo(persistentId, dateAvailable, fileIds); - } - } - - Instant getDateAvailable(Deposit deposit) { - return XPathEvaluator.strings(deposit.getDdm(), "/ddm:DDM/ddm:profile/ddm:available") - .map(DatasetEditor::parseDate) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Deposit without a ddm:available element")); - } - - void deleteDraftIfExists(String persistentId) throws IOException, DataverseException { - if (deleteDraftOnFailure) { - var data = dataverseClient.dataset(persistentId).getLatestVersion().getData(); - - if (data.getLatestVersion().getVersionState().contains("DRAFT")) { - log.warn("Deleting draft version of dataset {} because deposit failed AND deleteDraftOnFailure = true", persistentId); - dataverseClient.dataset(persistentId).deleteDraft(); - } - } - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/DatasetUpdater.java b/src/main/java/nl/knaw/dans/ingest/core/service/DatasetUpdater.java deleted file mode 100644 index f368143..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/service/DatasetUpdater.java +++ /dev/null @@ -1,439 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.extern.slf4j.Slf4j; -import nl.knaw.dans.ingest.core.dataverse.DatasetService; -import nl.knaw.dans.ingest.core.domain.Deposit; -import nl.knaw.dans.ingest.core.domain.FileInfo; -import nl.knaw.dans.ingest.core.exception.CannotUpdateDraftDatasetException; -import nl.knaw.dans.lib.dataverse.DatasetApi; -import nl.knaw.dans.lib.dataverse.DataverseException; -import nl.knaw.dans.lib.dataverse.Version; -import nl.knaw.dans.lib.dataverse.model.dataset.Dataset; -import nl.knaw.dans.lib.dataverse.model.file.FileMeta; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Collections.singletonMap; - -@Slf4j -public class DatasetUpdater extends DatasetEditor { - - protected DatasetUpdater(boolean isMigration, Dataset dataset, - Deposit deposit, List supportedLicenses, - Pattern fileExclusionPattern, ZipFileHandler zipFileHandler, - ObjectMapper objectMapper, DatasetService datasetService, String vaultMetadataKey, boolean deleteDraftOnFailure) { - super(isMigration, dataset, deposit, supportedLicenses, - fileExclusionPattern, - zipFileHandler, objectMapper, datasetService, vaultMetadataKey, deleteDraftOnFailure); - } - - @Override - public String performEdit() throws IOException, DataverseException, InterruptedException { - // TODO wait for the doi to become available (uses sleep in scala, but this was frowned upon!) - FileInfo originalMetadata = null; - - try { - var doi = deposit.getDataverseDoi(); - - try { - Thread.sleep(4000); - var api = dataverseClient.dataset(doi); - - api.awaitUnlock(); - Thread.sleep(8000); - api.awaitUnlock(); - - var state = api.getLatestVersion().getData().getLatestVersion().getVersionState(); - - if (state.contains("DRAFT")) { - throw new CannotUpdateDraftDatasetException(deposit); - } - - var latestVersion = api.getLatestVersion().getData().getLatestVersion(); - var datasetVersion = dataset.getDatasetVersion(); - var fileAccessRequest = deposit.allowAccessRequests() && api.getLatestVersion().getData().getLatestVersion().getFileAccessRequest(); - // Possibly disable file access request (never enable if it was not already). - datasetVersion.setFileAccessRequest(fileAccessRequest); - - // Copy the terms of access from the latest version (not automatically inherited, alas!) if no new terms were provided. - if (StringUtils.isBlank(datasetVersion.getTermsOfAccess())) { - datasetVersion.setTermsOfAccess(StringUtils.isNotBlank(latestVersion.getTermsOfAccess()) ? - latestVersion.getTermsOfAccess() : - // if fileAccessRequest ends up false, Dataverse requires termsOfAccess to be filled in. - (fileAccessRequest ? "" : "N/a")); - } - - if (!isMigration) { - // CIT025B - // Copy the dateOfDeposit from the latest version. We make an exception for migration, because the dateOfDeposit is taken - // from the AMD. It should not actually make a difference though. - latestVersion.getMetadataBlocks().get("citation").getFields().stream() - .filter(field -> field.getTypeName().equals("dateOfDeposit")) - .findFirst() - .ifPresent(field -> datasetVersion.getMetadataBlocks().get("citation").getFields().add(field)); - } - - var keyMap = new HashMap(singletonMap("dansDataVaultMetadata", vaultMetadataKey)); - api.updateMetadata(datasetVersion, keyMap); - api.awaitUnlock(); - - var license = toJson(Map.of("http://schema.org/license", getLicense(deposit.getDdm()))); - api.updateMetadataFromJsonLd(license, true); - api.awaitUnlock(); - - var pathToFileInfo = getFileInfo(); - - if (!isMigration) { - originalMetadata = createOriginalMetadataFileInfo(); - pathToFileInfo.put(Paths.get(ORIGINAL_METADATA_ZIP), originalMetadata); - } - - log.debug("pathToFileInfo = {}", pathToFileInfo); - var pathToFileMetaInLatestVersion = getFilesInfoInLatestVersion(api); - - validateFileMetas(pathToFileMetaInLatestVersion); - - var versions = api.getAllVersions().getData(); - var publishedVersions = versions.stream().filter(v -> v.getVersionState().equals("RELEASED")).count(); - log.debug("Number of published versions so far: {}", publishedVersions); - - // move old paths to new paths - var oldToNewPathMovedFiles = getOldToNewPathOfFilesToMove(pathToFileMetaInLatestVersion, pathToFileInfo); - var fileMovements = oldToNewPathMovedFiles.keySet().stream() - .map(path -> Map.entry(pathToFileMetaInLatestVersion.get(path).getDataFile().getId(), pathToFileInfo.get(oldToNewPathMovedFiles.get(path)).getMetadata())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - log.debug("fileMovements = {}", fileMovements); - - /* - * File replacement can only happen on files with paths that are not also involved in a rename/move action. Otherwise, we end up with: - * - * - trying to update the file metadata by a database ID that is not the "HEAD" of a file version history (which Dataverse doesn't allow anyway, it - * fails with "You cannot edit metadata on a dataFile that has been replaced"). This happens when a file A is renamed to B, but a different file A - * is also added in the same update. Schematically, (1) means unique file 1, A is a file name: - * V1 -> V2 - * - * (1)A -> (1)B (move) - * .. -> (2)A (add) - * - * - trying to add a file with a name that already exists. This happens when a file A is renamed to B, while B is also part of the latest version (V1) - * - * V1 -> V2 - * (1)A -> (1)B (move) - * (2)B -> .. (delete) - * - */ - var fileReplacementCandidates = pathToFileMetaInLatestVersion.entrySet().stream() - .filter(pathToFileInfoEntry -> !oldToNewPathMovedFiles.containsKey(pathToFileInfoEntry.getKey())) - .filter(pathToFileInfoEntry -> !oldToNewPathMovedFiles.containsValue(pathToFileInfoEntry.getKey())) // TODO in the OG version, this was a Set; check what performance is like - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - var filesToReplace = getFilesToReplace(pathToFileInfo, fileReplacementCandidates); - log.debug("filesToReplace = {}", filesToReplace); - var fileReplacements = replaceFiles(api, filesToReplace); - log.debug("fileReplacements = {}", fileReplacements); - - /* - * To find the files to delete we start from the paths in the deposit payload. In principle, these paths are remaining, so should NOT be deleted. - * However, if a file is moved/renamed to a path that was also present in the latest version, then the old file at that path must first be deleted - * (and must therefore NOT be included in candidateRemainingFiles). Otherwise, we'll end up trying to use an existing (directoryLabel, label) pair. - */ - var newPathsOfMovedFiles = new HashSet<>(oldToNewPathMovedFiles.values()); - var candidateRemainingFiles = diff(pathToFileInfo.keySet(), newPathsOfMovedFiles); - - /* - * The paths to delete, now, are the paths in the latest version minus the remaining files. We further subtract the old paths of the moved files. - * This may be a bit confusing, but the goal is to make sure that the underlying FILE remains present (after all, it is to be renamed/moved). The - * path itself WILL be "removed" from the latest version by the move. (It MAY be filled again by a file addition in the same update, though.) - */ - var oldPathsOfMovedFiles = new HashSet<>(oldToNewPathMovedFiles.keySet()); - var pathsToDelete = diff(diff(pathToFileMetaInLatestVersion.keySet(), candidateRemainingFiles), oldPathsOfMovedFiles); - log.debug("pathsToDelete = {}", pathsToDelete); - - var fileDeletions = getFileDeletions(pathsToDelete, pathToFileMetaInLatestVersion); - log.debug("fileDeletions = {}", fileDeletions); - - deleteFiles(api, fileDeletions); - - /* - * After the movements have been performed, which paths are occupied? We start from the paths of the latest version (pathToFileMetaInLatestVersion.keySet) - * - * The old paths of the moved files (oldToNewPathMovedFiles.keySet) are no longer occupied, so they must be subtracted. This is important in the case where - * a deposit renames/moves a file (leaving the checksum unchanges) but provides a new file for the vacated path. - * - * The paths of the deleted files (pathsToDelete) are no longer occupied, so must be subtracted. (It is not strictly necessary for the calculation - * of pathsToAdd, but done anyway to keep the logic consistent.) - * - * The new paths of the moved files (oldToNewPathMovedFiles.values.toSet) *are* now occupied, so the must be added. This is important to - * avoid those files from being marked as "new" files, i.e. files to be added. - * - * All paths in the deposit that are not occupied, are new files to be added. - */ - - var diffed = diff(diff(pathToFileMetaInLatestVersion.keySet(), newPathsOfMovedFiles), pathsToDelete); - var occupiedPaths = union(diffed, oldToNewPathMovedFiles.values()); - - log.debug("occupiedPaths = {}", occupiedPaths); - var pathsToAdd = diff(pathToFileInfo.keySet(), occupiedPaths); - var filesToAdd = pathsToAdd.stream().map(pathToFileInfo::get).collect(Collectors.toList()); - var fileAdditions = addFiles(doi, filesToAdd).entrySet().stream() - .map(e -> Map.entry(e.getKey(), e.getValue().getMetadata())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - api.awaitUnlock(); - - // TODO: check that only updating the file metadata works (from scala code) - updateFileMetadata(fileReplacements, fileMovements, fileAdditions); - api.awaitUnlock(); - - // embargo - var dateAvailable = getDateAvailable(deposit); - var fileIdsToEmbargo = union(fileReplacements.keySet(), fileAdditions.keySet()) - .stream() - .map(key -> Map.entry(key, fileReplacements.getOrDefault(key, fileAdditions.get(key)))) - .filter(entry -> !embargoExclusions.contains(entry.getValue().getLabel())) - .map(Map.Entry::getKey) - .collect(Collectors.toSet()); - - embargoFiles(doi, dateAvailable, fileIdsToEmbargo); - - return doi; - } - catch (Exception e) { - log.error("Error updating dataset" + (deleteDraftOnFailure ? ", deleting draft" : ""), e); - deleteDraftIfExists(doi); - throw e; - } - } - catch (Exception e) { - log.error("Error updating dataset", e); - throw e; - } - finally { - if (originalMetadata != null) { - FileUtils.deleteQuietly(originalMetadata.getPath().toFile()); - } - } - } - - @SafeVarargs - private void updateFileMetadata(Map... fileMaps) throws IOException, DataverseException { - var seen = new HashSet(); - - for (var fileMap : fileMaps) { - - for (var file : fileMap.entrySet()) { - var id = file.getKey(); - var fileMeta = file.getValue(); - - // don't do duplicates - if (seen.contains(id)) { - continue; - } - - seen.add(id); - - var result = dataverseClient.file(id).updateMetadata(fileMeta); - log.debug("Called updateFileMetadata for id = {}; result = {} {}", id, result.getHttpResponse().getCode(), result.getHttpResponse().getReasonPhrase()); - } - } - } - - private void deleteFiles(DatasetApi api, Set fileDeletions) throws IOException, DataverseException { - for (var id : fileDeletions) { - log.debug("Deleting file, databaseId = {}", id); - dataverseClient.sword().deleteFile(id); - api.awaitUnlock(); - } - } - - private Set getFileDeletions(Set pathsToDelete, Map pathToFileInfoInLatestVersion) { - return pathsToDelete.stream() - .map(p -> pathToFileInfoInLatestVersion.get(p).getDataFile().getId()) - .collect(Collectors.toSet()); - } - - Set diff(Collection a, Collection b) { - return a.stream().filter(k -> !b.contains(k)).collect(Collectors.toSet()); - } - - Set intersection(Collection a, Collection b) { - return a.stream().filter(b::contains).collect(Collectors.toSet()); - } - - Set union(Collection a, Collection b) { - return Stream.of(a.stream(), b.stream()).flatMap(i -> i).collect(Collectors.toSet()); - // a.stream().filter(b::contains).collect(Collectors.toSet()); - } - - private Map replaceFiles(DatasetApi api, Map filesToReplace) throws IOException, DataverseException { - var results = new HashMap(); - - for (var entry : filesToReplace.entrySet()) { - log.debug("Replacing file with ID = {}", entry.getKey()); - var fileApi = dataverseClient.file(entry.getKey()); - var filePath = entry.getValue().getPhysicalPath(); - - var wrappedZip = zipFileHandler.wrapIfZipFile(filePath); - var file = wrappedZip.orElse(filePath); - - var meta = new FileMeta(); - meta.setForceReplace(true); - meta.setLabel(file.getFileName().toString()); - var result = fileApi.replaceFile(file, meta); - - if (wrappedZip.isPresent()) { - try { - Files.deleteIfExists(wrappedZip.get()); - } - catch (IOException e) { - log.error("Unable to delete zipfile {}", wrappedZip.get(), e); - } - } - - var id = -1; - - try { - id = result.getData().getFiles().get(0).getDataFile().getId(); - } - // TODO figure out what kind of exceptions can be thrown here, besides DataverseException and NPE - catch (Throwable e) { - log.error("Unable to get ID from result", e); - throw new IllegalStateException("Could not get ID of replacement file after replace action", e); - } - - results.put(id, entry.getValue().getMetadata()); - api.awaitUnlock(); - } - - return results; - } - - private Map getFilesToReplace(Map pathToFileInfo, Map fileReplacementCandidates) { - - var intersection = intersection(pathToFileInfo.keySet(), fileReplacementCandidates.keySet()); - - log.debug("Intersection paths for replacing = {}", intersection); - - return intersection.stream() - .filter(p -> !pathToFileInfo.get(p).getChecksum().equals(fileReplacementCandidates.get(p).getDataFile().getChecksum().getValue())) - .map(p -> Map.entry(fileReplacementCandidates.get(p).getDataFile().getId(), pathToFileInfo.get(p))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - /** - * Creatings a mapping for moving files to a new location. To determine this, the file needs to be unique in the old and the new version, because its checksum is used to locate it. Files that - * occur multiple times in either the old or the new version cannot be moved in this way. They will appear to have been deleted in the old version and added in the new. This has the same net - * result, except that the "Changes" overview in Dataverse does not record that the file was effectively moved. - * - * @param pathToFileMetaInLatestVersion map from path to file metadata in the old version - * @param pathToFileInfo map from path to file info in the new version (i.e. the deposit). - * @return - */ - Map getOldToNewPathOfFilesToMove(Map pathToFileMetaInLatestVersion, Map pathToFileInfo) { - - var depositChecksums = pathToFileInfo.entrySet().stream() - .map(e -> Map.entry(e.getKey(), e.getValue().getChecksum())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - var latestFileChecksums = pathToFileMetaInLatestVersion.entrySet().stream() - .map(e -> Map.entry(e.getKey(), e.getValue().getDataFile().getChecksum().getValue())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - var checksumsToPathNonDuplicatedFilesInDeposit = getChecksumsToPathOfNonDuplicateFiles(depositChecksums); - var checksumsToPathNonDuplicatedFilesInLatestVersion = getChecksumsToPathOfNonDuplicateFiles(latestFileChecksums); - - var intersects = checksumsToPathNonDuplicatedFilesInDeposit.keySet().stream() - .filter(checksumsToPathNonDuplicatedFilesInLatestVersion::containsKey) - .collect(Collectors.toSet()); - - return intersects.stream() - .map(c -> Map.entry(checksumsToPathNonDuplicatedFilesInLatestVersion.get(c), checksumsToPathNonDuplicatedFilesInDeposit.get(c))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - } - - private Map getChecksumsToPathOfNonDuplicateFiles(Map pathToChecksum) { - // inverse map first - var inversed = pathToChecksum.entrySet().stream() - .collect(Collectors.groupingBy(Map.Entry::getValue, Collectors.mapping(Map.Entry::getKey, Collectors.toList()))); - - // filter out items with 0 or more than 1 items - return inversed.entrySet().stream() - .filter(item -> item.getValue().size() == 1) - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().get(0))); - } - - private void validateFileMetas(Map pathToFileInfoInLatestVersion) { - // check for nulls - for (var fileMeta : pathToFileInfoInLatestVersion.values()) { - if (fileMeta.getDataFile() == null) { - throw new IllegalArgumentException("Found file metadata without dataFile element"); - } - } - - // check if any of them have a checksum that is not SHA-1restrictedFilesPresent - for (var fileMeta : pathToFileInfoInLatestVersion.values()) { - var checksumType = fileMeta.getDataFile().getChecksum().getType(); - log.trace("Filemeta checksum type for file {}: {}", fileMeta.getLabel(), checksumType); - if (!checksumType.equals("SHA-1")) { - throw new IllegalArgumentException("Not all file checksums are of type SHA-1"); - } - } - } - - private Map getFilesInfoInLatestVersion(DatasetApi datasetApi) throws IOException, DataverseException { - // N.B. If LATEST_PUBLISHED is not specified, it almost works, but the directoryLabel is not picked up somehow. - // N.B.2 it still returns an empty (null) directoryLabel if there is no directoryLabel (file is in root of file structure) - var response = datasetApi.getFiles(Version.LATEST_PUBLISHED.toString()); - - return response.getData().stream() - .map(item -> { - try { - log.trace("File item = {}", objectMapper.writeValueAsString(item)); - var path = Path.of( - Optional.ofNullable(item.getDirectoryLabel()).orElse(""), - item.getLabel() - ); - return Map.entry(path, item); - } - catch (Exception e) { - log.error("Error converting json", e); - return null; - } - }) - .filter(Objects::nonNull) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/ManifestHelper.java b/src/main/java/nl/knaw/dans/ingest/core/service/ManifestHelper.java deleted file mode 100644 index 63813b3..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/service/ManifestHelper.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.service; - -import gov.loc.repository.bagit.domain.Bag; - -import java.io.IOException; -import java.security.NoSuchAlgorithmException; - -public interface ManifestHelper { - void ensureSha1ManifestPresent(Bag bag) throws NoSuchAlgorithmException, IOException; - -} diff --git a/src/main/java/nl/knaw/dans/ingest/core/service/ZipFileHandler.java b/src/main/java/nl/knaw/dans/ingest/core/service/ZipFileHandler.java deleted file mode 100644 index 099c6af..0000000 --- a/src/main/java/nl/knaw/dans/ingest/core/service/ZipFileHandler.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package nl.knaw.dans.ingest.core.service; - -import lombok.extern.slf4j.Slf4j; -import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.model.ZipParameters; -import net.lingala.zip4j.model.enums.CompressionMethod; -import org.apache.tika.Tika; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; - -@Slf4j -public class ZipFileHandler { - - private final Tika tika = new Tika(); - private final Path tempDir; - - private final Set needToBeZipWrapped = Set.of( - "application/zip", - "application/zipped-shapefile", - "application/fits-gzipped" - ); - - public Path zipOriginalMetadata(Path... xml) throws IOException { - - var tempFilePath = tempDir.resolve(String.format("original-metadata-%s.zip", UUID.randomUUID())); - - try (var zip = new ZipFile(tempFilePath.toFile())) { - for(var path: xml){ - zip.addFile(path.toFile(), zipWithoutCompressing()); - } - } - return tempFilePath; - } - - public ZipFileHandler(Path tempDir) { - this.tempDir = tempDir; - } - - public Optional wrapIfZipFile(Path path) throws IOException { - if (!needsToBeWrapped(path)) { - return Optional.empty(); - } - - var filename = Optional.ofNullable(path.getFileName()) - .map(Path::toString) - .orElse(""); - - var randomName = String.format("zip-wrapped-%s-%s.zip", - filename, UUID.randomUUID()); - - var tempFile = tempDir.resolve(randomName); - - try (var zip = new ZipFile(tempFile.toFile())) { - zip.addFile(path.toFile(), zipWithoutCompressing()); - } - - return Optional.of(tempFile); - } - - private ZipParameters zipWithoutCompressing() { - var params = new ZipParameters(); - params.setCompressionMethod(CompressionMethod.STORE); - return params; - } - - boolean needsToBeWrapped(Path path) throws IOException { - var endsWithZip = Optional.ofNullable(path.getFileName()) - .map(Path::toString) - .map(x -> x.endsWith(".zip")) - .orElse(false); - - log.trace("Checking if path {} needs to be wrapped: endsWithZip={}", path, endsWithZip); - - return endsWithZip || needToBeZipWrapped.contains(getMimeType(path)); - } - - String getMimeType(Path path) throws IOException { - String result = tika.detect(path); - log.trace("MimeType of path {} is {}", path, result); - return result; - } -} diff --git a/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FileUploadInclusionPredicateTest.java b/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FileUploadInclusionPredicateTest.java index 17af4b4..b2f6631 100644 --- a/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FileUploadInclusionPredicateTest.java +++ b/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FileUploadInclusionPredicateTest.java @@ -33,32 +33,6 @@ public void setUp() { editFiles = new EditFiles(); } - @Test - public void ignored_files_are_skipped_for_unrestricted_file_upload() throws Exception { - // Given - editFiles.setIgnoreFiles(List.of("file1")); - var fileUploadInclusionPredicate = new FileUploadInclusionPredicate(editFiles, dataDir.toPath(), false); - - // When - var result = fileUploadInclusionPredicate.evaluate(new File("dataDir/file1")); - - // Then - assertThat(result).isFalse(); - } - - @Test - public void ignored_files_are_skipped_for_restricted_file_upload() throws Exception { - // Given - editFiles.setIgnoreFiles(List.of("file1")); - var fileUploadInclusionPredicate = new FileUploadInclusionPredicate(editFiles, dataDir.toPath(), true); - - // When - var result = fileUploadInclusionPredicate.evaluate(new File("dataDir/file1")); - - // Then - assertThat(result).isFalse(); - } - @Test public void restricted_files_are_included_for_restricted_file_upload() throws Exception { // Given @@ -114,9 +88,10 @@ public void replaced_files_are_skipped_for_restricted_file_upload() throws Excep @Test public void unrestricted_files_are_included_for_unrestricted_file_upload() throws Exception { // Given - // Default is unrestricted upload + editFiles.setAddUnrestrictedFiles(List.of("file1")); var fileUploadInclusionPredicate = new FileUploadInclusionPredicate(editFiles, dataDir.toPath(), false); + // When var result = fileUploadInclusionPredicate.evaluate(new File("dataDir/file1")); @@ -125,7 +100,7 @@ public void unrestricted_files_are_included_for_unrestricted_file_upload() throw } @Test - public void all_files_are_included_if_editFiles_is_null_for_unrestricted_file_upload() throws Exception { + public void all_files_are_excluded_if_editFiles_is_null_for_unrestricted_file_upload() throws Exception { // Given var fileUploadInclusionPredicate = new FileUploadInclusionPredicate(null, dataDir.toPath(), false); @@ -133,7 +108,7 @@ public void all_files_are_included_if_editFiles_is_null_for_unrestricted_file_up var result = fileUploadInclusionPredicate.evaluate(new File("dataDir/file1")); // Then - assertThat(result).isTrue(); + assertThat(result).isFalse(); } @Test diff --git a/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorTest.java b/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorTest.java index 05d8a79..5987fd7 100644 --- a/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorTest.java +++ b/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditorTest.java @@ -77,7 +77,7 @@ public void deleteFiles_deletes_files_from_dataset() throws Exception { // Then Mockito.verify(dataverseServiceMock).deleteFile(1); Mockito.verify(dataverseServiceMock).deleteFile(3); - assertThat(filesEditor.getFilesInDataset()).containsOnlyKeys("file2"); + assertThat(filesEditor.getFilesInDatasetCache().get("file1")).isNull(); } @Test @@ -97,4 +97,6 @@ public void deleteFiles_throws_exception_when_file_not_found() throws Exception .hasMessage("File to delete not found in dataset: file4"); } + + } diff --git a/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesInDatasetCacheNonNullTest.java b/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesInDatasetCacheNonNullTest.java new file mode 100644 index 0000000..cb7c76b --- /dev/null +++ b/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesInDatasetCacheNonNullTest.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.bagprocessor; + +import nl.knaw.dans.dvingest.core.service.DataverseService; +import nl.knaw.dans.lib.dataverse.model.file.FileMeta; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class FilesInDatasetCacheNonNullTest { + private final DataverseService dataverseServiceMock = Mockito.mock(DataverseService.class); + + @Test + public void constructor_throws_exception_when_dataverseService_is_null() { + assertThatThrownBy(() -> new FilesInDatasetCache(null, Map.of())) + .isInstanceOf(NullPointerException.class); + } + + @Test + public void constructor_throws_exception_when_autoRenamedFiles_is_null() { + assertThatThrownBy(() -> new FilesInDatasetCache(dataverseServiceMock, null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + public void get_throws_exception_when_filepath_is_null() { + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of()); + assertThatThrownBy(() -> filesInDatasetCache.get(null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + public void put_throws_exception_when_fileMeta_is_null() { + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of()); + assertThatThrownBy(() -> filesInDatasetCache.put(null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + public void createFileMetaForMovedFile_throws_exception_when_toPath_is_null() { + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of()); + var fileMeta = new FileMeta(); + assertThatThrownBy(() -> filesInDatasetCache.createFileMetaForMovedFile(null, fileMeta)) + .isInstanceOf(NullPointerException.class); + } + + @Test + public void createFileMetaForMovedFile_throws_exception_when_fileMeta_is_null() { + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of()); + assertThatThrownBy(() -> filesInDatasetCache.createFileMetaForMovedFile("path", null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + public void remove_throws_exception_when_filepath_is_null() { + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of()); + assertThatThrownBy(() -> filesInDatasetCache.remove(null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + public void downloadFromDataset_throws_exception_when_pid_is_null() { + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of()); + assertThatThrownBy(() -> filesInDatasetCache.downloadFromDataset(null)) + .isInstanceOf(NullPointerException.class); + } +} \ No newline at end of file diff --git a/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesInDatasetCacheTest.java b/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesInDatasetCacheTest.java new file mode 100644 index 0000000..6520f21 --- /dev/null +++ b/src/test/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesInDatasetCacheTest.java @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.bagprocessor; + +import nl.knaw.dans.dvingest.core.service.DataverseService; +import nl.knaw.dans.lib.dataverse.model.file.FileMeta; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class FilesInDatasetCacheTest { + private final DataverseService dataverseServiceMock = Mockito.mock(DataverseService.class); + + @BeforeEach + public void setUp() throws Exception { + Mockito.reset(dataverseServiceMock); + } + + @Test + public void get_returns_fileMeta_by_filepath() { + // Given + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of()); + var fileMeta = new FileMeta(); + fileMeta.setLabel("label"); + fileMeta.setDirectoryLabel("directoryLabel"); + + // When + filesInDatasetCache.put(fileMeta); + + // Then + assertThat(filesInDatasetCache.get("directoryLabel/label")).isEqualTo(fileMeta); + } + + @Test + public void get_returns_null_for_nonexistent_filepath() { + // Given + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of()); + + // When + var result = filesInDatasetCache.get("nonexistent/filepath"); + + // Then + assertThat(result).isNull(); + } + + @Test + public void put_adds_fileMeta_to_cache() { + // Given + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of()); + var fileMeta = new FileMeta(); + fileMeta.setLabel("label"); + fileMeta.setDirectoryLabel("directoryLabel"); + + // When + filesInDatasetCache.put(fileMeta); + + // Then + assertThat(filesInDatasetCache.get("directoryLabel/label")).isEqualTo(fileMeta); + } + + @Test + public void remove_deletes_fileMeta_from_cache() { + // Given + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of()); + var fileMeta = new FileMeta(); + fileMeta.setLabel("label"); + fileMeta.setDirectoryLabel("directoryLabel"); + filesInDatasetCache.put(fileMeta); + + // When + filesInDatasetCache.remove("directoryLabel/label"); + + // Then + assertThat(filesInDatasetCache.get("directoryLabel/label")).isNull(); + } + + @Test + public void downloadFromDataset_initializes_cache() throws Exception { + // Given + var fileMeta = new FileMeta(); + fileMeta.setLabel("label"); + fileMeta.setDirectoryLabel("directoryLabel"); + Mockito.when(dataverseServiceMock.getFiles("pid")).thenReturn(java.util.List.of(fileMeta)); + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of()); + + // When + filesInDatasetCache.downloadFromDataset("pid"); + + // Then + assertThat(filesInDatasetCache.get("directoryLabel/label")).isEqualTo(fileMeta); + } + + @Test + public void downloadFromDataset_throws_exception_if_already_initialized() throws Exception { + // Given + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of()); + filesInDatasetCache.downloadFromDataset("pid"); + + // When / Then + assertThatThrownBy(() -> filesInDatasetCache.downloadFromDataset("pid")) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Cache already initialized"); + } + + @Test + public void createFileMetaForMovedFile_updates_fileMeta_with_new_path() { + // Given + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of("oldPath/file.txt", "newPath/file.txt")); + var fileMeta = new FileMeta(); + fileMeta.setLabel("file.txt"); + fileMeta.setDirectoryLabel("oldPath"); + + // When + var updatedFileMeta = filesInDatasetCache.createFileMetaForMovedFile("newPath/file.txt", fileMeta); + + // Then + assertThat(updatedFileMeta.getLabel()).isEqualTo("file.txt"); + assertThat(updatedFileMeta.getDirectoryLabel()).isEqualTo("newPath"); + } + + @Test + public void get_returns_fileMeta_by_old_name_after_rename() { + // Given + var filesInDatasetCache = new FilesInDatasetCache(dataverseServiceMock, Map.of("oldPath/file.txt", "newPath/file.txt")); + var fileMeta = new FileMeta(); + fileMeta.setLabel("file.txt"); + fileMeta.setDirectoryLabel("newPath"); + filesInDatasetCache.put(fileMeta); + + // When + var result = filesInDatasetCache.get("oldPath/file.txt"); + + // Then + assertThat(result).isEqualTo(fileMeta); + } + +} \ No newline at end of file diff --git a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceTest.java b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceTest.java new file mode 100644 index 0000000..34176c5 --- /dev/null +++ b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceTest.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag; + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + +import java.nio.file.Paths; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +public class DansBagMappingServiceTest extends DansConversionFixture { + + @Test + public void getUpdatesDataset_finds_null_for_non_update_deposit() throws Exception { + // Given + var testDepositDir = testDir.resolve("00000000-0000-0000-0000-000000000001"); + FileUtils.copyDirectory(Paths.get("src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001").toFile(), testDepositDir.toFile()); + + // When / Then + assertThat(mappingService.getUpdatesDataset(testDepositDir)).isNull(); + } + + @Test + public void getUpdatesDataset_finds_dataset_for_update_deposit() throws Exception { + // Given + var doi = "doi:10.5072/dans-2xg-4yf"; + when(dataverseServiceMock.findDoiByMetadataField("dansSwordToken", "sword:00000000-0000-0000-0000-000000000001")).thenReturn(List.of(doi)); + var testDepositDir = testDir.resolve("00000000-0000-0000-0000-000000000002"); + FileUtils.copyDirectory(Paths.get("src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002").toFile(), testDepositDir.toFile()); + + // When / Then + assertThat(mappingService.getUpdatesDataset(testDepositDir)).isEqualTo(doi); + } + + + + + +} diff --git a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansConversionFixture.java b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansConversionFixture.java new file mode 100644 index 0000000..4005b8b --- /dev/null +++ b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansConversionFixture.java @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag; + +import gov.loc.repository.bagit.reader.BagReader; +import nl.knaw.dans.dvingest.core.TestDirFixture; +import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDepositReader; +import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDepositReaderImpl; +import nl.knaw.dans.dvingest.core.dansbag.mapper.DepositToDvDatasetMetadataMapper; +import nl.knaw.dans.dvingest.core.dansbag.xml.XmlReader; +import nl.knaw.dans.dvingest.core.dansbag.xml.XmlReaderImpl; +import nl.knaw.dans.dvingest.core.service.DataverseService; +import nl.knaw.dans.lib.dataverse.model.dataset.CompoundMultiValueField; +import nl.knaw.dans.lib.dataverse.model.dataset.ControlledMultiValueField; +import nl.knaw.dans.lib.dataverse.model.dataset.ControlledSingleValueField; +import nl.knaw.dans.lib.dataverse.model.dataset.License; +import nl.knaw.dans.lib.dataverse.model.dataset.MetadataField; +import nl.knaw.dans.lib.dataverse.model.dataset.PrimitiveMultiValueField; +import nl.knaw.dans.lib.dataverse.model.dataset.PrimitiveSingleValueField; +import nl.knaw.dans.lib.util.MappingLoader; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.BeforeEach; +import org.mockito.Mockito; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; + +public abstract class DansConversionFixture extends TestDirFixture { + protected final DataverseService dataverseServiceMock = Mockito.mock(DataverseService.class); + protected DansBagDepositReader dansBagDepositReader; + protected DansBagMappingService mappingService; + + @BeforeEach + public void setUp() throws Exception { + super.setUp(); + BagReader bagReader = new BagReader(); + XmlReader xmlReader = new XmlReaderImpl(); + + dansBagDepositReader = new DansBagDepositReaderImpl(xmlReader, bagReader); + var defaultConfigDir = Paths.get("src/main/assembly/dist/cfg"); + var mapper = new DepositToDvDatasetMetadataMapper( + false, + false, // Always false ? + Set.of("citation", "dansRights", "dansRelationMetadata", "dansArchaeologyMetadata", "dansTemporalSpatial", "dansDataVaultMetadata"), + MappingLoader.builder().csvFile(defaultConfigDir.resolve("iso639-1-to-dv.csv")).keyColumn("ISO639-1").valueColumn("Dataverse-language").build().load(), + MappingLoader.builder().csvFile(defaultConfigDir.resolve("iso639-2-to-dv.csv")).keyColumn("ISO639-2").valueColumn("Dataverse-language").build().load(), + MappingLoader.builder().csvFile(defaultConfigDir.resolve("abr-report-code-to-term.csv")).keyColumn("code").valueColumn("subject").build().load(), + MappingLoader.builder().csvFile(defaultConfigDir.resolve("verwervingswijzen-code-to-term.csv")).keyColumn("code").valueColumn("subject").build().load(), + MappingLoader.builder().csvFile(defaultConfigDir.resolve("abr-complextype-code-to-term.csv")).keyColumn("code").valueColumn("subject").build().load(), + MappingLoader.builder().csvFile(defaultConfigDir.resolve("abr-artifact-code-to-term.csv")).keyColumn("code").valueColumn("subject").build().load(), + MappingLoader.builder().csvFile(defaultConfigDir.resolve("abr-period-code-to-term.csv")).keyColumn("code").valueColumn("subject").build().load(), + FileUtils.readLines(defaultConfigDir.resolve("spatial-coverage-country-terms.txt").toFile(), StandardCharsets.UTF_8), + Collections.emptyMap(), + List.of()); + var supportedLicenses = new SupportedLicenses(licenses("http://opensource.org/licenses/MIT")); + mappingService = new DansBagMappingServiceImpl(mapper, dataverseServiceMock, supportedLicenses, Pattern.compile("a^"), List.of()); // never match + + Mockito.reset(dataverseServiceMock); + } + + private Map licenses(String... uri) { + var licenses = new HashMap(); + for (String s : uri) { + var license = new License(); + license.setUri(URI.create(s)); + licenses.put(license.getUri(), license); + } + return licenses; + } + + protected Path createValidDeposit(String validExample, String depositDir) throws Exception { + var deposit = testDir.resolve(depositDir); + Files.createDirectory(deposit); + // Create deposit.properties + var props = new Properties(); + props.setProperty("state.label", "SUBMITTED"); + props.setProperty("state.description", "Deposit is submitted"); + props.setProperty("deposit.origin", "SWORD"); + props.setProperty("creation.timestamp", DateTimeFormatter.ISO_INSTANT + .withZone(ZoneId.of("UTC")) + .format(Instant.now())); + props.setProperty("depositor.userId", "jdoe"); + try (var out = Files.newBufferedWriter(deposit.resolve("deposit.properties"))) { + props.store(out, null); + } + FileUtils.copyDirectoryToDirectory(Paths.get("target/test/example-bags/valid").resolve(validExample).toFile(), deposit.toFile()); + return deposit; + } + + protected void assertPrimitiveSinglevalueFieldContainsValue(List fields, String typeName, String value) { + assertThat(fields) + .filteredOn(f -> typeName.equals(f.getTypeName())) + .map(f -> (PrimitiveSingleValueField) f).extracting(PrimitiveSingleValueField::getValue) + .containsExactly(value); + } + + protected void assertPrimitiveMultiValueFieldContainsValues(List fields, String typeName, String... values) { + assertThat(fields) + .filteredOn(f -> typeName.equals(f.getTypeName())) + .map(f -> (PrimitiveMultiValueField) f).extracting(PrimitiveMultiValueField::getValue) + .containsExactly(List.of(values)); + } + + protected void assertControlledSingleValueFieldContainsValue(List fields, String typeName, String value) { + assertThat(fields) + .filteredOn(f -> typeName.equals(f.getTypeName())) + .map(f -> (ControlledSingleValueField) f).extracting(ControlledSingleValueField::getValue) + .containsExactly(value); + } + + protected void assertControlledMultiValueFieldContainsValues(List fields, String typeName, String... values) { + assertThat(fields) + .filteredOn(f -> typeName.equals(f.getTypeName())) + .map(f -> (ControlledMultiValueField) f).extracting(ControlledMultiValueField::getValue) + .containsExactly(List.of(values)); + } + + @SafeVarargs + protected final void assertCompoundMultiValueFieldContainsValues(List fields, String typeName, Map... expectedValues) { + var filteredFields = fields.stream() + .filter(f -> typeName.equals(f.getTypeName())) + .map(f -> (CompoundMultiValueField) f) + .toList(); + + assertThat(filteredFields).as("Field not found: " + typeName).isNotEmpty(); + assertThat(filteredFields).as("Field appearing more than once: " + typeName).hasSize(1); + + var actualValues = filteredFields.get(0).getValue(); + assertThat(actualValues).as("Different number of actual and expected values: " + actualValues.size() + " vs " + expectedValues.length).hasSize(expectedValues.length); + + List> actualValuesList = new ArrayList<>(); + for (var actualValue : actualValues) { + var actualValueMap = actualValue.entrySet().stream() + .map(e -> Map.entry(e.getKey(), e.getValue().getValue())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + actualValuesList.add(actualValueMap); + } + + assertThat(actualValuesList).containsExactlyInAnyOrderElementsOf(List.of(expectedValues)); + } +} diff --git a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverterTest.java b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverterTest.java index 0168275..7ce4573 100644 --- a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverterTest.java +++ b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverterTest.java @@ -15,114 +15,57 @@ */ package nl.knaw.dans.dvingest.core.dansbag; -import gov.loc.repository.bagit.reader.BagReader; -import nl.knaw.dans.lib.util.MappingLoader; -import nl.knaw.dans.dvingest.core.TestDirFixture; -import nl.knaw.dans.dvingest.core.service.DataverseService; import nl.knaw.dans.dvingest.core.service.YamlService; import nl.knaw.dans.dvingest.core.service.YamlServiceImpl; -import nl.knaw.dans.ingest.core.deposit.BagDirResolver; -import nl.knaw.dans.ingest.core.deposit.BagDirResolverImpl; -import nl.knaw.dans.ingest.core.deposit.DepositFileLister; -import nl.knaw.dans.ingest.core.deposit.DepositFileListerImpl; -import nl.knaw.dans.ingest.core.deposit.DepositReader; -import nl.knaw.dans.ingest.core.deposit.DepositReaderImpl; -import nl.knaw.dans.ingest.core.io.BagDataManager; -import nl.knaw.dans.ingest.core.io.BagDataManagerImpl; -import nl.knaw.dans.ingest.core.io.FileService; -import nl.knaw.dans.ingest.core.io.FileServiceImpl; -import nl.knaw.dans.ingest.core.service.ManifestHelper; -import nl.knaw.dans.ingest.core.service.ManifestHelperImpl; -import nl.knaw.dans.ingest.core.service.XmlReader; -import nl.knaw.dans.ingest.core.service.XmlReaderImpl; -import nl.knaw.dans.ingest.core.service.mapper.DepositToDvDatasetMetadataMapper; -import nl.knaw.dans.lib.dataverse.model.dataset.License; +import nl.knaw.dans.lib.dataverse.model.dataset.Dataset; import nl.knaw.dans.lib.dataverse.model.user.AuthenticatedUser; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; -import java.util.regex.Pattern; import static org.assertj.core.api.Assertions.assertThat; -public class DansDepositConverterTest extends TestDirFixture { - private final DataverseService dataverseServiceMock = Mockito.mock(DataverseService.class); +public class DansDepositConverterTest extends DansConversionFixture { - private DepositReader depositReader; - private DansBagMappingService mappingService; private final YamlService yamlService = new YamlServiceImpl(); - @BeforeEach - public void setUp() throws Exception { - super.setUp(); - BagReader bagReader = new BagReader(); - ManifestHelper manifestHelper = new ManifestHelperImpl(); - DepositFileLister depositFileLister = new DepositFileListerImpl(); - BagDataManager bagDataManager = new BagDataManagerImpl(bagReader); - XmlReader xmlReader = new XmlReaderImpl(); - FileService fileService = new FileServiceImpl(); - BagDirResolver bagDirResolver = new BagDirResolverImpl(fileService); - - depositReader = new DepositReaderImpl(xmlReader, bagDirResolver, fileService, bagDataManager, depositFileLister, manifestHelper); - var defaultConfigDir = Paths.get("src/main/assembly/dist/cfg"); - var mapper = new DepositToDvDatasetMetadataMapper( - false, - false, // Always false ? - Set.of("citation", "dansRights", "dansRelationMetadata", "dansArchaeologyMetadata", "dansTemporalSpatial", "dansDataVaultMetadata"), - MappingLoader.builder().csvFile(defaultConfigDir.resolve("iso639-1-to-dv.csv")).keyColumn("ISO639-1").valueColumn("Dataverse-language").build().load(), - MappingLoader.builder().csvFile(defaultConfigDir.resolve("iso639-2-to-dv.csv")).keyColumn("ISO639-2").valueColumn("Dataverse-language").build().load(), - MappingLoader.builder().csvFile(defaultConfigDir.resolve("abr-report-code-to-term.csv")).keyColumn("code").valueColumn("subject").build().load(), - MappingLoader.builder().csvFile(defaultConfigDir.resolve("verwervingswijzen-code-to-term.csv")).keyColumn("code").valueColumn("subject").build().load(), - MappingLoader.builder().csvFile(defaultConfigDir.resolve("abr-complextype-code-to-term.csv")).keyColumn("code").valueColumn("subject").build().load(), - MappingLoader.builder().csvFile(defaultConfigDir.resolve("abr-artifact-code-to-term.csv")).keyColumn("code").valueColumn("subject").build().load(), - MappingLoader.builder().csvFile(defaultConfigDir.resolve("abr-period-code-to-term.csv")).keyColumn("code").valueColumn("subject").build().load(), - FileUtils.readLines(defaultConfigDir.resolve("spatial-coverage-country-terms.txt").toFile(), StandardCharsets.UTF_8), - Collections.emptyMap(), - List.of()); - var supportedLicenses = new SupportedLicenses(licenses("http://opensource.org/licenses/MIT")); - mappingService = new DansBagMappingServiceImpl(mapper, dataverseServiceMock, supportedLicenses, Pattern.compile("a^"), List.of()); // never match - - Mockito.reset(dataverseServiceMock); - } - - private Map licenses(String... uri) { - var licenses = new HashMap(); - for (String s : uri) { - var license = new License(); - license.setUri(URI.create(s)); - licenses.put(license.getUri(), license); - } - return licenses; - } - @Test public void run_converts_dans_sword_all_mappings_example_to_dataverse_ingest_deposit() throws Exception { // Given - FileUtils.copyDirectoryToDirectory(Paths.get("src/test/resources/unit-test/d0919038-9866-49e8-986a-bcef54ae7566").toFile(), testDir.toFile()); - var depositDir = testDir.resolve("d0919038-9866-49e8-986a-bcef54ae7566"); - var deposit = depositReader.readDeposit(depositDir); + var depositDir = createValidDeposit("all-mappings", "00000000-0000-0000-0000-000000000001"); var authenticatedUser = new AuthenticatedUser(); authenticatedUser.setFirstName("John"); authenticatedUser.setLastName("Doe"); authenticatedUser.setEmail("jdoe@foo.com"); + authenticatedUser.setDisplayName("John Doe"); Mockito.when(dataverseServiceMock.getUserById(Mockito.anyString())).thenReturn(Optional.of(authenticatedUser)); + var deposit = dansBagDepositReader.readDeposit(depositDir); // When - new DansDepositConverter(deposit, mappingService, yamlService).run(); + new DansDepositConverter(deposit, null, mappingService, yamlService).run(); // Then assertThat(deposit.getBagDir().resolve("dataset.yml")).exists(); + var datasetYml = yamlService.readYaml(deposit.getBagDir().resolve("dataset.yml"), Dataset.class); + var citationBlockFields = datasetYml.getDatasetVersion().getMetadataBlocks().get("citation").getFields(); + // Find the metadata field with property typeName = "title" + assertPrimitiveSinglevalueFieldContainsValue(citationBlockFields, "title", "A bag containing examples for each mapping rule"); + assertPrimitiveMultiValueFieldContainsValues(citationBlockFields, "alternativeTitle", "DCTERMS title 1"); + assertCompoundMultiValueFieldContainsValues(citationBlockFields, "datasetContact", Map.of( + "datasetContactName", "John Doe", + "datasetContactEmail", "jdoe@foo.com" + )); + assertCompoundMultiValueFieldContainsValues(citationBlockFields, "otherId", + Map.of("otherIdAgency", "", "otherIdValue", "DCTERMS_ID001"), + Map.of("otherIdAgency", "", "otherIdValue", "DC_ID002"), + Map.of("otherIdAgency", "", "otherIdValue", "DCTERMS_ID003"), + Map.of("otherIdAgency", "TESTPREFIX", "otherIdValue", "1234")); + + + // TODO: CHECK ALL THE OTHER FIELDS + } } diff --git a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerFixture.java b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerFixture.java new file mode 100644 index 0000000..91ee33a --- /dev/null +++ b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerFixture.java @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag; + +import nl.knaw.dans.dvingest.core.bagprocessor.DataversePath; +import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo; +import nl.knaw.dans.dvingest.core.yaml.EditFiles; +import nl.knaw.dans.lib.dataverse.model.file.Checksum; +import nl.knaw.dans.lib.dataverse.model.file.DataFile; +import nl.knaw.dans.lib.dataverse.model.file.FileMeta; + +import java.nio.file.Path; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EditFilesComposerFixture { + protected static final Instant inThePast = Instant.parse("2010-01-01T00:00:00Z"); + protected final Instant inTheFuture = Instant.now().plus(1, ChronoUnit.DAYS); + protected EditFilesComposer editFilesComposer; + + /* + * Helper methods to set things up. + */ + protected FileInfo file(String path, String checksum, boolean restricted, String description, List categories) { + var fileMeta = new FileMeta(); + var dataversePath = new DataversePath(path); + fileMeta.setLabel(dataversePath.getLabel()); + fileMeta.setDirectoryLabel(dataversePath.getDirectoryLabel()); + fileMeta.setRestrict(restricted); + if (description != null) { + fileMeta.setDescription(description); + } + if (categories != null) { + fileMeta.setCategories(categories); + } + return new FileInfo(Path.of(path), checksum, false, fileMeta); + } + + private FileInfo sanitizedFile(String path, String sanitizedPath, String checksum, boolean restricted, String description, List categories) { + var fileMeta = new FileMeta(); + var dataversePath = new DataversePath(sanitizedPath); + fileMeta.setLabel(dataversePath.getLabel()); + fileMeta.setDirectoryLabel(dataversePath.getDirectoryLabel()); + fileMeta.setRestrict(restricted); + if (description != null) { + fileMeta.setDescription(description); + } + if (categories != null) { + fileMeta.setCategories(categories); + } + return new FileInfo(Path.of(path), checksum, true, fileMeta); + } + + protected FileInfo sanitizedFile(String path, String sanitizedPath, String checksum) { + return sanitizedFile(path, sanitizedPath, checksum, false, null, null); + } + + protected FileInfo file(String path, String checksum) { + return file(path, checksum, false, null, null); + } + + protected FileInfo file(String path, String checksum, boolean restricted) { + return file(path, checksum, restricted, null, null); + } + + protected FileInfo file(String path, String checksum, boolean restricted, String description) { + return file(path, checksum, restricted, description, null); + } + + protected void add(Map map, FileInfo fileInfo) { + map.put(fileInfo.getPath(), fileInfo); + } + + protected void assertEmptyFieldsExcept(EditFiles editFiles, String... except) { + List exceptList = List.of(except); + if (!exceptList.contains("addRestrictedFiles")) { + assertThat(editFiles.getAddRestrictedFiles()).as("expected addRestrictedFiles to remain empty").isEmpty(); + } + if (!exceptList.contains("addUnrestrictedFiles")) { + assertThat(editFiles.getAddUnrestrictedFiles()).as("expected addUnrestrictedFiles to remain empty").isEmpty(); + } + if (!exceptList.contains("autoRenameFiles")) { + assertThat(editFiles.getAutoRenameFiles()).as("expected autoRenameFiles to remain empty").isEmpty(); + } + if (!exceptList.contains("updateFileMetas")) { + assertThat(editFiles.getUpdateFileMetas()).as("expected updateFileMetas to remain empty").isEmpty(); + } + if (!exceptList.contains("addEmbargoes")) { + assertThat(editFiles.getAddEmbargoes()).as("expected addEmbargoes to remain empty").isEmpty(); + } + if (!exceptList.contains("deleteFiles")) { + assertThat(editFiles.getDeleteFiles()).as("expected deleteFiles to remain empty").isEmpty(); + } + if (!exceptList.contains("moveFiles")) { + assertThat(editFiles.getMoveFiles()).as("expected moveFiles to remain empty").isEmpty(); + } + } + + protected FileMeta fileMeta(String path, String checksum) { + var fileMeta = new FileMeta(); + var dataversePath = new DataversePath(path); + fileMeta.setLabel(dataversePath.getLabel()); + fileMeta.setDirectoryLabel(dataversePath.getDirectoryLabel()); + var dataFile = new DataFile(); + var cs = new Checksum(); + cs.setType("SHA-1"); + cs.setValue(checksum); + dataFile.setChecksum(cs); + dataFile.setFilename(dataversePath.getLabel()); + // No directoryLabel? + fileMeta.setDataFile(dataFile); + return fileMeta; + } +} diff --git a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdateTest.java b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdateTest.java new file mode 100644 index 0000000..a6ff5ce --- /dev/null +++ b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdateTest.java @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag; + +import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo; +import nl.knaw.dans.dvingest.core.service.DataverseService; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class EditFilesComposerForUpdateTest extends EditFilesComposerFixture { + private final DataverseService dataverseServiceMock = mock(DataverseService.class); + + @Test + public void file_with_same_path_and_different_checksum_is_replaced() throws Exception { + // Given + when(dataverseServiceMock.getFiles(anyString())).thenReturn(List.of(fileMeta("file1.txt", "oldchecksum"))); + Map map = new HashMap<>(); + add(map, file("file1.txt", "newchecksum")); + + editFilesComposer = new EditFilesComposerForUpdate(map, inThePast, "doi:some", null, List.of(), dataverseServiceMock); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + assertThat(editFiles.getReplaceFiles()).hasSize(1); + assertThat(editFiles.getReplaceFiles().get(0)).isEqualTo("file1.txt"); + + assertEmptyFieldsExcept(editFiles, "replaceFiles"); + } + + @Test + public void file_with_same_path_and_same_checksum_is_NOT_replaced() throws Exception { + // Given + when(dataverseServiceMock.getFiles(anyString())).thenReturn(List.of(fileMeta("file1.txt", "oldchecksum"))); + Map map = new HashMap<>(); + add(map, file("file1.txt", "oldchecksum")); + + editFilesComposer = new EditFilesComposerForUpdate(map, inThePast, "doi:some", null, List.of(), dataverseServiceMock); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + assertThat(editFiles.getReplaceFiles()).isEmpty(); + + assertEmptyFieldsExcept(editFiles); + } + + @Test + public void file_with_different_path_and_same_checksum_is_moved() throws Exception { + // Given + when(dataverseServiceMock.getFiles(anyString())).thenReturn(List.of(fileMeta("path/to/file1.txt", "oldchecksum"))); + Map map = new HashMap<>(); + add(map, file("path/three/file2.txt", "oldchecksum")); + + editFilesComposer = new EditFilesComposerForUpdate(map, inThePast, "doi:some", null, List.of(), dataverseServiceMock); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + assertThat(editFiles.getMoveFiles()).hasSize(1); + assertThat(editFiles.getMoveFiles().get(0).getFrom()).isEqualTo("path/to/file1.txt"); + assertThat(editFiles.getMoveFiles().get(0).getTo()).isEqualTo("path/three/file2.txt"); + + assertEmptyFieldsExcept(editFiles, "moveFiles"); + } + + @Test + public void unrestricted_file_with_different_path_and_different_checksum_is_added() throws Exception { + // Given + when(dataverseServiceMock.getFiles(anyString())).thenReturn(List.of(fileMeta("path/to/file1.txt", "oldchecksum"))); + Map map = new HashMap<>(); + add(map, file("path/to/file1.txt", "oldchecksum")); // Confirming that the file is to remain in the dataset + add(map, file("path/three/file2.txt", "newchecksum")); + editFilesComposer = new EditFilesComposerForUpdate(map, inThePast, "doi:some", null, List.of(), dataverseServiceMock); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + assertThat(editFiles.getAddUnrestrictedFiles()).hasSize(1); + assertThat(editFiles.getAddUnrestrictedFiles()).contains("path/three/file2.txt"); + + assertEmptyFieldsExcept(editFiles, "addUnrestrictedFiles"); + } + + @Test + public void restricted_file_with_different_path_and_different_checksum_is_added() throws Exception { + // Given + when(dataverseServiceMock.getFiles(anyString())).thenReturn(List.of(fileMeta("path/to/file1.txt", "oldchecksum"))); + Map map = new HashMap<>(); + add(map, file("path/to/file1.txt", "oldchecksum")); // Confirming that the file is to remain in the dataset + add(map, file("path/three/file2.txt", "newchecksum", true)); + editFilesComposer = new EditFilesComposerForUpdate(map, inThePast, "doi:some", null, List.of(), dataverseServiceMock); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + assertThat(editFiles.getAddRestrictedFiles()).hasSize(1); + assertThat(editFiles.getAddRestrictedFiles()).contains("path/three/file2.txt"); + + assertEmptyFieldsExcept(editFiles, "addRestrictedFiles"); + } + + @Test + public void ambiguous_move_is_implemented_add_delete_and_add() throws Exception { + // Given + when(dataverseServiceMock.getFiles(anyString())).thenReturn(List.of(fileMeta("path/to/file1.txt", "oldchecksum"))); + Map map = new HashMap<>(); + add(map, file("path/three/file1.txt", "oldchecksum")); + add(map, file("path/three/file2.txt", "oldchecksum")); + + editFilesComposer = new EditFilesComposerForUpdate(map, inThePast, "doi:some", null, List.of(), dataverseServiceMock); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + assertThat(editFiles.getDeleteFiles()).hasSize(1); + assertThat(editFiles.getDeleteFiles().get(0)).isEqualTo("path/to/file1.txt"); + assertThat(editFiles.getAddUnrestrictedFiles()).hasSize(2); + assertThat(editFiles.getAddUnrestrictedFiles()).contains("path/three/file1.txt", "path/three/file2.txt"); + + assertEmptyFieldsExcept(editFiles, "deleteFiles", "addUnrestrictedFiles"); + } + + @Test + public void file_not_replaced_nor_in_current_deposit_is_deleted() throws Exception { + // Given + when(dataverseServiceMock.getFiles(anyString())).thenReturn(List.of(fileMeta("path/to/file1.txt", "oldchecksum"))); + Map map = new HashMap<>(); + + editFilesComposer = new EditFilesComposerForUpdate(map, inThePast, "doi:some", null, List.of(), dataverseServiceMock); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + assertThat(editFiles.getDeleteFiles()).hasSize(1); + assertThat(editFiles.getDeleteFiles().get(0)).isEqualTo("path/to/file1.txt"); + + assertEmptyFieldsExcept(editFiles, "deleteFiles"); + } + + @Test + public void file_with_same_path_and_checksum_is_not_touched() throws Exception { + // Given + when(dataverseServiceMock.getFiles(anyString())).thenReturn(List.of(fileMeta("path/to/file1.txt", "oldchecksum"))); + Map map = new HashMap<>(); + add(map, file("path/to/file1.txt", "oldchecksum")); + + editFilesComposer = new EditFilesComposerForUpdate(map, inThePast, "doi:some", null, List.of(), dataverseServiceMock); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + assertEmptyFieldsExcept(editFiles); + } + +} diff --git a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerTest.java b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerTest.java new file mode 100644 index 0000000..444ed09 --- /dev/null +++ b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerTest.java @@ -0,0 +1,214 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag; + +import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo; +import nl.knaw.dans.dvingest.core.yaml.AddEmbargo; +import nl.knaw.dans.lib.dataverse.model.file.FileMeta; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EditFilesComposerTest extends EditFilesComposerFixture { + + @Test + public void adding_two_unrestricted_files_adds_them_to_addUnrestrictedFiles() { + // Given + Map map = new HashMap<>(); + add(map, file("file1.txt", "checksum1")); + add(map, file("file2.txt", "checksum2")); + editFilesComposer = new EditFilesComposer(map, inThePast, null, List.of()); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + var unrestrictedFiles = editFiles.getAddUnrestrictedFiles(); + assertThat(unrestrictedFiles).hasSize(2); + assertThat(unrestrictedFiles).contains("file1.txt", "file2.txt"); + + // Then + assertEmptyFieldsExcept(editFiles, "addUnrestrictedFiles"); + } + + @Test + public void adding_two_restricted_files_adds_them_to_addRestrictedFiles() { + // Given + Map map = new HashMap<>(); + add(map, file("file1.txt", "checksum1", true)); + add(map, file("file2.txt", "checksum2", true)); + add(map, file("file3.txt", "checksum3", false)); + editFilesComposer = new EditFilesComposer(map, inThePast, null, List.of()); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + var addRestrictedFiles = editFiles.getAddRestrictedFiles(); + assertThat(addRestrictedFiles).hasSize(2); + assertThat(addRestrictedFiles).contains("file1.txt", "file2.txt"); + + var unrestrictedFiles = editFiles.getAddUnrestrictedFiles(); + assertThat(unrestrictedFiles).hasSize(1); + assertThat(unrestrictedFiles).contains("file3.txt"); + + assertEmptyFieldsExcept(editFiles, "addRestrictedFiles", "addUnrestrictedFiles"); + } + + @Test + public void setting_description_adds_it_to_updateFileMetas() { + // Given + Map map = new HashMap<>(); + add(map, file("file1.txt", "checksum1", false, "description1")); + add(map, file("file2.txt", "checksum2", false, "description2")); + add(map, file("file3.txt", "checksum3", false)); + editFilesComposer = new EditFilesComposer(map, inThePast, null, List.of()); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + var updateFileMetas = editFiles.getUpdateFileMetas(); + assertThat(updateFileMetas).hasSize(2); + assertThat(updateFileMetas).extracting(FileMeta::getDescription).contains("description1", "description2"); + + var unrestrictedFiles = editFiles.getAddUnrestrictedFiles(); + assertThat(unrestrictedFiles).hasSize(3); + assertThat(unrestrictedFiles).contains("file1.txt", "file2.txt", "file3.txt"); + + assertEmptyFieldsExcept(editFiles, "updateFileMetas", "addUnrestrictedFiles"); + } + + @Test + public void setting_categories_adds_them_to_updateFileMetas() { + // Given + Map map = new HashMap<>(); + add(map, file("file1.txt", "checksum1", false, null, List.of("category1"))); + add(map, file("file2.txt", "checksum2", false, null, List.of("category2"))); + add(map, file("file3.txt", "checksum3", false)); + editFilesComposer = new EditFilesComposer(map, inThePast, null, List.of()); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + var updateFileMetas = editFiles.getUpdateFileMetas(); + assertThat(updateFileMetas).hasSize(2); + assertThat(updateFileMetas).extracting(FileMeta::getCategories).contains(List.of("category1"), List.of("category2")); + + var unrestrictedFiles = editFiles.getAddUnrestrictedFiles(); + assertThat(unrestrictedFiles).hasSize(3); + assertThat(unrestrictedFiles).contains("file1.txt", "file2.txt", "file3.txt"); + + assertEmptyFieldsExcept(editFiles, "updateFileMetas", "addUnrestrictedFiles"); + } + + @Test + public void setting_dateAvailable_to_future_date_embargoes_all_files() { + // Given + Map map = new HashMap<>(); + add(map, file("file1.txt", "checksum1")); + add(map, file("file2.txt", "checksum2")); + editFilesComposer = new EditFilesComposer(map, inTheFuture, null, List.of()); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + var addEmbargoes = editFiles.getAddEmbargoes(); + assertThat(addEmbargoes).hasSize(1); // There is only one embargo, covering all files + assertThat(addEmbargoes).extracting(AddEmbargo::getFilePaths).containsExactly(List.of("file1.txt", "file2.txt")); + + var unrestrictedFiles = editFiles.getAddUnrestrictedFiles(); + assertThat(unrestrictedFiles).hasSize(2); + assertThat(unrestrictedFiles).contains("file1.txt", "file2.txt"); + + assertEmptyFieldsExcept(editFiles, "addEmbargoes", "addUnrestrictedFiles"); + } + + @Test + public void sanitized_files_are_add_to_autoRenamedFiles() { + // Given + Map map = new HashMap<>(); + add(map, sanitizedFile("file1.txt", "file1_sanitized.txt", "checksum1")); + add(map, file("file2.txt", "checksum2")); + editFilesComposer = new EditFilesComposer(map, inThePast, null, List.of()); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + var autoRenameFiles = editFiles.getAutoRenameFiles(); + assertThat(autoRenameFiles).hasSize(1); + assertThat(autoRenameFiles.get(0).getFrom()).isEqualTo("file1.txt"); + assertThat(autoRenameFiles.get(0).getTo()).isEqualTo("file1_sanitized.txt"); + + var unrestrictedFiles = editFiles.getAddUnrestrictedFiles(); + assertThat(unrestrictedFiles).hasSize(2); + assertThat(unrestrictedFiles).contains("file1.txt", "file2.txt"); + + assertEmptyFieldsExcept(editFiles, "autoRenameFiles", "addUnrestrictedFiles"); + } + + @Test + public void fileExclusionPattern_ignores_files_matching_pattern() { + // Given + Map map = new HashMap<>(); + add(map, file("file1.txt", "checksum1")); + add(map, file("file2.txt", "checksum2")); + editFilesComposer = new EditFilesComposer(map, inThePast, Pattern.compile("file1.*"), List.of()); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + var addUnrestrictedFiles = editFiles.getAddUnrestrictedFiles(); + assertThat(addUnrestrictedFiles).hasSize(1); + assertThat(addUnrestrictedFiles).contains("file2.txt"); + + // Then + assertEmptyFieldsExcept(editFiles, "addUnrestrictedFiles"); + } + + @Test + public void embargoExclusions_ignores_files_matching_filepaths() { + // Given + Map map = new HashMap<>(); + add(map, file("file1.txt", "checksum1")); + add(map, file("file2.txt", "checksum2")); + add(map, file("subdir/file3.txt", "checksum3")); + editFilesComposer = new EditFilesComposer(map, inTheFuture, null, List.of("file2.txt", "subdir/file3.txt")); + + // When + var editFiles = editFilesComposer.composeEditFiles(); + + // Then + var addEmbargoes = editFiles.getAddEmbargoes(); + assertThat(addEmbargoes).hasSize(1); // There is only one embargo, covering all files + assertThat(addEmbargoes).extracting(AddEmbargo::getFilePaths).containsExactly(List.of("file1.txt")); + + var unrestrictedFiles = editFiles.getAddUnrestrictedFiles(); + assertThat(unrestrictedFiles).hasSize(3); + assertThat(unrestrictedFiles).contains("file1.txt", "file2.txt", "subdir/file3.txt"); + + assertEmptyFieldsExcept(editFiles, "addEmbargoes", "addUnrestrictedFiles"); + } + +} diff --git a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/LightweightBagInfoTest.java b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/LightweightBagInfoTest.java new file mode 100644 index 0000000..b751f86 --- /dev/null +++ b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/LightweightBagInfoTest.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag; + +import nl.knaw.dans.dvingest.core.TestDirFixture; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThat; + +public class LightweightBagInfoTest extends TestDirFixture { + + @Test + public void reads_key_value_pairs_from_bag_info_file() throws Exception { + // Given + var bagInfoTxt = testDir.resolve("bag-info.txt"); + FileUtils.writeStringToFile(bagInfoTxt.toFile(), """ + Bagging-Date: 2021-06-01 + Bag-Size: 12345 + Payload-Oxum: 12345.6 + """, StandardCharsets.UTF_8); + + // When + var bagInfo = new LightweightBagInfo(bagInfoTxt); + + // Then + assertThat(bagInfo.get("Bagging-Date")).isEqualTo("2021-06-01"); + assertThat(bagInfo.get("Bag-Size")).isEqualTo("12345"); + assertThat(bagInfo.get("Payload-Oxum")).isEqualTo("12345.6"); + } + + @Test + public void reads_Is_Version_Of_from_bag_info_file() throws Exception { + // Given + var bagInfoTxt = testDir.resolve("bag-info.txt"); + FileUtils.writeStringToFile(bagInfoTxt.toFile(), """ + Is-Version-Of: urn:nbn:nl:ui:13-4-5-6 + """, StandardCharsets.UTF_8); + + // When + var bagInfo = new LightweightBagInfo(bagInfoTxt); + + // Then + assertThat(bagInfo.get("Is-Version-Of")).isEqualTo("urn:nbn:nl:ui:13-4-5-6"); + } + +} diff --git "a/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/data/probl\303\250me/.txt" "b/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/data/probl\303\250me/.txt" new file mode 100644 index 0000000..f795387 --- /dev/null +++ "b/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/data/probl\303\250me/.txt" @@ -0,0 +1,2 @@ +A problematic name: problème/.txt +The sanitized name: probl_me/_problem_.txt \ No newline at end of file diff --git a/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/dataset.yml b/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/dataset.yml index 7a61515..1dc58e6 100644 --- a/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/dataset.yml +++ b/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/dataset.yml @@ -84,6 +84,11 @@ magna amet ipsum et ipsum dolor do lorem sed sit" multiple: false typeClass: "controlledVocabulary" value: "No" + - typeName: "dansMetadataLanguage" + multiple: true + typeClass: "controlledVocabulary" + value: + - "Dutch" dansRelationMetadata: displayName: "Relation Metadata" name: "dansRelationMetadata" diff --git a/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/edit-files.yml b/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/edit-files.yml index b91ee73..9798b6a 100644 --- a/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/edit-files.yml +++ b/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/edit-files.yml @@ -2,6 +2,16 @@ editFiles: addRestrictedFiles: - 'subdir/file2-restricted.txt' - 'file4-restricted.txt' + addUnrestrictedFiles: + - 'problème/.txt' + - 'subdir/file6-to-be-deleted-in-v2.txt' + - 'subdir/file7-to-be-moved-to-subdir2-in-v2.txt' + - 'file1.txt' + - 'file3-desc.txt' + - 'file5-to-be-replaced-in-v2.txt' updateFileMetas: - label: 'file3-desc.txt' - description: 'Description of file3' \ No newline at end of file + description: 'Description of file3' + autoRenameFiles: + - from: 'problème/.txt' + to: 'probl_me/_problem_.txt' \ No newline at end of file diff --git a/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/manifest-sha512.txt b/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/manifest-sha512.txt index 3b5d7d3..d30e831 100644 --- a/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/manifest-sha512.txt +++ b/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/1/manifest-sha512.txt @@ -1,4 +1,5 @@ 119c19f868a33109852c09d66f6a5c73a7cd52f38325020a461cd94a74edef88709fcbc547d96d0ad9da671260fc42322d177378bad7a285f5df03f8e28f8565 data/file1.txt 612c321523321eaf4ba171402ffec07378748d10b9bc579247367e012d16aa96cf084e8bc64d020ee2fc0db613d130259973b54e0b96ce982373e8e9dbb8745f data/file3-desc.txt 7a9bdc3e760e951ddbc2e910153018ace7f7ada1c277484a4baf4095783576933be5bc07d00b98640273f5ca79acdb853d20fa5a7cb1957c0efb82336e578115 data/subdir/file2-restricted.txt -a7ea02a2bab60f2dcd10939969f2da1b2c179094cc714b92b0e2b38cf0364922e1155948c66c8c48d055b55918d666eeb12daeca0b2c0880fc5d39d214ad4a94 data/file4-restricted.txt \ No newline at end of file +a7ea02a2bab60f2dcd10939969f2da1b2c179094cc714b92b0e2b38cf0364922e1155948c66c8c48d055b55918d666eeb12daeca0b2c0880fc5d39d214ad4a94 data/file4-restricted.txt +ed8a4ec583263446ab6409e93f7f520b947458e0c933d3e6bf8f39037eef395c7764c5456058ebbb23e2593d21ea3919c40224cc7e4714f6027226f4a6078553 data/problème/.txt \ No newline at end of file diff --git a/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/2/data/ignored-file.txt b/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/2/data/ignored-file.txt new file mode 100644 index 0000000..bf65270 --- /dev/null +++ b/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/2/data/ignored-file.txt @@ -0,0 +1 @@ +Ignored \ No newline at end of file diff --git a/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/2/edit-files.yml b/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/2/edit-files.yml index 7bf9e92..2e180ab 100644 --- a/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/2/edit-files.yml +++ b/src/test/resources/test-deposits/072625c6-c2a8-43a6-9f35-f49b2db9435c/2/edit-files.yml @@ -1,4 +1,7 @@ editFiles: + addUnrestrictedFiles: + - 'extra/added/extra-added-1.txt' + - 'extra/added/extra-added-2.txt' replaceFiles: - 'file5-to-be-replaced-in-v2.txt' deleteFiles: diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/deposit.properties b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/deposit.properties new file mode 100644 index 0000000..dd6c26d --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/deposit.properties @@ -0,0 +1,11 @@ +bag-store.bag-id = 00000000-0000-0000-0000-000000000001 +dataverse.bag-id = urn:uuid:00000000-0000-0000-0000-000000000001 +creation.timestamp = 2024-12-04T15:47:37.604306939+01:00 +deposit.origin = SWORD2 +depositor.userId = user001 +state.label = PUBLISHED +state.description = The deposit was successfully ingested in the Data Station and will be automatically archived. +bag-store.bag-name = revision01 +dataverse.sword-token = sword:00000000-0000-0000-0000-000000000001 +identifier.urn = urn:nbn:nl:ui:13-de0f2b1c-2ffc-429b-93a2-151248406eee +identifier.doi = 10.5072/FK2/UMUCKU diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/README.md b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/README.md new file mode 100644 index 0000000..bf73df1 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/README.md @@ -0,0 +1,28 @@ +revisions +========= + +The bags revision01, revision02 and revision03 demonstrate how deposits can update existing datasets: revision01 creates a new +dataset and revision02 and revision03 create versions 2 and 3. + +Revision01 +---------- +Revision01 has a file (fileD) with accessibleToRights set to RESTRICTED_REQUEST. After revision01 is ingested, the +dataset has File Access Requests enabled. No Terms of Access are filled in. They could be filled in but are +not mandatory, as they can be negotiated via a request. + +Revision02 +---------- +In revision 02 fileD has been further restricted to accessibleToRights NONE. This means that access can no longer be +requested. Since File Access Requests is a dataset-level attribute, this mean that the dataset has been closed for +access requests for all its restricted files in all versions including future versions. Opening it again cannot be +done via automated deposit, and requires manual actions by the administrator. + +An alternative way to close the dataset for file access requests is to set ddm:accessRights in dataset.xml to NO_ACCESS. + +In revision02 the available date is set to 2025. If, by the time you are reading this, this is still in the future, this would +embargo any files added to the new version, either as new files or as replacements for existing files. + +Revision03 +---------- +In revision 03 fileD has been reset to RESTRICTED_REQUEST. As explained in the previous section, this does +NOT enable file access request for the dataset again, because this is a dataset-level attribute. diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/bag-info.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/bag-info.txt new file mode 100644 index 0000000..de256d0 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.8.1 +Bagging-Date: 2023-02-21 +Payload-Oxum: 104.8 diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/bagit.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/bagit.txt new file mode 100644 index 0000000..c4aebb4 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file1.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file1.txt new file mode 100644 index 0000000..d88c464 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file1.txt @@ -0,0 +1 @@ +This is file1 \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file2.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file2.txt new file mode 100644 index 0000000..d9a0c4d --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file2.txt @@ -0,0 +1 @@ +This is file2 \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file3.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file3.txt new file mode 100644 index 0000000..9682004 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file3.txt @@ -0,0 +1 @@ +This is file3 \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file4.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file4.txt new file mode 100644 index 0000000..7be7387 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/file4.txt @@ -0,0 +1 @@ +This is file4 \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileA.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileA.txt new file mode 100644 index 0000000..c29eeab --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileA.txt @@ -0,0 +1 @@ +This is fileA \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileB.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileB.txt new file mode 100644 index 0000000..b621ba5 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileB.txt @@ -0,0 +1 @@ +This is fileB \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileC.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileC.txt new file mode 100644 index 0000000..679f99c --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileC.txt @@ -0,0 +1 @@ +This is fileC \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileD.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileD.txt new file mode 100644 index 0000000..6f435fd --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/data/subdir/fileD.txt @@ -0,0 +1 @@ +This is fileD \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/manifest-sha1.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/manifest-sha1.txt new file mode 100644 index 0000000..185a331 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/manifest-sha1.txt @@ -0,0 +1,8 @@ +912dcd8724d99d2ecef62f089ef1b81032845db8 data/file1.txt +02f1b44559a4a4e4d213e694fc082f039732ba5d data/file2.txt +5f0de8456a3e55bc316fdde3c10f10b35e69ddf3 data/file3.txt +27d811fcc8a62981486f473f172e201543178146 data/file4.txt +cfb9ed217d610ad6a32487889f6fe5319056ce6c data/subdir/fileA.txt +4e6867e27887a3df45412b557f561a081e516555 data/subdir/fileB.txt +c9ea1142a3c51817bcc163f6efc23dfdb9b195cd data/subdir/fileC.txt +a58371604201655035ff49711e58a4891536f1d6 data/subdir/fileD.txt \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/metadata/dataset.xml b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/metadata/dataset.xml new file mode 100644 index 0000000..34fcea6 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/metadata/dataset.xml @@ -0,0 +1,39 @@ + + + A bag to demonstrate revisions - 1 + + This series of bags demonstrates how to update datasets with new versions + + + + Dr + I + Lastname + + Example Org + + + + 2015-09-09 + 2015-09-09 + D16300 + D16100 + D16200 + D16400 + D16500 + E16000 + OPEN_ACCESS + + + + http://opensource.org/licenses/MIT + I Lastname + 10.17026/dans-z6y-5y2e + + diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/metadata/files.xml b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/metadata/files.xml new file mode 100644 index 0000000..217b414 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/metadata/files.xml @@ -0,0 +1,29 @@ + + + + text/plain + + + text/plain + + + text/plain + + + text/plain + + + text/plain + + + text/plain + + + text/plain + + + text/plain + RESTRICTED_REQUEST + + + diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/tagmanifest-sha1.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/tagmanifest-sha1.txt new file mode 100644 index 0000000..682b4cc --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000001/revision01/tagmanifest-sha1.txt @@ -0,0 +1,4 @@ +f850910760b637cc7d289d1ce89eabe8d50173aa manifest-sha1.txt +e2924b081506bac23f5fffe650ad1848a1c8ac1d bagit.txt +0d59ec57226a827c75ec1457fb243405194f3d07 metadata/dataset.xml +8d0862b0496d4d067bc594651e6e52574b21da42 metadata/files.xml diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/deposit.properties b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/deposit.properties new file mode 100644 index 0000000..d6606ed --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/deposit.properties @@ -0,0 +1,11 @@ +bag-store.bag-id = 00000000-0000-0000-0000-000000000002 +dataverse.bag-id = urn:uuid:00000000-0000-0000-0000-000000000002 +creation.timestamp = 2024-12-04T15:47:47.293558430+01:00 +deposit.origin = SWORD2 +depositor.userId = user001 +state.label = PUBLISHED +state.description = The deposit was successfully ingested in the Data Station and will be automatically archived. +bag-store.bag-name = revision02 +dataverse.sword-token = sword:00000000-0000-0000-0000-000000000001 +identifier.urn = urn:nbn:nl:ui:13-de0f2b1c-2ffc-429b-93a2-151248406eee +identifier.doi = 10.5072/FK2/UMUCKU diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/bag-info.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/bag-info.txt new file mode 100644 index 0000000..1cd8e55 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/bag-info.txt @@ -0,0 +1,4 @@ +Bag-Software-Agent: bagit.py v1.8.1 +Bagging-Date: 2023-02-21 +Is-Version-Of: urn:uuid:00000000-0000-0000-0000-000000000001 +Payload-Oxum: 252.8 diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/bagit.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/bagit.txt new file mode 100644 index 0000000..c4aebb4 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/file1.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/file1.txt new file mode 100644 index 0000000..d88c464 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/file1.txt @@ -0,0 +1 @@ +This is file1 \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/file2-renamed.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/file2-renamed.txt new file mode 100644 index 0000000..d9a0c4d --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/file2-renamed.txt @@ -0,0 +1 @@ +This is file2 \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/file4.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/file4.txt new file mode 100644 index 0000000..0e49f66 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/file4.txt @@ -0,0 +1 @@ +This is file4 with modifications. \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/from-root-to-subdir.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/from-root-to-subdir.txt new file mode 100644 index 0000000..abee171 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/from-root-to-subdir.txt @@ -0,0 +1 @@ +This file is moved from root dir to subdir between version 2 and 3. \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/fileA.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/fileA.txt new file mode 100644 index 0000000..c29eeab --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/fileA.txt @@ -0,0 +1 @@ +This is fileA \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/fileB-renamed.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/fileB-renamed.txt new file mode 100644 index 0000000..b621ba5 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/fileB-renamed.txt @@ -0,0 +1 @@ +This is fileB \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/fileD.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/fileD.txt new file mode 100644 index 0000000..61424df --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/fileD.txt @@ -0,0 +1 @@ +This is fileD with modifications. \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/from-subdir-to-root.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/from-subdir-to-root.txt new file mode 100644 index 0000000..7905f20 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/data/subdir/from-subdir-to-root.txt @@ -0,0 +1 @@ +This file is moved from subdir to root dir between version 2 and 3. \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/manifest-sha1.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/manifest-sha1.txt new file mode 100644 index 0000000..b49a124 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/manifest-sha1.txt @@ -0,0 +1,8 @@ +38346b95fadd5fce5cc512a8e2b101ec0b4bd766 data/file4.txt +803c3d2458982137364ef85c50f28d904cf8b25e data/subdir/from-subdir-to-root.txt +4e6867e27887a3df45412b557f561a081e516555 data/subdir/fileB-renamed.txt +cfb9ed217d610ad6a32487889f6fe5319056ce6c data/subdir/fileA.txt +271490baca1915b0592d94f2b7b6357fad2b9306 data/from-root-to-subdir.txt +912dcd8724d99d2ecef62f089ef1b81032845db8 data/file1.txt +02f1b44559a4a4e4d213e694fc082f039732ba5d data/file2-renamed.txt +205c6d07fdbc2e456c976f8650da360d2bf68b79 data/subdir/fileD.txt diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/metadata/dataset.xml b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/metadata/dataset.xml new file mode 100644 index 0000000..3b51c27 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/metadata/dataset.xml @@ -0,0 +1,34 @@ + + + A bag to demonstrate revisions - 2 + + Updated description for revision 2 + + + + Dr + I + Lastname + + Example Org + + + + 2020-01-01 + 2025-01-01 + D16300 + OPEN_ACCESS + + + + http://opensource.org/licenses/MIT + I Lastname + 10.17026/dans-z6y-5y2e + + diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/metadata/files.xml b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/metadata/files.xml new file mode 100644 index 0000000..7efb877 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/metadata/files.xml @@ -0,0 +1,33 @@ + + + + text/plain + + + text/plain + + + Content of this file has changed + text/plain + + + text/plain + + + text/plain + + + Content of this file has changed + text/plain + NONE + + + I am in a subdir + text/plain + + + I am in the root dir + text/plain + + + diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/tagmanifest-sha1.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/tagmanifest-sha1.txt new file mode 100644 index 0000000..f57dccd --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000002/revision02/tagmanifest-sha1.txt @@ -0,0 +1,4 @@ +2a05f2af17aceba2307ea93706d83c95dff25511 metadata/dataset.xml +e2924b081506bac23f5fffe650ad1848a1c8ac1d bagit.txt +2206d43e98a0608eec54f2e3feda0dcd0d977eb3 manifest-sha1.txt +389232664ba41566040278ce85d8d8c8e5887a3d metadata/files.xml diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/deposit.properties b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/deposit.properties new file mode 100644 index 0000000..ef50f20 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/deposit.properties @@ -0,0 +1,11 @@ +bag-store.bag-id = 00000000-0000-0000-0000-000000000003 +dataverse.bag-id = urn:uuid:00000000-0000-0000-0000-000000000003 +creation.timestamp = 2024-12-04T15:48:09.387152259+01:00 +deposit.origin = SWORD2 +depositor.userId = user001 +state.label = PUBLISHED +state.description = The deposit was successfully ingested in the Data Station and will be automatically archived. +bag-store.bag-name = revision03 +dataverse.sword-token = sword:00000000-0000-0000-0000-000000000001 +identifier.urn = urn:nbn:nl:ui:13-de0f2b1c-2ffc-429b-93a2-151248406eee +identifier.doi = 10.5072/FK2/UMUCKU diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/bag-info.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/bag-info.txt new file mode 100644 index 0000000..4f45dc4 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/bag-info.txt @@ -0,0 +1,4 @@ +Bag-Software-Agent: bagit.py v1.8.1 +Bagging-Date: 2023-02-21 +Is-Version-Of: urn:uuid:00000000-0000-0000-0000-000000000001 +Payload-Oxum: 299.10 diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/bagit.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/bagit.txt new file mode 100644 index 0000000..c4aebb4 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file1.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file1.txt new file mode 100644 index 0000000..d88c464 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file1.txt @@ -0,0 +1 @@ +This is file1 \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file2-renamed.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file2-renamed.txt new file mode 100644 index 0000000..d9a0c4d --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file2-renamed.txt @@ -0,0 +1 @@ +This is file2 \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file3.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file3.txt new file mode 100644 index 0000000..9682004 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file3.txt @@ -0,0 +1 @@ +This is file3 \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file4.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file4.txt new file mode 100644 index 0000000..4d1f644 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/file4.txt @@ -0,0 +1 @@ +This is file 4 with even more modifications. \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/from-subdir-to-root.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/from-subdir-to-root.txt new file mode 100644 index 0000000..7905f20 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/from-subdir-to-root.txt @@ -0,0 +1 @@ +This file is moved from subdir to root dir between version 2 and 3. \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileA.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileA.txt new file mode 100644 index 0000000..c29eeab --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileA.txt @@ -0,0 +1 @@ +This is fileA \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileB-renamed.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileB-renamed.txt new file mode 100644 index 0000000..b621ba5 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileB-renamed.txt @@ -0,0 +1 @@ +This is fileB \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileC.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileC.txt new file mode 100644 index 0000000..679f99c --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileC.txt @@ -0,0 +1 @@ +This is fileC \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileD.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileD.txt new file mode 100644 index 0000000..5f524a2 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/fileD.txt @@ -0,0 +1 @@ +This is fileD with even more modifications. \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/from-root-to-subdir.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/from-root-to-subdir.txt new file mode 100644 index 0000000..abee171 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/data/subdir/from-root-to-subdir.txt @@ -0,0 +1 @@ +This file is moved from root dir to subdir between version 2 and 3. \ No newline at end of file diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/manifest-sha1.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/manifest-sha1.txt new file mode 100644 index 0000000..a7741e0 --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/manifest-sha1.txt @@ -0,0 +1,10 @@ +02f1b44559a4a4e4d213e694fc082f039732ba5d data/file2-renamed.txt +1a0f96d77513d88bef8fa063163044a6ed5f8566 data/file4.txt +64d5460524fd1d5a259ea65d3292383570967b7c data/subdir/fileD.txt +4e6867e27887a3df45412b557f561a081e516555 data/subdir/fileB-renamed.txt +c9ea1142a3c51817bcc163f6efc23dfdb9b195cd data/subdir/fileC.txt +271490baca1915b0592d94f2b7b6357fad2b9306 data/subdir/from-root-to-subdir.txt +cfb9ed217d610ad6a32487889f6fe5319056ce6c data/subdir/fileA.txt +803c3d2458982137364ef85c50f28d904cf8b25e data/from-subdir-to-root.txt +912dcd8724d99d2ecef62f089ef1b81032845db8 data/file1.txt +5f0de8456a3e55bc316fdde3c10f10b35e69ddf3 data/file3.txt diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/metadata/dataset.xml b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/metadata/dataset.xml new file mode 100644 index 0000000..2dd5e7f --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/metadata/dataset.xml @@ -0,0 +1,34 @@ + + + A bag to demonstrate revisions - 3 + + An yet another description for version 3 + + + + Dr + I + Lastname + + Example Org + + + + 2020-01-01 + 2025-01-01 + D16300 + OPEN_ACCESS + + + + http://opensource.org/licenses/MIT + I Lastname + 10.17026/dans-z6y-5y2e + + diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/metadata/files.xml b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/metadata/files.xml new file mode 100644 index 0000000..ac1179c --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/metadata/files.xml @@ -0,0 +1,39 @@ + + + + text/plain + + + text/plain + + + text/plain + + + Content of this file has changed again + text/plain + + + text/plain + + + text/plain + + + text/plain + + + Content of this file has changed again + text/plain + RESTRICTED_REQUEST + + + I am in the root dir + text/plain + + + I am in the subdir + text/plain + + + diff --git a/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/tagmanifest-sha1.txt b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/tagmanifest-sha1.txt new file mode 100644 index 0000000..dc22fce --- /dev/null +++ b/src/test/resources/unit-test/update-deposits/00000000-0000-0000-0000-000000000003/revision03/tagmanifest-sha1.txt @@ -0,0 +1,4 @@ +9002fadbf0f10e5a58ebd4d66f8d54a2e41c9214 metadata/dataset.xml +e5d5ae25938cfa37c70a7d948453d69d4d5eeae1 manifest-sha1.txt +24b520e80fbc6dbad56a1149470af3638a8d0e7c metadata/files.xml +e2924b081506bac23f5fffe650ad1848a1c8ac1d bagit.txt