diff --git a/docs/test-on-vm.md b/docs/test-on-vm.md new file mode 100644 index 0000000..806cfa8 --- /dev/null +++ b/docs/test-on-vm.md @@ -0,0 +1,329 @@ + +set up terminals +================ + +```bash +gnome-terminal -q --tab --title="Vagrant boxes" --working-directory ~/git/dtap/dd-dtap -- \ + bash -c 'ls; vagrant status; $SHELL' + +gnome-terminal -q --tab --title="dd-validate-dans-bag" --working-directory ~/git/service/data-station/dd-validate-dans-bag -- \ + bash -c 'pwd; ls; sleep 10; start-service.sh; $SHELL' + +gnome-terminal -q --tab --title="dd-ingest-flow DB" --working-directory ~/git/service/data-station/dd-ingest-flow -- \ + bash -c 'start-hsqldb-server.sh; $SHELL' +gnome-terminal -q --tab --title="dd-ingest-flow SERVICE" --working-directory ~/git/service/data-station/dd-ingest-flow -- \ + bash -c 'sleep 10; start-service.sh; $SHELL' +gnome-terminal -q --tab --title="dd-ingest-flow DATA" --working-directory ~/git/service/data-station/dd-ingest-flow/data -- \ + bash -c 'echo ''ignore *data'';tree -I ''*data''; SHELL' +``` + +preparations +============ + + +```bash +cd ~/git/dtap/dd-dtap/ # at least PR 359 for a debug port on dd-manage-deposit +start-preprovisioned-box.py -s dev_archaeology dev_vocabs dev_vaas dev_transfer +deploy.py -m dd-manage-deposit dev_vaas +deploy.py -m dd-manage-deposit dev_archaeology +deploy.py -m dd-ingest-flow dev_archaeology # for an additional sleep in DepositManagerImpl.updateAndMoveDeposit +deploy.py -m dd-vault-ingest-flow dev_vaas # at least PR 13; which also has a dev.md to assemble deposits +deploy.py -m dd-transfer-to-vault dev_transfer +vagrant ssh dev_vaas +``` + +Set `pollingInterval: 50` and the log level in + +```bash +sudo su +vi /etc/opt/dans.knaw.nl/dd-manage-deposit/config.yml +``` + +``` + loggers: + "nl.knaw.dans": + level: DEBUG +``` + + +You might have wrapped other deposits into deposits in `dtap/shared` than `revision*` + +```bash +systemctl restart dd-manage-deposit +cd /var/opt/dans.knaw.nl/tmp/auto-ingest +cp -r `readlink -m /vagrant/shared/*/revision*/..` .. # or the actually wrapped bags +chmod -R 777 ../*7* +``` + +Scenario 1a +=========== + + cd dd-dans-sword2-examples + ./run-deposit.sh Simple https://dev.sword2.archaeology.datastations.nl/collection/1 user001 user001 src/main/resources/example-bags/valid/embargoed/ + +State in the database is not changed from SUBMITTED to PUBLISHED + +``` +curl http://localhost:20355/report +[{"depositor":"user001","depositId":"01a1d38b-6517-4dab-abc9-e0da85db04e7","bagName":"revision01","depositState":"","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5887,"deleted":false}, +{"depositor":"user001","depositId":"a78c2cad-d473-401d-8751-a447ef573b7a","bagName":"revision02","depositState":"","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":4616,"deleted":false}, +{"depositor":"user001","depositId":"3d54d5e9-41f8-49f4-92e1-8296e38e7d16","bagName":"revision03","depositState":"","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5299,"deleted":false}, +{"depositor":"","depositId":"revision03","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1695117265.863579000,"depositUpdateTimestamp":1695117265.863579000,"description":"Bag was not valid according to Profile Version 1.0.0. Violations: \n- [1.1.1] Bag is not valid: /home/jokep/git/service/data-station/dd-vault-ingest-flow/data/inbox/revision03/data/bagit.txt\n- [1.2.1] bag-info.txt does not exist","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed/3d54d5e9-41f8-49f4-92e1-8296e38e7d16","storageInBytes":4974,"deleted":false}, +{"depositor":"user001","depositId":"98cfb44d-f1d8-46ff-bc38-97ec3d3b8b8f","bagName":"embargoed","depositState":"SUBMITTED","depositCreationTimestamp":1695130860.327553000,"depositUpdateTimestamp":1695130860.327553000,"description":"Deposit is valid and ready for post-submission processing","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":3216491,"deleted":true}] +``` + +Scenario 1a +=========== + +On dev_vaas (for 1b: use dev_archaeology, at first sight same results) + +For 01, 02 and 03, execute: +```bash +mv -v `readlink -m ../*/revision01/..` inbox +``` + +``` +grep core.service /var/opt/dans.knaw.nl/log/dd-manage-deposit/dd-manage-deposit.log | sed 's/.*core.service.//' +onFileCreate: '/var/opt/dans.knaw.nl/tmp/auto-ingest/inbox/01a1d38b-6517-4dab-abc9-e0da85db04e7/deposit.properties' +onFileCreate: '/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed/01a1d38b-6517-4dab-abc9-e0da85db04e7/deposit.properties' +onFileDelete: '/var/opt/dans.knaw.nl/tmp/auto-ingest/inbox/01a1d38b-6517-4dab-abc9-e0da85db04e7/deposit.properties' +onFileCreate: '/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed/a78c2cad-d473-401d-8751-a447ef573b7a/deposit.properties' +onFileCreate: '/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed/3d54d5e9-41f8-49f4-92e1-8296e38e7d16/deposit.properties' +onFileCreate: '/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed/3d54d5e9-41f8-49f4-92e1-8296e38e7d16/revision03/deposit.properties' +``` + +It looks like the status is only set when the inbox events are missed. + +One bag accidentally contained a `deposit.properties` in the bag. +The payload could also contain a `deposit.properties` file. +These should be ignored. +https://stackoverflow.com/questions/31026294/visit-only-directories-of-a-specific-depth-with-java-7 + +With a polling interval of 5000 I saw less inbox-delete events than with 100. The create event could also be missed. + +Saw no update events unless I changed deposit-properties manually. + +``` +curl http://localhost:20355/report +[{"depositor":"user001","depositId":"01a1d38b-6517-4dab-abc9-e0da85db04e7","bagName":"revision01","depositState":"","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5887,"deleted":true}, +{"depositor":"user001","depositId":"a78c2cad-d473-401d-8751-a447ef573b7a","bagName":"revision02","depositState":"","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":4616,"deleted":true}, +{"depositor":"user001","depositId":"3d54d5e9-41f8-49f4-92e1-8296e38e7d16","bagName":"revision03","depositState":"","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5299,"deleted":true}, +{"depositor":"","depositId":"revision03","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1694775791.343441000,"depositUpdateTimestamp":1694775791.343441000,"description":"Bag was not valid according to Profile Version 1.0.0. Violations: \n- [1.1.1] Bag is not valid: /home/jokep/git/service/data-station/dd-vault-ingest-flow/data/inbox/revision03/data/bagit.txt\n- [1.2.1] bag-info.txt does not exist","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed/3d54d5e9-41f8-49f4-92e1-8296e38e7d16","storageInBytes":4974,"deleted":true}] +``` + +On dev_transfer: +``` +\c dd_transfer_to_vault +select id,bag_id,creation_time,nbn from transfer_item; +id | bag_id | creation_time | nbn +----+-----------------------------------------------+----------------------------+------------------------------------------------------- +1 | urn:uuid:05070843-a477-4ac4-b67a-8a286efc5b4a | 2023-08-16 18:17:04.632966 | urn:nbn:nl:ui:13-43ed5939-a554-42a2-9c7a-d892ac143f2a +2 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 11:46:39.964773 | urn:nbn:nl:ui:13-c88a1706-f01c-4ed9-bb0b-115f859379db +3 | urn:uuid:a78c2cad-d473-401d-8751-a447ef573b7a | 2023-09-14 11:48:47.863567 | urn:nbn:nl:ui:13-99599f84-7503-48bb-86fe-2882bab3d094 +4 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 11:50:52.242299 | urn:nbn:nl:ui:13-5186ee72-91db-4e1c-a7e0-1b00c3be7225 +``` + +scenario 1b +=========== + + + +scenario 2 +========== + +```shell +mv ../01a1d38b-6517-4dab-abc9-e0da85db04e7 inbox +tree -L 3 +. +├── inbox +└── outbox + ├── failed + ├── processed + │   ├── 01a1d38b-6517-4dab-abc9-e0da85db04e7 + │   └── 05070843-a477-4ac4-b67a-8a286efc5b4a ### came with the VM + └── rejected +curl http://localhost:20355/report +[{"depositor":"user001","depositId":"01a1d38b-6517-4dab-abc9-e0da85db04e7","bagName":"revision01","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5947,"deleted":false}] +mv outbox/processed/01a1d38b-6517-4dab-abc9-e0da85db04e7/ .. +curl http://localhost:20355/report +[{"depositor":"user001","depositId":"01a1d38b-6517-4dab-abc9-e0da85db04e7","bagName":"revision01","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5947,"deleted":true}] +systemctl stop dd-vault-ingest-flow +``` + +Stopping the dd-vault-ingest-flow service and moving the same deposit again to the inbox doesn't change the report, after restart still nothing changes, not even the deleted value. + +Moving the revision03 after the 1st to the inbox: moved to processed and gets two records in the report. + +Creating an empty directory gets immediately moved to rejected as expected. + +``` +[{"depositor":"user001","depositId":"01a1d38b-6517-4dab-abc9-e0da85db04e7","bagName":"revision01","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5947,"deleted":true}, +{"depositor":"user001","depositId":"3d54d5e9-41f8-49f4-92e1-8296e38e7d16","bagName":"revision03","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5359,"deleted":false}, +{"depositor":"","depositId":"revision03","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1694676355.982606000,"depositUpdateTimestamp":1694676355.982606000,"description":"Bag was not valid according to Profile Version 1.0.0. Violations: \n- [1.1.1] Bag is not valid: /home/jokep/git/service/data-station/dd-vault-ingest-flow/data/inbox/revision03/data/bagit.txt\n- [1.2.1] bag-info.txt does not exist","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed/3d54d5e9-41f8-49f4-92e1-8296e38e7d16","storageInBytes":4974,"deleted":false}, +{"depositor":"","depositId":"dummy","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1694676559.653176000,"depositUpdateTimestamp":1694676559.653176000,"description":"Bag directory cannot be null","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/rejected","storageInBytes":72,"deleted":false}] +``` + +Meanwhile, on dev_transfer: + +```psql +\c dd_vault_catalog +select id,bag_id,created from ocfl_object_versions; + id | bag_id | created +----+-----------------------------------------------+---------------------------- + 1 | urn:uuid:05070843-a477-4ac4-b67a-8a286efc5b4a | 2023-08-16 18:17:02.593967 + 2 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 08:57:10.071497 + 3 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 09:12:56.108685 + 4 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 09:25:52.051765 +(4 rows) + +\c dd_transfer_to_vault +select id,bag_id,creation_time,nbn from transfer_item; + id | bag_id | creation_time | nbn +----+-----------------------------------------------+----------------------------+------------------------------------------------------- + 1 | urn:uuid:05070843-a477-4ac4-b67a-8a286efc5b4a | 2023-08-16 18:17:04.632966 | urn:nbn:nl:ui:13-43ed5939-a554-42a2-9c7a-d892ac143f2a + 2 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 08:57:12.32885 | urn:nbn:nl:ui:13-d61e674d-a17e-43ef-89a2-93a2586129cd + 3 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 09:12:57.258831 | urn:nbn:nl:ui:13-30f3f36b-2d50-4586-9296-a39a3cbce545 + 4 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 09:25:52.522885 | urn:nbn:nl:ui:13-c669b61f-e0c9-4aab-9df7-5ca91448de35 +(4 rows) +``` + +Subsequently moving revision2 to the inbox: +``` +curl http://localhost:20355/report +[{"depositor":"user001","depositId":"01a1d38b-6517-4dab-abc9-e0da85db04e7","bagName":"revision01","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5947,"deleted":true}, +{"depositor":"user001","depositId":"3d54d5e9-41f8-49f4-92e1-8296e38e7d16","bagName":"revision03","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5359,"deleted":false}, +{"depositor":"","depositId":"revision03","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1694676355.982606000,"depositUpdateTimestamp":1694676355.982606000,"description":"Bag was not valid according to Profile Version 1.0.0. Violations: \n- [1.1.1] Bag is not valid: /home/jokep/git/service/data-station/dd-vault-ingest-flow/data/inbox/revision03/data/bagit.txt\n- [1.2.1] bag-info.txt does not exist","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed/3d54d5e9-41f8-49f4-92e1-8296e38e7d16","storageInBytes":4974,"deleted":false}, +{"depositor":"","depositId":"dummy","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1694676559.653176000,"depositUpdateTimestamp":1694676559.653176000,"description":"Bag directory cannot be null","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/rejected","storageInBytes":72,"deleted":false}, +{"depositor":"user001","depositId":"a78c2cad-d473-401d-8751-a447ef573b7a","bagName":"revision02","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":4676,"deleted":false}] + +select id,bag_id,creation_time,nbn from transfer_item; + id | bag_id | creation_time | nbn +----+-----------------------------------------------+----------------------------+------------------------------------------------------- + 1 | urn:uuid:05070843-a477-4ac4-b67a-8a286efc5b4a | 2023-08-16 18:17:04.632966 | urn:nbn:nl:ui:13-43ed5939-a554-42a2-9c7a-d892ac143f2a + 2 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 08:57:12.32885 | urn:nbn:nl:ui:13-d61e674d-a17e-43ef-89a2-93a2586129cd + 3 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 09:12:57.258831 | urn:nbn:nl:ui:13-30f3f36b-2d50-4586-9296-a39a3cbce545 + 4 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 09:25:52.522885 | urn:nbn:nl:ui:13-c669b61f-e0c9-4aab-9df7-5ca91448de35 + 5 | urn:uuid:a78c2cad-d473-401d-8751-a447ef573b7a | 2023-09-14 10:14:44.644248 | urn:nbn:nl:ui:13-dd161e82-c444-43a6-903f-f41dd9b2eb89 +(5 rows) +``` +Subsequently moving revision3 from processed to the inbox: + +``` +curl http://localhost:20355/report +[{"depositor":"user001","depositId":"01a1d38b-6517-4dab-abc9-e0da85db04e7","bagName":"revision01","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5947,"deleted":true}, +{"depositor":"","depositId":"revision03","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1694676355.982606000,"depositUpdateTimestamp":1694676355.982606000,"description":"Bag was not valid according to Profile Version 1.0.0. Violations: \n- [1.1.1] Bag is not valid: /home/jokep/git/service/data-station/dd-vault-ingest-flow/data/inbox/revision03/data/bagit.txt\n- [1.2.1] bag-info.txt does not exist","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed/3d54d5e9-41f8-49f4-92e1-8296e38e7d16","storageInBytes":4974,"deleted":false}, +{"depositor":"","depositId":"dummy","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1694676559.653176000,"depositUpdateTimestamp":1694676559.653176000,"description":"Bag directory cannot be null","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/rejected","storageInBytes":72,"deleted":false}, +{"depositor":"user001","depositId":"a78c2cad-d473-401d-8751-a447ef573b7a","bagName":"revision02","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":4676,"deleted":false}, +{"depositor":"user001","depositId":"3d54d5e9-41f8-49f4-92e1-8296e38e7d16","bagName":"revision03","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1694680276.481675000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5359,"deleted":false}] + +select id,bag_id,creation_time,nbn from transfer_item; + id | bag_id | creation_time | nbn +----+-----------------------------------------------+----------------------------+------------------------------------------------------- + 1 | urn:uuid:05070843-a477-4ac4-b67a-8a286efc5b4a | 2023-08-16 18:17:04.632966 | urn:nbn:nl:ui:13-43ed5939-a554-42a2-9c7a-d892ac143f2a + 2 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 08:57:12.32885 | urn:nbn:nl:ui:13-d61e674d-a17e-43ef-89a2-93a2586129cd + 3 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 09:12:57.258831 | urn:nbn:nl:ui:13-30f3f36b-2d50-4586-9296-a39a3cbce545 + 4 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 09:25:52.522885 | urn:nbn:nl:ui:13-c669b61f-e0c9-4aab-9df7-5ca91448de35 + 5 | urn:uuid:a78c2cad-d473-401d-8751-a447ef573b7a | 2023-09-14 10:14:44.644248 | urn:nbn:nl:ui:13-dd161e82-c444-43a6-903f-f41dd9b2eb89 + 6 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 10:31:14.524386 | urn:nbn:nl:ui:13-08eda420-01b1-47a1-add8-648a8f4145f9 + +``` + +and again: +``` +curl http://localhost:20355/report +[{"depositor":"user001","depositId":"01a1d38b-6517-4dab-abc9-e0da85db04e7","bagName":"revision01","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5947,"deleted":true}, +{"depositor":"","depositId":"revision03","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1694676355.982606000,"depositUpdateTimestamp":1694676355.982606000,"description":"Bag was not valid according to Profile Version 1.0.0. Violations: \n- [1.1.1] Bag is not valid: /home/jokep/git/service/data-station/dd-vault-ingest-flow/data/inbox/revision03/data/bagit.txt\n- [1.2.1] bag-info.txt does not exist","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed/3d54d5e9-41f8-49f4-92e1-8296e38e7d16","storageInBytes":4974,"deleted":false}, +{"depositor":"","depositId":"dummy","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1694676559.653176000,"depositUpdateTimestamp":1694676559.653176000,"description":"Bag directory cannot be null","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/rejected","storageInBytes":72,"deleted":false}, +{"depositor":"user001","depositId":"3d54d5e9-41f8-49f4-92e1-8296e38e7d16","bagName":"revision03","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1694680276.481675000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5359,"deleted":false}, +{"depositor":"user001","depositId":"a78c2cad-d473-401d-8751-a447ef573b7a","bagName":"revision02","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1694681298.370201000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":4676,"deleted":false}] + +select id,bag_id,creation_time,nbn from transfer_item; + id | bag_id | creation_time | nbn +----+-----------------------------------------------+----------------------------+------------------------------------------------------- + 1 | urn:uuid:05070843-a477-4ac4-b67a-8a286efc5b4a | 2023-08-16 18:17:04.632966 | urn:nbn:nl:ui:13-43ed5939-a554-42a2-9c7a-d892ac143f2a + 2 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 08:57:12.32885 | urn:nbn:nl:ui:13-d61e674d-a17e-43ef-89a2-93a2586129cd + 3 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 09:12:57.258831 | urn:nbn:nl:ui:13-30f3f36b-2d50-4586-9296-a39a3cbce545 + 4 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 09:25:52.522885 | urn:nbn:nl:ui:13-c669b61f-e0c9-4aab-9df7-5ca91448de35 + 5 | urn:uuid:a78c2cad-d473-401d-8751-a447ef573b7a | 2023-09-14 10:14:44.644248 | urn:nbn:nl:ui:13-dd161e82-c444-43a6-903f-f41dd9b2eb89 + 6 | urn:uuid:01a1d38b-6517-4dab-abc9-e0da85db04e7 | 2023-09-14 10:31:14.524386 | urn:nbn:nl:ui:13-08eda420-01b1-47a1-add8-648a8f4145f9 + 7 | urn:uuid:a78c2cad-d473-401d-8751-a447ef573b7a | 2023-09-14 10:48:15.677044 | urn:nbn:nl:ui:13-5b65c9d1-2062-4078-959f-3fee53a0904f +(7 rows) + + + +mv outbox/processed/*/revision02 inbox +curl http://localhost:20355/report +[{"depositor":"user001","depositId":"01a1d38b-6517-4dab-abc9-e0da85db04e7","bagName":"revision01","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1692200441.390209000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5947,"deleted":true}, +{"depositor":"","depositId":"revision03","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1694676355.982606000,"depositUpdateTimestamp":1694676355.982606000,"description":"Bag was not valid according to Profile Version 1.0.0. Violations: \n- [1.1.1] Bag is not valid: /home/jokep/git/service/data-station/dd-vault-ingest-flow/data/inbox/revision03/data/bagit.txt\n- [1.2.1] bag-info.txt does not exist","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed/3d54d5e9-41f8-49f4-92e1-8296e38e7d16","storageInBytes":4974,"deleted":false}, +{"depositor":"","depositId":"dummy","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1694676559.653176000,"depositUpdateTimestamp":1694676559.653176000,"description":"Bag directory cannot be null","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/rejected","storageInBytes":72,"deleted":false}, +{"depositor":"user001","depositId":"3d54d5e9-41f8-49f4-92e1-8296e38e7d16","bagName":"revision03","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1694680276.481675000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":5359,"deleted":false}, +{"depositor":"user001","depositId":"a78c2cad-d473-401d-8751-a447ef573b7a","bagName":"revision02","depositState":"ACCEPTED","depositCreationTimestamp":1692200441.390209000,"depositUpdateTimestamp":1694681298.370201000,"description":"Deposit accepted","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/processed","storageInBytes":4676,"deleted":false}, +{"depositor":"","depositId":"revision02","bagName":"","depositState":"REJECTED","depositCreationTimestamp":1694682069.832226000,"depositUpdateTimestamp":1694682069.832226000,"description":"Bag was not valid according to Profile Version 1.0.0. Violations: \n- [1.1.1] Bag is not valid: /var/opt/dans.knaw.nl/tmp/auto-ingest/inbox/revision02/data/bagit.txt\n- [1.2.1] bag-info.txt does not exist","location":"/var/opt/dans.knaw.nl/tmp/auto-ingest/outbox/rejected","storageInBytes":4539,"deleted":false}] + +ls -la outbox/*/*/* +-rwxrwxrwx. 1 root root 385 Sep 14 09:12 outbox/processed/01a1d38b-6517-4dab-abc9-e0da85db04e7/deposit.properties +-rw-rw----+ 1 dd-sword2 deposits 387 Aug 16 18:17 outbox/processed/05070843-a477-4ac4-b67a-8a286efc5b4a/deposit.properties +-rwxrwxrwx. 1 root root 385 Sep 14 10:31 outbox/processed/3d54d5e9-41f8-49f4-92e1-8296e38e7d16/deposit.properties +-rwxrwxrwx. 1 root root 385 Sep 14 10:48 outbox/processed/a78c2cad-d473-401d-8751-a447ef573b7a/deposit.properties +-rw-rw----+ 1 dd-vault-ingest-flow deposits 72 Sep 14 09:29 outbox/rejected/dummy/deposit.properties +-rwxrwxrwx. 1 root root 172 Sep 14 08:54 outbox/rejected/revision02/bag-info.txt +-rwxrwxrwx. 1 root root 55 Sep 14 08:54 outbox/rejected/revision02/bagit.txt +-rw-r--r--. 1 dd-vault-ingest-flow dd-vault-ingest-flow 248 Sep 14 11:01 outbox/rejected/revision02/deposit.properties +-rwxrwxrwx. 1 root root 528 Sep 14 08:54 outbox/rejected/revision02/manifest-sha1.txt +-rwxrwxrwx. 1 root root 286 Sep 14 08:54 outbox/rejected/revision02/tagmanifest-sha1.txt + +outbox/processed/01a1d38b-6517-4dab-abc9-e0da85db04e7/revision01: +total 20 +drwxrwxrwx. 4 root root 141 Sep 14 08:54 . +drwxrwxrwx. 3 root root 50 Sep 14 08:54 .. +-rwxrwxrwx. 1 root root 172 Sep 14 08:54 bag-info.txt +-rwxrwxrwx. 1 root root 55 Sep 14 08:54 bagit.txt +drwxrwxrwx. 3 root root 88 Sep 14 08:54 data +-rwxrwxrwx. 1 root root 483 Sep 14 08:54 manifest-sha1.txt +drwxrwxrwx. 2 root root 42 Sep 14 08:54 metadata +-rwxrwxrwx. 1 root root 1543 Sep 14 08:54 README.md +-rwxrwxrwx. 1 root root 286 Sep 14 08:54 tagmanifest-sha1.txt + +outbox/processed/05070843-a477-4ac4-b67a-8a286efc5b4a/all-mappings: +total 32 +drwxrws---+ 4 dd-sword2 deposits 4096 Aug 16 18:17 . +drwxrws---+ 3 dd-sword2 deposits 52 Aug 16 18:17 .. +-rw-rw----+ 1 dd-sword2 deposits 148 Aug 16 18:17 bag-info.txt +-rw-rw----+ 1 dd-sword2 deposits 55 Aug 16 18:17 bagit.txt +drwxrws---+ 2 dd-sword2 deposits 4096 Aug 16 18:17 data +-rw-rw----+ 1 dd-sword2 deposits 251 Aug 16 18:17 manifest-sha1.txt +drwxrws---+ 2 dd-sword2 deposits 42 Aug 16 18:17 metadata +-rw-rw----+ 1 dd-sword2 deposits 219 Aug 16 18:17 original-filepaths.txt +-rw-rw----+ 1 dd-sword2 deposits 253 Aug 16 18:17 README.md +-rw-rw----+ 1 dd-sword2 deposits 353 Aug 16 18:17 tagmanifest-sha1.txt + +outbox/processed/3d54d5e9-41f8-49f4-92e1-8296e38e7d16/revision03: +total 20 +drwxrwxrwx. 4 root root 150 Sep 14 08:54 . +drwxrwxrwx. 3 root root 50 Sep 14 08:54 .. +-rwxrwxrwx. 1 root root 172 Sep 14 08:54 bag-info.txt +-rwxrwxrwx. 1 root root 55 Sep 14 08:54 bagit.txt +drwxrwxrwx. 3 root root 127 Sep 14 08:54 data +-rwxrwxrwx. 1 root root 273 Sep 14 08:54 deposit.properties +-rwxrwxrwx. 1 root root 648 Sep 14 08:54 manifest-sha1.txt +drwxrwxrwx. 2 root root 42 Sep 14 08:54 metadata +-rwxrwxrwx. 1 root root 286 Sep 14 08:54 tagmanifest-sha1.txt + +outbox/rejected/revision02/data: +total 16 +drwxrwxrwx. 3 root root 110 Sep 14 08:54 . +drwxrwxrwx. 4 root root 150 Sep 14 11:01 .. +-rwxrwxrwx. 1 root root 13 Sep 14 08:54 file1.txt +-rwxrwxrwx. 1 root root 13 Sep 14 08:54 file2-renamed.txt +-rwxrwxrwx. 1 root root 33 Sep 14 08:54 file4.txt +-rwxrwxrwx. 1 root root 67 Sep 14 08:54 from-root-to-subdir.txt +drwxrwxrwx. 2 root root 96 Sep 14 08:54 subdir + +outbox/rejected/revision02/metadata: +total 8 +drwxrwxrwx. 2 root root 42 Sep 14 08:54 . +drwxrwxrwx. 4 root root 150 Sep 14 11:01 .. +-rwxrwxrwx. 1 root root 1596 Sep 14 08:54 dataset.xml +-rwxrwxrwx. 1 root root 1402 Sep 14 08:54 files.xml + +``` diff --git a/src/main/java/nl/knaw/dans/managedeposit/core/DepositProperties.java b/src/main/java/nl/knaw/dans/managedeposit/core/DepositProperties.java index 177416a..6e0836d 100644 --- a/src/main/java/nl/knaw/dans/managedeposit/core/DepositProperties.java +++ b/src/main/java/nl/knaw/dans/managedeposit/core/DepositProperties.java @@ -169,4 +169,20 @@ public boolean equals(Object o) { public int hashCode() { return 31 * depositId.hashCode() + depositor.hashCode(); } + + @Override + public String toString() { + return "DepositProperties{" + + "depositor='" + depositor + '\'' + + ", depositId='" + depositId + '\'' + + ", bagName='" + bagName + '\'' + + ", depositState='" + depositState + '\'' + + ", depositCreationTimestamp=" + depositCreationTimestamp + + ", depositUpdateTimestamp=" + depositUpdateTimestamp + + ", description='" + description + '\'' + + ", location='" + location + '\'' + + ", storageInBytes=" + storageInBytes + + ", deleted=" + deleted + + '}'; + } } \ No newline at end of file diff --git a/src/main/java/nl/knaw/dans/managedeposit/core/service/DepositStatusUpdater.java b/src/main/java/nl/knaw/dans/managedeposit/core/service/DepositStatusUpdater.java index 936fa79..46c3231 100644 --- a/src/main/java/nl/knaw/dans/managedeposit/core/service/DepositStatusUpdater.java +++ b/src/main/java/nl/knaw/dans/managedeposit/core/service/DepositStatusUpdater.java @@ -47,12 +47,16 @@ public void onCreateDeposit(File depositPropertiesFile) { Path depositLocationFolder = Path.of(depositPropertiesFile.getParentFile().getParentFile().getAbsolutePath()); Optional updatedNumber = depositPropertiesDAO.updateDepositLocation(dp.get().getDepositId(), depositLocationFolder); if (updatedNumber.isPresent()) - log.debug("onCreateDeposit - `location` of deposit '{}' has been updated to '{}' ", dp.get().getDepositId(), depositLocationFolder); + log.debug("onCreateDeposit - `location` has been updated '{}' ", dp.get()); } else { Optional dpObject = depositPropertiesAssembler.assembleObject(depositPropertiesFile, false); dpObject.ifPresent(depositPropertiesDAO::save); - log.debug("onCreateDeposit: A new deposit has been registered `{}`", depositPropertiesFile.getParentFile().getAbsolutePath()); + if (dpObject.isEmpty()) + log.debug("onChangeDeposit: new directory has been detected but no deposit.properties is found `{}`", dpObject.get()); + else log.debug("onChangeDeposit: deposit.properties has been changed `{}`", dpObject.get()); + + dpObject.ifPresent(depositProperties -> log.debug("onCreateDeposit: A new deposit has been registered `{}`", depositProperties)); } } @@ -60,7 +64,7 @@ public void onCreateDeposit(File depositPropertiesFile) { public void onChangeDeposit(File depositPropertiesFile) { Optional dpObject = depositPropertiesAssembler.assembleObject(depositPropertiesFile, true); dpObject.ifPresent(depositPropertiesDAO::save); - log.debug("onChangeDeposit: deposit.properties has been changed `{}`", depositPropertiesFile.getParentFile().getAbsolutePath()); + dpObject.ifPresent(depositProperties -> log.debug("onChangeDeposit: deposit.properties has been changed `{}`", dpObject.get())); } @UnitOfWork @@ -73,7 +77,7 @@ public void onDeleteDeposit(File depositPropertiesFile) { if (dp.isPresent() && Files.isSameFile(Path.of(dp.get().getLocation()), Path.of(depositPropertiesFile.getParentFile().getParentFile().getAbsolutePath()))) { String depositId = depositPropertiesFile.getParentFile().getName(); Optional deletedNumber = depositPropertiesDAO.updateDeleteFlag(depositId, true); - log.debug("onDeleteDeposit - 'deleted' mark has been set to '{}' for deposit.properties from '{}' ", deletedNumber.isPresent(), depositId); + log.debug("onDeleteDeposit - 'deleted' mark has been set to '{}' for deposit.properties from '{}', found '{}' ", deletedNumber.isPresent(), depositId, dp.get()); } } catch (IOException e) {