From 96c4cf97c5e5da5a46734d805ff780ef0cd62eb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Oct 2021 10:24:20 +0000 Subject: [PATCH 01/43] build(deps): Bump faker from 9.5.1 to 9.5.2 Bumps [faker](https://github.com/joke2k/faker) from 9.5.1 to 9.5.2. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.md) - [Commits](https://github.com/joke2k/faker/compare/v9.5.1...v9.5.2) --- updated-dependencies: - dependency-name: faker dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index 99a771133..2c3469723 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -29,7 +29,7 @@ coveralls # Django # ------------------------------------------------------------------------------ factory-boy==3.2.0 # https://github.com/FactoryBoy/factory_boy -faker==9.5.1 +faker==9.5.2 django-debug-toolbar==3.2.2 # https://github.com/jazzband/django-debug-toolbar django-extensions==3.1.3 # https://github.com/django-extensions/django-extensions From a1337002f5908f8f2e2dd14c7aa7c25c9f618f88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 10:22:49 +0000 Subject: [PATCH 02/43] build(deps): Bump boto3 from 1.19.0 to 1.19.2 Bumps [boto3](https://github.com/boto/boto3) from 1.19.0 to 1.19.2. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.19.0...1.19.2) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 9ed5e507c..e62695ae7 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -55,7 +55,7 @@ gitpython==3.1.24 django-background-tasks==1.2.5 # S3 Uploads -boto3==1.19.0 +boto3==1.19.2 django-storages==1.12.2 # https://github.com/jschneier/django-storages # Python function to stream unzip all the files in a ZIP archive, without loading the entire ZIP file into memory or any of its uncompressed files. From 3cbfaf32b155c69773e72b731653a2bb8de70c8e Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Mon, 25 Oct 2021 11:28:35 -0400 Subject: [PATCH 03/43] Fix incorrect absolute URL in success email --- apps/ingest/mail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/ingest/mail.py b/apps/ingest/mail.py index 60351458c..0ad5fd8d2 100644 --- a/apps/ingest/mail.py +++ b/apps/ingest/mail.py @@ -50,7 +50,7 @@ def send_email_on_success(task_watcher=None): 'admin:manifests_manifest_change', args=(task_watcher.associated_manifest.id,) ) context['manifest_pid'] = task_watcher.associated_manifest.pid - context['volume_url'] = task_watcher.associated_manifest.get_absolute_url() + context['volume_url'] = task_watcher.associated_manifest.get_volume_url() else: context['manifests_list_url'] = settings.HOSTNAME + reverse( 'admin:manifests_manifest_changelist' From ee4cbd98aed667f8d563046ae17c3b6b865e49e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 10:22:04 +0000 Subject: [PATCH 04/43] build(deps): Bump moto from 2.2.10 to 2.2.11 Bumps [moto](https://github.com/spulec/moto) from 2.2.10 to 2.2.11. - [Release notes](https://github.com/spulec/moto/releases) - [Changelog](https://github.com/spulec/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/spulec/moto/compare/2.2.10...2.2.11) --- updated-dependencies: - dependency-name: moto dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index 2c3469723..143947b04 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -18,7 +18,7 @@ httpretty==1.1.4 # https://pypi.org/project/httpretty/ #mock==4.0.2 cssutils==2.3.0 # https://pypi.org/project/cssutils/ pytest-django==4.4.0 # https://github.com/pytest-dev/pytest-django -moto==2.2.10 # https://github.com/spulec/moto +moto==2.2.11 # https://github.com/spulec/moto # Code quality # ------------------------------------------------------------------------------ From cb0cfbdbce0f45d83d42d57256e679558ada6b40 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 10:22:23 +0000 Subject: [PATCH 05/43] build(deps): Bump faker from 9.5.2 to 9.6.0 Bumps [faker](https://github.com/joke2k/faker) from 9.5.2 to 9.6.0. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.md) - [Commits](https://github.com/joke2k/faker/compare/v9.5.2...v9.6.0) --- updated-dependencies: - dependency-name: faker dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index 2c3469723..b31a45165 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -29,7 +29,7 @@ coveralls # Django # ------------------------------------------------------------------------------ factory-boy==3.2.0 # https://github.com/FactoryBoy/factory_boy -faker==9.5.2 +faker==9.6.0 django-debug-toolbar==3.2.2 # https://github.com/jazzband/django-debug-toolbar django-extensions==3.1.3 # https://github.com/django-extensions/django-extensions From 65cee68b92c5a5bcdfcdce3d6997b992e131b047 Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Tue, 26 Oct 2021 14:08:27 -0400 Subject: [PATCH 06/43] PID automigrations --- .../migrations/0014_auto_20211026_1736.py | 18 ++++++++++++++++++ .../migrations/0010_auto_20211026_1736.py | 18 ++++++++++++++++++ .../migrations/0031_auto_20211026_1736.py | 18 ++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 apps/iiif/canvases/migrations/0014_auto_20211026_1736.py create mode 100644 apps/iiif/kollections/migrations/0010_auto_20211026_1736.py create mode 100644 apps/iiif/manifests/migrations/0031_auto_20211026_1736.py diff --git a/apps/iiif/canvases/migrations/0014_auto_20211026_1736.py b/apps/iiif/canvases/migrations/0014_auto_20211026_1736.py new file mode 100644 index 000000000..72f5f3a99 --- /dev/null +++ b/apps/iiif/canvases/migrations/0014_auto_20211026_1736.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.24 on 2021-10-26 17:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('canvases', '0013_auto_20211018_1913'), + ] + + operations = [ + migrations.AlterField( + model_name='canvas', + name='pid', + field=models.CharField(default='2qkw6szd', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255), + ), + ] diff --git a/apps/iiif/kollections/migrations/0010_auto_20211026_1736.py b/apps/iiif/kollections/migrations/0010_auto_20211026_1736.py new file mode 100644 index 000000000..36a440d89 --- /dev/null +++ b/apps/iiif/kollections/migrations/0010_auto_20211026_1736.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.24 on 2021-10-26 17:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('kollections', '0009_auto_20211018_1913'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='pid', + field=models.CharField(default='2qkw6szd', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255), + ), + ] diff --git a/apps/iiif/manifests/migrations/0031_auto_20211026_1736.py b/apps/iiif/manifests/migrations/0031_auto_20211026_1736.py new file mode 100644 index 000000000..b34039be8 --- /dev/null +++ b/apps/iiif/manifests/migrations/0031_auto_20211026_1736.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.24 on 2021-10-26 17:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('manifests', '0030_auto_20211018_1913'), + ] + + operations = [ + migrations.AlterField( + model_name='manifest', + name='pid', + field=models.CharField(default='2qkw6szd', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255), + ), + ] From cf65a8f0422698efd136eaa58f08582bf6555763 Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Tue, 26 Oct 2021 17:23:10 -0400 Subject: [PATCH 07/43] fix: offload batch upload to s3 to celery task --- apps/ingest/admin.py | 45 +++++++++-------- .../migrations/0032_auto_20211026_1736.py | 25 ++++++++++ apps/ingest/models.py | 3 +- apps/ingest/tasks.py | 50 ++++++++++++++++++- .../admin/ingest/bulk/change_form.html | 29 ++++------- apps/ingest/tests/test_admin.py | 3 +- 6 files changed, 112 insertions(+), 43 deletions(-) create mode 100644 apps/ingest/migrations/0032_auto_20211026_1736.py diff --git a/apps/ingest/admin.py b/apps/ingest/admin.py index 4b29bb983..fae157567 100644 --- a/apps/ingest/admin.py +++ b/apps/ingest/admin.py @@ -2,8 +2,8 @@ import logging from mimetypes import guess_type from os import environ, path, remove, listdir, rmdir +from django.core.files.base import ContentFile from django.contrib import admin -from django.core.files.storage import FileSystemStorage from django.shortcuts import redirect from django.urls import reverse from django.utils.html import format_html @@ -83,7 +83,6 @@ class BulkAdmin(admin.ModelAdmin): form = BulkVolumeUploadForm def save_model(self, request, obj, form, change): - form.storage = IngestStorage() obj.save() # Get files from multi upload form files = request.FILES.getlist("volume_files") @@ -100,35 +99,36 @@ def save_model(self, request, obj, form, change): else: file_meta = {} - # Save in storage - bundle_path = form.storage.save( - path.join("bulk", str(obj.id), file.name), file - ) - - # Create local + # Create a Local object new_local = Local.objects.create( bulk=obj, - bundle=bundle_path, image_server=obj.image_server, - metadata=file_meta, creator=request.user ) - new_local.save() - new_local.manifest = create_manifest(new_local) + if file_meta: + new_local.metadata=file_meta + + # Save tempfile in bundle_from_bulk + with ContentFile(file.read()) as file_content: + new_local.bundle_from_bulk.save(file.name, file_content) new_local.save() new_local.refresh_from_db() + + # Queue task to upload to S3 if environ["DJANGO_ENV"] != 'test': - local_task_id = tasks.create_canvas_form_local_task.delay(new_local.id) - local_task_result = TaskResult(task_id=local_task_id) - local_task_result.save() + upload_task = tasks.upload_to_s3_task.delay( + local_id=new_local.id, + file_path=path.join("bulk", str(obj.id), file.name), + user_id=request.user.id, + ) + upload_task_result = TaskResult(task_id=upload_task.id) + upload_task_result.save() IngestTaskWatcher.manager.create_watcher( - task_id=local_task_id, - task_result=local_task_result, + task_id=upload_task.id, + task_result=upload_task_result, task_creator=request.user, - associated_manifest=new_local.manifest, filename=file.name ) - obj.refresh_from_db() super().save_model(request, obj, form, change) @@ -137,11 +137,14 @@ def response_add(self, request, obj, post_url_continue=None): file_path = obj.volume_files.path if path.isfile(file_path): remove(file_path) + else: + LOGGER.error(f"Could not cleanup {file_path}") dir_path = file_path[0:file_path.rindex('/')] - if len(listdir(dir_path)) == 0: + if not path.isfile(file_path) and len(listdir(dir_path)) == 0: rmdir(dir_path) obj.delete() - return redirect("/admin/manifests/manifest/?o=-4") + url_to = reverse('admin:ingest_ingesttaskwatcher_changelist') + return redirect(url_to) class Meta: # pylint: disable=too-few-public-methods, missing-class-docstring model = Bulk diff --git a/apps/ingest/migrations/0032_auto_20211026_1736.py b/apps/ingest/migrations/0032_auto_20211026_1736.py new file mode 100644 index 000000000..02a285ddc --- /dev/null +++ b/apps/ingest/migrations/0032_auto_20211026_1736.py @@ -0,0 +1,25 @@ +# Generated by Django 2.2.24 on 2021-10-26 17:36 + +import apps.ingest.models +import apps.ingest.storages +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ingest', '0031_ingesttaskwatcher_associated_manifest'), + ] + + operations = [ + migrations.AddField( + model_name='local', + name='bundle_from_bulk', + field=models.FileField(null=True, upload_to=apps.ingest.models.bulk_path), + ), + migrations.AlterField( + model_name='local', + name='bundle', + field=models.FileField(null=True, storage=apps.ingest.storages.IngestStorage(), upload_to=''), + ), + ] diff --git a/apps/ingest/models.py b/apps/ingest/models.py index 6c10017db..d32dde1c5 100644 --- a/apps/ingest/models.py +++ b/apps/ingest/models.py @@ -77,7 +77,8 @@ class Meta: class Local(IngestAbstractModel): """ Model class for ingesting a volume from local files. """ bulk = models.ForeignKey(Bulk, related_name='local_uploads', on_delete=models.SET_NULL, null=True) - bundle = models.FileField(blank=False, storage=IngestStorage()) + bundle_from_bulk = models.FileField(null=True, blank=True, upload_to=bulk_path) + bundle = models.FileField(null=True, blank=True, storage=IngestStorage()) image_server = models.ForeignKey(ImageServer, on_delete=models.DO_NOTHING, null=True) creator = models.ForeignKey( settings.AUTH_USER_MODEL, diff --git a/apps/ingest/tasks.py b/apps/ingest/tasks.py index c046f57ae..18aa67376 100644 --- a/apps/ingest/tasks.py +++ b/apps/ingest/tasks.py @@ -2,10 +2,14 @@ """ Common tasks for ingest. """ import logging +from os import path, remove, listdir, rmdir from celery import Celery from celery.signals import task_success, task_failure from django.apps import apps from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.files.base import ContentFile +from django_celery_results.models import TaskResult from apps.ingest.models import IngestTaskWatcher from .services import create_manifest from .mail import send_email_on_failure, send_email_on_success @@ -64,6 +68,50 @@ def create_remote_canvases(ingest_id, *args, **kwargs): remote_ingest.create_canvases() +@app.task(name='uploading_to_s3', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) +def upload_to_s3_task(local_id, file_path, user_id): + """Task to create Canavs objects from remote IIIF manifest + + :param local_id: Primary key for .models.Local object + :type local_id: UUID + :param file_path: File path for uploaded file + :param user_id: Primary key for User object + :type local_id: UUID + """ + local = Local.objects.get(pk=local_id) + # Upload tempfile to S3 + with ContentFile(local.bundle_from_bulk.read()) as file_content: + local.bundle.save(file_path, file_content) + local.save() + + # Create manifest now that we have a file + local.manifest = create_manifest(local) + local.save() + local.refresh_from_db() + + # Delete tempfile + if local.bundle is not None and local.bundle_from_bulk is not None: + old_path = local.bundle_from_bulk.path + if path.isfile(old_path): + remove(old_path) + else: + LOGGER.error(f"Could not find for cleanup: {old_path}") + dir_path = old_path[0:old_path.rindex('/')] + if not path.isfile(old_path) and len(listdir(dir_path)) == 0: + rmdir(dir_path) + + # Queue task to create canvases etc. + local_task_id = create_canvas_form_local_task.delay(local_id) + local_task_result = TaskResult(task_id=local_task_id) + local_task_result.save() + file_name = local.bundle.name + IngestTaskWatcher.manager.create_watcher( + task_id=local_task_id, + task_result=local_task_result, + task_creator=get_user_model().objects.get(pk=user_id), + associated_manifest=local.manifest, + filename=file_name[file_name.rindex('/')+1:] + ) @task_failure.connect def send_email_on_failure_task(sender=None, exception=None, task_id=None, traceback=None, *args, **kwargs): @@ -88,4 +136,4 @@ def send_email_on_success_task(sender=None, **kwargs): task_id = sender.request.id task_watcher = IngestTaskWatcher.manager.get(task_id=task_id) if task_watcher is not None: - send_email_on_success(task_watcher) + send_email_on_success(task_watcher) diff --git a/apps/ingest/templates/admin/ingest/bulk/change_form.html b/apps/ingest/templates/admin/ingest/bulk/change_form.html index 6ef03a4cb..3d779de04 100644 --- a/apps/ingest/templates/admin/ingest/bulk/change_form.html +++ b/apps/ingest/templates/admin/ingest/bulk/change_form.html @@ -61,7 +61,6 @@ const percentage = document.getElementById('overlay-percentage'); const progressBar = document.getElementById('overlay-progress'); const buttonUsed = django.jQuery('#bulk_form').data("callerid"); - let errored = false; // Set up POST req const form = e.currentTarget; @@ -76,36 +75,28 @@ label.textContent = 'Uploading...'; }; - xhr.upload.onprogress = function (e) { + xhr.upload.onprogress = (e) => { const percent = e.lengthComputable ? (e.loaded / e.total) * 100 : 0; progressBar.value = percent.toFixed(2); progressBar.setAttribute('aria-valuenow', percent.toFixed(2)); percentage.textContent = percent.toFixed(2) + '%'; }; - function handleError (e) { - errored = true; - label.textContent = e.type.toString(); + xhr.upload.onloadend = (e) => { + label.textContent = 'Upload complete, please wait...'; } - xhr.upload.addEventListener('abort', handleError); - xhr.upload.addEventListener('error', handleError); - xhr.addEventListener('abort', handleError); - xhr.addEventListener('error', handleError); - - xhr.upload.onloadend = (e) => { - // Redirect to manifests page - if (!errored) { + xhr.onreadystatechange = (e) => { + if (xhr.readyState == XMLHttpRequest.DONE) { if (buttonUsed === 'start_upload') { - location.href = '/admin/manifests/manifest/?o=-4'; + location.href = '/admin/ingest/ingesttaskwatcher/'; } else { location.reload(); } } - }; - + } // Send POST req - xhr.open('POST', url); + xhr.open('POST', url, true); xhr.send(new FormData(form)); }) @@ -123,8 +114,8 @@ You must leave this window open during upload.

- Once upload completes, you will be sent to the list of manifests. - You may navigate away while the rest of the ingest completes; you + Once upload completes, you will be sent to a new page. + You may then navigate away while the rest of the ingest completes; you will receive an email to notify you when the ingest has completed.

diff --git a/apps/ingest/tests/test_admin.py b/apps/ingest/tests/test_admin.py index 5c7a16d63..615e1faae 100644 --- a/apps/ingest/tests/test_admin.py +++ b/apps/ingest/tests/test_admin.py @@ -6,6 +6,7 @@ from django.http import HttpResponseRedirect from django.test import TestCase from django.test.client import RequestFactory +from django.urls.base import reverse from django_celery_results.models import TaskResult from moto import mock_s3 from apps.ingest.forms import BulkVolumeUploadForm @@ -177,7 +178,7 @@ def test_bulk_admin_response_add(self): with self.assertRaises(Bulk.DoesNotExist): bulk.refresh_from_db() assert isinstance(response, HttpResponseRedirect) - assert response.url == '/admin/manifests/manifest/?o=-4' + assert response.url == reverse('admin:ingest_ingesttaskwatcher_changelist') def test_bulk_admin_with_external_metadata(self): """It should add the metadata to the matching Local object""" From f6f9f867a677e566eb74eff9a1b05886690667f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Oct 2021 10:26:11 +0000 Subject: [PATCH 08/43] build(deps): Bump boto3 from 1.19.2 to 1.19.4 Bumps [boto3](https://github.com/boto/boto3) from 1.19.2 to 1.19.4. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.19.2...1.19.4) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index e62695ae7..be0a2af88 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -55,7 +55,7 @@ gitpython==3.1.24 django-background-tasks==1.2.5 # S3 Uploads -boto3==1.19.2 +boto3==1.19.4 django-storages==1.12.2 # https://github.com/jschneier/django-storages # Python function to stream unzip all the files in a ZIP archive, without loading the entire ZIP file into memory or any of its uncompressed files. From 62ba27d74428488e57264c0d3601f5387207cc67 Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Wed, 27 Oct 2021 09:13:03 -0400 Subject: [PATCH 09/43] fix: Error handling in upload overlay --- .../templates/admin/ingest/bulk/change_form.html | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/apps/ingest/templates/admin/ingest/bulk/change_form.html b/apps/ingest/templates/admin/ingest/bulk/change_form.html index 3d779de04..051b8962d 100644 --- a/apps/ingest/templates/admin/ingest/bulk/change_form.html +++ b/apps/ingest/templates/admin/ingest/bulk/change_form.html @@ -86,6 +86,16 @@ label.textContent = 'Upload complete, please wait...'; } + function handleError (e) { + errored = true; + label.textContent = 'Error: Please see dev tools console for details'; + } + + xhr.upload.addEventListener('abort', handleError); + xhr.upload.addEventListener('error', handleError); + xhr.addEventListener('abort', handleError); + xhr.addEventListener('error', handleError); + xhr.onreadystatechange = (e) => { if (xhr.readyState == XMLHttpRequest.DONE) { if (buttonUsed === 'start_upload') { From 7b2ff0525c67ffa4e95699e65dae6a1c71220307 Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Wed, 27 Oct 2021 14:52:06 -0400 Subject: [PATCH 10/43] fix: Move bundle_to_s3 func to Local model, +tests --- apps/ingest/admin.py | 3 -- apps/ingest/models.py | 23 ++++++++++++ apps/ingest/tasks.py | 34 +++++------------- apps/ingest/tests/test_local.py | 64 ++++++++++++++++++++++++++------- 4 files changed, 84 insertions(+), 40 deletions(-) diff --git a/apps/ingest/admin.py b/apps/ingest/admin.py index fae157567..4d6819f77 100644 --- a/apps/ingest/admin.py +++ b/apps/ingest/admin.py @@ -9,7 +9,6 @@ from django.utils.html import format_html from django_celery_results.models import TaskResult from apps.ingest import tasks -from apps.ingest.storages import IngestStorage from .models import Bulk, IngestTaskWatcher, Local, Remote from .services import clean_metadata, create_manifest, get_associated_meta, get_metadata_from from .forms import BulkVolumeUploadForm @@ -118,8 +117,6 @@ def save_model(self, request, obj, form, change): if environ["DJANGO_ENV"] != 'test': upload_task = tasks.upload_to_s3_task.delay( local_id=new_local.id, - file_path=path.join("bulk", str(obj.id), file.name), - user_id=request.user.id, ) upload_task_result = TaskResult(task_id=upload_task.id) upload_task_result.save() diff --git a/apps/ingest/models.py b/apps/ingest/models.py index d32dde1c5..6dc7ea19e 100644 --- a/apps/ingest/models.py +++ b/apps/ingest/models.py @@ -4,6 +4,7 @@ import logging from io import BytesIO from mimetypes import guess_type +from django.core.files.base import ContentFile import httpretty from stream_unzip import stream_unzip, TruncatedDataError from boto3 import client @@ -153,6 +154,28 @@ def volume_to_s3(self): f'{self.manifest.pid}/_*ocr*_/{file_name}' ) + def bundle_to_s3(self): + """Uploads the zipfile stored in bundle_from_bulk to S3 + :param file_path: File path for uploaded file + :type file_path: str + """ + if bool(self.bundle_from_bulk): + # Save to bundle + if not bool(self.bundle): + if not os.path.isfile(self.bundle_from_bulk.path): + raise Exception(f"Could not find file: {self.bundle_from_bulk.path}") + bulk_name = self.bundle_from_bulk.name + with ContentFile(self.bundle_from_bulk.read()) as file_content: + self.bundle.save(bulk_name, file_content) + # Delete tempfile + if bool(self.bundle): + old_path = self.bundle_from_bulk.path + os.remove(old_path) + dir_path = old_path[0:old_path.rindex('/')] + if not os.path.isfile(old_path) and len(os.listdir(dir_path)) == 0: + os.rmdir(dir_path) + self.bundle_from_bulk.delete() + @property def file_list(self): """Returns a list of files in the zip. Used for testing. diff --git a/apps/ingest/tasks.py b/apps/ingest/tasks.py index 18aa67376..bcc07a8cb 100644 --- a/apps/ingest/tasks.py +++ b/apps/ingest/tasks.py @@ -2,7 +2,6 @@ """ Common tasks for ingest. """ import logging -from os import path, remove, listdir, rmdir from celery import Celery from celery.signals import task_success, task_failure from django.apps import apps @@ -69,48 +68,33 @@ def create_remote_canvases(ingest_id, *args, **kwargs): remote_ingest.create_canvases() @app.task(name='uploading_to_s3', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) -def upload_to_s3_task(local_id, file_path, user_id): +def upload_to_s3_task(local_id): """Task to create Canavs objects from remote IIIF manifest :param local_id: Primary key for .models.Local object :type local_id: UUID - :param file_path: File path for uploaded file - :param user_id: Primary key for User object - :type local_id: UUID """ local = Local.objects.get(pk=local_id) + # Upload tempfile to S3 - with ContentFile(local.bundle_from_bulk.read()) as file_content: - local.bundle.save(file_path, file_content) - local.save() + local.bundle_to_s3() # Create manifest now that we have a file - local.manifest = create_manifest(local) - local.save() - local.refresh_from_db() - - # Delete tempfile - if local.bundle is not None and local.bundle_from_bulk is not None: - old_path = local.bundle_from_bulk.path - if path.isfile(old_path): - remove(old_path) - else: - LOGGER.error(f"Could not find for cleanup: {old_path}") - dir_path = old_path[0:old_path.rindex('/')] - if not path.isfile(old_path) and len(listdir(dir_path)) == 0: - rmdir(dir_path) + if local.manifest is None: + local.manifest = create_manifest(local) # Queue task to create canvases etc. + local.save() + local.refresh_from_db() local_task_id = create_canvas_form_local_task.delay(local_id) local_task_result = TaskResult(task_id=local_task_id) local_task_result.save() - file_name = local.bundle.name IngestTaskWatcher.manager.create_watcher( task_id=local_task_id, task_result=local_task_result, - task_creator=get_user_model().objects.get(pk=user_id), + task_creator=get_user_model().objects.get(pk=local.creator.id), associated_manifest=local.manifest, - filename=file_name[file_name.rindex('/')+1:] + filename=local.bundle.name ) @task_failure.connect diff --git a/apps/ingest/tests/test_local.py b/apps/ingest/tests/test_local.py index 16dc318f1..3ed3855f3 100644 --- a/apps/ingest/tests/test_local.py +++ b/apps/ingest/tests/test_local.py @@ -1,17 +1,14 @@ """ Tests for local ingest """ +from os import path, remove +from tempfile import gettempdir import pytest import boto3 from moto import mock_s3 -from shutil import copy -from os.path import exists, join -from tempfile import gettempdir -from uuid import UUID -from unittest import mock +from os.path import join from django.test import TestCase -from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.files.uploadedfile import SimpleUploadedFile, TemporaryUploadedFile from django.conf import settings from apps.iiif.canvases.models import Canvas -from apps.iiif.manifests.models import Manifest from apps.iiif.manifests.tests.factories import ManifestFactory, ImageServerFactory from ..models import Local from ..services import create_manifest @@ -35,16 +32,21 @@ def setUp(self): conn.create_bucket(Bucket=self.image_server.storage_path) conn.create_bucket(Bucket='readux-ingest') - def mock_local(self, bundle, with_manifest=False, metadata={}): + def mock_local(self, bundle, with_manifest=False, metadata={}, from_bulk=False): # Note, I tried to use the factory here, but could not get it to override the file for bundle. local = Local( image_server = self.image_server, metadata = metadata ) - local.bundle = SimpleUploadedFile( - name=bundle, - content=open(join(self.fixture_path, bundle), 'rb').read() - ) + local.save() + file = SimpleUploadedFile( + name=bundle, + content=open(join(self.fixture_path, bundle), 'rb').read() + ) + if from_bulk: + local.bundle_from_bulk.save(bundle, file) + else: + local.bundle = file local.save() @@ -252,3 +254,41 @@ def test_it_creates_mainfest_with_metadata_property(self): local.manifest = create_manifest(local) assert local.manifest.pid == '808' assert local.manifest.title == 'Goodie Mob' + + def test_moving_bulk_bundle_to_s3(self): + """ + It should upload Local.bundle_from_bulk to mock S3 by saving it to + Local.bundle, then it should clean up tempfiles + """ + # Make sure local with from_bulk is mocked correctly + local = self.mock_local('bundle.zip', from_bulk=True) + assert bool(local.bundle_from_bulk) is True + bulk_name = local.bundle_from_bulk.name + bulk_path = local.bundle_from_bulk.path + dir_path = bulk_path[0:bulk_path.rindex('/')] + assert bulk_name[bulk_name.rindex('/')+1:] == 'bundle.zip' + assert path.isfile(bulk_path) is True + assert path.isdir(dir_path) is True + + # Call bundle_to_s3() and test that it uploaded the file + local.bundle_to_s3() + assert bool(local.bundle) is True + assert local.bundle.name[local.bundle.name.rindex('/')+1:] == 'bundle.zip' + assert local.bundle.storage.exists(f'bulk/{local.id}/bundle.zip') # pylint: disable=no-member + + # Test tempfile cleanup + assert bool(local.bundle_from_bulk) is False + assert path.isfile(bulk_path) is False + assert path.isdir(dir_path) is False + + def test_bundle_to_s3_fails_for_deleted_tempfile(self): + """ + It should raise an exception because we deleted the tempfile before + running bundle_to_s3 + """ + local = self.mock_local('bundle.zip', from_bulk=True) + bulk_path = local.bundle_from_bulk.path + assert path.isfile(bulk_path) is True + remove(bulk_path) + assert path.isfile(bulk_path) is False + self.assertRaises(Exception, local.bundle_to_s3) From 9ef7a418604974ed9de6e311a5f0ea78e4afe5d2 Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Wed, 27 Oct 2021 14:53:33 -0400 Subject: [PATCH 11/43] fix(test): Cleanup unused imports --- apps/ingest/tests/test_local.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/ingest/tests/test_local.py b/apps/ingest/tests/test_local.py index 3ed3855f3..0ac55ba79 100644 --- a/apps/ingest/tests/test_local.py +++ b/apps/ingest/tests/test_local.py @@ -1,12 +1,11 @@ """ Tests for local ingest """ from os import path, remove -from tempfile import gettempdir import pytest import boto3 from moto import mock_s3 from os.path import join from django.test import TestCase -from django.core.files.uploadedfile import SimpleUploadedFile, TemporaryUploadedFile +from django.core.files.uploadedfile import SimpleUploadedFile from django.conf import settings from apps.iiif.canvases.models import Canvas from apps.iiif.manifests.tests.factories import ManifestFactory, ImageServerFactory From 2625df8c5da1a9465430bef2ec6b28b7ffc7a3f7 Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Wed, 27 Oct 2021 14:54:20 -0400 Subject: [PATCH 12/43] refactor(test): Consolidate imports --- apps/ingest/tests/test_local.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/ingest/tests/test_local.py b/apps/ingest/tests/test_local.py index 0ac55ba79..733f026a8 100644 --- a/apps/ingest/tests/test_local.py +++ b/apps/ingest/tests/test_local.py @@ -3,7 +3,6 @@ import pytest import boto3 from moto import mock_s3 -from os.path import join from django.test import TestCase from django.core.files.uploadedfile import SimpleUploadedFile from django.conf import settings @@ -20,7 +19,7 @@ class LocalTest(TestCase): """ Tests for ingest.models.Local """ def setUp(self): """ Set instance variables. """ - self.fixture_path = join(settings.APPS_DIR, 'ingest/fixtures/') + self.fixture_path = path.join(settings.APPS_DIR, 'ingest/fixtures/') self.image_server = ImageServerFactory( server_base='http://readux.s3.amazonaws.com', storage_service='s3', @@ -40,7 +39,7 @@ def mock_local(self, bundle, with_manifest=False, metadata={}, from_bulk=False): local.save() file = SimpleUploadedFile( name=bundle, - content=open(join(self.fixture_path, bundle), 'rb').read() + content=open(path.join(self.fixture_path, bundle), 'rb').read() ) if from_bulk: local.bundle_from_bulk.save(bundle, file) From bfd7037b19e6542b440208f88b230c68171b5551 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Oct 2021 19:42:50 +0000 Subject: [PATCH 13/43] build(deps): Bump factory-boy from 3.2.0 to 3.2.1 Bumps [factory-boy](https://github.com/FactoryBoy/factory_boy) from 3.2.0 to 3.2.1. - [Release notes](https://github.com/FactoryBoy/factory_boy/releases) - [Changelog](https://github.com/FactoryBoy/factory_boy/blob/3.2.1/docs/changelog.rst) - [Commits](https://github.com/FactoryBoy/factory_boy/compare/3.2.0...3.2.1) --- updated-dependencies: - dependency-name: factory-boy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index 2150a8d63..d483969ed 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -28,7 +28,7 @@ coveralls # Django # ------------------------------------------------------------------------------ -factory-boy==3.2.0 # https://github.com/FactoryBoy/factory_boy +factory-boy==3.2.1 # https://github.com/FactoryBoy/factory_boy faker==9.6.0 django-debug-toolbar==3.2.2 # https://github.com/jazzband/django-debug-toolbar From 607faf6f0b85f354b0125f146c9b4a3dcbc2dfa9 Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Thu, 28 Oct 2021 14:38:24 -0400 Subject: [PATCH 14/43] feat: Allow attaching collection(s) to bulk ingest --- .../migrations/0015_auto_20211028_1527.py | 18 ++++++ .../migrations/0011_auto_20211028_1527.py | 18 ++++++ .../migrations/0032_auto_20211028_1527.py | 18 ++++++ apps/ingest/admin.py | 7 ++- apps/ingest/forms.py | 2 +- .../migrations/0033_auto_20211028_1527.py | 36 +++++++++++ apps/ingest/models.py | 13 +++- apps/ingest/services.py | 12 +++- apps/ingest/tests/test_admin.py | 61 +++++++++++++++++++ apps/ingest/tests/test_local.py | 4 +- 10 files changed, 181 insertions(+), 8 deletions(-) create mode 100644 apps/iiif/canvases/migrations/0015_auto_20211028_1527.py create mode 100644 apps/iiif/kollections/migrations/0011_auto_20211028_1527.py create mode 100644 apps/iiif/manifests/migrations/0032_auto_20211028_1527.py create mode 100644 apps/ingest/migrations/0033_auto_20211028_1527.py diff --git a/apps/iiif/canvases/migrations/0015_auto_20211028_1527.py b/apps/iiif/canvases/migrations/0015_auto_20211028_1527.py new file mode 100644 index 000000000..9b6e7cf4a --- /dev/null +++ b/apps/iiif/canvases/migrations/0015_auto_20211028_1527.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.24 on 2021-10-28 15:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('canvases', '0014_auto_20211026_1736'), + ] + + operations = [ + migrations.AlterField( + model_name='canvas', + name='pid', + field=models.CharField(default='2qpxp706', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255), + ), + ] diff --git a/apps/iiif/kollections/migrations/0011_auto_20211028_1527.py b/apps/iiif/kollections/migrations/0011_auto_20211028_1527.py new file mode 100644 index 000000000..cb26d18d1 --- /dev/null +++ b/apps/iiif/kollections/migrations/0011_auto_20211028_1527.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.24 on 2021-10-28 15:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('kollections', '0010_auto_20211026_1736'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='pid', + field=models.CharField(default='2qpxp706', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255), + ), + ] diff --git a/apps/iiif/manifests/migrations/0032_auto_20211028_1527.py b/apps/iiif/manifests/migrations/0032_auto_20211028_1527.py new file mode 100644 index 000000000..2e6831216 --- /dev/null +++ b/apps/iiif/manifests/migrations/0032_auto_20211028_1527.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.24 on 2021-10-28 15:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('manifests', '0031_auto_20211026_1736'), + ] + + operations = [ + migrations.AlterField( + model_name='manifest', + name='pid', + field=models.CharField(default='2qpxp706', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255), + ), + ] diff --git a/apps/ingest/admin.py b/apps/ingest/admin.py index 4d6819f77..b99b34954 100644 --- a/apps/ingest/admin.py +++ b/apps/ingest/admin.py @@ -16,7 +16,7 @@ LOGGER = logging.getLogger(__name__) class LocalAdmin(admin.ModelAdmin): """Django admin ingest.models.local resource.""" - fields = ('bundle', 'image_server') + fields = ('bundle', 'image_server', 'collections') show_save_and_add_another = False def save_model(self, request, obj, form, change): @@ -82,6 +82,10 @@ class BulkAdmin(admin.ModelAdmin): form = BulkVolumeUploadForm def save_model(self, request, obj, form, change): + # Save M2M relationships with collections so we can access them later + if form.is_valid(): + form.save(commit=False) + form.save_m2m() obj.save() # Get files from multi upload form files = request.FILES.getlist("volume_files") @@ -104,6 +108,7 @@ def save_model(self, request, obj, form, change): image_server=obj.image_server, creator=request.user ) + new_local.collections.set(obj.collections.all()) if file_meta: new_local.metadata=file_meta diff --git a/apps/ingest/forms.py b/apps/ingest/forms.py index 564e402ca..eb7414535 100644 --- a/apps/ingest/forms.py +++ b/apps/ingest/forms.py @@ -5,7 +5,7 @@ class BulkVolumeUploadForm(forms.ModelForm): class Meta: model = Bulk - fields = ['image_server', 'volume_files'] + fields = ['image_server', 'volume_files', 'collections'] widgets = { 'volume_files': ClearableFileInput(attrs={'multiple': True}), } diff --git a/apps/ingest/migrations/0033_auto_20211028_1527.py b/apps/ingest/migrations/0033_auto_20211028_1527.py new file mode 100644 index 000000000..a10e2faec --- /dev/null +++ b/apps/ingest/migrations/0033_auto_20211028_1527.py @@ -0,0 +1,36 @@ +# Generated by Django 2.2.24 on 2021-10-28 15:27 + +import apps.ingest.models +import apps.ingest.storages +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('kollections', '0011_auto_20211028_1527'), + ('ingest', '0032_auto_20211026_1736'), + ] + + operations = [ + migrations.AddField( + model_name='bulk', + name='collections', + field=models.ManyToManyField(blank=True, help_text='Optional: Collections to attach to ALL volumes ingested in this form.', null=True, to='kollections.Collection'), + ), + migrations.AddField( + model_name='local', + name='collections', + field=models.ManyToManyField(blank=True, help_text='Optional: Collections to attach to the volume ingested in this form.', null=True, to='kollections.Collection'), + ), + migrations.AlterField( + model_name='local', + name='bundle', + field=models.FileField(blank=True, null=True, storage=apps.ingest.storages.IngestStorage(), upload_to=''), + ), + migrations.AlterField( + model_name='local', + name='bundle_from_bulk', + field=models.FileField(blank=True, null=True, upload_to=apps.ingest.models.bulk_path), + ), + ] diff --git a/apps/ingest/models.py b/apps/ingest/models.py index 6dc7ea19e..6af1c2a40 100644 --- a/apps/ingest/models.py +++ b/apps/ingest/models.py @@ -4,7 +4,6 @@ import logging from io import BytesIO from mimetypes import guess_type -from django.core.files.base import ContentFile import httpretty from stream_unzip import stream_unzip, TruncatedDataError from boto3 import client @@ -12,9 +11,11 @@ from django.db import models from django.conf import settings from django.contrib.postgres.fields import JSONField +from django.core.files.base import ContentFile from django_celery_results.models import TaskResult from apps.iiif.canvases.models import Canvas from apps.iiif.canvases.tasks import add_ocr_task +from apps.iiif.kollections.models import Collection from apps.iiif.manifests.models import Manifest, ImageServer from apps.ingest import services from apps.utils.fetch import fetch_url @@ -71,6 +72,11 @@ class Bulk(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) image_server = models.ForeignKey(ImageServer, on_delete=models.DO_NOTHING, null=True) volume_files = models.FileField(blank=False, upload_to=bulk_path) + collections = models.ManyToManyField( + Collection, + blank=True, + help_text="Optional: Collections to attach to ALL volumes ingested in this form." + ) class Meta: verbose_name_plural = 'Bulk' @@ -87,6 +93,11 @@ class Local(IngestAbstractModel): null=True, related_name='created_locals' ) + collections = models.ManyToManyField( + Collection, + blank=True, + help_text="Optional: Collections to attach to the volume ingested in this form." + ) class Meta: verbose_name_plural = 'Local' diff --git a/apps/ingest/services.py b/apps/ingest/services.py index cb7052aa8..18857a6b9 100644 --- a/apps/ingest/services.py +++ b/apps/ingest/services.py @@ -1,11 +1,9 @@ """ Module of service classes and methods for ingest. """ from mimetypes import guess_type -from time import time from urllib.parse import unquote, urlparse from django.apps import apps from tablib.core import Dataset from apps.iiif.manifests.models import Manifest, RelatedLink -from apps.utils.noid import encode_noid def clean_metadata(metadata): """Remove keys that do not aligin with Manifest fields. @@ -54,10 +52,18 @@ def create_manifest(ingest): manifest = Manifest() manifest.image_server = ingest.image_server - manifest.save() # This was giving me a 'django.core.exceptions.AppRegistryNotReady: Models aren't loaded yet' error. Remote = apps.get_model('ingest.remote') + + # Ensure that manifest has an ID before updating the M2M relationship + manifest.save() + if not isinstance(ingest, Remote): + manifest.refresh_from_db() + manifest.collections.set(ingest.collections.all()) + # Save again once relationship is set + manifest.save() + # if type(ingest, .models.Remote): if isinstance(ingest, Remote): RelatedLink( diff --git a/apps/ingest/tests/test_admin.py b/apps/ingest/tests/test_admin.py index 615e1faae..7b20eab8e 100644 --- a/apps/ingest/tests/test_admin.py +++ b/apps/ingest/tests/test_admin.py @@ -11,6 +11,8 @@ from moto import mock_s3 from apps.ingest.forms import BulkVolumeUploadForm from apps.iiif.canvases.models import Canvas +from apps.iiif.kollections.models import Collection +from apps.iiif.kollections.tests.factories import CollectionFactory from apps.iiif.manifests.models import Manifest from apps.iiif.manifests.tests.factories import ManifestFactory, ImageServerFactory from apps.ingest.models import Bulk, Local, Remote, IngestTaskWatcher @@ -80,6 +82,42 @@ def test_local_admin_save(self): assert Manifest.objects.count() == original_manifest_count + 1 assert Canvas.objects.count() == original_canvas_count + 10 + def test_local_ingest_with_collections(self): + """It should add chosen collections to the Local's manifests""" + local = LocalFactory.build( + image_server=self.image_server + ) + + # Force evaluation to get the true current list of manifests + manifests_before = list(Manifest.objects.all()) + + # Assign collections to Local + for _ in range(3): + CollectionFactory.create() + collections = Collection.objects.all() + local.save() + local.collections.set(collections) + assert len(local.collections.all()) == 3 + + # Make a local ingest + request_factory = RequestFactory() + with open(join(self.fixture_path, 'no_meta_file.zip'), 'rb') as f: + content = files.base.ContentFile(f.read()) + local.bundle = files.File(content.file, 'no_meta_file.zip') + req = request_factory.post('/admin/ingest/local/add/', data={}) + req.user = self.user + local_model_admin = LocalAdmin(model=Local, admin_site=AdminSite()) + local_model_admin.save_model(obj=local, request=req, form=None, change=None) + + # Get the newly created manifest by comparing current list to the list before + manifests_after = list(Manifest.objects.all()) + new_manifests = [x for x in manifests_after if x not in manifests_before] + assert len(new_manifests) == 1 + assert isinstance(new_manifests[0], Manifest) + + # The new manifest shouhld get the Local's collections + assert new_manifests[0].collections.count() == 3 + def test_local_admin_response_add(self): """It should redirect to new manifest""" @@ -230,6 +268,29 @@ def test_task_watcher_admin_functions(self): assert watcher_admin.task_name(watcher) == 'fake_task' assert 'PENDING' in watcher_admin.task_status(watcher) + def test_bulk_ingest_with_collections(self): + """It should add the collections to the matching Local object""" + bulk = BulkFactory.create() + for _ in range(3): + CollectionFactory.create() + collections = Collection.objects.all() + bulk.save() + bulk.collections.set(collections) + data = {} + data['volume_files'] = [bulk.volume_files] + + request_factory = RequestFactory() + req = request_factory.post('/admin/ingest/bulk/add/', data=data) + req.user = self.user + + bulk_model_admin = BulkAdmin(model=Bulk, admin_site=AdminSite()) + mock_form = BulkVolumeUploadForm() + bulk_model_admin.save_model(obj=bulk, request=req, form=mock_form, change=None) + bulk.refresh_from_db() + created_local = bulk.local_uploads.first() + + assert created_local.collections.count() == 3 + # def test_local_admin_save_update_manifest(self): # """It should add a manifest to the Local object""" diff --git a/apps/ingest/tests/test_local.py b/apps/ingest/tests/test_local.py index 733f026a8..d676f588e 100644 --- a/apps/ingest/tests/test_local.py +++ b/apps/ingest/tests/test_local.py @@ -81,13 +81,13 @@ def test_ocr_upload_to_s3(self): assert f'{local.manifest.pid}/_*ocr*_/00000008.tsv' in ocr_files def test_metadata_from_excel(self): - """ It should create a manifest with metadat supplied in an Excel file. """ + """ It should create a manifest with metadata supplied in an Excel file. """ local = self.mock_local('bundle.zip', with_manifest=True) assert 'pid' in local.metadata.keys() for key in local.metadata.keys(): - assert local.metadata[key] == getattr(local.manifest, key) + assert str(local.metadata[key]) == str(getattr(local.manifest, key)) def test_metadata_from_csv(self): """ It should create a manifest with metadata supplied in a CSV file. """ From 670c7b65800cda4c6dfab23079467ad0dcd7f6d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Nov 2021 10:20:04 +0000 Subject: [PATCH 15/43] build(deps): Bump django-storages from 1.12.2 to 1.12.3 Bumps [django-storages](https://github.com/jschneier/django-storages) from 1.12.2 to 1.12.3. - [Release notes](https://github.com/jschneier/django-storages/releases) - [Changelog](https://github.com/jschneier/django-storages/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jschneier/django-storages/compare/1.12.2...1.12.3) --- updated-dependencies: - dependency-name: django-storages dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index be0a2af88..af15a7395 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -56,7 +56,7 @@ django-background-tasks==1.2.5 # S3 Uploads boto3==1.19.4 -django-storages==1.12.2 # https://github.com/jschneier/django-storages +django-storages==1.12.3 # https://github.com/jschneier/django-storages # Python function to stream unzip all the files in a ZIP archive, without loading the entire ZIP file into memory or any of its uncompressed files. stream-unzip>=0.0.58 From 31063d558d770851ce5a1e019435f5b624f4309f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Nov 2021 10:20:15 +0000 Subject: [PATCH 16/43] build(deps): Bump moto from 2.2.11 to 2.2.12 Bumps [moto](https://github.com/spulec/moto) from 2.2.11 to 2.2.12. - [Release notes](https://github.com/spulec/moto/releases) - [Changelog](https://github.com/spulec/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/spulec/moto/compare/2.2.11...2.2.12) --- updated-dependencies: - dependency-name: moto dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index d483969ed..737c366d0 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -18,7 +18,7 @@ httpretty==1.1.4 # https://pypi.org/project/httpretty/ #mock==4.0.2 cssutils==2.3.0 # https://pypi.org/project/cssutils/ pytest-django==4.4.0 # https://github.com/pytest-dev/pytest-django -moto==2.2.11 # https://github.com/spulec/moto +moto==2.2.12 # https://github.com/spulec/moto # Code quality # ------------------------------------------------------------------------------ From 3ad49a8d81762ba3e6bfda429c37b3aee9682511 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Nov 2021 10:49:21 +0000 Subject: [PATCH 17/43] build(deps): Bump faker from 9.6.0 to 9.8.0 Bumps [faker](https://github.com/joke2k/faker) from 9.6.0 to 9.8.0. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.md) - [Commits](https://github.com/joke2k/faker/compare/v9.6.0...v9.8.0) --- updated-dependencies: - dependency-name: faker dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index d483969ed..e28c0d147 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -29,7 +29,7 @@ coveralls # Django # ------------------------------------------------------------------------------ factory-boy==3.2.1 # https://github.com/FactoryBoy/factory_boy -faker==9.6.0 +faker==9.8.0 django-debug-toolbar==3.2.2 # https://github.com/jazzband/django-debug-toolbar django-extensions==3.1.3 # https://github.com/django-extensions/django-extensions From 870b90598dee651658242bb3180c4467a4c60db4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Nov 2021 16:06:00 +0000 Subject: [PATCH 18/43] build(deps): Bump boto3 from 1.19.4 to 1.19.9 Bumps [boto3](https://github.com/boto/boto3) from 1.19.4 to 1.19.9. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.19.4...1.19.9) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index af15a7395..dfe077c63 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -55,7 +55,7 @@ gitpython==3.1.24 django-background-tasks==1.2.5 # S3 Uploads -boto3==1.19.4 +boto3==1.19.9 django-storages==1.12.3 # https://github.com/jschneier/django-storages # Python function to stream unzip all the files in a ZIP archive, without loading the entire ZIP file into memory or any of its uncompressed files. From 6cf730e2619d19c33ebd2b76955f318eb1fbe304 Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Wed, 3 Nov 2021 17:04:28 -0400 Subject: [PATCH 19/43] feat: Action/form to add manifests to collections --- apps/iiif/manifests/admin.py | 24 ++++++++++++ apps/iiif/manifests/forms.py | 17 ++++++++ .../add_manifests_to_collections.html | 38 ++++++++++++++++++ apps/iiif/manifests/views.py | 39 ++++++++++++++++++- 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 apps/iiif/manifests/templates/add_manifests_to_collections.html diff --git a/apps/iiif/manifests/admin.py b/apps/iiif/manifests/admin.py index 459537181..3b4072836 100644 --- a/apps/iiif/manifests/admin.py +++ b/apps/iiif/manifests/admin.py @@ -1,11 +1,15 @@ """Django admin module for maninfests""" from django.contrib import admin +from django.http import HttpResponseRedirect +from django.urls.conf import path from import_export import resources, fields from import_export.admin import ImportExportModelAdmin from import_export.widgets import ManyToManyWidget, ForeignKeyWidget from django_summernote.admin import SummernoteModelAdmin + from .models import Manifest, Note, ImageServer from .forms import ManifestAdminForm +from .views import AddToCollectionsView from ..kollections.models import Collection class ManifestResource(resources.ModelResource): @@ -38,6 +42,26 @@ class ManifestAdmin(ImportExportModelAdmin, SummernoteModelAdmin, admin.ModelAdm search_fields = ('id', 'label', 'author', 'published_date') summernote_fields = ('summary',) form = ManifestAdminForm + actions = ['add_to_collections_action'] + + def add_to_collections_action(self, request, queryset): + """Action choose manifests to add to collections""" + selected = queryset.values_list('pk', flat=True) + selected_ids = ','.join(str(pk) for pk in selected) + return HttpResponseRedirect(f'add_to_collections/?ids={selected_ids}') + add_to_collections_action.short_description = 'Add selected manifests to collection(s)' + + def get_urls(self): + urls = super().get_urls() + my_urls = [ + path( + 'add_to_collections/', + self.admin_site.admin_view(AddToCollectionsView.as_view()), + {'model_admin': self, }, + name="AddManifestsToCollections", + ) + ] + return my_urls + urls class NoteAdmin(admin.ModelAdmin): """Django admin configuration for a note.""" diff --git a/apps/iiif/manifests/forms.py b/apps/iiif/manifests/forms.py index c03257548..f8ef13bc6 100644 --- a/apps/iiif/manifests/forms.py +++ b/apps/iiif/manifests/forms.py @@ -1,6 +1,9 @@ """Django Forms for export.""" import logging from django import forms +from django.contrib.admin import site as admin_site, widgets + +from apps.iiif.kollections.models import Collection from .models import Manifest from ..canvases.models import Canvas @@ -84,3 +87,17 @@ def __init__(self, *args, **kwargs): self.fields['start_canvas'].queryset = kwargs['instance'].canvas_set.all() else: self.fields['start_canvas'].queryset = Canvas.objects.none() + +class ManifestsCollectionsForm(forms.ModelForm): + """Form to add manifests to collections.""" + class Meta: + model = Manifest + fields=('collections',) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['collections'].widget = widgets.RelatedFieldWidgetWrapper( + self.fields['collections'].widget, + self.instance._meta.get_field('collections').remote_field, + admin_site, + ) diff --git a/apps/iiif/manifests/templates/add_manifests_to_collections.html b/apps/iiif/manifests/templates/add_manifests_to_collections.html new file mode 100644 index 000000000..3aefa8236 --- /dev/null +++ b/apps/iiif/manifests/templates/add_manifests_to_collections.html @@ -0,0 +1,38 @@ +{% extends "admin/base_site.html" %} +{% load i18n admin_urls %} +{% load static %} + +{% block extrastyle %} + +{% endblock %} + +{% block content %} +
{% csrf_token %} +
+
+
+ +
+
    + {% for manifest in manifests %} +
  • + {% trans 'unlabeled' as unlabeled %} + {{ manifest.label|default:unlabeled }} + ({{ manifest.pk }}) +
    • + {% endfor %} +
+
+
+
+
+ {{ form }} +
+
+ {% block submit_buttons_bottom %} +
+ +
+ {% endblock %} +
+{% endblock %} \ No newline at end of file diff --git a/apps/iiif/manifests/views.py b/apps/iiif/manifests/views.py index 32e0da38c..70e1d325f 100644 --- a/apps/iiif/manifests/views.py +++ b/apps/iiif/manifests/views.py @@ -2,18 +2,21 @@ import json import logging from datetime import datetime +from django.contrib import messages from django.http import JsonResponse from django.http import HttpResponse from django.shortcuts import render +from django.template.response import TemplateResponse from django.views import View from django.views.generic.base import TemplateView +from django.views.generic.edit import FormView from django.core.serializers import serialize from django.contrib.sitemaps import Sitemap from django.urls import reverse from ..canvases.models import Canvas from .models import Manifest from .export import IiifManifestExport -from .forms import JekyllExportForm +from .forms import JekyllExportForm, ManifestsCollectionsForm from .tasks import github_export_task from .tasks import download_export_task @@ -198,3 +201,37 @@ def get(self, request, *args, **kwargs): for canvas in self.get_queryset() : annotations.append(canvas.result) return HttpResponse(' '.join(annotations)) + +class AddToCollectionsView(FormView): + """Intermediate page to choose collections to which you are adding manifests""" + + template_name = 'add_manifests_to_collections.html' + form_class = ManifestsCollectionsForm + + def get_context_data(self, **kwargs): + ids = self.request.GET.get('ids', '').split(',') + manifests = Manifest.objects.filter(pk__in=ids) + model_admin = self.kwargs['model_admin'] + context = super().get_context_data(**kwargs) + context['model_admin'] = model_admin.admin_site.each_context(self.request) + context['manifests'] = manifests + context['title'] = 'Add selected manifests to collection(s)' + return context + + def form_valid(self, form): + self.add_manifests_to_collections(form) + return super().form_valid(form) + + def add_manifests_to_collections(self, form): + context = self.get_context_data() + manifests = context['manifests'] + collections = form.cleaned_data['collections'] + for manifest in manifests: + manifest.collections.add(*collections) + manifest.save() + + def get_success_url(self): + messages.add_message( + self.request, messages.SUCCESS, 'Successfully added manifests to collections' + ) + return reverse('admin:manifests_manifest_changelist') From 76b3af61674e8a7017d87a2f3852b9edfa6442c8 Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Thu, 4 Nov 2021 10:31:12 -0400 Subject: [PATCH 20/43] feat(test): Basic tests for adding to collections --- apps/iiif/manifests/admin.py | 1 - apps/iiif/manifests/tests/test_views.py | 53 +++++++++++++++++++++++-- apps/iiif/manifests/views.py | 4 +- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/apps/iiif/manifests/admin.py b/apps/iiif/manifests/admin.py index 3b4072836..64d496621 100644 --- a/apps/iiif/manifests/admin.py +++ b/apps/iiif/manifests/admin.py @@ -6,7 +6,6 @@ from import_export.admin import ImportExportModelAdmin from import_export.widgets import ManyToManyWidget, ForeignKeyWidget from django_summernote.admin import SummernoteModelAdmin - from .models import Manifest, Note, ImageServer from .forms import ManifestAdminForm from .views import AddToCollectionsView diff --git a/apps/iiif/manifests/tests/test_views.py b/apps/iiif/manifests/tests/test_views.py index 673fc2120..1a62c2250 100644 --- a/apps/iiif/manifests/tests/test_views.py +++ b/apps/iiif/manifests/tests/test_views.py @@ -2,21 +2,24 @@ ''' import json from datetime import datetime -from time import sleep from django.test import TestCase, Client from django.test import RequestFactory from django.conf import settings +from django.contrib.admin.sites import AdminSite from django.contrib.auth import get_user_model from django.urls import reverse from django.core.serializers import serialize from allauth.socialaccount.models import SocialAccount -from iiif_prezi.loader import ManifestReader -from ..views import ManifestSitemap, ManifestRis + +from ..admin import ManifestAdmin +from ..views import AddToCollectionsView, ManifestSitemap, ManifestRis from ..models import Manifest -from ..forms import JekyllExportForm +from ..forms import JekyllExportForm, ManifestsCollectionsForm from .factories import ManifestFactory, EmptyManifestFactory from ...canvases.models import Canvas from ...canvases.tests.factories import CanvasFactory +from ...kollections.models import Collection +from ...kollections.tests.factories import CollectionFactory USER = get_user_model() @@ -137,3 +140,45 @@ def test_no_starting_canvases(self): ) ) assert manifest.canvas_set.all().first().pid in serialized_manifest['thumbnail']['@id'] + + # TODO: Test with 0 manifests, 1 manifests, non-manifest obj + def test_add_to_collections_view_context(self): + """it should add the passed manifests to view context""" + manifest1 = ManifestFactory.create() + manifest2 = ManifestFactory.create() + ids = ','.join([str(manifest1.pk), str(manifest2.pk)]) + request = self.factory.get( + reverse('admin:manifests_manifest_changelist') + 'add_to_collections/?ids=' + ids + ) + request.user = self.user + view = AddToCollectionsView() + model_admin = ManifestAdmin(model=Manifest, admin_site=AdminSite()) + view.setup(request, model_admin=model_admin) + + context = view.get_context_data() + self.assertIn('manifests', context) + self.assertIn(manifest1, context['manifests']) + self.assertIn(manifest2, context['manifests']) + + def test_add_to_collections_view_function(self): + """it should add the passed manifests to the selected collections""" + manifest1 = ManifestFactory.create() + manifest2 = ManifestFactory.create() + collection1 = CollectionFactory.create() + collection2 = CollectionFactory.create() + ids = ','.join([str(manifest1.pk), str(manifest2.pk)]) + request = self.factory.get( + reverse('admin:manifests_manifest_changelist') + 'add_to_collections/?ids=' + ids + ) + request.user = self.user + view = AddToCollectionsView() + model_admin = ManifestAdmin(model=Manifest, admin_site=AdminSite()) + view.setup(request, model_admin=model_admin) + qs = Collection.objects.filter(pk__in=[collection1.pk, collection2.pk]) + form = ManifestsCollectionsForm(data={"collections": qs}) + + assert len(manifest1.collections.all()) == 0 + assert len(manifest2.collections.all()) == 0 + view.add_manifests_to_collections(form) + assert len(manifest1.collections.all()) == 2 + assert len(manifest2.collections.all()) == 2 diff --git a/apps/iiif/manifests/views.py b/apps/iiif/manifests/views.py index 70e1d325f..c03a4a4af 100644 --- a/apps/iiif/manifests/views.py +++ b/apps/iiif/manifests/views.py @@ -223,9 +223,11 @@ def form_valid(self, form): return super().form_valid(form) def add_manifests_to_collections(self, form): + """Adds selected manifests to selected collections from form""" context = self.get_context_data() manifests = context['manifests'] - collections = form.cleaned_data['collections'] + if form.is_valid(): + collections = form.cleaned_data['collections'] for manifest in manifests: manifest.collections.add(*collections) manifest.save() From 882787b0ff67bde91acd187a75f82a51e5fbd866 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Nov 2021 10:30:36 +0000 Subject: [PATCH 21/43] build(deps): Bump boto3 from 1.19.9 to 1.19.11 Bumps [boto3](https://github.com/boto/boto3) from 1.19.9 to 1.19.11. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.19.9...1.19.11) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index dfe077c63..7b09dcee3 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -55,7 +55,7 @@ gitpython==3.1.24 django-background-tasks==1.2.5 # S3 Uploads -boto3==1.19.9 +boto3==1.19.11 django-storages==1.12.3 # https://github.com/jschneier/django-storages # Python function to stream unzip all the files in a ZIP archive, without loading the entire ZIP file into memory or any of its uncompressed files. From dd8ad25cf7426fd00e7c66b2497e53fab67645ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Nov 2021 10:21:46 +0000 Subject: [PATCH 22/43] build(deps): Bump django-extensions from 3.1.3 to 3.1.5 Bumps [django-extensions](https://github.com/django-extensions/django-extensions) from 3.1.3 to 3.1.5. - [Release notes](https://github.com/django-extensions/django-extensions/releases) - [Changelog](https://github.com/django-extensions/django-extensions/blob/main/CHANGELOG.md) - [Commits](https://github.com/django-extensions/django-extensions/compare/3.1.3...3.1.5) --- updated-dependencies: - dependency-name: django-extensions dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index 9a0249c22..77d2af942 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -32,7 +32,7 @@ factory-boy==3.2.1 # https://github.com/FactoryBoy/factory_boy faker==9.8.0 django-debug-toolbar==3.2.2 # https://github.com/jazzband/django-debug-toolbar -django-extensions==3.1.3 # https://github.com/django-extensions/django-extensions +django-extensions==3.1.5 # https://github.com/django-extensions/django-extensions django-coverage-plugin==2.0.1 # https://github.com/nedbat/django_coverage_plugin pyopenssl # for running dev server under https From 562ee607d8aeb91db11cd3c039593ad495500395 Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Wed, 10 Nov 2021 10:27:42 -0500 Subject: [PATCH 23/43] Add test for Jekyll export error --- .coveragerc | 2 +- .gitignore | 3 +-- apps/iiif/manifests/export.py | 2 +- apps/readux/tests/test_export.py | 11 ++++++++++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.coveragerc b/.coveragerc index 4ae91d22f..48f76908b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -11,7 +11,7 @@ omit = exclude_lines = # Ignore imports from - import + import\s logger LOGGER pragma: no cover \ No newline at end of file diff --git a/.gitignore b/.gitignore index 295e976e4..cc2843b83 100644 --- a/.gitignore +++ b/.gitignore @@ -67,11 +67,10 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ -.coverage .cache nosetests.xml coverage.xml -.coveragerc +.coverage* # Packages *.egg *.egg-info diff --git a/apps/iiif/manifests/export.py b/apps/iiif/manifests/export.py index 03f61a5cc..4145f017d 100644 --- a/apps/iiif/manifests/export.py +++ b/apps/iiif/manifests/export.py @@ -673,7 +673,7 @@ def update_gitrepo(self): if os.environ['DJANGO_ENV'] != 'test': # run the script to import IIIF as jekyll site content - self.import_iiif_jekyll(self.manifest, self.jekyll_site_dir) + self.import_iiif_jekyll(self.manifest, self.jekyll_site_dir) # pragma: no cover # add any files that could be updated to the git index repo.index.add([ # pragma: no cover diff --git a/apps/readux/tests/test_export.py b/apps/readux/tests/test_export.py index bd9942ef0..46e98a49c 100644 --- a/apps/readux/tests/test_export.py +++ b/apps/readux/tests/test_export.py @@ -14,7 +14,7 @@ from apps.iiif.manifests.models import Manifest from apps.iiif.manifests.views import ManifestExport, JekyllExport from apps.iiif.canvases.models import Canvas -from apps.iiif.manifests.export import IiifManifestExport, JekyllSiteExport, GithubExportException +from apps.iiif.manifests.export import IiifManifestExport, JekyllSiteExport, GithubExportException, ExportException from apps.iiif.manifests.github import GithubApi from apps.users.tests.factories import UserFactory, SocialAccountFactory, SocialAppFactory, SocialTokenFactory from iiif_prezi.loader import ManifestReader @@ -121,6 +121,15 @@ def test_jekyll_site_export(self): # verify user annotation count is correct assert len(os.listdir(os.path.join(jekyll_path, '_annotations'))) == 1 + def test_jekyll_export_error(self): + export = JekyllSiteExport(self.volume, 'v2', owners=[self.user.id], deep_zoom='exclude') + export.jekyll_site_dir = 'nope' + try: + export.import_iiif_jekyll(self.volume, 'non_existing_directory') + assert False + except ExportException: + assert True + def test_get_zip_file(self): # Make an empty file dummy_file = os.path.join(tempfile.tempdir, 'file.txt') From df70aa051b995e48be682bc03abf35046e042b31 Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Wed, 10 Nov 2021 12:59:00 -0500 Subject: [PATCH 24/43] Move export tasks to new app and convert to celery --- apps/export/__init__.py | 0 apps/export/apps.py | 6 ++++ apps/export/celery.py | 16 ++++++++++ apps/{iiif/manifests => export}/export.py | 0 apps/{iiif/manifests => export}/github.py | 0 apps/{iiif/manifests => export}/tasks.py | 18 ++++++----- apps/{readux => export}/tests/test_export.py | 4 +-- apps/{readux => export}/tests/test_github.py | 4 +-- apps/iiif/canvases/tasks.py | 1 - apps/iiif/manifests/views.py | 10 +++--- .../commands/restart_background_tasks.py | 31 ------------------- apps/readux/views.py | 2 +- config/settings/base.py | 3 +- requirements/base.txt | 2 -- 14 files changed, 45 insertions(+), 52 deletions(-) create mode 100644 apps/export/__init__.py create mode 100644 apps/export/apps.py create mode 100644 apps/export/celery.py rename apps/{iiif/manifests => export}/export.py (100%) rename apps/{iiif/manifests => export}/github.py (100%) rename apps/{iiif/manifests => export}/tasks.py (82%) rename apps/{readux => export}/tests/test_export.py (98%) rename apps/{readux => export}/tests/test_github.py (97%) delete mode 100644 apps/readux/management/commands/restart_background_tasks.py diff --git a/apps/export/__init__.py b/apps/export/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/export/apps.py b/apps/export/apps.py new file mode 100644 index 000000000..e990d32c8 --- /dev/null +++ b/apps/export/apps.py @@ -0,0 +1,6 @@ +"""Django app configuration for Export""" +from django.apps import AppConfig + +class ExportConfig(AppConfig): + """Configuration for Export Django app""" + name = 'apps.export' diff --git a/apps/export/celery.py b/apps/export/celery.py new file mode 100644 index 000000000..7dfbe4c55 --- /dev/null +++ b/apps/export/celery.py @@ -0,0 +1,16 @@ +""" +Celery config for ingest tasks. +""" +import os +from celery import Celery +from django.conf import settings +# import config.settings.local as settings + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') +app = Celery('apps.export', result_extended=True) + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff --git a/apps/iiif/manifests/export.py b/apps/export/export.py similarity index 100% rename from apps/iiif/manifests/export.py rename to apps/export/export.py diff --git a/apps/iiif/manifests/github.py b/apps/export/github.py similarity index 100% rename from apps/iiif/manifests/github.py rename to apps/export/github.py diff --git a/apps/iiif/manifests/tasks.py b/apps/export/tasks.py similarity index 82% rename from apps/iiif/manifests/tasks.py rename to apps/export/tasks.py index d1946443e..6966dbc0c 100644 --- a/apps/iiif/manifests/tasks.py +++ b/apps/export/tasks.py @@ -1,15 +1,19 @@ """Module to manage background export task.""" import logging import os -from background_task import background +from celery import Celery +from django.conf import settings from apps.users.models import User +from apps.iiif.manifests.models import Manifest from .export import JekyllSiteExport -from .models import Manifest - LOGGER = logging.getLogger(__name__) -@background(schedule=1) +app = Celery('apps.ingest', result_extended=True) +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) + +@app.task(name='github_export', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) def github_export_task( manifest_pid, version, github_repo=None, user_id=None, owner_ids=None, deep_zoom=False): @@ -45,7 +49,7 @@ def github_export_task( jekyll_exporter.github_export(user.email) LOGGER.info('Background github export finished.') -@background(schedule=1) +@app.task(name='download_export', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) def download_export_task( manifest_pid, version, github_repo=None, user_id=None, owner_ids=None, deep_zoom=False): @@ -79,10 +83,10 @@ def download_export_task( ) zipfile_name = jekyll_exporter.download_export(user.email, manifest) - delete_download_task(zipfile_name) + delete_download_task.apply_async((zipfile_name), countdown=86400) LOGGER.info('Background download export finished.') -@background(schedule=86400) +@app.task(name='delete_download', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) def delete_download_task(download_path): """Background delete download task. diff --git a/apps/readux/tests/test_export.py b/apps/export/tests/test_export.py similarity index 98% rename from apps/readux/tests/test_export.py rename to apps/export/tests/test_export.py index 46e98a49c..d59aa58bb 100644 --- a/apps/readux/tests/test_export.py +++ b/apps/export/tests/test_export.py @@ -14,8 +14,8 @@ from apps.iiif.manifests.models import Manifest from apps.iiif.manifests.views import ManifestExport, JekyllExport from apps.iiif.canvases.models import Canvas -from apps.iiif.manifests.export import IiifManifestExport, JekyllSiteExport, GithubExportException, ExportException -from apps.iiif.manifests.github import GithubApi +from apps.export.export import IiifManifestExport, JekyllSiteExport, GithubExportException, ExportException +from apps.export.github import GithubApi from apps.users.tests.factories import UserFactory, SocialAccountFactory, SocialAppFactory, SocialTokenFactory from iiif_prezi.loader import ManifestReader diff --git a/apps/readux/tests/test_github.py b/apps/export/tests/test_github.py similarity index 97% rename from apps/readux/tests/test_github.py rename to apps/export/tests/test_github.py index ae5b544c2..162ab2633 100644 --- a/apps/readux/tests/test_github.py +++ b/apps/export/tests/test_github.py @@ -1,7 +1,7 @@ from django.test import TestCase -from apps.iiif.manifests.github import GithubApi, GithubApiException, GithubAccountNotFound -from apps.users.tests.factories import UserFactory, SocialAccountFactory, SocialAppFactory, SocialTokenFactory import httpretty +from apps.users.tests.factories import UserFactory, SocialAccountFactory, SocialAppFactory, SocialTokenFactory +from apps.export.github import GithubApi, GithubApiException, GithubAccountNotFound class TestGithubApi(TestCase): def setUp(self): diff --git a/apps/iiif/canvases/tasks.py b/apps/iiif/canvases/tasks.py index e447d9314..778806d5b 100644 --- a/apps/iiif/canvases/tasks.py +++ b/apps/iiif/canvases/tasks.py @@ -1,6 +1,5 @@ """ Common tasks for canvases. """ from celery import Celery -from background_task import background from django.apps import apps from ..annotations.models import Annotation from .models import Canvas diff --git a/apps/iiif/manifests/views.py b/apps/iiif/manifests/views.py index c03a4a4af..b6a37b7b3 100644 --- a/apps/iiif/manifests/views.py +++ b/apps/iiif/manifests/views.py @@ -15,10 +15,10 @@ from django.urls import reverse from ..canvases.models import Canvas from .models import Manifest -from .export import IiifManifestExport +from apps.export.export import IiifManifestExport from .forms import JekyllExportForm, ManifestsCollectionsForm -from .tasks import github_export_task -from .tasks import download_export_task +from apps.export.tasks import github_export_task +from apps.export.tasks import download_export_task LOGGER = logging.getLogger(__name__) @@ -149,7 +149,7 @@ def post(self, request, *args, **kwargs): # pylint: disable = unused-argument if export_mode == 'download': context = self.get_context_data() manifest_pid = manifest.pid - download_export_task( + download_export_task.delay( manifest_pid, kwargs['version'], github_repo=github_repo, @@ -164,7 +164,7 @@ def post(self, request, *args, **kwargs): # pylint: disable = unused-argument #github exports context = self.get_context_data() manifest_pid = manifest.pid - github_export_task( + github_export_task.delay( manifest_pid, kwargs['version'], github_repo=github_repo, diff --git a/apps/readux/management/commands/restart_background_tasks.py b/apps/readux/management/commands/restart_background_tasks.py deleted file mode 100644 index 8dd8dd3e1..000000000 --- a/apps/readux/management/commands/restart_background_tasks.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Manage command to restart background tasks. -""" -from os import kill -from signal import SIGKILL -from subprocess import check_output, Popen -from django.core.management.base import BaseCommand -from background_task.models import CompletedTask - -class Command(BaseCommand): - """ - Manage command that looks up system pids for the background tasks and kills them. - - Finally, it starts three new processes for the background tasks. - - TODO: Instead of assuming all background processes have been locked by a completed task, - we might should look to some other way to track the processes. The danger is that an - outdated process will pickup a job and it will be hard to figure out why something - bad happened. - """ - def handle(self, *args, **options): - pids = [task.locked_by for task in CompletedTask.objects.all()] - for pid in pids: - try: - kill(pid, SIGKILL) - except ProcessLookupError: - pass - - python_path = check_output(['which', 'python']).decode('utf-8').rstrip() - for _ in range(3): - Popen(['nohup', python_path, 'manage.py', 'process_tasks']) diff --git a/apps/readux/views.py b/apps/readux/views.py index 118af9f4b..e624b9ce0 100644 --- a/apps/readux/views.py +++ b/apps/readux/views.py @@ -11,13 +11,13 @@ from django.urls import reverse from django.utils.datastructures import MultiValueDictKeyError import config.settings.local as settings +from apps.export.export import JekyllSiteExport from .models import UserAnnotation from ..cms.models import Page, CollectionsPage, VolumesPage from ..iiif.kollections.models import Collection from ..iiif.canvases.models import Canvas from ..iiif.manifests.models import Manifest from ..iiif.manifests.forms import JekyllExportForm -from ..iiif.manifests.export import JekyllSiteExport SORT_OPTIONS = ['title', 'author', 'date published', 'date added'] ORDER_OPTIONS = ['asc', 'desc'] diff --git a/config/settings/base.py b/config/settings/base.py index ce493a5a5..bc75125e0 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -114,7 +114,8 @@ 'apps.cms.apps.CmsConfig', 'apps.templates', 'apps.custom_styles.apps.CustomStylesConfig', - 'apps.ingest.apps.IngestConfig' + 'apps.ingest.apps.IngestConfig', + 'apps.export.apps.ExportConfig' ] # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS diff --git a/requirements/base.txt b/requirements/base.txt index 7b09dcee3..a1f94c126 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -52,8 +52,6 @@ wagtail-cache==1.0.2 django-import-export==2.6.1 gitpython==3.1.24 -django-background-tasks==1.2.5 - # S3 Uploads boto3==1.19.11 django-storages==1.12.3 # https://github.com/jschneier/django-storages From fed291cb0dd97ea855e006488636a18caa8bce95 Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Wed, 10 Nov 2021 13:10:04 -0500 Subject: [PATCH 25/43] Update deploy to mount extra volume and remove restart background task --- fabfile.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/fabfile.py b/fabfile.py index fcdf11d7d..427ba0422 100644 --- a/fabfile.py +++ b/fabfile.py @@ -8,7 +8,7 @@ env.user = 'deploy' -def deploy(branch='release', path='/readux.io/readux'): +def deploy(branch='release', path='/readux.io/readux', volume=None): """Execute group of tasks for deployment. :param branch: Git branch to clone, defaults to 'master' @@ -30,7 +30,9 @@ def deploy(branch='release', path='/readux.io/readux'): _get_latest_source(branch, options) _update_virtualenv(options) _link_settings(options) - _create_static_media_symlinks(options) + _create_staticfiles_symlink(options) + if volume is not None: + _mount_media(volume) _update_static_files(options) _update_database(options) _update_symlink(options) @@ -63,7 +65,11 @@ def _link_settings(options): with cd('config/settings'): run('ln -s {rp}/local.py local.py'.format(rp=options['ROOT_PATH'])) -def _create_static_media_symlinks(options): +def _mount_media(volume): + with cd('apps'): + run(f'mkdir media && sudo mount /dev/{volume} media') + +def _create_staticfiles_symlink(options): run('ln -s {rp}/staticfiles staticfiles'.format(rp=options['ROOT_PATH'])) with cd('apps'): run('ln -s {rp}/media media'.format(rp=options['ROOT_PATH'])) @@ -84,8 +90,6 @@ def _restart_webserver(): run('sudo /bin/systemctl reload apache2') def _restart_background_tasks(options): - with cd(options['ROOT_PATH']): - run('nohup ./restart_export_tasks.sh &') run('sudo /bin/systemctl restart celeryd') def _clean_old_builds(options): From 61cf673f0f9e737c2cc36b8b3414cc38fa55eb22 Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Wed, 10 Nov 2021 13:16:58 -0500 Subject: [PATCH 26/43] Remove references to background_tasks --- apps/ingest/tasks.py | 5 ----- config/settings/base.py | 1 - fabfile.py | 4 ++-- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/apps/ingest/tasks.py b/apps/ingest/tasks.py index bcc07a8cb..adfb66b88 100644 --- a/apps/ingest/tasks.py +++ b/apps/ingest/tasks.py @@ -19,11 +19,6 @@ Remote = apps.get_model('ingest.remote') LOGGER = logging.getLogger(__name__) -logging.getLogger("background_task").setLevel(logging.ERROR) -logging.getLogger('boto3').setLevel(logging.ERROR) -logging.getLogger('botocore').setLevel(logging.ERROR) -logging.getLogger('s3transfer').setLevel(logging.ERROR) -logging.getLogger('factory').setLevel(logging.ERROR) app = Celery('apps.ingest', result_extended=True) app.config_from_object('django.conf:settings') diff --git a/config/settings/base.py b/config/settings/base.py index bc75125e0..9e934e58c 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -81,7 +81,6 @@ 'allauth.socialaccount.providers.github', 'allauth.socialaccount.providers.google', 'allauth.socialaccount.providers.twitter', - 'background_task', 'corsheaders', 'crispy_forms', 'django_celery_results', diff --git a/fabfile.py b/fabfile.py index 427ba0422..64481a21e 100644 --- a/fabfile.py +++ b/fabfile.py @@ -37,7 +37,7 @@ def deploy(branch='release', path='/readux.io/readux', volume=None): _update_database(options) _update_symlink(options) _restart_webserver() - _restart_background_tasks(options) + _restart_celery(options) _clean_old_builds(options) def _get_latest_source(branch, options): @@ -89,7 +89,7 @@ def _update_symlink(options): def _restart_webserver(): run('sudo /bin/systemctl reload apache2') -def _restart_background_tasks(options): +def _restart_celery(options): run('sudo /bin/systemctl restart celeryd') def _clean_old_builds(options): From d09b389ac64237e22d577e272310f4ee65902dc9 Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Wed, 10 Nov 2021 13:48:22 -0500 Subject: [PATCH 27/43] Remove media symlink from deploy --- fabfile.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fabfile.py b/fabfile.py index 64481a21e..e3f4a18b1 100644 --- a/fabfile.py +++ b/fabfile.py @@ -71,8 +71,6 @@ def _mount_media(volume): def _create_staticfiles_symlink(options): run('ln -s {rp}/staticfiles staticfiles'.format(rp=options['ROOT_PATH'])) - with cd('apps'): - run('ln -s {rp}/media media'.format(rp=options['ROOT_PATH'])) def _update_static_files(options): run('{v}/bin/python manage.py collectstatic --noinput'.format(v=options['VENV_PATH'])) From 4a3ce1f8d9faa2fba885bc0531c80dd03b20d8e0 Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Thu, 11 Nov 2021 08:48:58 -0500 Subject: [PATCH 28/43] Move celery module --- apps/export/tasks.py | 4 +--- apps/ingest/celery.py | 16 ---------------- apps/ingest/tasks.py | 4 +--- apps/{export => readux}/celery.py | 2 +- config/settings/__init__.py | 2 +- 5 files changed, 4 insertions(+), 24 deletions(-) delete mode 100644 apps/ingest/celery.py rename apps/{export => readux}/celery.py (90%) diff --git a/apps/export/tasks.py b/apps/export/tasks.py index 6966dbc0c..77df29244 100644 --- a/apps/export/tasks.py +++ b/apps/export/tasks.py @@ -9,9 +9,7 @@ LOGGER = logging.getLogger(__name__) -app = Celery('apps.ingest', result_extended=True) -app.config_from_object('django.conf:settings') -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) +app = Celery('apps.readux', result_extended=True) @app.task(name='github_export', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) def github_export_task( diff --git a/apps/ingest/celery.py b/apps/ingest/celery.py deleted file mode 100644 index 0b506b526..000000000 --- a/apps/ingest/celery.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Celery config for ingest tasks. -""" -import os -from celery import Celery -from django.conf import settings -# import config.settings.local as settings - -# set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') -app = Celery('apps.ingest', result_extended=True) - -# Using a string here means the worker will not have to -# pickle the object when using Windows. -app.config_from_object('django.conf:settings') -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff --git a/apps/ingest/tasks.py b/apps/ingest/tasks.py index adfb66b88..7bf3e6cc6 100644 --- a/apps/ingest/tasks.py +++ b/apps/ingest/tasks.py @@ -20,9 +20,7 @@ LOGGER = logging.getLogger(__name__) -app = Celery('apps.ingest', result_extended=True) -app.config_from_object('django.conf:settings') -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) +app = Celery('apps.readux', result_extended=True) @app.task(name='creating_canvases_from_local', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) def create_canvas_form_local_task(ingest_id): diff --git a/apps/export/celery.py b/apps/readux/celery.py similarity index 90% rename from apps/export/celery.py rename to apps/readux/celery.py index 7dfbe4c55..2ddf125b8 100644 --- a/apps/export/celery.py +++ b/apps/readux/celery.py @@ -8,7 +8,7 @@ # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') -app = Celery('apps.export', result_extended=True) +app = Celery('apps.readux', result_extended=True) # Using a string here means the worker will not have to # pickle the object when using Windows. diff --git a/config/settings/__init__.py b/config/settings/__init__.py index a417f4cc9..e6bd9d2d3 100644 --- a/config/settings/__init__.py +++ b/config/settings/__init__.py @@ -1 +1 @@ -from apps.ingest import celery \ No newline at end of file +from apps.readux import celery \ No newline at end of file From 29f6da0c8cb7577f7919d8074b4e801a3d6d8611 Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Thu, 11 Nov 2021 08:49:26 -0500 Subject: [PATCH 29/43] Fix failing export test --- apps/export/tests/test_export.py | 10 ++++++++++ apps/iiif/canvases/tasks.py | 4 +--- apps/iiif/manifests/views.py | 28 ++++++++++++++++++++-------- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/apps/export/tests/test_export.py b/apps/export/tests/test_export.py index d59aa58bb..f02e5374f 100644 --- a/apps/export/tests/test_export.py +++ b/apps/export/tests/test_export.py @@ -183,10 +183,20 @@ def test_jekyll_export_include_download(self): ) assert isinstance(response.getvalue(), bytes) + @httpretty.httprettified(allow_net_connect=False) def test_jekyll_export_to_github(self): ''' Docstring ''' + httpretty.register_uri( + httpretty.GET, + 'https://api.github.com/users/{u}/repos?per_page=3'.format(u=self.jse.github_username), + body='[{"name":"marx"}]', + content_type="text/json" + ) + httpretty.register_uri( + httpretty.POST, 'https://api.github.com/user/repos' + ) kwargs = {'pid': self.volume.pid, 'version': 'v2'} url = reverse('JekyllExport', kwargs=kwargs) kwargs['deep_zoom'] = 'exclude' diff --git a/apps/iiif/canvases/tasks.py b/apps/iiif/canvases/tasks.py index 778806d5b..dcc8e1fe6 100644 --- a/apps/iiif/canvases/tasks.py +++ b/apps/iiif/canvases/tasks.py @@ -10,9 +10,7 @@ # create a background task have to be serializable, we can't just pass in the model object. # Canvas = apps.get_model('canvases.canvas') -app = Celery('apps.ingest') -app.config_from_object('django.conf:settings') -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) +app = Celery('apps.readux') @app.task(name='adding_ocr_to_canvas', autoretry_for=(Canvas.DoesNotExist,), retry_backoff=5) def add_ocr_task(canvas_id, *args, **kwargs): diff --git a/apps/iiif/manifests/views.py b/apps/iiif/manifests/views.py index b6a37b7b3..ae6adfada 100644 --- a/apps/iiif/manifests/views.py +++ b/apps/iiif/manifests/views.py @@ -1,6 +1,7 @@ """Django views for manifests""" import json import logging +from os import environ from datetime import datetime from django.contrib import messages from django.http import JsonResponse @@ -164,14 +165,25 @@ def post(self, request, *args, **kwargs): # pylint: disable = unused-argument #github exports context = self.get_context_data() manifest_pid = manifest.pid - github_export_task.delay( - manifest_pid, - kwargs['version'], - github_repo=github_repo, - deep_zoom=False, - owner_ids=owners, - user_id=self.request.user.id - ) + + if environ['DJANGO_ENV'] != 'test': # pragma: no cover + github_export_task.delay( + manifest_pid, + kwargs['version'], + github_repo=github_repo, + deep_zoom=False, + owner_ids=owners, + user_id=self.request.user.id + ) + else: + github_export_task( + manifest_pid, + kwargs['version'], + github_repo=github_repo, + deep_zoom=False, + owner_ids=owners, + user_id=self.request.user.id + ) context['email'] = request.user.email context['mode'] = "github" From 61cf6b39106e9036948d92eb5d10720319ab0c3a Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Thu, 11 Nov 2021 09:22:55 -0500 Subject: [PATCH 30/43] Fix hanging test trying to start celery task --- apps/export/tests/test_export.py | 7 ++----- apps/iiif/manifests/views.py | 17 +++++++++-------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/apps/export/tests/test_export.py b/apps/export/tests/test_export.py index f02e5374f..1dbaf6064 100644 --- a/apps/export/tests/test_export.py +++ b/apps/export/tests/test_export.py @@ -146,8 +146,8 @@ def test_manifest_export(self): response = self.manifest_export_view(request, pid=self.volume.pid, version='v2') assert isinstance(response.getvalue(), bytes) - def test_setting_jekyll_site_dir(self): - self.jse + # def test_setting_jekyll_site_dir(self): + # self.jse # Things I want to test: # * Unzip the IIIF zip file @@ -185,9 +185,6 @@ def test_jekyll_export_include_download(self): @httpretty.httprettified(allow_net_connect=False) def test_jekyll_export_to_github(self): - ''' - Docstring - ''' httpretty.register_uri( httpretty.GET, 'https://api.github.com/users/{u}/repos?per_page=3'.format(u=self.jse.github_username), diff --git a/apps/iiif/manifests/views.py b/apps/iiif/manifests/views.py index ae6adfada..6cb45d94a 100644 --- a/apps/iiif/manifests/views.py +++ b/apps/iiif/manifests/views.py @@ -150,14 +150,15 @@ def post(self, request, *args, **kwargs): # pylint: disable = unused-argument if export_mode == 'download': context = self.get_context_data() manifest_pid = manifest.pid - download_export_task.delay( - manifest_pid, - kwargs['version'], - github_repo=github_repo, - deep_zoom=False, - owner_ids=owners, - user_id=self.request.user.id - ) + if environ['DJANGO_ENV'] != 'test': # pragma: no cover + download_export_task.delay( + manifest_pid, + kwargs['version'], + github_repo=github_repo, + deep_zoom=False, + owner_ids=owners, + user_id=self.request.user.id + ) context['email'] = request.user.email context['mode'] = "download" return render(request, self.template_name, context) From 261a7ac005bb4a1352dcf86a513cfdaddcd24f92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 14:40:44 +0000 Subject: [PATCH 31/43] build(deps): Bump coverage from 5.2.1 to 6.1.2 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.2.1 to 6.1.2. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.2.1...6.1.2) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index 77d2af942..3f26a851d 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -23,7 +23,7 @@ moto==2.2.12 # https://github.com/spulec/moto # Code quality # ------------------------------------------------------------------------------ flake8==4.0.1 # https://github.com/PyCQA/flake8 -coverage==5.2.1 # https://github.com/nedbat/coveragepy +coverage==6.1.2 # https://github.com/nedbat/coveragepy coveralls # Django From 7b349c6273dd8ccbc436abad2631da0e72140917 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 14:40:49 +0000 Subject: [PATCH 32/43] build(deps): Bump celery from 5.1.2 to 5.2.0 Bumps [celery](https://github.com/celery/celery) from 5.1.2 to 5.2.0. - [Release notes](https://github.com/celery/celery/releases) - [Changelog](https://github.com/celery/celery/blob/master/Changelog.rst) - [Commits](https://github.com/celery/celery/compare/v5.1.2...v5.2.0) --- updated-dependencies: - dependency-name: celery dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index 77d2af942..2d35229b9 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -41,5 +41,5 @@ pyopenssl # for running dev server under https # TODO: Why are we using this fork? Fabric3==1.14.post1 # https://github.com/mathiasertl/fabric -celery==5.1.2 +celery==5.2.0 django-celery-results==2.2.0 From d535fe85c1b5f5978baf15facff113f7f5e1a892 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 14:49:15 +0000 Subject: [PATCH 33/43] build(deps): Bump moto from 2.2.12 to 2.2.13 Bumps [moto](https://github.com/spulec/moto) from 2.2.12 to 2.2.13. - [Release notes](https://github.com/spulec/moto/releases) - [Changelog](https://github.com/spulec/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/spulec/moto/compare/2.2.12...2.2.13) --- updated-dependencies: - dependency-name: moto dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index 2d35229b9..0425f6778 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -18,7 +18,7 @@ httpretty==1.1.4 # https://pypi.org/project/httpretty/ #mock==4.0.2 cssutils==2.3.0 # https://pypi.org/project/cssutils/ pytest-django==4.4.0 # https://github.com/pytest-dev/pytest-django -moto==2.2.12 # https://github.com/spulec/moto +moto==2.2.13 # https://github.com/spulec/moto # Code quality # ------------------------------------------------------------------------------ From 3005d3a631fb8754eb156a73450b3ad6dbfce611 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 15:00:50 +0000 Subject: [PATCH 34/43] build(deps): Bump boto3 from 1.19.11 to 1.20.3 Bumps [boto3](https://github.com/boto/boto3) from 1.19.11 to 1.20.3. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.19.11...1.20.3) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index a1f94c126..67a81052b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -53,7 +53,7 @@ django-import-export==2.6.1 gitpython==3.1.24 # S3 Uploads -boto3==1.19.11 +boto3==1.20.3 django-storages==1.12.3 # https://github.com/jschneier/django-storages # Python function to stream unzip all the files in a ZIP archive, without loading the entire ZIP file into memory or any of its uncompressed files. From d8253ae14b085ffa1e1dad7b8db583ea058d03d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 15:08:25 +0000 Subject: [PATCH 35/43] build(deps): Bump sphinx from 4.2.0 to 4.3.0 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 4.2.0 to 4.3.0. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/4.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v4.2.0...v4.3.0) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index e0f4302bb..fb5cfe7ed 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -2,7 +2,7 @@ Werkzeug==2.0.2 # https://github.com/pallets/werkzeug ipdb==0.13.9 # https://github.com/gotcha/ipdb -Sphinx==4.2.0 # https://github.com/sphinx-doc/sphinx +Sphinx==4.3.0 # https://github.com/sphinx-doc/sphinx # TODO: Upgrade after moving to Django 3 psycopg2-binary==2.8.4 # https://github.com/psycopg/psycopg2 From 7675a35d6e8aae837bb94d6f8a7c45f2b23bed2c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Nov 2021 10:21:40 +0000 Subject: [PATCH 36/43] build(deps): Bump django-coverage-plugin from 2.0.1 to 2.0.2 Bumps [django-coverage-plugin](https://github.com/nedbat/django_coverage_plugin) from 2.0.1 to 2.0.2. - [Release notes](https://github.com/nedbat/django_coverage_plugin/releases) - [Commits](https://github.com/nedbat/django_coverage_plugin/compare/v2.0.1...v2.0.2) --- updated-dependencies: - dependency-name: django-coverage-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index fb5cfe7ed..9a0efc471 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -33,7 +33,7 @@ faker==9.8.0 django-debug-toolbar==3.2.2 # https://github.com/jazzband/django-debug-toolbar django-extensions==3.1.5 # https://github.com/django-extensions/django-extensions -django-coverage-plugin==2.0.1 # https://github.com/nedbat/django_coverage_plugin +django-coverage-plugin==2.0.2 # https://github.com/nedbat/django_coverage_plugin pyopenssl # for running dev server under https # Deployment From cb1daa0966059de0e02543738d120afd97f0bb67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Nov 2021 10:21:54 +0000 Subject: [PATCH 37/43] build(deps): Bump moto from 2.2.13 to 2.2.15 Bumps [moto](https://github.com/spulec/moto) from 2.2.13 to 2.2.15. - [Release notes](https://github.com/spulec/moto/releases) - [Changelog](https://github.com/spulec/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/spulec/moto/compare/2.2.13...2.2.15) --- updated-dependencies: - dependency-name: moto dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index fb5cfe7ed..c627b7155 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -18,7 +18,7 @@ httpretty==1.1.4 # https://pypi.org/project/httpretty/ #mock==4.0.2 cssutils==2.3.0 # https://pypi.org/project/cssutils/ pytest-django==4.4.0 # https://github.com/pytest-dev/pytest-django -moto==2.2.13 # https://github.com/spulec/moto +moto==2.2.15 # https://github.com/spulec/moto # Code quality # ------------------------------------------------------------------------------ From f38c70ab9401c236b87cc9181471e5686ff8d29f Mon Sep 17 00:00:00 2001 From: J Date: Mon, 15 Nov 2021 13:54:07 -0500 Subject: [PATCH 38/43] flagging color for collection banner text text color is getting overwritten by a general 'p' tag. --- apps/static/css/project.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/static/css/project.scss b/apps/static/css/project.scss index 074e8becf..15812e468 100644 --- a/apps/static/css/project.scss +++ b/apps/static/css/project.scss @@ -332,7 +332,7 @@ header#page-bg .collection-image-info h3 { } header#page-bg .collection-image-info p { width: 100%; - color: #dedede; + color: #dedede !important; text-align: left; margin-bottom: 0px; margin-top: 0px; From f69d6f52d35a6f76615cfeb7ef59042a86ba4a82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Nov 2021 10:24:59 +0000 Subject: [PATCH 39/43] build(deps): Bump redis from 3.5.3 to 4.0.0 Bumps [redis](https://github.com/redis/redis-py) from 3.5.3 to 4.0.0. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/3.5.3...v4.0.0) --- updated-dependencies: - dependency-name: redis dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 67a81052b..ea05b9c46 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,7 +2,7 @@ pytz==2021.3 # https://github.com/stub42/pytz python-slugify==5.0.2 # https://github.com/un33k/python-slugify Pillow argon2-cffi==21.1.0 # https://github.com/hynek/argon2_cffi -redis==3.5.3 # https://github.com/antirez/redis +redis==4.0.0 # https://github.com/antirez/redis bs4 pyyaml progress From 5b22a3f7c4e6d7983d7c206c0b15efcb537bd5f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Nov 2021 10:25:15 +0000 Subject: [PATCH 40/43] build(deps): Bump faker from 9.8.0 to 9.8.2 Bumps [faker](https://github.com/joke2k/faker) from 9.8.0 to 9.8.2. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.md) - [Commits](https://github.com/joke2k/faker/compare/v9.8.0...v9.8.2) --- updated-dependencies: - dependency-name: faker dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index fb5cfe7ed..0f1dfdc5e 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -29,7 +29,7 @@ coveralls # Django # ------------------------------------------------------------------------------ factory-boy==3.2.1 # https://github.com/FactoryBoy/factory_boy -faker==9.8.0 +faker==9.8.2 django-debug-toolbar==3.2.2 # https://github.com/jazzband/django-debug-toolbar django-extensions==3.1.5 # https://github.com/django-extensions/django-extensions From 367e0477c4b0995bbad3e336f93378325596e5c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Nov 2021 10:22:41 +0000 Subject: [PATCH 41/43] build(deps): Bump celery from 5.2.0 to 5.2.1 Bumps [celery](https://github.com/celery/celery) from 5.2.0 to 5.2.1. - [Release notes](https://github.com/celery/celery/releases) - [Changelog](https://github.com/celery/celery/blob/master/Changelog.rst) - [Commits](https://github.com/celery/celery/compare/v5.2.0...v5.2.1) --- updated-dependencies: - dependency-name: celery dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index fb5cfe7ed..15e19fc6c 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -41,5 +41,5 @@ pyopenssl # for running dev server under https # TODO: Why are we using this fork? Fabric3==1.14.post1 # https://github.com/mathiasertl/fabric -celery==5.2.0 +celery==5.2.1 django-celery-results==2.2.0 From 99570745a060c9e74e349f45ec46fb40a8bba24b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Nov 2021 10:22:58 +0000 Subject: [PATCH 42/43] build(deps): Bump boto3 from 1.20.3 to 1.20.7 Bumps [boto3](https://github.com/boto/boto3) from 1.20.3 to 1.20.7. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.20.3...1.20.7) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 67a81052b..96576a11f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -53,7 +53,7 @@ django-import-export==2.6.1 gitpython==3.1.24 # S3 Uploads -boto3==1.20.3 +boto3==1.20.7 django-storages==1.12.3 # https://github.com/jschneier/django-storages # Python function to stream unzip all the files in a ZIP archive, without loading the entire ZIP file into memory or any of its uncompressed files. From 063819fd5797f190a9440252e332a93ad417ab4d Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Wed, 17 Nov 2021 14:23:08 -0500 Subject: [PATCH 43/43] Bump version number and update changelog --- CHANGELOG.rst | 4 ++++ apps/__init__.py | 2 +- apps/readux/__init__.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1a6cef5ae..f4d606484 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,10 @@ CHANGELOG ========= +Release 2.2.1 +--------------------- +* Improvements to bulk ingest +* Export tasks converted to Celery. Release 2.2.0 --------------------- diff --git a/apps/__init__.py b/apps/__init__.py index c88b4e185..b73cbe3cc 100644 --- a/apps/__init__.py +++ b/apps/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.2.0" +__version__ = "2.2.1" __version_info__ = tuple( [ int(num) if num.isdigit() else num diff --git a/apps/readux/__init__.py b/apps/readux/__init__.py index 04188a16d..36a511eca 100644 --- a/apps/readux/__init__.py +++ b/apps/readux/__init__.py @@ -1 +1 @@ -__version__ = '2.2.0' +__version__ = '2.2.1'