WIP allow importing EADs nested under specified fonds on a dataset

Typically the fonds ID specifier is for `sync` update, where the info for the fonds is in the EAD to be imported. But sometimes we want to import items _under_ an existing fonds. This new `nest` option on the dataset allows this.
EHRI · Apr 9, 2024 · 7b7c1ea · 7b7c1ea
1 parent 50adfd0
commit 7b7c1ea
Show file tree

Hide file tree

Showing 12 changed files with 62 additions and 19 deletions.
diff --git a/conf/evolutions/default/1.sql b/conf/evolutions/default/1.sql
@@ -122,12 +122,12 @@ CREATE TABLE import_dataset (
     created         TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     item_id         TEXT,
     sync            BOOLEAN DEFAULT FALSE,
+    nest            BOOLEAN DEFAULT FALSE,
     status          VARCHAR(10) NOT NULL DEFAULT 'active',
     comments        TEXT,
     PRIMARY KEY (id, repo_id),
     UNIQUE (id, repo_id),
-    CONSTRAINT import_dataset_id_pattern CHECK (id ~ '^[a-z0-9_]+$') ,
-    CONSTRAINT import_dataset_item_id_pattern CHECK(item_id IS NULL OR item_id ~ concat('^', repo_id, '\-.+') )
+    CONSTRAINT import_dataset_id_pattern CHECK (id ~ '^[a-z0-9_]+$')
 );
 
 CREATE TABLE oaipmh_config (

diff --git a/etc/db_migrations/20240409_add_import_dataset_nest_column.sql b/etc/db_migrations/20240409_add_import_dataset_nest_column.sql
@@ -0,0 +1,3 @@
+ALTER TABLE import_dataset
+    ADD COLUMN status VARCHAR(10) NOT NULL DEFAULT 'active',
+    DROP CONSTRAINT import_dataset_item_id_pattern;
diff --git a/modules/admin/app/assets/js/datasets/components/_manager-dataset-list.vue b/modules/admin/app/assets/js/datasets/components/_manager-dataset-list.vue
@@ -94,6 +94,9 @@ export default {
       if (ds && this.editing && this.editing.id === ds.id) {
         this.editing = ds;
       }
+      if (ds && this.dataset && this.dataset.id === ds.id) {
+        this.dataset = ds;
+      }
     },
     stageName: function (code: ImportDatasetSrc): string {
       switch (code) {

diff --git a/modules/admin/app/assets/js/datasets/components/_modal-dataset-config.vue b/modules/admin/app/assets/js/datasets/components/_modal-dataset-config.vue
@@ -19,6 +19,7 @@ export default {
       src: this.info ? this.info.src : null,
       fonds: this.info ? this.info.fonds : null,
       sync: this.info ? this.info.sync : false,
+      nest: this.info ? this.info.nest : false,
       status: this.info ? this.info.status : "active",
       contentType: this.info ? this.info.contentType : null,
       notes: this.info ? this.info.notes : null,
@@ -39,6 +40,7 @@ export default {
         src: this.src,
         fonds: this.fonds,
         sync: this.sync,
+        nest: this.nest,
         status: this.status,
         contentType: this.contentType,
         notes: this.notes,
@@ -77,6 +79,7 @@ export default {
       if (!newValue) {
         this.fonds = null;
         this.sync = false;
+        this.nest = false;
       }
     }
   },
@@ -100,7 +103,8 @@ export default {
               || this.info.notes !== this.notes
               || this.info.fonds !== this.fonds
               || this.info.contentType !== this.contentType
-              || Boolean(this.info.sync) !== Boolean(this.sync))
+              || Boolean(this.info.sync) !== Boolean(this.sync)
+              || Boolean(this.info.nest) !== Boolean(this.nest))
           || this.info.status !== this.status;
     }
   },
@@ -177,10 +181,18 @@ export default {
         <input type="text" v-model="fonds" id="dataset-fonds" class="form-control" placeholder="(optional)"/>
       </div>
       <div class="form-group form-check">
-        <input v-bind:disabled="!(this.fonds && this.isValidFonds)" v-model="sync" class="form-check-input"
-               id="opt-sync" type="checkbox"/>
-        <label class="form-check-label" for="opt-sync">
-          Synchronise fonds with dataset
+          <input v-bind:disabled="!(this.fonds && this.isValidFonds)" v-model="sync" class="form-check-input"
+                 id="opt-sync" type="checkbox"/>
+          <label class="form-check-label" for="opt-sync" data-toggle="tooltip" title="Move or remove existing data to match EAD structure.">
+              Synchronise fonds with dataset
+          </label>
+      </div>
+      <div class="form-group form-check">
+        <input v-bind:disabled="!(this.fonds && this.isValidFonds)" v-model="nest" class="form-check-input"
+               id="opt-nest" type="checkbox"/>
+        <label class="form-check-label" for="opt-nest" data-toggle="tooltip"
+               title="Default (non-nested) behavior assumes EAD includes the specified fonds and imports at repository level.">
+            Nest items beneath specified fonds
         </label>
       </div>
       <div class="form-group">

diff --git a/modules/admin/app/assets/js/datasets/types.ts b/modules/admin/app/assets/js/datasets/types.ts
@@ -96,6 +96,7 @@ export interface ImportDatasetInfo {
   src: ImportDatasetSrc,
   fonds?: string,
   sync: boolean,
+  nest: boolean,
   status: ImportDatasetStatus,
   contentType?: string,
   notes?: string,

diff --git a/modules/admin/app/controllers/datasets/ImportConfigs.scala b/modules/admin/app/controllers/datasets/ImportConfigs.scala
@@ -66,12 +66,18 @@ case class ImportConfigs @Inject()(
     val urlsF = getUrlMap(request.body, prefix(id, ds, FileStage.Output))
     for (urls <- urlsF; dataset <- datasets.get(id, ds)) yield {
 
+      val scopeType = if (dataset.fonds.isDefined && dataset.nest)
+        models.ContentTypes.DocumentaryUnit else models.ContentTypes.Repository
+
+      val scopeId = if (dataset.fonds.isDefined && dataset.nest)
+        dataset.fonds.get else id
+
       // In the normal case this will be a list of one item, but
       // if batchSize is set, it will be a list of batches.
       val params: List[IngestParams] = urls.map { urlBatch =>
         IngestParams(
-          scopeType = models.ContentTypes.Repository,
-          scope = id,
+          scopeType = scopeType,
+          scope = scopeId,
           data = UrlMapPayload(urlBatch),
           allowUpdate = request.body.config.allowUpdates,
           useSourceId = request.body.config.useSourceId,
@@ -94,7 +100,7 @@ case class ImportConfigs @Inject()(
       // Use the sync endpoint if this fonds is synced and we a) are doing the complete
       // set and b) have a fonds.
       // Don't allow sync on the repository scope, because it is too dangerous.
-      val taskType = if (dataset.sync && request.body.files.isEmpty && dataset.fonds.isDefined)
+      val taskType = if (dataset.fonds.isDefined && request.body.files.isEmpty && dataset.sync)
         IngestDataType.EadSync else IngestDataType.Ead
 
       val ingestTasks = params.zipWithIndex.map { case (batchParams, i) =>

diff --git a/modules/admin/app/controllers/datasets/ImportDatasets.scala b/modules/admin/app/controllers/datasets/ImportDatasets.scala
@@ -153,6 +153,7 @@ case class ImportDatasets @Inject()(
   }
 
   def create(id: String): Action[ImportDatasetInfo] = EditAction(id).async(apiJson[ImportDatasetInfo]) { implicit request =>
+    // TODO: Check that the fondsId is valid
     datasets.create(id, request.body).map { ds =>
       Created(Json.toJson(ds))
     }.recover {
@@ -161,6 +162,7 @@ case class ImportDatasets @Inject()(
   }
 
   def update(id: String, ds: String): Action[ImportDatasetInfo] = EditAction(id).async(apiJson[ImportDatasetInfo]) { implicit request =>
+    // TODO: Check that the fondsId is valid
     datasets.update(id, ds, request.body).map { ds =>
       Ok(Json.toJson(ds))
     }
@@ -187,4 +189,12 @@ case class ImportDatasets @Inject()(
       }))
     }
   }
+
+  private def checkFondsId(id: String, dataset: ImportDatasetInfo)(implicit userOpt: Option[UserProfile]): Future[Boolean] = {
+    dataset.fonds.map { fondsId =>
+      userDataApi.get[DocumentaryUnit](fondsId).map { unit =>
+        unit.holder.exists(_.id == id)
+      }
+    }.getOrElse(Future.successful(true))
+  }
 }
diff --git a/modules/admin/app/models/ImportDataset.scala b/modules/admin/app/models/ImportDataset.scala
@@ -16,6 +16,7 @@ case class ImportDataset(
   created: Instant,
   fonds: Option[String] = None,
   sync: Boolean = false,
+  nest: Boolean = false,
   status: ImportDataset.Status.Value = ImportDataset.Status.Active,
   notes: Option[String] = None,
 )
@@ -49,6 +50,7 @@ case class ImportDatasetInfo(
   contentType: Option[String] = None,
   fonds: Option[String] = None,
   sync: Boolean = false,
+  nest: Boolean = false,
   status: ImportDataset.Status.Value = ImportDataset.Status.Active,
   notes: Option[String] = None,
 )

diff --git a/modules/admin/app/services/datasets/SqlImportDatasetService.scala b/modules/admin/app/services/datasets/SqlImportDatasetService.scala
@@ -15,7 +15,7 @@ case class SqlImportDatasetService @Inject()(db: Database, actorSystem: ActorSys
 
   private implicit val parser: RowParser[ImportDataset] =
     Macro.parser[ImportDataset](
-      "repo_id", "id", "name", "type", "content_type", "created", "item_id", "sync", "status", "comments")
+      "repo_id", "id", "name", "type", "content_type", "created", "item_id", "sync", "nest", "status", "comments")
 
   override def listAll(): Future[Map[String, Seq[ImportDataset]]] = Future {
     db.withConnection { implicit conn =>
@@ -54,7 +54,7 @@ case class SqlImportDatasetService @Inject()(db: Database, actorSystem: ActorSys
   override def create(repoId: String, info: ImportDatasetInfo): Future[ImportDataset] = Future {
     db.withConnection { implicit conn =>
       try {
-        SQL"""INSERT INTO import_dataset (repo_id, id, name, type, content_type, item_id, sync, status, comments)
+        SQL"""INSERT INTO import_dataset (repo_id, id, name, type, content_type, item_id, sync, nest, status, comments)
           VALUES (
             $repoId,
             ${info.id},
@@ -63,6 +63,7 @@ case class SqlImportDatasetService @Inject()(db: Database, actorSystem: ActorSys
             ${info.contentType},
             ${info.fonds.filter(_.trim.nonEmpty)},
             ${info.sync},
+            ${info.nest},
             ${info.status},
             ${info.notes}
           )
@@ -83,6 +84,7 @@ case class SqlImportDatasetService @Inject()(db: Database, actorSystem: ActorSys
               content_type = ${info.contentType},
               item_id = ${info.fonds.filter(_.trim.nonEmpty)},
               sync = ${info.sync},
+              nest = ${info.nest},
               status = ${info.status},
               comments = ${info.notes.filter(_.trim.nonEmpty)}
             WHERE repo_id = $repoId
@@ -101,6 +103,7 @@ case class SqlImportDatasetService @Inject()(db: Database, actorSystem: ActorSys
           'type -> item.src,
           'content_type -> item.contentType,
           'item_id -> item.fonds.filter(_.trim.nonEmpty),
+          'nest -> item.nest,
           'sync -> item.sync,
           'status -> item.status,
           'comments -> item.notes
@@ -114,6 +117,7 @@ case class SqlImportDatasetService @Inject()(db: Database, actorSystem: ActorSys
                       content_type = {content_type},
                       item_id = {item_id},
                       sync = {sync},
+                      nest = {nest},
                       status = {status},
                       comments = {comments}"""
       val batch = BatchSql(q, inserts.head, inserts.tail: _*)

diff --git a/modules/admin/app/services/ingest/WSIngestService.scala b/modules/admin/app/services/ingest/WSIngestService.scala
@@ -128,7 +128,6 @@ case class WSIngestService @Inject()(
     }
   }
 
-  // Run the actual data ingest on the backend
   override def importData(data: IngestData): Future[IngestResult] = {
     import scala.concurrent.duration._
 

diff --git a/test/integration/admin/ImportDatasetsSpec.scala b/test/integration/admin/ImportDatasetsSpec.scala
@@ -28,6 +28,7 @@ class ImportDatasetsSpec extends IntegrationTestRunner with ResourceUtils {
         "fonds" -> None,
         "notes" -> None,
         "sync" -> false,
+        "nest" -> false,
         "status" -> ImportDataset.Status.Active,
       )
       val r = FakeRequest(routes.create("r1")).withUser(privilegedUser).callWith(data)
@@ -43,6 +44,7 @@ class ImportDatasetsSpec extends IntegrationTestRunner with ResourceUtils {
         "fonds" -> None,
         "notes" -> None,
         "sync" -> true,
+        "nest" -> false,
         "status" -> ImportDataset.Status.Active,
       )
       val r = FakeRequest(routes.update("r1", "default")).withUser(privilegedUser).callWith(data)

diff --git a/test/resources/import-dataset-fixtures.sql b/test/resources/import-dataset-fixtures.sql
@@ -1,8 +1,9 @@
-INSERT INTO import_dataset (repo_id, id, name, type, status, comments)
-VALUES ('r1', 'default', 'Default', 'upload', 'active', 'test')
-     , ('r2', 'default', 'Default', 'oaipmh', 'active', 'test')
-     , ('r1', 'oaipmh_test', 'OaiPmh Test', 'oaipmh', 'active', 'test')
-     , ('r1', 'rs_test', 'RS Test', 'rs', 'active', 'test')
-     , ('r1', 'urlset_test', 'URL Set Test', 'urlset', 'active', 'test')
+INSERT INTO import_dataset (repo_id, id, name, type, item_id, nest, status, comments)
+VALUES ('r1', 'default', 'Default', 'upload', NULL, false, 'active', 'test')
+     , ('r2', 'default', 'Default', 'oaipmh', NULL, false, 'active', 'test')
+     , ('r1', 'oaipmh_test', 'OaiPmh Test', 'oaipmh', NULL, false, 'active', 'test')
+     , ('r1', 'rs_test', 'RS Test', 'rs', NULL, false, 'active', 'test')
+     , ('r1', 'urlset_test', 'URL Set Test', 'urlset', NULL, false, 'active', 'test')
+     , ('r1', 'nest_test', 'URL Set Test', 'urlset', 'nl-r1-m19', true, 'active', 'test')
      ;
-Original file line number
+Diff line change
@@ Expand Up / @@ -128,7 +128,6 @@ case class WSIngestService @Inject()( @@
         }
       }
-      // Run the actual data ingest on the backend
       override def importData(data: IngestData): Future[IngestResult] = {
         import scala.concurrent.duration._
@@ Expand Down @@