Configure GenericFileImporter for MEDIA

dtinit · Nov 15, 2024 · 40e18b1 · 40e18b1
1 parent 312862b
commit 40e18b1
Show file tree

Hide file tree

Showing 7 changed files with 107 additions and 2 deletions.
diff --git a/...r-generic/src/main/java/org/datatransferproject/datatransfer/generic/GenericImporter.java b/...r-generic/src/main/java/org/datatransferproject/datatransfer/generic/GenericImporter.java
@@ -2,6 +2,7 @@
 
 import static java.lang.String.format;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
@@ -18,13 +19,25 @@
 import org.datatransferproject.spi.transfer.provider.ImportResult.ResultType;
 import org.datatransferproject.spi.transfer.provider.Importer;
 import org.datatransferproject.types.common.models.ContainerResource;
+import org.datatransferproject.types.common.models.photos.PhotoModel;
+import org.datatransferproject.types.common.models.videos.VideoModel;
 import org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData;
 
 @FunctionalInterface
 interface ContainerMapper<C extends ContainerResource> {
   public Iterable<ImportableData> apply(C containerResource, ObjectMapper om);
 }
 
+@JsonIgnoreProperties({
+  "contentUrl",
+  "fetchableUrl",
+  "inTempStore",
+  "identifier",
+  "dataId",
+  "headline"
+})
+abstract class MediaSkipFieldsMixin {}
+
 public class GenericImporter<C extends ContainerResource>
     implements Importer<TokensAndUrlAuthData, C> {
   ContainerMapper<C> containerUnpacker;
@@ -38,6 +51,9 @@ public GenericImporter(ContainerMapper<C> containerUnpacker, Monitor monitor) {
     this.monitor = monitor;
     this.containerUnpacker = containerUnpacker;
     om.registerModule(new JavaTimeModule());
+    // TODO: this probably shouldn't live here
+    om.addMixIn(VideoModel.class, MediaSkipFieldsMixin.class);
+    om.addMixIn(PhotoModel.class, MediaSkipFieldsMixin.class);
   }
 
   @Override

diff --git a/...neric/src/main/java/org/datatransferproject/datatransfer/generic/GenericImporterTest.java b/...neric/src/main/java/org/datatransferproject/datatransfer/generic/GenericImporterTest.java
@@ -8,6 +8,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Date;
 import java.util.Random;
 import java.util.UUID;
 import org.datatransferproject.api.launcher.Constants.Environment;
@@ -25,13 +26,17 @@
 import org.datatransferproject.types.common.models.blob.BlobbyStorageContainerResource;
 import org.datatransferproject.types.common.models.blob.DigitalDocumentWrapper;
 import org.datatransferproject.types.common.models.blob.DtpDigitalDocument;
+import org.datatransferproject.types.common.models.media.MediaAlbum;
+import org.datatransferproject.types.common.models.media.MediaContainerResource;
+import org.datatransferproject.types.common.models.photos.PhotoModel;
 import org.datatransferproject.types.common.models.social.SocialActivityActor;
 import org.datatransferproject.types.common.models.social.SocialActivityAttachment;
 import org.datatransferproject.types.common.models.social.SocialActivityAttachmentType;
 import org.datatransferproject.types.common.models.social.SocialActivityContainerResource;
 import org.datatransferproject.types.common.models.social.SocialActivityLocation;
 import org.datatransferproject.types.common.models.social.SocialActivityModel;
 import org.datatransferproject.types.common.models.social.SocialActivityType;
+import org.datatransferproject.types.common.models.videos.VideoModel;
 import org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData;
 import org.datatransferproject.types.transfer.errors.ErrorDetail;
 
@@ -218,9 +223,41 @@ public static void main(String args[]) throws Exception {
                     "foodir",
                     Arrays.asList(
                         new DigitalDocumentWrapper(
-                            new DtpDigitalDocument("bar", "2020-02-01", "video/mp4"),
+                            new DtpDigitalDocument("bar.mp4", "2020-02-01", "video/mp4"),
                             "video/mp4",
                             "foobarfile")),
                     new ArrayList<>()))));
+
+    @SuppressWarnings("unchecked")
+    Importer<TokensAndUrlAuthData, MediaContainerResource> mediaImporter =
+        (Importer<TokensAndUrlAuthData, MediaContainerResource>)
+            extension.getImporter(DataVertical.MEDIA);
+    mediaImporter.importItem(
+        jobId,
+        idempotentImporter,
+        new TokensAndUrlAuthData("foo", "bar", "baz"),
+        new MediaContainerResource(
+            Arrays.asList(new MediaAlbum("album123", "Album 123", "Album description")),
+            Arrays.asList(
+                new PhotoModel(
+                    "bar.jpeg",
+                    "cachedVal2",
+                    "Bar description",
+                    "image/jpeg",
+                    "cachedVal2",
+                    "album123",
+                    true,
+                    null,
+                    new Date())),
+            Arrays.asList(
+                new VideoModel(
+                    "foo.mp4",
+                    "cachedVal1",
+                    "Foo description",
+                    "video/mp4",
+                    "cachedVal1",
+                    "album123",
+                    true,
+                    new Date()))));
   }
 }
diff --git a/.../src/main/java/org/datatransferproject/datatransfer/generic/GenericTransferExtension.java b/.../src/main/java/org/datatransferproject/datatransfer/generic/GenericTransferExtension.java
@@ -2,6 +2,7 @@
 
 import static java.lang.String.format;
 import static org.datatransferproject.types.common.models.DataVertical.BLOBS;
+import static org.datatransferproject.types.common.models.DataVertical.MEDIA;
 import static org.datatransferproject.types.common.models.DataVertical.SOCIAL_POSTS;
 
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
@@ -13,6 +14,7 @@
 import java.util.Map;
 import java.util.Queue;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.datatransferproject.api.launcher.ExtensionContext;
 import org.datatransferproject.spi.cloud.storage.JobStore;
 import org.datatransferproject.spi.transfer.extension.TransferExtension;
@@ -22,10 +24,16 @@
 import org.datatransferproject.types.common.models.DataVertical;
 import org.datatransferproject.types.common.models.blob.BlobbyStorageContainerResource;
 import org.datatransferproject.types.common.models.blob.DigitalDocumentWrapper;
+import org.datatransferproject.types.common.models.media.MediaContainerResource;
 import org.datatransferproject.types.common.models.social.SocialActivityContainerResource;
 
+/**
+ * Wrapper to adapt items known to be in temp storage (e.g. BLOB data) into {@link DownloadableItem}
+ *
+ * <p>It's useful to wrap such items so upstream code can consume either known temp store'd items or
+ * items the Importer has to download itself (some MEDIA items) from the same interface.
+ */
 class CachedDownloadableItem implements DownloadableItem {
-
   private String cachedId;
   private String name;
 
@@ -129,6 +137,41 @@ public void initialize(ExtensionContext context) {
             jobStore,
             context.getMonitor()));
 
+    importerMap.put(
+        MEDIA,
+        new GenericFileImporter<MediaContainerResource>(
+            (container, om) ->
+                Stream.concat(
+                        container.getAlbums().stream()
+                            .map(
+                                album ->
+                                    new ImportableData(
+                                        om.valueToTree(album),
+                                        album.getIdempotentId(),
+                                        album.getName())),
+                        Stream.concat(
+                            container.getVideos().stream()
+                                .map(
+                                    (video) -> {
+                                      return new ImportableFileData(
+                                          video,
+                                          om.valueToTree(video),
+                                          video.getIdempotentId(),
+                                          video.getName());
+                                    }),
+                            container.getPhotos().stream()
+                                .map(
+                                    photo -> {
+                                      return new ImportableFileData(
+                                          photo,
+                                          om.valueToTree(photo),
+                                          photo.getIdempotentId(),
+                                          photo.getName());
+                                    })))
+                    .collect(Collectors.toList()),
+            jobStore,
+            context.getMonitor()));
+
     importerMap.put(
         SOCIAL_POSTS,
         new GenericImporter<SocialActivityContainerResource>(

diff --git a/...-types-common/src/main/java/org/datatransferproject/types/common/models/FavoriteInfo.java b/...-types-common/src/main/java/org/datatransferproject/types/common/models/FavoriteInfo.java
@@ -1,10 +1,13 @@
 package org.datatransferproject.types.common.models;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
 import java.io.Serializable;
 import java.util.Date;
 
 /** FavoriteInfo about a {@code MediaItem}. */
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME)
 public class FavoriteInfo implements Serializable {
 
   @JsonProperty("favorite")

diff --git a/...es-common/src/main/java/org/datatransferproject/types/common/models/media/MediaAlbum.java b/...es-common/src/main/java/org/datatransferproject/types/common/models/media/MediaAlbum.java
@@ -3,6 +3,7 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Preconditions;
 import java.util.ArrayList;
@@ -15,6 +16,7 @@
 import javax.annotation.Nonnull;
 import org.datatransferproject.types.common.models.videos.VideoAlbum;
 
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME)
 public class MediaAlbum implements ImportableItem {
   private final String id;
   private String name;

diff --git a/...s-common/src/main/java/org/datatransferproject/types/common/models/photos/PhotoModel.java b/...s-common/src/main/java/org/datatransferproject/types/common/models/photos/PhotoModel.java
@@ -18,6 +18,7 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Objects;
 
@@ -28,6 +29,7 @@
 import org.datatransferproject.types.common.DownloadableFile;
 import org.datatransferproject.types.common.models.FavoriteInfo;
 
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME)
 public class PhotoModel implements DownloadableFile {
 
   private String title;

diff --git a/...s-common/src/main/java/org/datatransferproject/types/common/models/videos/VideoModel.java b/...s-common/src/main/java/org/datatransferproject/types/common/models/videos/VideoModel.java
@@ -19,6 +19,7 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Objects;
 import java.util.Date;
@@ -29,6 +30,7 @@
 import javax.annotation.Nonnull;
 
 
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME)
 public class VideoModel extends MediaObject implements DownloadableFile {
 
   private String dataId;