[DONE] Correct archive packing (#1133)

* Do not truncate chars in dublincore metadata, to avoid an html entity being truncated an thus preventing xml to be parsed.
+ now ignored video are printed in create_archive_package

* Add i18n lang files

pod/locale/fr/LC_MESSAGES/django.po

@@ -5,7 +5,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: Pod\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-05-15 08:40+0200\n"
+"POT-Creation-Date: 2024-05-17 15:36+0000\n"
 "PO-Revision-Date: \n"
 "Last-Translator: obado <[email protected]>\n"
 "Language-Team: Pod Team [email protected]\n"
@@ -1238,7 +1238,6 @@ msgstr ""
 #: pod/completion/models.py pod/enrichment/models.py pod/playlist/models.py
 #: pod/recorder/models.py pod/video/models.py
 #: pod/video/templates/channel/channel.html
-#: pod/video/templates/videos/video_breadcrumbs.html
 #: pod/video/templates/videos/videos.html pod/video_encode_transcript/models.py
 msgid "Video"
 msgstr "Vidéo"
@@ -2078,6 +2077,7 @@ msgid "Ending credits"
 msgstr "Générique de fin"
 
 #: pod/dressing/models.py pod/video/apps.py pod/video/models.py
+#: pod/video/templates/videos/video_breadcrumbs.html
 #: pod/video/templates/videos/videos.html
 msgid "Videos"
 msgstr "Vidéos"
@@ -8184,13 +8184,13 @@ msgstr ""
 "l’adresse suivante : %(email_address)s</p>\n"
 
 #: pod/video/management/commands/check_obsolete_videos.py
-msgid "Your video will be obsolete"
-msgstr "Votre vidéo sera obsolète"
+#, python-format
+msgid "Sending mail to %(to_email)s for video %(title)s."
+msgstr "Envoi d’un courriel à %(to_email)s pour la vidéo %(title)s."
 
 #: pod/video/management/commands/check_obsolete_videos.py
-#, python-format
-msgid "Mail sent to %(to_email)s for video %(title)s."
-msgstr "Courriel envoyé à %(to_email)s pour la vidéo %(title)s."
+msgid "Your video will be obsolete"
+msgstr "Votre vidéo sera obsolète"
 
 #: pod/video/management/commands/check_obsolete_videos.py
 #, python-format
@@ -8262,10 +8262,11 @@ msgstr "Dans %(deadline)s jours :"
 #: pod/video/management/commands/create_archive_package.py
 #, python-format
 msgid ""
-"Package archiving done. %(amount)s video packaged (%(weight)s - "
-"[%(duration)s])"
+"Package archiving done. %(amount)s video(s) packaged (%(weight)s - "
+"[%(duration)s]) - %(nb_ignored)s video(s) ignored."
 msgstr ""
-"Archivage effectué. %(amount)s vidéos traitées (%(weight)s - [%(duration)s])"
+"Archivage effectué. %(amount)s vidéos traitées (%(weight)s - [%(duration)s]) "
+"- %(nb_ignored)s vidéo(s) ignorée(s)."
 
 #: pod/video/management/commands/create_archive_package.py
 #, python-format
@@ -8285,8 +8286,17 @@ msgstr ""
 "code> sans l’option <code>--dry</code> pour les supprimer de %s."
 
 #: pod/video/management/commands/create_archive_package.py
-msgid "Packaging archived videos on Pod"
-msgstr "Déplacement des vidéos archivées sur Pod"
+msgid ""
+"And below is the list of ignored videos that were not packaged because they "
+"have been recently viewed."
+msgstr ""
+"Vous trouverez ci-dessous la liste des vidéos ignorées qui n'ont pas été "
+"regroupées parce qu'elles ont été visionnées récemment."
+
+#: pod/video/management/commands/create_archive_package.py
+#, python-format
+msgid "Packaging %s archived videos on Pod"
+msgstr "Déplacement de %s vidéos archivées sur Pod"
 
 #: pod/video/models.py pod/video/views.py
 msgid "None / All"

pod/locale/nl/LC_MESSAGES/django.po

@@ -5,7 +5,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: Pod\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-05-15 08:40+0200\n"
+"POT-Creation-Date: 2024-05-17 15:36+0000\n"
 "PO-Revision-Date: 2024-05-15 09:01+0200\n"
 "Last-Translator: obado <[email protected]>\n"
 "Language-Team: \n"
@@ -1173,7 +1173,6 @@ msgstr ""
 #: pod/completion/models.py pod/enrichment/models.py pod/playlist/models.py
 #: pod/recorder/models.py pod/video/models.py
 #: pod/video/templates/channel/channel.html
-#: pod/video/templates/videos/video_breadcrumbs.html
 #: pod/video/templates/videos/videos.html pod/video_encode_transcript/models.py
 msgid "Video"
 msgstr ""
@@ -1973,6 +1972,7 @@ msgid "Ending credits"
 msgstr ""
 
 #: pod/dressing/models.py pod/video/apps.py pod/video/models.py
+#: pod/video/templates/videos/video_breadcrumbs.html
 #: pod/video/templates/videos/videos.html
 msgid "Videos"
 msgstr ""
@@ -7612,12 +7612,12 @@ msgid ""
 msgstr ""
 
 #: pod/video/management/commands/check_obsolete_videos.py
-msgid "Your video will be obsolete"
+#, python-format
+msgid "Sending mail to %(to_email)s for video %(title)s."
 msgstr ""
 
 #: pod/video/management/commands/check_obsolete_videos.py
-#, python-format
-msgid "Mail sent to %(to_email)s for video %(title)s."
+msgid "Your video will be obsolete"
 msgstr ""
 
 #: pod/video/management/commands/check_obsolete_videos.py
@@ -7680,8 +7680,8 @@ msgstr ""
 #: pod/video/management/commands/create_archive_package.py
 #, python-format
 msgid ""
-"Package archiving done. %(amount)s video packaged (%(weight)s - "
-"[%(duration)s])"
+"Package archiving done. %(amount)s video(s) packaged (%(weight)s - "
+"[%(duration)s]) - %(nb_ignored)s video(s) ignored."
 msgstr ""
 
 #: pod/video/management/commands/create_archive_package.py
@@ -7699,7 +7699,14 @@ msgid ""
 msgstr ""
 
 #: pod/video/management/commands/create_archive_package.py
-msgid "Packaging archived videos on Pod"
+msgid ""
+"And below is the list of ignored videos that were not packaged because they "
+"have been recently viewed."
+msgstr ""
+
+#: pod/video/management/commands/create_archive_package.py
+#, python-format
+msgid "Packaging %s archived videos on Pod"
 msgstr ""
 
 #: pod/video/models.py pod/video/views.py

pod/video/management/commands/create_archive_package.py

@@ -126,7 +126,7 @@ def export_complement(
         """Store a video complement as json."""
         if len(export_objects) > 0:
             export_file = os.path.join(folder, "%s.json" % export_type)
-            print("Export %s %s." % (len(export_objects), export_type))
+            print("  * Export %s %s." % (len(export_objects), export_type))
             if not self.dry_mode:
                 with open(export_file, "w") as out:
                     content = serialize("json", export_objects)
@@ -135,7 +135,7 @@ def export_complement(
     def move_video_to_archive(self, mediaPackage_dir: str, vid: Video) -> None:
         """Move video source file to mediaPackage_dir."""
         if os.access(vid.video.path, os.F_OK):
-            print("Moving %s" % vid.video.path)
+            print("  * Moving %s" % vid.video.path)
             if not self.dry_mode:
                 shutil.move(
                     vid.video.path,
@@ -146,7 +146,7 @@ def move_video_to_archive(self, mediaPackage_dir: str, vid: Video) -> None:
             # Deletes the video object and the associated folder (encoding, logs, etc.)
             # Remove thumbnails (x3)
         else:
-            print("ERROR: Cannot acces to file '%s'." % vid.video.path)
+            print("ERROR: Cannot access to file '%s'." % vid.video.path)
 
     def archive_pack(self, video_dir: str, user_name: str, vid: Video) -> None:
         """Create a archive package for Video vid."""
@@ -178,24 +178,26 @@ def archive_pack(self, video_dir: str, user_name: str, vid: Video) -> None:
             self.export_complement(
                 mediaPackage_dir, model.__name__, model.objects.filter(video=vid)
             )
+        # Export also the video itself as json
+        self.export_complement(mediaPackage_dir, "Video", vid)
 
         # Store also files linked to Enrichments
         for enrich in Enrichment.objects.filter(video=vid):
             if enrich.document:
-                print("Copying %s..." % enrich.document.file.path)
+                print("  * Copying %s..." % enrich.document.file.path)
                 shutil.copy(enrich.document.file.path, mediaPackage_dir)
             if enrich.image:
-                print("Copying %s..." % enrich.image.file.path)
+                print("  * Copying %s..." % enrich.image.file.path)
                 shutil.copy(enrich.image.file.path, mediaPackage_dir)
 
         # Store file complements.
         for file in Document.objects.filter(video=vid):
-            print("Copying %s..." % file.document.file.path)
+            print("  * Copying %s..." % file.document.file.path)
             shutil.copy(file.document.file.path, mediaPackage_dir)
 
         # Store additional tracks (caption / subtitles)
         for track in Track.objects.filter(video=vid):
-            print("Copying %s..." % track.src.file.path)
+            print("  * Copying %s..." % track.src.file.path)
             shutil.copy(track.src.file.path, mediaPackage_dir)
 
         # TODO:
@@ -219,6 +221,7 @@ def handle(self, *args, **options) -> None:
         total_processed = 0
         total_weight = 0
         list_video = []
+        ignored_video = []
 
         if options["dry"]:
             self.dry_mode = True
@@ -245,9 +248,9 @@ def handle(self, *args, **options) -> None:
                 # Do not archive a video with recent views.
                 # (if video has been shared with a token, it can still be viewed)
                 print(
-                    "- Video %s ignored (%s recent views)"
-                    % (vid.slug, vid.recentViewcount)
+                    "  * IGNORED (%s recent views)" % vid.recentViewcount
                 )
+                ignored_video.append(str(vid))
                 continue
 
             # Recover original video slug
@@ -263,23 +266,27 @@ def handle(self, *args, **options) -> None:
                 list_video.append(str(vid))
                 self.archive_pack(video_dir, csv_entry["User name"], vid)
             else:
-                print("Video %s not present in archived file" % vid.id)
+                print("  * Video %s not present in archived file" % vid.id)
             print("---")
         # Convert seconds in human readable time
         total_duration = str(timedelta(seconds=total_duration))
         total_msg = _(
-            "Package archiving done. %(amount)s video packaged (%(weight)s - [%(duration)s])"
+            "Package archiving done. %(amount)s video(s) packaged (%(weight)s - [%(duration)s])"
+            " - %(nb_ignored)s video(s) ignored."
         ) % {
             "amount": total_processed,
             "weight": sizeof_fmt(total_weight),
             "duration": total_duration,
+            "nb_ignored": len(ignored_video)
         }
 
         print(total_msg)
         if total_processed > 0:
-            self.inform_managers(list_video, total_msg)
+            self.inform_managers(list_video, ignored_video, total_msg, total_processed)
 
-    def inform_managers(self, list_video: list, total_msg: str) -> None:
+    def inform_managers(
+        self, list_video: list, ignored_video: list, total_msg: str, total_processed: int
+    ) -> None:
         """Inform site managers of packaged archives."""
         msg_html = [_("Hello manager(s) of  %s,") % __TITLE_SITE__]
         msg_html.append("<br>")
@@ -307,10 +314,19 @@ def inform_managers(self, list_video: list, total_msg: str) -> None:
 
         msg_html.append("<p>%s</p>" % msg)
         msg_html.append(self.get_list_video_html(list_video))
+
+        msg = _(
+            "And below is the list of ignored videos that were not packaged "
+            "because they have been recently viewed."
+        )
+
+        msg_html.append("<p>%s</p>" % msg)
+        msg_html.append(self.get_list_video_html(ignored_video))
+
         msg_html.append("<p>%s</p>" % total_msg)
         msg_html.append("<p>%s</p>" % _("Regards."))
         msg_html = "\n".join(msg_html)
-        subject = _("Packaging archived videos on Pod")
+        subject = _("Packaging %s archived videos on Pod") % total_processed
         mail_managers(
             subject,
             striptags(msg_html),

pod/video/templates/videos/dublincore.html

@@ -2,7 +2,7 @@
 {% if xml == True %}<rdf:Description rdf:about="http://dublincore.org/">{% endif %}
 {% for key, value in video.get_dublin_core.items %}
   {% if xml == True %}
-    <{{key}}>{{value|metaformat|striptags|truncatechars:300}}</{{key}}>
+    <{{key}}>{{value|metaformat|striptags}}</{{key}}>
   {% else %}
     <meta name="{{key}}" content="{{key|metaformat|safe|striptags|truncatechars:300}}" />
   {% endif %}

setup.cfg

@@ -1,5 +1,5 @@
 [flake8]
-exclude = .git,pod/*/migrations/*.py,*_settings.py,node_modules/*/*.py,pod/static/*.py,pod/custom/tenants/*/*.py
+exclude = .git,pod/*/migrations/*.py,*_settings.py,node_modules/*/*.py,pod/static/*.py,pod/custom/tenants/*/*.py,./venv/*
 max-complexity = 7
 max-line-length = 90
 # See https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8