Upload fixes

.ansible/roles/web/templates/nginx.j2

@@ -4,21 +4,28 @@ upstream {{ project_name }} {
 
 server {
     listen 80 default_server;
+    listen [::]:80 default_server;
     server_name {{ web_domain }} {{ web_domain_alias }};
     return 301 https://{{ web_domain }}$request_uri;
 }
 
 server {
-    listen 443 ssl;
+    listen 443 http2 ssl;
+    listen [::]:443 http2 ssl;
+    server_name {{ web_domain_alias }};
+
     ssl_certificate {{ certbot_key_dir }}/fullchain.pem;
     ssl_certificate_key {{ certbot_key_dir }}/privkey.pem;
-    server_name {{ web_domain_alias }};
+    include {{ certbot_conf_dir }}/options-ssl-nginx.conf;
+    ssl_dhparam {{ certbot_conf_dir }}/ssl-dhparams.pem;
+
     return 301 https://{{ web_domain }}$request_uri;
 }
 
 server {
 
-    listen 443 ssl default_server;
+    listen 443 http2 ssl default_server;
+    listen [::]:443 http2 ssl default_server;
     server_name {{ web_domain }};
 
     ssl_certificate {{ certbot_key_dir }}/fullchain.pem;

.devcontainer/Dockerfile

@@ -21,6 +21,7 @@ ENV APP_ENV=${APP_ENV} \
 RUN apt-get update \
     && apt-get --no-install-recommends -y install \
     bash-completion \
+    ftp \
     libexempi8 \
     libmemcached-dev \
     postgresql-client \

creator/tasks/id3.py

@@ -1,9 +1,9 @@
 import celery
-from django import utils as django_utils
 from django import urls
+from django import utils as django_utils
 
-from frontrowcrew.utils import sites
 from creator import models
+from frontrowcrew.utils import sites
 
 
 @celery.shared_task
@@ -55,9 +55,9 @@ def apply_id3_tags(
     album_image = None
     album_image_data = episode.image
     if album_image_data:
-        album_image = album_image_data.open("rb")
+        album_image = album_image_data.file
     elif podcast.image:
-        album_image = podcast.image.open("rb")
+        album_image = podcast.image.file
 
     id3_data["album_image"] = album_image
 
@@ -80,7 +80,4 @@ def apply_id3_tags(
 
     mp3.set_id3(**id3_data)
 
-    if album_image:
-        album_image.close()
-
     return mp3.id

creator/templates/creator/base.html

@@ -22,7 +22,7 @@
     {% block nav%}
     <nav class="navbar navbar-expand-lg navbar-light bg-light">
         <div class="container-fluid">
-            <a class="navbar-brand" href="#">Navbar</a>
+            <a class="navbar-brand" href="#">Podcast Creator</a>
             <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarNav" aria-controls="navbarNav" aria-expanded="false" aria-label="Toggle navigation">
                 <span class="navbar-toggler-icon"></span>
             </button>

creator/views.py

@@ -1,16 +1,14 @@
 import http
-from django import forms
+
+from django import forms, shortcuts, urls
 from django import http as django_http
-from django import shortcuts
-from django import urls
-from django.views.decorators import vary as vary_decorators
 from django.contrib.auth import decorators as auth_decorators
+from django.views.decorators import vary as vary_decorators
 
 from media import models as media_models
 
 from . import forms as creator_forms
-from . import models
-from . import utils
+from . import models, utils
 
 
 @auth_decorators.login_required

etl/data/mkforumdb.py

@@ -63,9 +63,6 @@ def load_csv(csv_name: str):
             username = row[4]
             category_id = row[5]
             category_name = row[6]
-            if not category_id.isnumeric():
-                breakpoint()
-                print(row)
             csv_data.append(
                 (
                     id,

etl/management/commands/etl.py

@@ -1,18 +1,17 @@
-import tqdm
-import typing
 import os
 import sqlite3
+import typing
 
+import tqdm
 from django.apps import apps as django_apps
 from django.conf import settings
 from django.core import management as django_management
 from django.core.management.base import BaseCommand
 from django.db import connection as django_db_connection
 from django.db import models as django_models
 
-from frontrowcrew.utils import db as db_utils
 from etl import py_steps
-from etl import utils as etl_utils
+from frontrowcrew.utils import db as db_utils
 
 
 class Command(BaseCommand):
@@ -86,12 +85,13 @@ def repair_media(self, *args, app_name: str, model_name: str, field_name: str, *
         results = db_utils.namedtuplefetchall(self.postgres_cursor)
         for result in results:
             old_path = getattr(result, db_column_name, "")
-            new_base_path = getattr(field, "upload_to")
+            # new_base_path = getattr(field, "upload_to")
             if old_path:
-                new_path = etl_utils.download_to_default_storage(
-                    old_path=old_path,
-                    new_base_path=new_base_path,
-                )
+                # new_path = etl_utils.download_to_default_storage(
+                #     old_path=old_path,
+                #     new_base_path=new_base_path,
+                # )
+                new_path = ""
                 update_media_query = f"UPDATE {db_table_name} SET {db_column_name} = %s WHERE id = %s;"
                 self.postgres_cursor.execute(update_media_query, [new_path, result.id])
 
@@ -168,9 +168,9 @@ def handle(self, *args, **options) -> None:
                 "method": self._execute_sql_file,
                 "kwargs": {"filename": "07_videos.sql"},
             },
-            {
-                "method": py_steps.video_thumbnails.run,
-            },
+            # {
+            #     "method": py_steps.video_thumbnails.run,
+            # },
             {
                 "method": self._execute_sql_file,
                 "kwargs": {"filename": "08_geeknights.sql"},

etl/management/commands/etl_local_dev.py

@@ -0,0 +1,31 @@
+from django.core import management as django_management
+from django.contrib import auth
+from media import models as media_models
+
+
+class Command(django_management.base.BaseCommand):
+    help = "Run the ETL then create other things for local dev"
+
+    def handle(self, *args, **options) -> None:
+        # Reset database and perform etl
+        django_management.call_command("etl")
+
+        # Create superuser
+        django_management.call_command(
+            "createsuperuser",
+            "--noinput",
+            username="admin",
+            email="[email protected]",
+        )
+        User = auth.get_user_model()
+        user = User.objects.all()[0]
+        user.set_password("admin")
+        user.save()
+
+        # Create FTPDestination
+        media_models.FTPDestination.objects.create(
+            name="AlpineFTP",
+            host="ftp",
+            username="alpineftp",
+            password="alpineftp",
+        )

etl/py_steps/forum_link_fill_in.py

@@ -148,9 +148,6 @@ def run() -> None:
                 url=good_result["url"],
                 author="",
             )
-        elif len(resultset) > 0:
-            breakpoint()
-            print(resultset)
 
     # Remove repeats
     repeats = show_models.RelatedLink.objects.filter(

etl/py_steps/podcasts_create.py

@@ -32,11 +32,13 @@ def create_podcast_for_show(
 ) -> None:
     """ Create a podcast for a show using the given data """
     show = show_models.Show.objects.get(slug=show_slug)
-    logo_file_name = os.path.basename(show.logo.name)
-    logo_file = django_files_base.ContentFile(
-        show.logo.read(),
-        name=logo_file_name,
-    )
+    logo_file = ""
+    if show.logo:
+        logo_file_name = os.path.basename(show.logo.name)
+        logo_file = django_files_base.ContentFile(
+            show.logo.read(),
+            name=logo_file_name,
+        )
     podcast = podcast_models.Podcast(
         title=show.title,
         description=show.description,

legacy_redirects/tests/test_redirects.py

@@ -54,8 +54,6 @@ def test_most_redirects(self):
             testreader = csv.reader(testfile, delimiter=",", quotechar='"')
             for from_url, to_url in testreader:
                 response = self.client.get(from_url)
-                if response.status_code == HTTPStatus.NOT_FOUND:
-                    breakpoint()
                 self.assertEqual(
                     response.status_code,
                     HTTPStatus.MOVED_PERMANENTLY,

media/id3.py

@@ -1,13 +1,11 @@
+import io
 import os
-import PIL
-import tempfile
 
+import magic
+import mutagen
 from django.utils import timezone
-
-from mutagen import (
-    mp3 as mutagen_mp3,
-    id3 as mutagen_id3
-)
+from mutagen import id3 as mutagen_id3
+from mutagen import mp3 as mutagen_mp3
 
 
 def get_info(mp3_file):
@@ -79,22 +77,23 @@ def set_id3(
             )
 
     # APIC frame
-    permitted_image_formats = {
-        "JPEG": "image/jpeg",
-        "PNG": "image/png",
-    }
+    permitted_image_formats = [
+        "image/jpeg",
+        "image/png",
+    ]
     if album_image:
-        image = PIL.Image.open(album_image)
-        mime_type = permitted_image_formats.get(image.format, None)
-        if mime_type is not None:
-            image.seek(0)
-            APIC_frame = mutagen_id3.APIC(
-                mime=mime_type,
-                data=image.fp.read(),
+        image_data = album_image.file.read()
+        mime_checker = magic.Magic(mime=True)
+        image_mime = mime_checker.from_buffer(image_data)
+        if image_mime in permitted_image_formats:
+            apic_frame = mutagen_id3.APIC(
+                type=mutagen.id3.PictureType.COVER_FRONT,
+                mime=image_mime,
+                data=image_data,
             )
             if album_image_description:
-                APIC_frame.desc = album_image_description
-            frames.append(APIC_frame)
+                apic_frame.desc = album_image_description
+            frames.append(apic_frame)
 
     # Chapter frames
     if chapters:
@@ -122,19 +121,23 @@ def set_id3(
                 chapter_sub_frames.append(chapter_url_frame)
 
             chapter_image = chapter.get("image", None)
+
             if chapter_image:
-                chapter_image = PIL.Image.open(chapter_image)
-                mime_type = permitted_image_formats.get(chapter_image.format, None)
-                if mime_type is not None:
-                    chapter_image.seek(0)
-                    APIC_frame = mutagen_id3.APIC(
-                        mime=mime_type,
-                        data=chapter_image.fp.read(),
+                image_data = chapter_image.file.read()
+                # The same image might get used on many chapters, so reset it
+                chapter_image.file.seek(0)
+                mime_checker = magic.Magic(mime=True)
+                image_mime = mime_checker.from_buffer(image_data)
+                if image_mime in permitted_image_formats:
+                    apic_frame = mutagen_id3.APIC(
+                        type=mutagen.id3.PictureType.COVER_FRONT,
+                        mime=image_mime,
+                        data=image_data,
                     )
                     image_description = chapter.get("image_description", None)
                     if image_description:
-                        APIC_frame.desc = image_description
-                    chapter_sub_frames.append(APIC_frame)
+                        apic_frame.desc = image_description
+                    chapter_sub_frames.append(apic_frame)
 
             chapter_id = f"ch{number:02}"
             chapter_ids.append(chapter_id)
@@ -165,11 +168,13 @@ def set_id3(
         )
 
     # Write frames to file
-    with tempfile.NamedTemporaryFile() as temp3:
-        temp3.write(mp3_file.read())
-        tags = mutagen_mp3.MP3(temp3).tags
-        for frame in frames:
-            tags.add(frame)
-        tags.save(temp3, v1=mutagen_id3.ID3v1SaveOptions.CREATE)
-        mp3_basename = os.path.basename(mp3_file.name)
-        mp3_file.save(mp3_basename, temp3)
+    ramfile = io.BytesIO(mp3_file.read())
+    mp3_data = mutagen_mp3.MP3(ramfile)
+    mp3_data.delete(ramfile)
+    mp3_data.save(ramfile)
+    tags = mp3_data.tags
+    for frame in frames:
+        tags.add(frame)
+    tags.save(ramfile, v1=mutagen_id3.ID3v1SaveOptions.CREATE)
+    mp3_basename = os.path.basename(mp3_file.name)
+    mp3_file.save(mp3_basename, ramfile)