feat: switch to yt-dlp, search multiple videos with ytsearch

VERSION

@@ -1 +1 @@
-1.3.0
+1.4.0

pyproject.toml

@@ -17,11 +17,12 @@ dependencies = [
     "aiohttp",
     "beautifulsoup4",
     "click",
+    "defusedxml",
     "gTTS",
     "geoip2",
     "lxml",
     "python-magic",
-    "youtube-dl @ git+https://github.com/ytdl-org/youtube-dl@master#egg=youtube-dl",
+    "yt-dlp @ git+https://github.com/yt-dlp/yt-dlp@master#egg=yt-dlp"
 ]
 
 [project.scripts]

requirements.txt

@@ -16,8 +16,12 @@ attrs==22.1.0
     # via aiohttp
 beautifulsoup4==4.11.1
     # via torchlight (pyproject.toml)
+brotli==1.0.9
+    # via yt-dlp
 certifi==2022.9.24
-    # via requests
+    # via
+    #   requests
+    #   yt-dlp
 charset-normalizer==2.1.1
     # via
     #   aiohttp
@@ -26,6 +30,8 @@ click==8.1.3
     # via
     #   gtts
     #   torchlight (pyproject.toml)
+defusedxml==0.7.1
+    # via torchlight (pyproject.toml)
 frozenlist==1.3.3
     # via
     #   aiohttp
@@ -46,8 +52,12 @@ multidict==6.0.2
     # via
     #   aiohttp
     #   yarl
+mutagen==1.46.0
+    # via yt-dlp
 pillow==9.3.0
     # via torchlight (pyproject.toml)
+pycryptodomex==3.18.0
+    # via yt-dlp
 python-magic==0.4.27
     # via torchlight (pyproject.toml)
 requests==2.28.1
@@ -62,7 +72,9 @@ urllib3==1.26.13
     # via
     #   geoip2
     #   requests
+websockets==11.0.3
+    # via yt-dlp
 yarl==1.8.1
     # via aiohttp
-youtube-dl @ git+https://github.com/ytdl-org/youtube-dl@master
+yt-dlp @ git+https://github.com/yt-dlp/yt-dlp@master
     # via torchlight (pyproject.toml)

src/torchlight/Commands.py

@@ -1,3 +1,4 @@
+import ast
 import asyncio
 import datetime
 import json
@@ -8,11 +9,11 @@
 import sys
 import tempfile
 import traceback
-import xml.etree.ElementTree as etree
 from re import Match, Pattern
 from typing import Any
 
 import aiohttp
+import defusedxml.ElementTree as etree
 import geoip2.database
 import gtts
 
@@ -23,6 +24,8 @@
 from torchlight.PlayerManager import PlayerManager
 from torchlight.Torchlight import Torchlight
 from torchlight.URLInfo import (
+    get_audio_format,
+    get_first_valid_entry,
     get_url_real_time,
     get_url_text,
     get_url_youtube_info,
@@ -829,7 +832,10 @@ async def _func(self, message: list[str], player: Player) -> int:
             return -1
 
         input_keywords = message[1]
-        input_url = f"ytsearch: {input_keywords}"
+        if URLFilter.youtube_compile().search(input_keywords):
+            input_url = input_keywords
+        else:
+            input_url = f"ytsearch3: {input_keywords}"
 
         try:
             info = get_url_youtube_info(url=input_url)
@@ -845,13 +851,10 @@ async def _func(self, message: list[str], player: Player) -> int:
             return 1
 
         if info["extractor_key"] == "YoutubeSearch":
-            input_url = (
-                f"https://youtube.com/watch?v={info['entries'][0]['id']}"
-            )
-            info = get_url_youtube_info(url=input_url)
+            info = get_first_valid_entry(entries=info["entries"])
 
         title = info["title"]
-        url = info["formats"][0]["url"]
+        url = get_audio_format(info=info)
         title_words = title.split()
         keywords_banned: list[str] = []
 
@@ -1288,7 +1291,7 @@ class Exec(BaseCommand):
     async def _func(self, message: list[str], player: Player) -> int:
         self.logger.debug(sys._getframe().f_code.co_name + " " + str(message))
         try:
-            resp = eval(message[1])
+            resp = ast.literal_eval(message[1])
         except Exception as e:
             self.torchlight.SayChat(f"Error: {str(e)}")
             return 1

src/torchlight/URLInfo.py

@@ -1,15 +1,20 @@
 import asyncio
 import io
+import json
+import logging
 from collections.abc import Callable
+from typing import Any
 
 import aiohttp
 import magic
-import youtube_dl
+import yt_dlp
 from bs4 import BeautifulSoup
 from PIL import Image
 
 from torchlight.Utils import Utils
 
+logger = logging.getLogger(__name__)
+
 
 async def get_url_data(url: str) -> tuple[bytes, str, int]:
     async with aiohttp.ClientSession() as session:
@@ -106,11 +111,37 @@ def get_url_real_time(url: str) -> int:
 
 def get_url_youtube_info(url: str) -> dict:
     # https://github.com/ytdl-org/youtube-dl/blob/3e4cedf9e8cd3157df2457df7274d0c842421945/youtube_dl/YoutubeDL.py#L137-L312
+    # https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L192
     ydl_opts = {
         "extract_flat": True,
+        "skip_download": True,
+        "debug_printtraffic": False,
         "quiet": True,
-        "format": "bestaudio/best",
+        "no_warnings": True,
+        "format": "m4a/bestaudio/best",
+        "simulate": True,
+        "keepvideo": False,
     }
-    ydl = youtube_dl.YoutubeDL(ydl_opts)
+    ydl = yt_dlp.YoutubeDL(ydl_opts)
     ydl.add_default_info_extractors()
     return ydl.extract_info(url, download=False)
+
+
+def get_first_valid_entry(entries: list[Any]) -> dict[str, Any]:
+    for entry in entries:
+        input_url = f"https://youtube.com/watch?v={entry['id']}"
+        try:
+            info = get_url_youtube_info(url=input_url)
+            return info
+        except yt_dlp.utils.DownloadError:
+            logger.warn(f"Error trying to download <{input_url}>")
+            pass
+    raise Exception("No compatible youtube video found, try something else")
+
+
+def get_audio_format(info: dict[str, Any]) -> str:
+    for format in info["formats"]:
+        if "audio_channels" in format:
+            logger.debug(json.dumps(format, indent=2))
+            return format["url"]
+    raise Exception("No compatible audio format found, try something else")