diff --git a/Tests/video_system/youtube/test_youtube.py b/Tests/video_system/youtube/test_youtube.py index 670a7e3..ee42df6 100644 --- a/Tests/video_system/youtube/test_youtube.py +++ b/Tests/video_system/youtube/test_youtube.py @@ -1,10 +1,9 @@ -import logging import os import shutil import pytest -import yt_dlp from Tests.video_system.download_tester import DownloadTester +from src.downloader import DownloadFailedError from src.Youtube import YoutubeDownloader TEST_YOUTUBE_1 = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" @@ -26,10 +25,10 @@ def remove_cache(self): async def test_basic_download(self): try: videos = await YoutubeDownloader.download_video_from_link(TEST_YOUTUBE_1, DOWNLOAD_PATH) - except yt_dlp.DownloadError as e: + except DownloadFailedError as e: assert e.msg import warnings - if "Sign in" not in e.msg: + if "Sign in" not in str(e.with_traceback(None)): raise e # re-raise the exception if it's not a sign in error warnings.warn(e.msg) return diff --git a/src/Helpers/logging_system.py b/src/Helpers/logging_system.py index 4c03c68..ace1ca1 100644 --- a/src/Helpers/logging_system.py +++ b/src/Helpers/logging_system.py @@ -23,8 +23,12 @@ def is_server(only_true_if_cloud: bool = True) -> bool: if is_server(only_true_if_cloud=False): - google_client = google.cloud.logging.Client() - google_client.setup_logging(log_level=logging.DEBUG) + try: + google_client = google.cloud.logging.Client() + google_client.get_default_handler() + google_client.setup_logging(log_level=logging.DEBUG) + except Exception as e: + logging.error("Failed to setup google cloud logging", exc_info=e) else: format_string = '%(asctime)s: %(name)s - %(levelname)s - %(message)s in %(filename)s:%(lineno)d' logging.basicConfig(level=logging.DEBUG, filename=f'{BOT_NAME}.log', filemode='w', format=format_string) diff --git a/src/Youtube.py b/src/Youtube.py index 7b2325b..c93533e 100644 --- a/src/Youtube.py +++ b/src/Youtube.py @@ -1,4 +1,3 @@ -import asyncio import functools import logging import os @@ -7,7 +6,7 @@ import yt_dlp from Constants import MAX_VIDEO_DOWNLOAD_SIZE -from src.downloader import VIDEO_RETURN_TYPE, VideoDownloader, VideoFile +from src.downloader import VIDEO_RETURN_TYPE, AlternateVideoDownloader ydl_opts = { 'format': 'bestaudio', @@ -94,9 +93,9 @@ def youtube_download(video_url, progress_queue: LifoQueue, file_path_with_name): def get_last_played_guilded() -> video_data_guild: return last_played -class YoutubeDownloader(VideoDownloader): - @staticmethod - async def download_video_from_link(url: str, path: str | None = None) -> VIDEO_RETURN_TYPE: +class YoutubeDownloader(AlternateVideoDownloader): + @classmethod + async def download_video_from_link(cls, url: str, path: str | None = None) -> VIDEO_RETURN_TYPE: if path is None: path = os.path.join("downloads", "youtube") @@ -111,21 +110,4 @@ async def download_video_from_link(url: str, path: str | None = None) -> VIDEO_R 'quiet': True, } - with yt_dlp.YoutubeDL(costum_options) as ydl: - ydt = await asyncio.to_thread(ydl.extract_info, url, download=True) - - if ydt is None: - return [] - - info = ydt.get("entries", [None])[0] or ydt - video_id = info["id"] - video_extension = info["ext"] - if video_id is None: - return [] - - if video_extension != "mp4": - logging.error("Got a non-mp4 file that is %s from this link: %s", video_extension, url) - - file_path = os.path.join(path, f"{video_id}.{video_extension}") - - return [VideoFile(file_path, info.get("title", None))] + return await cls._get_list_from_ydt(url, costum_options, path) diff --git a/src/download_commands.py b/src/download_commands.py index 66998e4..fc49361 100644 --- a/src/download_commands.py +++ b/src/download_commands.py @@ -1,10 +1,18 @@ import asyncio import logging -from typing import Optional +from typing import Optional, Type import discord +from src.downloader import ( + VIDEO_RETURN_TYPE, + AbstractClassUsedError, + DownloadFailedError, + NoVideoFoundError, + VideoDownloader, +) from src.other import UnknownAlternateDownloader -from src.downloading_system import get_downloader +from src.downloading_system import get_downloader, get_url_from_text + def _convert_paths_to_discord_files(paths: list[str]) -> list[discord.File]: return [discord.File(path) for path in paths] @@ -16,40 +24,126 @@ def _get_shortest_punctuation_index(caption: str) -> int | None: comma = caption.find(",") question_mark = caption.find("?") exclamation_mark = caption.find("!") - filtered_list = list(filter(lambda x: x != -1, [dot, comma, question_mark, exclamation_mark])) + filtered_list = list( + filter(lambda x: x != -1, [dot, comma, question_mark, exclamation_mark]) + ) if len(filtered_list) == 0: return None return min(filtered_list) + def _get_shortened_caption(caption: str) -> str: # check if we have a punctuation mark in the caption caption = caption.split("\n")[0] punctuation_index = _get_shortest_punctuation_index(caption) if punctuation_index: - return caption[:punctuation_index + 1] + return caption[: punctuation_index + 1] return caption[:100] def _get_view(shortened_caption: str, caption: str): view = discord.ui.View() button = discord.ui.Button(label="🔽\nExpand", style=discord.ButtonStyle.secondary) + async def callback(interaction: discord.Interaction): revert_view = discord.ui.View() - button = discord.ui.Button(label="🔼\nShorten", style=discord.ButtonStyle.secondary) + button = discord.ui.Button( + label="🔼\nShorten", style=discord.ButtonStyle.secondary + ) + async def callback(interaction: discord.Interaction): - await interaction.response.edit_message(content=shortened_caption, view=view) + await interaction.response.edit_message( + content=shortened_caption, view=view + ) + button.callback = callback revert_view.add_item(button) await interaction.response.edit_message(content=caption, view=revert_view) + button.callback = callback view.add_item(button) return view +def _get_caption_and_view( + real_caption: str, include_title: Optional[bool] +) -> tuple[Optional[str], discord.ui.View]: + shortened_caption = _get_shortened_caption(real_caption) + " ***...***" + view = discord.utils.MISSING + + if include_title is False: + caption = None + + elif include_title is True: + caption = real_caption + + elif len(shortened_caption) < len(real_caption): + view = _get_view(shortened_caption, real_caption) + caption = shortened_caption + else: + caption = real_caption + + return caption, view + + +async def get_details( + downloader: Type[VideoDownloader], url: str, interaction: discord.Interaction +) -> Optional[VIDEO_RETURN_TYPE]: + try: + return await downloader.download_video_from_link(url) + except DownloadFailedError: + await interaction.followup.send( + "Video indirilirken başarısız olundu, hata raporu alındı. Lütfen daha sonra tekrar deneyin", + ephemeral=True, + ) + logging.exception("Failed Downloading Link: %s", url, exc_info=True) + return + except NoVideoFoundError: + await interaction.followup.send( + "Linkte bir video bulamadım, linkte **video** olduğuna emin misin?", + ephemeral=True, + ) + logging.exception("Couldn't find link on url %s", url, exc_info=True) + return + except AbstractClassUsedError: + await interaction.followup.send( + "Bir şeyler ÇOK ters gitti, hata raporu alındı.", ephemeral=True + ) + logging.exception( + "An abstract class was used, this should not happen", exc_info=True + ) + return + except Exception as e: + await interaction.followup.send( + "Bilinmeyen bir hata oluştu, lütfen tekrar deneyin", ephemeral=True + ) + raise e + + +async def _convert_to_discord_files( + interaction: discord.Interaction, attachments: VIDEO_RETURN_TYPE +) -> list[discord.File]: + try: + file_paths = [attachment.path for attachment in attachments] + return _convert_paths_to_discord_files(file_paths) + except Exception as e: + await interaction.followup.send( + "Bilinmeyen bir hata oluştu, lütfen tekrar deneyin", ephemeral=True + ) + raise e # re-raise the exception so we can see what went wrong + + +async def download_video_command( + interaction: discord.Interaction, + url: str, + is_ephemeral: bool = False, + include_title: bool | None = None, +): + url = get_url_from_text(url) -async def download_video_command(interaction: discord.Interaction, url: str, is_ephemeral: bool = False, include_title: bool | None = None): downloader = get_downloader(url) + if downloader is None: logging.info("Found an unsupported link: %s", url) await interaction.response.defer(ephemeral=True) @@ -57,39 +151,28 @@ async def download_video_command(interaction: discord.Interaction, url: str, is_ await interaction.response.defer(ephemeral=is_ephemeral) - try: - attachments = await downloader.download_video_from_link(url) - file_paths = [attachment.path for attachment in attachments] - discord_files = _convert_paths_to_discord_files(file_paths) - except Exception as e: - await interaction.followup.send("Bir şey ters gitti... lütfen tekrar deneyin", ephemeral=True) - raise e # re-raise the exception so we can see what went wrong - if len(attachments) == 0: - await interaction.followup.send("Videoyu Bulamadım, lütfen daha sonra tekrar deneyin ya da hatayı bildirin", ephemeral=True) + attachments = await get_details(downloader, url, interaction) + if attachments is None: return - returned_content = " + ".join(filter(None, [attachment.caption for attachment in attachments])) - default_caption = f"Video{'s' if len(attachments) > 1 else ''} Downloaded" - caption = "" - view = discord.utils.MISSING - shortened_content = _get_shortened_caption(returned_content) + " ***...***" - if include_title is False or not returned_content: - caption = default_caption + discord_files = await _convert_to_discord_files(interaction, attachments) - elif include_title is True: - caption = returned_content + real_caption = ( + attachments.caption or f"Video{'s' if len(attachments) > 1 else ''} Downloaded" + ) + caption, view = _get_caption_and_view(real_caption, include_title) + caption = caption or discord.utils.MISSING - elif len(shortened_content) < len(returned_content): - view = _get_view(shortened_content, returned_content) - caption = shortened_content - else: - caption = returned_content + await interaction.followup.send( + caption, files=discord_files, ephemeral=is_ephemeral, view=view + ) - await interaction.followup.send(caption, files=discord_files, ephemeral=is_ephemeral, view=view) async def loading_animation(message: discord.WebhookMessage): original_text = message.content - sleep_time = 0 # we don't actually need to sleep thanks to ``message.edit`` being async + sleep_time = ( + 0 # we don't actually need to sleep thanks to ``message.edit`` being async + ) while True: await message.edit(content=original_text + ".", view=discord.ui.View()) await asyncio.sleep(sleep_time) @@ -98,8 +181,11 @@ async def loading_animation(message: discord.WebhookMessage): await message.edit(content=original_text + "...", view=discord.ui.View()) await asyncio.sleep(sleep_time) -async def try_unknown_link(interaction: discord.Interaction, url: str, include_title: Optional[bool] = None): - """ edits the sent message if the download is successful, otherwise sends an error message + +async def try_unknown_link( + interaction: discord.Interaction, url: str, include_title: Optional[bool] = None +): + """edits the sent message if the download is successful, otherwise sends an error message Args: interaction (discord.Interaction): the interaction to edit with ``interaction.response.edit_message`` @@ -107,7 +193,11 @@ async def try_unknown_link(interaction: discord.Interaction, url: str, include_t """ downloader = UnknownAlternateDownloader - sent_message = await interaction.followup.send("Bu link resmi olarak desteklenmiyor, yine de indirmeyi deniyorum", ephemeral=True, wait=True) + sent_message = await interaction.followup.send( + "Bu link resmi olarak desteklenmiyor, yine de indirmeyi deniyorum", + ephemeral=True, + wait=True, + ) loading_task = asyncio.create_task(loading_animation(sent_message)) try: @@ -117,31 +207,13 @@ async def try_unknown_link(interaction: discord.Interaction, url: str, include_t except Exception as e: loading_task.cancel() await sent_message.edit(content="Linki ne yazıkki indiremedim") - raise e # re-raise the exception so we can see what went wrong - - if len(attachments) == 0: - loading_task.cancel() - await sent_message.edit(content="Videoyu Bulamadım, lütfen daha sonra tekrar deneyin ya da hatayı bildirin") - return - - returned_content = " + ".join(filter(None, [attachment.caption for attachment in attachments])) - default_caption = f"Video{'s' if len(attachments) > 1 else ''} Downloaded" - caption = "" - view = discord.utils.MISSING - shortened_content = _get_shortened_caption(returned_content) + " ***...***" + raise e # re-raise the exception so we can see what went wrong - - if include_title is False or not returned_content: - caption = default_caption - - elif include_title is True: - caption = returned_content - - elif len(shortened_content) < len(returned_content): - view = _get_view(shortened_content, returned_content) - caption = shortened_content - else: - caption = returned_content + real_caption = ( + attachments.caption or f"Video{'s' if len(attachments) > 1 else ''} Downloaded" + ) + caption, view = _get_caption_and_view(real_caption, include_title) + caption = caption or "" loading_task.cancel() await sent_message.edit(content=f"{url} downloaded") diff --git a/src/downloader.py b/src/downloader.py index 2aa4d5c..3c412ff 100644 --- a/src/downloader.py +++ b/src/downloader.py @@ -14,7 +14,15 @@ class DownloaderError(Exception): pass class DownloadFailedError(DownloaderError): - pass + """Base exception for any errors created in downloading.""" + msg = None + + def __init__(self, msg=None): + if msg is not None: + self.msg = msg + elif self.msg is None: + self.msg = type(self).__name__ + super().__init__(self.msg) class NoVideoFoundError(DownloaderError): pass @@ -73,7 +81,7 @@ def caption(self) -> str | None: -VIDEO_RETURN_TYPE = list[VideoFile] +VIDEO_RETURN_TYPE = VideoFiles class VideoDownloader(ABC): """ @@ -87,14 +95,20 @@ async def download_video_from_link(cls, url: str, path: str | None = None) -> VI Downloads Videos from a url if path is None, the default path is downloads/{website_name} - if the download fails, it returns an empty list + if the download fails, it will raise an Error + + Raises: + DownloadFailedError: if the download fails + NoVideoFoundError: if no video is found + AbstractClassUsedError: if the interface is directly called, should never happen + All the errors are subclasses of ``DownloaderError`` """ logging.error( "VideoDownloader download_url interface was directly called, this should not happen! url was: %s for path: %s", url, path, ) - return [] + raise AbstractClassUsedError("VideoDownloader download_url interface was directly called") @classmethod async def _download_link(cls, url: str, download_to: str) -> str | None: @@ -160,19 +174,19 @@ class AlternateVideoDownloader(VideoDownloader): async def _get_list_from_ydt(cls, url: str, ydl_opts: dict[str, Any], path: str, title_key: str = "title", cookies: dict | None = None) -> VIDEO_RETURN_TYPE: with yt_dlp.YoutubeDL(ydl_opts) as ydl: if cookies: - requests.utils.cookiejar_from_dict(cookies, ydl.cookiejar) + requests.utils.cookiejar_from_dict(cookies, ydl.cookiejar) try: ydt = await asyncio.to_thread(ydl.extract_info, url, download=True) except yt_dlp.DownloadError as e: logging.error("Couldn't download video from url: %s, Error: %s", url, e, exc_info=True) - return [] + raise DownloadFailedError(f"Couldn't download video from url: {url}") from e if ydt is None: - return [] + raise DownloadFailedError(f"Couldn't download video from url: {url}") infos: list[dict[str, Any]] = ydt.get("entries", [ydt]) - attachment_list: VIDEO_RETURN_TYPE = [] + attachment_list: list[VideoFile] = [] title = infos[0].get(title_key, None) url = infos[0].get("webpage_url", "URL-NOT-FOUND") for info in infos: @@ -190,8 +204,6 @@ async def _get_list_from_ydt(cls, url: str, ydl_opts: dict[str, Any], path: str, file_path = os.path.join(path, f"{video_id}.{video_extension}") - attachment_list.append(VideoFile(file_path, title)) - # only add the title to the first video, or else we duplicate the title for each video - title = None + attachment_list.append(VideoFile(file_path)) - return attachment_list + return VideoFiles(attachment_list, title) diff --git a/src/downloading_system.py b/src/downloading_system.py index 83611ea..11926db 100644 --- a/src/downloading_system.py +++ b/src/downloading_system.py @@ -13,6 +13,15 @@ _YOUTUBE_REGEX = re.compile(r"\b(?:https?:\/\/)?(?:www\.)?(?:youtube\.com\/|youtu\.be\/)\S*") +def get_url_from_text(text: str) -> str: + """ + Returns the first url found in the text, if no url is found it returns the text itself + """ + + if (result := re.search(_URL_PARSE_REGEX, text)): + return result.group(0) + return text + def get_downloader(url: str) -> Type[VideoDownloader] | None: """ Returns the correct downloader for the given url if it can't find it @@ -20,13 +29,13 @@ def get_downloader(url: str) -> Type[VideoDownloader] | None: if it still can't find a downloader, it returns None """ + url = get_url_from_text(url) + if re.match(_TWITTER_REGEX, url): return TwitterDownloader if re.match(_INSTAGRAM_REGEX, url): return InstagramDownloader if re.match(_YOUTUBE_REGEX, url): return YoutubeDownloader - # try to extract the url from the text incase there is extra text - if (result := re.search(_URL_PARSE_REGEX, url)) and result.group(0) != url: - return get_downloader(result.group(0)) + return None diff --git a/src/instagram.py b/src/instagram.py index 9b95289..fb84853 100644 --- a/src/instagram.py +++ b/src/instagram.py @@ -9,7 +9,7 @@ from instaloader.instaloader import Instaloader from instaloader.structures import Post -from src.downloader import VIDEO_RETURN_TYPE, AlternateVideoDownloader, VideoFile, VideoDownloader +from src.downloader import VIDEO_RETURN_TYPE, AlternateVideoDownloader, VideoFile, VideoDownloader, VideoFiles from src.Read import json_read, write_json _SHORTCODE_REGEX = ( @@ -133,7 +133,7 @@ async def download_video_from_link(cls, url: str, path: str | None = None) -> VI class InstagramDownloader(VideoDownloader): @classmethod async def download_video_from_link(cls, url: str, path: str | None = None) -> VIDEO_RETURN_TYPE: - attachment_list: VIDEO_RETURN_TYPE = [] + attachment_list: list[VideoFile] = [] _try_login() # try to login if not already logged in if path is None: @@ -155,8 +155,7 @@ async def download_video_from_link(cls, url: str, path: str | None = None) -> VI caption = post.caption for index in range(1, video_count + 1): # will run once if not sidecar file_path = os.path.join(path, _get_file_name(post, index)) - file = VideoFile(file_path, caption) - caption = None + file = VideoFile(file_path) if not os.path.exists(file.path) and not downloaded: await asyncio.to_thread(downloader.download_post, post, Path(path)) @@ -164,4 +163,4 @@ async def download_video_from_link(cls, url: str, path: str | None = None) -> VI attachment_list.append(file) - return attachment_list + return VideoFiles(attachment_list, caption) diff --git a/src/other.py b/src/other.py index c918804..dfbf055 100644 --- a/src/other.py +++ b/src/other.py @@ -13,7 +13,6 @@ async def download_video_from_link( os.makedirs(path, exist_ok=True) ydt_opts = { - "format": "best", "outtmpl": os.path.join(path, "%(id)s.%(ext)s"), "noplaylist": True, "default_search": "auto", diff --git a/src/twitter.py b/src/twitter.py index b0f6166..76afc25 100644 --- a/src/twitter.py +++ b/src/twitter.py @@ -5,7 +5,7 @@ from dotenv import load_dotenv import requests -from src.downloader import AlternateVideoDownloader, VideoDownloader, VideoFile, VIDEO_RETURN_TYPE +from src.downloader import AlternateVideoDownloader, DownloadFailedError, VideoDownloader, VideoFile, VIDEO_RETURN_TYPE, VideoFiles load_dotenv() @@ -88,7 +88,7 @@ class TwitterDownloader(VideoDownloader): async def download_video_from_link( cls, url: str, path: str | None = None ) -> VIDEO_RETURN_TYPE: - attachment_list: VIDEO_RETURN_TYPE = [] + attachment_list: list[VideoFile] = [] tweet_id = _get_tweet_id(url) if path is None: path = os.path.join("downloads", "twitter") @@ -108,14 +108,12 @@ async def download_video_from_link( if tweet_id is None: logging.error("No tweet id found in URL: %s", url) - return attachment_list + raise DownloadFailedError(f"No tweet id found in URL: {url}") download_urls = _get_highest_quality_url_list(response) title = _get_title(response) downloaded_file_paths = await cls._download_links(download_urls, path, tweet_id) attachment_list = [VideoFile(path) for path in downloaded_file_paths] - attachment_list[0]._title = title - - return attachment_list + return VideoFiles(attachment_list, title) diff --git a/tox.ini b/tox.ini index 24411b6..795ccba 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,6 @@ description = run the tests with pytest deps = pytest>=8 pytest-asyncio - static-ffmpeg -r requirements.txt passenv = * commands =