From b1f8f2cc585068e957650d390ce64f9d34ccc79f Mon Sep 17 00:00:00 2001 From: Collin Heist Date: Wed, 21 Aug 2024 22:09:44 -0600 Subject: [PATCH] [102] Reorganize and move webhook endpoints - Separate "rating key" logic into a standalone internal file - Create new /webhooks/ API router - Deprecate POST /api/series/sonarr/delete; new endpoint is POST /api/webhooks/sonarr/delete - Deprecate POST /api/cards/key endpoint; new one is /api/webhooks/plex/rating-key - Deprecate POST /api/cards/sonarr endpoint; new one is /api/webhooks/sonarr/cards - Use new endpoint in the Tautulli endpoint creator --- app/internal/webhooks.py | 126 +++++++++++++++++++ app/routers/api.py | 2 + app/routers/cards.py | 110 +++-------------- app/routers/series.py | 5 +- app/routers/webhooks.py | 224 ++++++++++++++++++++++++++++++++++ app/schemas/connection.py | 27 +--- app/schemas/webhooks.py | 121 ++++++++++++++++++ modules/TautulliInterface2.py | 4 +- modules/ref/version_webui | 2 +- 9 files changed, 496 insertions(+), 125 deletions(-) create mode 100755 app/internal/webhooks.py create mode 100644 app/routers/webhooks.py create mode 100755 app/schemas/webhooks.py diff --git a/app/internal/webhooks.py b/app/internal/webhooks.py new file mode 100755 index 00000000..2a579ec0 --- /dev/null +++ b/app/internal/webhooks.py @@ -0,0 +1,126 @@ +from logging import Logger + +from fastapi import HTTPException +from sqlalchemy.orm import Session + +from app.database.query import get_interface +from app.dependencies import PlexInterface +from app.internal.cards import create_episode_cards +from app.internal.episodes import refresh_episode_data +from app.internal.series import load_episode_title_card +from app.internal.snapshot import take_snapshot +from app.internal.sources import download_episode_source_images +from app.internal.translate import translate_episode +from app.models.episode import Episode +from app.models.series import Series + +from modules.Debug import log + + +def process_rating_key( + db: Session, + plex_interface: PlexInterface, + key: int, + *, + snapshot: bool = True, + log: Logger = log, + ) -> None: + """ + Create the Title Card for the item associated with the given Plex + Rating Key. This item can be a Show, Season, or Episode. + + Args: + db: Database to query for Card details. + plex_interface: Interface to Plex which has the details + associated with this Key. + key: Rating Key within Plex that identifies the item to create + the Card(s) for. + snapshot: Whether to take a snapshot of the database afterwards. + log: Logger for all log messages. + + Raises: + HTTPException (404): There are no details associated with the + given Rating Key. + """ + + # Get details of each key from Plex, raise 404 if not found/invalid + if len(details := plex_interface.get_episode_details(key, log=log)) == 0: + raise HTTPException( + status_code=404, + detail=f'Rating key {key} does not correspond to any content' + ) + log.debug(f'Identified {len(details)} entries from RatingKey={key}') + + # Process each set of details + episodes_to_load: list[Episode] = [] + for series_info, episode_info, watched_status in details: + # Find all matching Episodes + episodes = db.query(Episode)\ + .filter(episode_info.filter_conditions(Episode))\ + .all() + + # Episode does not exist, refresh episode data and try again + if not episodes: + # Try and find associated Series, skip if DNE + series = db.query(Series)\ + .filter(series_info.filter_conditions(Series))\ + .first() + if series is None: + log.info(f'Cannot find Series for {series_info}') + continue + + # Series found, refresh data and look for Episode again + refresh_episode_data(db, series, log=log) + episodes = db.query(Episode)\ + .filter(episode_info.filter_conditions(Episode))\ + .all() + if not episodes: + log.info(f'Cannot find Episode for {series_info} {episode_info}') + continue + + # Get first Episode that matches this Series + episode, found = None, False + for episode in episodes: + if episode.series.as_series_info == series_info: + found = True + break + + # If no match, exit + if not found: + log.info(f'Cannot find Episode for {series_info} {episode_info}') + continue + + # Update Episode watched status + episode.add_watched_status(watched_status) + + # Look for source, add translation, create card if source exists + images = download_episode_source_images(db, episode, log=log) + translate_episode(db, episode, log=log) + if not any(images): + log.info(f'{episode} has no source image - skipping') + continue + create_episode_cards(db, episode, log=log) + + # Add this Series to list of Series to load + if episode not in episodes_to_load: + episodes_to_load.append(episode) + + # Load all Episodes that require reloading + for episode in episodes_to_load: + # Refresh this Episode so that relational Card objects are + # updated, preventing stale (deleted) Cards from being used in + # the Loaded asset evaluation. Not sure why this is required + # because SQLAlchemy should update child objects when the DELETE + # is committed; but this does not happen. + db.refresh(episode) + + # Reload into all associated libraries + for library in episode.series.libraries: + interface = get_interface(library['interface_id']) + load_episode_title_card( + episode, db, library['name'], library['interface_id'], + interface, attempts=6, log=log, + ) + + if snapshot: + take_snapshot(db, log=log) diff --git a/app/routers/api.py b/app/routers/api.py index c1814da3..83d97950 100755 --- a/app/routers/api.py +++ b/app/routers/api.py @@ -27,6 +27,7 @@ from app.routers.sync import sync_router from app.routers.templates import template_router from app.routers.translate import translation_router +from app.routers.webhooks import webhook_router # Create sub router for all API requests api_router = APIRouter(prefix='/api') @@ -50,6 +51,7 @@ api_router.include_router(sync_router) api_router.include_router(template_router) api_router.include_router(translation_router) +api_router.include_router(webhook_router) @api_router.get('/healthcheck') def health_check( diff --git a/app/routers/cards.py b/app/routers/cards.py index 1674648c..5c2cc6b5 100755 --- a/app/routers/cards.py +++ b/app/routers/cards.py @@ -38,18 +38,18 @@ load_series_title_cards, update_series_config ) -from app.internal.snapshot import take_snapshot from app.internal.sources import download_episode_source_images from app.internal.translate import translate_episode +from app.internal.webhooks import process_rating_key from app.models.episode import Episode from app.models.series import Series from app.schemas.card import CardActions, TitleCard, PreviewTitleCard -from app.schemas.connection import SonarrWebhook from app.schemas.episode import Episode as EpisodeSchema, UpdateEpisode from app.schemas.font import DefaultFont from app.schemas.series import UpdateSeries -from modules.Debug import InvalidCardSettings, MissingSourceImage +from app.schemas.webhooks import SonarrWebhook +from modules.Debug import InvalidCardSettings, MissingSourceImage from modules.EpisodeInfo2 import EpisodeInfo from modules.FormatString import FormatString from modules.SeriesInfo2 import SeriesInfo @@ -60,6 +60,7 @@ card_router = APIRouter( prefix='/cards', tags=['Title Cards'], + # dependencies=[Depends(get_current_user)], # TODO add after webhooks are removed ) @@ -541,10 +542,11 @@ def create_card_for_episode( ) from exc -@card_router.post('/key', tags=['Plex', 'Tautulli']) +@card_router.post('/key', tags=['Plex', 'Webhooks'], deprecated=True) def create_cards_for_plex_rating_key( request: Request, key: int = Body(...), + snapshot: bool = Query(default=True), db: Session = Depends(get_database), plex_interface: PlexInterface = Depends(require_plex_interface), ) -> None: @@ -552,100 +554,22 @@ def create_cards_for_plex_rating_key( Create the Title Card for the item associated with the given Plex Rating Key. This item can be a Show, Season, or Episode. This endpoint does NOT require an authenticated User so that Tautulli can - trigger this without any credentials. The `interface_id` of the - appropriate Plex Connection must be passed via a Query parameter. + trigger this without any credentials. - - plex_rating_keys: Unique keys within Plex that identifies the item - to remake the card of. + - interface_id: Interface ID of the Plex Connection associated with + this Key. + - key: Rating Key within Plex that identifies the item to create the + Card(s) for. + - snapshot: Whether to take snapshot of the database after all Cards + have been processed. """ - # Get contextual logger - log = request.state.log - - # Get details of each key from Plex, raise 404 if not found/invalid - if len(details := plex_interface.get_episode_details(key, log=log)) == 0: - raise HTTPException( - status_code=404, - detail=f'Rating key {key} is invalid' - ) - log.debug(f'Identified {len(details)} entries from RatingKey={key}') - - # Process each set of details - episodes_to_load: list[Episode] = [] - for series_info, episode_info, watched_status in details: - # Find all matching Episodes - episodes = db.query(Episode)\ - .filter(episode_info.filter_conditions(Episode))\ - .all() - - # Episode does not exist, refresh episode data and try again - if not episodes: - # Try and find associated Series, skip if DNE - series = db.query(Series)\ - .filter(series_info.filter_conditions(Series))\ - .first() - if series is None: - log.info(f'Cannot find Series for {series_info}') - continue - - # Series found, refresh data and look for Episode again - refresh_episode_data(db, series, log=log) - episodes = db.query(Episode)\ - .filter(episode_info.filter_conditions(Episode))\ - .all() - if not episodes: - log.info(f'Cannot find Episode for {series_info} {episode_info}') - continue - - # Get first Episode that matches this Series - episode, found = None, False - for episode in episodes: - if episode.series.as_series_info == series_info: - found = True - break - - # If no match, exit - if not found: - log.info(f'Cannot find Episode for {series_info} {episode_info}') - continue - - # Update Episode watched status - episode.add_watched_status(watched_status) - - # Look for source, add translation, create card if source exists - images = download_episode_source_images(db, episode, log=log) - translate_episode(db, episode, log=log) - if not any(images): - log.info(f'{episode} has no source image - skipping') - continue - create_episode_cards(db, episode, log=log) - - # Add this Series to list of Series to load - if episode not in episodes_to_load: - episodes_to_load.append(episode) - - # Load all Episodes that require reloading - for episode in episodes_to_load: - # Refresh this Episode so that relational Card objects are - # updated, preventing stale (deleted) Cards from being used in - # the Loaded asset evaluation. Not sure why this is required - # because SQLAlchemy should update child objects when the DELETE - # is committed; but this does not happen. - db.refresh(episode) - - # Reload into all associated libraries - for library in episode.series.libraries: - interface = get_interface(library['interface_id']) - load_episode_title_card( - episode, db, library['name'], library['interface_id'], - interface, attempts=6, log=log, - ) - - take_snapshot(db, log=log) - return None + return process_rating_key( + db, plex_interface, key, snapshot=snapshot, log=request.state.log, + ) -@card_router.post('/sonarr', tags=['Sonarr']) +@card_router.post('/sonarr', tags=['Webhooks'], deprecated=True) def create_cards_for_sonarr_webhook( request: Request, webhook: SonarrWebhook = Body(...), diff --git a/app/routers/series.py b/app/routers/series.py index 30e403a3..92961005 100755 --- a/app/routers/series.py +++ b/app/routers/series.py @@ -14,7 +14,6 @@ from fastapi_pagination.ext.sqlalchemy import paginate from fastapi_pagination import paginate as paginate_sequence from PIL import Image -from requests import get from sqlalchemy import and_, desc, func, or_ from sqlalchemy.orm import Session from unidecode import unidecode @@ -43,7 +42,7 @@ from app.models.card import Card from app.models.loaded import Loaded from app.models.series import Series as SeriesModel -from app.schemas.connection import SonarrWebhook +from app.schemas.webhooks import SonarrWebhook from app.schemas.series import ( BatchUpdateSeries, NewSeries, @@ -214,7 +213,7 @@ def delete_series_( delete_series(db, series, log=request.state.log) -@series_router.post('/sonarr/delete') +@series_router.post('/sonarr/delete', deprecated=True) def delete_series_via_sonarr_webhook( request: Request, webhook: SonarrWebhook, diff --git a/app/routers/webhooks.py b/app/routers/webhooks.py new file mode 100644 index 00000000..d2a19e99 --- /dev/null +++ b/app/routers/webhooks.py @@ -0,0 +1,224 @@ +from logging import Logger +from time import sleep +from typing import Optional + +from fastapi import ( + APIRouter, + Body, + Depends, + Query, + Request +) +from fastapi.exceptions import HTTPException +from sqlalchemy.orm import Session + +from app.database.query import get_interface +from app.dependencies import get_database, require_plex_interface, PlexInterface +from app.internal.cards import create_episode_cards, delete_cards +from app.internal.episodes import refresh_episode_data +from app.internal.series import delete_series, load_episode_title_card +from app.internal.sources import download_episode_source_images +from app.internal.translate import translate_episode +from app.internal.webhooks import process_rating_key +from app.models.card import Card +from app.models.episode import Episode +from app.models.loaded import Loaded +from app.models.series import Series +from app.schemas.webhooks import PlexWebhook, SonarrWebhook +from modules.EpisodeInfo2 import EpisodeInfo +from modules.SeriesInfo2 import SeriesInfo + + +# Create sub router for all /webhooks API requests +webhook_router = APIRouter( + prefix='/webhooks', + tags=['Webhooks'], +) + + +@webhook_router.post('/plex/rating-key', tags=['Plex']) +def create_cards_for_plex_rating_key( + request: Request, + key: int = Body(...), + snapshot: bool = Query(default=True), + db: Session = Depends(get_database), + plex_interface: PlexInterface = Depends(require_plex_interface), + ) -> None: + """ + Create the Title Card for the item associated with the given Plex + Rating Key. This item can be a Show, Season, or Episode. This + endpoint does NOT require an authenticated User so that Tautulli can + trigger this without any credentials. + + - interface_id: Interface ID of the Plex Connection associated with + this Key. + - key: Rating Key within Plex that identifies the item to create the + Card(s) for. + - snapshot: Whether to take snapshot of the database after all Cards + have been processed. + """ + + return process_rating_key( + db, plex_interface, key, snapshot=snapshot, log=request.state.log + ) + + +@webhook_router.post('/sonarr/cards', tags=['Sonarr']) +def create_cards_for_sonarr_webhook( + request: Request, + webhook: SonarrWebhook = Body(...), + db: Session = Depends(get_database), + ) -> None: + """ + Create the Title Card for the items associated with the given Sonarr + Webhook payload. This is practically identical to the `/key` + endpoint. + + - webhook: Webhook payload containing Series and Episode details to + create the Title Cards of. + """ + + # Skip if payload has no Episodes to create Cards for + if not webhook.episodes: + return None + + # Get contextual logger + log: Logger = request.state.log + + # Create SeriesInfo for this payload's series + series_info = SeriesInfo( + name=webhook.series.title, + year=webhook.series.year, + imdb_id=webhook.series.imdbId, + tvdb_id=webhook.series.tvdbId, + tvrage_id=webhook.series.tvRageId, + ) + + # Search for this Series + series = db.query(Series)\ + .filter(series_info.filter_conditions(Series))\ + .first() + + # Series is not found, exit + if series is None: + log.info(f'Cannot find Series {series_info}') + return None + + def _find_episode(episode_info: EpisodeInfo) -> Optional[Episode]: + """Attempt to find the associated Episode up to three times.""" + + for _ in range(3): + # Search for this Episode + episode = db.query(Episode)\ + .filter(Episode.series_id==series.id, + episode_info.filter_conditions(Episode))\ + .first() + + # Episode exists, return it + if episode: + return episode + + # Sleep and re-query Episode data + log.debug(f'Cannot find Episode, waiting..') + sleep(30) + refresh_episode_data(db, series, log=log) + + return None + + # Find each Episode in the payload + for webhook_episode in webhook.episodes: + episode_info = EpisodeInfo( + title=webhook_episode.title, + season_number=webhook_episode.seasonNumber, + episode_number=webhook_episode.episodeNumber, + tvdb_id=webhook_episode.tvdbId, + ) + + # Find this Episode + if (episode := _find_episode(episode_info)) is None: + log.info(f'Cannot find Episode for {series_info} {episode_info}') + return None + + # Assume Episode is unwatched + episode.watched = False + + # Look for source, add translation, create Card if source exists + images = download_episode_source_images(db, episode, log=log) + translate_episode(db, episode, log=log) + if not images: + log.info(f'{episode} has no source image - skipping') + continue + create_episode_cards(db, episode, log=log) + + # Refresh this Episode so that relational Card objects are + # updated, preventing stale (deleted) Cards from being used in + # the Loaded asset evaluation. Not sure why this is required + # because SQLAlchemy should update child objects when the DELETE + # is committed; but this does not happen. + db.refresh(episode) + + # Reload into all associated libraries + for library in series.libraries: + if (interface := get_interface(library['interface_id'], raise_exc=False)): + load_episode_title_card( + episode, db, library['name'], library['interface_id'], interface, + attempts=6, log=log, + ) + else: + log.debug(f'Not loading {series_info} {episode_info} into ' + f'library "{library["name"]}" - no valid Connection') + continue + + return None + + +@webhook_router.post('/sonarr/delete', tags=['Sonarr']) +def delete_series_via_sonarr_webhook( + request: Request, + webhook: SonarrWebhook, + delete_title_cards: bool = Query(default=True), + db: Session = Depends(get_database) + ) -> None: + """ + Delete the Series defined in the given Webhook. + + - webhook: Webhook payload containing the details of the Series to + delete. + - delete_title_cards: Whether to delete Title Cards. + """ + + # Skip if Webhook type is not a Series deletion + if webhook.eventType != 'SeriesDelete': + return None + + # Create SeriesInfo for this payload's series + series_info = SeriesInfo( + name=webhook.series.title, + year=webhook.series.year, + imdb_id=webhook.series.imdbId, + tvdb_id=webhook.series.tvdbId, + tvrage_id=webhook.series.tvRageId, + ) + + # Search for this Series + series = db.query(Series)\ + .filter(series_info.filter_conditions(Series))\ + .first() + + # Series is not found, exit + if series is None: + raise HTTPException( + status_code=404, + detail=f'Series {series_info} not found', + ) + + # Delete Card, Loaded, and Series, as well all child content + if delete_title_cards: + delete_cards( + db, + db.query(Card).filter_by(series_id=series.id), + db.query(Loaded).filter_by(series_id=series.id), + log=request.state.log, + ) + delete_series(db, series, log=request.state.log) + return None diff --git a/app/schemas/connection.py b/app/schemas/connection.py index 4a98b95a..3835ae69 100755 --- a/app/schemas/connection.py +++ b/app/schemas/connection.py @@ -1,6 +1,6 @@ # pylint: disable=missing-class-docstring,missing-function-docstring,no-self-argument # pyright: reportInvalidTypeForm=false -from typing import Any, Literal, Optional, Union +from typing import Literal, Optional, Union from pydantic import AnyUrl, constr, validator @@ -257,28 +257,3 @@ class TVDbConnection(Base): TMDbConnection, TVDbConnection, ] - -""" -Sonarr Webhooks -""" -class WebhookSeries(Base): - id: int - title: str - year: int - imdbId: Any = None - tvdbId: Any = None - tvRageId: Any = None - -class WebhookEpisode(Base): - id: int - episodeNumber: int - seasonNumber: int - title: str - seriesId: int - # Added in v4.0.0.717 (https://github.com/Sonarr/Sonarr/pull/6151) - tvdbId: Optional[int] = None - -class SonarrWebhook(Base): - series: WebhookSeries - episodes: list[WebhookEpisode] = [] - eventType: str # Literal['SeriesAdd', 'Download', 'SeriesDelete', 'EpisodeFileDelete'] diff --git a/app/schemas/webhooks.py b/app/schemas/webhooks.py new file mode 100755 index 00000000..d492dd80 --- /dev/null +++ b/app/schemas/webhooks.py @@ -0,0 +1,121 @@ +# pylint: disable=missing-class-docstring,missing-function-docstring,no-self-argument +# pyright: reportInvalidTypeForm=false +from typing import Any, Literal, Optional + +from pydantic import AnyUrl + +from app.schemas.base import Base + +""" +Sonarr models +""" +class SonarrWebhookSeries(Base): + id: int + title: str + year: int + imdbId: Any = None + tvdbId: Any = None + tvRageId: Any = None + +class SonarrWebhookEpisode(Base): + id: int + episodeNumber: int + seasonNumber: int + title: str + seriesId: int + # Added in v4.0.0.717 (https://github.com/Sonarr/Sonarr/pull/6151) + tvdbId: Optional[int] = None + +class SonarrWebhook(Base): + series: SonarrWebhookSeries + episodes: list[SonarrWebhookEpisode] = [] + eventType: str # Literal['SeriesAdd', 'Download', 'SeriesDelete', 'EpisodeFileDelete'] + + +""" +Plex models - see https://support.plex.tv/articles/115002267687-webhooks/ +""" +PlexEvent = Literal[ + # New content + 'library.on.deck', + # A new item is added that appears in the user’s On Deck. A + # poster is also attached to this event. + 'library.new', + # A new item is added to a library to which the user has access. + # A poster is also attached to this event. + # Playback + 'media.pause', + # Media playback pauses. + 'media.play', + # Media starts playing. An appropriate poster is attached. + 'media.rate', + # Media is rated. A poster is also attached to this event. + 'media.resume', + # Media playback resumes. + 'media.scrobble', + # Media is viewed (played past the 90% mark). + 'media.stop', + # Media playback stops. + # Server Owner + 'admin.database.backup', + # A database backup is completed successfully via Scheduled Tasks. + 'admin.database.corrupted', + # Corruption is detected in the server database. + 'device.new', + # A device accesses the owner’s server for any reason, which may + # come from background connection testing and doesn’t + # necessarily indicate active browsing or playback. + 'playback.started', + # Playback is started by a shared user for the server. A poster + # is also attached to this event. +] + +class PlexAccount(Base): + id: int + thumb: AnyUrl + title: str + +class PlexServer(Base): + title: str + uuid: str + +class PlexPlayer(Base): + local: bool + publicAddress: str + title: str + uuid: str + +class PlexMetadata(Base): + # librarySectionType: str + ratingKey: int + # key: str + # parentRatingKey: int + # grandparentRatingKey: int + # guid: str + # librarySectionID: int + # type: str + # title: str + # grandparentKey: str + # parentKey: str + # grandparentTitle: str + # parentTitle: str + # summary: str + # index: int + # parentIndex: int + # ratingCount: int + # thumb: str + # art: str + # parentThumb: str + # grandparentThumb: str + # grandparentArt: str + # addedAt: int + # updatedAt: int + +class PlexWebhook(Base): + event: PlexEvent + # user: bool + # owner: bool + # Account: PlexAccoun + # Server: PlexServer + # Player: PlexPlayer + Metadata: PlexMetadata diff --git a/modules/TautulliInterface2.py b/modules/TautulliInterface2.py index b20e2f60..7db0edc1 100755 --- a/modules/TautulliInterface2.py +++ b/modules/TautulliInterface2.py @@ -68,8 +68,8 @@ def __init__(self, super().__init__('Tautulli', use_ssl, cache=False) # Get correct TCM URL - tcm_url = tcm_url.removesuffix('/') + '/' - self.tcm_url =f'{tcm_url}api/cards/key?interface_id={plex_interface_id}' + self.tcm_url = f'{tcm_url.removesuffix("/")}/api/webhooks/plex/rating-key?' \ + + f'interface_id={plex_interface_id}' # Get correct Tautulli URL tautulli_url = tautulli_url.removesuffix('/') + '/' diff --git a/modules/ref/version_webui b/modules/ref/version_webui index 90d67fcb..67882b3f 100755 --- a/modules/ref/version_webui +++ b/modules/ref/version_webui @@ -1 +1 @@ -v2.0-alpha.11.0-webui101 \ No newline at end of file +v2.0-alpha.11.0-webui102 \ No newline at end of file