From e5e42ba8afa767d7c375d9a70619bd45d52063f8 Mon Sep 17 00:00:00 2001 From: Ptitloup Date: Mon, 2 Oct 2023 15:59:19 +0200 Subject: [PATCH 1/5] Use cache to store video data - get it in context - create command to store video data --- pod/video/context_processors.py | 74 +++++++++++-------- .../management/commands/cache_video_data.py | 62 ++++++++++++++++ 2 files changed, 106 insertions(+), 30 deletions(-) create mode 100644 pod/video/management/commands/cache_video_data.py diff --git a/pod/video/context_processors.py b/pod/video/context_processors.py index f76f2e337b..c7b90e72f8 100644 --- a/pod/video/context_processors.py +++ b/pod/video/context_processors.py @@ -10,6 +10,7 @@ from django.db.models import OuterRef from datetime import timedelta +from django.core.cache import cache from django.contrib.sites.shortcuts import get_current_site from pod.video_encode_transcript.models import EncodingVideo from pod.video_encode_transcript.models import PlaylistVideo @@ -19,7 +20,7 @@ HIDE_USER_FILTER = getattr(django_settings, "HIDE_USER_FILTER", False) OEMBED = getattr(django_settings, "OEMBED", False) USE_STATS_VIEW = getattr(django_settings, "USE_STATS_VIEW", False) - +CACHE_VIDEO_DEFAULT_TIMEOUT = getattr(django_settings, "CACHE_VIDEO_DEFAULT_TIMEOUT", 60) __AVAILABLE_VIDEO_FILTER__ = { "encoding_in_progress": False, "is_draft": False, @@ -78,37 +79,50 @@ def context_video_settings(request): return new_settings -def context_navbar(request): - types = ( - Type.objects.filter( - sites=get_current_site(request), - video__is_draft=False, - video__sites=get_current_site(request), +def context_video_data(request): + """Get video data in cache, if not, create and add it in cache.""" + types = cache.get('TYPES') + if types is None: + types = ( + Type.objects.filter( + sites=get_current_site(request), + video__is_draft=False, + video__sites=get_current_site(request), + ) + .distinct() + .annotate(video_count=Count("video", distinct=True)) ) - .distinct() - .annotate(video_count=Count("video", distinct=True)) - ) - - disciplines = ( - Discipline.objects.filter( - site=get_current_site(request), - video__is_draft=False, - video__sites=get_current_site(request), + cache.set("TYPES", types, timeout=CACHE_VIDEO_DEFAULT_TIMEOUT) + + disciplines = cache.get('DISCIPLINES') + if disciplines is None: + disciplines = ( + Discipline.objects.filter( + site=get_current_site(request), + video__is_draft=False, + video__sites=get_current_site(request), + ) + .distinct() + .annotate(video_count=Count("video", distinct=True)) ) - .distinct() - .annotate(video_count=Count("video", distinct=True)) - ) - - v_filter = get_available_videos_filter(request) - - aggregate_videos = v_filter.aggregate(duration=Sum("duration"), number=Count("id")) - - VIDEOS_COUNT = aggregate_videos["number"] - VIDEOS_DURATION = ( - str(timedelta(seconds=aggregate_videos["duration"])) - if aggregate_videos["duration"] - else 0 - ) + cache.set("DISCIPLINES", disciplines, timeout=CACHE_VIDEO_DEFAULT_TIMEOUT) + + VIDEOS_COUNT = cache.get('VIDEOS_COUNT') + VIDEOS_DURATION = cache.get('VIDEOS_DURATION') + if VIDEOS_COUNT is None: + v_filter = get_available_videos_filter(request) + aggregate_videos = v_filter.aggregate( + duration=Sum("duration"), + number=Count("id") + ) + VIDEOS_COUNT = aggregate_videos["number"] + cache.set("VIDEOS_COUNT", VIDEOS_COUNT, timeout=CACHE_VIDEO_DEFAULT_TIMEOUT) + VIDEOS_DURATION = ( + str(timedelta(seconds=aggregate_videos["duration"])) + if aggregate_videos["duration"] + else 0 + ) + cache.set("VIDEOS_DURATION", VIDEOS_DURATION, timeout=CACHE_VIDEO_DEFAULT_TIMEOUT) return { "TYPES": types, diff --git a/pod/video/management/commands/cache_video_data.py b/pod/video/management/commands/cache_video_data.py new file mode 100644 index 0000000000..3b329062c9 --- /dev/null +++ b/pod/video/management/commands/cache_video_data.py @@ -0,0 +1,62 @@ +from django.core.management.base import BaseCommand +from django.core.cache import cache +from datetime import timedelta +from django.contrib.sites.shortcuts import get_current_site +from django.db.models import Count, Sum +from django.conf import settings as django_settings + +from pod.video.models import Type +from pod.video.models import Discipline +from pod.video.context_processors import get_available_videos_filter + +CACHE_VIDEO_DEFAULT_TIMEOUT = getattr(django_settings, "CACHE_VIDEO_DEFAULT_TIMEOUT", 60) + + +class Command(BaseCommand): + help = "Store video data in django cache : " \ + + "types, discipline, video count and videos duration" + + def handle(self, *args, **options): + request = None + types = ( + Type.objects.filter( + sites=get_current_site(request), + video__is_draft=False, + video__sites=get_current_site(request), + ) + .distinct() + .annotate(video_count=Count("video", distinct=True)) + ) + cache.delete('TYPES') + cache.set("TYPES", types, timeout=None) + + disciplines = ( + Discipline.objects.filter( + site=get_current_site(request), + video__is_draft=False, + video__sites=get_current_site(request), + ) + .distinct() + .annotate(video_count=Count("video", distinct=True)) + ) + cache.delete('DISCIPLINES') + cache.set("DISCIPLINES", disciplines, timeout=None) + + v_filter = get_available_videos_filter(request) + + aggregate_videos = v_filter.aggregate(duration=Sum("duration"), number=Count("id")) + + VIDEOS_COUNT = aggregate_videos["number"] + cache.delete('VIDEOS_COUNT') + cache.set("VIDEOS_COUNT", VIDEOS_COUNT, timeout=None) + VIDEOS_DURATION = ( + str(timedelta(seconds=aggregate_videos["duration"])) + if aggregate_videos["duration"] + else 0 + ) + cache.delete('VIDEOS_DURATION') + cache.set("VIDEOS_DURATION", VIDEOS_DURATION, timeout=None) + + self.stdout.write( + self.style.SUCCESS('Successfully store video data in cache') + ) From 093233b43a1e39058ee142a201bbe98f3c277888 Mon Sep 17 00:00:00 2001 From: Ptitloup Date: Mon, 2 Oct 2023 16:16:15 +0200 Subject: [PATCH 2/5] change name of video context processor --- pod/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pod/settings.py b/pod/settings.py index 39f0867b10..00a1179592 100644 --- a/pod/settings.py +++ b/pod/settings.py @@ -117,7 +117,7 @@ # Local contexts "pod.main.context_processors.context_settings", "pod.main.context_processors.context_footer", - "pod.video.context_processors.context_navbar", + "pod.video.context_processors.context_video_data", "pod.video.context_processors.context_video_settings", "pod.authentication.context_processors.context_authentication_settings", "pod.recorder.context_processors.context_recorder_settings", From badf73a5ca7981f551711bbe4dc9e7582c8d9d20 Mon Sep 17 00:00:00 2001 From: Ptitloup Date: Mon, 2 Oct 2023 16:17:32 +0200 Subject: [PATCH 3/5] add some pydoc --- pod/video/management/commands/cache_video_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pod/video/management/commands/cache_video_data.py b/pod/video/management/commands/cache_video_data.py index 3b329062c9..3fd7a7a8c8 100644 --- a/pod/video/management/commands/cache_video_data.py +++ b/pod/video/management/commands/cache_video_data.py @@ -13,10 +13,12 @@ class Command(BaseCommand): + """Command to store video data in cache.""" help = "Store video data in django cache : " \ + "types, discipline, video count and videos duration" def handle(self, *args, **options): + """Function called to store video data in cache.""" request = None types = ( Type.objects.filter( From e712e731002b9be85167665aca9bc1e950315e3c Mon Sep 17 00:00:00 2001 From: Ptitloup Date: Tue, 3 Oct 2023 10:13:38 +0200 Subject: [PATCH 4/5] update cache timeout to 600 seconds - use DRY and call context cache video data instead of repeat code --- pod/video/context_processors.py | 2 +- .../management/commands/cache_video_data.py | 65 +++++-------------- 2 files changed, 16 insertions(+), 51 deletions(-) diff --git a/pod/video/context_processors.py b/pod/video/context_processors.py index c7b90e72f8..c0bc18a2b4 100644 --- a/pod/video/context_processors.py +++ b/pod/video/context_processors.py @@ -20,7 +20,7 @@ HIDE_USER_FILTER = getattr(django_settings, "HIDE_USER_FILTER", False) OEMBED = getattr(django_settings, "OEMBED", False) USE_STATS_VIEW = getattr(django_settings, "USE_STATS_VIEW", False) -CACHE_VIDEO_DEFAULT_TIMEOUT = getattr(django_settings, "CACHE_VIDEO_DEFAULT_TIMEOUT", 60) +CACHE_VIDEO_DEFAULT_TIMEOUT = getattr(django_settings, "CACHE_VIDEO_DEFAULT_TIMEOUT", 600) __AVAILABLE_VIDEO_FILTER__ = { "encoding_in_progress": False, "is_draft": False, diff --git a/pod/video/management/commands/cache_video_data.py b/pod/video/management/commands/cache_video_data.py index 3fd7a7a8c8..eb7849d034 100644 --- a/pod/video/management/commands/cache_video_data.py +++ b/pod/video/management/commands/cache_video_data.py @@ -1,15 +1,8 @@ from django.core.management.base import BaseCommand from django.core.cache import cache -from datetime import timedelta -from django.contrib.sites.shortcuts import get_current_site -from django.db.models import Count, Sum -from django.conf import settings as django_settings - -from pod.video.models import Type -from pod.video.models import Discipline -from pod.video.context_processors import get_available_videos_filter - -CACHE_VIDEO_DEFAULT_TIMEOUT = getattr(django_settings, "CACHE_VIDEO_DEFAULT_TIMEOUT", 60) +from pod.video.context_processors import context_video_data +from django.core import serializers +import json class Command(BaseCommand): @@ -19,46 +12,18 @@ class Command(BaseCommand): def handle(self, *args, **options): """Function called to store video data in cache.""" + cache.delete(['DISCIPLINES', 'VIDEOS_COUNT', 'VIDEOS_DURATION', 'TYPES']) request = None - types = ( - Type.objects.filter( - sites=get_current_site(request), - video__is_draft=False, - video__sites=get_current_site(request), - ) - .distinct() - .annotate(video_count=Count("video", distinct=True)) - ) - cache.delete('TYPES') - cache.set("TYPES", types, timeout=None) - - disciplines = ( - Discipline.objects.filter( - site=get_current_site(request), - video__is_draft=False, - video__sites=get_current_site(request), - ) - .distinct() - .annotate(video_count=Count("video", distinct=True)) - ) - cache.delete('DISCIPLINES') - cache.set("DISCIPLINES", disciplines, timeout=None) - - v_filter = get_available_videos_filter(request) - - aggregate_videos = v_filter.aggregate(duration=Sum("duration"), number=Count("id")) - - VIDEOS_COUNT = aggregate_videos["number"] - cache.delete('VIDEOS_COUNT') - cache.set("VIDEOS_COUNT", VIDEOS_COUNT, timeout=None) - VIDEOS_DURATION = ( - str(timedelta(seconds=aggregate_videos["duration"])) - if aggregate_videos["duration"] - else 0 - ) - cache.delete('VIDEOS_DURATION') - cache.set("VIDEOS_DURATION", VIDEOS_DURATION, timeout=None) - + video_data = context_video_data(request) + msg = 'Successfully store video data in cache' + for data in video_data: + try: + msg += "\n %s : %s" % ( + data, + json.dumps(serializers.serialize("json", video_data[data])) + ) + except (TypeError, AttributeError): + msg += "\n %s : %s" % (data, video_data[data]) self.stdout.write( - self.style.SUCCESS('Successfully store video data in cache') + self.style.SUCCESS(msg) ) From 58f331b83c1ed9c5354f740fe1f40b72a765cda6 Mon Sep 17 00:00:00 2001 From: Ptitloup Date: Tue, 3 Oct 2023 10:46:22 +0200 Subject: [PATCH 5/5] use delete many to clear cache --- pod/video/management/commands/cache_video_data.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pod/video/management/commands/cache_video_data.py b/pod/video/management/commands/cache_video_data.py index eb7849d034..9e7a7c33dc 100644 --- a/pod/video/management/commands/cache_video_data.py +++ b/pod/video/management/commands/cache_video_data.py @@ -12,9 +12,8 @@ class Command(BaseCommand): def handle(self, *args, **options): """Function called to store video data in cache.""" - cache.delete(['DISCIPLINES', 'VIDEOS_COUNT', 'VIDEOS_DURATION', 'TYPES']) - request = None - video_data = context_video_data(request) + cache.delete_many(['DISCIPLINES', 'VIDEOS_COUNT', 'VIDEOS_DURATION', 'TYPES']) + video_data = context_video_data(request=None) msg = 'Successfully store video data in cache' for data in video_data: try: