Add FastAPI Redis Caching

requirements.txt

@@ -41,7 +41,7 @@ PyMySQL==1.0.2
 pymssql==2.2.8
 python-jose[cryptography]==3.3.0
 pyyaml==6.0.1
-redis==3.5.3
+redis==4.6.0
 rich-click==1.6.1
 sendgrid==6.9.7
 slowapi==0.1.8

src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py

@@ -1,14 +1,12 @@
 import asyncio
 import uuid
+from functools import lru_cache
 from html import escape, unescape
 from typing import Dict, List, Optional
 
 from fastapi import Depends, HTTPException
 from fastapi import Query as FastAPIQuery
 from fastapi import Request, Response
-from fastapi_pagination import Page, Params
-from fastapi_pagination import paginate as fastapi_paginate
-from fastapi_pagination.bases import AbstractPage
 from loguru import logger
 from sqlalchemy.orm import Query, Session
 from starlette.status import (
@@ -25,7 +23,6 @@
 )
 from fides.api.models.privacy_notice import PrivacyNotice
 from fides.api.models.privacy_request import ProvidedIdentity
-from fides.api.schemas.privacy_experience import PrivacyExperienceResponse
 from fides.api.util.api_router import APIRouter
 from fides.api.util.consent_util import (
     PRIVACY_EXPERIENCE_ESCAPE_FIELDS,
@@ -46,7 +43,12 @@
 
 router = APIRouter(tags=["Privacy Experience"], prefix=urls.V1_URL_PREFIX)
 
+BUST_CACHE_HEADER = "bust-endpoint-cache"
+CACHE_HEADER = "X-Endpoint-Cache"
+PRIVACY_EXPERIENCE_CACHE: Dict[str, Dict] = {}
 
+
+@lru_cache(maxsize=20, typed=True)
 def get_privacy_experience_or_error(
     db: Session, experience_id: str
 ) -> PrivacyExperience:
@@ -64,6 +66,7 @@ def get_privacy_experience_or_error(
     return privacy_experience
 
 
+@lru_cache(maxsize=20, typed=True)
 def _filter_experiences_by_region_or_country(
     db: Session, region: Optional[str], experience_query: Query
 ) -> Query:
@@ -119,16 +122,14 @@ def _filter_experiences_by_region_or_country(
     return db.query(PrivacyExperience).filter(False)
 
 
+# TODO: readd the fides limiter
 @router.get(
     urls.PRIVACY_EXPERIENCE,
     status_code=HTTP_200_OK,
-    response_model=Page[PrivacyExperienceResponse],
 )
-@fides_limiter.limit(CONFIG.security.public_request_rate_limit)
 async def privacy_experience_list(
     *,
     db: Session = Depends(deps.get_db),
-    params: Params = Depends(),
     show_disabled: Optional[bool] = True,
     region: Optional[str] = None,
     component: Optional[ComponentType] = None,
@@ -140,7 +141,7 @@ async def privacy_experience_list(
     include_meta: Optional[bool] = False,
     request: Request,  # required for rate limiting
     response: Response,  # required for rate limiting
-) -> AbstractPage[PrivacyExperience]:
+) -> Dict:
     """
     Public endpoint that returns a list of PrivacyExperience records for individual regions with
     relevant privacy notices or tcf contents embedded in the response.
@@ -149,7 +150,6 @@ async def privacy_experience_list(
     notices as well.
 
     :param db:
-    :param params:
     :param show_disabled: If False, returns only enabled Experiences and Notices
     :param region: Return the Experiences for the given region
     :param component: Returns Experiences of the given component type
@@ -163,7 +163,33 @@ async def privacy_experience_list(
     :param response:
     :return:
     """
-    logger.info("Finding all Privacy Experiences with pagination params '{}'", params)
+
+    # These are the parameters that get used to create the cache.
+    param_hash_list = [
+        show_disabled,
+        region,
+        component,
+        content_required,
+        has_config,
+        fides_user_device_id,
+        systems_applicable,
+        include_gvl,
+        include_meta,
+    ]
+    # Create a custom hash that avoids unhashable parameters
+    cache_hash = "_".join([repr(x) for x in param_hash_list])
+
+    if request.headers.get(BUST_CACHE_HEADER):
+        PRIVACY_EXPERIENCE_CACHE.clear()
+
+    if PRIVACY_EXPERIENCE_CACHE.get(cache_hash):
+        logger.debug("Cache HIT: {}", cache_hash)
+        response.headers[CACHE_HEADER] = "HIT"
+        return PRIVACY_EXPERIENCE_CACHE[cache_hash]
+
+    logger.debug("Cache MISS: {}", cache_hash)
+    response.headers[CACHE_HEADER] = "MISS"
+
     fides_user_provided_identity: Optional[ProvidedIdentity] = None
     if fides_user_device_id:
         try:
@@ -259,7 +285,11 @@ async def privacy_experience_list(
 
         results.append(privacy_experience)
 
-    return fastapi_paginate(results, params=params)
+    # This is structured to look like a paginated result to minimize impact from
+    # the caching changes
+    api_result = {"items": results, "total": len(results)}
+    PRIVACY_EXPERIENCE_CACHE[cache_hash] = api_result
+    return api_result
 
 
 def embed_experience_details(

src/fides/api/util/cache.py

@@ -7,7 +7,6 @@
 from bson.objectid import ObjectId
 from loguru import logger
 from redis import Redis
-from redis.client import Script  # type: ignore
 from redis.exceptions import ConnectionError as ConnectionErrorFromRedis
 
 from fides.api import common_exceptions
@@ -63,6 +62,7 @@ def _custom_decoder(json_dict: Dict[str, Any]) -> Dict[str, Any]:
     return json_dict
 
 
+# pylint: disable=abstract-method
 class FidesopsRedis(Redis):
     """
     An extension to Redis' python bindings to support auto expiring data input. This class
@@ -95,10 +95,9 @@ def get_keys_by_prefix(self, prefix: str, chunk_size: int = 1000) -> List[str]:
 
     def delete_keys_by_prefix(self, prefix: str) -> None:
         """Delete all keys starting with a given prefix"""
-        s: Script = self.register_script(
+        self.register_script(
             f"for _,k in ipairs(redis.call('keys','{prefix}*')) do redis.call('del',k) end"
-        )
-        s()
+        )()
 
     def get_values(self, keys: List[str]) -> Dict[str, Optional[Any]]:
         """Retrieve all values corresponding to the set of input keys and return them as a