From 73ce95f687f938ad5d4cc87a6e7af948c08b7bfa Mon Sep 17 00:00:00 2001 From: Greg V Date: Sun, 29 Sep 2024 14:38:27 -0700 Subject: [PATCH] [Performance] Split out volunteer data from hackathon payload for faster loading and better caching --- api/messages/messages_service.py | 231 +++++++++---------------------- api/messages/messages_views.py | 43 ++++-- common/utils/firebase.py | 47 +++++++ 3 files changed, 149 insertions(+), 172 deletions(-) diff --git a/api/messages/messages_service.py b/api/messages/messages_service.py index 047bcf4..339a57b 100644 --- a/api/messages/messages_service.py +++ b/api/messages/messages_service.py @@ -1,6 +1,6 @@ from common.utils import safe_get_env_var from common.utils.slack import send_slack_audit, create_slack_channel, send_slack, invite_user_to_channel -from common.utils.firebase import get_hackathon_by_event_id, upsert_news, upsert_praise, get_github_contributions_for_user +from common.utils.firebase import get_hackathon_by_event_id, upsert_news, upsert_praise, get_github_contributions_for_user,get_volunteer_from_db_by_event from common.utils.openai_api import generate_and_save_image_to_cdn from common.utils.github import create_github_repo from api.messages.message import Message @@ -35,7 +35,6 @@ logger = logging.getLogger("myapp") logger.setLevel(logging.DEBUG) -logger.setLevel(logging.DEBUG) google_recaptcha_key = safe_get_env_var("GOOGLE_CAPTCHA_SECRET_KEY") @@ -176,6 +175,25 @@ def get_single_hackathon_id(id): return result return {} +@cached(cache=TTLCache(maxsize=100, ttl=600)) +@limits(calls=2000, period=ONE_MINUTE) +def get_volunteer_by_event(event_id, volunteer_type): + logger.debug(f"get {volunteer_type} start event_id={event_id}") + + if event_id is None: + logger.warning(f"get {volunteer_type} end (no results)") + return [] + + results = get_volunteer_from_db_by_event(event_id, volunteer_type) + + if results is None: + logger.warning(f"get {volunteer_type} end (no results)") + return [] + else: + logger.debug(f"get {volunteer_type} end (with result):{results}") + return results + + @cached(cache=TTLCache(maxsize=100, ttl=600)) @limits(calls=2000, period=ONE_MINUTE) def get_single_hackathon_event(hackathon_id): @@ -922,68 +940,33 @@ def single_add_volunteer(event_id, json, propel_id): # Since we know this user is an admin, prefix all vars with admin_ admin_email, admin_user_id, admin_last_login, admin_profile_image, admin_name, admin_nickname = get_propel_user_details_by_id(propel_id) + + # Rename json["type"] to volunteer_type + if "type" in json: + json["volunteer_type"] = json["type"] + del json["type"] + + # Add created_by and created_timestamp + json["created_by"] = admin_name + json["created_timestamp"] = datetime.now().isoformat() - # Query for event_id column - doc = db.collection('hackathons').where("event_id", "==", event_id).stream() - doc = list(doc) - doc = doc[0] if len(doc) > 0 else None - - # Convert from DocumentSnapshot to DocumentReference - if isinstance(doc, firestore.DocumentSnapshot): - doc = doc.reference + fields_that_should_always_be_present = ["name", "timestamp"] + # We don't want to add the same name for the same event_id, so check that first + doc = db.collection('volunteers').where("event_id", "==", event_id).where("name", "==", json["name"]).stream() + # If we don't have a duplicate, then return + if len(list(doc)) > 0: + return Message("Volunteer already exists") + + # Query for event_id column in hackathons to ensure it exists + doc = db.collection('hackathons').where("event_id", "==", event_id).stream() # If we don't find the event, return - if doc is None: + if len(list(doc)) == 0: return Message("No Hackathon Found") - volunteer_type = json["type"] - - # This will help to make sure we don't always have all data in the Google Sheets/Forms and can add it later - fields_that_should_always_be_present = ["name", "timestamp"] - - if volunteer_type == "mentors": - logger.info("Adding Mentor") - # Get the mentor block - mentor_block = doc.get().to_dict()["mentors"] - # Add the new mentor - mentor = json - mentor["created_by"] = admin_name - mentor["created_timestamp"] = datetime.now().isoformat() - mentor_block.append(mentor) - # Update the mentor block with the new mentor - doc.update({ - "mentors": mentor_block - }) - elif volunteer_type == "judges": - logger.info("Adding Judge") - # Get the judge block - judge_block = doc.get().to_dict()["judges"] - # Add the new judge - judge = json - judge["created_by"] = admin_name - judge["created_timestamp"] = datetime.now().isoformat() - judge_block.append(judge) - # Update the judge block with the new judge - doc.update({ - "judges": judge_block - }) - elif volunteer_type == "volunteers": - logger.info("Adding Volunteer") - # Get the volunteer block - volunteer_block = doc.get().to_dict()["volunteers"] - # Add the new volunteer - volunteer = json - volunteer["created_by"] = admin_name - volunteer["created_timestamp"] = datetime.now().isoformat() - volunteer_block.append(volunteer) - # Update the volunteer block with the new volunteer - doc.update({ - "volunteers": volunteer_block - }) - - # Clear cache - get_single_hackathon_event.cache_clear() - + # Add the volunteer + doc = db.collection('volunteers').add(json) + return Message( "Added Hackathon Volunteer" ) @@ -1097,132 +1080,52 @@ def bulk_add_volunteers(event_id, json, propel_id): @limits(calls=50, period=ONE_MINUTE) -def update_hackathon_volunteers(event_id, json, propel_id): +def update_hackathon_volunteers(event_id, volunteer_type, json, propel_id): db = get_db() - logger.info("Update Hackathon Volunteers") + logger.info(f"update_hackathon_volunteers for event_id={event_id} propel_id={propel_id}") logger.info("JSON: " + str(json)) send_slack_audit(action="update_hackathon_volunteers", message="Updating", payload=json) + if "id" not in json: + logger.error("Missing id field") + return Message("Missing id field") + + volunteer_id = json["id"] + # Since we know this user is an admin, prefix all vars with admin_ admin_email, admin_user_id, admin_last_login, admin_profile_image, admin_name, admin_nickname = get_propel_user_details_by_id(propel_id) # Query for event_id column - doc = db.collection('hackathons').where("event_id", "==", event_id).stream() - doc = list(doc) - doc = doc[0] if len(doc) > 0 else None - - # Convert from DocumentSnapshot to DocumentReference - if isinstance(doc, firestore.DocumentSnapshot): - doc = doc.reference + doc = db.collection("volunteers").document(volunteer_id) # If we don't find the event, return if doc is None: - return Message("No Hackathon Found") - - volunteer_type = json["type"] - timestamp = json["timestamp"] - name = json["name"] - # We will use timestamp and name to find the volunteer - + return Message("No volunteer for Hackathon Found") + # This will help to make sure we don't always have all data in the Google Sheets/Forms and can add it later fields_that_should_always_be_present = ["slack_user_id", "pronouns", "linkedinProfile"] - if volunteer_type == "mentors": - logger.info("Updating Mentor") - # Get the mentor block - mentor_block = doc.get().to_dict()["mentors"] - - # Find the mentor - for mentor in mentor_block: - # Check if fields_that_should_always_be_present are present and if not add empty string - for field in fields_that_should_always_be_present: - if field not in mentor: - mentor[field] = "" - + # Make sure that fields are present in json + for field in fields_that_should_always_be_present: + if field not in json: + logger.error(f"Missing field {field} in {json}") + return Message("Missing field") + + # Update doc with timestamp and admin_name + json["updated_by"] = admin_name + json["updated_timestamp"] = datetime.now().isoformat() - logger.info(f"Comparing {mentor['name']} with {name} and {mentor['timestamp']} with {timestamp}") - if mentor["name"] == name and mentor["timestamp"] == timestamp: - # For each field in mentor, update with the new value from json - for key in mentor.keys(): - if key in json: - if key == "timestamp" or key == "name": # Don't want to update these since they are primary keys - continue - mentor[key] = json[key] - - mentor["updated_timestamp"] = datetime.now().isoformat() - mentor["updated_by"] = admin_name - - logger.info(f"Found mentor {mentor}") - break - # Update the mentor block with the json for this mentor - doc.update({ - "mentors": mentor_block - }) - elif volunteer_type == "judges": - # Get the judge block - judge_block = doc.get().to_dict()["judges"] - # Find the judge - for judge in judge_block: - # Check if fields_that_should_always_be_present are present and if not add empty string - for field in fields_that_should_always_be_present: - if field not in judge: - judge[field] = "" - - if judge["name"] == name and judge["timestamp"] == timestamp: - # For each field in judge, update with the new value from json - for key in judge.keys(): - if key in json: - if key == "timestamp" or key == "name": - continue - judge[key] = json[key] - judge["updated_timestamp"] = datetime.now().isoformat() - judge["updated_by"] = admin_name - - logger.info(f"Found judge {judge}") - break - # Update the judge block with the json for this judge - doc.update({ - "judges": judge_block - }) - elif volunteer_type == "volunteers": - # Get the volunteer block - volunteer_block = doc.get().to_dict()["volunteers"] - # Find the volunteer - for volunteer in volunteer_block: - # Check if fields_that_should_always_be_present are present and if not add empty string - for field in fields_that_should_always_be_present: - if field not in volunteer: - volunteer[field] = "" - - if volunteer["name"] == name and volunteer["timestamp"] == timestamp: - # For each field in volunteer, update with the new value from json - for key in volunteer.keys(): - if key in json: - if key == "timestamp" or key == "name": - continue - volunteer[key] = json[key] - volunteer["updated_timestamp"] = datetime.now().isoformat() - volunteer["updated_by"] = admin_name - - logger.info(f"Found volunteer {volunteer}") - break - # Update the volunteer block with the json for this volunteer - doc.update({ - "volunteers": volunteer_block - }) + # Update the volunteer record with the new data + doc.update(json) + + # Clear cache for get_volunteer_by_event + get_volunteer_by_event.cache_clear() - # Clear cache - get_single_hackathon_event.cache_clear() - return Message( "Updated Hackathon Volunteers" ) - - - - @limits(calls=50, period=ONE_MINUTE) def save_hackathon(json): db = get_db() # this connects to our Firestore database diff --git a/api/messages/messages_views.py b/api/messages/messages_views.py index 05ce63f..2d5a488 100644 --- a/api/messages/messages_views.py +++ b/api/messages/messages_views.py @@ -47,7 +47,8 @@ get_github_profile, save_praise, save_feedback, - get_user_feedback + get_user_feedback, + get_volunteer_by_event ) logger = logging.getLogger("myapp") @@ -139,13 +140,6 @@ def update_npo_application_api(application_id): def add_hackathon(): return vars(save_hackathon(request.get_json())) -@bp.route("/hackathon//volunteers", methods=["PATCH"]) -@auth.require_user -@auth.require_org_member_with_permission("volunteer.admin", req_to_org_id=getOrgId) -def update_hackathon_volunteers_mentors_judges(event_id): - if auth_user and auth_user.user_id: - return vars(update_hackathon_volunteers(event_id, request.get_json(), auth_user.user_id)) - @bp.route("/hackathon//volunteers/bulk", methods=["POST"]) @auth.require_user @@ -176,6 +170,39 @@ def list_hackathons(): def get_single_hackathon_by_event(event_id): return (get_single_hackathon_event(event_id)) +@bp.route("/hackathon//mentor", methods=["GET"]) +def get_volunteer_mentor_by_event_api(event_id): + return (get_volunteer_by_event(event_id, "mentor")) + +@bp.route("/hackathon//judge", methods=["GET"]) +def get_volunteer_judge_by_event_api(event_id): + return (get_volunteer_by_event(event_id, "judge")) + +@bp.route("/hackathon//volunteer", methods=["GET"]) +def get_volunteer_volunteers_by_event_api(event_id): + return (get_volunteer_by_event(event_id, "volunteer")) + +# ------------------- PATCH ------------------- # +@bp.route("/hackathon//mentor", methods=["PATCH"]) +@auth.require_user +@auth.require_org_member_with_permission("volunteer.admin", req_to_org_id=getOrgId) +def update_mentor_by_event_id(event_id): + if auth_user and auth_user.user_id: + return vars(update_hackathon_volunteers(event_id, "mentors", request.get_json(), auth_user.user_id)) + +@bp.route("/hackathon//judge", methods=["PATCH"]) +@auth.require_user +@auth.require_org_member_with_permission("volunteer.admin", req_to_org_id=getOrgId) +def update_judge_by_event_id(event_id): + return vars(update_hackathon_volunteers(event_id, "judges", request.get_json(), auth_user.user_id)) + +@bp.route("/hackathon//volunteer", methods=["PATCH"]) +@auth.require_user +@auth.require_org_member_with_permission("volunteer.admin", req_to_org_id=getOrgId) +def update_volunteer_by_event_id(event_id): + return vars(update_hackathon_volunteers(event_id, "volunteers", request.get_json(), auth_user.user_id)) + + @bp.route("/hackathon/id/", methods=["GET"]) def get_single_hackathon_by_id(id): return (get_single_hackathon_id(id)) diff --git a/common/utils/firebase.py b/common/utils/firebase.py index 2490252..43696c7 100644 --- a/common/utils/firebase.py +++ b/common/utils/firebase.py @@ -5,6 +5,8 @@ from mockfirestore import MockFirestore import datetime import re +from google.cloud.firestore import FieldFilter + cert_env = json.loads(safe_get_env_var("FIREBASE_CERT_CONFIG")) cred = credentials.Certificate(cert_env) @@ -1059,3 +1061,48 @@ def upsert_praise(praise): db.collection("praises").add(praise) logger.info("praise successfully saved") + + +def get_volunteer_from_db_by_event(event_id: str, volunteer_type: str) -> dict: + """ + Retrieve volunteers for a specific event and type. + + Args: + event_id (str): The ID of the event. + volunteer_type (str): The type of volunteer (e.g., 'mentor', 'judge', 'volunteer'). + + Returns: + dict: A dictionary containing a list of volunteer data. + """ + logger.debug(f"Getting {volunteer_type}s for event_id={event_id}") + + if not event_id: + logger.warning(f"get {volunteer_type}s end (no event_id provided)") + return {"data": []} + + db = get_db() + + try: + # Use FieldFilter for more explicit and type-safe queries + query = db.collection("volunteers").where( + filter=FieldFilter("event_id", "==", event_id) + ).where( + filter=FieldFilter("volunteer_type", "==", volunteer_type) + ) + + # Stream the documents and convert to list of dicts also with their id from the database + volunteers = [ {**doc.to_dict(), "id": doc.id} for doc in query.stream() ] + + if not volunteers: + logger.info(f"No {volunteer_type}s found for event_id={event_id}") + logger.debug(f"get {volunteer_type}s end (no results)") + return {"data": []} + + logger.info(f"Retrieved {len(volunteers)} {volunteer_type}s for event_id={event_id}") + logger.debug(f"get {volunteer_type}s end (with results)") + + return {"data": volunteers} + + except Exception as e: + logger.error(f"Error retrieving {volunteer_type}s: {str(e)}") + return {"data": [], "error": str(e)} \ No newline at end of file