Merge refresh-button to dev

backend/aws_helpers/dynamo_db_utils/base_db.py

@@ -221,7 +221,7 @@ def get_record(self, partition_id: Any) -> BaseData:
         item = self.__number_decoder(item)        
         self.__param_checker("approve", **item)
         return self.DataClass(**item)
-
+        
     def update_record(self, partition_id: Any, **kwargs) -> Literal['Success']:
         """Function to update a record with the partition_key values as attribute 'partition_id' from 
             the associated DynamoDB table. It takes in changes in attributes as keyword arguments"""

backend/aws_helpers/dynamo_db_utils/execution_db.py

@@ -6,6 +6,7 @@
 from backend.aws_helpers.dynamo_db_utils.base_db import BaseData, BaseDDBUtil, enumclass, changevar
 from backend.common.constants import EXECUTION_TABLE_NAME, AWS_REGION
 from backend.common.utils import get_current_timestamp
+from boto3.dynamodb.conditions import Key
 from typing import Union
 
 @dataclass
@@ -65,6 +66,48 @@ def createUserExecutionsData(entryData: dict) -> str:
 def updateUserExecutionsData(requestData: dict) -> str:
     """
     Updates an entry from the `execution-table` DynamoDB table given an `execution_id`.
+    @param requestData: A dictionary containing the execution_id and other table attributes to be updated, with user_id as a required field
+    @return a success status message if the update is successful
+    """
+
+    required_keys = ["execution_id"]
+    if not validate_keys(requestData, required_keys):
+        raise ValueError(f"Missing keys {required_keys} in request body")
+
+    dynamoTable = ExecutionDDBUtil(EXECUTION_TABLE_NAME, AWS_REGION)
+    execution_id = requestData["execution_id"]
+    updatedRecord = ExecutionData(**requestData).__dict__
+    updatedRecord.pop("execution_id")
+    dynamoTable.update_record(execution_id, **updatedRecord)
+    return "{\"status\": \"success\"}"
+
+def getAllUserExecutionsData(user_id: str) -> str:
+    """
+    Retrieves an entry from the `execution-table` DynamoDB table given an `execution_id`. Example output: {"execution_id": "blah", "user_id": "blah", "name": "blah", "timestamp": "blah", "data_source": "TABULAR", "status": "QUEUED", "progress": 1}
+
+    @param execution_id: The execution_id of the entry to be retrieved
+    @return: A JSON string of the entry retrieved from the table
+    """
+    dynamoTable = ExecutionDDBUtil(EXECUTION_TABLE_NAME, AWS_REGION)
+    response = dynamoTable.table.query(IndexName='user_id', KeyConditionExpression=Key('user_id').eq(user_id))
+    items = response["Items"]
+    record = []
+    for item in items:
+        data_source, name, progress, status, timestamp, execution_id = item["data_source"], item["name"], item["progress"], item["status"], item["timestamp"], item["execution_id"]
+        record.append({"data_source": data_source, "name": name, "progress": float(progress), "status": status, "timestamp": timestamp, "execution_id": execution_id})
+
+
+    while 'LastEvaluatedKey' in response:
+        key = response['LastEvaluatedKey']
+        response = dynamoTable.table.query(KeyConditionExpression=Key('user_id').eq(user_id), ExclusiveStartKey=key)
+        for item in items:
+            data_source, name, progress, status, timestamp, execution_id = item["data_source"], item["name"], item["progress"], item["status"], item["timestamp"], item["execution_id"]
+            record.append({"data_source": data_source, "name": name, "progress": float(progress), "status": status, "timestamp": timestamp, "execution_id": execution_id})
+    return json.dumps(record)
+
+def updateUserExecutionsData_(requestData: dict) -> str:
+    """
+    Updates an entry from the `execution-table` DynamoDB table given an `execution_id`.
 
     @param requestData: A dictionary containing the execution_id and other table attributes to be updated, with user_id as a required field
     @return a success status message if the update is successful

backend/aws_helpers/s3_utils/s3_client.py

@@ -37,4 +37,20 @@ def read_from_bucket(bucket_name: str, bucket_path: str, output_file_name: str,
     s3.Bucket(bucket_name).download_file(bucket_path, output_file_name)
     shutil.move(f"{os.getcwd()}/{output_file_name}", f"{output_file_path}/{output_file_name}")
 
+def get_presigned_url_from_bucket(bucket_name: str, bucket_path: str):
+    """
+    Given S3 URI, read the file from the S3 bucket
+
+    Args:
+        bucket_name (str): name of s3 bucket
+        bucket_path (str): path within s3 bucket where the file resides
+        output_file_name (str): name of file to download S3 object to (this is because of the way the boto3 endpoint works)
+        output_file_path (str): filepath to download file to (ie: what folder/directory)
+
+    """
+    s3 = boto3.client('s3')
+    return s3.generate_presigned_url("get_object", Params={'Bucket': bucket_name, 'Key': bucket_path})
+
+def get_presigned_url_from_exec_file(bucket_name: str, exec_id: str, filename: str):
+    return get_presigned_url_from_bucket(bucket_name, exec_id + "/" + filename)
 

backend/driver.py

@@ -5,6 +5,7 @@
 import shutil
 
 from flask import Flask, request, send_from_directory
+from backend.aws_helpers.s3_utils.s3_client import get_presigned_url_from_exec_file
 from backend.middleware import middleware
 from flask_cors import CORS
 
@@ -15,8 +16,8 @@
 from backend.common.utils import *
 from backend.firebase_helpers.firebase import init_firebase
 from backend.aws_helpers.dynamo_db_utils.learnmod_db import UserProgressDDBUtil, UserProgressData
-from backend.aws_helpers.dynamo_db_utils.execution_db import ExecutionDDBUtil, ExecutionData, createUserExecutionsData
 from backend.aws_helpers.sqs_utils.sqs_client import add_to_training_queue
+from backend.aws_helpers.dynamo_db_utils.execution_db import createUserExecutionsData, getAllUserExecutionsData
 from backend.common.constants import EXECUTION_TABLE_NAME, AWS_REGION, USERPROGRESS_TABLE_NAME, POINTS_PER_QUESTION
 from backend.dl.detection import detection_img_drive
 
@@ -282,6 +283,29 @@ def send_columns():
         print(traceback.format_exc())
         return send_traceback_error()
 
+@app.route("/api/getExecutionsData", methods=["POST"])
+def executions_table():
+    try:
+        request_data = json.loads(request.data)
+        user_id = request_data['user_id']
+        record = getAllUserExecutionsData(user_id)
+        return send_success({"record": record})
+    except Exception:
+        print(traceback.format_exc())
+        return send_traceback_error()
+
+@app.route("/api/getExecutionsFilesPresignedUrls", methods=["POST"])
+def executions_files():
+    try:
+        request_data = json.loads(request.data)
+        exec_id = request_data['exec_id']
+        dl_results = get_presigned_url_from_exec_file("dlp-executions-bucket", exec_id, "dl_results.csv")
+        model_pt = get_presigned_url_from_exec_file("dlp-executions-bucket", exec_id, "model.pt")
+        model_onnx = get_presigned_url_from_exec_file("dlp-executions-bucket", exec_id, "my_deep_learning_model.onnx")
+        return send_success({"dl_results": dl_results, "model_pt": model_pt, "model_onnx": model_onnx})
+    except Exception:
+        print(traceback.format_exc())
+        return send_traceback_error()
 
 @app.route("/api/upload", methods=["POST"])
 def upload():
@@ -317,23 +341,6 @@ def writeToQueue() -> str:
             return send_success({"message": "Successfully added your training request to the queue"})
     except Exception:
         return send_error("Failed to queue data")
-
-
-@app.route("/api/getUserExecutionsData", methods=["POST"])
-def getUserExecutionsData() -> str:
-    """
-    Retrieves an entry from the `execution-table` DynamoDB table given an `execution_id`. If does not exist, create a new entry corresponding to the given user_id.
-
-    E.g.
-    POST request to http://localhost:8000/api/getUserExecutionsData with body
-    {"execution_id": "fsdh", "user_id": "fweadshas"}
-    will return the execution_id = "fsdh" entry if it exists, else create a new entry with the given execution_id and other attributes present e.g. user_id (user_id must be present upon creating a new entry)
-
-    @return: A JSON string of the entry retrieved or created from the table
-    """
-    entryData = json.loads(request.data)
-    return getOrCreateUserExecutionsData(entryData)
-
 
 @app.route("/api/getUserProgressData", methods=["POST"])
 def getUserProgressData():