Add methods to log the results of the api runner to a server

eval/api_runner.py

@@ -189,6 +189,7 @@ def run_api_eval(args):
     logprobs = args.logprobs
     cot_table_alias = args.cot_table_alias
     sql_lora_path = args.adapter if args.adapter else None
+    run_name = args.run_name if args.run_name else None
     if sql_lora_path:
         print("Using LoRA adapter at:", sql_lora_path)
     if logprobs:
@@ -302,11 +303,17 @@ def run_api_eval(args):
         # with open(prompt_file, "r") as f:
         #     prompt = f.read()
 
-        # if args.upload_url is not None:
-        #     upload_results(
-        #         results=results,
-        #         url=args.upload_url,
-        #         runner_type="api_runner",
-        #         prompt=prompt,
-        #         args=args,
-        #     )
+        if run_name is None:
+            run_name = output_file.split("/")[-1].replace(".csv", "")
+            print(
+                "Run name not provided. Using a output filename for run name:", run_name
+            )
+
+        if args.upload_url is not None:
+            upload_results(
+                results=results,
+                url=args.upload_url,
+                runner_type="api_runner",
+                args=args,
+                run_name=run_name,
+            )

main.py

@@ -33,6 +33,7 @@
     parser.add_argument("-v", "--verbose", action="store_true")
     parser.add_argument("-l", "--logprobs", action="store_true")
     parser.add_argument("--upload_url", type=str)
+    parser.add_argument("--run_name", type=str, required=False)
     parser.add_argument(
         "-qz", "--quantized", default=False, action=argparse.BooleanOptionalAction
     )

utils/creds.py

@@ -23,7 +23,7 @@
         "creds": os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"),
     },
     "sqlite": {
-        "path_to_folder": f"{os.path.expanduser('~')}/workspace/defog-data/sqlite_dbs/",  # Path to folder containing sqlite dbs
+        "path_to_folder": f"./defog-data/sqlite_dbs/",  # Path to folder containing sqlite dbs
     },
     "tsql": {
         "server": os.getenv("TSQL_SERVER"),

utils/reporting.py

@@ -1,9 +1,7 @@
-import json
 import requests
 from uuid import uuid4
 from datetime import datetime
 import os
-import hashlib
 
 
 # get the GPU name this is running on
@@ -81,8 +79,8 @@ def num_gpus():
 def upload_results(
     results: list,
     url: str,
+    run_name: str,
     runner_type: str,
-    prompt: str,
     args: dict,
 ):
     """
@@ -94,26 +92,16 @@ def upload_results(
     # Create a unique id for the request
     run_id = uuid4().hex
 
-    # Create a unique id for the prompt, based on a hash of the prompt
-    prompt_id = hashlib.md5(prompt.encode()).hexdigest()
-
     # Create a dictionary with the request id and the results
     data = {
         "run_id": run_id,
         "results": results,
         "timestamp": datetime.now().isoformat(),
         "runner_type": runner_type,
-        "prompt": prompt,
-        "prompt_id": prompt_id,
         "model": args.model,
         "num_beams": args.num_beams,
         "db_type": args.db_type,
-        "gpu_name": get_gpu_name(),
-        "gpu_memory": get_gpu_memory(),
-        "gpu_driver_version": get_gpu_driver_version(),
-        "gpu_cuda_version": get_gpu_cuda_version(),
-        "num_gpus": num_gpus(),
-        "run_args": vars(args),
+        "run_name": run_name,
     }
     # Send the data to the server
     response = requests.post(url, json=data)

utils/upload_report_gcloud.py

@@ -0,0 +1,19 @@
+# this is a Google cloud function for receiving the data from the web app and storing it in the database
+
+import functions_framework
+from google.cloud import storage
+import json
+
+BUCKET_NAME = "YOUR-BUCKET-NAME"
+
+
+@functions_framework.http
+def hello_http(request):
+    request_json = request.get_json(silent=True)
+    results = request_json["results"]
+    run_name = request_json["run_name"]
+    storage_client = storage.Client()
+    bucket = storage_client.bucket(BUCKET_NAME)
+    blob = bucket.blob(run_name + ".json")
+    blob.upload_from_string(json.dumps(results))
+    return "success"