current status

JannikSchneider12 · Oct 22, 2024 · f5f7a56 · f5f7a56
1 parent a3042aa
commit f5f7a56
Show file tree

Hide file tree

Showing 5 changed files with 2,830 additions and 5 deletions.
diff --git a/6858dceb-1a53-4507-b1e8-cd507ed9a2ad.csv b/6858dceb-1a53-4507-b1e8-cd507ed9a2ad.csv
diff --git a/70240a21-a925-4b9d-8879-3d0475d7c660.csv b/70240a21-a925-4b9d-8879-3d0475d7c660.csv
diff --git a/remote_challenge_evaluation/eval_ai_interface.py b/remote_challenge_evaluation/eval_ai_interface.py
@@ -115,8 +115,13 @@ def update_submission_data(self, data):
             [JSON]: JSON response data
         """
         url = URLS.get("update_submission").format(self.CHALLENGE_PK)
+        print(f"update_submission_debug get url: {url}")
         url = self.return_url_per_environment(url)
+        print(f"update_submission_debug return url: {url}")
+        # Log the data being sent
+        print("Data being sent to API:", data)
         response = self.make_request(url, "PUT", data=data)
+        print(f"response: {response}")
         return response
 
     def update_submission_status(self, data):

diff --git a/remote_challenge_evaluation/evaluate.py b/remote_challenge_evaluation/evaluate.py
@@ -104,6 +104,7 @@ def evaluate(test_annotation_file_path, user_submission_file_path, phase_codenam
             }
         ]
 
+    '''
     # To display the results in the result file
     if phase_codename == "dev":
 
@@ -114,7 +115,7 @@ def evaluate(test_annotation_file_path, user_submission_file_path, phase_codenam
         output["submission_result"] = output["result"][1]
 
     # print(output['submission_result'])
-
+    '''
     print(f"Completed evaluation for {phase_codename}")
 
     return output
diff --git a/remote_challenge_evaluation/main.py b/remote_challenge_evaluation/main.py
@@ -58,7 +58,14 @@ def load_config(file_path='github/host_config.json'):
 challenge_pk = "2389" #os.environ["CHALLENGE_PK"]
 save_dir = os.environ.get("SAVE_DIR", "./")
 
-
+'''
+print("Debugging for parameters")
+print(f"auth_token: {auth_token}")
+print(f"evalai_api_server: {evalai_api_server}")
+print(f"queue_name: {queue_name}")
+print(f"challenge_pk: {challenge_pk}")
+print(f"save_dir: {save_dir}")
+'''
 
 def download(submission, save_dir):
     response = requests.get(submission["input_file"])
@@ -120,33 +127,43 @@ def update_finished(
         # Get the message from the queue
         message = evalai.get_message_from_sqs_queue()
         message_body = message.get("body")
+        print(f"Fetched message: {message}")
+        print(f'message_body: {message_body}')
         if message_body:
             submission_pk = message_body.get("submission_pk")
             challenge_pk = message_body.get("challenge_pk")
             phase_pk = message_body.get("phase_pk")
             # Get submission details -- This will contain the input file URL
             submission = evalai.get_submission_by_pk(submission_pk)
             challenge_phase = evalai.get_challenge_phase_by_pk(phase_pk)
+
             if (
                 submission.get("status") == "finished"
                 or submission.get("status") == "failed"
                 or submission.get("status") == "cancelled"
             ):
+                print("if condition successful")
                 message_receipt_handle = message.get("receipt_handle")
                 evalai.delete_message_from_sqs_queue(message_receipt_handle)
 
             else:
+                print("else condition successful")
                 if submission.get("status") == "submitted":
                     update_running(evalai, submission_pk)
                 submission_file_path = download(submission, save_dir)
+                print(f'submission_file_path: {submission_file_path}')
                 try:
-                    results = evaluate(
-                        submission_file_path, challenge_phase["codename"]
+                    print("try evaluate")
+                    results = evaluate(test_annotation_file_path='./annotations/test_annotations_testsplit.json',
+                        user_submission_file_path=submission_file_path, phase_codename=challenge_phase["codename"]
                     )
+                    print(f"results: {results}")
+                    print(f'evalai: {evalai}, \n phase_pk: {phase_pk}, \n submission_pk: {submission_pk}, json_dumps: {json.dumps(results["result"])}')
                     update_finished(
-                        evalai, phase_pk, submission_pk, json.dumps(results["result"])
+                        evalai, phase_pk, submission_pk, json.dumps(results["result"]) # json.dumps(results["results"])
                     )
                 except Exception as e:
+                    print(f"Error: {str(e)}")
                     update_failed(evalai, phase_pk, submission_pk, str(e))
         # Poll challenge queue for new submissions
         time.sleep(60)