Fetch name from commit and use in lb

infocusp · Apr 21, 2024 · d439c77 · d439c77
1 parent 0f6aeab
commit d439c77
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 31 deletions.
diff --git a/.github/workflows/update_leaderboard_after_merge.yaml b/.github/workflows/update_leaderboard_after_merge.yaml
@@ -34,6 +34,7 @@ jobs:
         with:
           script: core.setFailed('More than one submissions are not allowed at once.')
 
+      # TODO(hetul): Get github profile from email https://api.github.com/search/users?q=EMAIL
       - if: ${{ (steps.changes.outputs.src == 'true') && (steps.changes.outputs.src_count == 1) }}
         name: Get author's name from last commit
         id: author_name
@@ -62,8 +63,8 @@ jobs:
           filename_without_extension="${filename%.*}"  # Remove extension          
           python -m scripts.leaderboard --github_user="${{ steps.author_name.outputs.author_name }}" --prompt="$filename_without_extension"
 
-      # - if: ${{ (steps.changes.outputs.src == 'true') && (steps.changes.outputs.src_count == 1) }}
-      #   name: Commit changes
-      #   uses: EndBug/add-and-commit@v9
-      #   with:
-      #     default_author: github_actions          
+      - if: ${{ (steps.changes.outputs.src == 'true') && (steps.changes.outputs.src_count == 1) }}
+        name: Commit changes
+        uses: EndBug/add-and-commit@v9
+        with:
+          default_author: github_actions          
diff --git a/session_2/challenge/leaderboard.md b/session_2/challenge/leaderboard.md
@@ -12,15 +12,10 @@ Check [participation guide](how_to_participate.md).
 <center>
 
 <!-- leader-board-begins -->
-|   Rank | Profile Image                                                                                   | GitHub Username                            | Solution                                                                                                                                 |   Accuracy % |
-|-------:|:------------------------------------------------------------------------------------------------|:-------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------|-------------:|
-|      1 | <img src="https://github.com/.png" width="50px" height="50px" class="profile-image">            | [](https://github.com/)                    | [baseline](https://github.com/infocusp/llm_seminar_series/blob/main/session_2/challenge/submissions/baseline.py)                         |        100   |
-|      2 | <img src="https://github.com/hetulvp.png" width="50px" height="50px" class="profile-image">     | [hetulvp](https://github.com/hetulvp)      | [baseline_copy](https://github.com/infocusp/llm_seminar_series/blob/main/session_2/challenge/submissions/baseline_copy.py)               |        100   |
-|      3 | <img src="https://github.com/hetul-patel.png" width="50px" height="50px" class="profile-image"> | [New User](https://github.com/new_user)    | [New Solution](https://github.com/new_solution)                                                                                          |         99.5 |
-|      4 | <img src="https://github.com/hetul-patel.png" width="50px" height="50px" class="profile-image"> | [Username 2](https://github.com/username2) | [Baseline](https://github.com/infocusp/llm_seminar_series/blob/hetul/prompting-leader-board/session_2/challenge/submissions/baseline.py) |         95   |
-|      5 | <img src="https://github.com/hetul-patel.png" width="50px" height="50px" class="profile-image"> | [Username 4](https://github.com/username4) | [Baseline](https://github.com/infocusp/llm_seminar_series/blob/hetul/prompting-leader-board/session_2/challenge/submissions/baseline.py) |         95   |
-|      6 | <img src="https://github.com/hetul-patel.png" width="50px" height="50px" class="profile-image"> | [Username 3](https://github.com/username3) | [Baseline](https://github.com/infocusp/llm_seminar_series/blob/hetul/prompting-leader-board/session_2/challenge/submissions/baseline.py) |         10   |
-|      7 | <img src="https://github.com/hetul-patel.png" width="50px" height="50px" class="profile-image"> | [Username 1](https://github.com/username1) | [Baseline](https://github.com/infocusp/llm_seminar_series/blob/hetul/prompting-leader-board/session_2/challenge/submissions/baseline.py) |          0   |
+|   Rank | Name   | Solution                                                                                                         |   Accuracy % |
+|-------:|:-------|:-----------------------------------------------------------------------------------------------------------------|-------------:|
+|      1 | XYZ    | [baseline](https://github.com/infocusp/llm_seminar_series/blob/main/session_2/challenge/submissions/baseline.py) |          100 |
+|      2 | Hetul  | [baseline](https://github.com/infocusp/llm_seminar_series/blob/main/session_2/challenge/submissions/baseline.py) |           50 |
 <!-- leader-board-ends -->
 
 </center>
diff --git a/session_2/challenge/scripts/leaderboard.py b/session_2/challenge/scripts/leaderboard.py
@@ -2,7 +2,7 @@
 
 Sample command:
 python -m scripts.leaderboard \
-    --github_user=your_github_user \
+    --github_name=your_github_user \
     --prompt=baseline
 """
 
@@ -18,15 +18,15 @@
     "prompt", None, "Name of the submitted prompt to evaluate."
 )
 
-_GITHUB_USER = flags.DEFINE_string(
-    "github_user", None, "Github username to add an entry in leaderboard."
+_GITHUB_NAME = flags.DEFINE_string(
+    "github_name", None, "Github name to add an entry in leaderboard."
 )
 
 
 _LEADERBORAD = "leaderboard.md"  # current leaderboard
 
 
-def generate_leaderboard(prompt_name: str, accuracy: float, github_user: str):
+def generate_leaderboard(prompt_name: str, accuracy: float, github_name: str):
     """Generates leaderboard."""
     # Read the markdown table into a DataFrame
     with open(_LEADERBORAD, "r") as file:
@@ -39,16 +39,15 @@ def generate_leaderboard(prompt_name: str, accuracy: float, github_user: str):
 
     # Extract rows using regex
     rows = re.findall(
-        r"\|([^|]+)\|([^|]+)\|([^|]+)\|([^|]+)\|([^|]+)\|", table_content
+        r"\|([^|]+)\|([^|]+)\|([^|]+)\|([^|]+)\|", table_content
     )[2:]
 
     # Create a DataFrame from the extracted rows
     df = pd.DataFrame(
         rows,
         columns=[
             "Rank",
-            "Profile Image",
-            "GitHub Username",
+            "Name",
             "Solution",
             "Accuracy %",
         ],
@@ -65,17 +64,15 @@ def generate_leaderboard(prompt_name: str, accuracy: float, github_user: str):
     repo_url = "https://github.com/infocusp/llm_seminar_series/blob/main/session_2/challenge/submissions"
     new_entry = {
         "Rank": len(df) + 1,
-        "Profile Image": f'<img src="https://github.com/{github_user}.png" '
-        + 'width="50px" height="50px" class="profile-image">',
-        "GitHub Username": f"[{github_user}](https://github.com/{github_user})",
+        "Name": github_name,
         "Solution": f"[{prompt_name}]({repo_url}/{prompt_name}.py)",
         "Accuracy %": accuracy,
     }
 
     df = pd.concat([df, pd.DataFrame([new_entry])], ignore_index=True)
 
     # Keep only the highest submission for each user
-    highest_indices = df.groupby("GitHub Username")["Accuracy %"].idxmax()
+    highest_indices = df.groupby("Name")["Accuracy %"].idxmax()
     df_highest = df.loc[highest_indices]
 
     # Sort the DataFrame by "Accuracy %" column in descending order
@@ -102,19 +99,19 @@ def generate_leaderboard(prompt_name: str, accuracy: float, github_user: str):
 
     logging.info(
         "Submission by %s with prompt %s updated in the leaderboard.",
-        github_user,
+        github_name,
         prompt_name,
     )
 
 
-def update_leaderboard(prompt_name: str, github_user: str):
+def update_leaderboard(prompt_name: str, github_name: str):
     """Generates a public leaderboard by evaluating given submission."""
     sample_dataset = dataset.load_dataset_from_dir(samples_dir="dataset")
     acc = evaluate_lib.evaluate(
         dataset=sample_dataset, prompt_name=prompt_name
     )
     generate_leaderboard(
-        prompt_name=prompt_name, accuracy=acc, github_user=github_user
+        prompt_name=prompt_name, accuracy=acc, github_name=github_name
     )
 
 
@@ -124,11 +121,11 @@ def main(argv: Sequence[str]) -> None:
         raise app.UsageError("Too many command-line arguments.")
     logging.getLogger().setLevel(logging.INFO)
     update_leaderboard(
-        prompt_name=_PROMPT.value, github_user=_GITHUB_USER.value
+        prompt_name=_PROMPT.value, github_name=_GITHUB_NAME.value
     )
 
 
 if __name__ == "__main__":
     flags.mark_flag_as_required("prompt")
-    flags.mark_flag_as_required("github_user")
+    flags.mark_flag_as_required("github_name")
     app.run(main)
diff --git a/session_2/challenge/submissions/baseline.py b/session_2/challenge/submissions/baseline.py
@@ -14,7 +14,7 @@ def build_prompt(self, job_description: str) -> str:
         Say "YES" if the given job description is suitable for
         a freshers other wise say "NO".
 
-        {job_description}.
+        {job_description}
         
         """
         return prompt.strip()