diff --git a/.github/workflows/update_leaderboard.yaml b/.github/workflows/update_leaderboard.yaml deleted file mode 100644 index 337b6a9..0000000 --- a/.github/workflows/update_leaderboard.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: Update leaderboard on PR Merge. - -on: - pull_request: - types: [opened, reopened, synchronize] - # pull_request: - # branches: - # - development - # types: - # [closed] - -jobs: - leaderboard_evaluation: - runs-on: ubuntu-latest - steps: - # Check changes in the source branch of the PR - - name: Check if there are any changes in submissions dir - uses: dorny/paths-filter@v3.0.2 - id: changes - with: - filters: | - src: - - 'session_2/challenge/submissions/**' - list-files: "shell" - - - name: Print changed files - run: | - echo '${{ toJSON(steps.changes.outputs) }}' - - - if: ${{ (steps.changes.outputs.src_count > 1) }} - uses: actions/github-script@v7 - with: - script: core.setFailed('More than one submissions are not allowed at once.') - - # Update leaderboard only if single file is changed in submission dir - - if: ${{ (steps.changes.outputs.src == 'true') && (steps.changes.outputs.src_count == 1) }} - name: Checkout code - uses: actions/checkout@v4 - with: - # Here we checkout the target branch of the PR - repository: ${{ github.event.pull_request.base.repo.full_name }} - ref: ${{ github.event.pull_request.base.ref }} - token: ${{ secrets.GH_PAT }} - - - if: ${{ (steps.changes.outputs.src == 'true') && (steps.changes.outputs.src_count == 1) }} - name: Install Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - if: ${{ (steps.changes.outputs.src == 'true') && (steps.changes.outputs.src_count == 1) }} - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r session_2/challenge/requirements.txt - - - if: ${{ (steps.changes.outputs.src == 'true') && (steps.changes.outputs.src_count == 1) }} - name: Run leaderboard update script - id: leaderboard-update - run: | - cd session_2/challenge - filename=$(basename "${{ steps.changes.outputs.src_files }}") - filename_without_extension="${filename%.*}" # Remove extension - python -m scripts.leaderboard --github_user="${{ github.event.pull_request.user.login }}" --prompt="$filename_without_extension" - - - if: ${{ (steps.changes.outputs.src == 'true') && (steps.changes.outputs.src_count == 1) }} - name: Commit changes - uses: EndBug/add-and-commit@v9 - with: - default_author: github_actions \ No newline at end of file diff --git a/.github/workflows/update_leaderboard_after_merge.yaml b/.github/workflows/update_leaderboard_after_merge.yaml new file mode 100644 index 0000000..2f5cbc3 --- /dev/null +++ b/.github/workflows/update_leaderboard_after_merge.yaml @@ -0,0 +1,45 @@ +name: Update File After PR Closed + +on: + push: + branches: + - development +jobs: + update-file: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check if there are any changes in submissions dir + uses: dorny/paths-filter@v3.0.2 + id: changes + with: + # Use context to get the branch where commits were pushed. + # If there is only one long-lived branch (e.g. master), + # you can specify it directly. + # If it's not configured, the repository default branch is used. + base: ${{ github.ref }} + filters: | + src: + - 'session_2/challenge/submissions/**' + list-files: "shell" + + - name: Print changed files + run: | + echo '${{ toJSON(steps.changes.outputs) }}' + + - if: ${{ (steps.changes.outputs.file_count > 1) }} + uses: actions/github-script@v7 + with: + script: core.setFailed('More than one submissions are not allowed at once.') + + # - name: Update file + # if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + # run: | + # git config --global user.name "GitHub Actions" + # git config --global user.email "actions@github.com" + # echo "Updated content" > hello.txt + # git add hello.txt + # git commit -m "Update file after PR merge" + # git push origin HEAD:${{ github.base_ref }} \ No newline at end of file diff --git a/changed_files.txt b/changed_files.txt new file mode 100644 index 0000000..e69de29 diff --git a/session_2/challenge/submissions/baseline_copy_copy.py b/session_2/challenge/submissions/baseline_copy_copy.py new file mode 100644 index 0000000..c8f6b3e --- /dev/null +++ b/session_2/challenge/submissions/baseline_copy_copy.py @@ -0,0 +1,34 @@ +"""Baseline submission for the job description classification challenge.""" + +from scripts import base, registry + + +@registry.register() +class Baseline(base.PromptSubmission): + """Baseline submission.""" + + def build_prompt(self, job_description: str) -> str: + """Builds a prompt for classification of job description.""" + prompt = f""" + + Say "YES" if the given job description is suitable for + a freshers other wise say "NO". + + {job_description}. + + """ + return prompt.strip() + + def parse_response(self, model_response: str) -> bool: + """Parses a response from the LLM to decide the final answer. + + Args: + model_response: Output of the llm for the given prompt. + + Returns: + True is the job_description is for a fresher otherwise False. + """ + model_response = model_response.lower() + if "yes" in model_response: + return True + return False