test_scraper.yml

name: Run Scraper Daily

# Set the schedule to run every 24 hours
# on:
#   schedule:
#     - cron: "0 0 * * *" # This runs at midnight UTC every day

#testing
# Set the schedule to run every 5 minutes
on:
  push:
      branches:
        - scraper-tecq
  # schedule:
  #   - cron: "*/5 * * * *" # This runs every 5 minutes
  workflow_dispatch: # This allows you to run the workflow manually

jobs:
  run-scraper:
    runs-on: ubuntu-latest
    
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.13'  # Use your required Python version

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
      - name: Setup Chrome
        uses: browser-actions/setup-chrome@v1.7.2
        with:
          chrome-version: 120
          install-chromedriver: true
      - name: Run Scraper
        env:
          CHROME_BIN: "/usr/bin/google-chrome"  # Optional, in case your script needs the Chrome path
          CHROMEDRIVER_PATH: "/opt/hostedtoolcache/setup-chrome/chromedriver/120.0.6099.109/x64"
        run: |
          python main.py
        
      - name: Save Excel Output
        # Create a file name with the date and time
        run: |
          timestamp=$(date +"%Y-%m-%d_%H-%M-%S")
          mv output.xlsx "output_$timestamp.xlsx"
          mv "output_$timestamp.xlsx" results/
        
      - name: Commit and Push the Excel file
        run: |
          git config --local user.email "nn36@rice.edu"
          git config --local user.name "GitHub Actions"
          git add results/output_*.xlsx
          git commit -m "Add data for $timestamp"
          git push --force
        env:
          # GitHub Token for permissions to push
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

# make sure to push to scraper-tecq branch