issue #70: github metrics script

.github/workflows/github-metrics-workflow.yml

@@ -0,0 +1,74 @@
+name: Github metrics workflow
+
+on:
+  workflow_dispatch:
+    inputs:
+      start_date:
+        required: true
+        description: "Enter start date (format: yyyy-mm-dd)"
+        type: string
+      end_date:
+        required: true
+        description: "Enter end date (format: yyyy-mm-dd)"
+        type: string
+      selected_members:
+        required: false
+        description: "Enter selected members (format: Bob,Alice,...,)"
+        default: '*'
+      selected_repositories:
+        required: false
+        description: "Enter selected repositories (format: fertiscan-backend,fertiscan-frontend,...,)"
+        default: '*'
+
+jobs:
+  generate-report:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Generate token from Github application (GH app for workflows)
+        id: generate-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.GH_WORKFLOW_APP_ID }}
+          private-key: ${{ secrets.GH_WORKFLOW_APP_PEM }}
+
+      - name: Checkout repo
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.8
+
+      - name: Install the packAge from github.com/ai-cfia/devops inside the user-site
+        run: >
+          python -m pip install --user \
+          git+https://$DEVOPS_USER:[email protected]/ai-cfia/devops.git@main
+        env:
+          USER: ${{ secrets.DEVOPS_USER }}
+          USER_TOKEN: ${{ secrets.DEVOPS_USER_TOKEN }}
+
+      - name: Access user site-packages
+        run: |
+          USER_SITE=$(python -m site --user-site)
+          echo "Path to site-packages is $USER_SITE"
+          echo "USER_SITE=$USER_SITE" >> $GITHUB_ENV
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Run github metrics script
+        run: python $USER_SITE/github-metrics/github_metrics.py
+        env:
+          GITHUB_ACCESS_TOKEN: ${{ steps.generate-token.outputs.token }}
+          START_DATE: ${{ github.event.inputs.start_date }}
+          END_DATE: ${{ github.event.inputs.end_date }}
+          SELECTED_REPOSITORIES: ${{ github.event.inputs.selected_repositories }}
+
+      - name: Upload PDF artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: github_metrics-${{ github.event.inputs.start_date }}-${{ github.event.inputs.end_date }}
+          path: $USER_SITE/github-metrics/github_metrics-${{ github.event.inputs.start_date }}-${{ github.event.inputs.end_date }}.pdf

github-metrics/export.py

@@ -0,0 +1,124 @@
+import io
+
+from reportlab.lib import colors
+from reportlab.lib.pagesizes import letter
+from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle, PageBreak
+from reportlab.lib.styles import getSampleStyleSheet
+
+def print_results(username, assigned_issues, commits_per_issue, issues_with_linked_pr,
+                  reviews_done, issue_comments, issues_created, prs_created, prs_merged, prs_closed):
+    print("\n--- Statistics ---\n")
+
+    print(f"Assigned issue {username}:")
+    for issue in assigned_issues:
+        print(f"- [{issue.state}] {issue.title} ({issue.html_url})")
+
+    print(f"\nCommits per issue:")
+    for issue_number, count in commits_per_issue.items():
+        print(f"- Issue #{issue_number}: {count} commits")
+
+    print(f"\nIssue linked PR:")
+    for issue, pr in issues_with_linked_pr:
+        print(f"- Issue #{issue.number} linked PR #{pr.number}")
+
+    print(f"\n Reviews made by {username}:")
+    for review in reviews_done:
+        print(f"- PR url [{review.pull_request_url}] : {review.state} ({review.submitted_at})")
+
+    print(f"\nComments per issue:")
+    for comment in issue_comments:
+        print(f"- Issue url [{comment.issue_url}]: {comment.body[:30]}...")
+
+    print(f"\nIssue created by {username}:")
+    for issue in issues_created:
+        print(f"- {issue.title} ({issue.html_url})")
+
+    print(f"\nPRs created by {username}:")
+    for pr in prs_created:
+        print(f"- PR #{pr.number}: {pr.title} ({pr.html_url})")
+
+    print(f"\nPRs merged by {username}:")
+    for pr in prs_merged:
+        print(f"- PR #{pr.number}: {pr.title} ({pr.html_url})")
+
+    print(f"\nPRs closed (not merged) by {username}:")
+    for pr in prs_closed:
+        print(f"- PR #{pr.number}: {pr.title} ({pr.html_url})")
+
+def generate_pdf_for_all_users(users_data, start_date_str, end_date_str):
+    buffer = io.BytesIO()
+    doc = SimpleDocTemplate(buffer, pagesize=letter)
+    styles = getSampleStyleSheet()
+    flowables = []
+
+    for user_data in users_data:
+        USERNAME = user_data['username']
+        assigned_issues = user_data['assigned_issues']
+        commits_per_issue = user_data['commits_per_issue']
+        issues_with_linked_pr = user_data['issues_with_linked_pr']
+        reviews_done = user_data['reviews_done']
+        issue_comments = user_data['issue_comments']
+        issues_created = user_data['issues_created']
+        prs_created = user_data['prs_created']
+        prs_merged = user_data['prs_merged']
+        prs_closed = user_data['prs_closed']
+
+        flowables.append(Paragraph(f"<b>Statistics for {USERNAME}</b>", styles['Title']))
+        flowables.append(Spacer(1, 12))
+
+        flowables.append(Paragraph("Assigned Issues:", styles['Heading2']))
+        for issue in assigned_issues:
+            flowables.append(Paragraph(f"- [{issue.state}] {issue.title} ({issue.html_url})", styles['Normal']))
+
+        flowables.append(Spacer(1, 12))
+        flowables.append(Paragraph("Commits per Issue:", styles['Heading2']))
+        for issue_number, count in commits_per_issue.items():
+            flowables.append(Paragraph(f"- Issue #{issue_number}: {count} commits", styles['Normal']))
+
+        flowables.append(Spacer(1, 12))
+        flowables.append(Paragraph("Issues with Linked PRs:", styles['Heading2']))
+        for issue, pr in issues_with_linked_pr:
+            flowables.append(Paragraph(f"- Issue #{issue.number} linked PR #{pr.number}", styles['Normal']))
+
+        flowables.append(Spacer(1, 12))
+        flowables.append(Paragraph("Reviews Done:", styles['Heading2']))
+        for review in reviews_done:
+            flowables.append(Paragraph(f"- PR url [{review.pull_request_url}] : {review.state} ({review.submitted_at})", styles['Normal']))
+
+        flowables.append(Spacer(1, 12))
+        flowables.append(Paragraph("Issue Comments:", styles['Heading2']))
+        for comment in issue_comments:
+            flowables.append(Paragraph(f"- Issue url [{comment.issue_url}]: {comment.body[:30]}...", styles['Normal']))
+
+        flowables.append(Spacer(1, 12))
+        flowables.append(Paragraph("Issues Created:", styles['Heading2']))
+        for issue in issues_created:
+            flowables.append(Paragraph(f"- {issue.title} ({issue.html_url})", styles['Normal']))
+
+        flowables.append(Spacer(1, 12))
+        flowables.append(Paragraph("PRs Created:", styles['Heading2']))
+        for pr in prs_created:
+            flowables.append(Paragraph(f"- PR #{pr.number}: {pr.title} ({pr.html_url})", styles['Normal']))
+
+        flowables.append(Spacer(1, 12))
+        flowables.append(Paragraph("PRs Merged:", styles['Heading2']))
+        for pr in prs_merged:
+            flowables.append(Paragraph(f"- PR #{pr.number}: {pr.title} ({pr.html_url})", styles['Normal']))
+
+        flowables.append(Spacer(1, 12))
+        flowables.append(Paragraph("PRs Closed (but not merged):", styles['Heading2']))
+        for pr in prs_closed:
+            flowables.append(Paragraph(f"- PR #{pr.number}: {pr.title} ({pr.html_url})", styles['Normal']))
+
+        flowables.append(PageBreak())
+
+    # Build the PDF
+    doc.build(flowables)
+
+    # Write the buffer to a PDF file
+    pdf_filename = f"github_metrics-{start_date_str}-{end_date_str}.pdf"
+    with open(pdf_filename, 'wb') as f:
+        f.write(buffer.getvalue())
+
+    buffer.close()
+    print(f"PDF report generated: {pdf_filename}")

github-metrics/github_metrics.py

@@ -0,0 +1,129 @@
+import os
+import pytz
+
+from github import Github
+from datetime import datetime, timedelta, timezone
+from collections import defaultdict
+from dotenv import load_dotenv
+from concurrent.futures import ThreadPoolExecutor
+
+from export import print_results,generate_pdf_for_all_users
+from utils import get_commits_related_to_issue, get_linked_pr
+
+EST = pytz.timezone('America/Toronto')
+ANY = '*'
+ORGANIZATION_NAME = 'ai-cfia'
+MAX_WORKERS = 10
+
+def collect_user_data(member, repos, start_date, end_date, selected_repository):
+    username = member.login
+    assigned_issues = []
+    commits_per_issue = defaultdict(int)
+    issues_with_linked_pr = []
+    reviews_done = []
+    issue_comments = []
+    issues_created = []
+    prs_created = []
+    prs_closed = []
+    prs_merged = []
+
+    for repo in repos:
+        if repo.name in selected_repository or selected_repository == ANY:
+            for issue in repo.get_issues(assignee=member, since=start_date, state='all'):
+                if issue.created_at <= end_date:
+                    assigned_issues.append(issue)
+                    commits = get_commits_related_to_issue(repo, issue, member, start_date, end_date)
+                    commits_per_issue[issue.number] += len(commits)
+                    linked_pr = get_linked_pr(repo, issue)
+                    if linked_pr:
+                        issues_with_linked_pr.append((issue, linked_pr))
+
+            for pr in repo.get_pulls(state='all'):
+                reviews = pr.get_reviews()
+                for review in reviews:
+                    if review.user == member and review.submitted_at is not None and start_date <= review.submitted_at <= end_date:
+                        reviews_done.append(review)
+
+            for issue in repo.get_issues(state='all', since=start_date):
+                comments = issue.get_comments()
+                for comment in comments:
+                    if comment.user == member and start_date <= comment.created_at <= end_date:
+                        issue_comments.append(comment)
+
+            for issue in repo.get_issues(creator=member.login, since=start_date, state='all'):
+                if issue.created_at <= end_date:
+                    issues_created.append(issue)
+
+            for pr in repo.get_pulls(state='all'):
+                if pr.user == member and start_date <= pr.created_at <= end_date:
+                    prs_created.append(pr)
+                if pr.merged and start_date <= pr.merged_at <= end_date:
+                    prs_merged.append(pr)
+                if pr.closed_at and not pr.merged and start_date <= pr.closed_at <= end_date:
+                    prs_closed.append(pr)
+
+            print(f"=== repo {repo.name} done for {username}")
+
+    return {
+        'username': username,
+        'assigned_issues': assigned_issues,
+        'commits_per_issue': commits_per_issue,
+        'issues_with_linked_pr': issues_with_linked_pr,
+        'reviews_done': reviews_done,
+        'issue_comments': issue_comments,
+        'issues_created': issues_created,
+        'prs_created': prs_created,
+        'prs_merged': prs_merged,
+        'prs_closed': prs_closed
+    }
+
+def main(gh_access_token, start_date_str, end_date_str, selected_repository, selected_members):
+    g = Github(gh_access_token)
+    org = g.get_organization(ORGANIZATION_NAME)
+    repos = org.get_repos()
+    members = org.get_members()
+    start_date = EST.localize(datetime.strptime(start_date_str, '%Y-%m-%d'))
+    end_date = EST.localize(datetime.strptime(end_date_str, '%Y-%m-%d'))
+    users_data = []
+
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+        futures = []
+        for member in members:
+            if member.login in selected_members or selected_members == ANY:
+                print(f"fetching github metrics for {member.login}")
+                futures.append(executor.submit(collect_user_data, member, repos, start_date, end_date, selected_repository))
+
+        for future in futures:
+            user_data = future.result()
+            users_data.append(user_data)
+            print_results(user_data['username'], user_data['assigned_issues'], user_data['commits_per_issue'], 
+                          user_data['issues_with_linked_pr'], user_data['reviews_done'], user_data['issue_comments'], 
+                          user_data['issues_created'], user_data['prs_created'], user_data['prs_merged'], user_data['prs_closed'])
+
+    generate_pdf_for_all_users(users_data, start_date_str, end_date_str)
+
+if __name__ == "__main__":
+    load_dotenv()
+
+    gh_access_token = os.getenv("GITHUB_ACCESS_TOKEN")
+
+    # e.g 2024-12-01 (yyyy-MM-dd)
+    start_date_str = os.getenv("START_DATE")
+    end_date_str = os.getenv("END_DATE")
+
+    # from https://github.com/ai-cfia/github-workflows/blob/main/repo_project.txt
+    # e.g fertiscan-backend,fertiscan-frontend,fertiscan-pipeline,nachet-backend,nachet-frontend,nachet-model,howard,github-workflows,devops,finesse-backend,finesse-frontend,finesse-data
+    selected_repositories = os.getenv("SELECTED_REPOSITORIES")
+    if not selected_repositories:
+        repos = ANY
+    else:
+        repos = selected_repositories.split(',')
+
+    # e.g Bob,Alice,...
+    selected_members = os.getenv("SELECTED_MEMBERS")
+    if not selected_members:
+        members = ANY
+    else:
+        members = selected_members.split(',')
+
+    main(gh_access_token, start_date_str, end_date_str, repos, members)

github-metrics/requirements.txt

@@ -0,0 +1,4 @@
+PyGithub
+python-dotenv
+pytz
+reportlab

github-metrics/utils.py

@@ -0,0 +1,13 @@
+
+def get_commits_related_to_issue(repo, issue, user, start_date, end_date):
+    commits = []
+    for commit in repo.get_commits(author=user, since=start_date, until=end_date):
+        if f"#{issue.number}" in commit.commit.message:
+            commits.append(commit)
+    return commits
+
+def get_linked_pr(repo, issue):
+    for pr in repo.get_pulls(state='all'):
+        if (f"#{issue.number}" in (pr.body or '')) or (f"#{issue.number}" in (pr.title or '')):
+            return pr
+    return None

setup.py

@@ -8,6 +8,7 @@
         'console_scripts': [
             'remove-previous-images=remove_previous_image.remove_previous_image:main',
             'webtop-template=webtop_template.webtop_template:main',
+            'github-metrics=github_metrics.github_metrics:main',
         ],
     },
     url='https://github.com/ai-cfia/devops.git',
@@ -20,6 +21,8 @@
         'requests',
         'jinja2',
         'PyGithub',
-        'python-dotenv'
+        'python-dotenv',
+        'pytz',
+        'reportlab'
     ],
 )