diff --git a/.autorelease/autorelease.yml b/.autorelease/autorelease.yml index 4d35090..9732e95 100644 --- a/.autorelease/autorelease.yml +++ b/.autorelease/autorelease.yml @@ -12,6 +12,7 @@ repo: release-branches: - stable release-tag: "v{BASE_VERSION}" + dev-branch: main # writing release notes notes: @@ -22,6 +23,9 @@ notes: heading: Bugs fixed - label: misc PR heading: Miscellaneous improvements + topics: + - label: docs + name: Improvements to documentation standard_contributors: - dwhswenson diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 3fbf0d2..6055aed 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -27,7 +27,6 @@ jobs: - "3.10" - "3.9" - "3.8" - - "3.7" steps: - uses: actions/checkout@v2 diff --git a/autorelease-travis.yml b/autorelease-travis.yml index c9c28ba..8755571 100644 --- a/autorelease-travis.yml +++ b/autorelease-travis.yml @@ -1,4 +1,4 @@ -# AUTORELEASE v0.4.2 +# AUTORELEASE v0.5.0 # for nonrelease, use @main # for release, use v${VERSION}, e.g., v1.0.0 stages: @@ -9,7 +9,7 @@ stages: - deploy pypi import: - - dwhswenson/autorelease:travis_stages/deploy_testpypi.yml@v0.4.2 - - dwhswenson/autorelease:travis_stages/test_testpypi.yml@v0.4.2 - - dwhswenson/autorelease:travis_stages/cut_release.yml@v0.4.2 - - dwhswenson/autorelease:travis_stages/deploy_pypi.yml@v0.4.2 + - dwhswenson/autorelease:travis_stages/deploy_testpypi.yml@0.5.0 + - dwhswenson/autorelease:travis_stages/test_testpypi.yml@0.5.0 + - dwhswenson/autorelease:travis_stages/cut_release.yml@0.5.0 + - dwhswenson/autorelease:travis_stages/deploy_pypi.yml@0.5.0 diff --git a/autorelease/gh_api4/__init__.py b/autorelease/gh_api4/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/autorelease/gh_api4/notes4.py b/autorelease/gh_api4/notes4.py new file mode 100644 index 0000000..e559eeb --- /dev/null +++ b/autorelease/gh_api4/notes4.py @@ -0,0 +1,137 @@ +""" +Writing release notes based on the GitHub v4 (GraphQL) API +""" + +import typing +import enum +import datetime + +from .pull_requests import PRStatus, PR, graphql_get_all_prs +from .releases import latest_release + +import logging +_logger = logging.getLogger(__name__) + +def filter_release_prs(all_prs, prev_release_date, target_branch="main"): + def is_release_pr(pr): + return ( + pr.status == PRStatus.MERGED + and pr.merge_time > prev_release_date + and pr.target == target_branch + ) + + for pr in all_prs: + _logger.info(f"{pr}") + if is_release_pr(pr): + _logger.info(f"Including {pr}") + yield pr + else: + _logger.info("Skipping") + + +def prs_since_latest_release(owner, repo, auth, target_branch="main"): + latest = latest_release(owner, repo, auth) + _logger.info(f"Latest release: {latest}") + + release_date = latest.date + all_prs = [PR.from_api_response(pr) + for pr in graphql_get_all_prs(owner, repo, auth, + target_branch)] + + _logger.info(f"Loaded {len(all_prs)} PRs") + new_prs = list(filter_release_prs( + all_prs=all_prs, + prev_release_date=latest.date, + target_branch=target_branch + )) + _logger.info(f"After filtering, found {len(new_prs)} new PRs") + return new_prs + + +class PRCategory: + def __init__(self, label, heading, topics): + self.label = label + self.heading = heading + self.topics = topics + self.prs = [] + self.topic_prs = {l: [] for l in topics} + + def append(self, pr): + if topics := set(pr.labels) & set(self.topics): + for topic in topics: + self.topic_prs[topic].append(pr) + else: + self.prs.append(pr) + + +class NotesWriter: + def __init__(self, category_labels, topics, standard_contributors): + self.category_labels = category_labels + self.topics = topics + self.standard_contributors = set(standard_contributors) + + @staticmethod + def assign_prs_to_categories(prs, categories): + category_labels = set(categories) + for pr in prs: + selected = [categories[label] + for label in set(pr.labels) & category_labels] + + if not selected: + selected = [categories[None]] + + for category in selected: + category.append(pr) + + def _write_pr_details(self, pr, category_label, topic_label): + out = f"[#{pr.number}]({pr.url})" + if pr.author not in self.standard_contributors: + out += f" @{pr.author}" + + out_labels = [label for label in pr.labels + if label not in {category_label, topic_label}] + if out_labels: + out += " " + out += " ".join(f"#{label}" for label in out_labels) + return out + + def write_single_pr(self, pr, category_label): + details = self._write_pr_details(pr, category_label, "") + out = f"* {pr.title} ({details})\n" + return out + + def write_topic(self, category, topic): + out = "" + topic_prs = category.topic_prs[topic] + topic_text = category.topics[topic] + if len(topic_prs): + out += f"* {topic_text} (" + out += ", ".join( + self._write_pr_details(pr, category.label, topic) + for pr in topic_prs + ) + out += ")\n" + + return out + + def write_category(self, category): + out = f"## {category.heading}\n\n" + for pr in category.prs: + out += self.write_single_pr(pr, category.label) + + for topic in category.topics: + out += self.write_topic(category, topic) + + return out + + def write(self, prs): + categories = { + label: PRCategory(label, heading, self.topics.get(label, {})) + for label, heading in self.category_labels.items() + } + categories[None] = PRCategory(None, "Unlabeled PRs", {}) + self.assign_prs_to_categories(prs, categories) + + out = "\n".join(self.write_category(category) + for category in categories.values()) + return out diff --git a/autorelease/gh_api4/pull_requests.py b/autorelease/gh_api4/pull_requests.py new file mode 100644 index 0000000..81d58f8 --- /dev/null +++ b/autorelease/gh_api4/pull_requests.py @@ -0,0 +1,117 @@ +from .query_runner import QueryRunner + +import typing +import enum +import datetime + +from .utils import string_to_datetime + + + +class PRStatus(enum.Enum): + OPEN = "open" + CLOSED = "closed" + MERGED = "merged" + + +class PR(typing.NamedTuple): + number: int + target: str + title: str + status: PRStatus + author: str + labels: typing.List[str] + url: str + merge_time: typing.Optional[datetime.datetime] + + @classmethod + def from_api_response(cls, api_pr): + return cls( + number=int(api_pr["number"]), + target=api_pr["baseRefName"], + title=api_pr["title"], + status=getattr(PRStatus, api_pr["state"]), + author=api_pr["author"]["login"], + labels=[node["name"] for node in api_pr["labels"]["nodes"]], + url=api_pr["url"], + merge_time=string_to_datetime(api_pr["mergedAt"]), + ) + +PR_QUERY = """ +{ + repository(name: "$repo_name", owner: "$repo_owner") { + pullRequests( + orderBy: {field: UPDATED_AT, direction: DESC} + first: 100 + $after + states: MERGED + baseRefName: "$target_branch" + ) { + nodes { + author { + login + } + merged + mergedAt + number + title + headRefName + closed + baseRefName + state + url + labels(first: 100) { + nodes { + name + } + pageInfo { + endCursor + hasNextPage + startCursor + } + } + } + pageInfo { + startCursor + endCursor + hasNextPage + hasPreviousPage + } + } + } +} +""" + +def graphql_get_all_prs(owner, repo, auth, target_branch): + # TODO: query needs to take repo and owner + def extractor(result): + return result["data"]["repository"]["pullRequests"]["nodes"] + + def next_page_cursor(result): + info = result["data"]["repository"]["pullRequests"]["pageInfo"] + next_cursor = info["endCursor"] if info["hasNextPage"] else None + return next_cursor + + + query_runner = QueryRunner(PR_QUERY, auth=auth, + api_endpoint="https://api.github.com/graphql") + extracted_results = [] + + # TODO: how to manage nested inner loops? + # actually.. better choice is to post-process to remove inner + # paginations: get additional labels for anything with more than 100 + # labels + default_kwargs = { + 'repo_owner': owner, + 'repo_name': repo, + 'target_branch': target_branch, + } + result = query_runner(after="", **default_kwargs) + extracted_results.extend(extractor(result)) + while cursor := next_page_cursor(result): + result = query_runner(after=f'after: "{cursor}"', **default_kwargs) + extracted_results.extend(extractor(result)) + + return extracted_results + + diff --git a/autorelease/gh_api4/query_runner.py b/autorelease/gh_api4/query_runner.py new file mode 100644 index 0000000..7239409 --- /dev/null +++ b/autorelease/gh_api4/query_runner.py @@ -0,0 +1,16 @@ +import requests +import string + +GITHUB_API_ENDPOINT = "https://api.github.com/graphql" + +class QueryRunner: + def __init__(self, query_template, auth, + api_endpoint=GITHUB_API_ENDPOINT): + self.query_template = string.Template(query_template) + self.auth = auth + self.api_endpoint = api_endpoint + + def __call__(self, **kwargs): + query = self.query_template.substitute(**kwargs) + return requests.post(self.api_endpoint, json={'query': query}, + auth=self.auth).json() diff --git a/autorelease/gh_api4/releases.py b/autorelease/gh_api4/releases.py new file mode 100644 index 0000000..7609f25 --- /dev/null +++ b/autorelease/gh_api4/releases.py @@ -0,0 +1,50 @@ +from typing import NamedTuple +from .query_runner import QueryRunner +from .utils import string_to_datetime +import datetime + +class Release(NamedTuple): + name: str + tag: str + draft: bool + prerelease: bool + latest: bool + date: datetime.datetime + + @classmethod + def from_api(cls, api_release): + return cls( + name=api_release['name'], + tag=api_release["tagName"], + draft=api_release["isDraft"], + prerelease=api_release["isPrerelease"], + latest=api_release["isLatest"], + date=string_to_datetime(api_release["publishedAt"]), + ) + +RELEASES_QUERY = """ +{ + repository(name: "$repo_name", owner: "$repo_owner") { + releases(orderBy: {field: CREATED_AT, direction: DESC}, first: 100) { + nodes { + publishedAt + isLatest + isPrerelease + isDraft + name + tagName + } + } + } +} +""" + +def latest_release(owner, repo, auth): + # TODO: support paginated releases + runner = QueryRunner(RELEASES_QUERY, auth) + result = runner(repo_name=repo, repo_owner=owner) + api_release_info = result['data']['repository']['releases']['nodes'] + releases = [Release.from_api(rel) for rel in api_release_info] + claim_latest = [rel for rel in releases if rel.latest] + assert len(claim_latest) == 1 + return claim_latest[0] diff --git a/autorelease/gh_api4/utils.py b/autorelease/gh_api4/utils.py new file mode 100644 index 0000000..1b18715 --- /dev/null +++ b/autorelease/gh_api4/utils.py @@ -0,0 +1,5 @@ +import datetime +def string_to_datetime(string): + # TODO: move this elsewhere + return datetime.datetime.strptime(string, "%Y-%m-%dT%H:%M:%SZ") + diff --git a/autorelease/release_notes.py b/autorelease/release_notes.py index 2726432..0e67802 100644 --- a/autorelease/release_notes.py +++ b/autorelease/release_notes.py @@ -18,6 +18,7 @@ def __init__(self, config, since_release=None, project=None, self.config = config self.since_release = since_release + import pdb; pdb.set_trace() project = self._apply_config_key(project, 'project', ProjectOptions) github_user = self._apply_config_key(github_user, 'github_user', GitHubUser) diff --git a/autorelease/scripts/cli.py b/autorelease/scripts/cli.py index 70db92c..faf63bf 100644 --- a/autorelease/scripts/cli.py +++ b/autorelease/scripts/cli.py @@ -4,7 +4,10 @@ from autorelease.scripts.vendor import vendor_actions from autorelease.scripts.check import run_checks -from autorelease import ReleaseNoteWriter +# from autorelease import ReleaseNoteWriter +from autorelease.gh_api4.notes4 import NotesWriter, prs_since_latest_release + +import logging def _find_first_file(pathlist): @@ -46,8 +49,11 @@ def load_auth(user_input): @click.group() -def cli(): - pass +@click.option("--loglevel", type=str, default="WARNING") +def cli(loglevel): + logging.basicConfig(level=getattr(logging, loglevel)) + logger = logging.getLogger("autorelease") + logger.setLevel(getattr(logging, loglevel)) @cli.command() @click.option('--conf', type=click.File('r')) @@ -64,20 +70,45 @@ def config(conf): config = load_config(conf) pprint(config) +@cli.command() +@click.option('--auth', type=click.File('r')) +def auth(auth): + from pprint import pprint + auth = load_auth(auth) + pprint(auth) + @cli.command() @click.option('--conf', type=click.File('r')) @click.option('--auth', type=click.File('r')) -@click.option('--since-release', type=str, default=None) -@click.option('-o', '--output', type=str) -def notes(conf, auth, since_release, output): +def notes(conf, auth): config = load_config(conf) github_user = load_auth(auth) + target_branch = config['repo'].get('dev-branch', 'main') notes_conf = config['notes'] notes_conf.update(github_user) notes_conf['project'] = config['project'] - writer = ReleaseNoteWriter(config=notes_conf) - writer.write_release_notes(outfile=output) + category_labels = { + lab['label']: lab['heading'] for lab in notes_conf['labels'] + } + topics = {} + for label in notes_conf['labels']: + if tops := label.get('topics'): + topic_dict = {top['label']: top['name'] for top in tops} + topics[label['label']] = topic_dict + + writer = NotesWriter( + category_labels=category_labels, + topics=topics, + standard_contributors=notes_conf['standard_contributors'] + ) + new_prs = prs_since_latest_release( + owner=config['project']['repo_owner'], + repo=config['project']['repo_name'], + auth=(None, github_user['github_user']['token']), + target_branch=target_branch, + ) + print(writer.write(new_prs)) @click.group() def vendor(): diff --git a/autorelease/scripts/write_release_notes.py b/autorelease/scripts/write_release_notes.py index 7768159..c611276 100644 --- a/autorelease/scripts/write_release_notes.py +++ b/autorelease/scripts/write_release_notes.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import argparse +import yaml from autorelease import ReleaseNoteWriter, AutoreleaseParsingHelper +from autorelease.gh_api4.notes4 import NotesWriter, prs_since_latest_release def make_parser(): parser = argparse.ArgumentParser() @@ -8,13 +10,46 @@ def make_parser(): auto_parser.add_github_parsing() auto_parser.add_project_parsing() auto_parser.parser.add_argument("--conf", type=str) - auto_parser.parser.add_argument("-o", "--output", type=str) - auto_parser.parser.add_argument("--since-release", type=str) + # auto_parser.parser.add_argument("-o", "--output", type=str) + # auto_parser.parser.add_argument("--since-release", type=str) return auto_parser def main(): auto_parser = make_parser() opts = auto_parser.parse_args() + with open(opts.conf, mode='r') as f: + config = yaml.load(f.read(), Loader=yaml.SafeLoader) + + + token = None + if auto_parser.github_user is not None: + token = auto_parser.github_user.token + elif tok : + + notes_conf = config['notes'] + category_labels = { + lab['label']: lab['heading'] for lab in notes_conf['labels'] + } + topics = {} + for label in notes_conf['labels']: + if tops := label.get('topics'): + topic_dict = {top['label']: top['name'] for top in tops} + topics[label['label']] = topic_dict + + writer = NotesWriter( + category_labels=category_labels, + topics=topics, + standard_contributors=notes_conf['standard_contributors'] + ) + import pdb; pdb.set_trace() + + new_prs = prs_since_latest_release( + owner=config['project']['repo_owner'], + repo=config['project']['repo_name'], + auth=opts.github_user + ) + print(writer_write(new_prs)) + writer = ReleaseNoteWriter(config=opts.conf, since_release=opts.since_release, project=opts.project, diff --git a/devtools/conda-recipe/meta.yaml b/devtools/conda-recipe/meta.yaml index 6b12656..c9d7ce8 100644 --- a/devtools/conda-recipe/meta.yaml +++ b/devtools/conda-recipe/meta.yaml @@ -1,7 +1,7 @@ package: name: autorelease # add ".dev0" for unreleased versions - version: "0.4.2" + version: "0.5.0" source: path: ../../ diff --git a/setup.cfg b/setup.cfg index 4a85bf7..82acc84 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = autorelease -version = 0.4.2 +version = 0.5.0 # version should end in .dev0 if this isn't to be released short_description = Tools to keep the release process clean. description = Tools to keep the release process clean.