From 55cd0cba5605ee270f850feec052d2f9897d23b2 Mon Sep 17 00:00:00 2001 From: Gabriel Bustamante Date: Wed, 15 May 2024 13:10:09 -0500 Subject: [PATCH] Query the whattrainisitnow release schedule API instead of hard-coding scheduled release dates (#1464) * Query the whattrainisitnow release schedule API instead of hard-coding scheduled release dates * Import datetime names to decrease verbosity in product_details module * https://bugzilla.mozilla.org/show_bug.cgi?id=1890753 --- api/src/shipit_api/admin/product_details.py | 117 +++++++++++++++++--- api/src/shipit_api/common/config.py | 15 --- 2 files changed, 100 insertions(+), 32 deletions(-) diff --git a/api/src/shipit_api/admin/product_details.py b/api/src/shipit_api/admin/product_details.py index a9eaff669..023194778 100644 --- a/api/src/shipit_api/admin/product_details.py +++ b/api/src/shipit_api/admin/product_details.py @@ -5,7 +5,6 @@ import asyncio import collections -import datetime import functools import hashlib import io @@ -17,6 +16,7 @@ import shutil import typing import urllib.parse +from datetime import datetime, timedelta, timezone import aiohttp import arrow @@ -25,6 +25,7 @@ import mypy_extensions import sqlalchemy import sqlalchemy.orm +from mozilla_version.gecko import FirefoxVersion from mozilla_version.mobile import MobileVersion import cli_common.command @@ -126,14 +127,37 @@ def with_default(a: typing.Optional[A], func: typing.Callable[[A], B], default: return func(a) -def to_isoformat(d: datetime.datetime) -> str: +def to_isoformat(d: datetime) -> str: return arrow.get(d).isoformat() -def to_format(d: datetime.datetime, format: str) -> str: +def from_isoformat(s: str) -> datetime: + return datetime.fromisoformat(s) + + +def to_format(d: datetime, format: str) -> str: return arrow.get(d).format(format) +def from_format(s: str, format: str) -> datetime: + return datetime.strptime(s, format) + + +YMD_DATE_FORMAT = "%Y-%m-%d" + + +def iso_to_ymd(s: str) -> str: + return from_isoformat(s).strftime(YMD_DATE_FORMAT) + + +def dt_to_ymd(d: datetime) -> str: + return d.strftime(YMD_DATE_FORMAT) + + +def from_ymd_format(s: str) -> datetime: + return from_format(s, YMD_DATE_FORMAT) + + def create_index_listing_html(folder: pathlib.Path, items: typing.Set[pathlib.Path]) -> str: folder = "/" / folder # noqa : T484 Unsupported left operand type for / ("str") with io.StringIO() as html: @@ -495,7 +519,7 @@ def get_release_history( return ordered_history -def get_primary_builds( +async def get_primary_builds( breakpoint_version: int, product: Product, releases: typing.List[shipit_api.common.models.Release], @@ -531,7 +555,7 @@ def get_primary_builds( """ if product is Product.FIREFOX: - firefox_versions = get_firefox_versions(releases) + firefox_versions = await get_firefox_versions(releases) # make sure that Devedition is included in the list products = [Product.FIREFOX, Product.DEVEDITION] versions = set( @@ -624,7 +648,62 @@ def get_firefox_esr_next_version(releases: typing.List[shipit_api.common.models. return get_latest_version(releases, product, branch) -def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release]) -> FirefoxVersions: +@backoff.on_exception(backoff.expo, (aiohttp.ClientError, asyncio.TimeoutError), max_time=60) +async def fetch_firefox_release_schedule_data(releases: typing.List[shipit_api.common.models.Release], session: aiohttp.ClientSession): + firefox_nightly_mozilla_version = FirefoxVersion.parse(shipit_api.common.config.FIREFOX_NIGHTLY) + current_nightly_version_major_number = firefox_nightly_mozilla_version.major_number + previous_nightly_version_major_number = current_nightly_version_major_number - 1 + url_template = "https://whattrainisitnow.com/api/release/schedule/?version={version}" + try: + url = url_template.format(version=current_nightly_version_major_number) + async with session.get(url) as response: + response.raise_for_status() + logger.debug(f"Fetched {url}") + current_nightly_version_schedule = await response.json() + url = url_template.format(version=previous_nightly_version_major_number) + async with session.get(url) as response: + response.raise_for_status() + logger.debug(f"Fetched {url}") + previous_nightly_version_schedule = await response.json() + except Exception: + logger.info("Failed to fetch %s", url) + raise + last_softfreeze_date = iso_to_ymd(previous_nightly_version_schedule["soft_code_freeze"]) + last_merge_date = iso_to_ymd(previous_nightly_version_schedule["merge_day"]) + releases_after_last_merge_date = sorted( + [ + release + for release in releases + if release.product == Product.FIREFOX.value + and FirefoxVersion.parse(release.version).is_release + and release.status == "shipped" + and release.completed is not None + and release.completed.replace(tzinfo=timezone.utc) > from_isoformat(previous_nightly_version_schedule["merge_day"]) + ], + key=lambda release: FirefoxVersion.parse(release.version), + ) + if not releases_after_last_merge_date: + raise ValueError(f"No Firefox releases shipped after the last merge date ({last_merge_date})") + first_release_after_last_merge_date = releases_after_last_merge_date[0] + last_release_date = dt_to_ymd(first_release_after_last_merge_date.completed) + next_softfreeze_date = iso_to_ymd(current_nightly_version_schedule["soft_code_freeze"]) + next_merge_date = iso_to_ymd(current_nightly_version_schedule["merge_day"]) + next_release_date = dt_to_ymd(from_isoformat(current_nightly_version_schedule["merge_day"]) + timedelta(days=1)) + last_stringfreeze_date = dt_to_ymd(from_ymd_format(last_softfreeze_date) + timedelta(days=1)) + next_stringfreeze_date = dt_to_ymd(from_ymd_format(next_softfreeze_date) + timedelta(days=1)) + return { + "LAST_SOFTFREEZE_DATE": last_softfreeze_date, + "LAST_MERGE_DATE": last_merge_date, + "LAST_RELEASE_DATE": last_release_date, + "NEXT_SOFTFREEZE_DATE": next_softfreeze_date, + "NEXT_MERGE_DATE": next_merge_date, + "NEXT_RELEASE_DATE": next_release_date, + "LAST_STRINGFREEZE_DATE": last_stringfreeze_date, + "NEXT_STRINGFREEZE_DATE": next_stringfreeze_date, + } + + +async def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release]) -> FirefoxVersions: """All the versions we ship for Firefox for Desktop This function will output to the following files: @@ -651,6 +730,8 @@ def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release] "NEXT_RELEASE_DATE": "2019-05-14" } """ + async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit_per_host=50), timeout=aiohttp.ClientTimeout(total=30)) as session: + firefox_release_schedule_data = await fetch_firefox_release_schedule_data(releases, session) return dict( FIREFOX_NIGHTLY=shipit_api.common.config.FIREFOX_NIGHTLY, @@ -664,14 +745,14 @@ def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release] LATEST_FIREFOX_RELEASED_DEVEL_VERSION=get_latest_version(releases, Product.FIREFOX, shipit_api.common.config.BETA_BRANCH), FIREFOX_DEVEDITION=get_latest_version(releases, Product.DEVEDITION, shipit_api.common.config.BETA_BRANCH), LATEST_FIREFOX_OLDER_VERSION=shipit_api.common.config.LATEST_FIREFOX_OLDER_VERSION, - LAST_SOFTFREEZE_DATE=shipit_api.common.config.LAST_SOFTFREEZE_DATE, - LAST_STRINGFREEZE_DATE=shipit_api.common.config.LAST_STRINGFREEZE_DATE, - LAST_MERGE_DATE=shipit_api.common.config.LAST_MERGE_DATE, - LAST_RELEASE_DATE=shipit_api.common.config.LAST_RELEASE_DATE, - NEXT_SOFTFREEZE_DATE=shipit_api.common.config.NEXT_SOFTFREEZE_DATE, - NEXT_STRINGFREEZE_DATE=shipit_api.common.config.NEXT_STRINGFREEZE_DATE, - NEXT_MERGE_DATE=shipit_api.common.config.NEXT_MERGE_DATE, - NEXT_RELEASE_DATE=shipit_api.common.config.NEXT_RELEASE_DATE, + LAST_SOFTFREEZE_DATE=firefox_release_schedule_data["LAST_SOFTFREEZE_DATE"], + LAST_STRINGFREEZE_DATE=firefox_release_schedule_data["LAST_STRINGFREEZE_DATE"], + LAST_MERGE_DATE=firefox_release_schedule_data["LAST_MERGE_DATE"], + LAST_RELEASE_DATE=firefox_release_schedule_data["LAST_RELEASE_DATE"], + NEXT_SOFTFREEZE_DATE=firefox_release_schedule_data["NEXT_SOFTFREEZE_DATE"], + NEXT_STRINGFREEZE_DATE=firefox_release_schedule_data["NEXT_STRINGFREEZE_DATE"], + NEXT_MERGE_DATE=firefox_release_schedule_data["NEXT_MERGE_DATE"], + NEXT_RELEASE_DATE=firefox_release_schedule_data["NEXT_RELEASE_DATE"], ) @@ -1104,8 +1185,8 @@ async def rebuild( "firefox_history_stability_releases.json": get_release_history( breakpoint_version, Product.FIREFOX, ProductCategory.STABILITY, releases, old_product_details ), - "firefox_primary_builds.json": get_primary_builds(breakpoint_version, Product.FIREFOX, combined_releases, combined_l10n, old_product_details), - "firefox_versions.json": get_firefox_versions(releases), + "firefox_primary_builds.json": await get_primary_builds(breakpoint_version, Product.FIREFOX, combined_releases, combined_l10n, old_product_details), + "firefox_versions.json": await get_firefox_versions(releases), "languages.json": get_languages(old_product_details), "mobile_android.json": get_releases(breakpoint_version, [Product.FENNEC, Product.FENIX, Product.FIREFOX_ANDROID], releases, old_product_details), "mobile_details.json": get_mobile_details(releases), @@ -1128,7 +1209,9 @@ async def rebuild( "thunderbird_history_stability_releases.json": get_release_history( breakpoint_version, Product.THUNDERBIRD, ProductCategory.STABILITY, releases, old_product_details ), - "thunderbird_primary_builds.json": get_primary_builds(breakpoint_version, Product.THUNDERBIRD, combined_releases, combined_l10n, old_product_details), + "thunderbird_primary_builds.json": await get_primary_builds( + breakpoint_version, Product.THUNDERBIRD, combined_releases, combined_l10n, old_product_details + ), "thunderbird_versions.json": get_thunderbird_versions(releases), } diff --git a/api/src/shipit_api/common/config.py b/api/src/shipit_api/common/config.py index f8fef7446..0ea5e259a 100644 --- a/api/src/shipit_api/common/config.py +++ b/api/src/shipit_api/common/config.py @@ -5,7 +5,6 @@ import pathlib import tempfile -from datetime import datetime, timedelta from functools import cache from decouple import config @@ -54,20 +53,6 @@ # This version also defines the mobile nightly version (i.e.: Fenix) FIREFOX_NIGHTLY = "128.0a1" -# The next 6 dates are information about the current and next release -# They must be updated at the same time as FIREFOX_NIGHTLY -# They can be found on https://whattrainisitnow.com/calendar/ -LAST_SOFTFREEZE_DATE = "2024-05-09" -LAST_MERGE_DATE = "2024-05-13" -LAST_RELEASE_DATE = "2024-05-14" -NEXT_SOFTFREEZE_DATE = "2024-06-06" -NEXT_MERGE_DATE = "2024-06-10" -NEXT_RELEASE_DATE = "2024-06-11" - -DATE_FORMAT = "%Y-%m-%d" -LAST_STRINGFREEZE_DATE = (datetime.strptime(LAST_SOFTFREEZE_DATE, DATE_FORMAT) + timedelta(days=1)).strftime(DATE_FORMAT) -NEXT_STRINGFREEZE_DATE = (datetime.strptime(NEXT_SOFTFREEZE_DATE, DATE_FORMAT) + timedelta(days=1)).strftime(DATE_FORMAT) - # Aurora has been replaced by Dev Edition, but some 3rd party applications may # still rely on this value. FIREFOX_AURORA = ""