diff --git a/api/src/shipit_api/admin/product_details.py b/api/src/shipit_api/admin/product_details.py index a9eaff669..2b7118e80 100644 --- a/api/src/shipit_api/admin/product_details.py +++ b/api/src/shipit_api/admin/product_details.py @@ -25,6 +25,7 @@ import mypy_extensions import sqlalchemy import sqlalchemy.orm +from mozilla_version.gecko import GeckoVersion from mozilla_version.mobile import MobileVersion import cli_common.command @@ -495,7 +496,7 @@ def get_release_history( return ordered_history -def get_primary_builds( +async def get_primary_builds( breakpoint_version: int, product: Product, releases: typing.List[shipit_api.common.models.Release], @@ -531,7 +532,7 @@ def get_primary_builds( """ if product is Product.FIREFOX: - firefox_versions = get_firefox_versions(releases) + firefox_versions = await get_firefox_versions(releases) # make sure that Devedition is included in the list products = [Product.FIREFOX, Product.DEVEDITION] versions = set( @@ -624,7 +625,49 @@ def get_firefox_esr_next_version(releases: typing.List[shipit_api.common.models. return get_latest_version(releases, product, branch) -def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release]) -> FirefoxVersions: +@backoff.on_exception(backoff.expo, (aiohttp.ClientError, asyncio.TimeoutError), max_time=60) +async def fetch_release_schedule_data(session: aiohttp.ClientSession): + gecko_version = GeckoVersion.parse(shipit_api.common.config.FIREFOX_NIGHTLY) + current_version = gecko_version.major_number + previous_version = current_version - 1 + url_template = "https://whattrainisitnow.com/api/release/schedule/?version={version}" + data = {} + try: + url = url_template.format(version=current_version) + async with session.get(url) as response: + response.raise_for_status() + logger.debug(f"Fetched {url}") + data["current_version"] = await response.json() + url = url_template.format(version=previous_version) + async with session.get(url) as response: + response.raise_for_status() + logger.debug(f"Fetched {url}") + data["previous_version"] = await response.json() + except Exception: + logger.info("Failed to fetch %s, %s", url) + raise + DATE_FORMAT = "%Y-%m-%d" + LAST_SOFTFREEZE_DATE = datetime.datetime.fromisoformat(data["previous_version"]["soft_code_freeze"]).strftime("%Y-%m-%d") + LAST_MERGE_DATE = datetime.datetime.fromisoformat(data["previous_version"]["merge_day"]).strftime("%Y-%m-%d") + LAST_RELEASE_DATE = (datetime.datetime.fromisoformat(data["previous_version"]["merge_day"]) + datetime.timedelta(days=1)).strftime("%Y-%m-%d") + NEXT_SOFTFREEZE_DATE = datetime.datetime.fromisoformat(data["current_version"]["soft_code_freeze"]).strftime("%Y-%m-%d") + NEXT_MERGE_DATE = datetime.datetime.fromisoformat(data["current_version"]["merge_day"]).strftime("%Y-%m-%d") + NEXT_RELEASE_DATE = (datetime.datetime.fromisoformat(data["current_version"]["merge_day"]) + datetime.timedelta(days=1)).strftime("%Y-%m-%d") + LAST_STRINGFREEZE_DATE = (datetime.datetime.strptime(LAST_SOFTFREEZE_DATE, DATE_FORMAT) + datetime.timedelta(days=1)).strftime(DATE_FORMAT) + NEXT_STRINGFREEZE_DATE = (datetime.datetime.strptime(NEXT_SOFTFREEZE_DATE, DATE_FORMAT) + datetime.timedelta(days=1)).strftime(DATE_FORMAT) + return { + "LAST_SOFTFREEZE_DATE": LAST_SOFTFREEZE_DATE, + "LAST_MERGE_DATE": LAST_MERGE_DATE, + "LAST_RELEASE_DATE": LAST_RELEASE_DATE, + "NEXT_SOFTFREEZE_DATE": NEXT_SOFTFREEZE_DATE, + "NEXT_MERGE_DATE": NEXT_MERGE_DATE, + "NEXT_RELEASE_DATE": NEXT_RELEASE_DATE, + "LAST_STRINGFREEZE_DATE": LAST_STRINGFREEZE_DATE, + "NEXT_STRINGFREEZE_DATE": NEXT_STRINGFREEZE_DATE, + } + + +async def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release]) -> FirefoxVersions: """All the versions we ship for Firefox for Desktop This function will output to the following files: @@ -651,6 +694,8 @@ def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release] "NEXT_RELEASE_DATE": "2019-05-14" } """ + async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit_per_host=50), timeout=aiohttp.ClientTimeout(total=30)) as session: + release_schedule_data = await fetch_release_schedule_data(session) return dict( FIREFOX_NIGHTLY=shipit_api.common.config.FIREFOX_NIGHTLY, @@ -664,14 +709,14 @@ def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release] LATEST_FIREFOX_RELEASED_DEVEL_VERSION=get_latest_version(releases, Product.FIREFOX, shipit_api.common.config.BETA_BRANCH), FIREFOX_DEVEDITION=get_latest_version(releases, Product.DEVEDITION, shipit_api.common.config.BETA_BRANCH), LATEST_FIREFOX_OLDER_VERSION=shipit_api.common.config.LATEST_FIREFOX_OLDER_VERSION, - LAST_SOFTFREEZE_DATE=shipit_api.common.config.LAST_SOFTFREEZE_DATE, - LAST_STRINGFREEZE_DATE=shipit_api.common.config.LAST_STRINGFREEZE_DATE, - LAST_MERGE_DATE=shipit_api.common.config.LAST_MERGE_DATE, - LAST_RELEASE_DATE=shipit_api.common.config.LAST_RELEASE_DATE, - NEXT_SOFTFREEZE_DATE=shipit_api.common.config.NEXT_SOFTFREEZE_DATE, - NEXT_STRINGFREEZE_DATE=shipit_api.common.config.NEXT_STRINGFREEZE_DATE, - NEXT_MERGE_DATE=shipit_api.common.config.NEXT_MERGE_DATE, - NEXT_RELEASE_DATE=shipit_api.common.config.NEXT_RELEASE_DATE, + LAST_SOFTFREEZE_DATE=release_schedule_data["LAST_SOFTFREEZE_DATE"], + LAST_STRINGFREEZE_DATE=release_schedule_data["LAST_STRINGFREEZE_DATE"], + LAST_MERGE_DATE=release_schedule_data["LAST_MERGE_DATE"], + LAST_RELEASE_DATE=release_schedule_data["LAST_RELEASE_DATE"], + NEXT_SOFTFREEZE_DATE=release_schedule_data["NEXT_SOFTFREEZE_DATE"], + NEXT_STRINGFREEZE_DATE=release_schedule_data["NEXT_STRINGFREEZE_DATE"], + NEXT_MERGE_DATE=release_schedule_data["NEXT_MERGE_DATE"], + NEXT_RELEASE_DATE=release_schedule_data["NEXT_RELEASE_DATE"], ) @@ -1104,8 +1149,8 @@ async def rebuild( "firefox_history_stability_releases.json": get_release_history( breakpoint_version, Product.FIREFOX, ProductCategory.STABILITY, releases, old_product_details ), - "firefox_primary_builds.json": get_primary_builds(breakpoint_version, Product.FIREFOX, combined_releases, combined_l10n, old_product_details), - "firefox_versions.json": get_firefox_versions(releases), + "firefox_primary_builds.json": await get_primary_builds(breakpoint_version, Product.FIREFOX, combined_releases, combined_l10n, old_product_details), + "firefox_versions.json": await get_firefox_versions(releases), "languages.json": get_languages(old_product_details), "mobile_android.json": get_releases(breakpoint_version, [Product.FENNEC, Product.FENIX, Product.FIREFOX_ANDROID], releases, old_product_details), "mobile_details.json": get_mobile_details(releases), @@ -1128,7 +1173,9 @@ async def rebuild( "thunderbird_history_stability_releases.json": get_release_history( breakpoint_version, Product.THUNDERBIRD, ProductCategory.STABILITY, releases, old_product_details ), - "thunderbird_primary_builds.json": get_primary_builds(breakpoint_version, Product.THUNDERBIRD, combined_releases, combined_l10n, old_product_details), + "thunderbird_primary_builds.json": await get_primary_builds( + breakpoint_version, Product.THUNDERBIRD, combined_releases, combined_l10n, old_product_details + ), "thunderbird_versions.json": get_thunderbird_versions(releases), }