Skip to content

Commit

Permalink
Query the whattrainisitnow release schedule API instead of hard-codin…
Browse files Browse the repository at this point in the history 
…g scheduled release dates (#1464)

* Query the whattrainisitnow release schedule API instead of hard-coding scheduled release dates
* Import datetime names to decrease verbosity in product_details module
* https://bugzilla.mozilla.org/show_bug.cgi?id=1890753
  • Loading branch information
@gabrielBusta
gabrielBusta authored May 15, 2024
1 parent ac8421f commit 55cd0cb
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 32 deletions.
117 changes: 100 additions & 17 deletions api/src/shipit_api/admin/product_details.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

import asyncio
import collections
import datetime
import functools
import hashlib
import io
Expand All @@ -17,6 +16,7 @@
import shutil
import typing
import urllib.parse
from datetime import datetime, timedelta, timezone

import aiohttp
import arrow
Expand All @@ -25,6 +25,7 @@
import mypy_extensions
import sqlalchemy
import sqlalchemy.orm
from mozilla_version.gecko import FirefoxVersion
from mozilla_version.mobile import MobileVersion

import cli_common.command
Expand Down Expand Up @@ -126,14 +127,37 @@ def with_default(a: typing.Optional[A], func: typing.Callable[[A], B], default:
return func(a)


def to_isoformat(d: datetime.datetime) -> str:
def to_isoformat(d: datetime) -> str:
return arrow.get(d).isoformat()


def to_format(d: datetime.datetime, format: str) -> str:
def from_isoformat(s: str) -> datetime:
return datetime.fromisoformat(s)


def to_format(d: datetime, format: str) -> str:
return arrow.get(d).format(format)


def from_format(s: str, format: str) -> datetime:
return datetime.strptime(s, format)


YMD_DATE_FORMAT = "%Y-%m-%d"


def iso_to_ymd(s: str) -> str:
return from_isoformat(s).strftime(YMD_DATE_FORMAT)


def dt_to_ymd(d: datetime) -> str:
return d.strftime(YMD_DATE_FORMAT)


def from_ymd_format(s: str) -> datetime:
return from_format(s, YMD_DATE_FORMAT)


def create_index_listing_html(folder: pathlib.Path, items: typing.Set[pathlib.Path]) -> str:
folder = "/" / folder # noqa : T484 Unsupported left operand type for / ("str")
with io.StringIO() as html:
Expand Down Expand Up @@ -495,7 +519,7 @@ def get_release_history(
return ordered_history


def get_primary_builds(
async def get_primary_builds(
breakpoint_version: int,
product: Product,
releases: typing.List[shipit_api.common.models.Release],
Expand Down Expand Up @@ -531,7 +555,7 @@ def get_primary_builds(
"""

if product is Product.FIREFOX:
firefox_versions = get_firefox_versions(releases)
firefox_versions = await get_firefox_versions(releases)
# make sure that Devedition is included in the list
products = [Product.FIREFOX, Product.DEVEDITION]
versions = set(
Expand Down Expand Up @@ -624,7 +648,62 @@ def get_firefox_esr_next_version(releases: typing.List[shipit_api.common.models.
return get_latest_version(releases, product, branch)


def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release]) -> FirefoxVersions:
@backoff.on_exception(backoff.expo, (aiohttp.ClientError, asyncio.TimeoutError), max_time=60)
async def fetch_firefox_release_schedule_data(releases: typing.List[shipit_api.common.models.Release], session: aiohttp.ClientSession):
firefox_nightly_mozilla_version = FirefoxVersion.parse(shipit_api.common.config.FIREFOX_NIGHTLY)
current_nightly_version_major_number = firefox_nightly_mozilla_version.major_number
previous_nightly_version_major_number = current_nightly_version_major_number - 1
url_template = "https://whattrainisitnow.com/api/release/schedule/?version={version}"
try:
url = url_template.format(version=current_nightly_version_major_number)
async with session.get(url) as response:
response.raise_for_status()
logger.debug(f"Fetched {url}")
current_nightly_version_schedule = await response.json()
url = url_template.format(version=previous_nightly_version_major_number)
async with session.get(url) as response:
response.raise_for_status()
logger.debug(f"Fetched {url}")
previous_nightly_version_schedule = await response.json()
except Exception:
logger.info("Failed to fetch %s", url)
raise
last_softfreeze_date = iso_to_ymd(previous_nightly_version_schedule["soft_code_freeze"])
last_merge_date = iso_to_ymd(previous_nightly_version_schedule["merge_day"])
releases_after_last_merge_date = sorted(
[
release
for release in releases
if release.product == Product.FIREFOX.value
and FirefoxVersion.parse(release.version).is_release
and release.status == "shipped"
and release.completed is not None
and release.completed.replace(tzinfo=timezone.utc) > from_isoformat(previous_nightly_version_schedule["merge_day"])
],
key=lambda release: FirefoxVersion.parse(release.version),
)
if not releases_after_last_merge_date:
raise ValueError(f"No Firefox releases shipped after the last merge date ({last_merge_date})")
first_release_after_last_merge_date = releases_after_last_merge_date[0]
last_release_date = dt_to_ymd(first_release_after_last_merge_date.completed)
next_softfreeze_date = iso_to_ymd(current_nightly_version_schedule["soft_code_freeze"])
next_merge_date = iso_to_ymd(current_nightly_version_schedule["merge_day"])
next_release_date = dt_to_ymd(from_isoformat(current_nightly_version_schedule["merge_day"]) + timedelta(days=1))
last_stringfreeze_date = dt_to_ymd(from_ymd_format(last_softfreeze_date) + timedelta(days=1))
next_stringfreeze_date = dt_to_ymd(from_ymd_format(next_softfreeze_date) + timedelta(days=1))
return {
"LAST_SOFTFREEZE_DATE": last_softfreeze_date,
"LAST_MERGE_DATE": last_merge_date,
"LAST_RELEASE_DATE": last_release_date,
"NEXT_SOFTFREEZE_DATE": next_softfreeze_date,
"NEXT_MERGE_DATE": next_merge_date,
"NEXT_RELEASE_DATE": next_release_date,
"LAST_STRINGFREEZE_DATE": last_stringfreeze_date,
"NEXT_STRINGFREEZE_DATE": next_stringfreeze_date,
}


async def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release]) -> FirefoxVersions:
"""All the versions we ship for Firefox for Desktop
This function will output to the following files:
Expand All @@ -651,6 +730,8 @@ def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release]
"NEXT_RELEASE_DATE": "2019-05-14"
}
"""
async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit_per_host=50), timeout=aiohttp.ClientTimeout(total=30)) as session:
firefox_release_schedule_data = await fetch_firefox_release_schedule_data(releases, session)

return dict(
FIREFOX_NIGHTLY=shipit_api.common.config.FIREFOX_NIGHTLY,
Expand All @@ -664,14 +745,14 @@ def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release]
LATEST_FIREFOX_RELEASED_DEVEL_VERSION=get_latest_version(releases, Product.FIREFOX, shipit_api.common.config.BETA_BRANCH),
FIREFOX_DEVEDITION=get_latest_version(releases, Product.DEVEDITION, shipit_api.common.config.BETA_BRANCH),
LATEST_FIREFOX_OLDER_VERSION=shipit_api.common.config.LATEST_FIREFOX_OLDER_VERSION,
LAST_SOFTFREEZE_DATE=shipit_api.common.config.LAST_SOFTFREEZE_DATE,
LAST_STRINGFREEZE_DATE=shipit_api.common.config.LAST_STRINGFREEZE_DATE,
LAST_MERGE_DATE=shipit_api.common.config.LAST_MERGE_DATE,
LAST_RELEASE_DATE=shipit_api.common.config.LAST_RELEASE_DATE,
NEXT_SOFTFREEZE_DATE=shipit_api.common.config.NEXT_SOFTFREEZE_DATE,
NEXT_STRINGFREEZE_DATE=shipit_api.common.config.NEXT_STRINGFREEZE_DATE,
NEXT_MERGE_DATE=shipit_api.common.config.NEXT_MERGE_DATE,
NEXT_RELEASE_DATE=shipit_api.common.config.NEXT_RELEASE_DATE,
LAST_SOFTFREEZE_DATE=firefox_release_schedule_data["LAST_SOFTFREEZE_DATE"],
LAST_STRINGFREEZE_DATE=firefox_release_schedule_data["LAST_STRINGFREEZE_DATE"],
LAST_MERGE_DATE=firefox_release_schedule_data["LAST_MERGE_DATE"],
LAST_RELEASE_DATE=firefox_release_schedule_data["LAST_RELEASE_DATE"],
NEXT_SOFTFREEZE_DATE=firefox_release_schedule_data["NEXT_SOFTFREEZE_DATE"],
NEXT_STRINGFREEZE_DATE=firefox_release_schedule_data["NEXT_STRINGFREEZE_DATE"],
NEXT_MERGE_DATE=firefox_release_schedule_data["NEXT_MERGE_DATE"],
NEXT_RELEASE_DATE=firefox_release_schedule_data["NEXT_RELEASE_DATE"],
)


Expand Down Expand Up @@ -1104,8 +1185,8 @@ async def rebuild(
"firefox_history_stability_releases.json": get_release_history(
breakpoint_version, Product.FIREFOX, ProductCategory.STABILITY, releases, old_product_details
),
"firefox_primary_builds.json": get_primary_builds(breakpoint_version, Product.FIREFOX, combined_releases, combined_l10n, old_product_details),
"firefox_versions.json": get_firefox_versions(releases),
"firefox_primary_builds.json": await get_primary_builds(breakpoint_version, Product.FIREFOX, combined_releases, combined_l10n, old_product_details),
"firefox_versions.json": await get_firefox_versions(releases),
"languages.json": get_languages(old_product_details),
"mobile_android.json": get_releases(breakpoint_version, [Product.FENNEC, Product.FENIX, Product.FIREFOX_ANDROID], releases, old_product_details),
"mobile_details.json": get_mobile_details(releases),
Expand All @@ -1128,7 +1209,9 @@ async def rebuild(
"thunderbird_history_stability_releases.json": get_release_history(
breakpoint_version, Product.THUNDERBIRD, ProductCategory.STABILITY, releases, old_product_details
),
"thunderbird_primary_builds.json": get_primary_builds(breakpoint_version, Product.THUNDERBIRD, combined_releases, combined_l10n, old_product_details),
"thunderbird_primary_builds.json": await get_primary_builds(
breakpoint_version, Product.THUNDERBIRD, combined_releases, combined_l10n, old_product_details
),
"thunderbird_versions.json": get_thunderbird_versions(releases),
}

Expand Down
15 changes: 0 additions & 15 deletions api/src/shipit_api/common/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

import pathlib
import tempfile
from datetime import datetime, timedelta
from functools import cache

from decouple import config
Expand Down Expand Up @@ -54,20 +53,6 @@
# This version also defines the mobile nightly version (i.e.: Fenix)
FIREFOX_NIGHTLY = "128.0a1"

# The next 6 dates are information about the current and next release
# They must be updated at the same time as FIREFOX_NIGHTLY
# They can be found on https://whattrainisitnow.com/calendar/
LAST_SOFTFREEZE_DATE = "2024-05-09"
LAST_MERGE_DATE = "2024-05-13"
LAST_RELEASE_DATE = "2024-05-14"
NEXT_SOFTFREEZE_DATE = "2024-06-06"
NEXT_MERGE_DATE = "2024-06-10"
NEXT_RELEASE_DATE = "2024-06-11"

DATE_FORMAT = "%Y-%m-%d"
LAST_STRINGFREEZE_DATE = (datetime.strptime(LAST_SOFTFREEZE_DATE, DATE_FORMAT) + timedelta(days=1)).strftime(DATE_FORMAT)
NEXT_STRINGFREEZE_DATE = (datetime.strptime(NEXT_SOFTFREEZE_DATE, DATE_FORMAT) + timedelta(days=1)).strftime(DATE_FORMAT)

# Aurora has been replaced by Dev Edition, but some 3rd party applications may
# still rely on this value.
FIREFOX_AURORA = ""
Expand Down

0 comments on commit 55cd0cb

Please sign in to comment.