Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Query the whattrainisitnow release schedule API instead of hard-coding scheduled release dates #1464

Merged
merged 5 commits into from
May 15, 2024
Merged

Query the whattrainisitnow release schedule API instead of hard-coding scheduled release dates #1464

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d73fe8d
Query the whattrainisitnow release schedule API instead of hard-codin…
gabrielBusta May 7, 2024
0f72799
Raise a ValueError if no value is found for releases_after_last_merge…
gabrielBusta May 15, 2024
b61ba6e
Fix logging on url fetch error
gabrielBusta May 15, 2024
1f8fec9
Add datetime helpers to decrease verbosity in fetch_firefox_release_s…
gabrielBusta May 15, 2024
63eecb2
Import datetime names to decrease verbosity in product details
gabrielBusta May 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 100 additions & 17 deletions api/src/shipit_api/admin/product_details.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

import asyncio
import collections
import datetime
import functools
import hashlib
import io
Expand All @@ -17,6 +16,7 @@
import shutil
import typing
import urllib.parse
from datetime import datetime, timedelta, timezone

import aiohttp
import arrow
Expand All @@ -25,6 +25,7 @@
import mypy_extensions
gabrielBusta marked this conversation as resolved.
Show resolved Hide resolved
import sqlalchemy
import sqlalchemy.orm
from mozilla_version.gecko import FirefoxVersion
from mozilla_version.mobile import MobileVersion

import cli_common.command
Expand Down Expand Up @@ -126,14 +127,37 @@ def with_default(a: typing.Optional[A], func: typing.Callable[[A], B], default:
return func(a)


def to_isoformat(d: datetime.datetime) -> str:
def to_isoformat(d: datetime) -> str:
return arrow.get(d).isoformat()


def to_format(d: datetime.datetime, format: str) -> str:
def from_isoformat(s: str) -> datetime:
return datetime.fromisoformat(s)


def to_format(d: datetime, format: str) -> str:
return arrow.get(d).format(format)


def from_format(s: str, format: str) -> datetime:
return datetime.strptime(s, format)


YMD_DATE_FORMAT = "%Y-%m-%d"


def iso_to_ymd(s: str) -> str:
return from_isoformat(s).strftime(YMD_DATE_FORMAT)


def dt_to_ymd(d: datetime) -> str:
return d.strftime(YMD_DATE_FORMAT)


def from_ymd_format(s: str) -> datetime:
return from_format(s, YMD_DATE_FORMAT)


gabrielBusta marked this conversation as resolved.
Show resolved Hide resolved
def create_index_listing_html(folder: pathlib.Path, items: typing.Set[pathlib.Path]) -> str:
folder = "/" / folder # noqa : T484 Unsupported left operand type for / ("str")
with io.StringIO() as html:
Expand Down Expand Up @@ -495,7 +519,7 @@ def get_release_history(
return ordered_history


def get_primary_builds(
async def get_primary_builds(
breakpoint_version: int,
product: Product,
releases: typing.List[shipit_api.common.models.Release],
Expand Down Expand Up @@ -531,7 +555,7 @@ def get_primary_builds(
"""

if product is Product.FIREFOX:
firefox_versions = get_firefox_versions(releases)
firefox_versions = await get_firefox_versions(releases)
# make sure that Devedition is included in the list
products = [Product.FIREFOX, Product.DEVEDITION]
versions = set(
Expand Down Expand Up @@ -624,7 +648,62 @@ def get_firefox_esr_next_version(releases: typing.List[shipit_api.common.models.
return get_latest_version(releases, product, branch)


def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release]) -> FirefoxVersions:
@backoff.on_exception(backoff.expo, (aiohttp.ClientError, asyncio.TimeoutError), max_time=60)
async def fetch_firefox_release_schedule_data(releases: typing.List[shipit_api.common.models.Release], session: aiohttp.ClientSession):
firefox_nightly_mozilla_version = FirefoxVersion.parse(shipit_api.common.config.FIREFOX_NIGHTLY)
current_nightly_version_major_number = firefox_nightly_mozilla_version.major_number
previous_nightly_version_major_number = current_nightly_version_major_number - 1
url_template = "https://whattrainisitnow.com/api/release/schedule/?version={version}"
try:
url = url_template.format(version=current_nightly_version_major_number)
async with session.get(url) as response:
response.raise_for_status()
logger.debug(f"Fetched {url}")
current_nightly_version_schedule = await response.json()
url = url_template.format(version=previous_nightly_version_major_number)
async with session.get(url) as response:
response.raise_for_status()
logger.debug(f"Fetched {url}")
previous_nightly_version_schedule = await response.json()
except Exception:
logger.info("Failed to fetch %s", url)
raise
last_softfreeze_date = iso_to_ymd(previous_nightly_version_schedule["soft_code_freeze"])
last_merge_date = iso_to_ymd(previous_nightly_version_schedule["merge_day"])
releases_after_last_merge_date = sorted(
[
release
for release in releases
if release.product == Product.FIREFOX.value
and FirefoxVersion.parse(release.version).is_release
and release.status == "shipped"
and release.completed is not None
and release.completed.replace(tzinfo=timezone.utc) > from_isoformat(previous_nightly_version_schedule["merge_day"])
],
key=lambda release: FirefoxVersion.parse(release.version),
)
if not releases_after_last_merge_date:
raise ValueError(f"No Firefox releases shipped after the last merge date ({last_merge_date})")
first_release_after_last_merge_date = releases_after_last_merge_date[0]
gabrielBusta marked this conversation as resolved.
Show resolved Hide resolved
last_release_date = dt_to_ymd(first_release_after_last_merge_date.completed)
next_softfreeze_date = iso_to_ymd(current_nightly_version_schedule["soft_code_freeze"])
next_merge_date = iso_to_ymd(current_nightly_version_schedule["merge_day"])
next_release_date = dt_to_ymd(from_isoformat(current_nightly_version_schedule["merge_day"]) + timedelta(days=1))
last_stringfreeze_date = dt_to_ymd(from_ymd_format(last_softfreeze_date) + timedelta(days=1))
next_stringfreeze_date = dt_to_ymd(from_ymd_format(next_softfreeze_date) + timedelta(days=1))
return {
"LAST_SOFTFREEZE_DATE": last_softfreeze_date,
"LAST_MERGE_DATE": last_merge_date,
"LAST_RELEASE_DATE": last_release_date,
"NEXT_SOFTFREEZE_DATE": next_softfreeze_date,
"NEXT_MERGE_DATE": next_merge_date,
"NEXT_RELEASE_DATE": next_release_date,
"LAST_STRINGFREEZE_DATE": last_stringfreeze_date,
"NEXT_STRINGFREEZE_DATE": next_stringfreeze_date,
}


async def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release]) -> FirefoxVersions:
"""All the versions we ship for Firefox for Desktop

This function will output to the following files:
Expand All @@ -651,6 +730,8 @@ def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release]
"NEXT_RELEASE_DATE": "2019-05-14"
}
"""
async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit_per_host=50), timeout=aiohttp.ClientTimeout(total=30)) as session:
firefox_release_schedule_data = await fetch_firefox_release_schedule_data(releases, session)

return dict(
FIREFOX_NIGHTLY=shipit_api.common.config.FIREFOX_NIGHTLY,
Expand All @@ -664,14 +745,14 @@ def get_firefox_versions(releases: typing.List[shipit_api.common.models.Release]
LATEST_FIREFOX_RELEASED_DEVEL_VERSION=get_latest_version(releases, Product.FIREFOX, shipit_api.common.config.BETA_BRANCH),
FIREFOX_DEVEDITION=get_latest_version(releases, Product.DEVEDITION, shipit_api.common.config.BETA_BRANCH),
LATEST_FIREFOX_OLDER_VERSION=shipit_api.common.config.LATEST_FIREFOX_OLDER_VERSION,
LAST_SOFTFREEZE_DATE=shipit_api.common.config.LAST_SOFTFREEZE_DATE,
LAST_STRINGFREEZE_DATE=shipit_api.common.config.LAST_STRINGFREEZE_DATE,
LAST_MERGE_DATE=shipit_api.common.config.LAST_MERGE_DATE,
LAST_RELEASE_DATE=shipit_api.common.config.LAST_RELEASE_DATE,
NEXT_SOFTFREEZE_DATE=shipit_api.common.config.NEXT_SOFTFREEZE_DATE,
NEXT_STRINGFREEZE_DATE=shipit_api.common.config.NEXT_STRINGFREEZE_DATE,
NEXT_MERGE_DATE=shipit_api.common.config.NEXT_MERGE_DATE,
NEXT_RELEASE_DATE=shipit_api.common.config.NEXT_RELEASE_DATE,
LAST_SOFTFREEZE_DATE=firefox_release_schedule_data["LAST_SOFTFREEZE_DATE"],
LAST_STRINGFREEZE_DATE=firefox_release_schedule_data["LAST_STRINGFREEZE_DATE"],
LAST_MERGE_DATE=firefox_release_schedule_data["LAST_MERGE_DATE"],
LAST_RELEASE_DATE=firefox_release_schedule_data["LAST_RELEASE_DATE"],
NEXT_SOFTFREEZE_DATE=firefox_release_schedule_data["NEXT_SOFTFREEZE_DATE"],
NEXT_STRINGFREEZE_DATE=firefox_release_schedule_data["NEXT_STRINGFREEZE_DATE"],
NEXT_MERGE_DATE=firefox_release_schedule_data["NEXT_MERGE_DATE"],
NEXT_RELEASE_DATE=firefox_release_schedule_data["NEXT_RELEASE_DATE"],
)


Expand Down Expand Up @@ -1104,8 +1185,8 @@ async def rebuild(
"firefox_history_stability_releases.json": get_release_history(
breakpoint_version, Product.FIREFOX, ProductCategory.STABILITY, releases, old_product_details
),
"firefox_primary_builds.json": get_primary_builds(breakpoint_version, Product.FIREFOX, combined_releases, combined_l10n, old_product_details),
"firefox_versions.json": get_firefox_versions(releases),
"firefox_primary_builds.json": await get_primary_builds(breakpoint_version, Product.FIREFOX, combined_releases, combined_l10n, old_product_details),
"firefox_versions.json": await get_firefox_versions(releases),
"languages.json": get_languages(old_product_details),
"mobile_android.json": get_releases(breakpoint_version, [Product.FENNEC, Product.FENIX, Product.FIREFOX_ANDROID], releases, old_product_details),
"mobile_details.json": get_mobile_details(releases),
Expand All @@ -1128,7 +1209,9 @@ async def rebuild(
"thunderbird_history_stability_releases.json": get_release_history(
breakpoint_version, Product.THUNDERBIRD, ProductCategory.STABILITY, releases, old_product_details
),
"thunderbird_primary_builds.json": get_primary_builds(breakpoint_version, Product.THUNDERBIRD, combined_releases, combined_l10n, old_product_details),
"thunderbird_primary_builds.json": await get_primary_builds(
breakpoint_version, Product.THUNDERBIRD, combined_releases, combined_l10n, old_product_details
),
"thunderbird_versions.json": get_thunderbird_versions(releases),
}

Expand Down
15 changes: 0 additions & 15 deletions api/src/shipit_api/common/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

import pathlib
import tempfile
from datetime import datetime, timedelta
from functools import cache

from decouple import config
Expand Down Expand Up @@ -54,20 +53,6 @@
# This version also defines the mobile nightly version (i.e.: Fenix)
FIREFOX_NIGHTLY = "128.0a1"

# The next 6 dates are information about the current and next release
# They must be updated at the same time as FIREFOX_NIGHTLY
# They can be found on https://whattrainisitnow.com/calendar/
LAST_SOFTFREEZE_DATE = "2024-05-09"
LAST_MERGE_DATE = "2024-05-13"
LAST_RELEASE_DATE = "2024-05-14"
NEXT_SOFTFREEZE_DATE = "2024-06-06"
NEXT_MERGE_DATE = "2024-06-10"
NEXT_RELEASE_DATE = "2024-06-11"

DATE_FORMAT = "%Y-%m-%d"
LAST_STRINGFREEZE_DATE = (datetime.strptime(LAST_SOFTFREEZE_DATE, DATE_FORMAT) + timedelta(days=1)).strftime(DATE_FORMAT)
NEXT_STRINGFREEZE_DATE = (datetime.strptime(NEXT_SOFTFREEZE_DATE, DATE_FORMAT) + timedelta(days=1)).strftime(DATE_FORMAT)

# Aurora has been replaced by Dev Edition, but some 3rd party applications may
# still rely on this value.
FIREFOX_AURORA = ""
Expand Down