From 2111ae542b3c0ee6f65c15a71ec3b48b47f17c1c Mon Sep 17 00:00:00 2001 From: Francesco Ballarin Date: Sat, 31 Aug 2024 19:20:09 +0200 Subject: [PATCH] Add live_turing_to_html script (although without unit and integration testing) --- mathrace_interaction/README.md | 48 +++ .../mathrace_interaction/__init__.py | 3 +- .../live_turing_to_html.py | 278 ++++++++++++++++++ .../live_turing_to_live_journal.py | 5 +- mathrace_interaction/pyproject.toml | 3 + 5 files changed, 333 insertions(+), 4 deletions(-) create mode 100644 mathrace_interaction/mathrace_interaction/live_turing_to_html.py diff --git a/mathrace_interaction/README.md b/mathrace_interaction/README.md index e8982bf..b579c27 100644 --- a/mathrace_interaction/README.md +++ b/mathrace_interaction/README.md @@ -175,3 +175,51 @@ while true; do sleep 5 done ``` + +## Live `turing` to HTML output + +The script `mathrace_interaction/live_turing_to_html.py` transfers race events from a live `turing` session to a sequence of HTML files, which are suitable to update a mirror website and warn about podium position changes. + +### Before the race + +Set up the race through the turing web interface, and make a note of the value of the race id as `${LIVE_TURING_PRIMARY_KEY}`. + +### At the beginning of the race + +#### Step 1: send live updates from `turing` + +``` +python3 -m mathrace_interaction.live_turing_to_html -u "http://0.0.0.0" -p $(cat /mnt/secrets/.django_superuser_initial_password) -t "${LIVE_TURING_PRIMARY_KEY}" -s 10 -o "/shared/host-tmp/live_${LIVE_TURING_PRIMARY_KEY}" +``` + +#### Step 2: sync from docker host to another machine + +``` +while true; do + rsync -arvz ${TURING_HOST_USER}@${TURING_HOST}:/tmp/shared-turing-dmf/live_${LIVE_TURING_PRIMARY_KEY}/ /tmp/live_${LIVE_TURING_PRIMARY_KEY} + sleep 5 +done +``` + +#### Step 3: copy current classification to a remote website + +``` +HTML_TURING_OUTPUT=/tmp/shared-turing-dmf/live_${LIVE_TURING_PRIMARY_KEY}/html_files +REMOTE_WEBSITE_PATH=/tmp/remote-website +for EXT in css eot ttf woff woff2; do + scp ${TURING_HOST_USER}@${TURING_HOST}:${HTML_TURING_OUTPUT}/*.${EXT} ${REMOTE_WEBSITE_HOST_USER}@${REMOTE_WEBSITE_HOST}:${REMOTE_WEBSITE_PATH} +done +ssh -t ${TURING_HOST_USER}@${TURING_HOST} "tail -n 0 -f ${HTML_TURING_OUTPUT}/watch.txt" | while read -r LINE; do + scp ${TURING_HOST_USER}@${TURING_HOST}:${HTML_TURING_OUTPUT}/latest.html ${REMOTE_WEBSITE_HOST_USER}@${REMOTE_WEBSITE_HOST}:${REMOTE_WEBSITE_PATH}/index.html +done +``` + +#### Step 4: execute an action on podium change + +``` +PODIUM_CHANGE_TURING_OUTPUT=/tmp/shared-turing-dmf/live_${LIVE_TURING_PRIMARY_KEY}/podium_change_files +ssh -t ${TURING_HOST_USER}@${TURING_HOST} "tail -n 0 -f ${PODIUM_CHANGE_TURING_OUTPUT}/watch.txt" | while read -r LINE; do + POSITION=$(echo "${LINE}" | tr -d "\r" | rev | cut -d" " -f1 | rev) + echo "Podium position ${POSITION} changed" >> /tmp/podium-change-history +done +``` diff --git a/mathrace_interaction/mathrace_interaction/__init__.py b/mathrace_interaction/mathrace_interaction/__init__.py index 8a090b9..27e902b 100644 --- a/mathrace_interaction/mathrace_interaction/__init__.py +++ b/mathrace_interaction/mathrace_interaction/__init__.py @@ -13,12 +13,13 @@ from mathrace_interaction.journal_writer import journal_writer from mathrace_interaction.list_journal_versions import list_journal_versions from mathrace_interaction.live_journal_to_live_turing import live_journal_to_live_turing +from mathrace_interaction.live_turing_to_html import live_turing_to_html from mathrace_interaction.live_turing_to_live_journal import live_turing_to_live_journal # Silence warning when trying to run modules as entrypoint for entrypoint in ( "determine_journal_version", "journal_reader", "journal_version_converter", "journal_writer", - "list_journal_versions", "live_journal_to_live_turing", "live_turing_to_live_journal" + "list_journal_versions", "live_journal_to_live_turing", "live_turing_to_html", "live_turing_to_live_journal" ): warnings.filterwarnings( "ignore", message=( diff --git a/mathrace_interaction/mathrace_interaction/live_turing_to_html.py b/mathrace_interaction/mathrace_interaction/live_turing_to_html.py new file mode 100644 index 0000000..06b798c --- /dev/null +++ b/mathrace_interaction/mathrace_interaction/live_turing_to_html.py @@ -0,0 +1,278 @@ +# Copyright (C) 2024 by the Turing @ DMF authors +# +# This file is part of Turing @ DMF. +# +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Follow a live session in turing and convert it into a sequence of html files.""" + +import argparse +import json +import datetime +import pathlib +import shutil +import time +import types +import typing + +import prettytable +import pytz + +from mathrace_interaction.network import TuringClassificationSelenium + + +def live_turing_to_html( + turing_url: str, turing_models: types.ModuleType, turing_race_id: int, turing_race_admin_password: str, + sleep: float, output_directory: pathlib.Path, compute_current_time: typing.Callable[[int], datetime.datetime], + termination_condition: typing.Callable[[int], bool] +) -> None: # pragma: no cover + """ + Follow a live session in turing and convert it into a sequence of html files. + + Parameters + ---------- + turing_url + The URL of the live turing instance. + turing_models + The python module containing the turing model Gara. + turing_race_id + The ID of the turing race to follow. + turing_race_admin_password + The password of the administrator of the turing race. + sleep + The amount of time to wait between consecutive reads of the turing state. + output_directory + The path of the output directory + compute_current_time + A function that computes the current time given the current time counter + termination_condition + A function to determine whether to terminate the processing given the current time counter. + """ + # Get the actual turing models out of the turing_models argument + Gara = getattr(turing_models, "Gara") # noqa: N806 + + # Get the turing race from its ID + turing_race = Gara.objects.get(pk=turing_race_id) + + # The race must have been started before running this script + if turing_race.inizio is None: + raise RuntimeError(f"Please start race {turing_race_id} from the turing web interface") + + # Create the output directory if it does not exist yet + output_directory.mkdir(parents=True, exist_ok=True) + + # Constants associated to the two browsers we will open + LIVE = 0 # noqa: N806 + COMPARISON = 1 # noqa: N806 + + # Constants associated to the table we will create + POSITION_COLUMN = 0 # noqa: N806 + TEAM_ID_COLUMN = 1 # noqa: N806 + TEAM_NAME_COLUMN = 2 # noqa: N806 + SCORE_COLUMN = 3 # noqa: N806 + + # Create subdirectories in the output directory, if they do not exist yet + datetime_files_directory = output_directory / "datetime_files" + live_turing_json_files_directory = output_directory / "live_turing_json_files" + html_files_directory = [output_directory / "html_files", output_directory / "html_files_comparison"] + table_files_directory = [output_directory / "table_files", output_directory / "table_files_comparison"] + podium_change_files_directory = output_directory / "podium_change_files" + for directory in ( + datetime_files_directory, live_turing_json_files_directory, *html_files_directory, *table_files_directory, + podium_change_files_directory + ): + directory.mkdir(parents=True, exist_ok=True) + + # Read the current time counter if available, otherwise set it to zero + time_counter = 0 + time_counter_file = output_directory / "time_counter.txt" + if time_counter_file.exists(): + time_counter = int(time_counter_file.read_text()) + time_counter += 1 + + # Open two browsers to access the classification with querystring ?ended=False and ?ended=True + browsers = [TuringClassificationSelenium(turing_url, turing_race.pk, sleep // 2) for _ in range(2)] + for browser in browsers: + browser.login(turing_race.admin.username, turing_race_admin_password) + # Connect the live browser to the live instance with ?ended=False + browsers[LIVE].go_to_classification_page("unica", {"ended": "false", "computation_rate": str(sleep // 4)}) + + # Save CSS files for HTML export + if time_counter == 0: + browsers[LIVE].lock() + css_files, font_files = browsers[LIVE].get_auxiliary_files() + for (auxiliary_files, write_content) in ( + (css_files, lambda path, content: path.write_text(content)), + (font_files, lambda path, content: path.write_bytes(content)) + ): + for (filename, content) in auxiliary_files.items(): + write_content(html_files_directory[LIVE] / filename, content) # type: ignore[no-untyped-call] + shutil.copy( + html_files_directory[LIVE] / filename, + html_files_directory[COMPARISON] / filename) + browsers[LIVE].unlock() + + # Continuously read the turing state + previous_positions = None + previous_scores = None + while True: + print(f"{time_counter=}") + # Compute the current time, up to the microsecond + current_time = compute_current_time(time_counter) + actual_time = datetime.datetime.now(current_time.tzinfo) + inizio = turing_race.inizio.astimezone(current_time.tzinfo) + timestamp = (current_time - inizio).total_seconds() + # Precision to the second is more then enough for our goals: strip the microseconds, + # and recompute the dates + timestamp = int(timestamp) + current_time = inizio + datetime.timedelta(seconds=timestamp) + # Write out current time + (datetime_files_directory / f"{time_counter}.datetime").write_text("""Computed: {current_time} +Actual: {actual_time}""") + print(f"\tcomputed time is {current_time}") + print(f"\tactual time is {actual_time}") + print(f"\telapsed number of seconds {timestamp}") + # Backup the turing dictionary associated to the race at the time represented by the current counter + with open(live_turing_json_files_directory / f"{time_counter}.json", "w") as turing_json_file: + turing_json_file.write(json.dumps(turing_race.to_dict(), indent=4)) + shutil.copy( + live_turing_json_files_directory / f"{time_counter}.json", + live_turing_json_files_directory / "latest.json") + # Download browser content + html: list[str] = [None, None] # type: ignore[list-item] + table: list[prettytable.PrettyTable] = [None, None] # type: ignore[list-item] + for (INSTANCE, browser) in enumerate(browsers): # noqa: N806 + print(f'\tupdating {"live" if INSTANCE == LIVE else "comparison"} browser') + if INSTANCE == LIVE: + # Freeze the browser at the current time + browser.freeze_time(current_time) + else: + # Time does not get updated in the comparison browser, and hence go to the updated classification page + browser.go_to_classification_page("unica", { + "ended": "true", "computation_rate": "1", "race_time": str(timestamp)}) + # Save the content of the browser + browser.lock() + html[INSTANCE] = browser.get_cleaned_html_source() + table[INSTANCE] = browser.get_table() + browser.unlock() + # Do not bother unfreezing time in the live browser, since it would immediately be frozen again + # at the next iteration + # Write out the html files + for INSTANCE in (LIVE, COMPARISON): # noqa: N806 + assert html[INSTANCE] is not None + (html_files_directory[INSTANCE] / f"{time_counter}.html").write_text( + html[INSTANCE]) + # Add livejs script to the latest page so that it refreshes automatically + # when uploaded to an HTTP server. + # Note: livejs will not work when opening the file locally, since the file:// is not supported: + # to try it you need to have a real server and access it through http:// or https:// + # As a workaround, you can start a local HTTP server by running + # python3 -m http.server + # in the local directory. + (html_files_directory[INSTANCE] / "latest.html").write_text( + (html_files_directory[INSTANCE] / f"{time_counter}.html").read_text().replace( + "", '')) + with open(html_files_directory[INSTANCE] / "watch.txt", "a") as text_file: + text_file.write(f"updated at time counter {time_counter} ({current_time})\n") + # Determine if the live table and the comparison one are the same or not + assert table[LIVE] is not None + warn_table = (table[LIVE].get_string() != table[COMPARISON].get_string()) + # Compute team positions/scores, as a dictionary from the team ID to the team position/score + positions = {r[TEAM_ID_COLUMN]: r[POSITION_COLUMN] for r in table[LIVE].rows[1:]} + scores = {r[TEAM_ID_COLUMN]: r[SCORE_COLUMN] for r in table[LIVE].rows[1:]} + # Compute the difference between the scores at this time and at the previous time + print_fields: list[list[str]] = [None, None] # type: ignore[list-item] + print_fields[LIVE] = ["Position", "Team ID", "Team name", "Score"] + print_fields[COMPARISON] = ["Position", "Team ID", "Team name", "Score"] # do not assign the LIVE one! + podium_change: list[tuple[int, str, int, int]] = [] + if previous_positions is not None: + position_update = [ + previous_positions[r[TEAM_ID_COLUMN]] - positions[r[TEAM_ID_COLUMN]] for r in table[LIVE].rows[1:]] + for podium_position in (3, 2, 1): + if position_update[podium_position - 1] > 0: + team_id = table[LIVE].rows[podium_position][TEAM_ID_COLUMN] + team_name = table[LIVE].rows[podium_position][TEAM_NAME_COLUMN] + podium_change.append((team_id, team_name, podium_position, previous_positions[team_id])) + table[LIVE].add_column("Position update", [""] + [u if u != 0 else "" for u in position_update]) + print_fields[LIVE].append("Position update") + if previous_scores is not None: + score_update = [ + scores[r[TEAM_ID_COLUMN]] - previous_scores[r[TEAM_ID_COLUMN]] for r in table[LIVE].rows[1:]] + table[LIVE].add_column("Score update", [""] + [u if u != 0 else "" for u in score_update]) + print_fields[LIVE].append("Score update") + # Write out the table files + for INSTANCE in (LIVE, COMPARISON): # noqa: N806 + assert table[INSTANCE] is not None + (table_files_directory[INSTANCE] / f"{time_counter}.csv").write_text( + table[INSTANCE].get_formatted_string(out_format="csv")) + (table_files_directory[INSTANCE] / f"{time_counter}.html").write_text( + "" + + table[INSTANCE].get_formatted_string(fields=print_fields[INSTANCE], out_format="html", format=True) + + "") + shutil.copy( + table_files_directory[INSTANCE] / f"{time_counter}.csv", + table_files_directory[INSTANCE] / "latest.csv") + (table_files_directory[INSTANCE] / "latest.html").write_text( + (table_files_directory[INSTANCE] / f"{time_counter}.html").read_text().replace( + "", '')) + # Write out the podium change file + for (text_filename, mode) in ((f"{time_counter}.txt", "w"), ("watch.txt", "a")): + with open(podium_change_files_directory / text_filename, mode) as text_file: + for (team_id, team_name, current_position, previous_position) in podium_change: + text_file.write( + f"position change at time counter {time_counter} ({current_time}): " + f'team with ID {team_id} and name "{team_name}" improves from ' + f"position {previous_position} to podium position {current_position}\n" + ) + # Write out the time counter + time_counter_file.write_text(str(time_counter)) + # Print out table + print("\t" + table[LIVE].get_string(fields=print_fields[LIVE]).replace("\n", "\n\t")) + if warn_table: + print("\tWARNING: live and comparison tables are different") + # Break out of the loop if the race has ended + if termination_condition(time_counter): + break + # Upate the time counter + time_counter += 1 + # Replace previous positions/scores + previous_positions = positions + previous_scores = scores + # Wait before reading again the updated version of the turing state + actual_time_end = datetime.datetime.now(current_time.tzinfo) + wait_time = sleep - (actual_time_end - actual_time).total_seconds() + if wait_time > 0: + print(f"\twaiting {wait_time} seconds for next time iteration") + time.sleep(wait_time) + + +if __name__ == "__main__": # pragma: no cover + # This import requires turing to be available, and thus cannot be moved to the common section. + # We skip coverage testing of this part because we cannot cover this in unit tests, since they + # cannot interact with turing. Testing this entrypoint is delayed to integration testing. + import django + django.setup() + + import django.conf + import engine.models + + TIME_ZONE_SETTING = getattr(django.conf.settings, "TIME_ZONE", None) + assert TIME_ZONE_SETTING is not None + assert isinstance(TIME_ZONE_SETTING, str) + TIME_ZONE_SETTING = pytz.timezone(TIME_ZONE_SETTING) + + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("-u", "--turing-url", type=str, required=True, help="The URL of the live turing instance") + parser.add_argument("-t", "--turing-race-id", type=int, required=True, help="ID of the turing race to follow") + parser.add_argument( + "-p", "--turing-race-admin-password", type=str, required=True, + help="The password of the administrator of the turing race") + parser.add_argument( + "-s", "--sleep", type=float, required=False, default=1.0, + help="The amount of time to wait between consecutive turing race exports") + parser.add_argument("-o", "--output-directory", type=str, required=True, help="Path of the output directory") + args = parser.parse_args() + + live_turing_to_html( + args.turing_url, engine.models, args.turing_race_id, args.turing_race_admin_password, + args.sleep, pathlib.Path(args.output_directory), lambda time_counter: datetime.datetime.now(TIME_ZONE_SETTING), + lambda time_counter: False) diff --git a/mathrace_interaction/mathrace_interaction/live_turing_to_live_journal.py b/mathrace_interaction/mathrace_interaction/live_turing_to_live_journal.py index c9ed80b..0067cac 100644 --- a/mathrace_interaction/mathrace_interaction/live_turing_to_live_journal.py +++ b/mathrace_interaction/mathrace_interaction/live_turing_to_live_journal.py @@ -22,7 +22,7 @@ def live_turing_to_live_journal( sleep: float, output_directory: pathlib.Path, termination_condition: typing.Callable[[int], bool] ) -> None: """ - Follow the mathrace journal of a live race, and register all events into a live session in turing. + Follow a live session in turing and convert it into a sequence of mathrace journals. Parameters ---------- @@ -63,8 +63,7 @@ def live_turing_to_live_journal( time_counter_file = output_directory / "time_counter.txt" if time_counter_file.exists(): time_counter = int(time_counter_file.read_text()) - else: - time_counter_file.write_text(str(time_counter)) + time_counter += 1 # Continuously read the turing state while True: diff --git a/mathrace_interaction/pyproject.toml b/mathrace_interaction/pyproject.toml index aaf75d3..dbe5fa9 100644 --- a/mathrace_interaction/pyproject.toml +++ b/mathrace_interaction/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "jsondiff", "paramiko", "prettytable", + "pytz", "requests", "selenium", "tinycss2" @@ -54,6 +55,7 @@ lint = [ "ruff", "types-beautifulsoup4", "types-paramiko", + "types-pytz", "types-requests", "yamllint" ] @@ -89,6 +91,7 @@ warn_unused_ignores = true [[tool.mypy.overrides]] module = [ "django", + "django.conf", "engine", "engine.models", "jsondiff",