From bfa3be98baac77dc699815282f0d84a03efbe634 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 24 Oct 2023 10:32:43 -0400 Subject: [PATCH] Remove db, add flake8 etc. --- .vscode/settings.json | 3 ++ samples/ServalApp/.flake8 | 5 +++ samples/ServalApp/builds.db | Bin 20480 -> 0 bytes samples/ServalApp/db.py | 8 ++--- samples/ServalApp/pyproject.toml | 2 +- samples/ServalApp/serval_app.py | 40 ++++++++++++++-------- samples/ServalApp/serval_auth_module.py | 6 ++-- samples/ServalApp/serval_client_module.py | 35 ++++++------------- samples/ServalApp/serval_email_module.py | 9 +++-- 9 files changed, 59 insertions(+), 49 deletions(-) create mode 100644 samples/ServalApp/.flake8 delete mode 100644 samples/ServalApp/builds.db diff --git a/.vscode/settings.json b/.vscode/settings.json index d2907b44..5d6ec7eb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -30,4 +30,7 @@ } ], "dotnet.defaultSolution": "Serval.sln", + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + } } \ No newline at end of file diff --git a/samples/ServalApp/.flake8 b/samples/ServalApp/.flake8 new file mode 100644 index 00000000..ce6af763 --- /dev/null +++ b/samples/ServalApp/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 120 +per-file-ignores = serval_app.py:F821,W605 +exclude = + serval_client_module.py \ No newline at end of file diff --git a/samples/ServalApp/builds.db b/samples/ServalApp/builds.db deleted file mode 100644 index d0e91a7f0fb9dcc2820cd668fd61e8a3d81eb4ab..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20480 zcmeI(Jx{_w7{Ku>ib@j#8)3V|1tOXl=#^`X#7_P_p7;=vgb)dlc+=;M@jOBR0R#|0009ILKmY**5cnekD%E@W2sDi`IbPiD;6uvPb(^4hWsCCiqT zv#P9ImCHG4#@tTJ>$u-OZ9ASnc~1`yLPura1Qj?cMi-sVQ{lxqW=8 diff --git a/samples/ServalApp/db.py b/samples/ServalApp/db.py index fc065d23..e205bd9a 100644 --- a/samples/ServalApp/db.py +++ b/samples/ServalApp/db.py @@ -1,7 +1,8 @@ -from sqlalchemy.orm import declarative_base -from sqlalchemy import Column, MetaData, String, Enum, create_engine import enum +from sqlalchemy import Column, Enum, MetaData, String, create_engine +from sqlalchemy.orm import declarative_base + class State(enum.Enum): Pending = 0 @@ -37,7 +38,6 @@ def __repr__(self): return self.__str__() -def clear_and_regenerate_tables(): +def create_db_if_not_exists(): engine = create_engine("sqlite:///builds.db") - metadata.drop_all(bind=engine) metadata.create_all(bind=engine) diff --git a/samples/ServalApp/pyproject.toml b/samples/ServalApp/pyproject.toml index b84afe31..536c96ce 100644 --- a/samples/ServalApp/pyproject.toml +++ b/samples/ServalApp/pyproject.toml @@ -2,7 +2,7 @@ name = "servalapp" version = "0.1.0" description = "" -authors = ["Your Name "] +authors = ["Eli Lowry "] readme = "README.md" [tool.poetry.dependencies] diff --git a/samples/ServalApp/serval_app.py b/samples/ServalApp/serval_app.py index 3bd052d4..6abe56b8 100644 --- a/samples/ServalApp/serval_app.py +++ b/samples/ServalApp/serval_app.py @@ -1,16 +1,21 @@ +import json +import os +import re +import traceback +from threading import Thread +from time import sleep + import streamlit as st -from streamlit.runtime.scriptrunner import add_script_run_ctx -from serval_client_module import * -from serval_auth_module import * +from db import Build, State, create_db_if_not_exists +from serval_auth_module import ServalBearerAuth +from serval_client_module import (PretranslateCorpusConfig, RemoteCaller, TranslationBuildConfig, + TranslationCorpusConfig, TranslationCorpusFileConfig, TranslationEngineConfig) +from serval_email_module import ServalAppEmailServer from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from db import Build -from time import sleep -from threading import Thread -import os -from db import Build, State -from serval_email_module import ServalAppEmailServer -import re +from streamlit.runtime.scriptrunner import add_script_run_ctx + +create_db_if_not_exists() def send_emails(): @@ -80,7 +85,7 @@ def get_update(build: Build, email_server: ServalAppEmailServer): session.commit() def send_updates(email_server: ServalAppEmailServer): - print(f"Checking for updates...") + print("Checking for updates...") with session.no_autoflush: builds = session.query(Build).all() for build in builds: @@ -88,6 +93,7 @@ def send_updates(email_server: ServalAppEmailServer): get_update(build, email_server) except Exception as e: print(f"\tFailed to update {build} because of exception {e}") + traceback.print_exc() raise e with ServalAppEmailServer( @@ -133,7 +139,9 @@ def send_updates(email_server: ServalAppEmailServer): st.session_state["authorized"] = False st.session_state["authorization_failure"] = True st.rerun() - client = RemoteCaller(url_prefix="https://prod.serval-api.org", auth=serval_auth) + client = RemoteCaller( + url_prefix=os.environ.get("SERVAL_HOST_URL"), auth=serval_auth + ) engine = create_engine("sqlite:///builds.db") Session = sessionmaker(bind=engine) session = Session() @@ -213,7 +221,7 @@ def submit(): ) ], options='{"max_steps":' - + os.environ.get("SERVAL_APP_MAX_STEPS", 10) + + str(os.environ.get("SERVAL_APP_MAX_STEPS", 10)) + "}", ), ) @@ -290,7 +298,8 @@ def already_active_build_for(email: str): st.session_state["tried_to_submit"] = True st.session_state[ "error" - ] = "There is already an a pending or active build associated with this email address. Please wait for the previous build to finish." + ] = "There is already an a pending or active build associated with this email address. \ + Please wait for the previous build to finish." st.rerun() elif ( st.session_state["source_language"] != "" @@ -313,6 +322,7 @@ def already_active_build_for(email: str): ] = "Some required fields were left blank. Please fill in all fields above" st.rerun() st.markdown( - "\* Use IETF tags if possible. See [here](https://en.wikipedia.org/wiki/IETF_language_tag) for more information on IETF tags.", + "\* Use IETF tags if possible. See [here](https://en.wikipedia.org/wiki/IETF_language_tag) \ + for more information on IETF tags.", unsafe_allow_html=True, ) diff --git a/samples/ServalApp/serval_auth_module.py b/samples/ServalApp/serval_auth_module.py index 48020bfa..0fd265a8 100644 --- a/samples/ServalApp/serval_auth_module.py +++ b/samples/ServalApp/serval_auth_module.py @@ -1,8 +1,9 @@ -import requests import json import os import time +import requests + class ServalBearerAuth(requests.auth.AuthBase): def __init__(self, client_id="", client_secret=""): @@ -46,5 +47,6 @@ def update_token(self): self.token = r.json()["access_token"] if r is not None else None except Exception as e: raise ValueError( - f"Token cannot be None. Failed to retrieve token from auth server; responded with {r.status_code if r is not None else ''}. Original exception: {e}" + f"Token cannot be None. Failed to retrieve token from auth server; responded \ + with {r.status_code if r is not None else ''}. Original exception: {e}" ) diff --git a/samples/ServalApp/serval_client_module.py b/samples/ServalApp/serval_client_module.py index a44f0871..699fa5e1 100644 --- a/samples/ServalApp/serval_client_module.py +++ b/samples/ServalApp/serval_client_module.py @@ -2173,7 +2173,7 @@ def __init__( message: Optional[str] = None, queue_depth: Optional[int] = None, date_finished: Optional[str] = None, - options: Optional[str] = None) -> None: + options: Optional[Any] = None) -> None: """Initializes with the given values.""" self.id = id @@ -2322,14 +2322,7 @@ def translation_build_from_obj(obj: Any, path: str = "") -> TranslationBuild: else: date_finished_from_obj = None - obj_options = obj.get('options', None) - if obj_options is not None: - options_from_obj = from_obj( - obj_options, - expected=[str], - path=path + '.options') # type: Optional[str] - else: - options_from_obj = None + options_from_obj = obj.get('options', None) return TranslationBuild( id=id_from_obj, @@ -2492,7 +2485,7 @@ def __init__( self, name: Optional[str] = None, pretranslate: Optional[List['PretranslateCorpusConfig']] = None, - options: Optional[str] = None) -> None: + options: Optional[Any] = None) -> None: """Initializes with the given values.""" self.name = name @@ -2548,14 +2541,7 @@ def translation_build_config_from_obj(obj: Any, path: str = "") -> TranslationBu else: pretranslate_from_obj = None - obj_options = obj.get('options', None) - if obj_options is not None: - options_from_obj = from_obj( - obj_options, - expected=[str], - path=path + '.options') # type: Optional[str] - else: - options_from_obj = None + options_from_obj = obj.get('options', None) return TranslationBuildConfig( name=name_from_obj, @@ -3166,7 +3152,7 @@ def translation_engines_get_queue( self, engine_type: str) -> 'Queue': """ - Send a get request to /api/v1/translation/engines/queues. + Send a post request to /api/v1/translation/engines/queues. :param engine_type: A valid engine type: SmtTransfer, Nmt, or Echo @@ -3178,7 +3164,7 @@ def translation_engines_get_queue( resp = self.session.request( - method='get', + method='post', url=url, json=data, ) @@ -3572,7 +3558,7 @@ def translation_engines_start_build( self, id: str, build_config: 'TranslationBuildConfig') -> bytes: - r""" + """ Specify the corpora or textIds to pretranslate. Even when a corpus or textId is selected for pretranslation, only "untranslated" text will be pretranslated: that is, segments (lines of text) in the specified corpora or textId's that have @@ -3580,8 +3566,8 @@ def translation_engines_start_build( you may flag a subset of books for pretranslation by including their [abbreviations](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs) in the textIds parameter. If the engine does not support pretranslation, these fields have no effect. - The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON string. - A typical use case would be to set `"options"` to `"{\"max_steps\":10}"` in order to configure the maximum + The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. + A typical use case would be to set `"options"` to `{"max_steps":10}` in order to configure the maximum number of training iterations in order to reduce turnaround time for testing purposes. :param id: The translation engine id @@ -3622,7 +3608,8 @@ def translation_engines_get_build( will timeout. A use case is to actively query the state of the current build, where the subsequent request sets the `minRevision` to the returned `revision` + 1 and timeouts are handled gracefully. - Note: this method should use request throttling. + This method should use request throttling. + Note: Within the returned build, percentCompleted is a value between 0 and 1. :param id: The translation engine id :param build_id: The build job id diff --git a/samples/ServalApp/serval_email_module.py b/samples/ServalApp/serval_email_module.py index 1a5cfc20..2ad8c38a 100644 --- a/samples/ServalApp/serval_email_module.py +++ b/samples/ServalApp/serval_email_module.py @@ -1,5 +1,6 @@ +import smtplib +import ssl from email.message import EmailMessage -import smtplib, ssl class ServalAppEmailServer: @@ -36,7 +37,8 @@ def send_build_completed_email( msg.set_content( """Hi! -Your NMT engine has completed building. Attached are the translations of untranslated source text in the files you included. +Your NMT engine has completed building. Attached are the \ + translations of untranslated source text in the files you included. If you are experiencing difficulties using this application, please contact eli_lowry@sil.org. @@ -54,7 +56,8 @@ def send_build_faulted_email(self, recipient_address: str, error=""): msg.set_content( f"""Hi! -Your NMT engine has failed to build{" with the following error message: " + error if error != "" else ""}. Please make sure the information you specified is correct and try again after a while. +Your NMT engine has failed to build{" with the following error message: " + error if error != "" else ""}. \ + Please make sure the information you specified is correct and try again after a while. If you continue to experience difficulties using this application, please contact eli_lowry@sil.org.