From 8c6501947a73642d270673710b6ba864f55ff24e Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Mon, 23 Oct 2023 15:36:32 -0400 Subject: [PATCH] Formatting, gitignore, and poetry --- .gitignore | 2 + samples/ServalApp/db.py | 25 +- samples/ServalApp/pyproject.toml | 18 ++ samples/ServalApp/serval_app.py | 306 +++++++++++++++++------ samples/ServalApp/serval_auth_module.py | 43 ++-- samples/ServalApp/serval_email_module.py | 53 ++-- 6 files changed, 324 insertions(+), 123 deletions(-) create mode 100644 samples/ServalApp/pyproject.toml diff --git a/.gitignore b/.gitignore index 1aed303d..05f01900 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ lib/ .vs appsettings.user.json artifacts + +.db diff --git a/samples/ServalApp/db.py b/samples/ServalApp/db.py index ab59bbce..fc065d23 100644 --- a/samples/ServalApp/db.py +++ b/samples/ServalApp/db.py @@ -2,28 +2,41 @@ from sqlalchemy import Column, MetaData, String, Enum, create_engine import enum + class State(enum.Enum): Pending = 0 Active = 1 Completed = 2 Faulted = 3 + metadata = MetaData() Base = declarative_base(metadata=metadata) + class Build(Base): __tablename__ = "builds" - build_id = Column("build_id",String,primary_key=True) - engine_id = Column("engine_id",String,primary_key=True) - email = Column("email",String) - state = Column("state",Enum(State)) - corpus_id = Column("corpus_id",String) + build_id = Column("build_id", String, primary_key=True) + engine_id = Column("engine_id", String, primary_key=True) + email = Column("email", String) + state = Column("state", Enum(State)) + corpus_id = Column("corpus_id", String) def __str__(self): - return str({'build_id':self.build_id, 'engine_id':self.engine_id,'email':self.email,'state':self.state,'corpus_id':self.corpus_id}) + return str( + { + "build_id": self.build_id, + "engine_id": self.engine_id, + "email": self.email, + "state": self.state, + "corpus_id": self.corpus_id, + } + ) def __repr__(self): return self.__str__() + + def clear_and_regenerate_tables(): engine = create_engine("sqlite:///builds.db") metadata.drop_all(bind=engine) diff --git a/samples/ServalApp/pyproject.toml b/samples/ServalApp/pyproject.toml new file mode 100644 index 00000000..b84afe31 --- /dev/null +++ b/samples/ServalApp/pyproject.toml @@ -0,0 +1,18 @@ +[tool.poetry] +name = "servalapp" +version = "0.1.0" +description = "" +authors = ["Your Name "] +readme = "README.md" + +[tool.poetry.dependencies] +python = ">=3.8,<3.9.7 || >3.9.7,<4.0" +email = "^4.0.2" +streamlit = "^1.27.2" +requests = "^2.31.0" +SQLAlchemy = "^2.0.22" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/samples/ServalApp/serval_app.py b/samples/ServalApp/serval_app.py index 6e64a699..3bd052d4 100644 --- a/samples/ServalApp/serval_app.py +++ b/samples/ServalApp/serval_app.py @@ -12,44 +12,74 @@ from serval_email_module import ServalAppEmailServer import re + def send_emails(): engine = create_engine("sqlite:///builds.db") Session = sessionmaker(bind=engine) session = Session() try: - def started(build:Build, email_server:ServalAppEmailServer, data=None): + + def started(build: Build, email_server: ServalAppEmailServer, data=None): print(f"\tStarted {build}") email_server.send_build_started_email(build.email) session.delete(build) - session.add(Build(build_id=build.build_id, engine_id=build.engine_id, email=build.email, state=State.Active, corpus_id=build.corpus_id)) + session.add( + Build( + build_id=build.build_id, + engine_id=build.engine_id, + email=build.email, + state=State.Active, + corpus_id=build.corpus_id, + ) + ) - def faulted(build:Build, email_server:ServalAppEmailServer, data=None): + def faulted(build: Build, email_server: ServalAppEmailServer, data=None): print(f"\tFaulted {build}") email_server.send_build_faulted_email(build.email, error=data) session.delete(build) - def completed(build:Build, email_server:ServalAppEmailServer, data=None): + def completed(build: Build, email_server: ServalAppEmailServer, data=None): print(f"\tCompleted {build}") - pretranslations = client.translation_engines_get_all_pretranslations(build.engine_id, build.corpus_id) - email_server.send_build_completed_email(build.email, '\n'.join([f"{'|'.join(pretranslation.refs)}\t{pretranslation.translation}" for pretranslation in pretranslations])) + pretranslations = client.translation_engines_get_all_pretranslations( + build.engine_id, build.corpus_id + ) + email_server.send_build_completed_email( + build.email, + "\n".join( + [ + f"{'|'.join(pretranslation.refs)}\t{pretranslation.translation}" + for pretranslation in pretranslations + ] + ), + ) session.delete(build) - def update(build:Build, email_server:ServalAppEmailServer, data=None): + def update(build: Build, email_server: ServalAppEmailServer, data=None): print(f"\tUpdated {build}") serval_auth = ServalBearerAuth() - client = RemoteCaller(url_prefix=os.environ.get('SERVAL_HOST_URL'),auth=serval_auth) - responses:"dict[str,function]" = {"Completed":completed, "Faulted":faulted, "Canceled":faulted} + client = RemoteCaller( + url_prefix=os.environ.get("SERVAL_HOST_URL"), auth=serval_auth + ) + responses: "dict[str,function]" = { + "Completed": completed, + "Faulted": faulted, + "Canceled": faulted, + } - def get_update(build:Build, email_server:ServalAppEmailServer): - build_update = client.translation_engines_get_build(id=build.engine_id, build_id=build.build_id) + def get_update(build: Build, email_server: ServalAppEmailServer): + build_update = client.translation_engines_get_build( + id=build.engine_id, build_id=build.build_id + ) if build.state == State.Pending and build_update.state == "Active": started(build, email_server) else: - responses.get(build_update.state, update)(build, email_server, build_update.message) + responses.get(build_update.state, update)( + build, email_server, build_update.message + ) session.commit() - def send_updates(email_server:ServalAppEmailServer): + def send_updates(email_server: ServalAppEmailServer): print(f"Checking for updates...") with session.no_autoflush: builds = session.query(Build).all() @@ -60,107 +90,229 @@ def send_updates(email_server:ServalAppEmailServer): print(f"\tFailed to update {build} because of exception {e}") raise e - with ServalAppEmailServer(os.environ.get('SERVAL_APP_EMAIL_PASSWORD')) as email_server: - while(True): + with ServalAppEmailServer( + os.environ.get("SERVAL_APP_EMAIL_PASSWORD") + ) as email_server: + while True: send_updates(email_server) - sleep(os.environ.get('SERVAL_APP_UPDATE_FREQ_SEC',300)) + sleep(os.environ.get("SERVAL_APP_UPDATE_FREQ_SEC", 300)) except Exception as e: print(e) - st.session_state['background_process_has_started'] = False + st.session_state["background_process_has_started"] = False + -if not st.session_state.get('background_process_has_started',False): +if not st.session_state.get("background_process_has_started", False): cron_thread = Thread(target=send_emails) add_script_run_ctx(cron_thread) cron_thread.start() - st.session_state['background_process_has_started'] = True + st.session_state["background_process_has_started"] = True serval_auth = None -if not st.session_state.get('authorized',False): +if not st.session_state.get("authorized", False): with st.form(key="Authorization Form"): - st.session_state['client_id'] = st.text_input(label='Client ID') - st.session_state['client_secret'] = st.text_input(label='Client Secret', type='password') + st.session_state["client_id"] = st.text_input(label="Client ID") + st.session_state["client_secret"] = st.text_input( + label="Client Secret", type="password" + ) if st.form_submit_button("Authorize"): - st.session_state['authorized'] = True + st.session_state["authorized"] = True st.rerun() - if st.session_state.get('authorization_failure', False): - st.error('Invalid credentials. Please check your credentials.') + if st.session_state.get("authorization_failure", False): + st.error("Invalid credentials. Please check your credentials.") else: try: - serval_auth = ServalBearerAuth(client_id=st.session_state['client_id'] if st.session_state['client_id'] != "" else "", client_secret=st.session_state['client_secret'] if st.session_state['client_secret'] != "" else "") + serval_auth = ServalBearerAuth( + client_id=st.session_state["client_id"] + if st.session_state["client_id"] != "" + else "", + client_secret=st.session_state["client_secret"] + if st.session_state["client_secret"] != "" + else "", + ) except ValueError: - st.session_state['authorized'] = False - st.session_state['authorization_failure'] = True + st.session_state["authorized"] = False + st.session_state["authorization_failure"] = True st.rerun() - client = RemoteCaller(url_prefix="https://prod.serval-api.org",auth=serval_auth) + client = RemoteCaller(url_prefix="https://prod.serval-api.org", auth=serval_auth) engine = create_engine("sqlite:///builds.db") Session = sessionmaker(bind=engine) session = Session() def submit(): - engine = json.loads(client.translation_engines_create(TranslationEngineConfig(source_language=st.session_state['source_language'],target_language=st.session_state['target_language'],type='Nmt',name=f'serval_app_engine:{st.session_state["email"]}'))) - source_files = [json.loads(client.data_files_create(st.session_state['source_files'][i], format="Paratext" if st.session_state['source_files'][i].name[-4:] == '.zip' else "Text")) for i in range(len(st.session_state['source_files']))] - target_files = [json.loads(client.data_files_create(st.session_state['target_files'][i], format="Paratext" if st.session_state['target_files'][i].name[-4:] == '.zip' else "Text")) for i in range(len(st.session_state['target_files']))] - corpus = json.loads(client.translation_engines_add_corpus( - engine['id'], - TranslationCorpusConfig( - source_files=[TranslationCorpusFileConfig(file_id=file['id'], text_id=name) for file, name in zip(source_files, list(map(lambda f: f.name, st.session_state['source_files'])))], - target_files=[TranslationCorpusFileConfig(file_id=file['id'], text_id=name) for file, name in zip(target_files, list(map(lambda f: f.name, st.session_state['target_files'])))], - source_language=st.session_state['source_language'], - target_language=st.session_state['target_language'] + engine = json.loads( + client.translation_engines_create( + TranslationEngineConfig( + source_language=st.session_state["source_language"], + target_language=st.session_state["target_language"], + type="Nmt", + name=f'serval_app_engine:{st.session_state["email"]}', ) ) ) - build = json.loads(client.translation_engines_start_build(engine['id'], TranslationBuildConfig(pretranslate=[PretranslateCorpusConfig(corpus_id=corpus["id"], text_ids= [] if st.session_state['source_files'][0].name[-4:] == '.zip' else list(map(lambda f: f.name, st.session_state['source_files'])))], options="{\"max_steps\":" + os.environ.get('SERVAL_APP_MAX_STEPS',10) + "}"))) - session.add(Build(build_id=build['id'],engine_id=engine['id'],email=st.session_state['email'],state=build['state'],corpus_id=corpus['id'])) + source_files = [ + json.loads( + client.data_files_create( + st.session_state["source_files"][i], + format="Paratext" + if st.session_state["source_files"][i].name[-4:] == ".zip" + else "Text", + ) + ) + for i in range(len(st.session_state["source_files"])) + ] + target_files = [ + json.loads( + client.data_files_create( + st.session_state["target_files"][i], + format="Paratext" + if st.session_state["target_files"][i].name[-4:] == ".zip" + else "Text", + ) + ) + for i in range(len(st.session_state["target_files"])) + ] + corpus = json.loads( + client.translation_engines_add_corpus( + engine["id"], + TranslationCorpusConfig( + source_files=[ + TranslationCorpusFileConfig(file_id=file["id"], text_id=name) + for file, name in zip( + source_files, + list( + map(lambda f: f.name, st.session_state["source_files"]) + ), + ) + ], + target_files=[ + TranslationCorpusFileConfig(file_id=file["id"], text_id=name) + for file, name in zip( + target_files, + list( + map(lambda f: f.name, st.session_state["target_files"]) + ), + ) + ], + source_language=st.session_state["source_language"], + target_language=st.session_state["target_language"], + ), + ) + ) + build = json.loads( + client.translation_engines_start_build( + engine["id"], + TranslationBuildConfig( + pretranslate=[ + PretranslateCorpusConfig( + corpus_id=corpus["id"], + text_ids=[] + if st.session_state["source_files"][0].name[-4:] == ".zip" + else list( + map(lambda f: f.name, st.session_state["source_files"]) + ), + ) + ], + options='{"max_steps":' + + os.environ.get("SERVAL_APP_MAX_STEPS", 10) + + "}", + ), + ) + ) + session.add( + Build( + build_id=build["id"], + engine_id=engine["id"], + email=st.session_state["email"], + state=build["state"], + corpus_id=corpus["id"], + ) + ) session.commit() - def already_active_build_for(email:str): + def already_active_build_for(email: str): return len(session.query(Build).where(Build.email == email).all()) > 0 st.subheader("Neural Machine Translation") - tried_to_submit = st.session_state.get('tried_to_submit', False) + tried_to_submit = st.session_state.get("tried_to_submit", False) with st.form(key="NmtTranslationForm"): - st.session_state['source_language'] = st.text_input(label="Source language tag*", placeholder="en") - if st.session_state.get('source_language','') == '' and tried_to_submit: - st.error("Please enter a source language tag before submitting", icon='⬆️') - - st.session_state['source_files'] = st.file_uploader(label="Source File(s)", accept_multiple_files=True) - if len(st.session_state.get('source_files',[])) == 0 and tried_to_submit: - st.error("Please upload a source file before submitting", icon='⬆️') - if len(st.session_state.get('source_files',[])) > 1: - st.warning('Please note that source and target text files will be paired together by file name', icon='💡') - - st.session_state['target_language'] = st.text_input(label="Target language tag*", placeholder="es") - if st.session_state.get('target_language','') == '' and tried_to_submit: - st.error("Please enter a target language tag before submitting", icon='⬆️') - - st.session_state['target_files'] = st.file_uploader(label="Target File(s)", accept_multiple_files=True) - if len(st.session_state.get('target_files',[])) > 1: - st.warning('Please note that source and target text files will be paired together by file name', icon='💡') - - st.session_state['email'] = st.text_input(label="Email", placeholder="johndoe@example.com") - if st.session_state.get('email','') == '' and tried_to_submit: - st.error("Please enter an email address", icon='⬆️') - elif not re.match(r"^\S+@\S+\.\S+$", st.session_state['email']) and tried_to_submit: - st.error("Please enter a valid email address", icon='⬆️') - st.session_state['email'] = '' + st.session_state["source_language"] = st.text_input( + label="Source language tag*", placeholder="en" + ) + if st.session_state.get("source_language", "") == "" and tried_to_submit: + st.error("Please enter a source language tag before submitting", icon="⬆️") + + st.session_state["source_files"] = st.file_uploader( + label="Source File(s)", accept_multiple_files=True + ) + if len(st.session_state.get("source_files", [])) == 0 and tried_to_submit: + st.error("Please upload a source file before submitting", icon="⬆️") + if len(st.session_state.get("source_files", [])) > 1: + st.warning( + "Please note that source and target text files will be paired together by file name", + icon="💡", + ) + + st.session_state["target_language"] = st.text_input( + label="Target language tag*", placeholder="es" + ) + if st.session_state.get("target_language", "") == "" and tried_to_submit: + st.error("Please enter a target language tag before submitting", icon="⬆️") + + st.session_state["target_files"] = st.file_uploader( + label="Target File(s)", accept_multiple_files=True + ) + if len(st.session_state.get("target_files", [])) > 1: + st.warning( + "Please note that source and target text files will be paired together by file name", + icon="💡", + ) + + st.session_state["email"] = st.text_input( + label="Email", placeholder="johndoe@example.com" + ) + if st.session_state.get("email", "") == "" and tried_to_submit: + st.error("Please enter an email address", icon="⬆️") + elif ( + not re.match(r"^\S+@\S+\.\S+$", st.session_state["email"]) + and tried_to_submit + ): + st.error("Please enter a valid email address", icon="⬆️") + st.session_state["email"] = "" if tried_to_submit: - st.error(st.session_state.get('error',"Something went wrong. Please try again in a moment.")) + st.error( + st.session_state.get( + "error", "Something went wrong. Please try again in a moment." + ) + ) if st.form_submit_button("Generate translations"): - if already_active_build_for(st.session_state['email']): - st.session_state['tried_to_submit'] = True - st.session_state['error'] = "There is already an a pending or active build associated with this email address. Please wait for the previous build to finish." + if already_active_build_for(st.session_state["email"]): + st.session_state["tried_to_submit"] = True + st.session_state[ + "error" + ] = "There is already an a pending or active build associated with this email address. Please wait for the previous build to finish." st.rerun() - elif st.session_state['source_language'] != '' and st.session_state['target_language'] != '' and len(st.session_state['source_files']) > 0 and st.session_state['email'] != '': + elif ( + st.session_state["source_language"] != "" + and st.session_state["target_language"] != "" + and len(st.session_state["source_files"]) > 0 + and st.session_state["email"] != "" + ): with st.spinner(): submit() - st.session_state['tried_to_submit'] = False - st.toast("Translations are on their way! You'll receive an email when your translation job has begun.") + st.session_state["tried_to_submit"] = False + st.toast( + "Translations are on their way! You'll receive an email when your translation job has begun." + ) sleep(4) st.rerun() else: - st.session_state['tried_to_submit'] = True - st.session_state['error'] = "Some required fields were left blank. Please fill in all fields above" + st.session_state["tried_to_submit"] = True + st.session_state[ + "error" + ] = "Some required fields were left blank. Please fill in all fields above" st.rerun() - st.markdown("\* Use IETF tags if possible. See [here](https://en.wikipedia.org/wiki/IETF_language_tag) for more information on IETF tags.", unsafe_allow_html=True) \ No newline at end of file + st.markdown( + "\* Use IETF tags if possible. See [here](https://en.wikipedia.org/wiki/IETF_language_tag) for more information on IETF tags.", + unsafe_allow_html=True, + ) diff --git a/samples/ServalApp/serval_auth_module.py b/samples/ServalApp/serval_auth_module.py index dbb53ea6..48020bfa 100644 --- a/samples/ServalApp/serval_auth_module.py +++ b/samples/ServalApp/serval_auth_module.py @@ -3,18 +3,26 @@ import os import time + class ServalBearerAuth(requests.auth.AuthBase): def __init__(self, client_id="", client_secret=""): - self.__client_id = client_id if client_id != "" else os.environ.get("SERVAL_CLIENT_ID") - assert(self.__client_id is not None) - self.__client_secret = client_secret if client_secret != "" else os.environ.get("SERVAL_CLIENT_SECRET") - assert(self.__client_secret is not None) + self.__client_id = ( + client_id if client_id != "" else os.environ.get("SERVAL_CLIENT_ID") + ) + assert self.__client_id is not None + self.__client_secret = ( + client_secret + if client_secret != "" + else os.environ.get("SERVAL_CLIENT_SECRET") + ) + assert self.__client_secret is not None self.__auth_url = os.environ.get("SERVAL_AUTH_URL") - assert(self.__auth_url is not None) + assert self.__auth_url is not None self.update_token() self.__last_time_fetched = time.time() + def __call__(self, r): - if(time.time() - self.__last_time_fetched > 20*60): + if time.time() - self.__last_time_fetched > 20 * 60: self.update_token() r.headers["authorization"] = "Bearer " + self.token return r @@ -22,20 +30,21 @@ def __call__(self, r): def update_token(self): data = { "client_id": f"{self.__client_id}", - "client_secret":f"{self.__client_secret}", - "audience":"https://machine.sil.org", - "grant_type":"client_credentials" - } + "client_secret": f"{self.__client_secret}", + "audience": "https://machine.sil.org", + "grant_type": "client_credentials", + } - encoded_data = json.dumps(data).encode('utf-8') + encoded_data = json.dumps(data).encode("utf-8") r = None try: - r:requests.Response = requests.post( - url=f'{self.__auth_url}/oauth/token', + r: requests.Response = requests.post( + url=f"{self.__auth_url}/oauth/token", data=encoded_data, - headers={"content-type": "application/json"} + headers={"content-type": "application/json"}, ) - self.token = r.json()['access_token'] if r is not None else None + self.token = r.json()["access_token"] if r is not None else None except Exception as e: - raise ValueError(f"Token cannot be None. Failed to retrieve token from auth server; responded with {r.status_code if r is not None else ''}. Original exception: {e}") - + raise ValueError( + f"Token cannot be None. Failed to retrieve token from auth server; responded with {r.status_code if r is not None else ''}. Original exception: {e}" + ) diff --git a/samples/ServalApp/serval_email_module.py b/samples/ServalApp/serval_email_module.py index 5876e622..1a5cfc20 100644 --- a/samples/ServalApp/serval_email_module.py +++ b/samples/ServalApp/serval_email_module.py @@ -1,8 +1,15 @@ from email.message import EmailMessage import smtplib, ssl + class ServalAppEmailServer: - def __init__(self, password, sender_address = 'serval-app@languagetechnology.org', host='mail.languagetechnology.org', port=465) -> None: + def __init__( + self, + password, + sender_address="serval-app@languagetechnology.org", + host="mail.languagetechnology.org", + port=465, + ) -> None: self.__password = password self.sender_address = sender_address self.host = host @@ -11,7 +18,7 @@ def __init__(self, password, sender_address = 'serval-app@languagetechnology.org @property def password(self): - return len(self.__password)*"*" + return len(self.__password) * "*" def __enter__(self): context = ssl.create_default_context() @@ -22,57 +29,57 @@ def __enter__(self): def __exit__(self, *args): self.server.close() - def send_build_completed_email(self, recipient_address:str, pretranslations_file_data:str): + def send_build_completed_email( + self, recipient_address: str, pretranslations_file_data: str + ): msg = EmailMessage() msg.set_content( -'''Hi! + """Hi! Your NMT engine has completed building. Attached are the translations of untranslated source text in the files you included. If you are experiencing difficulties using this application, please contact eli_lowry@sil.org. Thank you! -''' +""" ) - msg['From'] = self.sender_address - msg['To'] = recipient_address - msg['Subject'] = 'Your NMT build job is complete!' - msg.add_attachment(pretranslations_file_data, filename='translations.txt') + msg["From"] = self.sender_address + msg["To"] = recipient_address + msg["Subject"] = "Your NMT build job is complete!" + msg.add_attachment(pretranslations_file_data, filename="translations.txt") self.server.send_message(msg) - def send_build_faulted_email(self, recipient_address:str, error=""): + def send_build_faulted_email(self, recipient_address: str, error=""): msg = EmailMessage() msg.set_content( -f'''Hi! + f"""Hi! Your NMT engine has failed to build{" with the following error message: " + error if error != "" else ""}. Please make sure the information you specified is correct and try again after a while. If you continue to experience difficulties using this application, please contact eli_lowry@sil.org. Thank you! -''' +""" ) - msg['From'] = self.sender_address - msg['To'] = recipient_address - msg['Subject'] = 'Your NMT build job has failed' + msg["From"] = self.sender_address + msg["To"] = recipient_address + msg["Subject"] = "Your NMT build job has failed" self.server.send_message(msg) - def send_build_started_email(self, recipient_address:str): + def send_build_started_email(self, recipient_address: str): msg = EmailMessage() msg.set_content( -'''Hi! + """Hi! Your NMT engine has started building. We will contact you when it is complete. If you are experiencing difficulties using this application, please contact eli_lowry@sil.org. Thank you! -''' +""" ) - msg['From'] = self.sender_address - msg['To'] = recipient_address - msg['Subject'] = 'Your NMT build job has started building!' + msg["From"] = self.sender_address + msg["To"] = recipient_address + msg["Subject"] = "Your NMT build job has started building!" self.server.send_message(msg) - -