Remove db, add flake8 etc.

.vscode/settings.json

@@ -30,4 +30,7 @@
     }
   ],
   "dotnet.defaultSolution": "Serval.sln",
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter"
+  }
 }

samples/ServalApp/.flake8

@@ -0,0 +1,5 @@
+[flake8]
+max-line-length = 120
+per-file-ignores = serval_app.py:F821,W605
+exclude =
+    serval_client_module.py

samples/ServalApp/db.py

@@ -1,7 +1,8 @@
-from sqlalchemy.orm import declarative_base
-from sqlalchemy import Column, MetaData, String, Enum, create_engine
 import enum
 
+from sqlalchemy import Column, Enum, MetaData, String, create_engine
+from sqlalchemy.orm import declarative_base
+
 
 class State(enum.Enum):
     Pending = 0
@@ -37,7 +38,6 @@ def __repr__(self):
         return self.__str__()
 
 
-def clear_and_regenerate_tables():
+def create_db_if_not_exists():
     engine = create_engine("sqlite:///builds.db")
-    metadata.drop_all(bind=engine)
     metadata.create_all(bind=engine)

samples/ServalApp/pyproject.toml

@@ -2,7 +2,7 @@
 name = "servalapp"
 version = "0.1.0"
 description = ""
-authors = ["Your Name <[email protected]>"]
+authors = ["Eli Lowry <[email protected]>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]

samples/ServalApp/serval_app.py

@@ -1,16 +1,21 @@
+import json
+import os
+import re
+import traceback
+from threading import Thread
+from time import sleep
+
 import streamlit as st
-from streamlit.runtime.scriptrunner import add_script_run_ctx
-from serval_client_module import *
-from serval_auth_module import *
+from db import Build, State, create_db_if_not_exists
+from serval_auth_module import ServalBearerAuth
+from serval_client_module import (PretranslateCorpusConfig, RemoteCaller, TranslationBuildConfig,
+                                  TranslationCorpusConfig, TranslationCorpusFileConfig, TranslationEngineConfig)
+from serval_email_module import ServalAppEmailServer
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
-from db import Build
-from time import sleep
-from threading import Thread
-import os
-from db import Build, State
-from serval_email_module import ServalAppEmailServer
-import re
+from streamlit.runtime.scriptrunner import add_script_run_ctx
+
+create_db_if_not_exists()
 
 
 def send_emails():
@@ -80,14 +85,15 @@ def get_update(build: Build, email_server: ServalAppEmailServer):
             session.commit()
 
         def send_updates(email_server: ServalAppEmailServer):
-            print(f"Checking for updates...")
+            print("Checking for updates...")
             with session.no_autoflush:
                 builds = session.query(Build).all()
                 for build in builds:
                     try:
                         get_update(build, email_server)
                     except Exception as e:
                         print(f"\tFailed to update {build} because of exception {e}")
+                        traceback.print_exc()
                         raise e
 
         with ServalAppEmailServer(
@@ -133,7 +139,9 @@ def send_updates(email_server: ServalAppEmailServer):
         st.session_state["authorized"] = False
         st.session_state["authorization_failure"] = True
         st.rerun()
-    client = RemoteCaller(url_prefix="https://prod.serval-api.org", auth=serval_auth)
+    client = RemoteCaller(
+        url_prefix=os.environ.get("SERVAL_HOST_URL"), auth=serval_auth
+    )
     engine = create_engine("sqlite:///builds.db")
     Session = sessionmaker(bind=engine)
     session = Session()
@@ -213,7 +221,7 @@ def submit():
                         )
                     ],
                     options='{"max_steps":'
-                    + os.environ.get("SERVAL_APP_MAX_STEPS", 10)
+                    + str(os.environ.get("SERVAL_APP_MAX_STEPS", 10))
                     + "}",
                 ),
             )
@@ -290,7 +298,8 @@ def already_active_build_for(email: str):
                 st.session_state["tried_to_submit"] = True
                 st.session_state[
                     "error"
-                ] = "There is already an a pending or active build associated with this email address. Please wait for the previous build to finish."
+                ] = "There is already an a pending or active build associated with this email address. \
+                    Please wait for the previous build to finish."
                 st.rerun()
             elif (
                 st.session_state["source_language"] != ""
@@ -313,6 +322,7 @@ def already_active_build_for(email: str):
                 ] = "Some required fields were left blank. Please fill in all fields above"
                 st.rerun()
         st.markdown(
-            "<sub>\* Use IETF tags if possible. See [here](https://en.wikipedia.org/wiki/IETF_language_tag) for more information on IETF tags.</sub>",
+            "<sub>\* Use IETF tags if possible. See [here](https://en.wikipedia.org/wiki/IETF_language_tag) \
+                for more information on IETF tags.</sub>",
             unsafe_allow_html=True,
         )

samples/ServalApp/serval_auth_module.py

@@ -1,8 +1,9 @@
-import requests
 import json
 import os
 import time
 
+import requests
+
 
 class ServalBearerAuth(requests.auth.AuthBase):
     def __init__(self, client_id="", client_secret=""):
@@ -46,5 +47,6 @@ def update_token(self):
             self.token = r.json()["access_token"] if r is not None else None
         except Exception as e:
             raise ValueError(
-                f"Token cannot be None. Failed to retrieve token from auth server; responded with {r.status_code if r is not None else '<unknown>'}. Original exception: {e}"
+                f"Token cannot be None. Failed to retrieve token from auth server; responded \
+                    with {r.status_code if r is not None else '<unknown>'}. Original exception: {e}"
             )

samples/ServalApp/serval_client_module.py

@@ -2173,7 +2173,7 @@ def __init__(
             message: Optional[str] = None,
             queue_depth: Optional[int] = None,
             date_finished: Optional[str] = None,
-            options: Optional[str] = None) -> None:
+            options: Optional[Any] = None) -> None:
         """Initializes with the given values."""
         self.id = id
 
@@ -2322,14 +2322,7 @@ def translation_build_from_obj(obj: Any, path: str = "") -> TranslationBuild:
     else:
         date_finished_from_obj = None
 
-    obj_options = obj.get('options', None)
-    if obj_options is not None:
-        options_from_obj = from_obj(
-            obj_options,
-            expected=[str],
-            path=path + '.options')  # type: Optional[str]
-    else:
-        options_from_obj = None
+    options_from_obj = obj.get('options', None)
 
     return TranslationBuild(
         id=id_from_obj,
@@ -2492,7 +2485,7 @@ def __init__(
             self,
             name: Optional[str] = None,
             pretranslate: Optional[List['PretranslateCorpusConfig']] = None,
-            options: Optional[str] = None) -> None:
+            options: Optional[Any] = None) -> None:
         """Initializes with the given values."""
         self.name = name
 
@@ -2548,14 +2541,7 @@ def translation_build_config_from_obj(obj: Any, path: str = "") -> TranslationBu
     else:
         pretranslate_from_obj = None
 
-    obj_options = obj.get('options', None)
-    if obj_options is not None:
-        options_from_obj = from_obj(
-            obj_options,
-            expected=[str],
-            path=path + '.options')  # type: Optional[str]
-    else:
-        options_from_obj = None
+    options_from_obj = obj.get('options', None)
 
     return TranslationBuildConfig(
         name=name_from_obj,
@@ -3166,7 +3152,7 @@ def translation_engines_get_queue(
             self,
             engine_type: str) -> 'Queue':
         """
-        Send a get request to /api/v1/translation/engines/queues.
+        Send a post request to /api/v1/translation/engines/queues.
 
         :param engine_type: A valid engine type: SmtTransfer, Nmt, or Echo
 
@@ -3178,7 +3164,7 @@ def translation_engines_get_queue(
 
 
         resp = self.session.request(
-            method='get',
+            method='post',
             url=url,
             json=data,
         )
@@ -3572,16 +3558,16 @@ def translation_engines_start_build(
             self,
             id: str,
             build_config: 'TranslationBuildConfig') -> bytes:
-        r"""
+        """
         Specify the corpora or textIds to pretranslate.  Even when a corpus or textId
         is selected for pretranslation, only "untranslated" text will be pretranslated:
         that is, segments (lines of text) in the specified corpora or textId's that have
         untranslated text but no translated text. If a corpus is a Paratext project,
         you may flag a subset of books for pretranslation by including their [abbreviations](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs)
         in the textIds parameter. If the engine does not support pretranslation, these fields have no effect.
                     
-        The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON string.
-        A typical use case would be to set `"options"` to `"{\"max_steps\":10}"` in order to configure the maximum
+        The `"options"` parameter of the build config provides the ability to pass build configuration parameters as a JSON object.
+        A typical use case would be to set `"options"` to `{"max_steps":10}` in order to configure the maximum
         number of training iterations in order to reduce turnaround time for testing purposes.
 
         :param id: The translation engine id
@@ -3622,7 +3608,8 @@ def translation_engines_get_build(
         will timeout.
         A use case is to actively query the state of the current build, where the subsequent
         request sets the `minRevision` to the returned `revision` + 1 and timeouts are handled gracefully.
-        Note: this method should use request throttling.
+        This method should use request throttling.
+        Note: Within the returned build, percentCompleted is a value between 0 and 1.
 
         :param id: The translation engine id
         :param build_id: The build job id

samples/ServalApp/serval_email_module.py

@@ -1,5 +1,6 @@
+import smtplib
+import ssl
 from email.message import EmailMessage
-import smtplib, ssl
 
 
 class ServalAppEmailServer:
@@ -36,7 +37,8 @@ def send_build_completed_email(
         msg.set_content(
             """Hi!
 
-Your NMT engine has completed building. Attached are the translations of untranslated source text in the files you included.
+Your NMT engine has completed building. Attached are the \
+    translations of untranslated source text in the files you included.
 
 If you are experiencing difficulties using this application, please contact [email protected].
 
@@ -54,7 +56,8 @@ def send_build_faulted_email(self, recipient_address: str, error=""):
         msg.set_content(
             f"""Hi!
 
-Your NMT engine has failed to build{" with the following error message: " + error if error != "" else ""}. Please make sure the information you specified is correct and try again after a while.
+Your NMT engine has failed to build{" with the following error message: " + error if error != "" else ""}. \
+    Please make sure the information you specified is correct and try again after a while.
 
 If you continue to experience difficulties using this application, please contact [email protected].