MCS-1815 MCS-1930 Webenabled app now uses a production-ready server b…

…y default. Also added configurable command line settings.

webenabled/mcs_interface.py

@@ -1,6 +1,7 @@
 import datetime
 import glob
 import json
+import logging
 import os
 import sys
 import time
@@ -35,9 +36,9 @@
 def convert_key_to_action(key: str, logger):
     for action in mcs.Action:
         if key.lower() == action.key:
-            logger.info(f"Converting '{key}' into {action.value}")
+            logger.debug(f"Converting '{key}' into {action.value}")
             return action.value
-    logger.info(f"Unable to convert '{key}'. Returning Pass...")
+    logger.debug(f"Unable to convert '{key}'. Returning Pass...")
     return "Pass"
 
 
@@ -52,7 +53,7 @@ class MCSInterface:
 
     def __init__(self, user: str):
         self.logger = current_app.logger
-        self.logger.info(f'MCS interface directory: {TMP_DIR_FULL_PATH}')
+        self.logger.debug(f'MCS interface directory: {TMP_DIR_FULL_PATH}')
         self.step_number = 0
         self.scene_id = None
         self.scene_filename = None
@@ -91,7 +92,11 @@ def get_latest_step_output(self):
     def start_mcs(self):
         # Start the unity controller.  (the function is in a different
         # file so we can pickle / store MCSInterface in the session)
-        self.pid = start_subprocess(self.command_out_dir, self.step_output_dir)
+        self.pid = start_subprocess(
+            self.command_out_dir,
+            self.step_output_dir,
+            self.logger.isEnabledFor(logging.DEBUG)
+        )
 
         # Read in the image
         images, _ = self.get_images_and_step_output(startup=True)
@@ -209,13 +214,13 @@ def get_images_and_step_output(self, startup=False, init_scene=False):
             timenow = time.time()
             elapsed = (timenow - timestart)
             if (startup and elapsed > UNITY_STARTUP_WAIT_TIMEOUT):
-                self.logger.info(
+                self.logger.debug(
                     "Display blank image on default when starting up.")
                 self.img_name = self.blank_path
                 return [self.img_name], self.step_output
 
             if elapsed > IMAGE_WAIT_TIMEOUT:
-                self.logger.info("Timeout waiting for image")
+                self.logger.warn("Timeout waiting for image")
 
                 list_of_error_files = glob.glob(
                     self.step_output_dir + "/error_*.json")
@@ -348,18 +353,18 @@ def get_goal_info(self, scene_filename):
 
     def get_task_desc(self, scene_filename):
         """Get task description based on filename from mcs_task_desc.py"""
-        self.logger.info(
+        self.logger.debug(
             f"Attempt to get task description based"
             f"on scene_filename: {scene_filename}")
         scene_type = scene_filename.split('/')[-1].split('0')[0][:-1].upper()
 
         for description in TaskDescription:
             if (description.name == scene_type):
-                self.logger.info(
+                self.logger.debug(
                     f"Scene type identified: {description.name}")
                 return description.value
 
-        self.logger.info("Scene type not found, returning 'N/A'")
+        self.logger.warn("Scene type not found, returning 'N/A'")
         return "N/A"
 
     def get_controller_pid(self):

webenabled/mcsweb.py

@@ -1,9 +1,11 @@
+import argparse
 import logging
 import random
 import string
 
 import psutil
 import typeguard
+import waitress
 
 
 # Override the typechecked decorator used in machine_common_sense to do nothing
@@ -74,22 +76,22 @@ def get_mcs_interface(request, label, on_exit=False):
 
     # If old user, get stored mcs interface
     if uniq_id_str is not None:
-        app.logger.info(f"{label}: existing user: {uniq_id_str}")
+        app.logger.debug(f"{label}: existing user: {uniq_id_str}")
         mcs_interface = session.get(uniq_id_str)
         if mcs_interface is not None:
             controller_alive = mcs_interface.is_controller_alive()
             if controller_alive:
                 return mcs_interface, uniq_id_str
-            app.logger.info("MCS controller is unavailable")
+            app.logger.debug("MCS controller is unavailable")
         else:
-            app.logger.info("MCS interface is unavailable")
+            app.logger.debug("MCS interface is unavailable")
 
     # skip for exit_unity route, since in that case, we don't
     # need to start a new interface/controller if one isn't found
     if (on_exit is False):
         letters = string.ascii_lowercase
         uniq_id_str = ''.join(random.choice(letters) for i in range(10))
-        app.logger.info(f"{label}: new user: {uniq_id_str}")
+        app.logger.debug(f"{label}: new user: {uniq_id_str}")
 
         # Don't recognize, create new mcs interface
         mcs_interface = MCSInterface(uniq_id_str)
@@ -101,8 +103,8 @@ def get_mcs_interface(request, label, on_exit=False):
 
 @app.route('/mcs')
 def show_mcs_page():
-    app.logger.info("=" * 30)
-    app.logger.info(
+    app.logger.debug("=" * 30)
+    app.logger.debug(
         "Initialize page before checking for "
         "controller and existing user session...")
     rendered_template = render_template(
@@ -116,7 +118,7 @@ def show_mcs_page():
 
 @app.route('/load_controller', methods=["POST"])
 def handle_load_controller():
-    app.logger.info("=" * 30)
+    app.logger.debug("=" * 30)
     mcs_interface, uniq_id_str = get_mcs_interface(request, "Load page")
     if mcs_interface is None:
         app.logger.warn("Cannot load MCS interface")
@@ -135,7 +137,7 @@ def handle_load_controller():
 
 @app.route("/keypress", methods=["POST"])
 def handle_keypress():
-    app.logger.info("=" * 30)
+    app.logger.debug("=" * 30)
     mcs_interface, _ = get_mcs_interface(request, "Key press")
     if mcs_interface is None:
         app.logger.warn("Cannot load MCS interface")
@@ -151,12 +153,12 @@ def handle_keypress():
     img = images[0]
     step_number = mcs_interface.step_number
     if key:
-        app.logger.info(
+        app.logger.debug(
             f"Key press: '{key}', action string: {action_string}, "
             f"step: {step_number}, img: {img}, output: {step_output}"
         )
     else:
-        app.logger.info(
+        app.logger.debug(
             f"Action: '{action}', "
             f"step: {step_number}, img: {img}, output: {step_output}"
         )
@@ -172,7 +174,7 @@ def handle_keypress():
 
 @app.route("/exit_unity", methods=["POST"])
 def exit_unity():
-    app.logger.info("=" * 30)
+    app.logger.debug("=" * 30)
     mcs_interface, unique_id = get_mcs_interface(
         request, "Exit Unity", on_exit=True)
     if mcs_interface is None:
@@ -188,26 +190,26 @@ def exit_unity():
 
     controller_pid = mcs_interface.get_controller_pid()
 
-    app.logger.info(
+    app.logger.debug(
         "Attempting to clean up processes after browser has been closed.")
 
     for p in psutil.process_iter(['pid']):
         if p.info['pid'] == controller_pid:
             children = p.children(recursive=True)
             for c_process in children:
-                app.logger.info(
+                app.logger.debug(
                     f"Found child process of controller: {c_process}, "
                     f"will attempt to end.")
                 c_process.kill()
 
-            app.logger.info(
+            app.logger.debug(
                 f"Found controller process: {p}, will attempt to end.")
             p.kill()
 
     if (unique_id is None):
         unique_id = request.cookies.get("uniq_id")
 
-    app.logger.info(
+    app.logger.debug(
         f"Clear user session for: {unique_id}")
     del session[unique_id]
 
@@ -222,7 +224,7 @@ def exit_unity():
 
 @app.route("/scene_selection", methods=["POST"])
 def handle_scene_selection():
-    app.logger.info("=" * 30)
+    app.logger.debug("=" * 30)
     mcs_interface, _ = get_mcs_interface(request, "Start scene")
     if mcs_interface is None:
         app.logger.warn("Cannot load MCS interface")
@@ -233,7 +235,7 @@ def handle_scene_selection():
     load_output = mcs_interface.load_scene("scenes/" + scene_filename)
     images, step_output, action_list, goal_info, task_desc = load_output
     img = convert_image_path(images[0])
-    app.logger.info(f"Start scene: {scene_filename}, output: {img}")
+    app.logger.debug(f"Start scene: {scene_filename}, output: {img}")
     resp = jsonify(
         last_action="Initialize",
         action_list=action_list,
@@ -249,4 +251,44 @@ def handle_scene_selection():
 
 
 if __name__ == "__main__":
-    app.run(host='0.0.0.0', port=8080, debug=True)
+    parser = argparse.ArgumentParser(description=(
+        'Machine Common Sense Web Interface'
+    ))
+    parser.add_argument(
+        '--host',
+        type=str,
+        default='0.0.0.0',
+        help='Host'
+    )
+    parser.add_argument(
+        '--port',
+        type=int,
+        default=8080,
+        help='Port'
+    )
+    parser.add_argument(
+        '--dev',
+        default=False,
+        action='store_true',
+        help='Development server'
+    )
+    parser.add_argument(
+        '--debug',
+        default=False,
+        action='store_true',
+        help='Debug logging'
+    )
+    args = parser.parse_args()
+    app.logger.info(
+        f'Starting MCS web interface: host={args.host} port={args.port} '
+        f'dev={args.dev} debug={args.debug}'
+    )
+
+    if args.dev:
+        app.run(host=args.host, port=args.port, debug=True)
+    else:
+        waitress.serve(app, host=args.host, port=args.port)
+        waitress_logger = logging.getLogger('waitress')
+        waitress_logger.setLevel(logging.DEBUG if args.debug else logging.INFO)
+
+    app.logger.setLevel(logging.DEBUG if args.debug else logging.INFO)

webenabled/run_scene_with_dir.py

@@ -40,7 +40,7 @@ def __init__(self, command_in_dir, output_dir):
         self.step_number = 0
 
     def run_loop(self):
-        logger.info(
+        logger.debug(
             f"Starting controller: watching command directory "
             f"{self.command_in_dir[(self.command_in_dir.rfind('/') + 1):]}"
             f", writing to output directory "
@@ -109,7 +109,7 @@ def load_command_file(self, command_text_file):
                 self.step_and_save(command)
 
     def load_scene(self):
-        logger.info(f"Loading file {self.scene_file}")
+        logger.debug(f"Loading file {self.scene_file}")
 
         if not exists(self.scene_file):
             logger.warn(f"Missing file {self.scene_file}")
@@ -125,7 +125,7 @@ def load_scene(self):
             logger.exception(f"Error loading file {self.scene_file}")
 
     def step_and_save(self, command):
-        logger.info(f"Executing command {command}")
+        logger.debug(f"Executing command {command}")
         try:
             output: StepMetadata = self.controller.step(command)
             if output is not None:
@@ -169,7 +169,7 @@ def log_error(self, error):
                 f"Error saving error output to file {error_output_file}")
 
     def save_output_info(self, output: StepMetadata):
-        logger.info(f"Saving output info at step {output.step_number}")
+        logger.debug(f"Saving output info at step {output.step_number}")
 
         output_to_save_dict = {
             'step_number': output.step_number,
@@ -188,7 +188,7 @@ def save_output_info(self, output: StepMetadata):
                 f'{self.output_dir}/step_output_{scene_id}_step_'
                 f'{output.step_number}.json'
             )
-            logger.info(
+            logger.debug(
                 f"Saved json file on step {output.step_number} to "
                 f"{output_path}"
             )
@@ -200,7 +200,7 @@ def save_output_info(self, output: StepMetadata):
                 f"Error saving output info on step {output.step_number}")
 
     def save_output_image(self, output: StepMetadata):
-        logger.info(f"Saving output image at step {output.step_number}")
+        logger.debug(f"Saving output image at step {output.step_number}")
         scene_id = self.scene_file[
             (self.scene_file.rfind('/') + 1):(self.scene_file.rfind('.'))
         ]
@@ -211,7 +211,7 @@ def save_output_image(self, output: StepMetadata):
                     f'{self.output_dir}/rgb_{scene_id}_step_'
                     f'{output.step_number}.png'
                 )
-                logger.info(
+                logger.debug(
                     f"Saved RGB image on step {output.step_number} to "
                     f"{img_path}"
                 )
@@ -236,13 +236,13 @@ def convert_file_path(self, path: str) -> str:
     # ----------------------------------
     def on_created(self, event):
         path = self.convert_file_path(event.src_path)
-        logger.info(f"File creation: {path}")
+        logger.debug(f"File creation: {path}")
         self.load_command_file(path)
         os.unlink(event.src_path)
 
     def on_modified(self, event):
         path = self.convert_file_path(event.src_path)
-        logger.info(f"File modified: {path}")
+        logger.debug(f"File modified: {path}")
         self.load_command_file(path)
         os.unlink(event.src_path)
 
@@ -264,6 +264,11 @@ def parse_args():
     parser.add_argument(
         '--mcs_output_dir',
         help='MCS directory that images will appear in')
+    parser.add_argument(
+        '--debug',
+        default=False,
+        action='store_true',
+        help='Debug logging')
     return parser.parse_args()
 
 
@@ -272,6 +277,7 @@ def parse_args():
     # scene_file = args.mcs_scene_json_file
     command_in_dir = args.mcs_command_in_dir
     output_dir = args.mcs_output_dir
+    logger.setLevel(logging.DEBUG if args.debug else logging.INFO)
 
     run_scene = RunSceneWithDir(command_in_dir, output_dir)
     run_scene.run_loop()

webenabled/subprocess_runner.py

@@ -11,14 +11,15 @@
 from flask import current_app
 
 
-def start_subprocess(command_dir, output_dir):
+def start_subprocess(command_dir, output_dir, debug):
     logger = current_app.logger
-    proc = subprocess.Popen(
-        ["python3", "run_scene_with_dir.py",
-         "--mcs_command_in_dir", command_dir,
-         "--mcs_output_dir", output_dir])
+    proc = subprocess.Popen([
+        "python3", "run_scene_with_dir.py",
+        "--mcs_command_in_dir", command_dir,
+        "--mcs_output_dir", output_dir
+    ] + (["--debug"] if debug else []))
     pid_str = str(proc.pid)
-    logger.info(
+    logger.debug(
         f"Running script to start the MCS Controller with command directory "
         f"{command_dir[(command_dir.rfind('/') + 1):]}"
         f" and output directory "