Merge pull request #685 from NextCenturyCorporation/MCS-1813-pyinstaller

MCS-1813 Updated the webenabled app to be packageable with pyinstaller

.gitignore

@@ -62,3 +62,7 @@ test-results.xml
 webenabled/flask_session/
 webenabled/scenes/*.json
 webenabled/static/mcsinterface/
+
+# webenabled pyinstaller
+mcsweb.spec
+pyinstaller.out

webenabled/README.md

@@ -29,17 +29,7 @@ scene files will appear on the web page
 
 First, run `cache_addressables` to cache all addressable assets and prevent possible timeout issues. You should do this each time you reboot your machine.
 
-For development, run the server directly:
-
-_(Note that we're calling `venv/bin/flask` to ensure it's the correct version of flask, which is especially important on Mac)_
-
-```FLASK_APP=mcsweb FLASK_DEBUG=1 venv/bin/flask run --port=8080 --host=0.0.0.0```
-
-Where:
-- FLASK_APP is the name of the python file to run.
-- FLASK_DEBUG=1 means that flask will auto-reload files when they change on disk
-- port 8080 makes the port flask runs on be 8080, rather than default of 5000
-- host=0.0.0.0 means that the page will be accessible from any machine, defaults to localhost
+For development, run `python mcsweb.py` to start the flask server with host `0.0.0.0` (so the page will be accessable from any machine on the network) and port `8080`.
 
 For production, use a WSGI server (not sure what to put here....)
 
@@ -58,3 +48,41 @@ http://<machine.ip.address>:8080/mcs
 3. The `run_scene_with_dir` script creates the MCS Controller and starts a watchdog Observer to watch for changes to the `cmd_<time>/` directory. Whenever a new "command" text file is created or modified, it triggers the Observer, which reads the command (either a scene filename ending in ".json" or an MCS action like Pass or MoveAhead), gives it to the MCS Controller via either its `start_scene` or `step` function, and saves step output and the output image in the `output_<time>` directory.
 4. When the user selects a scene in the UI, the `handle_scene_selection` function in `mcsweb` calls `load_scene` in `MCSInterface` which saves a "command" text file containing the scene filename.
 5. When the user presses a key in the UI, the `handle_keypress` function in `mcsweb` calls `perform_action` in `MCSInterface` which saves a "command" text file containing the action string.
+
+## Pyinstaller
+
+### Linux and Mac
+
+Setup:
+
+```
+pip install -U pyinstaller
+```
+
+Build:
+
+```
+pyinstaller --add-data 'templates:templates' --add-data 'static:static' --add-data 'scenes:scenes' --console mcsweb.py --log-level=DEBUG &> pyinstaller.out
+```
+
+Run:
+
+```
+./dist/mcsweb/mcsweb
+```
+
+Cleanup (do before you need to rebuild):
+
+```
+rm -rf build/ dist/
+```
+
+### Windows
+
+Same as the Linux/Mac instructions, except as noted below.
+
+Build:
+
+```
+pyinstaller --add-data "templates;templates" --add-data "static;static" --add-data "scenes;scenes" --console mcsweb.py --log-level=DEBUG -y *>&1 | Tee-Object -Append -FilePath pyinstaller.out
+```

webenabled/mcs_interface.py

@@ -2,8 +2,9 @@
 import glob
 import json
 import os
+import sys
 import time
-from os.path import exists
+from os.path import exists, getctime, relpath
 
 from flask import current_app
 from mcs_task_desc import TaskDescription
@@ -16,7 +17,15 @@
 
 IMG_WIDTH = 600
 IMG_HEIGHT = 400
-MCS_INTERFACE_TMP_DIR = "static/mcsinterface/"
+# Use _MEIPASS in the pyinstaller package (frozen=True).
+ROOT_PATH = f'{sys._MEIPASS}/' if getattr(sys, 'frozen', False) else ''
+RELATIVE_PATH = (
+    f'{relpath(ROOT_PATH, os.getcwd())}/'
+    # Convert the absolute path used within the pyinstaller package.
+    if ROOT_PATH.startswith('/') else ROOT_PATH
+)
+MCS_INTERFACE_TMP_DIR = 'static/mcsinterface/'
+TMP_DIR_FULL_PATH = f'{RELATIVE_PATH}{MCS_INTERFACE_TMP_DIR}'
 BLANK_IMAGE_NAME = 'blank_600x400.png'
 IMAGE_WAIT_TIMEOUT = 20.0
 UNITY_STARTUP_WAIT_TIMEOUT = 10.0
@@ -42,26 +51,26 @@ class MCSInterface:
 
     def __init__(self, user: str):
         self.logger = current_app.logger
-        # TODO FIXME Use the step number from the output metadata.
+        self.logger.info(f'MCS interface directory: {TMP_DIR_FULL_PATH}')
         self.step_number = 0
         self.scene_id = None
         self.scene_filename = None
         self.step_output = None
 
-        if not exists(MCS_INTERFACE_TMP_DIR):
-            os.mkdir(MCS_INTERFACE_TMP_DIR)
+        if not exists(TMP_DIR_FULL_PATH):
+            os.mkdir(TMP_DIR_FULL_PATH)
 
         time_str = datetime.datetime.now().strftime("%y%m%d_%H%M%S")
         suffix = f"{time_str}_{user}"
-        self.command_out_dir = f"{MCS_INTERFACE_TMP_DIR}cmd_{suffix}"
-        self.step_output_dir = f"{MCS_INTERFACE_TMP_DIR}output_{suffix}"
+        self.command_out_dir = f"{TMP_DIR_FULL_PATH}cmd_{suffix}"
+        self.step_output_dir = f"{TMP_DIR_FULL_PATH}output_{suffix}"
         if not exists(self.command_out_dir):
             os.mkdir(self.command_out_dir)
         if not exists(self.step_output_dir):
             os.mkdir(self.step_output_dir)
 
         # Make sure that there is a blank image (in case something goes wrong)
-        self.blank_path = MCS_INTERFACE_TMP_DIR + BLANK_IMAGE_NAME
+        self.blank_path = TMP_DIR_FULL_PATH + BLANK_IMAGE_NAME
         if not exists(self.blank_path):
             img = Image.new('RGB', (IMG_WIDTH, IMG_HEIGHT))
             img.save(self.blank_path)
@@ -84,7 +93,7 @@ def start_mcs(self):
         self.pid = start_subprocess(self.command_out_dir, self.step_output_dir)
 
         # Read in the image
-        self.img_name = self.get_image_name_and_step_output(startup=True)
+        self.img_name, _ = self.get_image_name_and_step_output(startup=True)
         return self.img_name
 
     def is_controller_alive(self):
@@ -203,7 +212,7 @@ def get_image_name_and_step_output(self, startup=False, init_scene=False):
 
                 if len(list_of_error_files) > 0:
                     latest_error_file = max(
-                        list_of_error_files, key=os.path.getctime
+                        list_of_error_files, key=getctime
                     )
                     if latest_error_file.endswith(
                             f"step_{self.step_number}.json") or init_scene:
@@ -230,14 +239,14 @@ def get_image_name_and_step_output(self, startup=False, init_scene=False):
             if len(list_of_img_files) > 0 and len(
                     list_of_output_files) > 0:  # noqa: E501
                 latest_json_file = max(
-                    list_of_output_files, key=os.path.getctime)
+                    list_of_output_files, key=getctime)
 
                 for file in list_of_output_files:
                     if file != latest_json_file:
                         os.unlink(file)
 
                 latest_image_file = max(
-                    list_of_img_files, key=os.path.getctime)
+                    list_of_img_files, key=getctime)
 
                 # Remove old files
                 for file in list_of_img_files:

webenabled/mcsweb.py

@@ -3,11 +3,24 @@
 import string
 
 import psutil
+import typeguard
+
+
+# Override the typechecked decorator used in machine_common_sense to do nothing
+# because it doesn't work with pyinstaller since the final package doesn't
+# include source code. (I tried setting PYTHONOPTIMIZE as recommended in the
+# typeguard docs but that didn't work.)
+def mock_decorator(func):
+    return func
+
+
+typeguard.typechecked = mock_decorator
+
 from flask import (Flask, jsonify, make_response, render_template, request,
                    session)
 # See: https://www.geeksforgeeks.org/how-to-use-flask-session-in-python-flask/
 from flask_session import Session
-from mcs_interface import MCSInterface
+from mcs_interface import MCS_INTERFACE_TMP_DIR, MCSInterface
 from webenabled_common import LOG_CONFIG
 
 # Configure logging _before_ creating the app oject
@@ -47,6 +60,14 @@ def clean_request_data(request, is_json=False):
     return data
 
 
+def convert_image_path(img: str) -> str:
+    # In packages made by pyinstaller, we need to adjust the image path for the
+    # HTML page, even though python is being run from a parent directory.
+    if img.find(MCS_INTERFACE_TMP_DIR) > 0:
+        return img[img.find(MCS_INTERFACE_TMP_DIR):]
+    return img
+
+
 def get_mcs_interface(request, label, on_exit=False):
     # Do we know who this is?
     uniq_id_str = request.cookies.get("uniq_id")
@@ -102,6 +123,7 @@ def handle_load_controller():
         return
 
     img = mcs_interface.blank_path
+    img = convert_image_path(img)
     scene_list = mcs_interface.get_scene_list()
 
     resp = jsonify(image=img, scene_list=scene_list)
@@ -122,6 +144,7 @@ def handle_keypress():
     params = clean_request_data(request, is_json=True)
     key = params["keypress"]
     action_string, img, step_output, action_list = mcs_interface.perform_action(params)  # noqa: E501
+    img = convert_image_path(img)
     step_number = mcs_interface.step_number
     app.logger.info(
         f"Key press: '{key}', action string: {action_string}, "
@@ -197,6 +220,7 @@ def handle_scene_selection():
     scene_filename = clean_request_data(request)
     img, step_output, action_list, goal_info, task_desc = mcs_interface.load_scene(  # noqa: E501
         "scenes/" + scene_filename)
+    img = convert_image_path(img)
     app.logger.info(f"Start scene: {scene_filename}, output: {img}")
     resp = jsonify(
         last_action="Initialize",
@@ -209,3 +233,7 @@ def handle_scene_selection():
         step_output=step_output
     )
     return resp
+
+
+if __name__ == "__main__":
+    app.run(host='0.0.0.0', port=8080, debug=True)

webenabled/run_scene_with_dir.py

@@ -17,7 +17,7 @@
 import logging
 import os
 import time
-from os.path import exists
+from os.path import exists, relpath
 
 from watchdog.events import PatternMatchingEventHandler
 from watchdog.observers import Observer
@@ -84,7 +84,7 @@ def create_command_file(self, command_text_file):
 
     def load_command_file(self, command_text_file):
 
-        if os.path.exists(command_text_file) is False:
+        if exists(command_text_file) is False:
             self.create_command_file(command_text_file)
 
         try:
@@ -224,17 +224,26 @@ def save_output_image(self, output: StepMetadata):
             logger.exception(
                 f"Error saving output image on step {output.step_number}")
 
+    def convert_file_path(self, path: str) -> str:
+        # Must convert a absolute file path to a relative file path in the
+        # pyinstaller package.
+        if path.startswith('/'):
+            return relpath(path, os.getcwd())
+        return path
+
     # ----------------------------------
     # Watchdog functions
     # ----------------------------------
     def on_created(self, event):
-        logger.info(f"File creation: {event.src_path}")
-        self.load_command_file(event.src_path)
+        path = self.convert_file_path(event.src_path)
+        logger.info(f"File creation: {path}")
+        self.load_command_file(path)
         os.unlink(event.src_path)
 
     def on_modified(self, event):
-        logger.info(f"File modified: {event.src_path}")
-        self.load_command_file(event.src_path)
+        path = self.convert_file_path(event.src_path)
+        logger.info(f"File modified: {path}")
+        self.load_command_file(path)
         os.unlink(event.src_path)
 
     def on_moved(self, event):

webenabled/templates/mcs_page.html

@@ -378,7 +378,6 @@ <h1>Machine Common Sense</h1>
         <p>This research was developed with funding from the Defense Advanced Research Projects Agency (DARPA). The views, opinions and/or findings expressed are those of the author and should not be interpreted as representing the official views or policies of the Department of Defense or the U.S. Government.<br/> <a href="https://www.darpa.mil/program/machine-common-sense" target="_blank">DARPA's Machine Common Sense (MCS) Program Page</a></p>
     </div>
 </div>
-<script src="https://unpkg.com/[email protected]/fetch.js"></script>
 <script>
     // ---------------------------------------------------------
     // Handle clicking on one of the scene file names