Merge pull request #706 from NextCenturyCorporation/MCS-1923-quick-ac…

…tions

MCS-1923 MCS-1924 Webenabled Updates: Previous Images, Quick Actions

webenabled/mcs_interface.py

@@ -27,8 +27,9 @@
 MCS_INTERFACE_TMP_DIR = 'static/mcsinterface/'
 TMP_DIR_FULL_PATH = f'{RELATIVE_PATH}{MCS_INTERFACE_TMP_DIR}'
 BLANK_IMAGE_NAME = 'blank_600x400.png'
-IMAGE_WAIT_TIMEOUT = 20.0
+IMAGE_WAIT_TIMEOUT = 60.0
 UNITY_STARTUP_WAIT_TIMEOUT = 10.0
+IMAGE_COUNT = 500
 
 
 def convert_key_to_action(key: str, logger):
@@ -93,8 +94,8 @@ def start_mcs(self):
         self.pid = start_subprocess(self.command_out_dir, self.step_output_dir)
 
         # Read in the image
-        self.img_name, _ = self.get_image_name_and_step_output(startup=True)
-        return self.img_name
+        images, _ = self.get_images_and_step_output(startup=True)
+        return images
 
     def is_controller_alive(self):
         # When we re-attach, we need to make sure that the controller
@@ -112,32 +113,40 @@ def load_scene(self, scene_filename: str):
         self.scene_id = scene_filename[
             (scene_filename.rfind('/') + 1):(scene_filename.rfind('.'))
         ]
-        _, img, step_output = self._post_step_and_get_output(scene_filename)
-        return img, step_output, action_list_str, goal_info, task_desc
+        _, images, step_output = self._post_step_and_get_output(scene_filename)
+        return images, step_output, action_list_str, goal_info, task_desc
 
     def perform_action(self, params: object):
-        key = params["keypress"]
-        del params["keypress"]
-        action = convert_key_to_action(key, self.logger)
-        full_action, img, step_output = self._post_step_and_get_output(
+        action = None
+        if "keypress" in params:
+            key = params["keypress"]
+            del params["keypress"]
+            action = convert_key_to_action(key, self.logger)
+        if "action" in params:
+            action = params["action"]
+            del params["action"]
+        full_action, images, step_output = self._post_step_and_get_output(
             action, params)
         if (step_output):
             self.step_number = step_output.get('step_number', self.step_number)
             action_list_str = self.get_action_list(
                 step_number=self.step_number)
-        return full_action, img, step_output, action_list_str
+        return full_action, images, step_output, action_list_str
 
     def _post_step_and_get_output(self, action: str, params=None):
         full_action_str = action
+
         image_coord_actions = ["CloseObject", "OpenObject", "PickupObject",
                                "PullObject", "PushObject", "PutObject",
                                "TorqueObject", "RotateObject", "MoveObject",
                                "InteractWithAgent"]
+
         command_file_name = (
             f'{self.command_out_dir}/command_{self.scene_id}_step_'
             f'{self.step_number}.txt'
         )
         f = open(command_file_name, "a")
+
         if (action in image_coord_actions and params is not None):
             x_coord = params["objectImageCoordsX"]
             y_coord = params["objectImageCoordsY"]
@@ -185,14 +194,15 @@ def _post_step_and_get_output(self, action: str, params=None):
         if is_initialize:
             action_to_return = self.scene_id
 
-        img, step_output = self.get_image_name_and_step_output(
+        images, step_output = self.get_images_and_step_output(
             init_scene=is_initialize)
 
-        return action_to_return, img, step_output
+        return action_to_return, images, step_output
 
-    def get_image_name_and_step_output(self, startup=False, init_scene=False):
-        """Watch the output directory, get image that appears.  If it does
-        not appear in timeout seconds, give up and return blank."""
+    def get_images_and_step_output(self, startup=False, init_scene=False):
+        """Watch the output directory and return the latest images and step
+        output that appears. If it does not appear in <timeout> seconds, then
+        give up and return a blank image."""
         timestart = time.time()
 
         while True:
@@ -202,7 +212,7 @@ def get_image_name_and_step_output(self, startup=False, init_scene=False):
                 self.logger.info(
                     "Display blank image on default when starting up.")
                 self.img_name = self.blank_path
-                return self.img_name, self.step_output
+                return [self.img_name], self.step_output
 
             if elapsed > IMAGE_WAIT_TIMEOUT:
                 self.logger.info("Timeout waiting for image")
@@ -229,29 +239,33 @@ def get_image_name_and_step_output(self, startup=False, init_scene=False):
                             'error': new_error_output["error"]
                         }
 
-                return self.img_name, self.step_output
+                return [self.img_name], self.step_output
 
-            list_of_output_files = glob.glob(
-                self.step_output_dir + "/step_output_*.json")
-            list_of_img_files = glob.glob(self.step_output_dir + "/rgb_*.png")
+            # Sort descending by date/time modified (newest first)
+            list_of_output_files = list(reversed(sorted(
+                glob.glob(self.step_output_dir + "/step_output_*.json"),
+                key=getctime
+            )))
+            list_of_img_files = list(reversed(sorted(
+                glob.glob(self.step_output_dir + "/rgb_*.png"),
+                key=getctime
+            )))
 
             # Image file logic
-            if len(list_of_img_files) > 0 and len(
-                    list_of_output_files) > 0:  # noqa: E501
-                latest_json_file = max(
-                    list_of_output_files, key=getctime)
+            if len(list_of_img_files) > 0 and len(list_of_output_files) > 0:
+                latest_json_file = list_of_output_files[0]
 
-                for file in list_of_output_files:
-                    if file != latest_json_file:
-                        os.unlink(file)
+                # Keep only the latest output file.
+                for file in list_of_output_files[1:]:
+                    os.unlink(file)
 
-                latest_image_file = max(
-                    list_of_img_files, key=getctime)
+                latest_image_file = list_of_img_files[0]
 
-                # Remove old files
-                for file in list_of_img_files:
-                    if file != latest_image_file:
-                        os.unlink(file)
+                # Keep only the latest image files. Use step_number if less
+                # than IMAGE_COUNT to remove images from previous scenes.
+                image_count = min(IMAGE_COUNT, self.step_number + 2)
+                for file in list_of_img_files[image_count:]:
+                    os.unlink(file)
 
                 # wait to make sure we've finished loading the new image
                 # (not sure why the is_file_open check below didn't
@@ -280,7 +294,7 @@ def get_image_name_and_step_output(self, startup=False, init_scene=False):
                 if latest_image_file != self.img_name and has_new_step_output:
                     self.step_output = new_step_output
                     self.img_name = latest_image_file
-                    return self.img_name, self.step_output
+                    return list_of_img_files[:image_count], self.step_output
 
             time.sleep(0.05)
 
@@ -313,7 +327,7 @@ def get_action_list(self, step_number=0):
                         ]
                     if is_passive:
                         actions = GoalMetadata.DEFAULT_PASSIVE_SCENE_ACTIONS
-                    if goal.get('action_list'):
+                    if len(goal.get('action_list', [])) > step_number:
                         actions = goal['action_list'][step_number]
                     return self.simplify_action_list(actions)
         except Exception:

webenabled/mcsweb.py

@@ -126,7 +126,7 @@ def handle_load_controller():
     img = convert_image_path(img)
     scene_list = mcs_interface.get_scene_list()
 
-    resp = jsonify(image=img, scene_list=scene_list)
+    resp = jsonify(image=img, previous_images=[], scene_list=scene_list)
 
     resp.set_cookie("uniq_id", uniq_id_str)
 
@@ -142,17 +142,29 @@ def handle_keypress():
         return
 
     params = clean_request_data(request, is_json=True)
-    key = params["keypress"]
-    action_string, img, step_output, action_list = mcs_interface.perform_action(params)  # noqa: E501
-    img = convert_image_path(img)
+    key = params["keypress"] if "keypress" in params else None
+    action = params["action"] if "action" in params else None
+    action_output = mcs_interface.perform_action(params)
+    action_string, images, step_output, action_list = action_output
+    for index in range(len(images)):
+        images[index] = convert_image_path(images[index])
+    img = images[0]
     step_number = mcs_interface.step_number
-    app.logger.info(
-        f"Key press: '{key}', action string: {action_string}, "
-        f"step {step_number}, img: {img}, output: {step_output}")
+    if key:
+        app.logger.info(
+            f"Key press: '{key}', action string: {action_string}, "
+            f"step: {step_number}, img: {img}, output: {step_output}"
+        )
+    else:
+        app.logger.info(
+            f"Action: '{action}', "
+            f"step: {step_number}, img: {img}, output: {step_output}"
+        )
     resp = jsonify(
         last_action=action_string,
         action_list=action_list,
         image=img,
+        previous_images=images[1:],
         step=step_number,
         step_output=step_output)
     return resp
@@ -218,14 +230,15 @@ def handle_scene_selection():
 
     # Get the scene filename and tell interface to load it.
     scene_filename = clean_request_data(request)
-    img, step_output, action_list, goal_info, task_desc = mcs_interface.load_scene(  # noqa: E501
-        "scenes/" + scene_filename)
-    img = convert_image_path(img)
+    load_output = mcs_interface.load_scene("scenes/" + scene_filename)
+    images, step_output, action_list, goal_info, task_desc = load_output
+    img = convert_image_path(images[0])
     app.logger.info(f"Start scene: {scene_filename}, output: {img}")
     resp = jsonify(
         last_action="Initialize",
         action_list=action_list,
         image=img,
+        previous_images=[],
         scene=scene_filename,
         goal=goal_info,
         task_desc=task_desc,

webenabled/run_scene_with_dir.py

@@ -174,7 +174,7 @@ def save_output_info(self, output: StepMetadata):
         output_to_save_dict = {
             'step_number': output.step_number,
             'return_status': output.return_status,
-            'reward': output.reward,
+            'reward': round(output.reward, 3),
             'steps_on_lava': output.steps_on_lava
         }