jsk-ros-pkg · sktometometo · Apr 13, 2023 · Apr 13, 2023 · Apr 13, 2023 · Apr 13, 2023
diff --git a/database_talker/.gitignore b/database_talker/.gitignore
@@ -0,0 +1,3 @@
+auth/*
+test_db/*
+.vscode/*
diff --git a/database_talker/CMakeLists.txt b/database_talker/CMakeLists.txt
@@ -0,0 +1,21 @@
+cmake_minimum_required(VERSION 3.0.2)
+project(database_talker)
+
+find_package(catkin REQUIRED COMPONENTS catkin_virtualenv)
+
+## This macro ensures modules and global scripts declared therein get installed
+catkin_python_setup()
+
+catkin_generate_virtualenv(
+  PYTHON_INTERPRETER python3
+  CHECK_VENV FALSE
+  USE_SYSTEM_PACKAGES FALSE  # Default TRUE
+)
+
+catkin_package(
+)
+
+catkin_install_python(PROGRAMS
+  sample/include/store_image_description.py scripts/make_diary.py
+  DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
+)
diff --git a/database_talker/README.md b/database_talker/README.md
@@ -0,0 +1,44 @@
+# database talker
+
+## What is this?
+
+This is sample code to generate a response/diary from a robot's experience stored in MongoDB.
+
+## How to setup
+
+Set up a workspace using the `rosinstall` file and compile it with `catkin build database_talker`.
+
+## How to use
+
+For a minimum setup, run the following command. This will start the mongodb/lifelog nodes and save the usb camera data to the database.
+```bash
+roslaunch database_talker sample.launch
+```
+
+To generate a diary using robot memory, execute the following command and talk to GoogleChat bot.
+
+```bash
+rosrun database_talker make_diary.py --prompt-type personality
+```
+
+## Tips
+
+### How to test using data from a specific date without using GoogleChat.
+```bash
+rosrun database_talker make_diary.py --test --prompt-type personality --date 2023-03-20
+```
+
+### Stop using external DBs, this is recommended during debug phase.
+
+Remove `mongodb_store_extras` in `jsk_robot_startup/lifelog/mongodb_replication_params.yaml`
+```
+-mongodb_store_extras: [["robot-database.jsk.imi.i.u-tokyo.ac.jp", 27017],["musca.jsk.imi.i.u-tokyo.ac.jp",27017]]
++mongodb_store_extras: []
+```
+
+### Force store image
+
+An image will only be saved if a significant change is found in the image within seconds. To force the image to be saved, use the following command.
+```
+rostopic pub -1 /publish_trigger_mongodb_event roseus/StringStamped '{header: auto, data: debug}'
+```
diff --git a/database_talker/aibo.rosinstall b/database_talker/aibo.rosinstall
@@ -0,0 +1,42 @@
+# database talker for aibo
+- git:
+    local-name: jsk_demos
+    uri: https://github.com/sktometometo/jsk_demos
+    version: PR/hoge-py
+# use k-okada' version of aibo_driver (for message)
+- git:
+    local-name: aibo_driver
+    uri: https://gitlab.jsk.imi.i.u-tokyo.ac.jp/k-okada/aibo_status
+    version: driver
+#
+# use latest google_chat_ros, until noetic is synced
+# armhf users need to use https://github.com/k-okada/jsk_3rdparty/commit/8524aaa4118cb7ab02c65f47c46343bbb4f7147c
+# rosinstall_generator google_chat_ros --rosdistro noetic
+#
+- git:
+    local-name: google_chat_ros
+    uri: https://github.com/tork-a/jsk_3rdparty-release.git
+    version: release/noetic/google_chat_ros/2.1.28-1
+#
+# use latest jsk_recognition_msgs, until noetic is synced
+#
+- git:
+    local-name: jsk_recognition/jsk_recognition_msgs
+    uri: https://github.com/tork-a/jsk_recognition-release.git
+    version: release/noetic/jsk_recognition_msgs/1.2.17-1
+#
+# need releasing
+# https://github.com/davesarmoury/openai_ros/pulls?q=is%3Apr
+#
+- git:
+    local-name: openai_ros
+    # uri: https://github.com/davesarmoury/openai_ros
+    uri: https://github.com/k-okada/openai_ros
+#
+# add sample launch code for database_talker #1792
+# https://github.com/jsk-ros-pkg/jsk_robot/pull/1792
+#
+- git:
+    local-name: jsk_robot
+    uri: https://github.com/k-okada/jsk_robot.git
+    version: ichikura_sample
diff --git a/database_talker/auth/.keepme b/database_talker/auth/.keepme
diff --git a/database_talker/launch/aibo_diary.launch b/database_talker/launch/aibo_diary.launch
@@ -0,0 +1,52 @@
+<launch>
+  <arg name="start_aibo_driver" value="false" />
+  <arg name="start_camera_driver" default="true" />
+  <arg name="camera_device" default="/dev/video0" />
+
+  <include file="$(find aibo_driver)/launch/aibo_driver.launch" if="$(arg start_aibo_driver)" />
+
+  <node name="usb_cam" pkg="usb_cam" type="usb_cam_node" output="screen" if="$(arg start_camera_driver)" respawn="true">
+    <param name="video_device" value="$(arg camera_device)" />
+    <param name="image_width" value="320" />
+    <param name="image_height" value="240" />
+    <param name="pixel_format" value="yuyv" />
+    <param name="color_format" value="yuv422p" />
+    <param name="camera_frame_id" value="usb_cam" />
+    <param name="io_method" value="mmap"/>
+    <param name="framrenate" value="10"/>
+  </node>
+
+  <include file="$(find database_talker)/launch/include/database_talker.launch" >
+    <arg name="google_credentials_json" value="$(find database_talker)/auth/jsk-aibo-37d42e158287.json" />
+    <arg name="google_credentials_project_id" value="jsk-aibo" />
+    <arg name="start_periodic_mongodb_trigger" default="false" />
+  </include>
+
+  <!-- start_periodic_mongodb_trigger -->
+  <node pkg="rostopic" type="rostopic" name="periodic_mongodb_trigger"
+        launch-prefix="bash -c 'set -x; while [ 1 ]; do ${@:0:$#-2} &quot;${@: -3}&quot; ; sleep 1000; done;' "
+        args="pub -s -1 /publish_trigger_mongodb_event aibo_driver/StringStatus &quot;{header: auto}&quot;" />
+  <!-- republish compressed/throttled image data only when /publish_trigger_mongodb_event is published -->
+  <!-- relay /aibo_driver/trigger -> /publish_trigger_mongodb_event to store /usb_cam/image_raw-->
+  <node name="relay_aibo_trigger_to_mongodb_trigger"
+	pkg="topic_tools" type="relay"
+	args="/aibo_driver/hungry_status /publish_trigger_mongodb_event" />
+
+  <!-- aibo data -->
+  <node name="aibo_driver_logger"
+        pkg="jsk_robot_startup" type="mongo_record.py"
+        respawn="true">
+    <rosparam subst_value="true">
+      topics:
+      - /aibo_driver/biting_status
+      - /aibo_driver/body_touched_status
+      - /aibo_driver/found_objects_status
+      - /aibo_driver/hungry_status
+      - /aibo_driver/name_called_status
+      - /aibo_driver/paw_pads_status
+      - /aibo_driver/posture_status
+      - /aibo_driver/sleepy_status
+      - /aibo_driver/voice_command_status
+    </rosparam>
+  </node>
+</launch>
diff --git a/database_talker/launch/aibo_example.launch b/database_talker/launch/aibo_example.launch
@@ -0,0 +1,20 @@
+<launch>
+  <arg name="openai_credentials_json" default="$(find aibo_driver)/auth/openai_credentials.json" />
+  <param name="robot/type" value="aibo" />
+  <param name="robot/name" value="aibo" />
+
+
+  <param name="mongodb_host" value="musca.jsk.imi.i.u-tokyo.ac.jp" />
+  <param name="mongodb_port" value="27017" />
+  <node name="message_store" pkg="mongodb_store" type="message_store_node.py" >
+    <param name="mongodb_use_localdatacenter" value="false" />
+    <param name="queue_size" value="100" />
+  </node>
+
+  <!-- openai -->
+  <group ns="openai">
+    <node pkg="openai_ros" type="openai_node.py" name="openai" output="screen" >
+      <rosparam command="load" file="$(arg openai_credentials_json)" />
+    </node>
+  </group>
+</launch>
diff --git a/database_talker/launch/include/database_talker.launch b/database_talker/launch/include/database_talker.launch
@@ -0,0 +1,72 @@
+<launch>
+  <arg name="openai_credentials_json" default="$(find database_talker)/auth/openai_credentials.json" />
+  <arg name="google_credentials_json" default="$(find database_talker)/auth/eternal-byte-236613-4bc6962824d1.json" />
+  <arg name="google_credentials_project_id" default="eternal-byte-236613" />
+  <arg name="gdrive_credentials_yaml" default="$(find database_talker)/auth/pydrive_settings.yaml" />
+  <arg name="start_periodic_mongodb_trigger" default="true" />
+
+  <arg name="start_mongodb_record_nodelet_manager" default="true" />
+  <arg name="start_mongodb" default="true" />
+
+  <include file="$(find jsk_robot_startup)/launch/lifelog.launch" >
+    <arg name="logger_save_rgb" value="false" />
+    <arg name="logger_save_action" value="false" />
+    <arg name="logger_save_smach" value="false" />
+    <arg name="logger_save_faces" value="false" />
+    <arg name="logger_save_dialogflow" value="false" />
+    <arg name="logger_save_google_chat" value="false" />
+    <arg name="logger_save_video_to_scene" value="false" /> <!-- to reduce CPU load we use /aibo_driver/trigger -->
+    <arg name="image" value="/usb_cam/image_raw" />
+    <arg name="logger_save_throttled_image" value="false" />
+    <arg name="start_mongodb_record_nodelet_manager" value="$(arg start_mongodb_record_nodelet_manager)" />
+    <arg name="start_mongodb" value="$(arg start_mongodb)" />
+  </include>
+  <group ns="lifelog">
+    <include file="$(find jsk_robot_startup)/lifelog/lifelog_rgb_image.launch" >
+      <arg name="node_name" value="thorottled_color_logger" />
+      <arg name="image" value="/store_image_description/result/image/compressed" />
+      <arg name="manager" value="mongodb_record_nodelet_manager" />
+      <arg name="respawn" value="true" />
+    </include>
+  </group>
+  <node pkg="rostopic" type="rostopic" name="periodic_mongodb_trigger" if="$(arg start_periodic_mongodb_trigger)"
+        launch-prefix="bash -c 'set -x; while [ 1 ]; do ${@:0:$#-2} &quot;${@: -3}&quot; ; sleep 1000; done;' "
+        args="pub -s -1 /publish_trigger_mongodb_event roseus/StringStamped &quot;{header: auto}&quot;" />
+
+  <!-- openai -->
+  <group ns="openai">
+    <node pkg="openai_ros" type="openai_node.py" name="openai" output="screen" >
+      <rosparam command="load" file="$(arg openai_credentials_json)" />
+    </node>
+  </group>
+  <!-- store vqa result -->
+  <node name="store_image_description"
+        pkg="database_talker" type="store_image_description.py"
+        output="screen" >
+    <remap from="image" to="/usb_cam/image_raw/compressed/throttled" />
+  </node>
+  <node name="vqa_logger"
+        pkg="jsk_robot_startup" type="mongo_record.py"
+        respawn="true">
+    <rosparam subst_value="true">
+      topics:
+      - /store_image_description/result
+    </rosparam>
+  </node>
+
+  <!-- google chat -->
+  <include file="$(find google_chat_ros)/launch/google_chat.launch" >
+    <arg name="use_helper" default="false" />
+    <arg name="receiving_mode" value="pubsub" />
+    <arg name="google_cloud_credentials_json" value="$(arg google_credentials_json)" />
+    <arg name="project_id" value="$(arg google_credentials_project_id)" />
+    <arg name="subscription_id" value="chat-sub" />
+  </include>
+
+  <!-- gdrive_ros -->
+  <include file="$(find gdrive_ros)/launch/gdrive_server.launch" >
+    <arg name="settings_yaml" value="$(arg gdrive_credentials_yaml)" />
+    <arg name="respawn" value="true" />
+    <arg name="node_name" value="gdrive_ros" />
+  </include>
+</launch>
diff --git a/database_talker/launch/lovot_diary.launch b/database_talker/launch/lovot_diary.launch
@@ -0,0 +1,37 @@
+<launch>
+  <arg name="start_lovot_driver" value="false" />
+  <arg name="start_camera_driver" default="false" />
+  <arg name="camera_device" default="/dev/video0" />
+
+  <include file="$(find lovot_driver)/launch/lovot_driver.launch" if="$(arg start_lovot_driver)" />
+
+  <node name="usb_cam" pkg="usb_cam" type="usb_cam_node" output="screen" if="$(arg start_camera_driver)" respawn="true">
+    <param name="video_device" value="$(arg camera_device)" />
+    <param name="image_width" value="320" />
+    <param name="image_height" value="240" />
+    <param name="pixel_format" value="yuyv" />
+    <param name="color_format" value="yuv422p" />
+    <param name="camera_frame_id" value="usb_cam" />
+    <param name="io_method" value="mmap"/>
+    <param name="framrenate" value="10"/>
+  </node>
+
+  <include file="$(find database_talker)/launch/include/database_talker.launch" >
+    <arg name="google_credentials_json" value="$(find database_talker)/auth/jsk-lovot-f2a011bf1060.json" />
+    <arg name="google_credentials_project_id" value="jsk-lovot" />
+    <arg name="start_periodic_mongodb_trigger" default="false" />
+    <arg name="start_mongodb_record_nodelet_manager" default="false" />
+    <arg name="start_mongodb" default="false" />
+  </include>
+
+  <!-- start_periodic_mongodb_trigger -->
+  <node pkg="rostopic" type="rostopic" name="periodic_mongodb_trigger"
+        launch-prefix="bash -c 'set -x; while [ 1 ]; do ${@:0:$#-2} &quot;${@: -3}&quot; ; sleep 1000; done;' "
+        args="pub -s -1 /publish_trigger_mongodb_event lovot_driver/StringStamped &quot;{header: auto}&quot;" />
+  <!-- republish compressed/throttled image data only when /publish_trigger_mongodb_event is published -->
+  <!-- relay /lovot_driver/trigger -> /publish_trigger_mongodb_event to store /usb_cam/image_raw-->
+  <node name="relay_lovot_trigger_to_mongodb_trigger"
+	pkg="topic_tools" type="relay"
+	args="/lovot_driver/hungry_status /publish_trigger_mongodb_event" />
+
+</launch>
diff --git a/database_talker/launch/make_aibo_diary.launch b/database_talker/launch/make_aibo_diary.launch
@@ -0,0 +1,4 @@
+<launch>
+  <node pkg="database_talker" type="make_aibo_diary.py" name="make_aibo_diary" output="screen"
+	args="--prompt-type personality" />
+</launch>
diff --git a/database_talker/package.xml b/database_talker/package.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<package format="3">
+  <name>database_talker</name>
+  <version>0.0.0</version>
+  <description>The database_talker package</description>
+
+  <maintainer
+    email="[email protected]">sktometometo</maintainer>
+
+  <license>BSD</license>
+
+  <buildtool_depend>catkin</buildtool_depend>
+  <build_depend>catkin_virtualenv</build_depend>
+
+  <exec_depend>mongodb_store</exec_depend>
+  <exec_depend>usb_cam</exec_depend>
+  <exec_depend>jsk_robot_startup</exec_depend>
+  <exec_depend>openai_ros</exec_depend>
+  <exec_depend>google_chat_ros</exec_depend>
+  <exec_depend>gdrive_ros</exec_depend>
+  <exec_depend>dialogflow_task_executive</exec_depend>
+  <exec_depend>ros_google_cloud_language</exec_depend>
+
+  <export>
+    <pip_requirements>requirements.txt</pip_requirements>
+  </export>
+</package>
diff --git a/database_talker/requirements.txt b/database_talker/requirements.txt
@@ -0,0 +1,2 @@
+pymongo==3.10.1
+opencv-python==4.2.0.34
diff --git a/database_talker/rosinstall b/database_talker/rosinstall
@@ -0,0 +1,36 @@
+##
+## rosdep install --from-path . --ignores-rc
+## catkin build database_talker
+##
+#
+# database_talker demos
+# see https://github.com/jsk-ros-pkg/jsk_demos/pull/1388
+- git:
+    local-name: jsk_demos
+    uri: https://github.com/sktometometo/jsk_demos.git
+    version: PR/hoge-py
+# https://github.com/jsk-ros-pkg/jsk_robot/pull/1792
+- git:
+    local-name: jsk_robot
+    uri: https://github.com/k-okada/jsk_robot.git
+    version: ichikura_sample
+# wait until https://github.com/jsk-ros-pkg/jsk_3rdparty/pull/504
+- git:
+    local-name: jsk_3rdparty
+    uri: https://github.com/jsk-ros-pkg/jsk_3rdparty.git
+    version: master
+# lovot driver
+- git:
+    local-name: lovot_driver
+    uri: https://gitlab.jsk.imi.i.u-tokyo.ac.jp/ichikura/lovot.git
+    version: okada_ros_version
+# aibo driver
+- git:
+    local-name: aibo_driver
+    uri: https://gitlab.jsk.imi.i.u-tokyo.ac.jp/k-okada/aibo_status.git
+    version: driver
+# openai_ros, with latest endpoint
+- git:
+    local-name: openai_ros
+    uri: https://github.com/k-okada/openai_ros.git
+    version: use_ros