bleh

README.md

@@ -0,0 +1,93 @@
+# FLAIR: Feeding via Long-horizon AcquIsition of Realistic dishes
+
+## Hardware Requirements
+
+The following robot plaforms are supported:
+- Kinova 6-DoF or 7-DoF robot arm
+- Franka Emika Panda 7-DoF robot arm
+
+FLAIR's uses a custom feeding utensil. You can access a list of components and detailed specifications [here](https://drive.google.com/drive/u/1/folders/1WjtiHdZtLfJFWJ-NYM1NFTaGlcZTuRHH).
+
+## Setting up FLAIR
+
+### Dependencies
+
+Ensure you have the following dependencies installed:
+- [Grounded-Segment-Anything](https://github.com/IDEA-Research/Grounded-Segment-Anything)
+- [Depth-Anything](https://github.com/LiheYoung/Depth-Anything)
+
+Further, download weights for the spaghetti segmentation model from [this link](https://drive.google.com/file/d/1MCgjYcFv6nTxO-e3mlMtdZIzmOgu3rNb/view?usp=sharing) and place it in `FLAIR/bite_acquisition`.
+
+For Kinova arms, use the kortex API included in this repository. For Franka arms, use Polymetis combined with Robot-Lighting. Setup instructions for Franka can be found [here](https://github.com/jhejna/robot-lightning).
+
+### ROS Workspace Setup
+
+Create and setup the feeding workspace:
+```
+mkdir -p flair_ws/src
+cd flair_ws/src
+git clone https://github.com/empriselab/FLAIR.git
+```
+
+Build the workspace after ensuring all dependencies are met:
+```
+catkin build
+cd ..
+source devel/setup.sh
+```
+
+### Feeding Utensil Setup
+
+Follow these steps to set up and control the feeding utensil using the Dynamixel SDK:
+
+1. Install Dynamixel SDK: [Dynamixel SDK Installation Guide](https://emanual.robotis.com/docs/en/software/dynamixel/dynamixel_sdk/download/#repository)
+
+2. Install Dynamixel Wizard 2.0: [Dynamixel Wizard 2.0 Installation Guide](https://emanual.robotis.com/docs/en/software/dynamixel/dynamixel_wizard2/)
+
+3. Follow the Quick Start Guide: [Dynamixel Quick Start Guide for Basic Setup and Assembly](https://emanual.robotis.com/docs/en/dxl/dxl-quick-start-guide/)
+
+4. Configure Dynamixel
+   - Set the IDs to 50 and 100 and scan Dynamixel. Refer to Section 6 in the Dynamixel Wizard 2.0 link.
+   - Manually change the baud rate to 1Mbps if scanning shows a different value.
+
+5. Start the feeding utensil: Source the workspace, plug in the wrist joint, add port permissions, and run the driver:
+   ```
+   source devel/setup.bash
+   sudo chmod a+rw /dev/ttyUSB0
+   rosrun wrist_driver_ros wrist_driver
+   ```
+
+6. Test the feeding utensil:
+   ```
+   rostopic pub -1 /cmd_wrist_joint_angles wrist_driver_interfaces/SimpleJointAngleCommand '{q0: 0.0, q1: 0.0}'
+   ```
+
+## Running FLAIR
+
+TO run FLAIR, follow these steps, running each command in a separate terminal window. Remember to source your environment in each terminal to ensure all dependencies are properly loaded.
+
+1. Start the ROS core:
+   ```
+   roscore
+   ```
+
+2. Open Rviz for visualizing robot actions:
+   ```
+   rviz
+   ```
+
+3. Start the wrist driver:
+   ```
+   rosrun wrist_driver_ros wrist_driver
+   ```
+
+4. Launch the bite acquisition process for Kinova or Franka setups:
+   ```
+   roslaunch bite_acquisition kinova/franka.launch
+   ```
+
+5. In a new terminal, set the OpenAI API key and run the main feeding bot script (these should be run in the same terminal):
+   ```
+   export OPENAI_API_KEY=<your_api_key_here>
+   python feeding_bot.py
+   ```

assets/make_gifs (1).py

@@ -0,0 +1,17 @@
+import os
+import numpy as np
+import cv2
+from PIL import Image, ImageSequence
+import imageio
+
+
+def create_gifs(video_folder):
+    for fn in os.listdir(video_folder):
+        output_gif_fn = fn[:-4] + '.gif'
+        cmd = 'ffmpeg -y -i %s -vf "fps=10,scale=600:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" -loop 0 %s'%(os.path.join(video_folder, fn), os.path.join(video_folder, output_gif_fn))
+        #cmd = 'ffmpeg -y -i %s -vf "crop=1200:1080:360:0,eq=gamma=1.4:saturation=1.3,fps=10,scale=400:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" -loop 0 %s'%(os.path.join(video_folder, fn), os.path.join(video_folder, output_gif_fn)) # for cropping / brightness increase for dessert plates
+        os.system(cmd)
+
+if __name__ == '__main__':
+    create_gifs('spaghetti/edits')
+

bite_acquisition/.gitignore

@@ -0,0 +1,17 @@
+/checkpoints/*
+/dependencies/*
+/spaghetti_checkpoints/*
+scripts/mashed_potatoes/*
+scripts/outputs/*
+scripts/log/*
+scripts/old_log/*
+scripts/skill_library.py
+src/controller.cpp
+launch/controller_utensil.launch
+task_planning_tests/noodle_plates/log/*
+task_planning_tests/noodle_plates/outputs/*
+task_planning_tests/study_plates/log/*
+task_planning_tests/study_plates/outputs/*
+*/__pycache__/*
+*.bag
+*.zip

bite_acquisition/CMakeLists.txt

@@ -0,0 +1,79 @@
+cmake_minimum_required(VERSION 3.0.2)
+project(bite_acquisition)
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
+
+## Compile as C++11, supported in ROS Kinetic and newer
+add_compile_options(-std=c++11)
+add_definitions(-D_OS_UNIX)
+
+find_package(catkin REQUIRED COMPONENTS
+  roscpp
+  rospy
+  std_msgs
+  geometry_msgs
+  sensor_msgs
+  message_generation
+)
+
+add_message_files(
+  FILES
+  CartesianState.msg
+)
+
+add_service_files(
+    FILES
+    JointCommand.srv
+    PoseCommand.srv
+    PoseWaypointsCommand.srv
+    TwistCommand.srv
+    Segmentation.srv
+    VQA.srv
+)
+
+generate_messages(
+    DEPENDENCIES
+    geometry_msgs
+    sensor_msgs
+)
+
+find_package(Boost REQUIRED COMPONENTS system)
+link_directories(${PROJECT_SOURCE_DIR}/lib/release)
+
+catkin_package(
+  INCLUDE_DIRS
+    include
+  CATKIN_DEPENDS
+    roscpp
+    rospy
+    std_msgs
+    geometry_msgs
+    sensor_msgs
+    message_runtime
+)
+
+include_directories(
+  include
+  ${Boost_INCLUDE_DIRS}
+  ${catkin_INCLUDE_DIRS}
+  ${libkindrv_INCLUDE_DIRS}
+  ${PROJECT_SOURCE_DIR}/include/client
+  ${PROJECT_SOURCE_DIR}/include/client_stubs
+  ${PROJECT_SOURCE_DIR}/include/messages
+  ${PROJECT_SOURCE_DIR}/include/common
+)
+
+add_executable(controller src/controller.cpp)
+
+set(KINOVA_ARM_LIBS ${catkin_LIBRARIES})
+set(KINOVA_ARM_LIBS
+    ${KINOVA_ARM_LIBS}
+    -l:libKortexApiCpp_without_protobuf.a
+    -l:libKortexApiCpp.a)
+
+## Specify libraries to link a library or executable target against
+target_link_libraries(controller
+  ${catkin_LIBRARIES}
+  ${Boost_LIBRARIES}
+  ${KINOVA_ARM_LIBS}
+)

bite_acquisition/README.md

@@ -0,0 +1,17 @@
+Directory Structure
+- include
+    - bite_acquisition
+- src
+    - bite_acquisition
+- scripts
+    - spaghetti
+    - grounded_sam
+- msg
+    - CartesianState.msg
+- srv
+    - JointCommand.srv
+    - PoseCommand.srv
+    - TareFTSensors.srv
+    - TwistCommand.srv
+- checkpoints
+- launch