diff --git a/Gemfile b/Gemfile index e057636..b52dc7a 100644 --- a/Gemfile +++ b/Gemfile @@ -25,8 +25,8 @@ platforms :mingw, :x64_mingw, :mswin, :jruby do gem "tzinfo-data" end -# Performance-booster for watching directories on Windows -gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin] +# # Performance-booster for watching directories on Windows +gem "wdm"#, "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin] # Lock `http_parser.rb` gem to `v0.6.x` on JRuby builds since newer versions of the gem # do not have a Java counterpart. diff --git a/Gemfile.lock b/Gemfile.lock index 5313ad3..ce01261 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,7 +10,8 @@ GEM eventmachine (>= 0.12.9) http_parser.rb (~> 0) eventmachine (1.2.7) - ffi (1.17.0) + ffi (1.17.0-x64-mingw-ucrt) + ffi (1.17.0-x86_64-linux-gnu) forwardable-extended (2.6.0) google-protobuf (4.27.2-x64-mingw-ucrt) bigdecimal @@ -81,7 +82,7 @@ GEM tzinfo-data (1.2024.1) tzinfo (>= 1.0.0) unicode-display_width (2.5.0) - wdm (0.1.1) + wdm (0.2.0) webrick (1.8.1) PLATFORMS @@ -98,7 +99,7 @@ DEPENDENCIES minima (~> 2.5) tzinfo (>= 1, < 3) tzinfo-data - wdm (~> 0.1.1) + wdm BUNDLED WITH 2.4.9 diff --git a/assets/images/projects/gigahands/caption_itw_small.mp4 b/assets/images/projects/gigahands/caption_itw_small.mp4 new file mode 100644 index 0000000..6bdd8c3 Binary files /dev/null and b/assets/images/projects/gigahands/caption_itw_small.mp4 differ diff --git a/assets/images/projects/gigahands/caption_itw_small.mp4.mp4 b/assets/images/projects/gigahands/caption_itw_small.mp4.mp4 new file mode 100644 index 0000000..6bdd8c3 Binary files /dev/null and b/assets/images/projects/gigahands/caption_itw_small.mp4.mp4 differ diff --git a/assets/images/projects/gigahands/caption_small.mp4 b/assets/images/projects/gigahands/caption_small.mp4 new file mode 100644 index 0000000..52bc6a6 Binary files /dev/null and b/assets/images/projects/gigahands/caption_small.mp4 differ diff --git a/assets/images/projects/gigahands/example_video_small.mp4 b/assets/images/projects/gigahands/example_video_small.mp4 new file mode 100644 index 0000000..1cd11d5 Binary files /dev/null and b/assets/images/projects/gigahands/example_video_small.mp4 differ diff --git a/assets/images/projects/gigahands/motion_small.mp4 b/assets/images/projects/gigahands/motion_small.mp4 new file mode 100644 index 0000000..3a1c676 Binary files /dev/null and b/assets/images/projects/gigahands/motion_small.mp4 differ diff --git a/assets/images/projects/gigahands/teaser.jpg b/assets/images/projects/gigahands/teaser.jpg new file mode 100644 index 0000000..99b8dde Binary files /dev/null and b/assets/images/projects/gigahands/teaser.jpg differ diff --git a/assets/images/projects/gigahands/teaser_video_small-cover.jpg b/assets/images/projects/gigahands/teaser_video_small-cover.jpg new file mode 100644 index 0000000..4fe2b12 Binary files /dev/null and b/assets/images/projects/gigahands/teaser_video_small-cover.jpg differ diff --git a/assets/images/projects/gigahands/teaser_video_small.mp4 b/assets/images/projects/gigahands/teaser_video_small.mp4 new file mode 100644 index 0000000..3b77191 Binary files /dev/null and b/assets/images/projects/gigahands/teaser_video_small.mp4 differ diff --git a/research/gigahands.md b/research/gigahands.md new file mode 100644 index 0000000..3085674 --- /dev/null +++ b/research/gigahands.md @@ -0,0 +1,95 @@ +--- +layout: project +title: "GigaHands: A Massive Annotated Dataset of Bimanual Hand Activities" +authors: + - author: + name: "Rao Fu" + affiliation: 1* + link: https://freddierao.github.io/ + - author: + name: "Dingxi Zhang" + affiliation: 2* + link: https://kristen-z.github.io/ + - author: + name: "Alex Jiang" + affiliation: 1 + link: https://www.alex-jiang.com/about/ + - author: + name: "Wanjia Fu" + affiliation: 1 + link: https://wanjia-fu.com/ + - author: + name: "Austin Funk" + affiliation: 1 + link: https://austin-funk.github.io/ + - author: + name: "Daniel Ritchie" + affiliation: 1 + link: https://dritchie.github.io/ + - author: + name: "Srinath Sridhar" + affiliation: 1 + link: https://cs.brown.edu/people/ssrinath/ +affiliations: + - "1[Brown University]" + - "2[ETH Zurich]" + - "*[Equal Contributions]" +journal: "" +redirect_from: + - add redirect_path here + - another path here +--- + +{% include icons.html paper="https://arxiv.org/\f" code="https://github.com/brown-ivl/gigahands" dataset="https://github.com/brown-ivl/gigahands" %} + +{% include full_image.html path="/assets/images/projects/gigahands/teaser.jpg" alt="[Teaser Figure]" %} +**We present GigaHands, a massive annotated bimanual hand activity dataset, unlocking new possibilities for animations, robotics and beyond.** +Each column above shows an activity sequence from the dataset. The bottom row show other annotations in the dataset including text, hand shape, object shape and pose (left half images). The right half images show novel views from dynamic radiance field fitting. + +--- +# Abstract +Understanding bimanual human hand activities is a critical problem in AI and robotics. We cannot build large models of bimanual activities because existing datasets lack the scale, coverage of diverse hand activities, and detailed annotations. We introduce GigaHands, a massive annotated dataset capturing 34 hours of bimanual hand activities from **56 subjects** and **417 objects**, totaling **14k motion clips** derived from **183 million frames** paired with **84k text annotations**. Our markerless capture setup and data acquisition protocol enable fully automatic 3D hand and object estimation while minimizing the effort required for text annotation. The scale and diversity of GigaHands enable broad applications, including text-driven action synthesis, hand motion +captioning, and dynamic radiance field reconstruction. +{% include video.html path="/assets/images/projects/gigahands/teaser_video_small.mp4" alt="[Video Teaser]" %} + +--- +# Dataset Annotations +**To be released soon** + +GigaHands is a diverse, massive, and fully-annotated 3D bimanual hand activities dataset. All sequences in GigaHands are fully annotated with: detailed activity text descriptions; 3D hand shape and pose; 3D object shape, pose and appearance; hand/object segmentation masks; 2D/3D hand keypoints; camera pose. +{% include video.html path="/assets/images/projects/gigahands/example_video_small.mp4" alt="[Dataset]" %} + +--- +# Application: Text-driven Motion Generation +We showcase text-driven motion generation enabled by training a generative model with GigaHands. +{% include video.html path="/assets/images/projects/gigahands/motion_small.mp4" alt="[Motion Results]" %} + +--- +# Application: Motion Captioning + +Within Dataset +We showcase 3D hand motion captioning by training a generative model with GigaHands. +{% include video.html path="/assets/images/projects/gigahands/caption_small.mp4" alt="[Caption Result]" %} + +In-the-Wild Dataset +Using only GigaHands, we enable 3D hand motion captioning for other datasets. +{% include video.html path="/assets/images/projects/gigahands/caption_itw_small.mp4" alt="[Caption in-th-wild Result]" %} + +--- +# Citations +``` +@misc{fu2024gigahands, + title={GigaHands: A Massive Annotated Dataset of Bimanual Hand Activities}, + author={Rao Fu, Dingxi Zhang, Alex Jiang, Wanjia Fu, Austin Funk, Daniel Ritchie and Srinath Sridhar}, + year={2024}, + eprint={}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} +``` + +# Acknowledgements +This research was supported by AFOSR grant FA9550-21-1-0214, NSF CAREER grant #2143576, and ONR DURIP grant N00014-23-1-2804. We would like to thank the OpenAI Research Access Program for API support and extend our gratitude to Ellie Pavlick, Tianran Zhang, Carmen Yu, Angela Xing, Chandradeep Pokhariya, Sudarshan Harithas, Hongyu Li, Chaerin Min, Xindi Qu, Xiaoquan Liu, Hao Sun, Melvin He and Brandon Woodard. + +# Contact +Rao Fu ([contact email](rao_fu@brown.edu))