From e5d951350b5acd07bf8fe00599e4327afd4da722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=AC=E5=BD=A6=E8=B5=A4=E5=B1=8B=E5=85=88?= Date: Sat, 2 Nov 2024 03:28:06 +0900 Subject: [PATCH] Joint input actions draft impl --- KinectHandler/KinectHandler.h | 10 ++ KinectHandler/KinectHandler.vcxproj | 2 +- KinectHandler/KinectWrapper.h | 28 ++- plugin_KinectOne/Assets/Strings/en.json | 212 +++++++++++++++++------ plugin_KinectOne/KinectOne.cs | 210 ++++++++++++++++++++-- plugin_KinectOne/PackageUtils.cs | 52 +++++- plugin_KinectOne/plugin_KinectOne.csproj | 74 ++++---- 7 files changed, 482 insertions(+), 106 deletions(-) diff --git a/KinectHandler/KinectHandler.h b/KinectHandler/KinectHandler.h index 6a3dd8b..2dbacbc 100644 --- a/KinectHandler/KinectHandler.h +++ b/KinectHandler/KinectHandler.h @@ -98,6 +98,16 @@ namespace KinectHandler return trackedKinectJoints; } + property bool LeftHandClosed + { + bool get() { return kinect_->left_hand_state(); } + } + + property bool RightHandClosed + { + bool get() { return kinect_->right_hand_state(); } + } + property bool IsInitialized { bool get() { return kinect_->is_initialized(); } diff --git a/KinectHandler/KinectHandler.vcxproj b/KinectHandler/KinectHandler.vcxproj index 3dab58f..fe22402 100644 --- a/KinectHandler/KinectHandler.vcxproj +++ b/KinectHandler/KinectHandler.vcxproj @@ -95,7 +95,7 @@ - 0.2.25 + 0.3.32-alpha 4.3.0 diff --git a/KinectHandler/KinectWrapper.h b/KinectHandler/KinectWrapper.h index bdcecd5..3fa3569 100644 --- a/KinectHandler/KinectWrapper.h +++ b/KinectHandler/KinectWrapper.h @@ -26,6 +26,9 @@ class KinectWrapper JointOrientation boneOrientations[JointType_Count]; IBody* kinectBodies[BODY_COUNT] = {nullptr}; + HandState leftHandState = HandState_Unknown; + HandState rightHandState = HandState_Unknown; + WAITABLE_HANDLE h_statusChangedEvent; WAITABLE_HANDLE h_multiFrameEvent; @@ -82,18 +85,25 @@ class KinectWrapper // Copy joint positions & orientations std::copy(std::begin(joints), std::end(joints), - skeleton_positions_.begin()); + skeleton_positions_.begin()); std::copy(std::begin(boneOrientations), std::end(boneOrientations), - bone_orientations_.begin()); + bone_orientations_.begin()); + + // Get hand states + i->get_HandLeftState(&leftHandState); + i->get_HandRightState(&rightHandState); - break; + break; // Enough } + skeleton_tracked_ = false; + leftHandState = HandState_Unknown; + rightHandState = HandState_Unknown; } // Don't process color if not requested if (!camera_enabled()) return; - + // Get the color frame and process it IColorFrameReference* colorFrameReference = nullptr; multiFrame->get_ColorFrameReference(&colorFrameReference); @@ -451,6 +461,16 @@ class KinectWrapper return KinectJointTypeDictionary.at(static_cast(kinectJointType)); } + bool left_hand_state() + { + return kinectSensor && leftHandState == HandState_Closed; + } + + bool right_hand_state() + { + return kinectSensor && rightHandState == HandState_Closed; + } + std::pair CameraImageSize() { return std::make_pair(1920, 1080); diff --git a/plugin_KinectOne/Assets/Strings/en.json b/plugin_KinectOne/Assets/Strings/en.json index e3f30de..c8c3cad 100644 --- a/plugin_KinectOne/Assets/Strings/en.json +++ b/plugin_KinectOne/Assets/Strings/en.json @@ -1,57 +1,161 @@ { "language": "en", - "messages": [ - { - "id": "/Plugins/KinectOne/Statuses/Success", - "translation": "Success!\nS_OK\nEverything's good!" - }, - { - "id": "/Plugins/KinectOne/Statuses/NotAvailable", - "translation": "Sensor Unavailable!\nE_NOTAVAILABLE\nCheck if the Kinect is plugged in to your PC's USB and power plugs." - }, - { - "id": "/Plugins/KinectOne/Stages/Downloading/WiX", - "translation": "Downloading WiX Toolset..." - }, - { - "id": "/Plugins/KinectOne/Stages/Exceptions/WiX/Extraction", - "translation": "Toolset extraction failed! Exception: {0}" - }, - { - "id": "/Plugins/KinectOne/Stages/Exceptions/WiX/Installation", - "translation": "Toolset installation failed! Exception: {0}" - }, - { - "id": "/Plugins/KinectOne/Stages/Downloading/Runtime", - "translation": "Downloading Kinect for Xbox One Runtime..." - }, - { - "id": "/Plugins/KinectOne/Stages/Exceptions/Runtime/Installation", - "translation": "Runtime installation failed! Exception: {0}" - }, - { - "id": "/Plugins/KinectOne/Stages/Unpacking", - "translation": "Unpacking {0}..." - }, - { - "id": "/Plugins/KinectOne/Stages/Installing", - "translation": "Installing {0}..." - }, - { - "id": "/Plugins/KinectOne/Stages/Exceptions/Other", - "translation": "Exception: {0}" - }, - { - "id": "/Plugins/KinectOne/Stages/Dark/Error/Timeout", - "translation": "Failed to execute dark.exe in the allocated time!" - }, - { - "id": "/Plugins/KinectOne/Stages/Dark/Error/Result", - "translation": "Dark.exe exited with error code: {0}" - }, - { - "id": "/Plugins/KinectOne/Dependencies/Runtime/Name", - "translation": "Kinect for Xbox One Runtime" - } - ] + "messages": [ + { + "id": "/Plugins/KinectOne/Statuses/Success", + "translation": "Success!\nS_OK\nEverything's good!" + }, + { + "id": "/Plugins/KinectOne/Statuses/NotAvailable", + "translation": "Sensor Unavailable!\nE_NOTAVAILABLE\nCheck if the Kinect is plugged in to your PC's USB and power plugs." + }, + { + "id": "/Plugins/KinectOne/Stages/Downloading/WiX", + "translation": "Downloading WiX Toolset..." + }, + { + "id": "/Plugins/KinectOne/Stages/Exceptions/WiX/Extraction", + "translation": "Toolset extraction failed! Exception: {0}" + }, + { + "id": "/Plugins/KinectOne/Stages/Exceptions/WiX/Installation", + "translation": "Toolset installation failed! Exception: {0}" + }, + { + "id": "/Plugins/KinectOne/Stages/Downloading/Runtime", + "translation": "Downloading Kinect for Xbox One Runtime..." + }, + { + "id": "/Plugins/KinectOne/Stages/Exceptions/Runtime/Installation", + "translation": "Runtime installation failed! Exception: {0}" + }, + { + "id": "/Plugins/KinectOne/Stages/Unpacking", + "translation": "Unpacking {0}..." + }, + { + "id": "/Plugins/KinectOne/Stages/Installing", + "translation": "Installing {0}..." + }, + { + "id": "/Plugins/KinectOne/Stages/Exceptions/Other", + "translation": "Exception: {0}" + }, + { + "id": "/Plugins/KinectOne/Stages/Dark/Error/Timeout", + "translation": "Failed to execute dark.exe in the allocated time!" + }, + { + "id": "/Plugins/KinectOne/Stages/Dark/Error/Result", + "translation": "Dark.exe exited with error code: {0}" + }, + { + "id": "/Plugins/KinectOne/Dependencies/Runtime/Name", + "translation": "Kinect for Xbox One Runtime" + }, + { + "id": "/JointsEnum/JointHead", + "translation": "Head" + }, + { + "id": "/JointsEnum/JointNeck", + "translation": "Neck" + }, + { + "id": "/JointsEnum/JointSpineShoulder", + "translation": "Spine (Shoulders)" + }, + { + "id": "/JointsEnum/JointShoulderLeft", + "translation": "Left Shoulder" + }, + { + "id": "/JointsEnum/JointElbowLeft", + "translation": "Left Elbow" + }, + { + "id": "/JointsEnum/JointWristLeft", + "translation": "Left Wrist" + }, + { + "id": "/JointsEnum/JointHandLeft", + "translation": "Left Hand" + }, + { + "id": "/JointsEnum/JointHandTipLeft", + "translation": "Left Hand Tip" + }, + { + "id": "/JointsEnum/JointThumbLeft", + "translation": "Left Thumb" + }, + { + "id": "/JointsEnum/JointShoulderRight", + "translation": "Right Shoulder" + }, + { + "id": "/JointsEnum/JointElbowRight", + "translation": "Right Elbow" + }, + { + "id": "/JointsEnum/JointWristRight", + "translation": "Right Wrist" + }, + { + "id": "/JointsEnum/JointHandRight", + "translation": "Right Hand" + }, + { + "id": "/JointsEnum/JointHandTipRight", + "translation": "Right Hand Tip" + }, + { + "id": "/JointsEnum/JointThumbRight", + "translation": "Right Thumb" + }, + { + "id": "/JointsEnum/JointSpineMiddle", + "translation": "Spine (Middle)" + }, + { + "id": "/JointsEnum/JointSpineWaist", + "translation": "Waist" + }, + { + "id": "/JointsEnum/JointHipLeft", + "translation": "Left Hip" + }, + { + "id": "/JointsEnum/JointKneeLeft", + "translation": "Left Knee" + }, + { + "id": "/JointsEnum/JointFootLeft", + "translation": "Left Foot" + }, + { + "id": "/JointsEnum/JointFootTipLeft", + "translation": "Left Foot Tip" + }, + { + "id": "/JointsEnum/JointHipRight", + "translation": "Right Hip" + }, + { + "id": "/JointsEnum/JointKneeRight", + "translation": "Right Knee" + }, + { + "id": "/JointsEnum/JointFootRight", + "translation": "Right Foot" + }, + { + "id": "/JointsEnum/JointFootTipRight", + "translation": "Right Foot Tip" + }, + { + "id": "/JointsEnum/JointManual", + "translation": "Manual" + } + ] } \ No newline at end of file diff --git a/plugin_KinectOne/KinectOne.cs b/plugin_KinectOne/KinectOne.cs index 56683d0..a81c160 100644 --- a/plugin_KinectOne/KinectOne.cs +++ b/plugin_KinectOne/KinectOne.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See LICENSE in the project root for license information. using System; +using System.Collections.Generic; using System.Collections.ObjectModel; using System.ComponentModel.Composition; using System.Drawing; @@ -39,29 +40,97 @@ public class KinectOne : KinectHandler.KinectHandler, ITrackingDevice public bool IsAppOrientationSupported => true; public object SettingsInterfaceRoot => null; public WriteableBitmap CameraImage { get; set; } + public static IAmethystHost HostStatic { get; set; } + + private readonly GestureDetector + _pauseDetectorLeft = new(), + _pauseDetectorRight = new(), + _pointDetectorLeft = new(), + _pointDetectorRight = new(); public ObservableCollection TrackedJoints { get; } = // Prepend all supported joints to the joints list new(Enum.GetValues() .Where(x => x is not TrackedJointType.JointManual) - .Select(x => new TrackedJoint { Name = x.ToString(), Role = x })); + .Select(x => new TrackedJoint + { + Name = HostStatic?.RequestLocalizedString($"/JointsEnum/{x.ToString()}") ?? x.ToString(), + Role = x, + SupportedInputActions = x switch + { + TrackedJointType.JointHandLeft => + [ + new KeyInputAction + { + Name = "Left Pause", Description = "Left hand pause gesture", + Guid = "5E4680F9-F232-4EA1-AE12-E96F7F8E0CC1", GetHost = () => HostStatic + }, + new KeyInputAction + { + Name = "Left Point", Description = "Left hand point gesture", + Guid = "8D83B89D-5FBD-4D52-B626-4E90BDD26B08", GetHost = () => HostStatic + }, + new KeyInputAction + { + Name = "Left Grab", Description = "Left hand grab gesture", + Guid = "E383258F-5918-4F1C-BC66-7325DB1F07E8", GetHost = () => HostStatic + } + ], + TrackedJointType.JointHandRight => + [ + new KeyInputAction + { + Name = "Right Pause", Description = "Right hand pause gesture", + Guid = "B8389FA6-75EF-4509-AEC2-1758AFE41D95", GetHost = () => HostStatic + }, + new KeyInputAction + { + Name = "Right Point", Description = "Right hand point gesture", + Guid = "C58EBCFE-0DF5-40FD-ABC1-06B415FA51BE", GetHost = () => HostStatic + }, + new KeyInputAction + { + Name = "Right Grab", Description = "Right hand grab gesture", + Guid = "801336BE-5BD5-4881-A390-D57D958592EF", GetHost = () => HostStatic + } + ], + _ => [] + } + })); public string DeviceStatusString => PluginLoaded ? DeviceStatus switch { 0 => Host.RequestLocalizedString("/Plugins/KinectOne/Statuses/Success"), 1 => Host.RequestLocalizedString("/Plugins/KinectOne/Statuses/NotAvailable"), - _ => $"Undefined: {DeviceStatus}\nE_UNDEFINED\nSomething weird has happened, though we can't tell what." + _ => $"Undefined: {DeviceStatus}\nE_UNDEFINED\nSomething weird has happened, although we can't tell what." } - : $"Undefined: {DeviceStatus}\nE_UNDEFINED\nSomething weird has happened, though we can't tell what."; + : $"Undefined: {DeviceStatus}\nE_UNDEFINED\nSomething weird has happened, although we can't tell what."; public Uri ErrorDocsUri => new($"https://docs.k2vr.tech/{Host?.DocsLanguageCode ?? "en"}/one/troubleshooting/"); public void OnLoad() { + // Backup the plugin host + HostStatic = Host; PluginLoaded = true; CameraImage = new WriteableBitmap(CameraImageWidth, CameraImageHeight); + + try + { + // Re-generate joint names + lock (Host.UpdateThreadLock) + { + for (var i = 0; i < TrackedJoints.Count; i++) + TrackedJoints[i] = TrackedJoints[i].WithName(Host?.RequestLocalizedString( + $"/JointsEnum/{TrackedJoints[i].Role.ToString()}") ?? TrackedJoints[i].Role.ToString()); + } + } + catch (Exception e) + { + Host?.Log($"Error setting joint names! Message: {e.Message}", LogSeverity.Error); + } } public void Initialize() @@ -112,14 +181,86 @@ public void Update() }); // Update camera feed - if (!IsCameraEnabled) return; - CameraImage.DispatcherQueue.TryEnqueue(async () => + if (IsCameraEnabled) + CameraImage.DispatcherQueue.TryEnqueue(async () => + { + var buffer = GetImageBuffer(); // Read from Kinect + if (buffer is null || buffer.Length <= 0) return; + await CameraImage.PixelBuffer.AsStream().WriteAsync(buffer); + CameraImage.Invalidate(); // Enqueue for preview refresh + }); + + + // Update gestures + if (trackedJoints.Count != 25) return; + + try { - var buffer = GetImageBuffer(); // Read from Kinect - if (buffer is null || buffer.Length <= 0) return; - await CameraImage.PixelBuffer.AsStream().WriteAsync(buffer); - CameraImage.Invalidate(); // Enqueue for preview refresh - }); + var shoulderLeft = trackedJoints[(int)TrackedJointType.JointShoulderLeft].Position; + var shoulderRight = trackedJoints[(int)TrackedJointType.JointShoulderRight].Position; + var elbowLeft = trackedJoints[(int)TrackedJointType.JointElbowLeft].Position; + var elbowRight = trackedJoints[(int)TrackedJointType.JointElbowRight].Position; + var handLeft = trackedJoints[(int)TrackedJointType.JointWristLeft].Position; + var handRight = trackedJoints[(int)TrackedJointType.JointWristRight].Position; + + // >0.9f when elbow is not bent and the arm is straight : LEFT + var armDotLeft = Vector3.Dot( + Vector3.Normalize(elbowLeft - shoulderLeft), + Vector3.Normalize(handLeft - elbowLeft)); + + // >0.9f when the arm is pointing down : LEFT + var armDownDotLeft = Vector3.Dot( + new Vector3(0.0f, -1.0f, 0.0f), + Vector3.Normalize(handLeft - elbowLeft)); + + // >0.4f <0.6f when the arm is slightly tilted sideways : RIGHT + var armTiltDotLeft = Vector3.Dot( + Vector3.Normalize(shoulderLeft - shoulderRight), + Vector3.Normalize(handLeft - elbowLeft)); + + // >0.9f when elbow is not bent and the arm is straight : LEFT + var armDotRight = Vector3.Dot( + Vector3.Normalize(elbowRight - shoulderRight), + Vector3.Normalize(handRight - elbowRight)); + + // >0.9f when the arm is pointing down : RIGHT + var armDownDotRight = Vector3.Dot( + new Vector3(0.0f, -1.0f, 0.0f), + Vector3.Normalize(handRight - elbowRight)); + + // >0.4f <0.6f when the arm is slightly tilted sideways : RIGHT + var armTiltDotRight = Vector3.Dot( + Vector3.Normalize(shoulderRight - shoulderLeft), + Vector3.Normalize(handRight - elbowRight)); + + /* Trigger the detected gestures */ + + if (TrackedJoints[(int)TrackedJointType.JointHandLeft].SupportedInputActions.IsUsed(0, out var pauseActionLeft)) + Host.ReceiveKeyInput(pauseActionLeft, _pauseDetectorLeft.Update(armDotLeft > 0.9f && armTiltDotLeft is > 0.4f and < 0.7f)); + + if (TrackedJoints[(int)TrackedJointType.JointHandRight].SupportedInputActions.IsUsed(0, out var pauseActionRight)) + Host.ReceiveKeyInput(pauseActionRight, _pauseDetectorRight.Update(armDotRight > 0.9f && armTiltDotRight is > 0.4f and < 0.7f)); + + if (TrackedJoints[(int)TrackedJointType.JointHandLeft].SupportedInputActions.IsUsed(1, out var pointActionLeft)) + Host.ReceiveKeyInput(pointActionLeft, _pointDetectorLeft + .Update(armDotLeft > 0.9f && armTiltDotLeft is > -0.5f and < 0.5f && armDownDotLeft is > -0.3f and < 0.7f)); + + if (TrackedJoints[(int)TrackedJointType.JointHandRight].SupportedInputActions.IsUsed(1, out var pointActionRight)) + Host.ReceiveKeyInput(pointActionRight, _pointDetectorRight + .Update(armDotRight > 0.9f && armTiltDotRight is > -0.5f and < 0.5f && armDownDotRight is > -0.3f and < 0.7f)); + + if (TrackedJoints[(int)TrackedJointType.JointHandLeft].SupportedInputActions.IsUsed(2, out var grabActionLeft)) + Host.ReceiveKeyInput(grabActionLeft, LeftHandClosed); + + if (TrackedJoints[(int)TrackedJointType.JointHandRight].SupportedInputActions.IsUsed(2, out var grabActionRight)) + Host.ReceiveKeyInput(grabActionRight, RightHandClosed); + + /* Trigger the detected gestures */ + } + catch (Exception ex) + { + Host?.Log(ex); + } } public void SignalJoint(int jointId) @@ -142,7 +283,7 @@ public override void StatusChangedHandler() public Func MapCoordinateDelegate => MapCoordinate; } -internal static class PoseUtils +internal static class Utils { public static Quaternion Safe(this Quaternion q) { @@ -158,4 +299,51 @@ public static Vector3 Safe(this Vector3 v) ? Vector3.Zero // Return a placeholder position vector : v; // If everything is fine, return the actual orientation } + + public static T At(this SortedSet set, int at) + { + return set.ElementAt(at); + } + + public static bool At(this SortedSet set, int at, out T result) + { + try + { + result = set.ElementAt(at); + } + catch + { + result = default; + return false; + } + + return true; + } + + public static bool IsUsed(this SortedSet set, int at) + { + return set.At(at, out var action) && (KinectOne.HostStatic?.CheckInputActionIsUsed(action) ?? false); + } + + public static bool IsUsed(this SortedSet set, int at, out IKeyInputAction action) + { + return set.At(at, out action) && (KinectOne.HostStatic?.CheckInputActionIsUsed(action) ?? false); + } + + public static TrackedJoint WithName(this TrackedJoint joint, string name) + { + return new TrackedJoint + { + Name = name, + Role = joint.Role, + Acceleration = joint.Acceleration, + AngularAcceleration = joint.AngularAcceleration, + AngularVelocity = joint.AngularVelocity, + Orientation = joint.Orientation, + Position = joint.Position, + SupportedInputActions = joint.SupportedInputActions, + TrackingState = joint.TrackingState, + Velocity = joint.Velocity + }; + } } \ No newline at end of file diff --git a/plugin_KinectOne/PackageUtils.cs b/plugin_KinectOne/PackageUtils.cs index 32623be..07cb6e6 100644 --- a/plugin_KinectOne/PackageUtils.cs +++ b/plugin_KinectOne/PackageUtils.cs @@ -1,4 +1,6 @@ -using System; +using Amethyst.Plugins.Contract; +using System; +using System.Diagnostics; using System.IO; using Windows.ApplicationModel; using Windows.Storage; @@ -45,4 +47,52 @@ public static void CopyToFolder(this DirectoryInfo source, string destination, b foreach (var newPath in source.GetFiles("*.*", SearchOption.AllDirectories)) newPath.CopyTo(newPath.FullName.Replace(source.FullName, destination), true); } +} + +public class GestureDetector +{ + private bool Value { get; set; } + private bool ValueBlock { get; set; } + private Stopwatch Timer { get; set; } = new(); + + public bool Update(bool value) + { + // ReSharper disable once ConvertIfStatementToSwitchStatement + if (!Value && value) + { + //Console.WriteLine("Restarting gesture timer..."); + ValueBlock = false; + Timer.Restart(); + Value = true; + return false; + } + + if (!Value && !value) + { + //Console.WriteLine("Resetting gesture timer..."); + ValueBlock = false; + Timer.Reset(); + return false; + } + + Value = value; + + switch (Timer.ElapsedMilliseconds) + { + case >= 1000 when !ValueBlock: + //Console.Write("Gesture detected! "); + KinectOne.HostStatic?.PlayAppSound(SoundType.Focus); + ValueBlock = true; + return true; + case >= 3000 when ValueBlock: + //Console.Write("Restarting timer..."); + KinectOne.HostStatic?.PlayAppSound(SoundType.Focus); + ValueBlock = false; + Timer.Restart(); + return true; + default: + //Console.WriteLine("Gesture detected! Waiting for the timer..."); + return false; + } + } } \ No newline at end of file diff --git a/plugin_KinectOne/plugin_KinectOne.csproj b/plugin_KinectOne/plugin_KinectOne.csproj index 7d19a41..ba122dc 100644 --- a/plugin_KinectOne/plugin_KinectOne.csproj +++ b/plugin_KinectOne/plugin_KinectOne.csproj @@ -1,40 +1,44 @@  - - net7.0 - 10.0.17763.0 - 10.0.22621.0 - Windows - plugin_KinectOne - win10-x64 - true - x64 - true - + + net7.0 + 10.0.17763.0 + 10.0.22621.0 + Windows + plugin_KinectOne + win10-x64 + x64 + true + latest + - - - - - - - - + + true + - - - + + + + + + + + - - - Always - - - Always - Microsoft.Kinect.dll - - + + + - - - - + + + Always + + + Always + Microsoft.Kinect.dll + + + + + + + \ No newline at end of file