From db5bb438f4abc60e7ba2228164c6f81f2e599ebe Mon Sep 17 00:00:00 2001 From: Max Palme Date: Tue, 5 Mar 2024 17:14:01 +0000 Subject: [PATCH 1/3] Change the target colour palette values for joint confidence targets in the render texture as they appear altered from the expected values --- .../Runtime/Scripts/JointOcclusion.cs | 141 +++++++++++++++++- Packages/Tracking/CHANGELOG.md | 24 +-- 2 files changed, 134 insertions(+), 31 deletions(-) diff --git a/Packages/Tracking Preview/AggregationProviders/Runtime/Scripts/JointOcclusion.cs b/Packages/Tracking Preview/AggregationProviders/Runtime/Scripts/JointOcclusion.cs index acb2049ea9..8db8657b00 100644 --- a/Packages/Tracking Preview/AggregationProviders/Runtime/Scripts/JointOcclusion.cs +++ b/Packages/Tracking Preview/AggregationProviders/Runtime/Scripts/JointOcclusion.cs @@ -1,3 +1,7 @@ + +// Tue this to log out the render colours seen by the camera in the render texture +#define LOGGING_RENDER_COLOURS + using Leap; using Leap.Unity; using Leap.Unity.Encoding; @@ -19,12 +23,82 @@ public class JointOcclusion : MonoBehaviour Camera cam; Texture2D tex; Rect regionToReadFrom; + + // Colours used for the capsule hand joint spheres Color[] occlusionSphereColorsLeft; Color[] occlusionSphereColorsRight; + + /// + /// For some reason the capsule hand colours that are rendered to the occlusion camera are not the original colours, but appear darker + /// This set is based on the colours as viewed in the render texture. The ComputMajorColour codeis used to log out the colours + /// found when looking for joints, when a hand is placed so all joints can be seen + /// + Color[] shiftedOcclusionSphereColorsLeft = + { + new Color(1, 0, 0, 1), + new Color(0.9294118f, 0.003921569f, 0, 1), + new Color(0.8627452f, 0.003921569f, 0, 1), + new Color(0.8000001f, 0.007843138f, 0, 1), + new Color(0.7372549f, 0.01568628f, 0, 1), + new Color(0.6784314f, 0.01960784f, 0, 1), + new Color(0.6235294f, 0.02745098f, 0, 1), + new Color(0.572549f, 0.03921569f, 0, 1), + new Color(0.0509804f, 0.5215687f, 0, 1), + new Color(0.4745098f, 0.0627451f, 0, 1), + new Color(0.4313726f, 0.07843138f, 0, 1), + new Color(0.3882353f, 0.09803922f, 0, 1), + new Color(0.0509804f, 0.5215687f, 0, 1), + new Color(0.3098039f, 0.1372549f, 0, 1), + new Color(0.2745098f, 0.1607843f, 0, 1), + new Color(0.2431373f, 0.1843137f, 0, 1), + new Color(0.2156863f, 0.2156863f, 0, 1), + new Color(0.1843137f, 0.2431373f, 0, 1), + new Color(0.1607843f, 0.2745098f, 0, 1), + new Color(0.1372549f, 0.3098039f, 0, 1) + }; + + /// + /// For some reason the capsule hand colours that are rendered to the occlusion camera are not the original colours, but appear darker + /// This set is based on the colours as viewed in the render texture. The ComputMajorColour codeis used to log out the colours + /// found when looking for joints, when a hand is placed so all joints can be seen + /// + Color[] shiftedOcclusionSphereColorsRight = + { + new Color(1f, 0.8313726f, 0f, 1f), + new Color(0.9294118f, 0.7725491f, 0.003921569f, 1f), + new Color(0.8627452f, 0.7176471f, 0.007843138f, 1f), + new Color(0.8000001f, 0.6666667f, 0.01176471f, 1f), + new Color(0.7372549f, 0.6156863f, 0.01568628f, 1f), + new Color(0.6784314f, 0.5647059f, 0.02352941f, 1f), + new Color(0.6235294f, 0.5215687f, 0.03137255f, 1f), + new Color(0.572549f, 0.4784314f, 0.04313726f, 1f), + new Color(0.0509804f, 0.04313726f, 0.5294118f, 1f), + new Color(0.4745098f, 0.3960785f, 0.07058824f, 1f), + new Color(0.4313726f, 0.3607843f, 0.08627451f, 1f), + new Color(0.3882353f, 0.3254902f, 0.1019608f, 1f), + new Color(0.3490196f, 0.2901961f, 0.1215686f, 1f), + new Color(0.3098039f, 0.2588235f, 0.145098f, 1f), + new Color(0.2745098f, 0.2313726f, 0.1686275f, 1f), + new Color(0.2431373f, 0.2039216f, 0.1921569f, 1f), + new Color(0.2156863f, 0.1803922f, 0.2196079f, 1f), + new Color(0.1843137f, 0.1568628f, 0.2509804f, 1f), + new Color(0.1607843f, 0.1333333f, 0.282353f, 1f), + new Color(0.1372549f, 0.1137255f, 0.3176471f, 1f), + }; + Mesh cubeMesh; Material cubeMaterial; string layerName; + private bool setup = false; + public void Update() + { + if (!setup) + { + Setup(); + } + } + /// /// this sets everything up, so that joint occlusion works (eg. rendering layers) /// @@ -65,11 +139,9 @@ public void Setup() occlusionHandLeft.SphereColors = occlusionSphereColorsLeft; occlusionHandRight.SphereColors = occlusionSphereColorsRight; - - cubeMesh = createCubeMesh(); cubeMaterial = new Material(Shader.Find("Standard")); - + setup = true; } private Mesh createCubeMesh() @@ -139,12 +211,13 @@ public float[] Confidence_JointOcclusion(float[] confidences, Transform deviceOr tex.ReadPixels(regionToReadFrom, 0, 0); tex.Apply(); - // loop through all joints that are visible (all joints that are rendered on a capsule hand), // and save how many pixels of a joint can be seen (in pixelsSeenCount) // and how many pixels of a joint would be seen if the joint was not occluded at all (in optimalPixelsCount) int[] pixelsSeenCount = new int[confidences.Length]; int[] optimalPixelsCount = new int[confidences.Length]; + Color[] majorColours = new Color[occlusionSphereColorsLeft.Length]; + foreach (var finger in hand.Fingers) { for (int j = 0; j < 4; j++) @@ -167,7 +240,6 @@ public float[] Confidence_JointOcclusion(float[] confidences, Transform deviceOr optimalPixelsCount[key] = (int)(Mathf.PI * radius * radius); - // only count pixels around where the sphere is supposed to be (+5 pixel margin) int margin = 5; int x0 = Mathf.Clamp((int)(screenPosCenter.x - radius - margin), 0, tex.width); @@ -179,11 +251,26 @@ public float[] Confidence_JointOcclusion(float[] confidences, Transform deviceOr if (hand.IsLeft) { - pixelsSeenCount[key] = tempPixels.Where(x => DistanceBetweenColors(x, occlusionSphereColorsLeft[capsuleHandKey]) < 0.01f).Count(); + pixelsSeenCount[key] = tempPixels.Where(x => DistanceBetweenColors(x, shiftedOcclusionSphereColorsLeft[capsuleHandKey]) < 0.01f).Count(); + +#if LOGGING_RENDER_COLOURS + majorColours[capsuleHandKey] = FindDominantColourInTarget(tempPixels); + + Debug.Log($"{capsuleHandKey} Original sphere colour {occlusionSphereColorsLeft[capsuleHandKey].r},{occlusionSphereColorsLeft[capsuleHandKey].g},{occlusionSphereColorsLeft[capsuleHandKey].b}, {occlusionSphereColorsLeft[capsuleHandKey].a} " + + $"Dominant colour actually found {majorColours[capsuleHandKey].r}, {majorColours[capsuleHandKey].g}, {majorColours[capsuleHandKey].b}, {majorColours[capsuleHandKey].a} " + + $"Difference {DistanceBetweenColors(majorColours[capsuleHandKey], occlusionSphereColorsLeft[capsuleHandKey])}"); +#endif } else { - pixelsSeenCount[key] = tempPixels.Where(x => DistanceBetweenColors(x, occlusionSphereColorsRight[capsuleHandKey]) < 0.01f).Count(); + pixelsSeenCount[key] = tempPixels.Where(x => DistanceBetweenColors(x, shiftedOcclusionSphereColorsRight[capsuleHandKey]) < 0.01f).Count(); + +#if LOGGING_RENDER_COLOURS + majorColours[capsuleHandKey] = FindDominantColourInTarget(tempPixels); + Debug.Log($"{capsuleHandKey} Original sphere colour {occlusionSphereColorsLeft[capsuleHandKey].r},{occlusionSphereColorsLeft[capsuleHandKey].g},{occlusionSphereColorsLeft[capsuleHandKey].b}, {occlusionSphereColorsLeft[capsuleHandKey].a} " + + $"Dominant colour actually found {majorColours[capsuleHandKey].r}, {majorColours[capsuleHandKey].g}, {majorColours[capsuleHandKey].b}, {majorColours[capsuleHandKey].a} " + + $"Difference {DistanceBetweenColors(majorColours[capsuleHandKey], occlusionSphereColorsLeft[capsuleHandKey])}"); +#endif } } } @@ -199,10 +286,48 @@ public float[] Confidence_JointOcclusion(float[] confidences, Transform deviceOr return confidences; } - float DistanceBetweenColors(Color color1, Color color2) + private float DistanceBetweenColors(Color color1, Color color2) { Color colorDifference = color1 - color2; Vector3 diffVetor = new Vector3(colorDifference.r, colorDifference.g, colorDifference.b); return diffVetor.magnitude; } + + private Color FindDominantColourInTarget(Color[] targetPixelRegion) + { + Dictionary colourStats = new Dictionary(); + + // Can get both black defined as 0,0,0,0 and 0,0,0,1 + Color black = new Color(0, 0, 0, 0); + + foreach (Color pixel in targetPixelRegion) + { + // Screen out white and black pixel(s) + if (pixel != Color.white && pixel != black && pixel != Color.black) + { + if (colourStats.ContainsKey(pixel)) + { + colourStats[pixel] = colourStats[pixel] + 1; + } + else + { + colourStats.Add(pixel, 1); + } + } + } + + int maxCount = 0; + Color mostCommonColor = Color.black; + + foreach (Color color in colourStats.Keys) + { + if (colourStats[color] > maxCount) + { + maxCount = colourStats[color]; + mostCommonColor = color; + } + } + + return mostCommonColor; + } } \ No newline at end of file diff --git a/Packages/Tracking/CHANGELOG.md b/Packages/Tracking/CHANGELOG.md index 2171208369..11c530b2fc 100644 --- a/Packages/Tracking/CHANGELOG.md +++ b/Packages/Tracking/CHANGELOG.md @@ -14,13 +14,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Android v5.17.1 ### Added -- LeapServiceProvider accessor for world space psoition of the Tracking Camera -- LeapXRServiceProvider accessor for world space psoition of the Tracking Camera -- LeapServiceProvider accessor for world space position of the Tracking Camera -- LeapXRServiceProvider accessor for world space position of the Tracking Camera -- (Physical Hands) Ability to ignore collisions on single object or all children -- (Physical Hands) Added toggle to GrabHelper to allow kinematic object movement -- (Physical Hands) Ignore Physical hands component can now choose which hand(s) it should be applied to ### Changed - (Config) Additional uses of Config marked as Obsolete @@ -28,26 +21,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Example content to be split by XR, Tabletop and URP Examples - Combined example content to be part of the main Ultraleap Tracking .unitypackage when importing via .unitypackage - Added fog and gradient sky for all XR example scenes -- (Hand Rays) Exposed dot product used to test if the hand is facing camera -- (Hinting) Added support for startup setting of Hand Tracking Hints via the Ultraleap Settings window -- (Hinting) Added support for runtime changing of Hand Tracking Hints via static HandTrackingHintManager -- Access of Physical Hands extensions -- Added public accessors to various Physical Hands utilities ### Fixed - Errors in Editor when using pre-2023.3.18 LTS due to FindObjectByType issue - (Physical Hands) Objects are sticky when they ignore collision with hard contact hands -- (Physical Hands) Ability to toggle ignore Physical hands options from the inspector at runtime. -- (Locomotion) IsPinching wouldn't fire when between Activate and Deactivate values in LightweightPinchDetector -- (Physical Hands) Soft contact button difficult to press in physical hands playground scene when not using UI layer -- (Physical Hands) Disabled Ignore Physical hands components still affecting grab and collision at runtime -- (Physical Hands) Button Prefab uses mesh from example assets -- (Physical Hands) Button gets stuck down if disabled after pressing -- (UI Input Preview) Null UIInput events cause unnecessary error logs -- Memory increase when repeatedly opening scenes with LeapServiceProviders -- ThreadAbort when changing scenes in editor that use multidevice or display the tracking device gizmo - - +- Update target colour palette values for JointOcclusion to account for unexpected changes when rendered to the render texture ## [6.14.0] - 24/01/24 From edf59f68185f6521acd866f20ddaf0b9b2482611 Mon Sep 17 00:00:00 2001 From: Max Palmer <53301524+MaxPalmer-UH@users.noreply.github.com> Date: Tue, 5 Mar 2024 17:19:10 +0000 Subject: [PATCH 2/3] Update CHANGELOG.md Restored omitted comments --- Packages/Tracking/CHANGELOG.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/Packages/Tracking/CHANGELOG.md b/Packages/Tracking/CHANGELOG.md index 11c530b2fc..efd412c11a 100644 --- a/Packages/Tracking/CHANGELOG.md +++ b/Packages/Tracking/CHANGELOG.md @@ -14,6 +14,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Android v5.17.1 ### Added +- LeapServiceProvider accessor for world space psoition of the Tracking Camera +- LeapXRServiceProvider accessor for world space psoition of the Tracking Camera +- LeapServiceProvider accessor for world space position of the Tracking Camera +- LeapXRServiceProvider accessor for world space position of the Tracking Camera +- (Physical Hands) Ability to ignore collisions on single object or all children +- (Physical Hands) Added toggle to GrabHelper to allow kinematic object movement +- (Physical Hands) Ignore Physical hands component can now choose which hand(s) it should be applied to ### Changed - (Config) Additional uses of Config marked as Obsolete @@ -21,11 +28,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Example content to be split by XR, Tabletop and URP Examples - Combined example content to be part of the main Ultraleap Tracking .unitypackage when importing via .unitypackage - Added fog and gradient sky for all XR example scenes +- (Hand Rays) Exposed dot product used to test if the hand is facing camera +- (Hinting) Added support for startup setting of Hand Tracking Hints via the Ultraleap Settings window +- (Hinting) Added support for runtime changing of Hand Tracking Hints via static HandTrackingHintManager +- Access of Physical Hands extensions +- Added public accessors to various Physical Hands utilities ### Fixed - Errors in Editor when using pre-2023.3.18 LTS due to FindObjectByType issue - (Physical Hands) Objects are sticky when they ignore collision with hard contact hands -- Update target colour palette values for JointOcclusion to account for unexpected changes when rendered to the render texture +- (Physical Hands) Ability to toggle ignore Physical hands options from the inspector at runtime. +- (Locomotion) IsPinching wouldn't fire when between Activate and Deactivate values in LightweightPinchDetector +- (Physical Hands) Soft contact button difficult to press in physical hands playground scene when not using UI layer +- (Physical Hands) Disabled Ignore Physical hands components still affecting grab and collision at runtime +- (Physical Hands) Button Prefab uses mesh from example assets +- (Physical Hands) Button gets stuck down if disabled after pressing +- (UI Input Preview) Null UIInput events cause unnecessary error logs +- Memory increase when repeatedly opening scenes with LeapServiceProviders +- ThreadAbort when changing scenes in editor that use multidevice or display the tracking device gizmo ## [6.14.0] - 24/01/24 From 0247c7cffbbc0af7f784958619a7ace5f3cf4687 Mon Sep 17 00:00:00 2001 From: Max Palmer <53301524+MaxPalmer-UH@users.noreply.github.com> Date: Tue, 5 Mar 2024 17:20:31 +0000 Subject: [PATCH 3/3] Update CHANGELOG.md --- Packages/Tracking/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Packages/Tracking/CHANGELOG.md b/Packages/Tracking/CHANGELOG.md index efd412c11a..c483808cfd 100644 --- a/Packages/Tracking/CHANGELOG.md +++ b/Packages/Tracking/CHANGELOG.md @@ -46,6 +46,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - (UI Input Preview) Null UIInput events cause unnecessary error logs - Memory increase when repeatedly opening scenes with LeapServiceProviders - ThreadAbort when changing scenes in editor that use multidevice or display the tracking device gizmo +- Updated target colour palette values for the JointOcclusion confidence code to account for unexpected changes when rendered to the render texture ## [6.14.0] - 24/01/24