Matrix math for model-space handling

+ add option to disable character texture computation + add further ktisis credits (alloc class for transform matrices) + use model transform in debug view option selected is in model space
PassiveModding · Sep 28, 2024 · a82213c · a82213c
1 parent 751a948
commit a82213c
Show file tree

Hide file tree

Showing 8 changed files with 148 additions and 53 deletions.
diff --git a/Meddle/Meddle.Plugin/Models/Composer/CharacterComposer.cs b/Meddle/Meddle.Plugin/Models/Composer/CharacterComposer.cs
@@ -22,13 +22,15 @@ public class CharacterComposer
     private static readonly object StaticFileLock = new();
     private readonly SkeletonUtils.PoseMode poseMode;
     private readonly bool includePose;
+    private readonly bool computeCharacterTextures;
 
     public CharacterComposer(DataProvider dataProvider, Configuration config, Action<ProgressEvent>? progress = null) 
     {
         this.dataProvider = dataProvider;
         this.progress = progress;
         includePose = config.IncludePose;
         poseMode = config.PoseMode;
+        computeCharacterTextures = config.ComputeCharacterTextures;
 
         lock (StaticFileLock)
         {
@@ -74,6 +76,12 @@ private void HandleModel(GenderRace genderRace, CustomizeParameter customizePara
             try
             {
                 var materialInfo = modelInfo.Materials[i];
+                if (!computeCharacterTextures)
+                {
+                    materialBuilders[i] = new MaterialBuilder(materialInfo.Path.FullPath);
+                    continue;
+                }
+
                 progress?.Invoke(new ProgressEvent(modelInfo.GetHashCode(), $"{materialInfo.Path.GamePath}", i + 1, modelInfo.Materials.Length));
                 var mtrlData = dataProvider.LookupData(materialInfo.Path.FullPath);
                 if (mtrlData == null)

diff --git a/Meddle/Meddle.Plugin/Models/Skeletons/ParsedHkaPose.cs b/Meddle/Meddle.Plugin/Models/Skeletons/ParsedHkaPose.cs
@@ -1,6 +1,9 @@
-using System.Text.Json.Serialization;
+using System.Runtime.InteropServices;
+using System.Text.Json.Serialization;
+using FFXIVClientStructs.FFXIV.Common.Math;
 using FFXIVClientStructs.Havok.Animation.Rig;
 using FFXIVClientStructs.Interop;
+using Meddle.Plugin.Utils;
 
 namespace Meddle.Plugin.Models.Skeletons;
 
@@ -10,32 +13,43 @@ public unsafe ParsedHkaPose(Pointer<hkaPose> pose) : this(pose.Value) { }
 
     public unsafe ParsedHkaPose(hkaPose* pose)
     {
+        var localSpaceTransforms = new List<Transform>();
+        var hkLocalSpaceMatrices = new List<Matrix4x4>();
+        var hkModelSpaceMatrices = new List<Matrix4x4>();
+
         var boneCount = pose->Skeleton->Bones.Length;
 
-        var transforms = new List<Transform>();
-        var syncedLocalPose = pose->GetSyncedPoseLocalSpace()->Data;
         for (var i = 0; i < boneCount; ++i)
         {
-            var localSpace = syncedLocalPose[i];
-            transforms.Add(new Transform(localSpace));
+            var localSpace = pose->AccessBoneLocalSpace(i);
+            if (localSpace == null)
+            {
+                throw new ArgumentException($"Failed to access bone {i}");
+            }
+            hkLocalSpaceMatrices.Add(Alloc.GetMatrix(localSpace));
+            localSpaceTransforms.Add(new Transform(*localSpace));
         }
-
-        var modelTransforms = new List<Transform>();
-        var modelSpace = pose->GetSyncedPoseModelSpace()->Data;
         for (var i = 0; i < boneCount; ++i)
         {
-            var model = modelSpace[i];
-            modelTransforms.Add(new Transform(model));
+            var modelSpace = pose->AccessBoneModelSpace(i, hkaPose.PropagateOrNot.DontPropagate);
+            if (modelSpace == null)
+            {
+                throw new ArgumentException($"Failed to access model bone {i}");
+            }
+            hkModelSpaceMatrices.Add(Alloc.GetMatrix(modelSpace));
         }
-
 
-        Pose = transforms;
-        ModelPose = modelTransforms;
+        Pose = localSpaceTransforms;
+        HkLocalSpaceMatrices = hkLocalSpaceMatrices;
+        HkModelSpaceMatrices = hkModelSpaceMatrices;
     }
 
     [JsonIgnore]
     public IReadOnlyList<Transform> Pose { get; }
+
+    [JsonIgnore]
+    public IReadOnlyList<Matrix4x4> HkLocalSpaceMatrices { get; }
 
     [JsonIgnore]
-    public IReadOnlyList<Transform> ModelPose { get; }
+    public IReadOnlyList<Matrix4x4> HkModelSpaceMatrices { get; }
 }
diff --git a/Meddle/Meddle.Plugin/Plugin.cs b/Meddle/Meddle.Plugin/Plugin.cs
@@ -33,6 +33,7 @@ public Plugin(IDalamudPluginInterface pluginInterface)
         {
             var config = pluginInterface.GetPluginConfig() as Configuration ?? new Configuration();
             pluginInterface.Inject(config);
+            Alloc.Init();
 
             var host = Host.CreateDefaultBuilder();
             host.ConfigureLogging(logging =>
@@ -93,6 +94,7 @@ public void Dispose()
         app?.WaitForShutdown();
         app?.Dispose();
         log?.LogDebug("Plugin disposed");
+        Alloc.Dispose();
     }
 }
 
@@ -131,6 +133,8 @@ public class Configuration : IPluginConfiguration
     /// </summary>
     public SkeletonUtils.PoseMode PoseMode { get; set; } = DefaultPoseMode;
 
+    public bool ComputeCharacterTextures { get; set; } = true;
+
     /// <summary>
     /// GLTF = GLTF JSON
     /// GLB = GLTF Binary

diff --git a/Meddle/Meddle.Plugin/UI/DebugTab.cs b/Meddle/Meddle.Plugin/UI/DebugTab.cs
@@ -416,7 +416,16 @@ private unsafe void DrawBoneTransformsOnScreen(PartialSkeleton partialSkeleton,
                 continue;
             }
 
-            var modelTransform = new Transform(pose->ModelPose[i]);
+            var t = boneMode switch
+            {
+                BoneMode.Local => new Transform(pose->LocalPose[i]),
+                BoneMode.ModelPropagate => new Transform(*pose->AccessBoneModelSpace(i, hkaPose.PropagateOrNot.Propagate)),
+                BoneMode.ModelNoPropagate => new Transform(*pose->AccessBoneModelSpace(i, hkaPose.PropagateOrNot.DontPropagate)),
+                BoneMode.ModelRaw => new Transform(pose->ModelPose[i]),
+                _ => new Transform(pose->ModelPose[i])
+            };
+
+            var modelTransform = boneMode == BoneMode.Local ? new Transform(pose->ModelPose[i]) : t;
             var worldMatrix = modelTransform.AffineTransform.Matrix * rootTransform.AffineTransform.Matrix;
             ImGui.TableNextRow();
             ImGui.TableSetColumnIndex(0);
@@ -427,15 +436,6 @@ private unsafe void DrawBoneTransformsOnScreen(PartialSkeleton partialSkeleton,
                 dotColorRgb = new Vector4(1, 0, 0, 0.5f);
             }
 
-            var t = boneMode switch
-            {
-                BoneMode.Local => new Transform(pose->LocalPose[i]),
-                BoneMode.ModelPropagate => new Transform(*pose->AccessBoneModelSpace(i, hkaPose.PropagateOrNot.Propagate)),
-                BoneMode.ModelNoPropagate => new Transform(*pose->AccessBoneModelSpace(i, hkaPose.PropagateOrNot.DontPropagate)),
-                BoneMode.ModelRaw => new Transform(pose->ModelPose[i]),
-                _ => new Transform(pose->ModelPose[i])
-            };
-
             ImGui.TableSetColumnIndex(1);
             var parentIndex = pose->Skeleton->ParentIndices[i];
             ImGui.Text(parentIndex.ToString());

diff --git a/Meddle/Meddle.Plugin/UI/OptionsTab.cs b/Meddle/Meddle.Plugin/UI/OptionsTab.cs
@@ -77,6 +77,13 @@ public void Draw()
 
         DrawPoseMode();
 
+        var computeCharacterTextures = config.ComputeCharacterTextures;
+        if (ImGui.Checkbox("Compute Character Textures", ref computeCharacterTextures))
+        {
+            config.ComputeCharacterTextures = computeCharacterTextures;
+            config.Save();
+        }
+
         ImGui.Separator();
 
         var playerNameOverride = config.PlayerNameOverride;

diff --git a/Meddle/Meddle.Plugin/Utils/Alloc.cs b/Meddle/Meddle.Plugin/Utils/Alloc.cs
@@ -0,0 +1,36 @@
+using System.Runtime.InteropServices;
+using FFXIVClientStructs.FFXIV.Common.Math;
+using FFXIVClientStructs.Havok.Common.Base.Math.Matrix;
+using FFXIVClientStructs.Havok.Common.Base.Math.QsTransform;
+
+namespace Meddle.Plugin.Utils;
+
+// https://github.com/ktisis-tools/Ktisis/blob/88c1af74f748298d1b1d01135aa58ce0a9530419/Ktisis/Interop/Alloc.cs
+internal static class Alloc {
+    // Allocations
+    private static IntPtr MatrixAlloc;
+
+    // Access
+    internal static unsafe Matrix4x4* Matrix; // Align to 16-byte boundary
+    internal static unsafe Matrix4x4 GetMatrix(hkQsTransformf* transform) {
+        transform->get4x4ColumnMajor((float*)Matrix);
+        return *Matrix;
+    }
+
+    // internal static unsafe void SetMatrix(hkQsTransformf* transform, Matrix4x4 matrix) {
+    //     *Matrix = matrix;
+    //     transform->set((hkMatrix4f*)Matrix);
+    // }
+
+    // Init & dispose
+    public static unsafe void Init() {
+        // Allocate space for our matrix to be aligned on a 16-byte boundary.
+        // This is required due to ffxiv's use of the MOVAPS instruction.
+        // Thanks to Fayti1703 for helping with debugging and coming up with this fix.
+        MatrixAlloc = Marshal.AllocHGlobal(sizeof(float) * 16 + 16);
+        Matrix = (Matrix4x4*)(16 * ((long)(MatrixAlloc + 15) / 16));
+    }
+    public static void Dispose() {
+        Marshal.FreeHGlobal(MatrixAlloc);
+    }
+}
diff --git a/Meddle/Meddle.Plugin/Utils/SkeletonUtils.cs b/Meddle/Meddle.Plugin/Utils/SkeletonUtils.cs
@@ -1,7 +1,10 @@
 using System.Numerics;
+using System.Runtime.CompilerServices;
+using FFXIVClientStructs.Havok.Common.Base.Math.QsTransform;
 using Meddle.Plugin.Models;
 using Meddle.Plugin.Models.Skeletons;
 using Meddle.Utils;
+using Microsoft.Extensions.Logging;
 using SharpGLTF.Scenes;
 using SharpGLTF.Transforms;
 
@@ -15,7 +18,9 @@ public enum PoseMode
         Model
     }
 
-    public static (List<BoneNodeBuilder> List, BoneNodeBuilder Root)[] GetBoneMaps(IReadOnlyList<ParsedPartialSkeleton> partialSkeletons, PoseMode? poseMode)
+    public static (List<BoneNodeBuilder> List, BoneNodeBuilder Root)[] GetBoneMaps(
+        Transform rootTransform,
+        IReadOnlyList<ParsedPartialSkeleton> partialSkeletons, PoseMode? poseMode)
     {
         List<BoneNodeBuilder> boneMap = new();
         List<BoneNodeBuilder> rootList = new();
@@ -93,41 +98,71 @@ public static (List<BoneNodeBuilder> List, BoneNodeBuilder Root)[] GetBoneMaps(I
             {
                 foreach (var bone in map.List)
                 {
-                    ApplyPose(bone, partialSkeletons, poseMode.Value, 0);
+                    ApplyPose(rootTransform, bone, partialSkeletons, poseMode.Value, 0);
                 }
             }
         }
 
         return boneMaps;
     }
 
-    private static void ApplyPose(
+    private static void ApplyPose(Transform rootTransform,
         BoneNodeBuilder bone, IReadOnlyList<ParsedPartialSkeleton> partialSkeletons, PoseMode poseMode, float time)
     {
         var partial = partialSkeletons[bone.PartialSkeletonIndex];
-        if (partial.Poses.FirstOrDefault() is not { } pose)
+        if (partial.Poses.Count == 0)
+        {
+            Plugin.Logger?.LogWarning("No poses found for {BoneName}", bone.BoneName);
             return;
+        }
+        var pose = partial.Poses[0];
 
         switch (poseMode)
         {
             case PoseMode.Model when bone.Parent is BoneNodeBuilder parent:
             {
-                var boneTransform = pose.ModelPose[bone.BoneIndex].AffineTransform;
-                var parentPose = partialSkeletons[parent.PartialSkeletonIndex].Poses.FirstOrDefault();
-                if (parentPose == null)
+                var boneMatrix = pose.HkModelSpaceMatrices[bone.BoneIndex];
+                if (partialSkeletons[parent.PartialSkeletonIndex].Poses.Count == 0)
+                {
+                    Plugin.Logger?.LogWarning("Parent pose not found for {BoneName} parent {ParentName}", bone.BoneName, parent.BoneName);
+                    return;
+                }
+
+                var parentMatrix = partialSkeletons[parent.PartialSkeletonIndex].Poses[0].HkModelSpaceMatrices[parent.BoneIndex];
+                var boneAffine = new AffineTransform(boneMatrix);
+                var parentAffine = new AffineTransform(parentMatrix);
+                if (!AffineTransform.TryInvert(parentAffine, out var invParentAffine))
+                {
+                    Plugin.Logger?.LogWarning("Failed to invert parent affine for {BoneName} parent {ParentName}", bone.BoneName, parent.BoneName);
+                    return;
+                }
+
+                var affine = boneAffine * invParentAffine;
+                if (!affine.TryDecompose(out var scale, out var rotation, out var translation))
+                {
+                    Plugin.Logger?.LogWarning("Failed to decompose affine for {BoneName} parent {ParentName}", bone.BoneName, parent.BoneName);
                     return;
+                }
 
-                var parentMatrix = parentPose.ModelPose[parent.BoneIndex].AffineTransform.Matrix;
-                var matrix = boneTransform.Matrix * AffineInverse(parentMatrix);
-                var affine = new AffineTransform(matrix).GetDecomposed();
-                bone.UseScale().UseTrackBuilder("pose").WithPoint(time, affine.Scale);
-                bone.UseRotation().UseTrackBuilder("pose").WithPoint(time, affine.Rotation);
-                bone.UseTranslation().UseTrackBuilder("pose").WithPoint(time, affine.Translation);
+                bone.UseScale().UseTrackBuilder("pose").WithPoint(time, scale);
+                bone.UseRotation().UseTrackBuilder("pose").WithPoint(time, rotation);
+                bone.UseTranslation().UseTrackBuilder("pose").WithPoint(time, translation);
                 break;
             }
             case PoseMode.Model:
             {
-                var boneTransform = pose.ModelPose[bone.BoneIndex].AffineTransform;
+                var boneMatrix = pose.HkModelSpaceMatrices[bone.BoneIndex];
+                var boneAffine = new AffineTransform(boneMatrix).GetDecomposed();
+                var scale = boneAffine.Scale * rootTransform.Scale;
+
+                bone.UseScale().UseTrackBuilder("pose").WithPoint(time, scale);
+                bone.UseRotation().UseTrackBuilder("pose").WithPoint(time, boneAffine.Rotation);
+                bone.UseTranslation().UseTrackBuilder("pose").WithPoint(time, boneAffine.Translation);
+                break;
+            }
+            case PoseMode.Local when bone.Parent is BoneNodeBuilder:
+            {
+                var boneTransform = pose.Pose[bone.BoneIndex].AffineTransform;
                 bone.UseScale().UseTrackBuilder("pose").WithPoint(time, boneTransform.Scale);
                 bone.UseRotation().UseTrackBuilder("pose").WithPoint(time, boneTransform.Rotation);
                 bone.UseTranslation().UseTrackBuilder("pose").WithPoint(time, boneTransform.Translation);
@@ -136,7 +171,9 @@ private static void ApplyPose(
             case PoseMode.Local:
             {
                 var boneTransform = pose.Pose[bone.BoneIndex].AffineTransform;
-                bone.UseScale().UseTrackBuilder("pose").WithPoint(time, boneTransform.Scale);
+                var scale = boneTransform.Scale * rootTransform.Scale;
+
+                bone.UseScale().UseTrackBuilder("pose").WithPoint(time, scale);
                 bone.UseRotation().UseTrackBuilder("pose").WithPoint(time, boneTransform.Rotation);
                 bone.UseTranslation().UseTrackBuilder("pose").WithPoint(time, boneTransform.Translation);
                 break;
@@ -148,7 +185,7 @@ private static void ApplyPose(
 
     public static List<BoneNodeBuilder> GetBoneMap(ParsedSkeleton skeleton, PoseMode? poseMode, out BoneNodeBuilder? root)
     {
-        var maps = GetBoneMaps(skeleton.PartialSkeletons, poseMode);
+        var maps = GetBoneMaps(skeleton.Transform, skeleton.PartialSkeletons, poseMode);
         if (maps.Length == 0)
         {
             throw new InvalidOperationException("No roots were found");
@@ -224,7 +261,7 @@ public static Dictionary<string, AttachGrouping> GetAnimatedBoneMap((DateTime Ti
                         bone = attachBone;
                     }
 
-                    ApplyPose(bone, frame.Attach.Skeleton.PartialSkeletons, poseMode, frameTime);
+                    ApplyPose(frame.Attach.Skeleton.Transform, bone, frame.Attach.Skeleton.PartialSkeletons, poseMode, frameTime);
                 }
 
                 var firstTranslation = firstAttach.Transform.Translation;
@@ -238,18 +275,6 @@ public static Dictionary<string, AttachGrouping> GetAnimatedBoneMap((DateTime Ti
         return attachDict;
     }
 
-    public static Matrix4x4 AffineInverse(Matrix4x4 matrix)
-    {
-        if (!Matrix4x4.Invert(matrix, out var invMatrix))
-            throw new InvalidOperationException("Failed to invert matrix");
-
-        // ReSharper disable once CompareOfFloatsByEqualityOperator
-        if (matrix.M44 == 1.0)
-            invMatrix.M44 = 1f;
-
-        return invMatrix;
-    }
-
     public static float TotalSeconds(DateTime time, DateTime startTime)
     {
         var value = (float)(time - startTime).TotalSeconds;

diff --git a/README.md b/README.md
@@ -33,6 +33,7 @@ Much of this code is from or based on the following projects and wouldn't have b
   - reference for world overlay logic
 - [Ktisis](https://github.com/ktisis-tools/Ktisis) [[GNU GPL v3](https://github.com/ktisis-tools/Ktisis/blob/main/LICENSE)]
   - lighting structs
+  - hkQsTransformf matrix handling
 
 Important contributors:
 - [WorkingRobot](https://github.com/WorkingRobot)