Refactored socket vs merging

packages/3d-web-avatar/src/character/Character.ts → packages/3d-web-avatar/src/character/MMLCharacter.ts

@@ -1,11 +1,24 @@
-import { BufferAttribute, Group, Matrix4, Object3D, Skeleton, SkinnedMesh } from "three";
+import {
+  Bone,
+  BufferAttribute,
+  Euler,
+  Group,
+  MathUtils,
+  Matrix4,
+  Mesh,
+  Object3D,
+  Skeleton,
+  SkinnedMesh,
+  Vector3,
+} from "three";
 import { GLTF } from "three/examples/jsm/loaders/GLTFLoader.js";
 
+import { MMLCharacterDescriptionPart } from "../helpers/parseMMLDescription";
 import { SkeletonHelpers } from "../helpers/SkeletonHelpers";
 
 import { ModelLoader } from "./ModelLoader";
 
-export class Character {
+export class MMLCharacter {
   private skeletonHelpers: SkeletonHelpers = new SkeletonHelpers();
   private skinnedMeshesParent: Group | null = null;
   private sharedSkeleton: Skeleton | null = null;
@@ -53,26 +66,34 @@ export class Character {
     );
   }
 
-  public async mergeBodyParts(fullBodyURL: string, bodyParts: Array<string>): Promise<Object3D> {
+  public async mergeBodyParts(
+    fullBodyURL: string,
+    bodyParts: Array<MMLCharacterDescriptionPart>,
+  ): Promise<Object3D> {
     const fullBodyAsset = await this.modelLoader.load(fullBodyURL);
     const fullBodyGLTF = this.skeletonHelpers.cloneGLTF(fullBodyAsset as GLTF, "fullBody");
 
-    const assetPromises: Array<Promise<{ asset: GLTF; partUrl: string }>> = bodyParts.map(
-      (partUrl) => {
+    const assetPromises: Array<Promise<{ asset: GLTF; part: MMLCharacterDescriptionPart }>> =
+      bodyParts.map((part) => {
         return new Promise((resolve) => {
-          this.modelLoader.load(partUrl).then((asset) => {
-            resolve({ asset: asset!, partUrl });
+          this.modelLoader.load(part.url).then((asset) => {
+            resolve({ asset: asset!, part });
           });
         });
-      },
-    );
+      });
     const assets = await Promise.all(assetPromises);
 
     const fullBodyModelGroup = fullBodyGLTF.gltf.scene;
 
     this.skinnedMeshesParent = null;
 
+    const availableBones = new Map<string, Bone>();
     fullBodyModelGroup.traverse((child) => {
+      const asBone = child as Bone;
+      if (asBone.isBone) {
+        availableBones.set(child.name, asBone);
+      }
+
       const asSkinnedMesh = child as SkinnedMesh;
       if (asSkinnedMesh.isSkinnedMesh) {
         asSkinnedMesh.castShadow = true;
@@ -86,19 +107,57 @@ export class Character {
     this.sharedMatrixWorld = fullBodyGLTF.matrixWorld;
 
     for (const loadingAsset of assets) {
-      const gltf = this.skeletonHelpers.cloneGLTF(loadingAsset.asset, loadingAsset.partUrl);
+      const part = loadingAsset.part;
+      const gltf = this.skeletonHelpers.cloneGLTF(loadingAsset.asset, part.url);
       const modelGroup = gltf.gltf.scene;
-      modelGroup.traverse((child) => {
-        const asSkinnedMesh = child as SkinnedMesh;
-        if (asSkinnedMesh.type === "SkinnedMesh") {
-          const skinnedMeshClone = child.clone(true) as SkinnedMesh;
-          this.remapBoneIndices(skinnedMeshClone);
-          skinnedMeshClone.castShadow = true;
-          skinnedMeshClone.receiveShadow = true;
-          skinnedMeshClone.bind(this.sharedSkeleton!, this.sharedMatrixWorld!);
-          this.skinnedMeshesParent?.add(skinnedMeshClone);
+      if (part.socket) {
+        const socketName = part.socket.socket;
+        let bone = availableBones.get("root");
+        if (availableBones.has(socketName)) {
+          bone = availableBones.get(socketName);
+        } else {
+          console.warn(
+            `WARNING: no bone found for [${socketName}] socket. Attatching to Root bone`,
+          );
         }
-      });
+        if (bone) {
+          modelGroup.position.set(0, 0, 0);
+          modelGroup.rotation.set(0, 0, 0);
+          modelGroup.scale.set(1, 1, 1);
+
+          bone.add(modelGroup);
+
+          modelGroup.rotateZ(-Math.PI / 2);
+
+          const offsetPosition = new Vector3(
+            part.socket.position.x,
+            part.socket.position.y,
+            part.socket.position.z,
+          );
+          modelGroup.position.copy(offsetPosition);
+
+          const offsetRotation = new Euler(
+            MathUtils.degToRad(part.socket.rotation.x),
+            MathUtils.degToRad(part.socket.rotation.y),
+            MathUtils.degToRad(part.socket.rotation.z),
+          );
+          modelGroup.setRotationFromEuler(offsetRotation);
+
+          modelGroup.scale.set(part.socket.scale.x, part.socket.scale.y, part.socket.scale.z);
+        }
+      } else {
+        modelGroup.traverse((child) => {
+          const asSkinnedMesh = child as SkinnedMesh;
+          if (asSkinnedMesh.isSkinnedMesh) {
+            const skinnedMeshClone = child.clone(true) as SkinnedMesh;
+            this.remapBoneIndices(skinnedMeshClone);
+            skinnedMeshClone.castShadow = true;
+            skinnedMeshClone.receiveShadow = true;
+            skinnedMeshClone.bind(this.sharedSkeleton!, this.sharedMatrixWorld!);
+            this.skinnedMeshesParent?.add(skinnedMeshClone);
+          }
+        });
+      }
     }
     return fullBodyGLTF!.gltf.scene as Object3D;
   }

packages/3d-web-avatar/src/index.ts

@@ -1,4 +1,4 @@
-export { Character } from "./character/Character";
+export { MMLCharacter } from "./character/MMLCharacter";
 export { cloneModel } from "./helpers/SkeletonHelpers";
 export { TimeManagerInterface } from "./character/types";
 export { ModelLoader } from "./character/ModelLoader";

packages/3d-web-client-core/src/character/Character.ts

@@ -5,7 +5,6 @@ import { Composer } from "../rendering/composer";
 
 import { CharacterModel } from "./CharacterModel";
 import { CharacterModelLoader } from "./CharacterModelLoader";
-import { CharacterSockets } from "./CharacterSockets";
 import { CharacterSpeakingIndicator } from "./CharacterSpeakingIndicator";
 import { AnimationState } from "./CharacterState";
 import { CharacterTooltip } from "./CharacterTooltip";
@@ -28,7 +27,6 @@ export class Character extends Group {
   public color: Color = new Color();
   public tooltip: CharacterTooltip | null = null;
   public speakingIndicator: CharacterSpeakingIndicator | null = null;
-  public sockets: CharacterSockets | null = null;
 
   constructor(
     private readonly characterDescription: CharacterDescription,
@@ -57,9 +55,6 @@ export class Character extends Group {
       this.speakingIndicator = new CharacterSpeakingIndicator(this.composer.postPostScene);
     }
     this.color = this.model.material.colorsCube216[this.characterId];
-    if (this.sockets === null) {
-      this.sockets = new CharacterSockets(this.model.mesh!, this.model.mmlCharacterDescription);
-    }
     this.modelLoadedCallback();
   }
 

packages/3d-web-client-core/src/character/CharacterModel.ts

@@ -1,5 +1,5 @@
 import {
-  Character as MMLCharacter,
+  MMLCharacter,
   type MMLCharacterDescription,
   ModelLoader,
   parseMMLDescription,
@@ -100,16 +100,13 @@ export class CharacterModel {
     let mergedCharacter: Object3D | null = null;
     if (mmlCharacterDescription) {
       const characterBase = mmlCharacterDescription.base?.url || null;
-      const parts: string[] = [];
-
-      mmlCharacterDescription.parts?.forEach((part) => {
-        if (part.url) parts.push(part.url);
-      });
-
       if (characterBase) {
         this.mmlCharacterDescription = mmlCharacterDescription;
         const mmlCharacter = new MMLCharacter(new ModelLoader());
-        mergedCharacter = await mmlCharacter.mergeBodyParts(characterBase, parts);
+        mergedCharacter = await mmlCharacter.mergeBodyParts(
+          characterBase,
+          mmlCharacterDescription.parts,
+        );
         if (mergedCharacter) {
           return mergedCharacter.children[0].children[0];
         }