Merge pull request #173 from Benjythebee/add/blendshapes-as-sub-traits

Looks good to me! :)

package.json

@@ -15,7 +15,7 @@
   },
   "dependencies": {
     "@ethersproject/providers": "^5.5.1",
-    "@pixiv/three-vrm": "^1.0.9",
+    "@pixiv/three-vrm": "^3.1.1",
     "@react-spring/three": "^9.2.4",
     "@use-gesture/react": "^10.0.0-beta.22",
     "@vitejs/plugin-basic-ssl": "^1.0.1",

src/library/CharacterManifestData.js

@@ -173,6 +173,9 @@ export class CharacterManifestData{
     getAllTraits(){
       return this.getRandomTraits(this.allTraits);
     }
+    getAllBlendShapeTraits(){
+      return this.modelTraits.map(traitGroup => traitGroup.getCollection()).flat().map((c)=>c.blendshapeTraits).flat().map((c)=>c?.collection).flat().filter((c)=>!!c);
+    }
     isColliderRequired(groupTraitID){
       if (this.colliderTraits.indexOf(groupTraitID) != -1)
         return true;
@@ -626,6 +629,8 @@ class TraitColorsGroup{
   }
 }
 class ModelTrait{
+  blendshapeTraits = []; 
+  blendshapeTraitsMap = new Map();
   constructor(traitGroup, options){
       const {
           id,
@@ -636,6 +641,7 @@ class ModelTrait{
           cullingLayer,
           type = [],
           textureCollection,
+          blendshapeTraits,
           colorCollection,
           fullDirectory,
           fullThumbnail,
@@ -678,8 +684,14 @@ class ModelTrait{
       this.targetTextureCollection = textureCollection ? traitGroup.manifestData.getTextureGroup(textureCollection) : null;
       this.targetColorCollection = colorCollection ? traitGroup.manifestData.getColorGroup(colorCollection) : null;
 
-      if (this.targetTextureCollection)
-        console.log(this.targetTextureCollection);
+      if(blendshapeTraits && Array.isArray(blendshapeTraits)){
+
+        this.blendshapeTraits = blendshapeTraits.map((blendshapeGroup) => {
+          return new BlendShapeGroup(this, blendshapeGroup)
+        })
+
+        this.blendshapeTraitsMap = new Map(this.blendshapeTraits.map(item => [item.trait, item]));
+      }
   }
   isRestricted(targetModelTrait){
     if (targetModelTrait == null)
@@ -691,7 +703,7 @@ class ModelTrait{
 
     if (this.type.length > 0 && this.manifestData.restrictedTypes > 0){
 
-      haveCommonValue = (arr1, arr2) => {
+      const haveCommonValue = (arr1, arr2) => {
         if (arr1 == null || arr2 == null)
           return false;
         for (let i = 0; i < arr1.length; i++) {
@@ -703,15 +715,128 @@ class ModelTrait{
       }
 
       const restrictedTypes = this.manifestData.restrictedTypes;
-      traitTypes = getAsArray(this.type);
+      const traitTypes = getAsArray(this.type);
       traitTypes.forEach(type => {
         return haveCommonValue(restrictedTypes[type], traitTypes)
       });
     }
     return false;
   }
+  getGroupBlendShapeTraits(){
+    return this.blendshapeTraits;
+  }
+
+  /**
+   * 
+   * @param {string} traitGroupID 
+   * @returns {BlendShapeTrait[]}
+   */
+  getBlendShapes(traitGroupID){
+    return this.blendshapeTraitsMap?.get(traitGroupID)?.collection
+  }
 
+  /**
+   * 
+   * @param {string} traitGroupID 
+   * @param {string} traitID 
+   * @returns {BlendShapeTrait | undefined}
+   */
+  getBlendShape(traitGroupID,traitID){
+    return this.blendshapeTraitsMap?.get(traitGroupID)?.getTrait(traitID);
+  }
 }
+
+
+export class BlendShapeGroup {
+  trait
+  name
+  isBlendShapeGroup = true
+  collection=[]
+  cameraTarget=null
+  collectionMap= null
+  /**
+   * @param {ModelTrait} modelTrait 
+   * @param {BlendShapeGroupModelTraitData} options 
+   */
+  constructor( modelTrait, options){
+    const {
+        trait,
+        name,
+        collection,
+        cameraTarget = modelTrait.traitGroup.cameraTarget || { distance:3 , height:1 }
+    }= options;
+    this.modelTrait = modelTrait;
+    this.trait = trait;
+    this.name = name;
+
+    this.cameraTarget = cameraTarget;
+    this.createCollection(collection);
+  }
+  /**
+   * @param {BlendShapeTraitData[]} itemCollection 
+   * @param {boolean} [replaceExisting] (default false)
+   */
+  createCollection(itemCollection, replaceExisting = false){
+    if (replaceExisting) this.collection = [];
+
+    getAsArray(itemCollection).forEach(item => {
+      this.collection.push(new BlendShapeTrait(this, item))
+    });
+    this.collectionMap = new Map(this.collection.map(item => [item.id, item]));
+  }
+
+  getTrait(traitID){
+    return this.collectionMap.get(traitID);
+  }
+
+  /**
+   * @param {number} index 
+   */
+  getTraitByIndex(index){
+    return this.collection[index];
+  }
+
+  getRandomTrait(){
+    return this.collection.length > 0 ? 
+      this.collection[Math.floor(Math.random() * this.collection.length)] : 
+      null;
+  }
+}
+
+export class BlendShapeTrait{
+  id
+  name
+  fullThumbnail=undefined
+  isBlendShape = true
+  /**
+   * @param {BlendShapeGroup} parentGroup 
+   * @param {BlendShapeTraitData} options 
+   */
+  constructor(parentGroup,options){
+      const {
+          id,
+          name,
+          fullThumbnail
+      }= options;
+
+      if(!id){
+        console.warn("BlendShapeTrait is missing id, parent trait: "+ parentGroup.trait)
+      }
+      if(!name){
+        console.warn("BlendShapeTrait is missing name, parent trait: "+ parentGroup.trait)
+      }
+
+      this.parentGroup = parentGroup;
+      this.id = id;
+      this.fullThumbnail = fullThumbnail;
+      this.name = name;
+  }
+
+  getGroupId(){
+    return this.parentGroup.trait;
+  }
+}
+
 class TextureTrait{
   constructor(traitGroup, options){
       const {

src/library/characterManager.js

@@ -298,6 +298,19 @@ export class CharacterManager {
       if (this.manifestData){
         return this.manifestData.getGroupModelTraits();
       }
+    }
+      /**
+     * Same as getGroupTraits() but for Blendshapes
+     * @param {string} traitGroupId - The ID of the trait group.
+     * @param {string} traitId - The ID of the trait.
+     * @returns {Array} Array of blendshape traits
+     */
+    getBlendShapeGroupTraits(traitGroupId, traitId){
+      if (this.manifestData){
+        return this.manifestData.getModelTrait(traitGroupId, traitId)?.getGroupBlendShapeTraits()
+      }else {
+        return []
+      }
     }
     getCurrentCharacterModel(){
       return this.characterModel;
@@ -337,6 +350,13 @@ export class CharacterManager {
     getCurrentTraitData(groupTraitID){
       return this.avatar[groupTraitID]?.traitInfo;
     }
+    /**
+     * @param {string} groupTraitID 
+     * @returns {Object} Returns the current blendshape trait info for the specified group trait ID.
+     */
+    getCurrentBlendShapeTraitData(groupTraitID){
+      return this.avatar[groupTraitID]?.blendShapeTraitsInfo||{};
+    }
     getCurrentTraitVRM(groupTraitID){
       return this.avatar[groupTraitID]?.vrm;
     }
@@ -510,6 +530,30 @@ export class CharacterManager {
         }
       });
     }
+    /**
+     * Load and activate blendshape trait
+     * @param {string} traitGroupID 
+     * @param {string} blendshapeGroupId 
+     * @param {string|null} blendshapeTraitId 
+     * @returns 
+     */
+    loadBlendShapeTrait(traitGroupID, blendshapeGroupId,blendshapeTraitId){
+      const currentTrait = this.avatar[traitGroupID];
+      if(!currentTrait){
+        console.warn(`Trait with name: ${traitGroupID} was not found or not selected.`)
+        return;
+      }
+      if(!this.manifestData){
+        console.warn("No manifest data was found.")
+        return;
+      }
+
+      try{
+        this._loadBlendShapeTrait(traitGroupID, blendshapeGroupId, blendshapeTraitId);
+      }catch{ 
+        console.error("Error loading blendshape trait "+traitGroupID, blendshapeGroupId, blendshapeTraitId);
+      }
+    }
 
     /**
      * Loads a specific trait based on group and trait IDs.
@@ -879,6 +923,65 @@ export class CharacterManager {
         cullHiddenMeshes(this.avatar);
       })
     }
+
+    /**
+     * 
+     * @param {string} traitGroupID 
+     * @param {string} blendshapeGroupId 
+     * @param {string|null} blendshapeTraitId 
+     * @returns 
+     */
+    async _loadBlendShapeTrait(traitGroupID, blendshapeGroupId,blendshapeTraitId){
+      const currentTrait = this.avatar[traitGroupID];
+      if(!currentTrait){
+        console.warn(`Trait with name: ${traitGroupID} was not found or not selected.`)
+        return;
+      }
+      if(!currentTrait.blendShapeTraitsInfo){
+        currentTrait.blendShapeTraitsInfo = {};
+      }
+      if(currentTrait.blendShapeTraitsInfo[blendshapeGroupId]){
+        // Deactivate the current blendshape trait
+        this.toggleBinaryBlendShape(currentTrait.model, currentTrait.blendShapeTraitsInfo[blendshapeGroupId], false);
+      }
+      if(blendshapeTraitId == null){
+        // Deactivated the blendshape trait; dont do anything else
+        delete this.avatar[traitGroupID].blendShapeTraitsInfo[blendshapeGroupId]
+        return
+      }
+
+      const blendShape = currentTrait.traitInfo.getBlendShape(blendshapeGroupId, blendshapeTraitId);
+      if(!blendShape){
+        console.warn(`Blendshape with name: ${blendshapeTraitId} was not found.`)
+        return;
+      }
+
+      // Apply blendshape to the model
+      this.toggleBinaryBlendShape(currentTrait.model, blendShape, true);
+
+      this.avatar[traitGroupID].blendShapeTraitsInfo[blendShape.getGroupId()] = blendShape;
+
+    }
+    /**
+     * 
+     * @param {THREE.Object3D} model 
+     * @param {BlendShapeTrait} blendshape 
+     * @param {boolean} enable 
+     */
+    toggleBinaryBlendShape = (model,blendshape,enable)=>{
+      model.traverse((child)=>{
+        if(child.isMesh || child.isSkinnedMesh){
+          const mesh = child;
+          if(!mesh.morphTargetDictionary || !mesh.morphTargetInfluences) return
+          const blendShapeIndex = mesh.morphTargetDictionary[blendshape.id];
+          if (blendShapeIndex != undefined){
+            mesh.morphTargetInfluences[blendShapeIndex] = enable?1:0;
+          }
+        }
+      })
+
+    }
+
     async _animationManagerSetup(paths, baseLocation, scale){
       const animationPaths = getAsArray(paths);
       if (this.animationManager){
@@ -1232,6 +1335,7 @@ export class CharacterManager {
       // to do, we are now able to load multiple vrm models per options, set the options to include vrm arrays
       this.avatar[traitGroupID] = {
         traitInfo: traitModel,
+        blendShapeTraitsInfo:{},
         textureInfo: textureTrait,
         colorInfo: colorTrait,
         name: traitModel.name,