fix(Runtime): Optimize performance of retargeting

Andrew Kim · facebook-github-bot · commit 7beeb4121041 · 2024-04-15T13:27:16.000-07:00
Summary: Optimize the performance of retargeting by reducing the usage of dictionaries, caching data, removing processors, and skipping the fixed update on retargeting layer.

Reviewed By: sohailshafiiWk

Differential Revision: D56087086

fbshipit-source-id: 113d361c826480c3df6130556b5a3afdcad6ffbc
diff --git a/Runtime/Scripts/AnimationRigging/BoneMappingsExtension.cs b/Runtime/Scripts/AnimationRigging/BoneMappingsExtension.cs
@@ -1,7 +1,7 @@
 // Copyright (c) Meta Platforms, Inc. and affiliates.
-
-using System.Collections.Generic;
-using UnityEngine;
+
+using System.Collections.Generic;
+using UnityEngine;
 
 namespace Oculus.Movement.AnimationRigging
 {
@@ -84,6 +84,68 @@ public static readonly Dictionary<HumanBodyBones, AvatarMaskBodyPart>
                 { HumanBodyBones.RightLittleDistal, AvatarMaskBodyPart.RightArm }
             };
 
+        /// <summary>
+        /// Maps HumanBodyBones to avatar mask body part. Includes legs as well.
+        /// </summary>
+        public static readonly AvatarMaskBodyPart[] HumanBoneToAvatarBodyPartArray = new AvatarMaskBodyPart[]
+            {
+                AvatarMaskBodyPart.Body, // Hips
+                AvatarMaskBodyPart.LeftLeg, // LeftUpperLeg
+                AvatarMaskBodyPart.RightLeg, // RightUpperLeg
+                AvatarMaskBodyPart.LeftLeg, // LeftLowerLeg
+                AvatarMaskBodyPart.RightLeg, // RightLowerLeg
+                AvatarMaskBodyPart.LeftLeg, // LeftFoot
+                AvatarMaskBodyPart.RightLeg, // RightFoot
+                AvatarMaskBodyPart.Body, // Spine
+                AvatarMaskBodyPart.Body, // Chest
+                AvatarMaskBodyPart.Head, // Neck
+                AvatarMaskBodyPart.Head, // Head
+                AvatarMaskBodyPart.LeftArm, // LeftShoulder
+                AvatarMaskBodyPart.RightArm, // RightShoulder
+                AvatarMaskBodyPart.LeftArm, // LeftUpperArm
+                AvatarMaskBodyPart.RightArm, // RightUpperArm
+                AvatarMaskBodyPart.LeftArm, // LeftLowerArm
+                AvatarMaskBodyPart.RightArm, // RightLowerArm
+                AvatarMaskBodyPart.LeftArm, // LeftHand
+                AvatarMaskBodyPart.RightArm, // RightHand
+                AvatarMaskBodyPart.LeftLeg, // LeftToes
+                AvatarMaskBodyPart.RightLeg, // RightToes
+                AvatarMaskBodyPart.Head, // LeftEye
+                AvatarMaskBodyPart.Head, // RightEye
+                AvatarMaskBodyPart.Head, // Jaw
+                AvatarMaskBodyPart.LeftArm, // Left Hand Fingers
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.LeftArm,
+                AvatarMaskBodyPart.RightArm, // Right Hand Fingers
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.RightArm,
+                AvatarMaskBodyPart.Body, // UpperChest
+            };
+
         /// <summary>
         /// Maps OVRSkeletonBoneId to avatar mask body part.
         /// </summary>
@@ -265,4 +327,4 @@ public static readonly Dictionary<HumanBodyBones, AvatarMaskBodyPart>
                 { OVRSkeleton.BoneId.FullBody_RightFootBall, AvatarMaskBodyPart.RightLeg },
             };
     }
-}
+}
diff --git a/Runtime/Scripts/AnimationRigging/ExternalBoneTargets.cs b/Runtime/Scripts/AnimationRigging/ExternalBoneTargets.cs
@@ -97,6 +97,9 @@ private void Initialize()
             {
                 retargetedBoneTarget.BoneId = humanBodyBoneToOVRBoneId[retargetedBoneTarget.HumanBodyBone];
             }
+
+            // Sort by bone id.
+            _boneTargets = _boneTargets.OrderBy(x => x.BoneId).ToArray();
             _initialized = true;
         }
 
@@ -110,22 +113,23 @@ public void ProcessSkeleton(OVRSkeleton skeleton)
             {
                 return;
             }
+
             Initialize();
-            IList<OVRBone> bones = skeleton.Bones;
+            var retargetedBoneTargetIndex = 0;
+            var bones = skeleton.Bones;
             for (var i = 0; i < bones.Count; i++)
             {
-                foreach (var retargetedBoneTarget in _boneTargets)
+                var retargetedBoneTarget = _boneTargets[retargetedBoneTargetIndex];
+                if (bones[i].Id == retargetedBoneTarget.BoneId)
                 {
-                    if (bones[i].Id == retargetedBoneTarget.BoneId)
+                    retargetedBoneTargetIndex++;
+                    var targetBone = bones[i].Transform;
+                    if (targetBone == null)
                     {
-                        var targetBone = bones[i].Transform;
-                        if (targetBone == null)
-                        {
-                            continue;
-                        }
-                        retargetedBoneTarget.Target.SetPositionAndRotation(
-                            targetBone.position, targetBone.rotation);
+                        continue;
                     }
+                    retargetedBoneTarget.Target.SetPositionAndRotation(
+                        targetBone.position, targetBone.rotation);
                 }
             }
         }
diff --git a/Runtime/Scripts/AnimationRigging/PlaybackAnimationJob.cs b/Runtime/Scripts/AnimationRigging/PlaybackAnimationJob.cs
@@ -176,7 +176,7 @@ public override void Update(PlaybackAnimationJob job, ref T data)
 
                 if (playbackType == PlaybackAnimationData.AnimationPlaybackType.None ||
                     (data.AnimationMask != null &&
-                     !data.AnimationMask.GetHumanoidBodyPartActive(BoneMappingsExtension.HumanBoneToAvatarBodyPart[i])))
+                     !data.AnimationMask.GetHumanoidBodyPartActive(BoneMappingsExtension.HumanBoneToAvatarBodyPartArray[(int)i])))
                 {
                     job.BonePositionDeltas[index] = Vector3.zero;
                     job.BoneRotationDeltas[index] = Quaternion.identity;
diff --git a/Runtime/Scripts/AnimationRigging/ProxyTransformLogic.cs b/Runtime/Scripts/AnimationRigging/ProxyTransformLogic.cs
@@ -171,7 +171,7 @@ private void CleanUpOldProxyTransforms()
             GameObject.Destroy(proxyParent.gameObject);
         }
 
-        // <summary>
+        /// <summary>
         /// Sometimes source transforms change even though the IDs are the same,
         /// and proxies are only recreated if the bone IDs of the sources change.
         /// Make sure they are set properly.
diff --git a/Runtime/Scripts/AnimationRigging/RetargetingLayer.cs b/Runtime/Scripts/AnimationRigging/RetargetingLayer.cs
@@ -53,12 +53,6 @@ public class JointPositionAdjustment
             public Vector3 GetPositionOffset()
             {
                 var targetPositionOffset = FinalPosition - OriginalPosition;
-                // The recorded positions will not be finite when we regenerate data for the rig.
-                if (!RiggingUtilities.IsFiniteVector3(FinalPosition) ||
-                    !RiggingUtilities.IsFiniteVector3(OriginalPosition))
-                {
-                    return Vector3.zero;
-                }
                 return targetPositionOffset;
             }
         }
@@ -214,6 +208,11 @@ public RetargetedBoneMappings RetargetedBoneMappingsInst
         private ProxyTransformLogic _proxyTransformLogic = new ProxyTransformLogic();
         private bool _isFocusedWhileInBuild = true;
 
+        // Cached array versions of dictionaries used in OVRUnityHumanoidSkeletonRetargeter
+        private HumanBodyBones[] _customBoneIdToHumanBodyBoneArray;
+        private OVRSkeletonMetadata.BoneData[] _targetSkeletonDataBodyToBoneDataArray;
+        private OVRHumanBodyBonesMappings.BodySection[] _boneToBodySectionArray;
+
         /// <summary>
         /// Check for required components.
         /// </summary>
@@ -263,6 +262,7 @@ protected override void Start()
                 ConstructDefaultPoseInformation();
                 ConstructBoneAdjustmentInformation();
                 CacheJointConstraints();
+                CacheBoneMapping();
                 ValidateHumanoid();
 
                 _retargetingAnimationRig.ValidateRig(this);
@@ -325,6 +325,26 @@ private void CacheJointConstraints()
             _jointConstraints = jointConstraints.ToArray();
         }
 
+        private void CacheBoneMapping()
+        {
+            _targetSkeletonDataBodyToBoneDataArray = new OVRSkeletonMetadata.BoneData[(int)HumanBodyBones.LastBone];
+            _customBoneIdToHumanBodyBoneArray = new HumanBodyBones[(int)BoneId.Max];
+            for (int i = 0; i < _customBoneIdToHumanBodyBoneArray.Length; i++)
+            {
+                _customBoneIdToHumanBodyBoneArray[i] = BodyBoneMappingsInterface.GetFullBodyBoneIdToHumanBodyBone.
+                    GetValueOrDefault((BoneId)i, HumanBodyBones.LastBone);
+            }
+
+            _boneToBodySectionArray =
+                new OVRHumanBodyBonesMappings.BodySection[OVRHumanBodyBonesMappings.BoneToBodySection.Count];
+            for (int i = 0; i < _boneToBodySectionArray.Length; i++)
+            {
+                _boneToBodySectionArray[i] =
+                    OVRHumanBodyBonesMappings.BoneToBodySection.GetValueOrDefault((HumanBodyBones)i,
+                        OVRHumanBodyBonesMappings.BodySection.Head);
+            }
+        }
+
         private void ValidateHumanoid()
         {
             bool validHumanoid = true;
@@ -386,6 +406,13 @@ protected override void Update()
             _retargetingAnimationRig.UpdateRig(this);
         }
 
+        /// <summary>
+        /// Empty fixed update to avoid updating OVRSkeleton during fixed update.
+        /// </summary>
+        private void FixedUpdate()
+        {
+        }
+
         /// <summary>
         /// Allows fixing joints via the retargeting processors. The avatar does not allow
         /// precise finger positions even with translate DoF checked.
@@ -397,6 +424,7 @@ protected virtual void LateUpdate()
                 return;
             }
 
+            UpdateBoneDataToArray();
             foreach (var retargetingProcessor in _retargetingProcessors)
             {
                 retargetingProcessor.PrepareRetargetingProcessor(this, Bones);
@@ -419,10 +447,18 @@ protected virtual void OnDrawGizmos()
             }
         }
 
+        protected virtual void UpdateBoneDataToArray()
+        {
+            for (int i = 0; i < TargetSkeletonData.BodyToBoneData.Count; i++)
+            {
+                _targetSkeletonDataBodyToBoneDataArray[i] =
+                    TargetSkeletonData.BodyToBoneData.GetValueOrDefault((HumanBodyBones)i, null);
+            }
+        }
+
         protected virtual bool ShouldUpdatePositionOfBone(HumanBodyBones humanBodyBone)
         {
-            var bodySectionOfJoint =
-                OVRHumanBodyBonesMappings.BoneToBodySection[humanBodyBone];
+            var bodySectionOfJoint =_boneToBodySectionArray[(int)humanBodyBone];
             return IsBodySectionInArray(bodySectionOfJoint,
                 _skeletonType == SkeletonType.FullBody ? FullBodySectionToPosition : BodySectionToPosition);
         }
@@ -547,7 +583,7 @@ public void UpdateAdjustments(Quaternion[] rotationOffsets,
                 if (avatarMask != null)
                 {
                     jointFailsMask = !avatarMask.GetHumanoidBodyPartActive(
-                        BoneMappingsExtension.HumanBoneToAvatarBodyPart[targetHumanBodyBone]);
+                        BoneMappingsExtension.HumanBoneToAvatarBodyPartArray[(int)targetHumanBodyBone]);
                 }
 
                 if (adjustment == null)
@@ -672,25 +708,33 @@ public int GetNumberOfTransformsRetargeted()
         /// <returns>The human body bone for a custom bone id. Returns null if it doesn't exist.</returns>
         public HumanBodyBones? GetCustomBoneIdToHumanBodyBone(BoneId boneId)
         {
-            if (CustomBoneIdToHumanBodyBone == null)
-            {
-                return null;
-            }
-            if (!CustomBoneIdToHumanBodyBone.TryGetValue(boneId, out var humanBodyBone))
+            var humanBodyBone = _customBoneIdToHumanBodyBoneArray[(int)boneId];
+            if (humanBodyBone == HumanBodyBones.LastBone)
             {
                 return null;
             }
             return humanBodyBone;
         }
 
+        /// <summary>
+        /// Returns the human body bone to body section pairing.
+        /// </summary>
+        /// <param name="humanBodyBone">The human body bone to check for.</param>
+        /// <returns>The body section for a human body bone.</returns>
+        public OVRHumanBodyBonesMappings.BodySection GetHumanBodyBoneToBodySection(HumanBodyBones humanBodyBone)
+        {
+            return _boneToBodySectionArray[(int)humanBodyBone];
+        }
+
         /// <summary>
         /// Returns the correction quaternion for a human body bone, if it exists.
         /// </summary>
         /// <param name="humanBodyBone">The human body bone to check for.</param>
         /// <returns>The correction quaternion for a human body bone. Returns null if it doesn't exist.</returns>
         public Quaternion? GetCorrectionQuaternion(HumanBodyBones humanBodyBone)
         {
-            if (!TargetSkeletonData.BodyToBoneData.TryGetValue(humanBodyBone, out var targetData))
+            var targetData = _targetSkeletonDataBodyToBoneDataArray[(int)humanBodyBone];
+            if (targetData == null)
             {
                 return null;
             }
@@ -708,7 +752,8 @@ public int GetNumberOfTransformsRetargeted()
         /// <returns>The original joint for a human body bone.</returns>
         public Transform GetOriginalJoint(HumanBodyBones humanBodyBone)
         {
-            if (!TargetSkeletonData.BodyToBoneData.TryGetValue(humanBodyBone, out var targetData))
+            var targetData = _targetSkeletonDataBodyToBoneDataArray[(int)humanBodyBone];
+            if (targetData == null)
             {
                 return null;
             }
diff --git a/Runtime/Scripts/RetargetingProcessing/RetargetingProcessorCorrectBones.cs b/Runtime/Scripts/RetargetingProcessing/RetargetingProcessorCorrectBones.cs
@@ -110,7 +110,7 @@ public override void ProcessRetargetingLayer(RetargetingLayer retargetingLayer,
                 var targetCorrectionQuaternion = (Quaternion)nullableTargetCorrectionQuaternion;
 
                 // Make sure body part passes mask, and bone's position should be updated.
-                var bodyPart = BoneMappingsExtension.HumanBoneToAvatarBodyPart[humanBodyBone];
+                var bodyPart = BoneMappingsExtension.HumanBoneToAvatarBodyPartArray[(int)humanBodyBone];
                 var targetJoint = retargetingLayer.GetOriginalJoint(humanBodyBone);
                 if (retargetingLayer.CustomPositionsToCorrectLateUpdateMask != null &&
                     !retargetingLayer.CustomPositionsToCorrectLateUpdateMask.GetHumanoidBodyPartActive(bodyPart))
@@ -123,16 +123,16 @@ public override void ProcessRetargetingLayer(RetargetingLayer retargetingLayer,
                     continue;
                 }
 
-                // Make sure the joint position is valid before fixing it.
+                // If this is the first target position, check its validity.
+                // If it's valid, assume that the remaining positions are valid as well.
                 var currentTargetPosition = targetJoint.position;
-                if (!RiggingUtilities.IsFiniteVector3(currentTargetPosition))
+                if (i == 0 && !RiggingUtilities.IsFiniteVector3(currentTargetPosition))
                 {
-                    continue;
+                    return;
                 }
 
                 var rtWeight = Weight * retargetingLayer.RetargetingConstraint.weight;
-                var bodySectionOfJoint =
-                    OVRUnityHumanoidSkeletonRetargeter.OVRHumanBodyBonesMappings.BoneToBodySection[humanBodyBone];
+                var bodySectionOfJoint = retargetingLayer.GetHumanBodyBoneToBodySection(humanBodyBone);
                 bool isLeftHandFingersOrWrist =
                     bodySectionOfJoint == OVRUnityHumanoidSkeletonRetargeter.OVRHumanBodyBonesMappings.BodySection.LeftHand ||
                     humanBodyBone == HumanBodyBones.LeftHand;
diff --git a/Runtime/Scripts/RetargetingProcessing/RetargetingProcessorCorrectHand.cs b/Runtime/Scripts/RetargetingProcessing/RetargetingProcessorCorrectHand.cs
@@ -179,9 +179,9 @@ public override void SetupRetargetingProcessor(RetargetingLayer retargetingLayer
                 }
 
                 if ((Handedness == Handedness.Left &&
-                     BoneMappingsExtension.HumanBoneToAvatarBodyPart[i] == AvatarMaskBodyPart.LeftArm) ||
+                     BoneMappingsExtension.HumanBoneToAvatarBodyPartArray[(int)i] == AvatarMaskBodyPart.LeftArm) ||
                     (Handedness == Handedness.Right &&
-                     BoneMappingsExtension.HumanBoneToAvatarBodyPart[i] == AvatarMaskBodyPart.RightArm))
+                     BoneMappingsExtension.HumanBoneToAvatarBodyPartArray[(int)i] == AvatarMaskBodyPart.RightArm))
                 {
                     armBones.Add(boneTransform);
                 }
diff --git a/Samples/Prefabs/Retargeting/ArmatureSkinningUpdateRetarget.prefab b/Samples/Prefabs/Retargeting/ArmatureSkinningUpdateRetarget.prefab
@@ -1562,7 +1562,6 @@ MonoBehaviour:
   - {fileID: 11400000, guid: 15440560904a85846a7a4b75e1f7b96e, type: 2}
   - {fileID: 11400000, guid: 8033477507feea54a9c44eb17a69be20, type: 2}
   - {fileID: 11400000, guid: b47ca2dcc0aa6e34484b90e7122d3db4, type: 2}
-  - {fileID: 11400000, guid: 8eb3b8157e16d26409685b880d73b84e, type: 2}
   _retargetingAnimationRig:
     _rebindAnimator: 1
     _reEnableRig: 1

Original file line number	Diff line number	Diff line change
`@@ -97,6 +97,9 @@ private void Initialize()`
`97`	`97`	`{`
`98`	`98`	`retargetedBoneTarget.BoneId = humanBodyBoneToOVRBoneId[retargetedBoneTarget.HumanBodyBone];`
`99`	`99`	`}`
	`100`	`+`
	`101`	`+ // Sort by bone id.`
	`102`	`+ _boneTargets = _boneTargets.OrderBy(x => x.BoneId).ToArray();`
`100`	`103`	`_initialized = true;`
`101`	`104`	`}`
`102`	`105`
`@@ -110,22 +113,23 @@ public void ProcessSkeleton(OVRSkeleton skeleton)`
`110`	`113`	`{`
`111`	`114`	`return;`
`112`	`115`	`}`
	`116`	`+`
`113`	`117`	`Initialize();`
`114`		`- IList<OVRBone> bones = skeleton.Bones;`
	`118`	`+ var retargetedBoneTargetIndex = 0;`
	`119`	`+ var bones = skeleton.Bones;`
`115`	`120`	`for (var i = 0; i < bones.Count; i++)`
`116`	`121`	`{`
`117`		`- foreach (var retargetedBoneTarget in _boneTargets)`
	`122`	`+ var retargetedBoneTarget = _boneTargets[retargetedBoneTargetIndex];`
	`123`	`+ if (bones[i].Id == retargetedBoneTarget.BoneId)`
`118`	`124`	`{`
`119`		`- if (bones[i].Id == retargetedBoneTarget.BoneId)`
	`125`	`+ retargetedBoneTargetIndex++;`
	`126`	`+ var targetBone = bones[i].Transform;`
	`127`	`+ if (targetBone == null)`
`120`	`128`	`{`
`121`		`- var targetBone = bones[i].Transform;`
`122`		`- if (targetBone == null)`
`123`		`- {`
`124`		`- continue;`
`125`		`- }`
`126`		`- retargetedBoneTarget.Target.SetPositionAndRotation(`
`127`		`- targetBone.position, targetBone.rotation);`
	`129`	`+ continue;`
`128`	`130`	`}`
	`131`	`+ retargetedBoneTarget.Target.SetPositionAndRotation(`
	`132`	`+ targetBone.position, targetBone.rotation);`
`129`	`133`	`}`
`130`	`134`	`}`
`131`	`135`	`}`
Original file line number	Diff line number	Diff line change
`@@ -176,7 +176,7 @@ public override void Update(PlaybackAnimationJob job, ref T data)`
`176`	`176`
`177`	`177`	`if (playbackType == PlaybackAnimationData.AnimationPlaybackType.None \|\|`
`178`	`178`	`(data.AnimationMask != null &&`
`179`		`- !data.AnimationMask.GetHumanoidBodyPartActive(BoneMappingsExtension.HumanBoneToAvatarBodyPart[i])))`
	`179`	`+ !data.AnimationMask.GetHumanoidBodyPartActive(BoneMappingsExtension.HumanBoneToAvatarBodyPartArray[(int)i])))`
`180`	`180`	`{`
`181`	`181`	`job.BonePositionDeltas[index] = Vector3.zero;`
`182`	`182`	`job.BoneRotationDeltas[index] = Quaternion.identity;`
Original file line number	Diff line number	Diff line change
`@@ -171,7 +171,7 @@ private void CleanUpOldProxyTransforms()`
`171`	`171`	`GameObject.Destroy(proxyParent.gameObject);`
`172`	`172`	`}`
`173`	`173`
`174`		`- // <summary>`
	`174`	`+ /// <summary>`
`175`	`175`	`/// Sometimes source transforms change even though the IDs are the same,`
`176`	`176`	`/// and proxies are only recreated if the bone IDs of the sources change.`
`177`	`177`	`/// Make sure they are set properly.`
Original file line number	Diff line number	Diff line change
`@@ -179,9 +179,9 @@ public override void SetupRetargetingProcessor(RetargetingLayer retargetingLayer`
`179`	`179`	`}`
`180`	`180`
`181`	`181`	`if ((Handedness == Handedness.Left &&`
`182`		`- BoneMappingsExtension.HumanBoneToAvatarBodyPart[i] == AvatarMaskBodyPart.LeftArm) \|\|`
	`182`	`+ BoneMappingsExtension.HumanBoneToAvatarBodyPartArray[(int)i] == AvatarMaskBodyPart.LeftArm) \|\|`
`183`	`183`	`(Handedness == Handedness.Right &&`
`184`		`- BoneMappingsExtension.HumanBoneToAvatarBodyPart[i] == AvatarMaskBodyPart.RightArm))`
	`184`	`+ BoneMappingsExtension.HumanBoneToAvatarBodyPartArray[(int)i] == AvatarMaskBodyPart.RightArm))`
`185`	`185`	`{`
`186`	`186`	`armBones.Add(boneTransform);`
`187`	`187`	`}`