feat(Runtime): Add tracking bone transforms by proxy

Summary: Adds tracking transforms by proxy. Any job that tracks the source bone transforms of the retargeting layer runs the risk of having to be recreated when the bones are re-initialized. The proxy transforms never get re-created unless bone ID that the proxy is associated changes.

Reviewed By: andkim-meta

Differential Revision: D46125799

fbshipit-source-id: d25588694fb9505c571ebf0c766ae353922e2e04

Author: sohailshafiiWk

Runtime/Scripts/AnimationRigging/AnimationRigSetup.cs

@@ -119,11 +119,28 @@ public RetargetingLayer RetargetingLayerComp
             set { _retargetingLayer = value; }
         }
 
+        /// <summary>
+        /// Use proxy transforms to check skeletal changes.
+        /// Proxy transforms can be used in case the original
+        /// skeleton updates too much.
+        /// </summary>
+        [SerializeField]
+        [Tooltip(AnimationRigSetupTooltips.CheckSkeletalUpdatesByProxy)]
+        protected bool _checkSkeletalUpdatesByProxy = false;
+
+        /// <inheritdoc cref="_checkSkeletalUpdatesByProxy"/>
+        public bool CheckSkeletalUpdatesByProxy
+        {
+            get { return _checkSkeletalUpdatesByProxy; }
+            set { _checkSkeletalUpdatesByProxy = value;  }
+        }
+
         private bool _ranSetup;
         private int _lastSkeletonChangeCount = -1;
         private bool _pendingSkeletalUpdateFromLastFrame = false;
         private IOVRSkeletonConstraint[] _iovrSkeletonConstraints;
 
+        private int _proxyChangeCount = 0;
         private int _lastRetargetedTransformCount = -1;
 
         /// <summary>
@@ -202,7 +219,7 @@ protected virtual void OnApplicationFocus(bool hasFocus)
             {
                 if (!hasFocus)
                 {
-                    DisableRig();
+                    DisableRigAndUpdateState();
                     _rigBuilder.Evaluate(Time.deltaTime);
                 }
                 else
@@ -239,7 +256,7 @@ private void RunInitialSetup()
             }
         }
 
-        private void DisableRig()
+        private void DisableRigAndUpdateState()
         {
             if (_rigBuilder)
             {
@@ -258,6 +275,11 @@ private void DisableRig()
             {
                 _lastRetargetedTransformCount = _retargetingLayer.GetNumberOfTransformsRetargeted();
             }
+
+            if (_checkSkeletalUpdatesByProxy && _retargetingLayer != null)
+            {
+                _proxyChangeCount = _retargetingLayer.ProxyChangeCount;
+            }
         }
 
         private void EnableRig()
@@ -293,19 +315,38 @@ private void CheckForSkeletalChanges()
             {
                 return;
             }
-            if (_lastSkeletonChangeCount != _skeleton.SkeletonChangedCount ||
-                HasRetargeterBeenUpdated())
+            if (_lastSkeletonChangeCount != _skeleton.SkeletonChangedCount)
             {
+                bool skeletalOrRetargeterChangeDetected =
+                   _checkSkeletalUpdatesByProxy && HasSkeletonProxiesBeenRecreated() ||
+                   HasRetargeterBeenUpdated();
+
+                // ONLY regenerate rig if change has been detected
+                if (!skeletalOrRetargeterChangeDetected)
+                {
+                    _lastSkeletonChangeCount = _skeleton.SkeletonChangedCount;
+                    return;
+                }
+
                 // Allow rig to initialize next frame so that constraints can
                 // catch up to skeletal changes in this frame.
                 _pendingSkeletalUpdateFromLastFrame = true;
                 _lastSkeletonChangeCount = _skeleton.SkeletonChangedCount;
 
-                DisableRig();
+                DisableRigAndUpdateState();
                 Debug.LogWarning("Detected skeletal change. Disabling the rig.");
             }
         }
 
+        private bool HasSkeletonProxiesBeenRecreated()
+        {
+            if (_retargetingLayer == null)
+            {
+                return false;
+            }
+            return _proxyChangeCount != _retargetingLayer.ProxyChangeCount;
+        }
+
         private bool HasRetargeterBeenUpdated()
         {
             if (_retargetingLayer == null)

Runtime/Scripts/AnimationRigging/ProxyTransformLogic.cs

@@ -0,0 +1,182 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace Oculus.Movement.AnimationRigging.Utils
+{
+    /// <summary>
+    /// Keeps track of bone transforms, which might get re-allocated, by
+    /// using proxies. Since proxies don't typically re-allocate, any jobs that
+    /// depend on them do not require re-allocation. Bone ID changes require
+    /// reallocation, however.
+    /// </summary>
+    public class ProxyTransformLogic
+    {
+        /// <summary>
+        /// Class that tracks proxy and source transforms.
+        /// </summary>
+        public class ProxyTransform
+        {
+            /// <summary>
+            /// Proxy transform constructor.
+            /// </summary>
+            /// <param name="drivenTransform">Transform to be driven by source;
+            ///  the actual proxy itself.</param>
+            /// <param name="sourceTransform">Source transform.</param>
+            /// <param name="boneId">The bone ID of the source.</param>
+            public ProxyTransform(Transform drivenTransform,
+                Transform sourceTransform,
+                OVRSkeleton.BoneId boneId)
+            {
+                DrivenTransform = drivenTransform;
+                SourceTransform = sourceTransform;
+                BoneId = boneId;
+            }
+
+            /// <summary>
+            ///  Update proxy position to track source.
+            /// </summary>
+            public void Update()
+            {
+                DrivenTransform.SetPositionAndRotation(
+                    SourceTransform.position,
+                    SourceTransform.rotation);
+            }
+
+            /// <summary>
+            /// Validate source transform to make sure it's
+            /// always correct w.r.t. the bone transform.
+            /// </summary>
+            /// <param name="bone">Bone to track.</param>
+            public void ValidateSource(OVRBone bone)
+            {
+                SourceTransform = bone.Transform;
+            }
+
+            /// <summary>
+            /// The driven, proxy transform.
+            /// </summary>
+            public Transform DrivenTransform { get; private set; }
+            /// <summary>
+            /// The source transform to drive the proxy.
+            /// </summary>
+            public Transform SourceTransform { get; private set; }
+            /// <summary>
+            /// The bone ID of the source.
+            /// </summary>
+            public OVRSkeleton.BoneId BoneId { get; private set; }
+        }
+
+        private ProxyTransform[] _proxyTransforms = null;
+        /// <summary>
+        /// Accessor for array of proxy transforms that track skeletal bones.
+        /// </summary>
+        public ProxyTransform[] ProxyTransforms => _proxyTransforms;
+
+        /// <summary>
+        /// Triggered if proxy transforms were recreated.
+        /// </summary>
+        public int ProxyChangeCount { get; private set; } = 0;
+
+        /// <summary>
+        /// Updates the state of the proxies using the skeletal
+        /// bones provided.
+        /// </summary>
+        /// <param name="bones">Bones of the skeleton tracked by
+        /// proxy transforms.</param>
+        public void UpdateState(IList<OVRBone> bones)
+        {
+            ReallocateProxiesIfBoneIdsHaveChanged(bones);
+
+            ValidateProxySourceTransforms(bones);
+
+            UpdateProxyTransforms(bones);
+        }
+
+        private void ReallocateProxiesIfBoneIdsHaveChanged(IList<OVRBone> bones)
+        {
+            int numBones = bones.Count;
+            if (!BoneIdsChanged(bones))
+            {
+                return;
+            }
+
+            Debug.Log($"Creating a new set of {numBones} bone proxies.");
+            CleanUpOldProxyTransforms();
+            _proxyTransforms = new ProxyTransform[numBones];
+            for (int boneIndex = 0; boneIndex < numBones; boneIndex++)
+            {
+                var originalBone = bones[boneIndex];
+                var drivenTransform = new GameObject($"Proxy {boneIndex}").transform;
+                _proxyTransforms[boneIndex] =
+                    new ProxyTransform(drivenTransform,
+                        originalBone.Transform, originalBone.Id);
+            }
+            ProxyChangeCount++;
+        }
+
+        private bool BoneIdsChanged(IList<OVRBone> bones)
+        {
+            int numBones = bones.Count;
+            if (_proxyTransforms == null ||
+                _proxyTransforms.Length != numBones)
+            {
+                return true;
+            }
+
+            for (int boneIndex = 0; boneIndex < numBones; boneIndex++)
+            {
+                var proxyBone = _proxyTransforms[boneIndex];
+                var originalBone = bones[boneIndex];
+                if (proxyBone.BoneId != originalBone.Id)
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        private void CleanUpOldProxyTransforms()
+        {
+            if (_proxyTransforms == null)
+            {
+                return;
+            }
+            foreach (var proxyTransform in _proxyTransforms)
+            {
+                GameObject.Destroy(proxyTransform.DrivenTransform.gameObject);
+            }
+            _proxyTransforms = null;
+        }
+
+        // <summary>
+        /// Sometimes source transforms change even though the IDs are the same,
+        /// and proxies are only recreated if the bone IDs of the sources change.
+        /// Make sure they are set properly.
+        /// </summary>
+        /// <param name="bones">The list of bones to track.</param>
+        private void ValidateProxySourceTransforms(IList<OVRBone> bones)
+        {
+            if (_proxyTransforms == null)
+            {
+                return;
+            }
+            int numBones = bones.Count;
+            for (int boneIndex = 0; boneIndex < numBones; boneIndex++)
+            {
+                _proxyTransforms[boneIndex].ValidateSource(bones[boneIndex]);
+            }
+        }
+
+        private void UpdateProxyTransforms(IList<OVRBone> bones)
+        {
+            var boneCount = bones.Count;
+            for (int boneIndex = 0; boneIndex < boneCount; boneIndex++)
+            {
+                _proxyTransforms[boneIndex].Update();
+            }
+        }
+    }
+}

Runtime/Scripts/AnimationRigging/ProxyTransformLogic.cs.meta

@@ -1,6 +1,7 @@
 // Copyright (c) Meta Platforms, Inc. and affiliates.
 
 using Oculus.Interaction;
+using Oculus.Movement.AnimationRigging.Utils;
 using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.Animations;
@@ -138,11 +139,37 @@ public AvatarMask MaskToSetToTPoseComp
             set { _maskToSetToTPose = value; }
         }
 
+        /// <summary>
+        /// Create proxy transforms that track the skeletal bones. If the
+        /// skeletal bone transforms change, that won't necessitate creating new
+        /// proxy transforms in most cases. This means any Animation jobs
+        /// that track the skeletal bone transform can use proxies
+        /// instead, which get re-allocated less often. Re-allocation would mean
+        /// having to create new animation jobs.
+        /// </summary>
+        [SerializeField]
+        [Tooltip(RetargetingLayerTooltips.EnableTrackingByProxy)]
+        private bool _enableTrackingByProxy = false;
+
+        /// <inheritdoc cref="_enableTrackingByProxy"/>
+        public bool EnableTrackingByProxy
+        {
+            get { return _enableTrackingByProxy; }
+            set { _enableTrackingByProxy = value; }
+        }
+
         private Pose[] _defaultPoses;
         private IJointConstraint[] _jointConstraints;
+        private ProxyTransformLogic _proxyTransformLogic = new ProxyTransformLogic();
 
         /// <summary>
-        /// Gets number of transforms being retargeted currently.
+        /// Triggered if proxy transforms were recreated.
+        /// </summary>
+        public int ProxyChangeCount => _proxyTransformLogic.ProxyChangeCount;
+
+        /// <summary>
+        /// Gets number of transforms being retargeted currently. This can change during
+        /// initialization.
         /// </summary>
         /// <returns>Number of transforms with a valid correction quaternion.</returns>
         public int GetNumberOfTransformsRetargeted()
@@ -209,6 +236,11 @@ protected override void Update()
             UpdateSkeleton();
 
             RecomputeSkeletalOffsetsIfNecessary();
+
+            if (_enableTrackingByProxy)
+            {
+                _proxyTransformLogic.UpdateState(Bones);
+            }
         }
 
         /// <summary>
@@ -376,7 +408,9 @@ public void FillTransformArrays(List<Transform> sourceTransforms,
                     }
                 }
 
-                sourceTransforms.Add(currentBone.Transform);
+                sourceTransforms.Add(_enableTrackingByProxy ?
+                    _proxyTransformLogic.ProxyTransforms[i].DrivenTransform :
+                    currentBone.Transform);
                 targetTransforms.Add(targetBoneData.OriginalJoint);
                 shouldUpdatePositions.Add(false);
                 rotationOffsets.Add(targetBoneData.CorrectionQuaternion.Value);

Runtime/Scripts/AnimationRigging/RetargetingLayer.cs

@@ -983,6 +983,11 @@ public static class AnimationRigSetupTooltips
 
         public const string RetargetingLayer =
             "Retargeting layer component to get data from.";
+
+        public const string CheckSkeletalUpdatesByProxy =
+            "Use proxy transforms to check skeletal changes. " +
+            "Proxy transforms can be used in case the original " +
+            "skeleton updates too much.";
     }
 
     public static class HipPinningConstraintCalibrationTooltips
@@ -1138,12 +1143,19 @@ public static class RetargetingLayerTooltips
     {
         public const string DisableAvatar =
             "Disable avatar for accurate positions, especially fingers.";
+
         public const string PositionsToCorrectLateUpdate =
             "Positions to correct after the fact. Avatar " +
             "masks prevent setting positions of the hands precisely.";
+
         public const string MaskToSetToTPose =
             "Since some bones are not affected by retargeting, " +
             "some joints should be reset to t-pose.";
+
+        public const string EnableTrackingByProxy =
+            "Create proxy transforms that track the skeletal bones. If the " +
+            "skeletal bone transforms change, that won't necessitate creating new " +
+            "proxy transforms in most cases.";
     }
 
     public static class LateMirroredObjectTooltips