fix(Runtime): Allow multiple rotation adjustments for retargeting

Summary: Allow a joint to be affected by multiple joint rotations. Eventually, this change would need to exist in `OVRUnityHumanoidSkeletonRetargeter`.

Reviewed By: andkim-meta

Differential Revision: D48768146

fbshipit-source-id: e03c4918caea42f1251466f6d4305c4087155c09

Author: sohailshafiiWk

Runtime/Scripts/AnimationRigging/RetargetingLayer.cs

@@ -62,6 +62,25 @@ public Vector3 GetPositionOffset()
             }
         }
 
+        /// <summary>
+        /// Allows one to adjust the per-joint rotation offsets computed between
+        /// source (OVRBody) and target characters. To avoid gimbal lock,
+        /// a series of rotations are permitted.
+        /// </summary>
+        [System.Serializable]
+        public class JointRotationTweaks
+        {
+            /// <summary>
+            /// Joint to affect.
+            /// </summary>
+            public HumanBodyBones Joint;
+
+            /// <summary>
+            /// A series of rotation tweaks.
+            /// </summary>
+            public Quaternion[] RotationTweaks;
+        }
+
         /// <summary>
         /// The array of joint position adjustments.
         /// </summary>
@@ -160,6 +179,25 @@ public RetargetingAnimationConstraint RetargetingConstraint
             set { _retargetingAnimationConstraint = value; }
         }
 
+        /// <summary>
+        /// Joint rotation tweaks array.
+        /// </summary>
+        [SerializeField]
+        [Tooltip(RetargetingLayerTooltips.JointRotationTweaks)]
+        protected JointRotationTweaks[] _jointRotationTweaks;
+        public JointRotationTweaks[] JointRotationTweaksArray
+        {
+            get => _jointRotationTweaks;
+            set => _jointRotationTweaks = value;
+        }
+        /// <summary>
+        /// Pre-compute these values each time the editor changes for the purposes
+        /// of efficiency.
+        /// </summary>
+        private Dictionary<HumanBodyBones, Quaternion> _humanBoneToAccumulatedRotationTweaks =
+            new Dictionary<HumanBodyBones, Quaternion>();
+        private List<HumanBodyBones> _bonesToRemove = new List<HumanBodyBones> ();
+
         private Pose[] _defaultPoses;
         private IJointConstraint[] _jointConstraints;
         private ProxyTransformLogic _proxyTransformLogic = new ProxyTransformLogic();
@@ -220,6 +258,8 @@ protected override void Start()
             CacheJointConstraints();
 
             ValidateHumanoid();
+
+            PrecomputeJointRotationTweaks();
         }
 
         private void ConstructDefaultPoseInformation()
@@ -305,6 +345,78 @@ protected virtual void OnApplicationFocus(bool hasFocus)
             _isFocusedWhileInBuild = hasFocus;
         }
 
+        protected virtual void OnValidate()
+        {
+            PrecomputeJointRotationTweaks();
+        }
+
+        /// <summary>
+        /// When the object's properties are modified, accumulate joint rotations
+        /// and cache those values. This saves on computation when the tweaks field is used.
+        /// </summary>
+        protected void PrecomputeJointRotationTweaks()
+        {
+#if UNITY_EDITOR
+            if (!UnityEditor.EditorApplication.isPlaying)
+            {
+                return;
+            }
+#endif
+
+            foreach (var rotationTweak in _jointRotationTweaks)
+            {
+                var allRotations = rotationTweak.RotationTweaks;
+                var joint = rotationTweak.Joint;
+
+                Quaternion accumulatedRotation = Quaternion.identity;
+                foreach (var rotationValue in allRotations)
+                {
+                    // Make sure the quaternion is valid. Quaternions are initialized to all
+                    // zeroes by default, which makes them invalid.
+                    if (rotationValue.w < Mathf.Epsilon && rotationValue.x < Mathf.Epsilon &&
+                        rotationValue.y < Mathf.Epsilon && rotationValue.z < Mathf.Epsilon)
+                    {
+                        continue;
+                    }
+
+                    accumulatedRotation *= rotationValue;
+                }
+
+                if (!_humanBoneToAccumulatedRotationTweaks.ContainsKey(joint))
+                {
+                    _humanBoneToAccumulatedRotationTweaks.Add(joint, accumulatedRotation);
+                }
+                else
+                {
+                    _humanBoneToAccumulatedRotationTweaks[joint] = accumulatedRotation;
+                }
+            }
+
+            _bonesToRemove.Clear();
+            // If the user removed bones from the UI, remove those from the dictionary.
+            foreach (var bone in _humanBoneToAccumulatedRotationTweaks.Keys)
+            {
+                bool boneFound = false;
+                foreach (var rotationTweak in _jointRotationTweaks)
+                {
+                    if (rotationTweak.Joint == bone)
+                    {
+                        boneFound = true;
+                        break;
+                    }
+                }
+
+                if (!boneFound)
+                {
+                    _bonesToRemove.Add(bone);
+                }
+            }
+            foreach(var boneToRemove in _bonesToRemove)
+            {
+                _humanBoneToAccumulatedRotationTweaks.Remove(boneToRemove);
+            }
+        }
+
         /// <inheritdoc />
         protected override void Update()
         {
@@ -419,7 +531,7 @@ private void CorrectBones()
                         targetJoint.rotation =
                             Quaternion.Slerp(targetJoint.rotation,
                                 Bones[i].Transform.rotation *
-                                targetData.CorrectionQuaternion.Value,
+                                targetData.CorrectionQuaternion.Value * GetRotationTweak(humanBodyBone),
                                 handWeight);
                     }
                     if (_correctPositionsLateUpdate)
@@ -438,7 +550,7 @@ private void CorrectBones()
                             targetJoint.rotation =
                                 Quaternion.Slerp(targetJoint.rotation,
                                     Bones[i].Transform.rotation *
-                                    targetData.CorrectionQuaternion.Value,
+                                    targetData.CorrectionQuaternion.Value * GetRotationTweak(humanBodyBone),
                                     handWeight);
                         }
 
@@ -455,6 +567,18 @@ private void CorrectBones()
             }
         }
 
+        protected Quaternion GetRotationTweak(HumanBodyBones humanBody)
+        {
+            Quaternion rotation;
+
+            if (!_humanBoneToAccumulatedRotationTweaks.TryGetValue(humanBody, out rotation))
+            {
+                rotation = Quaternion.identity;
+            }
+
+            return rotation;
+        }
+
         protected virtual bool ShouldUpdatePositionOfBone(HumanBodyBones humanBodyBone)
         {
             var bodySectionOfJoint = OVRHumanBodyBonesMappings.BoneToBodySection[humanBodyBone];
@@ -516,7 +640,8 @@ public void FillTransformArrays(List<Transform> sourceTransforms,
                 targetTransforms.Add(targetBoneData.OriginalJoint);
                 shouldUpdatePositions.Add(false);
                 shouldUpdateRotations.Add(false);
-                rotationOffsets.Add(targetBoneData.CorrectionQuaternion.Value);
+                rotationOffsets.Add(targetBoneData.CorrectionQuaternion.Value *
+                    GetRotationTweak(targetHumanBodyBone));
                 rotationAdjustments.Add(Quaternion.identity);
             }
         }
@@ -582,14 +707,16 @@ public void UpdateAdjustments(Quaternion[] rotationOffsets,
 
                 if (adjustment == null)
                 {
-                    SetUpDefaultAdjustment(rotationOffsets, shouldUpdatePositions,
+                    SetUpDefaultAdjustment(targetHumanBodyBone,
+                        rotationOffsets, shouldUpdatePositions,
                         shouldUpdateRotations, rotationAdjustments, arrayId,
                         targetBoneData, bodySectionInPositionArray,
                         jointFailsMask);
                 }
                 else
                 {
-                    SetUpCustomAdjustment(rotationOffsets, shouldUpdatePositions,
+                    SetUpCustomAdjustment(targetHumanBodyBone,
+                        rotationOffsets, shouldUpdatePositions,
                         shouldUpdateRotations,
                         rotationAdjustments, adjustment, arrayId,
                         targetBoneData, bodySectionInPositionArray,
@@ -600,26 +727,28 @@ public void UpdateAdjustments(Quaternion[] rotationOffsets,
             }
         }
 
-        private void SetUpDefaultAdjustment(Quaternion[] rotationOffsets,
+        private void SetUpDefaultAdjustment(HumanBodyBones humanBodyBone, Quaternion[] rotationOffsets,
             bool[] shouldUpdatePositions, bool[] shouldUpdateRotations,
             Quaternion[] rotationAdjustments, int arrayId,
             OVRSkeletonMetadata.BoneData targetBoneData,
             bool bodySectionInPositionArray, bool jointFailsMask)
         {
-            rotationOffsets[arrayId] = targetBoneData.CorrectionQuaternion.Value;
+            rotationOffsets[arrayId] = targetBoneData.CorrectionQuaternion.Value *
+                GetRotationTweak(humanBodyBone);
             shouldUpdatePositions[arrayId] = !jointFailsMask && bodySectionInPositionArray;
             shouldUpdateRotations[arrayId] = !jointFailsMask;
             rotationAdjustments[arrayId] = Quaternion.identity;
         }
 
-        private void SetUpCustomAdjustment(Quaternion[] rotationOffsets,
+        private void SetUpCustomAdjustment(HumanBodyBones humanBodyBone, Quaternion[] rotationOffsets,
             bool[] shouldUpdatePositions, bool[] shouldUpdateRotations,
             Quaternion[] rotationAdjustments,
             JointAdjustment adjustment, int arrayId,
             OVRSkeletonMetadata.BoneData targetBoneData,
             bool bodySectionInPositionArray, bool jointFailsMask)
         {
-            rotationOffsets[arrayId] = targetBoneData.CorrectionQuaternion.Value;
+            rotationOffsets[arrayId] = targetBoneData.CorrectionQuaternion.Value *
+                GetRotationTweak(humanBodyBone);
             shouldUpdatePositions[arrayId] =
                 !adjustment.DisablePositionTransform &&
                 bodySectionInPositionArray &&

Runtime/Scripts/Tooltips.cs

@@ -1234,6 +1234,9 @@ public static class RetargetingLayerTooltips
 
         public const string RetargetingAnimationContraint =
             "Related retargeting constraint.";
+
+        public const string JointRotationTweaks =
+            "Joint rotation tweaks array.";
     }
 
     public static class LateMirroredObjectTooltips

Samples/Prefabs/Retargeting/ArmatureSkinningUpdateRetarget.prefab

@@ -885,7 +885,7 @@ PrefabInstance:
       objectReference: {fileID: 0}
     - target: {fileID: 9121803696401781268, guid: 4d1160b6537bb3e42802d3a5219e72c4, type: 3}
       propertyPath: m_Name
-      value: ArmatureSkinningUpdateRetarget
+      value: ArmatureSkinningUpdateRetargetUser
       objectReference: {fileID: 0}
     m_RemovedComponents: []
   m_SourcePrefab: {fileID: 100100000, guid: 4d1160b6537bb3e42802d3a5219e72c4, type: 3}
@@ -1038,6 +1038,15 @@ MonoBehaviour:
   _applyAnimationConstraintsToCorrectedPositions: 1
   _enableTrackingByProxy: 0
   _retargetingAnimationConstraint: {fileID: 318886850}
+  _jointRotationTweaks:
+  - Joint: 17
+    RotationTweaks:
+    - {x: 0, y: 0.42261827, z: 0, w: 0.90630776}
+    - {x: 0, y: 0, z: 0.1736483, w: -0.9848077}
+  - Joint: 18
+    RotationTweaks:
+    - {x: 0, y: 0.42261827, z: 0, w: 0.90630776}
+    - {x: 0, y: 0, z: 0.1736483, w: -0.9848077}
 --- !u!114 &208026093
 MonoBehaviour:
   m_ObjectHideFlags: 0