// Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. using System; using System.Collections.Generic; using System.Linq; using Unity.Collections; using UnityEngine; using static Meta.XR.Movement.MSDKUtility; using static Meta.XR.Movement.Retargeting.SkeletonData; namespace Meta.XR.Movement.Editor { /// /// Utility class for aligning joints in a skeleton. /// public class JointAlignmentUtility { /// /// Aligns finger bones between source and target skeletons using a position-independent approach. /// /// The editor window. /// The array of starting bone IDs for fingers. /// The array of ending bone IDs for fingers. /// The wrist joint type. public static void PerformFingerMatching(MSDKUtilityEditorWindow win, FullBodyTrackingBoneId[] startBoneIds, FullBodyTrackingBoneId[] endBoneIds, KnownJointType wristType) { var source = win.SourceInfo; var target = win.TargetInfo; // Get the wrist joint from the target skeleton var wristJointIndex = GetKnownJointIndex(target, wristType); var wristTransform = target.SkeletonJoints[wristJointIndex]; if (wristTransform == null) { Debug.LogError($"Could not find wrist transform for {wristType}"); return; } // Update wrist rotation. ApplyWristFingerAlignment(win, startBoneIds, wristType); // Get finger chains using the simplified method Dictionary> fingerChains = GetFingerChains(wristTransform); // Process each finger chain foreach (var fingerPair in fingerChains) { string fingerName = fingerPair.Key; List fingerChain = fingerPair.Value; if (fingerChain.Count < 2) { // Need at least two joints for a proper finger continue; } // Find the corresponding source fingertip FullBodyTrackingBoneId? sourceFingertipId = null; // Map finger name to source bone ID if (fingerName.Equals("Thumb", StringComparison.OrdinalIgnoreCase)) { sourceFingertipId = wristType == KnownJointType.LeftWrist ? FullBodyTrackingBoneId.LeftHandThumbTip : FullBodyTrackingBoneId.RightHandThumbTip; } else if (fingerName.Equals("Index", StringComparison.OrdinalIgnoreCase)) { sourceFingertipId = wristType == KnownJointType.LeftWrist ? FullBodyTrackingBoneId.LeftHandIndexTip : FullBodyTrackingBoneId.RightHandIndexTip; } else if (fingerName.Equals("Middle", StringComparison.OrdinalIgnoreCase)) { sourceFingertipId = wristType == KnownJointType.LeftWrist ? FullBodyTrackingBoneId.LeftHandMiddleTip : FullBodyTrackingBoneId.RightHandMiddleTip; } else if (fingerName.Equals("Ring", StringComparison.OrdinalIgnoreCase)) { sourceFingertipId = wristType == KnownJointType.LeftWrist ? FullBodyTrackingBoneId.LeftHandRingTip : FullBodyTrackingBoneId.RightHandRingTip; } else if (fingerName.Equals("Pinky", StringComparison.OrdinalIgnoreCase)) { sourceFingertipId = wristType == KnownJointType.LeftWrist ? FullBodyTrackingBoneId.LeftHandLittleTip : FullBodyTrackingBoneId.RightHandLittleTip; } if (!sourceFingertipId.HasValue || (int)sourceFingertipId.Value >= source.ReferencePose.Length) { // Skip if we don't have a valid source fingertip continue; } // Get the source fingertip position Vector3 sourceFingertipPosition = source.ReferencePose[(int)sourceFingertipId.Value].Position; // Apply simple rotation to align the finger with the source fingertip AlignFingerToTarget(fingerChain, sourceFingertipPosition); } // Update the T-pose data after adjusting the fingers UpdateTPoseData(target); } /// /// Aligns wrist joints between source and target skeletons. /// /// The source skeleton configuration. /// The target skeleton configuration. public static void PerformWristMatching(MSDKUtilityEditorConfig source, MSDKUtilityEditorConfig target) { // Rotate to match left/right hand. var sourceRightHand = source.ReferencePose[(int)FullBodyTrackingBoneId.RightHandWrist]; var sourceLeftHand = source.ReferencePose[(int)FullBodyTrackingBoneId.LeftHandWrist]; var leftArmIndex = GetKnownJointIndex(target, KnownJointType.LeftUpperArm); var rightArmIndex = GetKnownJointIndex(target, KnownJointType.RightUpperArm); var leftUpperArm = target.SkeletonJoints[leftArmIndex]; var leftHand = target.SkeletonJoints[GetKnownJointIndex(target, KnownJointType.LeftWrist)]; leftUpperArm.rotation = GetBestRotation(leftUpperArm, leftHand, sourceLeftHand.Position); var rightUpperArm = target.SkeletonJoints[rightArmIndex]; var rightHand = target.SkeletonJoints[GetKnownJointIndex(target, KnownJointType.RightWrist)]; rightUpperArm.rotation = GetBestRotation(rightUpperArm, rightHand, sourceRightHand.Position); UpdateTPoseData(target); } /// /// Performs automatic alignment between source and target skeletons. /// /// The editor window. public static void AutoAlignment(MSDKUtilityEditorWindow win) { win.TargetInfo.SkeletonTPoseType = win.Step switch { MSDKUtilityEditorConfig.EditorStep.MinTPose => SkeletonTPoseType.MinTPose, MSDKUtilityEditorConfig.EditorStep.MaxTPose => SkeletonTPoseType.MaxTPose, _ => win.TargetInfo.SkeletonTPoseType }; win.ModifyConfig(false, true); if (win.FileReader.IsPlaying) { win.Previewer.DestroyPreviewCharacterRetargeter(); win.OpenPlaybackFile(win.CurrentPreviewPose); } } /// /// Loads the scale from the target configuration into the utility configuration. /// /// The target configuration. /// The utility configuration to update. public static void LoadScale(MSDKUtilityEditorConfig target, MSDKUtilityEditorConfig utilityConfig) { var targetTPose = new NativeArray(target.ReferencePose.Length, Allocator.Temp); targetTPose.CopyFrom(target.ReferencePose); ConvertJointPose(target.ConfigHandle, SkeletonType.TargetSkeleton, JointRelativeSpaceType.RootOriginRelativeSpace, JointRelativeSpaceType.LocalSpaceScaled, targetTPose, out var scaledTPose); GetJointIndexByKnownJointType(target.ConfigHandle, SkeletonType.TargetSkeleton, KnownJointType.Root, out var rootJointIndex); if (utilityConfig.Step != MSDKUtilityEditorConfig.EditorStep.Configuration) { var calculatedScale = scaledTPose[rootJointIndex].Scale; // Clamp scale components to the character retargeter range utilityConfig.RootScale = new Vector3( Mathf.Clamp(calculatedScale.x, MSDKUtilityEditorPreviewer.MinScale, MSDKUtilityEditorPreviewer.MaxScale), Mathf.Clamp(calculatedScale.y, MSDKUtilityEditorPreviewer.MinScale, MSDKUtilityEditorPreviewer.MaxScale), Mathf.Clamp(calculatedScale.z, MSDKUtilityEditorPreviewer.MinScale, MSDKUtilityEditorPreviewer.MaxScale) ); } else { utilityConfig.RootScale = Vector3.one; } } /// /// Updates the T-pose data in the target configuration based on the current skeleton joints. /// /// The target configuration to update. public static void UpdateTPoseData(MSDKUtilityEditorConfig target) { for (var i = 0; i < target.SkeletonJoints.Length; i++) { var joint = target.SkeletonJoints[i]; if (joint == null) { target.ReferencePose[i] = NativeTransform.Identity(); } else { target.ReferencePose[i] = new NativeTransform { Position = joint.position, Orientation = joint.rotation, Scale = Vector3.one * joint.localScale.x }; } } } /// /// Scales the arms of the target skeleton to match the source skeleton. /// /// The source configuration. /// The target configuration to modify. public static void PerformArmScaling(MSDKUtilityEditorConfig source, MSDKUtilityEditorConfig target) { // Scale arms - stretch/squash for aligning X-axis. var sourceRightHand = source.ReferencePose[(int)FullBodyTrackingBoneId.RightHandWrist]; var sourceLeftHand = source.ReferencePose[(int)FullBodyTrackingBoneId.LeftHandWrist]; var targetRightHand = GetTargetTPoseFromKnownJoint(target, KnownJointType.RightWrist); var targetLeftHand = GetTargetTPoseFromKnownJoint(target, KnownJointType.LeftWrist); var sourceRightWristVal = Mathf.Abs(sourceRightHand.Position.x); var targetRightWristVal = Mathf.Abs(targetRightHand.Position.x); var sourceLeftWristVal = Mathf.Abs(sourceLeftHand.Position.x); var targetLeftWristVal = Mathf.Abs(targetLeftHand.Position.x); var xScaleRatioRightWrist = sourceRightWristVal / (targetRightWristVal > 0.0f ? targetRightWristVal : 1.0f); var xScaleRatioLeftWrist = sourceLeftWristVal / (targetLeftWristVal > 0.0f ? targetLeftWristVal : 1.0f); GetJointIndexByKnownJointType(target.ConfigHandle, SkeletonType.TargetSkeleton, KnownJointType.RightUpperArm, out var rightArmIndex); GetJointIndexByKnownJointType(target.ConfigHandle, SkeletonType.TargetSkeleton, KnownJointType.LeftUpperArm, out var leftArmIndex); GetParentJointIndex(target.ConfigHandle, SkeletonType.TargetSkeleton, rightArmIndex, out var rightIndex); GetParentJointIndex(target.ConfigHandle, SkeletonType.TargetSkeleton, leftArmIndex, out var leftIndex); ApplyScalingToJointIndexAndChildren(target, rightIndex, xScaleRatioRightWrist, true); ApplyScalingToJointIndexAndChildren(target, leftIndex, xScaleRatioLeftWrist, true); } /// /// Applies scaling to a joint and all its children. /// /// The target configuration to modify. /// The index of the joint to scale. /// The scale factor to apply. /// Whether to skip scaling the specified joint and only scale its children. public static void ApplyScalingToJointIndexAndChildren(MSDKUtilityEditorConfig target, int jointIndex, float scaleFactor, bool skip = false) { // Apply the scaling to the current joint. if (!skip) { var joint = target.ReferencePose[jointIndex]; joint.Position.x *= scaleFactor; target.ReferencePose[jointIndex] = joint; } // Recursively apply the scaling to the children of the current joint. for (var i = 0; i < target.ParentIndices.Length; i++) { if (target.ParentIndices[i] == jointIndex) { ApplyScalingToJointIndexAndChildren(target, i, scaleFactor); } } } /// /// Gets the index of a known joint type in the target skeleton. /// /// The configuration to query. /// The known joint type to find. /// The index of the joint in the skeleton. public static int GetKnownJointIndex(MSDKUtilityEditorConfig config, KnownJointType jointType) { GetJointIndexByKnownJointType(config.ConfigHandle, SkeletonType.TargetSkeleton, jointType, out var index); return index; } /// /// Gets the T-pose transform for a known joint type in the target skeleton. /// /// The target configuration. /// The known joint type to find. /// The native transform of the joint in T-pose. public static NativeTransform GetTargetTPoseFromKnownJoint(MSDKUtilityEditorConfig target, KnownJointType knownJointType) { return target.ReferencePose[GetKnownJointIndex(target, knownJointType)]; } /// /// Calculates the desired scale factor between source and target skeletons. /// /// The source configuration. /// The target configuration. /// The type of T-pose to use for the source. /// Whether to use the current pose instead of the T-pose. /// The calculated scale factor. public static float GetDesiredScale(MSDKUtilityEditorConfig source, MSDKUtilityEditorConfig target, SkeletonTPoseType sourceTPoseType, bool useCurrentPose = false) { if (sourceTPoseType == SkeletonTPoseType.UnscaledTPose) { return 1.0f; } GetSkeletonTPose(source.ConfigHandle, SkeletonType.SourceSkeleton, sourceTPoseType, JointRelativeSpaceType.RootOriginRelativeSpace, out var sourceTPose); GetSkeletonTPose(target.ConfigHandle, SkeletonType.TargetSkeleton, SkeletonTPoseType.UnscaledTPose, JointRelativeSpaceType.RootOriginRelativeSpace, out var targetTPose); var rightHandJointIndex = GetKnownJointIndex(target, KnownJointType.RightWrist); var sourceRightHand = sourceTPose[(int)FullBodyTrackingBoneId.RightHandWrist]; var targetRightHand = useCurrentPose ? target.ReferencePose[rightHandJointIndex] : targetTPose[rightHandJointIndex]; // Calculate scale ratio using Y position (height-based scaling) var yScaleRatio = sourceRightHand.Position.y / (targetRightHand.Position.y > 0.0f ? targetRightHand.Position.y : 1.0f); // Clamp the scale ratio to prevent extreme scaling return Mathf.Clamp(yScaleRatio, MSDKUtilityEditorPreviewer.MinScale, MSDKUtilityEditorPreviewer.MaxScale); } /// /// Finds the best rotation for a parent transform to position a child transform at a target position. /// /// The parent transform to rotate. /// The child transform to position. /// The target position for the child. /// The maximum allowed rotation angle difference. /// The step size for rotation testing. /// The maximum rotation angle to test. /// The best rotation for the parent transform. public static Quaternion GetBestRotation(Transform parent, Transform child, Vector3 targetPosition, float rotationThreshold = 90f, float rotationStep = 0.1f, float maxRotation = 360f) { var initialRotation = parent.rotation; var childRotation = child.rotation; var minDistance = Mathf.Infinity; var bestRotation = initialRotation; foreach (var axis in new[] { parent.forward, parent.up, parent.right }) { for (var angle = -maxRotation; angle <= maxRotation; angle += rotationStep) { parent.rotation = initialRotation * Quaternion.AngleAxis(angle, axis); var distance = Vector3.Distance(child.position, targetPosition); if (distance < minDistance && Quaternion.Angle(childRotation, child.rotation) < rotationThreshold) { minDistance = distance; bestRotation = parent.rotation; } } } return bestRotation; } private static void ApplyWristFingerAlignment(MSDKUtilityEditorWindow win, FullBodyTrackingBoneId[] startBoneIds, KnownJointType wristType) { var source = win.SourceInfo; var target = win.TargetInfo; // Get the wrist joint from the target skeleton var wristJointIndex = GetKnownJointIndex(target, wristType); var wristTransform = target.SkeletonJoints[wristJointIndex]; if (wristTransform == null) { Debug.LogError($"Could not find wrist transform for {wristType}"); return; } // Calculate the average position of source metacarpal joints Vector3 sourceMetacarpalAverage = Vector3.zero; int validMetacarpalCount = 0; foreach (var startBoneId in startBoneIds) { if ((int)startBoneId >= 0 && (int)startBoneId < source.ReferencePose.Length) { sourceMetacarpalAverage += source.ReferencePose[(int)startBoneId].Position; validMetacarpalCount++; } } if (validMetacarpalCount > 0) { sourceMetacarpalAverage /= validMetacarpalCount; } else { Debug.LogError("No valid metacarpal joints found in source skeleton"); return; } // Store the initial rotation of the wrist Quaternion initialWristRotation = wristTransform.rotation; // Find the best rotation for the wrist to align child fingers with source metacarpals // We'll test rotations around different axes to find the best match float minDistance = float.MaxValue; Quaternion bestRotation = initialWristRotation; // Get all finger joints under the wrist List fingerJoints = new List(); for (int i = 0; i < target.ParentIndices.Length; i++) { if (IsChildOfJoint(target, i, wristJointIndex)) { fingerJoints.Add(target.SkeletonJoints[i]); } } // Test rotations around different axes foreach (var axis in new[] { wristTransform.forward, wristTransform.up, wristTransform.right }) { for (float angle = -180f; angle <= 180f; angle += 5f) // Use a larger step for initial search { wristTransform.rotation = initialWristRotation * Quaternion.AngleAxis(angle, axis); // Calculate the average position of all finger joints after rotation Vector3 targetFingerAverage = Vector3.zero; foreach (var joint in fingerJoints) { targetFingerAverage += joint.position; } targetFingerAverage /= fingerJoints.Count; // Calculate distance between averages float distance = Vector3.Distance(targetFingerAverage, sourceMetacarpalAverage); if (distance < minDistance) { minDistance = distance; bestRotation = wristTransform.rotation; } } } // Apply the best rotation wristTransform.rotation = bestRotation; // Update the T-pose data after rotating the wrist UpdateTPoseData(target); } private static Dictionary> GetFingerChains(Transform wristTransform) { Dictionary> fingerChains = new Dictionary>(); // Define finger names to search for string[] fingerNames = new[] { "Thumb", "Index", "Middle", "Ring", "Pinky", "Little" }; // Check each child of the wrist for potential finger roots foreach (Transform child in wristTransform) { // Try to determine which finger this might be string matchedFingerName = null; foreach (string fingerName in fingerNames) { if (child.name.IndexOf(fingerName, StringComparison.OrdinalIgnoreCase) >= 0) { matchedFingerName = fingerName; break; } } // If this child doesn't match any finger name, skip it if (matchedFingerName == null) { continue; } // Special case for "Little" which is an alternative name for "Pinky" if (matchedFingerName == "Little") { matchedFingerName = "Pinky"; } // Build the finger chain by following the hierarchy List chain = new List(); Transform current = child; // Add the starting joint chain.Add(current); // Follow the hierarchy to build the chain while (current.childCount > 0) { // Simply take the first child to continue the chain current = current.GetChild(0); chain.Add(current); // Limit chain length as a safety measure if (chain.Count >= 5) { break; } } // Only add the chain if it doesn't already exist or if this one is longer if (!fingerChains.ContainsKey(matchedFingerName) || chain.Count > fingerChains[matchedFingerName].Count) { fingerChains[matchedFingerName] = chain; } } return fingerChains; } private static void AlignFingerToTarget(List fingerChain, Vector3 targetPosition) { if (fingerChain.Count < 2) { return; } // Get the base and tip of the finger Transform baseJoint = fingerChain[0]; Transform tipJoint = fingerChain[^1]; // Calculate the current direction of the finger Vector3 currentDirection = (tipJoint.position - baseJoint.position).normalized; // Calculate the desired direction to the target Vector3 targetDirection = (targetPosition - baseJoint.position).normalized; // Create a rotation to align the current direction with the target direction Quaternion rotation = Quaternion.FromToRotation(currentDirection, targetDirection); // Check if the rotation angle is too large (more than 60 degrees) float rotationAngle = Quaternion.Angle(Quaternion.identity, rotation); if (rotationAngle > 60f) { Debug.LogWarning($"Excessive finger rotation detected: {rotationAngle} degrees. Ignoring rotation."); return; } // Store the initial position of the tip joint Vector3 initialTipPosition = tipJoint.position; // Apply the rotation to the base joint Quaternion originalRotation = baseJoint.rotation; baseJoint.rotation = rotation * baseJoint.rotation; // Check if the rotation actually improved the distance to the target float initialDistance = Vector3.Distance(initialTipPosition, targetPosition); float newDistance = Vector3.Distance(tipJoint.position, targetPosition); // If the new distance is greater than the initial distance, revert the rotation if (newDistance > initialDistance) { Debug.LogWarning( $"Finger rotation increased distance to target from {initialDistance} to {newDistance}. Reverting rotation."); baseJoint.rotation = originalRotation; } } private static bool IsChildOfJoint(MSDKUtilityEditorConfig config, int childIndex, int parentIndex) { if (childIndex == parentIndex) { return false; } int currentIndex = childIndex; while (currentIndex >= 0 && currentIndex < config.ParentIndices.Length) { int parent = config.ParentIndices[currentIndex]; if (parent == parentIndex) { return true; } if (parent == currentIndex || parent < 0) { break; } currentIndex = parent; } return false; } } }