// Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using Meta.XR.Movement.Retargeting; using Unity.Collections; using UnityEngine; using static Meta.XR.Movement.MSDKUtility; namespace Meta.XR.Movement { /// /// Utility helper functions for the native plugin. /// public static class MSDKUtilityHelper { public static readonly string[] AutoMapExcludedJointNames = { "LeftHandPalm", "LeftHandWristTwist", "RightHandPalm", "RightHandWristTwist" }; public static readonly string[] HalfBodyManifestationJointNames = { "Root", "Hips", "SpineLower", "SpineMiddle", "SpineUpper", "Chest", "Neck", "Head", "LeftShoulder", "LeftScapula", "LeftArmUpper", "LeftArmLower", "LeftHandWristTwist", "RightShoulder", "RightScapula", "RightArmUpper", "RightArmLower", "RightHandWristTwist", "LeftHandPalm", "LeftHandWrist", "LeftHandThumbMetacarpal", "LeftHandThumbProximal", "LeftHandThumbDistal", "LeftHandThumbTip", "LeftHandIndexMetacarpal", "LeftHandIndexProximal", "LeftHandIndexIntermediate", "LeftHandIndexDistal", "LeftHandIndexTip", "LeftHandMiddleMetacarpal", "LeftHandMiddleProximal", "LeftHandMiddleIntermediate", "LeftHandMiddleDistal", "LeftHandMiddleTip", "LeftHandRingMetacarpal", "LeftHandRingProximal", "LeftHandRingIntermediate", "LeftHandRingDistal", "LeftHandRingTip", "LeftHandLittlePinkyMetacarpal", "LeftHandLittlePinkyProximal", "LeftHandLittlePinkyIntermediate", "LeftHandLittlePinkyDistal", "LeftHandLittlePinkyTip", "RightHandPalm", "RightHandWrist", "RightHandThumbMetacarpal", "RightHandThumbProximal", "RightHandThumbDistal", "RightHandThumbTip", "RightHandIndexMetacarpal", "RightHandIndexProximal", "RightHandIndexIntermediate", "RightHandIndexDistal", "RightHandIndexTip", "RightHandMiddleMetacarpal", "RightHandMiddleProximal", "RightHandMiddleIntermediate", "RightHandMiddleDistal", "RightHandMiddleTip", "RightHandRingMetacarpal", "RightHandRingProximal", "RightHandRingIntermediate", "RightHandRingDistal", "RightHandRingTip", "RightHandLittlePinkyMetacarpal", "RightHandLittlePinkyProximal", "RightHandLittlePinkyIntermediate", "RightHandLittlePinkyDistal", "RightHandLittlePinkyTip" }; private static readonly string[] _sourceKnownJoints = { "Root", "Hips", "RightArmUpper", "LeftArmUpper", "RightHandWrist", "LeftHandWrist", "Chest", "Neck", "RightUpperLeg", "LeftUpperLeg", "RightFootAnkle", "LeftFootAnkle", }; /// /// Create the retargeting config string based on some data. /// /// The array of target blendshape names. /// The retargeting data of the source skeleton. /// The retargeting data of the target skeleton. /// The name of the config file. /// public static string CreateRetargetingConfig( string[] targetBlendshapeNames, SkeletonData sourceData, SkeletonData targetData, string configName) { // We need the following for the json: // 1. Source RetargetingBodyData // 2. Target RetargetingBodyData var sourceMinTPose = new NativeArray(sourceData.Joints.Length, Allocator.Temp, NativeArrayOptions.UninitializedMemory); var sourceMaxTPose = new NativeArray(sourceData.Joints.Length, Allocator.Temp, NativeArrayOptions.UninitializedMemory); for (var i = 0; i < sourceData.Joints.Length; i++) { sourceMinTPose[i] = sourceData.TPoseMin[i]; sourceMaxTPose[i] = sourceData.TPoseMax[i]; } var targetUnscaledTPose = new NativeArray(targetData.Joints.Length, Allocator.Temp, NativeArrayOptions.UninitializedMemory); var targetMinTPose = new NativeArray(targetData.Joints.Length, Allocator.Temp, NativeArrayOptions.UninitializedMemory); var targetMaxTPose = new NativeArray(targetData.Joints.Length, Allocator.Temp, NativeArrayOptions.UninitializedMemory); for (var i = 0; i < targetData.Joints.Length; i++) { targetUnscaledTPose[i] = targetData.TPose[i]; targetMinTPose[i] = targetData.TPoseMin[i]; targetMaxTPose[i] = targetData.TPoseMax[i]; } var minMappings = new NativeArray(0, Allocator.Temp, NativeArrayOptions.UninitializedMemory); var minMappingEntries = new NativeArray(0, Allocator.Temp, NativeArrayOptions.UninitializedMemory); var maxMappings = new NativeArray(0, Allocator.Temp, NativeArrayOptions.UninitializedMemory); var maxMappingEntries = new NativeArray(0, Allocator.Temp, NativeArrayOptions.UninitializedMemory); var targetKnownJoints = KnownJointFinder.FindKnownJoints(targetData.Joints, targetData.ParentJoints); // 3. Function calls here, then return the retargetingConfigJson variable var initParams = new ConfigInitParams(); initParams.SourceSkeleton.JointNames = sourceData.Joints; initParams.SourceSkeleton.ParentJointNames = sourceData.ParentJoints; initParams.SourceSkeleton.OptionalKnownSourceJointNamesById = _sourceKnownJoints; initParams.SourceSkeleton.OptionalAutoMapExcludedJointNames = AutoMapExcludedJointNames; initParams.SourceSkeleton.OptionalManifestationNames = new[] { MetaSourceDataProvider.HalfBodyManifestation }; initParams.SourceSkeleton.OptionalManifestationJointCounts = new[] { (int)SkeletonData.BodyTrackingBoneId.End }; initParams.SourceSkeleton.OptionalManifestationJointNames = HalfBodyManifestationJointNames; initParams.SourceSkeleton.MinTPose = sourceMinTPose; initParams.SourceSkeleton.MaxTPose = sourceMaxTPose; initParams.TargetSkeleton.BlendShapeNames = targetBlendshapeNames; initParams.TargetSkeleton.JointNames = targetData.Joints; initParams.TargetSkeleton.ParentJointNames = targetData.ParentJoints; initParams.TargetSkeleton.OptionalKnownSourceJointNamesById = targetKnownJoints; initParams.TargetSkeleton.MinTPose = targetMinTPose; initParams.TargetSkeleton.MaxTPose = targetMaxTPose; initParams.TargetSkeleton.UnscaledTPose = targetUnscaledTPose; initParams.MinMappings.Mappings = minMappings; initParams.MinMappings.MappingEntries = minMappingEntries; initParams.MaxMappings.Mappings = maxMappings; initParams.MaxMappings.MappingEntries = maxMappingEntries; if (!CreateOrUpdateUtilityConfig(configName, initParams, out var configHandle)) { Debug.LogError("Invalid config initialization params!\n\n" + initParams); return ""; } // 4. Try to auto align. AlignTargetToSource(configName, AlignmentFlags.All, configHandle, SkeletonType.SourceSkeleton, configHandle, out configHandle); // 5. Try to auto map. GenerateMappings(configHandle, AutoMappingFlags.EmptyFlag); WriteConfigDataToJson(configHandle, out var retargetingConfigJson); DestroyHandle(configHandle); return retargetingConfigJson; } /// /// Get the child to parent joint transform mappings for a target. /// /// The target game object to get mappings for. /// The root transform of the game object. /// The dictionary of child parent joint transform mappings. public static Dictionary GetChildParentJointMapping(Transform target, out Transform root) { // First, check if an animator exists so that we can get the hips. Transform hips = null; root = null; var animator = target.GetComponent(); if (animator != null && animator.avatar != null && animator.avatar.isHuman) { hips = animator.GetBoneTransform(HumanBodyBones.Hips); if (hips != null) { root = hips.parent; } } // If we couldn't find hips through the animator, use KnownJointFinder if (hips == null) { // Get all transforms in the hierarchy var allTransforms = new List { target }; allTransforms.AddRange(target.GetAllChildren()); // Filter out transforms with skinned mesh renderers as they're not joints var jointTransforms = allTransforms .Where(t => t.GetComponent() == null) .ToArray(); // Create joint names and parent joint names arrays var jointNames = jointTransforms.Select(t => t.name).ToArray(); var jointNameSet = new HashSet(jointNames); var parentJointNames = jointTransforms.Select(t => { if (t.parent == null) { return ""; } var parentName = t.parent.name; // Only return parent name if it exists in our joint list, otherwise return empty string return jointNameSet.Contains(parentName) ? parentName : ""; }).ToArray(); if (hips == null) { foreach (var jointTransform in jointTransforms) { // Check if position is greater than 0, 0, 0 if (!(jointTransform.position.x > 0) && !(jointTransform.position.y > 0) && !(jointTransform.position.z > 0)) { continue; } // Count children that are different in position or rotation var validChildrenCount = 0; for (var i = 0; i < jointTransform.childCount; i++) { var child = jointTransform.GetChild(i); // Check if child is different in position or rotation from parent if (child.localPosition != Vector3.zero || child.localRotation != Quaternion.identity) { validChildrenCount++; } } // If this transform has 3 or more valid children, consider it as hips if (validChildrenCount < 3) { continue; } hips = jointTransform; break; } } // Fallback - search by name. if (hips == null) { // Use KnownJointFinder to find known joints var knownJoints = KnownJointFinder.FindKnownJoints(jointNames, parentJointNames); // Find the hips joint from the known joints var hipsJointName = knownJoints.Length > (int)KnownJointType.Hips ? knownJoints[(int)KnownJointType.Hips] : ""; if (!string.IsNullOrEmpty(hipsJointName)) { hips = jointTransforms.FirstOrDefault(t => t.name == hipsJointName); } // Find the root joint from the known joints var rootJointName = knownJoints.Length > (int)KnownJointType.Root ? knownJoints[(int)KnownJointType.Root] : ""; if (!string.IsNullOrEmpty(rootJointName)) { root = jointTransforms.FirstOrDefault(t => t.name == rootJointName); } } // If we found hips but not root, assume root is parent of hips if (hips != null && root == null) { root = hips.parent; } } if (root == null) { Debug.LogError("Could not find root joint! Using the target transform as root"); root = target; } // Once we have the root, let's find all child transforms and create a dictionary mapping. var jointMapping = new Dictionary(); FillJointMapping(true, root, ref jointMapping); return jointMapping; } /// /// Gets the root joint in a transform's children. /// /// The handle. /// The root transform to search. /// The found root index. /// The root joint transform. public static void GetRootJoint(ulong handle, Transform root, out int index, out Transform rootJoint) { rootJoint = null; if (!GetJointIndexByKnownJointType(handle, SkeletonType.TargetSkeleton, KnownJointType.Root, out index)) { return; } GetJointNames(handle, SkeletonType.TargetSkeleton, out var jointNames); // Assume root joint is the parent of the hips joint. GetJointIndexByKnownJointType(handle, SkeletonType.TargetSkeleton, KnownJointType.Hips, out var hipsJointIndex); if (hipsJointIndex == -1) { return; } var hipsJoint = root.FindChildRecursive(jointNames[hipsJointIndex]); if (hipsJoint == null) { return; } rootJoint = hipsJoint.parent; if (rootJoint != null) { return; } var rootJointName = jointNames[index]; rootJoint = root.FindChildRecursive(rootJointName); } public static List GetAllChildren(this Transform parent) { var children = new List(); foreach (Transform child in parent) { children.Add(child); children.AddRange(child.GetAllChildren()); } return children; } public static Transform GetLowestChild(Transform parent) { if (parent.childCount == 0) { return parent; } Transform lowestChild = null; foreach (Transform child in parent) { Transform childLowestChild = GetLowestChild(child); if (childLowestChild != null && (lowestChild == null || childLowestChild.childCount < lowestChild.childCount)) { lowestChild = childLowestChild; } } return lowestChild; } public static Transform FindChildRecursiveExact(Transform parent, string name) { if (parent.name == name) { return parent; } for (var i = 0; i < parent.childCount; i++) { var child = parent.GetChild(i); if (child.name == name) { return child; } var result = FindChildRecursiveExact(child, name); if (result != null) { return result; } } return null; } public static Vector3 InverseTransformVector(NativeTransform t, Vector3 v) { return Quaternion.Inverse(t.Orientation) * v; } public static Vector3 TransformVector(NativeTransform t, Vector3 v) { return t.Orientation * v; } public static void ScaleSkeletonPose(UInt64 handle, SkeletonType skeletonType, ref NativeArray scaledTransforms, float scale) { for (var i = 0; i < scaledTransforms.Length; i++) { var jointTransform = scaledTransforms[i]; jointTransform.Position *= scale; scaledTransforms[i] = jointTransform; } } public static int GetIndexFromPropertyPath(string propertyPath) { var pathParts = propertyPath.Split('.'); foreach (var part in pathParts) { if (!part.Contains("[")) { continue; } int indexOfLeftBracket = part.IndexOf('['); int indexOfRightBracket = part.IndexOf(']'); return int.Parse(part.Substring( indexOfLeftBracket + 1, indexOfRightBracket - indexOfLeftBracket - 1)); } return -1; } private static void FillJointMapping( bool isRoot, Transform target, ref Dictionary jointMapping) { var targetChildCount = target.childCount; // We don't want to record skinned meshes if (target.GetComponent()) { return; } jointMapping.Add(target, isRoot ? null : target.parent); for (var i = 0; i < targetChildCount; i++) { var child = target.GetChild(i); FillJointMapping(false, child, ref jointMapping); } } } }