// Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.
using System;
using System.Collections.Generic;
using Unity.Collections;
using Unity.Jobs;
using UnityEngine;
using UnityEngine.Jobs;
using static Meta.XR.Movement.MSDKUtility;
using static Unity.Collections.Allocator;
using static Unity.Collections.NativeArrayOptions;
namespace Meta.XR.Movement.Retargeting
{
///
/// The retargeting handler that handles running of the retargeting job.
///
[Serializable]
public class SkeletonRetargeter
{
///
/// True if the pose was applied.
///
public bool AppliedPose { get; private set; }
///
/// True if scale should be applied.
///
public bool ApplyScale
{
get => _applyScale;
set => _applyScale = value;
}
///
/// The scale clamp range.
///
public Vector2 ScaleRange
{
get => _scaleRange;
set => _scaleRange = value;
}
///
/// The root scale.
///
public Vector3 RootScale =>
!_applyScale || !_isInitialized ? Vector3.one : RetargetedPose[_rootJointIndex].Scale;
///
/// The head scale.
///
public Vector3 HeadScale =>
!_applyScale || !_isInitialized ? Vector3.one : RetargetedPose[_headJointIndex].Scale;
///
/// The leg scale when hiding the lower body.
///
public Vector3 HideLegScale => _hideLegScale;
///
/// Hide the lower body if body tracking is set to UpperBody.
///
public bool HideLowerBodyWhenUpperBodyTracking => _hideLowerBodyWhenUpperBodyTracking;
///
/// True if this is initialized.
///
public bool IsInitialized => _isInitialized;
///
/// The retargeted pose in world space.
///
public NativeArray RetargetedPose;
///
/// The retargeted pose in local space.
///
public NativeArray RetargetedPoseLocal;
///
/// The source pose that's being retargeted.
///
public NativeArray SourcePose => _sourcePose;
///
/// The source reference pose.
///
public NativeArray SourceReferencePose => _sourceReferencePose;
///
/// The min T-Pose.
///
public NativeArray MinTPose => _minTPose;
///
/// The max T-Pose
///
public NativeArray MaxTPose => _maxTPose;
///
/// The native retargeting handle.
///
public ulong NativeHandle => _nativeHandle;
///
/// The source parent joint indices.
///
public NativeArray NativeSourceParentIndices => _nativeSourceParentIndices;
///
/// The target parent joint indices.
///
public NativeArray NativeTargetParentIndices => _nativeTargetParentIndices;
///
/// The number of joints in the source skeleton.
///
public int SourceJointCount => _sourceJointCount;
///
/// The number of joints in the target skeleton.
///
public int TargetJointCount => _targetJointCount;
///
/// The root joint index.
///
public int RootJointIndex => _rootJointIndex;
///
/// The head joint index.
///
public int HeadJointIndex => _headJointIndex;
///
/// The left leg joint index.
///
public int LeftUpperLegJointIndex => _leftUpperLegJointIndex;
///
/// The right leg joint index.
///
public int RightUpperLegJointIndex => _rightUpperLegJointIndex;
///
/// The left lower leg index.
///
public int LeftLowerLegJointIndex => _leftLowerLegJointIndex;
///
/// The right lower leg index.
///
public int RightLowerLegJointIndex => _rightLowerLegJointIndex;
///
/// The behavior type for retargeting operations.
///
[SerializeField]
private RetargetingBehavior _retargetingBehavior = RetargetingBehavior.RotationsAndPositions;
///
/// Hide the lower body if body tracking is set to UpperBody.
///
[SerializeField]
private bool _hideLowerBodyWhenUpperBodyTracking;
///
/// The scale for the legs if using
///
[SerializeField]
private Vector3 _hideLegScale = Vector3.zero;
///
/// Whether to apply scaling to the retargeted skeleton.
///
[SerializeField]
private bool _applyScale = true;
///
/// The current scale factor applied to the retargeted skeleton.
///
[SerializeField, ReadOnly]
private float _currentScale = 1.0f;
///
/// Scale factor applied to the head joint to prevent it from scaling as much as the rest of the body.
///
[SerializeField]
private float _headScaleFactor = 0.95f;
///
/// Minimum and maximum scale range for the retargeted skeleton.
///
[SerializeField]
private Vector2 _scaleRange = new(0.8f, 1.2f);
private SkeletonDraw _sourceSkeletonDraw = new()
{
IndexesToIgnore = new List
{
(int)SkeletonData.FullBodyTrackingBoneId.LeftHandWristTwist,
(int)SkeletonData.FullBodyTrackingBoneId.RightHandWristTwist
}
};
private SkeletonDraw _targetSkeletonDraw = new();
private ulong _nativeHandle = INVALID_HANDLE;
private bool _isInitialized;
private int _sourceJointCount;
private int _targetJointCount;
private int _rootJointIndex;
private int _headJointIndex;
private int _leftUpperLegJointIndex;
private int _rightUpperLegJointIndex;
private int _leftLowerLegJointIndex;
private int _rightLowerLegJointIndex;
private string[] _manifestations;
private int[] _sourceParentIndices;
private int[] _targetParentIndices;
private NativeArray _sourceReferencePose;
private NativeArray _sourcePose;
private NativeArray _minTPose;
private NativeArray _maxTPose;
private NativeArray _nativeSourceParentIndices;
private NativeArray _nativeTargetParentIndices;
private NativeArray _nativeFingerIndices;
private NativeArray _targetReferencePose;
private JobHandle _applyPoseJobHandle;
///
/// Destructor.
///
~SkeletonRetargeter()
{
Dispose();
}
///
/// Setup the retargeting info arrays.
///
public void Setup(string config)
{
// Create native handle.
if (!CreateOrUpdateHandle(config, out _nativeHandle))
{
throw new Exception("Failed to create a retargeting handle!.");
}
// Query skeleton info.
GetSkeletonJointCount(_nativeHandle, SkeletonType.SourceSkeleton, out _sourceJointCount);
GetSkeletonJointCount(_nativeHandle, SkeletonType.TargetSkeleton, out _targetJointCount);
GetJointIndexByKnownJointType(_nativeHandle, SkeletonType.TargetSkeleton, KnownJointType.Root,
out _rootJointIndex);
GetJointIndexByKnownJointType(_nativeHandle, SkeletonType.TargetSkeleton, KnownJointType.Neck,
out _headJointIndex);
GetJointIndexByKnownJointType(_nativeHandle, SkeletonType.TargetSkeleton, KnownJointType.LeftUpperLeg,
out _leftUpperLegJointIndex);
GetJointIndexByKnownJointType(_nativeHandle, SkeletonType.TargetSkeleton, KnownJointType.RightUpperLeg,
out _rightUpperLegJointIndex);
GetChildJointIndexes(_nativeHandle, SkeletonType.TargetSkeleton, _leftUpperLegJointIndex,
out var leftLowerLegIndices);
GetChildJointIndexes(_nativeHandle, SkeletonType.TargetSkeleton, _rightUpperLegJointIndex,
out var rightLowerLegIndices);
GetJointIndexByKnownJointType(_nativeHandle, SkeletonType.TargetSkeleton, KnownJointType.LeftWrist,
out var leftWristIndex);
GetJointIndexByKnownJointType(_nativeHandle, SkeletonType.TargetSkeleton, KnownJointType.RightWrist,
out var rightWristIndex);
_leftLowerLegJointIndex = leftLowerLegIndices[0];
_rightLowerLegJointIndex = rightLowerLegIndices[0];
// Query parent indices.
_nativeSourceParentIndices = new NativeArray(_sourceJointCount, Persistent, UninitializedMemory);
_nativeTargetParentIndices = new NativeArray(_targetJointCount, Persistent, UninitializedMemory);
GetParentJointIndexesByRef(_nativeHandle, SkeletonType.SourceSkeleton, ref _nativeSourceParentIndices);
GetParentJointIndexesByRef(_nativeHandle, SkeletonType.TargetSkeleton, ref _nativeTargetParentIndices);
_sourceParentIndices = _nativeSourceParentIndices.ToArray();
_targetParentIndices = _nativeTargetParentIndices.ToArray();
// Fill finger indices array with joints that have wrist parents
var fingerIndicesList = new List();
for (var i = 0; i < _targetJointCount; i++)
{
// Check if this joint or any of its parents are wrist joints
var currentJoint = i;
while (currentJoint != -1)
{
if (currentJoint == leftWristIndex || currentJoint == rightWristIndex)
{
fingerIndicesList.Add(i);
break;
}
currentJoint = _targetParentIndices[currentJoint];
}
}
_nativeFingerIndices = new NativeArray(fingerIndicesList.ToArray(), Persistent);
// Setup empty retargeting pose arrays.
_sourcePose = new NativeArray(_sourceJointCount, Persistent, UninitializedMemory);
_sourceReferencePose = new NativeArray(_sourceJointCount, Persistent, UninitializedMemory);
_minTPose = new NativeArray(_sourceJointCount, Persistent, UninitializedMemory);
_maxTPose = new NativeArray(_sourceJointCount, Persistent, UninitializedMemory);
// Retargeting pose arrays.
_targetReferencePose = new NativeArray(_targetJointCount, Persistent, UninitializedMemory);
RetargetedPose = new NativeArray(_targetJointCount, Persistent, UninitializedMemory);
RetargetedPoseLocal = new NativeArray(_targetJointCount, Persistent, UninitializedMemory);
// Setup T-Pose.
GetSkeletonTPoseByRef(_nativeHandle, SkeletonType.SourceSkeleton, SkeletonTPoseType.MinTPose,
JointRelativeSpaceType.RootOriginRelativeSpace, ref _minTPose);
GetSkeletonTPoseByRef(_nativeHandle, SkeletonType.SourceSkeleton, SkeletonTPoseType.MaxTPose,
JointRelativeSpaceType.RootOriginRelativeSpace, ref _maxTPose);
UpdateSourceReferenceTPose(_nativeHandle, _maxTPose);
_isInitialized = true;
}
///
/// Dispose the retargeting info arrays.
///
public void Dispose()
{
// Dispose of native arrays.
if (!_isInitialized)
{
return;
}
_isInitialized = false;
_sourceSkeletonDraw = null;
_targetSkeletonDraw = null;
_sourcePose.Dispose();
_sourceReferencePose.Dispose();
_minTPose.Dispose();
_maxTPose.Dispose();
_nativeSourceParentIndices.Dispose();
_nativeTargetParentIndices.Dispose();
_nativeFingerIndices.Dispose();
_targetReferencePose.Dispose();
RetargetedPose.Dispose();
RetargetedPoseLocal.Dispose();
// Destroy native handle.
if (!DestroyHandle(_nativeHandle))
{
Debug.LogError($"Failed to destroy retargeting handle {_nativeHandle}.");
}
else
{
_nativeHandle = INVALID_HANDLE;
}
}
///
/// Aligns the source input pose with the source reference pose.
///
/// The source pose.
public void Align(NativeArray sourcePose)
{
MatchPose(_nativeHandle, SkeletonType.SourceSkeleton, MatchPoseBehavior.MatchScale,
JointRelativeSpaceType.RootOriginRelativeSpace, ref _sourceReferencePose, ref sourcePose);
}
///
/// Run the retargeter and update the retargeted pose.
///
/// The source pose.
/// The manifestation.
public bool Update(NativeArray sourcePose, string manifestation)
{
// Get retargeting settings.
var retargetingBehaviorInfo = RetargetingBehaviorInfo.DefaultRetargetingSettings();
retargetingBehaviorInfo.RetargetingBehavior = _retargetingBehavior;
// If manifestation caused a change in the pose size, resize source pose.
if (_sourcePose.Length != sourcePose.Length)
{
_sourcePose.Dispose();
_sourcePose = new NativeArray(sourcePose, Persistent);
}
_sourcePose.CopyFrom(sourcePose);
// Run retargeting.
AppliedPose = false;
if (!RetargetFromSourceFrameData(
_nativeHandle,
retargetingBehaviorInfo,
_sourcePose,
ref RetargetedPose,
manifestation))
{
Debug.LogError("Failed to retarget source frame data!");
return false;
}
AppliedPose = true;
// Clamp and update scales.
var rootPose = RetargetedPose[RootJointIndex];
var headPose = RetargetedPose[HeadJointIndex];
var scaleVal = Mathf.Max(float.Epsilon,
Mathf.Clamp(_currentScale, _scaleRange.x, _scaleRange.y));
rootPose.Scale = Vector3.one * scaleVal;
headPose.Scale = Vector3.one * (1 + (1 - scaleVal) * (1 - _headScaleFactor));
RetargetedPose[RootJointIndex] = rootPose;
RetargetedPose[HeadJointIndex] = headPose;
return true;
}
public void ApplyPose(ref TransformAccessArray joints)
{
// Create job to apply the pose.
var job = new SkeletonJobs.ApplyPoseJob
{
BodyPose = RetargetedPoseLocal,
RotationOnlyIndices = _nativeFingerIndices,
CurrentRotationIndex =
_retargetingBehavior == RetargetingBehavior.RotationsAndPositionsHandsRotationOnly ? 0 : -1
};
_applyPoseJobHandle = job.Schedule(joints);
_applyPoseJobHandle.Complete();
}
///
/// Update the source reference pose of the retargeter and calculate the scale.
///
/// The source pose.
/// The manifestation.
public void UpdateSourceReferencePose(NativeArray sourcePose, string manifestation)
{
// If manifestation caused a change in the pose size, resize source reference pose.
if (_sourceReferencePose.Length != sourcePose.Length)
{
_sourceReferencePose.Dispose();
_sourceReferencePose = new NativeArray(sourcePose, Persistent);
}
// Save the source reference pose.
_sourceReferencePose.CopyFrom(sourcePose);
// Retarget from source t-pose to get scaling.
RetargetFromSourceFrameData(
NativeHandle,
RetargetingBehaviorInfo.DefaultRetargetingSettings(),
sourcePose,
ref _targetReferencePose,
manifestation);
ConvertJointPose(NativeHandle,
SkeletonType.TargetSkeleton,
JointRelativeSpaceType.RootOriginRelativeSpace,
JointRelativeSpaceType.LocalSpaceScaled,
_targetReferencePose,
out var jointPoseWithScale);
// Scale is uniform.
_currentScale = jointPoseWithScale[0].Scale.x;
}
///
/// Debug draw the source skeleton pose.
///
public void DrawDebugSourcePose(Transform offset, Color color)
{
_sourceSkeletonDraw ??= new SkeletonDraw
{
IndexesToIgnore = new List
{
(int)SkeletonData.FullBodyTrackingBoneId.LeftHandWristTwist,
(int)SkeletonData.FullBodyTrackingBoneId.RightHandWristTwist
}
};
if (_sourceSkeletonDraw.LineThickness <= float.Epsilon || _sourceSkeletonDraw.TintColor != color)
{
_sourceSkeletonDraw.InitDraw(color, 0.005f);
}
ApplyOffset(ref _sourcePose, offset, true);
_sourceSkeletonDraw.LoadDraw(_sourcePose.Length, _sourceParentIndices, _sourcePose);
_sourceSkeletonDraw.Draw();
}
///
/// Draws last valid debug source pose.
///
///
public void DrawInvalidSourcePose(Color color)
{
_sourceSkeletonDraw ??= new SkeletonDraw();
if (_sourceSkeletonDraw.LineThickness <= float.Epsilon || _sourceSkeletonDraw.TintColor != color)
{
_sourceSkeletonDraw.InitDraw(color, 0.005f);
}
_sourceSkeletonDraw.Draw();
}
///
/// Draws debug target pose.
///
/// Offset applied when drawing based on local transforms.
/// Color to use.
/// If rendering world transforms.
public void DrawDebugTargetPose(Transform localOffset, Color color, bool useWorldPose = false)
{
_targetSkeletonDraw ??= new SkeletonDraw();
if (_targetSkeletonDraw.LineThickness <= float.Epsilon || _targetSkeletonDraw.TintColor != color)
{
_targetSkeletonDraw.InitDraw(color, 0.005f);
}
NativeArray targetWorldPose;
if (useWorldPose)
{
targetWorldPose = new NativeArray(RetargetedPose.Length, Temp);
targetWorldPose.CopyFrom(RetargetedPose);
}
else
{
targetWorldPose = GetWorldPoseFromLocalPose(RetargetedPoseLocal);
}
ApplyOffset(ref targetWorldPose, localOffset, false);
_targetSkeletonDraw.LoadDraw(targetWorldPose.Length, _targetParentIndices, targetWorldPose);
_targetSkeletonDraw.Draw();
targetWorldPose.Dispose();
}
///
/// Draws last valid debug target pose.
///
///
public void DrawInvalidTargetPose(Color color)
{
_targetSkeletonDraw ??= new SkeletonDraw();
if (_targetSkeletonDraw.LineThickness <= float.Epsilon || _targetSkeletonDraw.TintColor != color)
{
_targetSkeletonDraw.InitDraw(color, 0.005f);
}
_targetSkeletonDraw.Draw();
}
///
/// Converts a local space pose to a world space pose.
///
/// The local space pose to convert.
/// Root position, optional.
/// Root rotation, optional.
/// A new NativeArray containing the converted world space pose.
public NativeArray GetWorldPoseFromLocalPose(
NativeArray localPose,
Vector3? rootPosition = null,
Quaternion? rootRotation = null)
{
var worldPose = new NativeArray(localPose.Length, TempJob);
var job = new SkeletonJobs.ConvertLocalToWorldPoseJob
{
LocalPose = localPose,
WorldPose = worldPose,
ParentIndices = _nativeTargetParentIndices,
RootScale = Vector3.one,
RootPosition = rootPosition ?? Vector3.zero,
RootRotation = rootRotation ?? Quaternion.identity
};
job.Schedule().Complete();
return worldPose;
}
private void ApplyOffset(ref NativeArray pose, Transform offset, bool uniformScale)
{
if (offset == null)
{
return;
}
var scale = offset.lossyScale;
if (uniformScale)
{
scale.x = scale.x >= 0.0f ? 1.0f : -1.0f;
scale.y = scale.y >= 0.0f ? 1.0f : -1.0f;
scale.z = scale.z >= 0.0f ? 1.0f : -1.0f;
}
for (var i = 0; i < pose.Length; i++)
{
var currentPose = pose[i];
var offsetPosition = offset.position +
offset.rotation * Vector3.Scale(scale, currentPose.Position);
var offsetRotation = offset.rotation * currentPose.Orientation;
pose[i] = new NativeTransform(offsetRotation, offsetPosition);
}
}
}
}