// Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. using Unity.Collections; using Unity.Jobs; using UnityEngine; using UnityEngine.Assertions; using UnityEngine.Jobs; using static Meta.XR.Movement.MSDKUtility; namespace Meta.XR.Movement.Retargeting { /// /// The CharacterRetargeter takes body tracking data from OVRBody and returns a retargeted output /// that can be applied to a target skeleton. /// public class CharacterRetargeter : CharacterRetargeterConfig { /// /// If the data is valid. /// public bool IsValid { get => _isValid; set => _isValid = value; } /// /// Retargeting handle. /// public ulong RetargetingHandle => _skeletonRetargeter?.NativeHandle ?? 0; /// /// Source processor containers. /// public SourceProcessorContainer[] SourceProcessorContainers => _sourceProcessorContainers; /// /// Source processor containers. /// public TargetProcessorContainer[] TargetProcessorContainers => _targetProcessorContainers; /// /// The skeleton retargeter. /// public SkeletonRetargeter SkeletonRetargeter { get => _skeletonRetargeter; set => _skeletonRetargeter = value; } /// /// The offset transform. /// public Transform DebugDrawTransform { get => _debugDrawTransform; set => _debugDrawTransform = value; } /// /// True if debug draw should be enabled. /// public bool DebugDrawSourceSkeleton { get => _debugDrawSourceSkeleton; set => _debugDrawSourceSkeleton = value; } /// /// True if debug draw should be enabled. /// public bool DebugDrawTargetSkeleton { get => _debugDrawTargetSkeleton; set => _debugDrawTargetSkeleton = value; } /// /// Set to true if the retargeter is valid or not. /// public bool RetargeterValid => _isValid && _skeletonRetargeter.IsInitialized; /// /// Whether to draw debug visualization for the source skeleton. /// [SerializeField] protected bool _debugDrawSourceSkeleton; /// /// The color to use when drawing the source skeleton debug visualization. /// [SerializeField] protected Color _debugDrawSourceSkeletonColor = Color.white; /// /// Whether to draw debug visualization for the target skeleton. /// [SerializeField] protected bool _debugDrawTargetSkeleton; /// /// The color to use when drawing the target skeleton debug visualization. /// [SerializeField] protected Color _debugDrawTargetSkeletonColor = Color.green; /// /// The color to use when drawing an invalid target skeleton debug visualization. /// [SerializeField] protected Color _debugDrawInvalidTargetSkeletonColor = Color.red; [SerializeField] protected Color _debugDrawInvalidSourceSkeletonColor = Color.magenta; /// /// The skeleton retargeter instance used for retargeting operations. /// [SerializeField] protected SkeletonRetargeter _skeletonRetargeter = new(); /// /// Array of source processor containers that modify the source skeleton data. /// [SerializeField] protected SourceProcessorContainer[] _sourceProcessorContainers; /// /// Array of target processor containers that modify the target skeleton data. /// [SerializeField] protected TargetProcessorContainer[] _targetProcessorContainers; // Skeleton data provider. private ISourceDataProvider _dataProvider; private string _currentManifestation; private bool _isValid; private bool _isCalibrated; // Jobs. protected Transform _debugDrawTransform; private JobHandle _convertPoseJobHandle; /// /// Initializes the character retargeter by finding required components and validating configuration. /// public virtual void Awake() { _dataProvider = gameObject.GetComponent(); _debugDrawTransform = transform; Assert.IsNotNull(_config, "Must have a reference to a config; none are defined."); Assert.IsNotNull(_dataProvider, "Must have a skeleton data provider; none are on this gameObject."); } /// /// Initializes the character retargeter and sets up retargeting with the provided configuration. /// public override void Start() { base.Start(); Setup(Config); } /// /// Updates the character retargeter each frame, processing source poses and applying retargeting. /// public virtual void Update() { var sourcePose = _dataProvider.GetSkeletonPose(); _currentManifestation = _dataProvider.GetManifestation(); _isValid = _dataProvider.IsPoseValid(); if (!_skeletonRetargeter.IsInitialized) { return; } if (!_isValid) { if (_debugDrawSourceSkeleton) { _skeletonRetargeter.DrawInvalidSourcePose(_debugDrawInvalidSourceSkeletonColor); } return; } // Perform retargeting. UpdateSkeletalTPose(); CalculatePose(sourcePose); } /// /// Performs late update processing for the character retargeter, applying the final pose to the character. /// public virtual void LateUpdate() { if (!_skeletonRetargeter.IsInitialized) { return; } if (!_skeletonRetargeter.AppliedPose) { if (_debugDrawTargetSkeleton) { _skeletonRetargeter.DrawInvalidTargetPose(_debugDrawInvalidTargetSkeletonColor); } return; } UpdatePose(); } /// /// Cleans up resources when the character retargeter is destroyed. /// public virtual void OnDestroy() { Dispose(); } /// /// Sets up the character retargeter with the specified configuration. /// /// The configuration string containing retargeting data. public void Setup(string config) { _skeletonRetargeter.Dispose(); _skeletonRetargeter.Setup(config); foreach (var sourceProcessorContainer in _sourceProcessorContainers) { sourceProcessorContainer?.GetCurrentProcessor()?.Initialize(this); } foreach (var targetProcessorContainer in _targetProcessorContainers) { targetProcessorContainer?.GetCurrentProcessor()?.Initialize(this); } if (_skeletonRetargeter.ApplyScale) { transform.localScale = Vector3.zero; } } /// /// Disposes of resources used by the character retargeter. /// public void Dispose() { _skeletonRetargeter?.Dispose(); _skeletonRetargeter = null; if (_joints.isCreated) { _joints.Dispose(); } } /// /// Calibrate the retargeter to be fixed to the last source T-Pose. /// public void Calibrate() { if (!_isValid) { return; } UpdateSkeletalTPose(true); _isCalibrated = true; } /// /// Calculates the retargeted pose from the source pose. /// /// The source pose to retarget. public void CalculatePose(NativeArray sourcePose) { // If calibrated, match the pose. if (_isCalibrated) { _skeletonRetargeter.Align(sourcePose); } // Run processors with source pose and retargeting target pose data. foreach (var processor in _sourceProcessorContainers) { processor.GetCurrentProcessor()?.ProcessSkeleton(sourcePose); } if (!_skeletonRetargeter.Update(sourcePose, _currentManifestation)) { return; } foreach (var targetProcessorContainer in _targetProcessorContainers) { targetProcessorContainer?.GetCurrentProcessor()?.UpdatePose(ref _skeletonRetargeter.RetargetedPose); } // Create job to convert world pose to local pose. var job = new SkeletonJobs.ConvertWorldToLocalPoseJob { RootJointIndex = _skeletonRetargeter.RootJointIndex, ParentIndices = _skeletonRetargeter.NativeTargetParentIndices, WorldPose = _skeletonRetargeter.RetargetedPose, LocalPose = _skeletonRetargeter.RetargetedPoseLocal }; _convertPoseJobHandle = job.Schedule(); if (!_skeletonRetargeter.IsInitialized) { return; } if (_debugDrawSourceSkeleton) { _skeletonRetargeter.DrawDebugSourcePose(_debugDrawTransform, _debugDrawSourceSkeletonColor); } } /// /// Updates the character's pose with the retargeted data. /// public void UpdatePose() { _convertPoseJobHandle.Complete(); // Run processors with current pose and retargeted target pose data if the character is non-zero. var currentPose = GetCurrentBodyPose(JointType.NoWorldSpace); foreach (var targetProcessorContainer in _targetProcessorContainers) { targetProcessorContainer?.GetCurrentProcessor()?.LateUpdatePose( ref currentPose, ref _skeletonRetargeter.RetargetedPoseLocal); } currentPose.Dispose(); ApplyPose(); if (_debugDrawTargetSkeleton) { if (_isValid) { _skeletonRetargeter.DrawDebugTargetPose(_debugDrawTransform, _debugDrawTargetSkeletonColor); } else { _skeletonRetargeter.DrawInvalidTargetPose(_debugDrawInvalidTargetSkeletonColor); } } } /// /// Updates the T-pose reference used for retargeting. /// /// The source pose to use as reference for the T-pose. public void UpdateTPose(NativeArray sourcePose) { UpdateSourceReferenceTPose(_skeletonRetargeter.NativeHandle, sourcePose); _skeletonRetargeter.UpdateSourceReferencePose(sourcePose, _currentManifestation); } /// /// Gets a processor of the specified type from the source processor containers. /// /// The type of processor to get. /// The processor of the specified type if found, otherwise null. public T GetSourceProcessor() where T : class { var containers = SourceProcessorContainers; foreach (var container in containers) { if (container?.GetCurrentProcessor() is T processor) { return processor; } } return null; } /// /// Gets a processor of the specified type from the target processor containers. /// /// The type of processor to get. /// The processor of the specified type if found, otherwise null. public T GetTargetProcessor() where T : class { var containers = TargetProcessorContainers; foreach (var container in containers) { if (container?.GetCurrentProcessor() is T processor) { return processor; } } return null; } private void UpdateSkeletalTPose(bool forceUpdate = false) { if (!forceUpdate && (_isCalibrated || !_dataProvider.IsNewTPoseAvailable())) { return; } var sourcePose = _dataProvider.GetSkeletonTPose(); UpdateTPose(sourcePose); } private void ApplyPose() { // Apply scale first. if (_skeletonRetargeter.ApplyScale && transform.localScale != _skeletonRetargeter.RootScale) { transform.localScale = _skeletonRetargeter.RootScale; _joints[_skeletonRetargeter.HeadJointIndex].localScale = _skeletonRetargeter.HeadScale; } // Hide the lower body by setting the scale of the leg joints to zero. if (_skeletonRetargeter.HideLowerBodyWhenUpperBodyTracking) { if (_dataProvider != null && _dataProvider.GetManifestation() == MetaSourceDataProvider.HalfBodyManifestation) { // Check only a single joint. if (_joints[_skeletonRetargeter.LeftUpperLegJointIndex].localScale != _skeletonRetargeter.HideLegScale) { _joints[_skeletonRetargeter.LeftUpperLegJointIndex].localScale = _skeletonRetargeter.HideLegScale; _joints[_skeletonRetargeter.RightUpperLegJointIndex].localScale = _skeletonRetargeter.HideLegScale; _joints[_skeletonRetargeter.LeftLowerLegJointIndex].localScale = Vector3.zero; _joints[_skeletonRetargeter.RightLowerLegJointIndex].localScale = Vector3.zero; } } else if (_joints[_skeletonRetargeter.LeftUpperLegJointIndex].localScale == _skeletonRetargeter.HideLegScale) { _joints[_skeletonRetargeter.LeftUpperLegJointIndex].localScale = Vector3.one; _joints[_skeletonRetargeter.RightUpperLegJointIndex].localScale = Vector3.one; _joints[_skeletonRetargeter.LeftLowerLegJointIndex].localScale = Vector3.one; _joints[_skeletonRetargeter.RightLowerLegJointIndex].localScale = Vector3.one; } } _skeletonRetargeter.ApplyPose(ref _joints); } } }