// Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. using Unity.Burst; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; using Unity.Jobs; using UnityEngine; using static Meta.XR.Movement.MSDKUtility; using static Meta.XR.Movement.Retargeting.SkeletonData; using static Unity.Collections.Allocator; using static Unity.Collections.NativeArrayOptions; namespace Meta.XR.Movement.Retargeting { /// /// Provides convenient functions that obtain data from the tracker. /// public static class SkeletonUtilities { /// /// Updates an array of source poses from arrays of /// positions and rotations. An offset can be provided to /// transform the entire set of poses. /// [BurstCompile] private struct UpdateSourcePoseJob : IJob { /// /// Offset to apply to the source poses. /// [ReadOnly] public Pose Offset; /// /// Input pose rotations to read from. /// [ReadOnly] public NativeArray InputPoseRotations; /// /// Input pose translations to read from. /// [ReadOnly] public NativeArray InputPoseTranslations; /// /// Output poses to write to. /// public NativeArray OutputPose; /// /// Execute the job. Updates an array of source poses from input /// position and rotations arrays. /// [BurstCompile] public void Execute() { var numberOfPoses = OutputPose.Length; for (var i = 0; i < numberOfPoses; i++) { var rotation = Offset.rotation * InputPoseRotations[i].FromFlippedZQuatf(); var position = Offset.rotation * InputPoseTranslations[i].FromFlippedZVector3f() + Offset.position; var pose = OutputPose[i]; pose.Position = position; pose.Orientation = rotation; pose.Scale = Vector3.one; OutputPose[i] = pose; } } } /// /// Computes local poses from the world poses provided. /// [BurstCompile] private struct ComputeLocalFromWorldPosesJob : IJob { /// /// World poses to read from. /// [ReadOnly] public NativeArray WorldPoses; /// /// Parent indices to read from. Maps a joint index to /// its parent index. This is necessary because a joint's /// local pose must be computed relative to its parent's. /// [ReadOnly] public NativeArray ParentIndices; /// /// Local poses to write to. /// public NativeArray LocalPoses; /// /// Computes local poses of each joint but compute the /// pose relative to its parent's pose. /// [BurstCompile] public void Execute() { var numberOfPoses = LocalPoses.Length; for (var i = 0; i < numberOfPoses; i++) { // do not transform if: // a) it's the hips/root // b) the number of parent indices provided by the config // is fewer than the number of poses provided // c) the parent index is out of bounds if (i == 0 || i >= ParentIndices.Length || ParentIndices[i] >= WorldPoses.Length) { LocalPoses[i] = WorldPoses[i]; continue; } var parentIndex = ParentIndices[i]; var parentPose = WorldPoses[parentIndex]; var currentPose = WorldPoses[i]; Vector3 fromPosition = parentPose.Position, toPosition = currentPose.Position; Quaternion fromRotation = parentPose.Orientation, toRotation = currentPose.Orientation; Pose localPose = ComputeDeltaPose(fromPosition, fromRotation, toPosition, toRotation); LocalPoses[i] = new NativeTransform( localPose.rotation, localPose.position, Vector3.one); } } private static Pose ComputeDeltaPose( Vector3 fromPosition, Quaternion fromRotation, Vector3 toPosition, Quaternion toRotation) { var result = new Pose(); var inverseFromRot = Quaternion.Inverse(fromRotation); result.position = inverseFromRot * (toPosition - fromPosition); result.rotation = inverseFromRot * toRotation; return result; } } private static OVRCameraRig _ovrCameraRig; private static OVRPlugin.Skeleton2 _skeleton; private static OVRSkeleton.SkeletonPoseData _data; private static NativeArray _outputPoses; private static float _timestamp; /// /// Get current body tracking frame data. /// /// Data provider. /// Gets tracker positions in world space. /// Latest available . public static FrameData GetCurrentFrameData( OVRSkeleton.IOVRSkeletonDataProvider dataProvider, bool trackerPositionsWorldSpace) { var skeletonPoseData = dataProvider.GetSkeletonPoseData(); // provider doesn't give us fidelity, so we have to fetch it ourselves OVRPlugin.BodyState bodyState = new OVRPlugin.BodyState(); OVRPlugin.BodyTrackingFidelity2 fidelity2 = OVRPlugin.BodyTrackingFidelity2.Low; if (!OVRPlugin.GetBodyState4(OVRPlugin.Step.Render, dataProvider.GetSkeletonType() == OVRSkeleton.SkeletonType.FullBody ? OVRPlugin.BodyJointSet.FullBody : OVRPlugin.BodyJointSet.UpperBody, ref bodyState)) { fidelity2 = bodyState.Fidelity; } ProvideInputTrackingState(out var leftSideHand, out var leftSideInputPos, out var leftSideInputRot, true); ProvideInputTrackingState(out var rightSideHand, out var rightSideInputPos, out var rightSideInputRot, false); Vector3 centerEyeTrackingPos = Vector3.zero; Quaternion centerEyeTrackingRot = Quaternion.identity; Transform centerEye = GetCenterEyeTransform(); if (centerEye == null) { Debug.Log("Center eye transform was not found.."); } else { centerEyeTrackingPos = centerEye.localPosition; centerEyeTrackingRot = centerEye.localRotation; } if (trackerPositionsWorldSpace) { var trackerSpaceTransform = GetTrackingSpaceTransform(); if (trackerSpaceTransform == null) { Debug.LogError("Cannot get tracker positions in world space because " + "the tracking space transform is null."); } else { (leftSideInputPos, leftSideInputRot) = TransformPositionAndRotationToWorld( trackerSpaceTransform, leftSideInputPos, leftSideInputRot); (rightSideInputPos, rightSideInputRot) = TransformPositionAndRotationToWorld( trackerSpaceTransform, rightSideInputPos, rightSideInputRot); (centerEyeTrackingPos, centerEyeTrackingRot) = TransformPositionAndRotationToWorld( trackerSpaceTransform, centerEyeTrackingPos, centerEyeTrackingRot); } } var frameData = new FrameData( (byte)fidelity2, bodyState.Time, skeletonPoseData.IsDataValid, bodyState.Confidence, (byte)bodyState.JointSet, (byte)bodyState.CalibrationStatus, bodyState.SkeletonChangedCount, leftSideHand, rightSideHand, new NativeTransform(leftSideInputRot, leftSideInputPos), new NativeTransform(rightSideInputRot, rightSideInputPos), new NativeTransform(centerEyeTrackingRot, centerEyeTrackingPos) ); return frameData; } /// /// Gets poses from the tracker. Checks against expected pose count. /// /// Data provider. /// /// /// /// public static NativeArray GetPosesFromTheTracker( OVRSkeleton.IOVRSkeletonDataProvider dataProvider, Pose offset, out int skeletonChangeCount, out bool validPoses) { if (Mathf.Abs(Time.time - _timestamp) <= float.Epsilon) { skeletonChangeCount = _data.SkeletonChangedCount; validPoses = _data.IsDataValid; return _outputPoses; } var allPoses = GetPosesFromTheTracker(dataProvider, offset); _timestamp = Time.time; _outputPoses = new NativeArray(allPoses.Length, Temp); _outputPoses.CopyFrom(allPoses); validPoses = _data.IsDataValid; skeletonChangeCount = _data.SkeletonChangedCount; allPoses.Dispose(); return _outputPoses; } /// /// Returns transforms from the tracker. /// /// Data provider to provide transforms. /// Offset to apply. /// Tracker poses. public static NativeArray GetPosesFromTheTracker( OVRSkeleton.IOVRSkeletonDataProvider dataProvider, Pose offset) { // Get body tracking data _data = dataProvider.GetSkeletonPoseData(); var isFullBody = dataProvider.GetSkeletonType() == OVRSkeleton.SkeletonType.FullBody; // If data isn't valid, just provide default poses if (!_data.IsDataValid) { var numBonesSpec = isFullBody ? (int)FullBodyTrackingBoneId.End : (int)BodyTrackingBoneId.End; var invalidPoses = new NativeArray(numBonesSpec, TempJob, UninitializedMemory); for (var i = 0; i < numBonesSpec; i++) { invalidPoses[i] = NativeTransform.Identity(); } return invalidPoses; } // Convert to native arrays var boneTranslations = GetBoneTranslations(_data.BoneTranslations); var boneRotations = GetBoneRotations(_data.BoneRotations); var sourcePoses = new NativeArray(_data.BoneTranslations.Length, TempJob); var job = new UpdateSourcePoseJob { Offset = offset, InputPoseRotations = boneRotations, InputPoseTranslations = boneTranslations, OutputPose = sourcePoses }; job.Schedule().Complete(); boneTranslations.Dispose(); boneRotations.Dispose(); return sourcePoses; } /// /// Returns the bind pose. /// /// Data provider. /// public static NativeArray GetBindPoses( OVRSkeleton.IOVRSkeletonDataProvider dataProvider) { var sourcePoses = new NativeArray(0, Temp, UninitializedMemory); var skeletonType = dataProvider.GetSkeletonType() == OVRSkeleton.SkeletonType.FullBody ? OVRPlugin.SkeletonType.FullBody : OVRPlugin.SkeletonType.Body; if (!OVRPlugin.GetSkeleton2(skeletonType, ref _skeleton)) { return sourcePoses; } var numBones = _skeleton.Bones.Length; sourcePoses = new NativeArray(numBones, Temp, UninitializedMemory); for (var i = 0; i < numBones; i++) { var skeletonPose = _skeleton.Bones[i].Pose; sourcePoses[i] = new NativeTransform( skeletonPose.Orientation.FromFlippedZQuatf(), skeletonPose.Position.FromFlippedZVector3f(), Vector3.one); } return sourcePoses; } /// /// Computes and returns transforms relative to their parents. /// /// Parent indices. /// Tracker positions in world space. /// Poses relative to their parents. public static NativeArray GetTransformsRelativeToParents( int[] parentIndices, NativeArray trackerPosesWorldSpace) { // don't modify the original poses; we need to have an untouched array of world positions // that we compute local poses from var localPoses = new NativeArray(trackerPosesWorldSpace.Length, TempJob, UninitializedMemory); var parentIndicesNativeArray = new NativeArray(parentIndices.Length, TempJob, UninitializedMemory); for (int i = 0; i < parentIndices.Length; i++) { parentIndicesNativeArray[i] = parentIndices[i]; } var localPosesJob = new ComputeLocalFromWorldPosesJob { WorldPoses = trackerPosesWorldSpace, ParentIndices = parentIndicesNativeArray, LocalPoses = localPoses }; localPosesJob.Schedule().Complete(); parentIndicesNativeArray.Dispose(); return localPoses; } /// /// Computes and returns the child transform affected by the parent. /// /// Parent pose. /// Child pose. /// Native transform representing the child transform. public static NativeTransform GetChildTransformAffectedByParent(Pose parentPose, Pose childPose) { Matrix4x4 parentMatrix = Matrix4x4.TRS(parentPose.position, parentPose.rotation, Vector3.one); Vector3 finalPosition = parentMatrix.MultiplyPoint3x4(childPose.position); Quaternion finalRotation = parentMatrix.rotation * childPose.rotation; return new NativeTransform( finalRotation, finalPosition); } /// /// Converts local poses to absolute space. Root is the pivot. /// /// Array of parent indices for each joint in the skeleton. /// Array of poses in local space to be converted to absolute space. They must be ordered /// from parents down to children, because the function updates transforms in that order. /// Optional root pose to apply to the root to anchor the character /// around a different point. public static void ConvertLocalPosesToAbsolute( int[] parentIndices, NativeArray posesToModify, Pose? anchorPose = null) { if (anchorPose.HasValue) { var finalPose = new Pose(); var childPose = new Pose(posesToModify[0].Position, posesToModify[0].Orientation); MultiplyPoses(anchorPose.Value, childPose, ref finalPose); posesToModify[0] = GetChildTransformAffectedByParent( anchorPose.Value, new Pose(posesToModify[0].Position, posesToModify[0].Orientation)); } // all joints are relative to parent, except for root. // This cannot be done as a job, because it computes the world poses for the parents // at the lower indices first, and then proceeds with the children. for (int i = 0; i < posesToModify.Length; i++) { // first joint is root, just skip. if (i == 0) { continue; } if (i >= parentIndices.Length) { continue; } // parent index var parentIndex = parentIndices[i]; if (parentIndex < posesToModify.Length - 1) { var parentTransform = posesToModify[parentIndex]; var currentTransform = posesToModify[i]; posesToModify[i] = GetChildTransformAffectedByParent( new Pose(parentTransform.Position, parentTransform.Orientation), new Pose(currentTransform.Position, currentTransform.Orientation)); } } } /// /// Get input state of hand or controller, depending /// on which one is active. /// /// If hand is active or not. /// Input tracking position. /// Input tracking rotation. /// If left side or not. public static void ProvideInputTrackingState( out bool isHandTracked, out Vector3 inputTrackingPosition, out Quaternion inputTrackingRotation, bool isLeftSide) { OVRPlugin.HandState handState = new OVRPlugin.HandState(); if (OVRPlugin.GetHandState(OVRPlugin.Step.Render, isLeftSide ? OVRPlugin.Hand.HandLeft : OVRPlugin.Hand.HandRight, ref handState)) { isHandTracked = true; inputTrackingPosition = handState.PointerPose.Position.FromFlippedZVector3f(); inputTrackingRotation = handState.PointerPose.Orientation.FromFlippedZQuatf(); } else { isHandTracked = false; inputTrackingPosition = OVRInput.GetLocalControllerPosition(isLeftSide ? OVRInput.Controller.LTouch : OVRInput.Controller.RTouch); inputTrackingRotation = OVRInput.GetLocalControllerRotation(isLeftSide ? OVRInput.Controller.LTouch : OVRInput.Controller.RTouch); } } /// /// Returns center eye transform. /// /// Center eye. public static Transform GetCenterEyeTransform() { if (_ovrCameraRig == null) { _ovrCameraRig = Object.FindAnyObjectByType(); } if (_ovrCameraRig == null) { Debug.LogError("OVRCameraRig is null; one needs to be present in the scene for the center eye."); return null; } return _ovrCameraRig.centerEyeAnchor; } /// /// Returns tracking space transform. /// /// Tracking space transform. public static Transform GetTrackingSpaceTransform() { if (_ovrCameraRig == null) { _ovrCameraRig = Object.FindAnyObjectByType(); } if (_ovrCameraRig == null) { Debug.LogError("OVRCameraRig is null; one needs to be present in the scene for the anchor."); return null; } return _ovrCameraRig.trackingSpace; } private static (Vector3, Quaternion) TransformPositionAndRotationToWorld( Transform worldTransform, Vector3 position, Quaternion rotation) { position = worldTransform.TransformPoint(position); rotation = worldTransform.rotation * rotation; return (position, rotation); } /// /// Multiplies two poses. /// /// First pose. /// Second pose. /// Resulting pose. private static void MultiplyPoses(in Pose a, in Pose b, ref Pose result) { result.position = a.position + a.rotation * b.position; result.rotation = a.rotation * b.rotation; } private static NativeArray GetBoneRotations(OVRPlugin.Quatf[] boneRotations) { unsafe { var rotations = new NativeArray(boneRotations.Length, TempJob, UninitializedMemory); fixed (void* boneRotationsPtr = boneRotations) { UnsafeUtility.MemCpy(NativeArrayUnsafeUtility.GetUnsafeBufferPointerWithoutChecks(rotations), boneRotationsPtr, boneRotations.Length * (long)UnsafeUtility.SizeOf()); } return rotations; } } private static NativeArray GetBoneTranslations(OVRPlugin.Vector3f[] boneTranslations) { unsafe { var translations = new NativeArray(boneTranslations.Length, TempJob, UninitializedMemory); fixed (void* boneTranslationsPtr = boneTranslations) { UnsafeUtility.MemCpy(NativeArrayUnsafeUtility.GetUnsafeBufferPointerWithoutChecks(translations), boneTranslationsPtr, boneTranslations.Length * (long)UnsafeUtility.SizeOf()); } return translations; } } } }