using Unity.XR.CoreUtils; using Unity.XR.CoreUtils.Bindings; using Unity.XR.CoreUtils.Bindings.Variables; using UnityEngine.Assertions; using UnityEngine.XR.Interaction.Toolkit.Inputs; using UnityEngine.XR.Interaction.Toolkit.Interactors; using UnityEngine.XR.Interaction.Toolkit.Utilities; using UnityEngine.XR.Interaction.Toolkit.Utilities.Tweenables.Primitives; using UnityEngine.XR.Interaction.Toolkit.Utilities.Tweenables.SmartTweenableVariables; namespace UnityEngine.XR.Interaction.Toolkit.UI.BodyUI { /// /// Makes a GameObject follow a tracked hand or motion controller with logic for setting visibility /// of the menu based on the palm orientation. This can be used, for example, to show a preferences /// menu when the user is looking at their palm. /// /// /// This class makes the assumption that the tracked offset has the following orientation: /// When the user's palm is facing down with fingers pointing away from the user, /// y-axis is up, z-axis is forward, x-axis is right according to OpenXR. /// /// Using controllers you will need different offsets. /// TODO: Disable GameObject automatically when hand tracking is lost. /// [AddComponentMenu("XR/Hand Menu", 22)] [HelpURL(XRHelpURLConstants.k_HandMenu)] public class HandMenu : MonoBehaviour { /// /// Enum dictating the up direction used in hand menu calculations. /// /// public enum UpDirection { /// /// Use the global world up direction (). /// WorldUp, /// /// Use this GameObject's world up direction (). /// Useful if this component is on a child GameObject of the XR Origin and the user can teleport to walls. /// TransformUp, /// /// Use the main camera up direction. /// The menu will stay oriented with the head when the user tilts their head left or right. /// CameraUp, } /// /// Enum determining which hand the hand menu will follow. /// /// public enum MenuHandedness { /// /// Make the menu not follow either hand. Effectively disables the hand menu. /// None, /// /// Make the menu follow the left hand. /// Left, /// /// Make the menu follow the right hand. /// Right, /// /// Make the menu follow either hand, choosing the first hand that satisfies requirements. /// Either, } [SerializeField] [Tooltip("Child GameObject used to hold the hand menu UI. This is the transform that moves each frame.")] GameObject m_HandMenuUIGameObject; /// /// Child GameObject used to hold the hand menu UI. This is the transform that moves each frame. /// public GameObject handMenuUIGameObject { get => m_HandMenuUIGameObject; set => m_HandMenuUIGameObject = value; } [Header("Hand alignment")] [SerializeField] [Tooltip("Which hand should the menu anchor to. None will disable the hand menu. Either will try to follow the first hand to meet requirements.")] MenuHandedness m_MenuHandedness = MenuHandedness.Either; /// /// Which hand should the menu anchor to. /// /// /// will disable the hand menu. /// will try to follow the first hand to meet requirements. /// /// public MenuHandedness menuHandedness { get => m_MenuHandedness; set => m_MenuHandedness = value; } [SerializeField] [Tooltip("Determines the up direction of the menu when the hand menu is looking at the camera.")] UpDirection m_HandMenuUpDirection = UpDirection.TransformUp; /// /// Determines the up direction of the menu when the hand menu is looking at the camera. /// /// public UpDirection handMenuUpDirection { get => m_HandMenuUpDirection; set => m_HandMenuUpDirection = value; } [Header("Palm anchor")] [SerializeField] [Tooltip("Anchor associated with the left palm pose for the hand.")] Transform m_LeftPalmAnchor; /// /// Anchor associated with the left palm pose for the hand. /// public Transform leftPalmAnchor { get => m_LeftPalmAnchor; set => m_LeftPalmAnchor = value; } [SerializeField] [Tooltip("Anchor associated with the right palm pose for the hand.")] Transform m_RightPalmAnchor; /// /// Anchor associated with the right palm pose for the hand. /// public Transform rightPalmAnchor { get => m_RightPalmAnchor; set => m_RightPalmAnchor = value; } [Header("Position follow config.")] [SerializeField] [Tooltip("Minimum distance in meters from target before which tween starts.")] float m_MinFollowDistance = 0.005f; /// /// Minimum distance in meters from target before which tween starts. /// public float minFollowDistance { get => m_MinFollowDistance; set { m_MinFollowDistance = value; m_HandAnchorSmartFollow.minDistanceAllowed = value; } } [SerializeField] [Tooltip("Maximum distance in meters from target before tween targets, when time threshold is reached.")] float m_MaxFollowDistance = 0.03f; /// /// Maximum distance in meters from target before tween targets, when time threshold is reached. /// public float maxFollowDistance { get => m_MaxFollowDistance; set { m_MaxFollowDistance = value; m_HandAnchorSmartFollow.maxDistanceAllowed = value; } } [SerializeField] [Tooltip("Time required to elapse before the max distance allowed goes from the min distance to the max.")] float m_MinToMaxDelaySeconds = 1f; /// /// Time required to elapse before the max distance allowed goes from the min distance to the max. /// public float minToMaxDelaySeconds { get => m_MinToMaxDelaySeconds; set { m_MinToMaxDelaySeconds = value; m_HandAnchorSmartFollow.minToMaxDelaySeconds = value; } } [Header("Gaze Alignment Config")] [SerializeField] [Tooltip("If true, menu will hide when gaze to menu origin's divergence angle is above the threshold. In other words, the menu will only show if looking roughly in it's direction.")] bool m_HideMenuWhenGazeDiverges = true; /// /// If true, menu will hide when gaze to menu origin's divergence angle is above the threshold. In other words, the menu will only show if looking roughly in it's direction. /// public bool hideMenuWhenGazeDiverges { get => m_HideMenuWhenGazeDiverges; set => m_HideMenuWhenGazeDiverges = value; } [SerializeField] [Tooltip("Only show menu if gaze to menu origin's divergence angle is below this value.")] float m_MenuVisibleGazeAngleDivergenceThreshold = 35f; float m_MenuVisibilityDotThreshold; /// /// Only show menu if gaze to menu origin's divergence angle is below this value. /// public float menuVisibleGazeDivergenceThreshold { get => m_MenuVisibleGazeAngleDivergenceThreshold; set { m_MenuVisibleGazeAngleDivergenceThreshold = value; m_MenuVisibilityDotThreshold = AngleToDot(value); } } #pragma warning disable CS0618 // Type or member is obsolete readonly SmartFollowVector3TweenableVariable m_HandAnchorSmartFollow = new SmartFollowVector3TweenableVariable(); readonly QuaternionTweenableVariable m_RotTweenFollow = new QuaternionTweenableVariable(); readonly Vector3TweenableVariable m_MenuScaleTweenable = new Vector3TweenableVariable(); #pragma warning restore CS0618 // Type or member is obsolete readonly BindingsGroup m_BindingsGroup = new BindingsGroup(); Transform m_CameraTransform; bool m_WasMenuHiddenLastFrame = true; MenuHandedness m_LastHandThatMetRequirements = MenuHandedness.Left; [Header("Animation Settings")] [SerializeField] [Tooltip("Should the menu animate when it is revealed or hidden.")] bool m_AnimateMenuHideAndReveal = true; /// /// Duration of the reveal/hide animation in seconds. /// public bool animateMenuHideAndRevel { get => m_AnimateMenuHideAndReveal; set => m_AnimateMenuHideAndReveal = value; } [SerializeField] [Tooltip("Duration of the reveal/hide animation in seconds.")] float m_RevealHideAnimationDuration = 0.15f; /// /// Duration of the reveal/hide animation in seconds. /// public float revealHideAnimationDuration { get => m_RevealHideAnimationDuration; set => m_RevealHideAnimationDuration = value; } [Header("Selection Behavior")] [SerializeField] [Tooltip("Should the menu hide when a selection is made with the hand for which the menu is anchored to.")] bool m_HideMenuOnSelect = true; /// /// Should the menu hide when a selection is made with the hand for which the menu is anchored to. /// public bool hideMenuOnSelect { get => m_HideMenuOnSelect; set => m_HideMenuOnSelect = value; } [SerializeField] [Tooltip("XR Interaction Manager used to determine if a hand is selecting. Will find one if None. Used for Hide Menu On Select.")] XRInteractionManager m_InteractionManager; /// /// XR Interaction Manager used to determine if a hand is selecting. Will find one if . /// Used for . /// /// public XRInteractionManager interactionManager { get => m_InteractionManager; set => m_InteractionManager = value; } [Header("Follow presets")] [SerializeField] FollowPresetDatumProperty m_HandTrackingFollowPreset; [SerializeField] FollowPresetDatumProperty m_ControllerFollowPreset; XRInputModalityManager.InputMode m_CurrentInputMode = XRInputModalityManager.InputMode.None; Transform m_LeftOffsetRoot = null; Transform m_RightOffsetRoot = null; Coroutine m_HideCoroutine = null; Coroutine m_ShowCoroutine = null; Transform m_LastValidCameraTransform = null; Transform m_LastValidPalmAnchor = null; Transform m_LastValidPalmAnchorOffset = null; Vector3 m_InitialMenuLocalScale = Vector3.one; readonly BindableVariable m_MenuVisibleBindableVariable = new BindableVariable(false); float m_LastValidTrackingTime = 0f; /// /// See . /// protected void Awake() { m_HandAnchorSmartFollow.minDistanceAllowed = m_MinFollowDistance; m_HandAnchorSmartFollow.maxDistanceAllowed = m_MaxFollowDistance; m_HandAnchorSmartFollow.minToMaxDelaySeconds = m_MinToMaxDelaySeconds; // Initialize anchors m_RightOffsetRoot = new GameObject("Right Offset Root").transform; m_RightOffsetRoot.transform.SetParent(m_RightPalmAnchor); m_LeftOffsetRoot = new GameObject("Left Offset Root").transform; m_LeftOffsetRoot.transform.SetParent(m_LeftPalmAnchor); } /// /// See . /// protected void OnEnable() { if (m_LeftPalmAnchor == null || m_RightPalmAnchor == null) { Debug.LogError($"Missing palm anchor transform reference. Disabling {this} component.", this); enabled = false; return; } if (m_HandMenuUIGameObject == null) { Debug.LogError($"Missing Hand Menu UI GameObject reference. Disabling {this} component.", this); enabled = false; return; } if (m_ControllerFollowPreset == null || m_HandTrackingFollowPreset == null) { Debug.LogError($"Missing Follow Preset reference. Disabling {this} component.", this); enabled = false; return; } m_HandAnchorSmartFollow.Value = m_HandMenuUIGameObject.transform.position; m_BindingsGroup.AddBinding(m_HandAnchorSmartFollow.Subscribe(newPosition => m_HandMenuUIGameObject.transform.position = newPosition)); m_RotTweenFollow.Value = m_HandMenuUIGameObject.transform.rotation; m_BindingsGroup.AddBinding(m_RotTweenFollow.Subscribe(newRot => m_HandMenuUIGameObject.transform.rotation = newRot)); m_InitialMenuLocalScale = m_HandMenuUIGameObject.transform.localScale; m_MenuScaleTweenable.Value = m_InitialMenuLocalScale; m_BindingsGroup.AddBinding(m_MenuScaleTweenable.Subscribe(value => m_HandMenuUIGameObject.transform.localScale = value)); m_BindingsGroup.AddBinding(XRInputModalityManager.currentInputMode.SubscribeAndUpdate(OnInputModeChanged)); m_MenuVisibleBindableVariable.Value = false; m_BindingsGroup.AddBinding(m_MenuVisibleBindableVariable.SubscribeAndUpdate(value => { if (value) ShowMenu(); else HideMenu(); })); } /// /// See . /// protected void OnDisable() { if (m_ShowCoroutine != null) { StopCoroutine(m_ShowCoroutine); m_ShowCoroutine = null; } if (m_HideCoroutine != null) { StopCoroutine(m_HideCoroutine); m_HideCoroutine = null; } m_BindingsGroup.Clear(); m_HandMenuUIGameObject.transform.localScale = m_InitialMenuLocalScale; m_HandMenuUIGameObject.SetActive(true); OnMenuVisible(); } /// /// See . /// protected void OnDestroy() { m_HandAnchorSmartFollow.Dispose(); } /// /// See . /// protected void OnValidate() { m_HandAnchorSmartFollow.minDistanceAllowed = m_MinFollowDistance; m_HandAnchorSmartFollow.maxDistanceAllowed = m_MaxFollowDistance; m_HandAnchorSmartFollow.minToMaxDelaySeconds = m_MinToMaxDelaySeconds; m_MenuVisibilityDotThreshold = AngleToDot(m_MenuVisibleGazeAngleDivergenceThreshold); } /// /// This method is called when the input mode changes in the XRInputModalityManager. /// It updates the current preset and applies it to the left and right offset roots /// based on the new input mode (MotionController or TrackedHand). /// /// The new input mode of the XRInputModalityManager. void OnInputModeChanged(XRInputModalityManager.InputMode newInputMode) { m_CurrentInputMode = newInputMode; GetCurrentPreset()?.ApplyPreset(m_LeftOffsetRoot, m_RightOffsetRoot); } FollowPreset GetCurrentPreset() { if (m_CurrentInputMode == XRInputModalityManager.InputMode.MotionController) return m_ControllerFollowPreset.Value; return m_HandTrackingFollowPreset.Value; } void ShowMenu() { if (m_HideCoroutine != null) { StopCoroutine(m_HideCoroutine); m_HideCoroutine = null; } m_HandMenuUIGameObject.SetActive(true); if (m_AnimateMenuHideAndReveal && m_ShowCoroutine == null) m_ShowCoroutine = StartCoroutine(m_MenuScaleTweenable.PlaySequence(m_MenuScaleTweenable.Value, m_InitialMenuLocalScale, m_RevealHideAnimationDuration, OnMenuVisible)); else OnMenuVisible(); } void OnMenuVisible() { m_ShowCoroutine = null; m_WasMenuHiddenLastFrame = false; } void HideMenu() { if (m_ShowCoroutine != null) { StopCoroutine(m_ShowCoroutine); m_ShowCoroutine = null; } if (m_AnimateMenuHideAndReveal && m_HideCoroutine == null) m_HideCoroutine = StartCoroutine(m_MenuScaleTweenable.PlaySequence(m_MenuScaleTweenable.Value, Vector3.zero, m_RevealHideAnimationDuration, OnMenuHidden)); else OnMenuHidden(); } void OnMenuHidden() { m_HandMenuUIGameObject.SetActive(false); m_WasMenuHiddenLastFrame = true; m_HideCoroutine = null; } /// /// See . /// protected void LateUpdate() { if (m_CurrentInputMode == XRInputModalityManager.InputMode.None) { m_MenuVisibleBindableVariable.Value = false; return; } bool showMenu = false; var currentPresent = GetCurrentPreset(); if (TryGetTrackedAnchors(m_MenuHandedness, currentPresent, out var targetHandedness, out var cameraTransform, out var palmAnchor, out var palmAnchorOffset)) { m_LastValidCameraTransform = cameraTransform; m_LastValidPalmAnchor = palmAnchor; m_LastValidPalmAnchorOffset = palmAnchorOffset; m_LastValidTrackingTime = Time.unscaledTime; showMenu = true; } // If trying to hide menu, but game object is still visible, we want to continue tracking during animation. if (!showMenu) { var timeSinceLastValidTracking = Time.unscaledTime - m_LastValidTrackingTime; // If within hide delay period, keep menu visible. if (timeSinceLastValidTracking > currentPresent.hideDelaySeconds) m_MenuVisibleBindableVariable.Value = false; // If any associated transforms are invalid - return. if (m_LastValidCameraTransform == null || m_LastValidPalmAnchor == null || m_LastValidPalmAnchorOffset == null) return; } var gazeToObject = (m_LastValidPalmAnchorOffset.position - m_LastValidCameraTransform.position).normalized; if (showMenu) { // Add extra gaze divergence validation if (m_HideMenuWhenGazeDiverges) { var gazeDirection = m_LastValidCameraTransform.forward; showMenu = Vector3.Dot(gazeToObject, gazeDirection) > m_MenuVisibilityDotThreshold; } m_MenuVisibleBindableVariable.Value = showMenu; } // Stop tracking if menu is not visible bool menuVisible = m_HandMenuUIGameObject.activeSelf; if (!menuVisible) return; var palmAnchorOffsetPose = m_LastValidPalmAnchorOffset.GetWorldPose(); var targetPos = palmAnchorOffsetPose.position; var targetRot = palmAnchorOffsetPose.rotation; // Check if head gaze is looking at palm if (targetHandedness == MenuHandedness.Left || targetHandedness == MenuHandedness.Right) { var referenceAxis = currentPresent.GetReferenceAxisForTrackingAnchor(m_LastValidPalmAnchor, targetHandedness == MenuHandedness.Right); var objectToGaze = -gazeToObject; // Gaze aligned with reference axis if (currentPresent.snapToGaze && Vector3.Dot(referenceAxis, objectToGaze) > currentPresent.snapToGazeDotThreshold) { var referenceUpDirection = GetReferenceUpDirection(m_LastValidCameraTransform); BurstMathUtility.OrthogonalLookRotation(gazeToObject, referenceUpDirection, out targetRot); } } m_HandAnchorSmartFollow.target = targetPos; m_RotTweenFollow.target = targetRot; // If the menu was previously hidden, we want to snap to the correct target if (m_WasMenuHiddenLastFrame || !currentPresent.allowSmoothing) { m_HandAnchorSmartFollow.HandleTween(1f); // If we allow smoothing, do not snap rotation as it looks jarring if (currentPresent.allowSmoothing) m_RotTweenFollow.HandleTween(Time.deltaTime * currentPresent.followLowerSmoothingValue); else m_RotTweenFollow.HandleTween(1f); } else { m_HandAnchorSmartFollow.HandleSmartTween(Time.deltaTime, currentPresent.followLowerSmoothingValue, currentPresent.followUpperSmoothingValue); m_RotTweenFollow.HandleTween(Time.deltaTime * currentPresent.followLowerSmoothingValue); } } bool TryGetTrackedAnchors(MenuHandedness desiredHandedness, in FollowPreset currentPreset, out MenuHandedness targetHandedness, out Transform cameraTransform, out Transform palmAnchor, out Transform palmAnchorOffset) { palmAnchor = null; palmAnchorOffset = null; targetHandedness = MenuHandedness.None; if (!TryGetCamera(out cameraTransform) || desiredHandedness == MenuHandedness.None) { return false; } // Check if each palm meets requirements. We expect the up vector to be aligned with the world up when the users palm is facing the ground. bool isLeftSelecting = false; bool isRightSelecting = false; if (m_HideMenuOnSelect && TryGetInteractionManager(out var manager)) { isLeftSelecting = manager.IsHandSelecting(InteractorHandedness.Left); isRightSelecting = manager.IsHandSelecting(InteractorHandedness.Right); } var leftMeetsRequirements = !isLeftSelecting && PalmMeetsRequirements(cameraTransform, m_LeftPalmAnchor, false, currentPreset); var rightMeetsRequirements = !isRightSelecting && PalmMeetsRequirements(cameraTransform, m_RightPalmAnchor, true, currentPreset); if (!leftMeetsRequirements && !rightMeetsRequirements) { return false; } if (desiredHandedness == MenuHandedness.Either) { // Check last hand to meet requirements if (leftMeetsRequirements && rightMeetsRequirements) { var handToTry = m_LastHandThatMetRequirements == MenuHandedness.Right ? MenuHandedness.Right : MenuHandedness.Left; GetTransformAnchorsForHandedness(handToTry, out palmAnchor, out palmAnchorOffset); targetHandedness = handToTry; return true; } if (leftMeetsRequirements) { GetTransformAnchorsForHandedness(MenuHandedness.Left, out palmAnchor, out palmAnchorOffset); m_LastHandThatMetRequirements = MenuHandedness.Left; targetHandedness = MenuHandedness.Left; return true; } else { GetTransformAnchorsForHandedness(MenuHandedness.Right, out palmAnchor, out palmAnchorOffset); m_LastHandThatMetRequirements = MenuHandedness.Right; targetHandedness = MenuHandedness.Right; return true; } } if (desiredHandedness == MenuHandedness.Left) { if (leftMeetsRequirements) { GetTransformAnchorsForHandedness(MenuHandedness.Left, out palmAnchor, out palmAnchorOffset); m_LastHandThatMetRequirements = MenuHandedness.Left; targetHandedness = MenuHandedness.Left; return true; } palmAnchor = null; palmAnchorOffset = null; return false; } if (desiredHandedness == MenuHandedness.Right) { if (rightMeetsRequirements) { GetTransformAnchorsForHandedness(MenuHandedness.Right, out palmAnchor, out palmAnchorOffset); m_LastHandThatMetRequirements = MenuHandedness.Right; targetHandedness = MenuHandedness.Right; return true; } palmAnchor = null; palmAnchorOffset = null; return false; } return false; } bool TryGetInteractionManager(out XRInteractionManager manager) { if (m_InteractionManager != null) { manager = m_InteractionManager; return true; } if (ComponentLocatorUtility.TryFindComponent(out m_InteractionManager)) { manager = m_InteractionManager; return true; } manager = null; return false; } void GetTransformAnchorsForHandedness(MenuHandedness handedness, out Transform palmAnchor, out Transform palmAnchorOffset) { if (handedness == MenuHandedness.Left) { palmAnchor = m_LeftPalmAnchor; palmAnchorOffset = m_LeftOffsetRoot; } else if (handedness == MenuHandedness.Right) { palmAnchor = m_RightPalmAnchor; palmAnchorOffset = m_RightOffsetRoot; } else { palmAnchor = null; palmAnchorOffset = null; } } Vector3 GetReferenceUpDirection(Transform cameraTransform) { switch (m_HandMenuUpDirection) { case UpDirection.WorldUp: return Vector3.up; case UpDirection.TransformUp: return transform.up; case UpDirection.CameraUp: return cameraTransform.up; default: Assert.IsTrue(false, $"Unhandled {nameof(UpDirection)}={m_HandMenuUpDirection}."); goto case UpDirection.TransformUp; } } bool PalmMeetsRequirements(Transform cameraTransform, Transform palmAnchor, bool isRightHand, in FollowPreset currentPresent) { if (currentPresent == null) return false; var palmAnchorUp = currentPresent.GetReferenceAxisForTrackingAnchor(palmAnchor, isRightHand); var referenceUpDirection = GetReferenceUpDirection(cameraTransform); // With hand tracking, palm faces the world up direction when the hand is lying down flat. // With controllers, we typically look for the right vector on the left controller, and the left vector on the right controller to fill this role. // Check if palm is looking at the camera and whether the palm is flipped over towards the sky bool meetsPalmFacingUserThreshold = !currentPresent.requirePalmFacingUser || Vector3.Dot(palmAnchorUp, -cameraTransform.forward) > currentPresent.palmFacingUserDotThreshold; bool meetsPalmFacingUpThreshold = !currentPresent.requirePalmFacingUp || Vector3.Dot(palmAnchorUp, referenceUpDirection) > currentPresent.palmFacingUpDotThreshold; return meetsPalmFacingUserThreshold && meetsPalmFacingUpThreshold; } // TODO: Handle the Camera becoming disabled and retry Camera.main bool TryGetCamera(out Transform cameraTransform) { if (m_CameraTransform == null) { var mainCamera = Camera.main; if (mainCamera == null) { cameraTransform = null; return false; } m_CameraTransform = mainCamera.transform; } cameraTransform = m_CameraTransform; return true; } static float AngleToDot(float angleDeg) { return Mathf.Cos(Mathf.Deg2Rad * angleDeg); } } }