/* * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under the license found in the * LICENSE file in the root directory of this source tree. */ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Text; using Meta.Voice; using Meta.Voice.Logging; using Meta.Voice.TelemetryUtilities; using Meta.WitAi.Configuration; using Meta.WitAi.Events; using Meta.WitAi.Json; using Meta.WitAi.Requests; using UnityEngine; using UnityEngine.Events; namespace Meta.WitAi { /// /// A simple base class for wrapping VoiceServiceRequest event callbacks /// [LogCategory(LogCategory.SpeechService)] public abstract class BaseSpeechService : MonoBehaviour { /// public IVLogger Logger { get; } = LoggerRegistry.Instance.GetLogger(LogCategory.SpeechService); /// /// Whether this script should wrap all request event setups /// public bool ShouldWrap = true; /// /// Whether this script should log /// public bool ShouldLog = true; /// /// All currently running requests /// public HashSet Requests { get; } = new HashSet(); /// /// Dictionary that holds generated actions that pass a request along with additional event data /// private ConcurrentDictionary _customRequestEvents = new ConcurrentDictionary(); /// /// Returns true if this voice service is currently active, listening with the mic or performing a networked request /// public virtual bool Active => Requests != null && Requests.Count > 0; /// /// If applicable, get all speech events /// protected virtual SpeechEvents GetSpeechEvents() => null; /// /// Returns true if this voice service is currently active, listening with the mic or performing a networked request /// public virtual bool IsAudioInputActive { get { var audioRequest = GetAudioRequest(); return audioRequest != null && audioRequest.IsAudioInputActivated; } } /// /// Get the first running audio request /// protected virtual VoiceServiceRequest GetAudioRequest() => Requests?.FirstOrDefault((request) => request.InputType == NLPRequestInputType.Audio); /// /// Check for error that will occur if attempting to activate audio /// /// Returns an error audio activation should not be allowed. public virtual string GetActivateAudioError() { // Ensure audio is not already active if (IsAudioInputActive) { return "Audio input is already being performed for this service."; } // No error return string.Empty; } /// /// Whether an audio request can be started or not /// public virtual bool CanActivateAudio() => string.IsNullOrEmpty(GetActivateAudioError()); /// /// Check for error that will occur if attempting to send data /// /// Returns an error if send will not be allowed. public virtual string GetSendError() { // No error return string.Empty; } /// /// Whether a voice service request can be sent or not /// public virtual bool CanSend() => string.IsNullOrEmpty(GetSendError()); /// /// On enable, begin watching for request initialized callbacks /// protected virtual void OnEnable() { // Cannot send if internet is not reachable (Only works on Mobile) if (Application.internetReachability == NetworkReachability.NotReachable) { Logger.Error("Unable to reach the internet. Check your connection."); } GetSpeechEvents()?.OnRequestInitialized.AddListener(OnRequestInit); } /// /// On enable, stop watching for request initialized callbacks /// protected virtual void OnDisable() { GetSpeechEvents()?.OnRequestInitialized.RemoveListener(OnRequestInit); } /// /// Deactivate all requests /// public virtual void Deactivate() { foreach (var request in Requests.ToArray()) { Deactivate(request); } } /// /// Deactivate a specific request /// public virtual void Deactivate(VoiceServiceRequest request) { if (request == null || !request.IsLocalRequest) { return; } request.DeactivateAudio(); } /// /// Deactivate and abort all locally originated requests /// public virtual void DeactivateAndAbortRequest() { foreach (var request in Requests.ToArray()) { DeactivateAndAbortRequest(request); } } /// /// Deactivate and abort a specific requests /// public virtual void DeactivateAndAbortRequest(VoiceServiceRequest request) { if (request == null || !request.IsLocalRequest) { return; } request.Cancel(); } /// /// Method to setup request events with provided base events /// /// Generate request events if empty public virtual void SetupRequestParameters(ref WitRequestOptions options, ref VoiceServiceRequestEvents events) { // Ensure options & events exist if (options == null) { options = new WitRequestOptions(); } if (events == null) { events = new VoiceServiceRequestEvents(); } // Call option setup if desired if (ShouldWrap) { GetSpeechEvents().OnRequestOptionSetup?.Invoke(options); } } /// /// Accepts a generated voice service request, wraps all request events & returns local methods /// for each /// /// The provided VoiceServiceRequest to be tracked /// Returns false if wrap fails public virtual bool WrapRequest(VoiceServiceRequest request) { // Cannot track if (request == null) { Log(null, "Cannot wrap a null VoiceServiceRequest", true); return false; } // Already complete, return if (request.State == VoiceRequestState.Canceled) { RuntimeTelemetry.Instance.LogEventTermination((OperationID)request.Options.OperationId, TerminationReason.Canceled); OnRequestCancel(request); OnRequestComplete(request); return true; } if (request.State == VoiceRequestState.Failed) { RuntimeTelemetry.Instance.LogEventTermination((OperationID)request.Options.OperationId, TerminationReason.Failed); OnRequestFailed(request); OnRequestComplete(request); return true; } if (request.State == VoiceRequestState.Successful) { RuntimeTelemetry.Instance.LogEventTermination((OperationID)request.Options.OperationId, TerminationReason.Successful); OnRequestPartialResponse(request, request?.ResponseData); OnRequestSuccess(request); OnRequestComplete(request); return true; } // Call init & add delegates if (ShouldWrap) { // Call request initialized method GetSpeechEvents()?.OnRequestInitialized?.Invoke(request); // Send if desired if (request.State == VoiceRequestState.Transmitting) { OnRequestSend(request); } } // Success return true; } // The desired log method for this script. Ensures request id is included in every call protected virtual void Log(VoiceServiceRequest request, string log, bool warn = false) { if (!ShouldLog) { return; } if (warn) { Logger.Error("{0}\nRequest Id: {1}", log, request?.Options?.RequestId); } else { Logger.Info(log); } } // Called via VoiceServiceRequest constructor protected virtual void OnRequestInit(VoiceServiceRequest request) { // Ignore if already set up if (Requests.Contains(request)) { return; } // Add main completion event callbacks SetEventListeners(request, true); // Add to request list Requests.Add(request); Log(request, "Request Initialized"); // Now initialized #pragma warning disable CS0618 GetSpeechEvents()?.OnRequestCreated?.Invoke(request is WitRequest witRequest ? witRequest : null); #pragma warning restore CS0618 } // Called when VoiceServiceRequest OnStartListening is returned protected virtual void OnRequestStartListening(VoiceServiceRequest request) { Log(request, "Request Start Listening"); GetSpeechEvents()?.OnStartListening?.Invoke(); } // Called when VoiceServiceRequest OnStopListening is returned protected virtual void OnRequestStopListening(VoiceServiceRequest request) { Log(request, "Request Stop Listening"); GetSpeechEvents()?.OnStoppedListening?.Invoke(); } // Called when VoiceServiceRequest OnPartialResponse is returned & tries to end early if possible protected virtual void OnRequestSend(VoiceServiceRequest request) { Log(request, "Request Send"); GetSpeechEvents()?.OnSend?.Invoke(request); } // Called when VoiceServiceRequest OnPartialTranscription is returned with early ASR protected virtual void OnRequestRawResponse(VoiceServiceRequest request, string rawResponse) { GetSpeechEvents()?.OnRawResponse?.Invoke(rawResponse); } // Called when VoiceServiceRequest OnPartialTranscription is returned with early ASR protected virtual void OnRequestPartialTranscription(VoiceServiceRequest request, string transcription) { Log(request, $"Request partial transcription received \nText: {transcription}"); GetSpeechEvents()?.OnPartialTranscription?.Invoke(transcription); GetSpeechEvents()?.OnUserPartialTranscription?.Invoke(request.Options.ClientUserId, transcription); } // Called when VoiceServiceRequest OnFullTranscription is returned from request with final ASR protected virtual void OnRequestFullTranscription(VoiceServiceRequest request, string transcription) { Log(request, $"Request Full Transcription received\nText: {transcription}"); GetSpeechEvents()?.OnFullTranscription?.Invoke(transcription); GetSpeechEvents()?.OnUserFullTranscription?.Invoke(request.Options.ClientUserId, transcription); } // Called when VoiceServiceRequest OnPartialResponse is returned & tries to end early if possible protected virtual void OnRequestPartialResponse(VoiceServiceRequest request, WitResponseNode responseData) { if (responseData != null) { GetSpeechEvents()?.OnPartialResponse?.Invoke(responseData); } } // Called when VoiceServiceRequest OnCancel is returned protected virtual void OnRequestCancel(VoiceServiceRequest request) { string message = request?.Results?.Message; Log(request, $"Request Canceled\nReason: {message}"); GetSpeechEvents()?.OnCanceled?.Invoke(message); if (!string.Equals(message, WitConstants.CANCEL_MESSAGE_PRE_SEND)) { GetSpeechEvents()?.OnAborted?.Invoke(); } } // Called when VoiceServiceRequest OnFailed is returned protected virtual void OnRequestFailed(VoiceServiceRequest request) { string code = $"HTTP Error {request.Results.StatusCode}"; string message = request?.Results?.Message; string debugMessage = message; if (string.Equals(debugMessage, WitConstants.ERROR_RESPONSE_TIMEOUT)) { debugMessage += $"\nTimeout Ms: {request.Options.TimeoutMs}"; } Log(request, $"Request Failed\n{code}: {debugMessage}", true); GetSpeechEvents()?.OnError?.Invoke(code, message); GetSpeechEvents()?.OnRequestCompleted?.Invoke(); } // Called when VoiceServiceRequest OnSuccess is returned protected virtual void OnRequestSuccess(VoiceServiceRequest request) { Log(request, $"Request Success"); GetSpeechEvents()?.OnResponse?.Invoke(request?.ResponseData); GetSpeechEvents()?.OnRequestCompleted?.Invoke(); } // Called when VoiceServiceRequest returns successfully, with an error or is cancelled protected virtual void OnRequestComplete(VoiceServiceRequest request) { // Remove from set & unwrap if found if (Requests.Contains(request)) { SetEventListeners(request, false); Requests.Remove(request); } // Perform log & event callbacks Log(request, $"Request Complete\nRemaining: {Requests.Count}"); GetSpeechEvents()?.OnComplete?.Invoke(request); } /// /// Adds or removes event listeners for every request event callback /// /// The request to begin or stop listening to /// If true, adds listeners and if false, removes listeners. protected virtual void SetEventListeners(VoiceServiceRequest request, bool addListeners) { // Get request events var events = request.Events; // Add/Remove 1 : 1 listeners events.OnStartListening.SetListener(OnRequestStartListening, addListeners); events.OnStopListening.SetListener(OnRequestStopListening, addListeners); events.OnSend.SetListener(OnRequestSend, addListeners); events.OnSuccess.SetListener(OnRequestSuccess, addListeners); events.OnFailed.SetListener(OnRequestFailed, addListeners); events.OnCancel.SetListener(OnRequestCancel, addListeners); events.OnComplete.SetListener(OnRequestComplete, addListeners); // Add/Remove custom actions SetRequestEventListener(events.OnRawResponse, request, OnRequestRawResponse, addListeners); SetRequestEventListener(events.OnPartialTranscription, request, OnRequestPartialTranscription, addListeners); SetRequestEventListener(events.OnFullTranscription, request, OnRequestFullTranscription, addListeners); SetRequestEventListener(events.OnPartialResponse, request, OnRequestPartialResponse, addListeners); } /// /// Adds or removes listener to a base event and passes base event's parameter along with the request to the added action. /// /// The base event to begin or stop listening to. /// The request to be passed to the callback along with the baseEvent parameter. /// The callback action that should occur whenever the baseEvent is called. /// If true, adds listener to baseEvent. If false, removes listener private void SetRequestEventListener(UnityEvent baseEvent, VoiceServiceRequest request, UnityAction callbackWithRequest, bool addListener) { var id = baseEvent.GetHashCode(); if (addListener) { UnityAction intermediaryEvent = (param) => callbackWithRequest?.Invoke(request, param); _customRequestEvents[id] = intermediaryEvent; baseEvent.AddListener(intermediaryEvent); } else if (_customRequestEvents.TryRemove(id, out var adapter) && adapter is UnityAction intermediaryEvent) { baseEvent.RemoveListener(intermediaryEvent); } } } }