/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under the license found in the
* LICENSE file in the root directory of this source tree.
*/
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Meta.Voice;
using Meta.Voice.Logging;
using Meta.Voice.TelemetryUtilities;
using Meta.WitAi.Configuration;
using Meta.WitAi.Events;
using Meta.WitAi.Json;
using Meta.WitAi.Requests;
using UnityEngine;
using UnityEngine.Events;
namespace Meta.WitAi
{
///
/// A simple base class for wrapping VoiceServiceRequest event callbacks
///
[LogCategory(LogCategory.SpeechService)]
public abstract class BaseSpeechService : MonoBehaviour
{
///
public IVLogger Logger { get; } = LoggerRegistry.Instance.GetLogger(LogCategory.SpeechService);
///
/// Whether this script should wrap all request event setups
///
public bool ShouldWrap = true;
///
/// Whether this script should log
///
public bool ShouldLog = true;
///
/// All currently running requests
///
public HashSet Requests { get; } = new HashSet();
///
/// Dictionary that holds generated actions that pass a request along with additional event data
///
private ConcurrentDictionary _customRequestEvents = new ConcurrentDictionary();
///
/// Returns true if this voice service is currently active, listening with the mic or performing a networked request
///
public virtual bool Active => Requests != null && Requests.Count > 0;
///
/// If applicable, get all speech events
///
protected virtual SpeechEvents GetSpeechEvents() => null;
///
/// Returns true if this voice service is currently active, listening with the mic or performing a networked request
///
public virtual bool IsAudioInputActive
{
get
{
var audioRequest = GetAudioRequest();
return audioRequest != null && audioRequest.IsAudioInputActivated;
}
}
///
/// Get the first running audio request
///
protected virtual VoiceServiceRequest GetAudioRequest() =>
Requests?.FirstOrDefault((request) => request.InputType == NLPRequestInputType.Audio);
///
/// Check for error that will occur if attempting to activate audio
///
/// Returns an error audio activation should not be allowed.
public virtual string GetActivateAudioError()
{
// Ensure audio is not already active
if (IsAudioInputActive)
{
return "Audio input is already being performed for this service.";
}
// No error
return string.Empty;
}
///
/// Whether an audio request can be started or not
///
public virtual bool CanActivateAudio() => string.IsNullOrEmpty(GetActivateAudioError());
///
/// Check for error that will occur if attempting to send data
///
/// Returns an error if send will not be allowed.
public virtual string GetSendError()
{
// No error
return string.Empty;
}
///
/// Whether a voice service request can be sent or not
///
public virtual bool CanSend() => string.IsNullOrEmpty(GetSendError());
///
/// On enable, begin watching for request initialized callbacks
///
protected virtual void OnEnable()
{
// Cannot send if internet is not reachable (Only works on Mobile)
if (Application.internetReachability == NetworkReachability.NotReachable)
{
Logger.Error("Unable to reach the internet. Check your connection.");
}
GetSpeechEvents()?.OnRequestInitialized.AddListener(OnRequestInit);
}
///
/// On enable, stop watching for request initialized callbacks
///
protected virtual void OnDisable()
{
GetSpeechEvents()?.OnRequestInitialized.RemoveListener(OnRequestInit);
}
///
/// Deactivate all requests
///
public virtual void Deactivate()
{
foreach (var request in Requests.ToArray())
{
Deactivate(request);
}
}
///
/// Deactivate a specific request
///
public virtual void Deactivate(VoiceServiceRequest request)
{
if (request == null || !request.IsLocalRequest)
{
return;
}
request.DeactivateAudio();
}
///
/// Deactivate and abort all locally originated requests
///
public virtual void DeactivateAndAbortRequest()
{
foreach (var request in Requests.ToArray())
{
DeactivateAndAbortRequest(request);
}
}
///
/// Deactivate and abort a specific requests
///
public virtual void DeactivateAndAbortRequest(VoiceServiceRequest request)
{
if (request == null || !request.IsLocalRequest)
{
return;
}
request.Cancel();
}
///
/// Method to setup request events with provided base events
///
/// Generate request events if empty
public virtual void SetupRequestParameters(ref WitRequestOptions options, ref VoiceServiceRequestEvents events)
{
// Ensure options & events exist
if (options == null)
{
options = new WitRequestOptions();
}
if (events == null)
{
events = new VoiceServiceRequestEvents();
}
// Call option setup if desired
if (ShouldWrap)
{
GetSpeechEvents().OnRequestOptionSetup?.Invoke(options);
}
}
///
/// Accepts a generated voice service request, wraps all request events & returns local methods
/// for each
///
/// The provided VoiceServiceRequest to be tracked
/// Returns false if wrap fails
public virtual bool WrapRequest(VoiceServiceRequest request)
{
// Cannot track
if (request == null)
{
Log(null, "Cannot wrap a null VoiceServiceRequest", true);
return false;
}
// Already complete, return
if (request.State == VoiceRequestState.Canceled)
{
RuntimeTelemetry.Instance.LogEventTermination((OperationID)request.Options.OperationId, TerminationReason.Canceled);
OnRequestCancel(request);
OnRequestComplete(request);
return true;
}
if (request.State == VoiceRequestState.Failed)
{
RuntimeTelemetry.Instance.LogEventTermination((OperationID)request.Options.OperationId, TerminationReason.Failed);
OnRequestFailed(request);
OnRequestComplete(request);
return true;
}
if (request.State == VoiceRequestState.Successful)
{
RuntimeTelemetry.Instance.LogEventTermination((OperationID)request.Options.OperationId, TerminationReason.Successful);
OnRequestPartialResponse(request, request?.ResponseData);
OnRequestSuccess(request);
OnRequestComplete(request);
return true;
}
// Call init & add delegates
if (ShouldWrap)
{
// Call request initialized method
GetSpeechEvents()?.OnRequestInitialized?.Invoke(request);
// Send if desired
if (request.State == VoiceRequestState.Transmitting)
{
OnRequestSend(request);
}
}
// Success
return true;
}
// The desired log method for this script. Ensures request id is included in every call
protected virtual void Log(VoiceServiceRequest request, string log, bool warn = false)
{
if (!ShouldLog)
{
return;
}
if (warn)
{
Logger.Error("{0}\nRequest Id: {1}", log, request?.Options?.RequestId);
}
else
{
Logger.Info(log);
}
}
// Called via VoiceServiceRequest constructor
protected virtual void OnRequestInit(VoiceServiceRequest request)
{
// Ignore if already set up
if (Requests.Contains(request))
{
return;
}
// Add main completion event callbacks
SetEventListeners(request, true);
// Add to request list
Requests.Add(request);
Log(request, "Request Initialized");
// Now initialized
#pragma warning disable CS0618
GetSpeechEvents()?.OnRequestCreated?.Invoke(request is WitRequest witRequest ? witRequest : null);
#pragma warning restore CS0618
}
// Called when VoiceServiceRequest OnStartListening is returned
protected virtual void OnRequestStartListening(VoiceServiceRequest request)
{
Log(request, "Request Start Listening");
GetSpeechEvents()?.OnStartListening?.Invoke();
}
// Called when VoiceServiceRequest OnStopListening is returned
protected virtual void OnRequestStopListening(VoiceServiceRequest request)
{
Log(request, "Request Stop Listening");
GetSpeechEvents()?.OnStoppedListening?.Invoke();
}
// Called when VoiceServiceRequest OnPartialResponse is returned & tries to end early if possible
protected virtual void OnRequestSend(VoiceServiceRequest request)
{
Log(request, "Request Send");
GetSpeechEvents()?.OnSend?.Invoke(request);
}
// Called when VoiceServiceRequest OnPartialTranscription is returned with early ASR
protected virtual void OnRequestRawResponse(VoiceServiceRequest request, string rawResponse)
{
GetSpeechEvents()?.OnRawResponse?.Invoke(rawResponse);
}
// Called when VoiceServiceRequest OnPartialTranscription is returned with early ASR
protected virtual void OnRequestPartialTranscription(VoiceServiceRequest request, string transcription)
{
Log(request, $"Request partial transcription received \nText: {transcription}");
GetSpeechEvents()?.OnPartialTranscription?.Invoke(transcription);
GetSpeechEvents()?.OnUserPartialTranscription?.Invoke(request.Options.ClientUserId, transcription);
}
// Called when VoiceServiceRequest OnFullTranscription is returned from request with final ASR
protected virtual void OnRequestFullTranscription(VoiceServiceRequest request, string transcription)
{
Log(request, $"Request Full Transcription received\nText: {transcription}");
GetSpeechEvents()?.OnFullTranscription?.Invoke(transcription);
GetSpeechEvents()?.OnUserFullTranscription?.Invoke(request.Options.ClientUserId, transcription);
}
// Called when VoiceServiceRequest OnPartialResponse is returned & tries to end early if possible
protected virtual void OnRequestPartialResponse(VoiceServiceRequest request, WitResponseNode responseData)
{
if (responseData != null)
{
GetSpeechEvents()?.OnPartialResponse?.Invoke(responseData);
}
}
// Called when VoiceServiceRequest OnCancel is returned
protected virtual void OnRequestCancel(VoiceServiceRequest request)
{
string message = request?.Results?.Message;
Log(request, $"Request Canceled\nReason: {message}");
GetSpeechEvents()?.OnCanceled?.Invoke(message);
if (!string.Equals(message, WitConstants.CANCEL_MESSAGE_PRE_SEND))
{
GetSpeechEvents()?.OnAborted?.Invoke();
}
}
// Called when VoiceServiceRequest OnFailed is returned
protected virtual void OnRequestFailed(VoiceServiceRequest request)
{
string code = $"HTTP Error {request.Results.StatusCode}";
string message = request?.Results?.Message;
string debugMessage = message;
if (string.Equals(debugMessage, WitConstants.ERROR_RESPONSE_TIMEOUT))
{
debugMessage += $"\nTimeout Ms: {request.Options.TimeoutMs}";
}
Log(request, $"Request Failed\n{code}: {debugMessage}", true);
GetSpeechEvents()?.OnError?.Invoke(code, message);
GetSpeechEvents()?.OnRequestCompleted?.Invoke();
}
// Called when VoiceServiceRequest OnSuccess is returned
protected virtual void OnRequestSuccess(VoiceServiceRequest request)
{
Log(request, $"Request Success");
GetSpeechEvents()?.OnResponse?.Invoke(request?.ResponseData);
GetSpeechEvents()?.OnRequestCompleted?.Invoke();
}
// Called when VoiceServiceRequest returns successfully, with an error or is cancelled
protected virtual void OnRequestComplete(VoiceServiceRequest request)
{
// Remove from set & unwrap if found
if (Requests.Contains(request))
{
SetEventListeners(request, false);
Requests.Remove(request);
}
// Perform log & event callbacks
Log(request, $"Request Complete\nRemaining: {Requests.Count}");
GetSpeechEvents()?.OnComplete?.Invoke(request);
}
///
/// Adds or removes event listeners for every request event callback
///
/// The request to begin or stop listening to
/// If true, adds listeners and if false, removes listeners.
protected virtual void SetEventListeners(VoiceServiceRequest request, bool addListeners)
{
// Get request events
var events = request.Events;
// Add/Remove 1 : 1 listeners
events.OnStartListening.SetListener(OnRequestStartListening, addListeners);
events.OnStopListening.SetListener(OnRequestStopListening, addListeners);
events.OnSend.SetListener(OnRequestSend, addListeners);
events.OnSuccess.SetListener(OnRequestSuccess, addListeners);
events.OnFailed.SetListener(OnRequestFailed, addListeners);
events.OnCancel.SetListener(OnRequestCancel, addListeners);
events.OnComplete.SetListener(OnRequestComplete, addListeners);
// Add/Remove custom actions
SetRequestEventListener(events.OnRawResponse, request, OnRequestRawResponse, addListeners);
SetRequestEventListener(events.OnPartialTranscription, request, OnRequestPartialTranscription, addListeners);
SetRequestEventListener(events.OnFullTranscription, request, OnRequestFullTranscription, addListeners);
SetRequestEventListener(events.OnPartialResponse, request, OnRequestPartialResponse, addListeners);
}
///
/// Adds or removes listener to a base event and passes base event's parameter along with the request to the added action.
///
/// The base event to begin or stop listening to.
/// The request to be passed to the callback along with the baseEvent parameter.
/// The callback action that should occur whenever the baseEvent is called.
/// If true, adds listener to baseEvent. If false, removes listener
private void SetRequestEventListener(UnityEvent baseEvent,
VoiceServiceRequest request,
UnityAction callbackWithRequest,
bool addListener)
{
var id = baseEvent.GetHashCode();
if (addListener)
{
UnityAction intermediaryEvent = (param) => callbackWithRequest?.Invoke(request, param);
_customRequestEvents[id] = intermediaryEvent;
baseEvent.AddListener(intermediaryEvent);
}
else if (_customRequestEvents.TryRemove(id, out var adapter)
&& adapter is UnityAction intermediaryEvent)
{
baseEvent.RemoveListener(intermediaryEvent);
}
}
}
}