From 47c7104d15367b441de4eb03130028425521dfa3 Mon Sep 17 00:00:00 2001
From: Nitwel <mail@nitwel.de>
Date: Fri, 15 Mar 2024 19:27:45 +0100
Subject: [PATCH] make voice assistant toggleable and fix bugs

---
 content/main.gd                             | 15 ++++++++++
 content/main.tscn                           |  6 +---
 content/system/assist/assist.gd             | 10 +++++--
 content/system/assist/assist.tscn           |  2 +-
 content/ui/menu/settings/settings_menu.gd   | 33 +++++++++++++++++++++
 content/ui/menu/settings/settings_menu.tscn | 15 ++++++++++
 lib/home_apis/hass_ws/handlers/assist.gd    |  5 +++-
 lib/stores/settings.gd                      |  7 +++--
 lib/stores/store.gd                         | 14 +++++----
 9 files changed, 89 insertions(+), 18 deletions(-)

diff --git a/content/main.gd b/content/main.gd
index 9feab7e..dbf478b 100644
--- a/content/main.gd
+++ b/content/main.gd
@@ -2,6 +2,7 @@ extends Node3D
 
 var sky = preload ("res://assets/materials/sky.material")
 var sky_passthrough = preload ("res://assets/materials/sky_passthrough.material")
+const VoiceAssistant = preload ("res://content/system/assist/assist.tscn")
 
 @onready var environment: WorldEnvironment = $WorldEnvironment
 @onready var camera: XRCamera3D = $XROrigin3D/XRCamera3D
@@ -10,6 +11,7 @@ var sky_passthrough = preload ("res://assets/materials/sky_passthrough.material"
 @onready var house = $House
 @onready var menu = $Menu
 @onready var keyboard = $Keyboard
+var voice_assistant = null
 
 func _ready():
 	# In case we're running on the headset, use the passthrough sky
@@ -20,6 +22,8 @@ func _ready():
 	else:
 		RenderingServer.set_debug_generate_wireframes(true)
 
+	update_voice_assistant()
+
 	controller_left.button_pressed.connect(func(name):
 		_emit_action(name, true, false)
 	)
@@ -61,6 +65,17 @@ func _ready():
 		remove_child(keyboard)
 	)
 
+func update_voice_assistant():
+	if Store.settings.is_loaded() == false:
+		await Store.settings.on_loaded
+
+	if Store.settings.voice_assistant&&voice_assistant == null:
+		voice_assistant = VoiceAssistant.instantiate()
+		add_child(voice_assistant)
+	elif !Store.settings.voice_assistant&&voice_assistant != null:
+		remove_child(voice_assistant)
+		voice_assistant.queue_free()
+
 func toggle_menu():
 	if menu.show_menu == false:
 		add_child(menu)
diff --git a/content/main.tscn b/content/main.tscn
index 19643a5..421b116 100644
--- a/content/main.tscn
+++ b/content/main.tscn
@@ -1,4 +1,4 @@
-[gd_scene load_steps=17 format=3 uid="uid://eecv28y6jxk4"]
+[gd_scene load_steps=16 format=3 uid="uid://eecv28y6jxk4"]
 
 [ext_resource type="PackedScene" uid="uid://clc5dre31iskm" path="res://addons/godot-xr-tools/xr/start_xr.tscn" id="1_i4c04"]
 [ext_resource type="Script" path="res://content/main.gd" id="1_uvrd4"]
@@ -11,7 +11,6 @@
 [ext_resource type="PackedScene" uid="uid://c3kdssrmv84kv" path="res://content/ui/menu/menu.tscn" id="8_du83w"]
 [ext_resource type="PackedScene" uid="uid://lrehk38exd5n" path="res://content/system/keyboard/keyboard.tscn" id="9_e5n3p"]
 [ext_resource type="PackedScene" uid="uid://cbemihbxkd4ll" path="res://content/system/house/house.tscn" id="9_np6mw"]
-[ext_resource type="PackedScene" uid="uid://oydbwnek6xb4" path="res://content/system/assist/assist.tscn" id="12_8av8q"]
 
 [sub_resource type="Sky" id="Sky_vhymk"]
 sky_material = ExtResource("5_wgwf8")
@@ -84,7 +83,4 @@ transform = Transform3D(0.499999, -0.000139169, -6.50204e-05, 5.24307e-05, 0.353
 
 [node name="House" parent="." instance=ExtResource("9_np6mw")]
 
-[node name="Assist" parent="." instance=ExtResource("12_8av8q")]
-transform = Transform3D(1, -1.39636e-11, 0, 9.47986e-12, 1, 0, 0, 0, 1, 0.000231838, -4.01369e-06, -0.855612)
-
 [editable path="XROrigin3D/XRControllerLeft"]
diff --git a/content/system/assist/assist.gd b/content/system/assist/assist.gd
index 81de3dc..ce3143a 100644
--- a/content/system/assist/assist.gd
+++ b/content/system/assist/assist.gd
@@ -8,7 +8,7 @@ const target_freq = 16000
 const sample_rate_ratio: float = audio_freq / target_freq * 1.5
 
 var effect: AudioEffectCapture
-@export var input_threshold: float = 0.1
+@export var input_threshold: float = 0.05
 @onready var audio_recorder: AudioStreamPlayer = $AudioStreamRecord
 @onready var audio_timer: Timer = $AudioTimer
 @onready var visual_timer: Timer = $VisualTimer
@@ -18,7 +18,7 @@ var effect: AudioEffectCapture
 @onready var loader: Node3D = $Loader
 @onready var camera = $"/root/Main/XROrigin3D/XRCamera3D"
 
-var running := true
+var running := false
 
 func _ready():
 	var index = AudioServer.get_bus_index("Record")
@@ -51,13 +51,17 @@ func _ready():
 	)
 
 	HomeApi.api.assist_handler.on_tts_sound.connect(func(audio):
-		print("Playing TTS ", audio.data.size())
 		audio_player_3d.stream=audio
 		audio_player_3d.play()
 		visual_timer.start()
 		running=false
 	)
 
+	HomeApi.api.assist_handler.on_error.connect(func():
+		running=false
+		finish()
+	)
+
 	visual_timer.timeout.connect(func():
 		if audio_player_3d.playing == false:
 			finish()
diff --git a/content/system/assist/assist.tscn b/content/system/assist/assist.tscn
index 1356257..76c7c44 100644
--- a/content/system/assist/assist.tscn
+++ b/content/system/assist/assist.tscn
@@ -21,7 +21,7 @@ one_shot = true
 [node name="AudioStreamPlayer3D" type="AudioStreamPlayer3D" parent="."]
 
 [node name="VisualTimer" type="Timer" parent="."]
-wait_time = 4.0
+wait_time = 5.0
 one_shot = true
 
 [node name="ChatUser" parent="." instance=ExtResource("2_laew1")]
diff --git a/content/ui/menu/settings/settings_menu.gd b/content/ui/menu/settings/settings_menu.gd
index c46d8c5..51e3afc 100644
--- a/content/ui/menu/settings/settings_menu.gd
+++ b/content/ui/menu/settings/settings_menu.gd
@@ -3,6 +3,7 @@ extends Node3D
 const credits_scene = preload ("./credits.tscn")
 
 @onready var connection_status = $Content/ConnectionStatus
+@onready var main = $"/root/Main"
 
 @onready var input_url = $Content/InputURL
 @onready var input_token = $Content/InputToken
@@ -11,6 +12,7 @@ const credits_scene = preload ("./credits.tscn")
 @onready var save = $Content/Save
 @onready var clear_save = $Content/ClearSave
 @onready var background = $Background
+@onready var voice_assist = $Content/VoiceAssist
 
 func _ready():
 	background.visible = false
@@ -53,6 +55,31 @@ func _ready():
 		House.body.update_house()
 	)
 
+	voice_assist.on_button_down.connect(func():
+		if Store.settings.is_loaded() == false:
+			await Store.settings.on_loaded
+
+		OS.request_permissions()
+
+		voice_assist.label="mic"
+
+		Store.settings.voice_assistant=true
+		main.update_voice_assistant()
+		Store.settings.save_local()
+	)
+
+	voice_assist.on_button_up.connect(func():
+		if Store.settings.is_loaded() == false:
+			await Store.settings.on_loaded
+
+		voice_assist.label="mic_off"
+
+		Store.settings.voice_assistant=false
+		main.update_voice_assistant()
+		Store.settings.save_local()
+
+	)
+
 	HomeApi.on_connect.connect(func():
 		connection_status.text="Connected"
 	)
@@ -60,3 +87,9 @@ func _ready():
 	HomeApi.on_disconnect.connect(func():
 		connection_status.text="Disconnected"
 	)
+
+	if Store.settings.is_loaded() == false:
+		await Store.settings.on_loaded
+
+	voice_assist.label = "mic_off" if Store.settings.voice_assistant == false else "mic"
+	voice_assist.active = Store.settings.voice_assistant
diff --git a/content/ui/menu/settings/settings_menu.tscn b/content/ui/menu/settings/settings_menu.tscn
index c41a242..7e45261 100644
--- a/content/ui/menu/settings/settings_menu.tscn
+++ b/content/ui/menu/settings/settings_menu.tscn
@@ -131,3 +131,18 @@ outline_size = 0
 horizontal_alignment = 0
 autowrap_mode = 3
 width = 150.0
+
+[node name="VoiceAssist" parent="Content" instance=ExtResource("1_faxng")]
+transform = Transform3D(1, 0, 0, 0, 1, 0, 0, 0, 1, 0.1, 0, 0.12)
+label = "mic_off"
+icon = true
+toggleable = true
+
+[node name="LabelVoiceAssist" type="Label3D" parent="Content"]
+transform = Transform3D(1, 0, 0, 0, -4.37114e-08, 1, 0, -1, -4.37114e-08, 0.01, 0, 0.12)
+pixel_size = 0.001
+text = "Voice-
+Assist:"
+font_size = 18
+outline_size = 0
+horizontal_alignment = 0
diff --git a/lib/home_apis/hass_ws/handlers/assist.gd b/lib/home_apis/hass_ws/handlers/assist.gd
index a7fac54..f90ef35 100644
--- a/lib/home_apis/hass_ws/handlers/assist.gd
+++ b/lib/home_apis/hass_ws/handlers/assist.gd
@@ -4,6 +4,7 @@ signal on_wake_word(wake_word: String)
 signal on_stt_message(message: String)
 signal on_tts_message(message: String)
 signal on_tts_sound(sound: AudioStreamMP3)
+signal on_error()
 
 var api: HASS_API
 var pipe_running := false
@@ -127,7 +128,9 @@ func handle_message(message: Dictionary):
 			sound.data = response[3]
 
 			tts_sound = sound
-		
+		"error":
+			if event["data"]["code"] == "stt-no-text-recognized":
+				on_error.emit()
 		"run-end":
 			pipe_running = false
 			wake_word = null
diff --git a/lib/stores/settings.gd b/lib/stores/settings.gd
index e2fc1ee..d277210 100644
--- a/lib/stores/settings.gd
+++ b/lib/stores/settings.gd
@@ -1,11 +1,11 @@
 extends StoreClass
 
-const StoreClass = preload("./store.gd")
-
+const StoreClass = preload ("./store.gd")
 
 var type: String = "HASS_WS"
 var url: String = ""
 var token: String = ""
+var voice_assistant: bool = false
 
 func _init():
 	_save_path = "user://settings.json"
@@ -13,4 +13,5 @@ func _init():
 func clear():
 	type = "HASS_WS"
 	url = ""
-	token = ""
\ No newline at end of file
+	token = ""
+	voice_assistant = false
\ No newline at end of file
diff --git a/lib/stores/store.gd b/lib/stores/store.gd
index 2a68ba8..f2ac103 100644
--- a/lib/stores/store.gd
+++ b/lib/stores/store.gd
@@ -1,6 +1,6 @@
 extends RefCounted
 
-const VariantSerializer = preload("res://lib/utils/variant_serializer.gd")
+const VariantSerializer = preload ("res://lib/utils/variant_serializer.gd")
 
 signal on_loaded
 signal on_saved
@@ -18,7 +18,7 @@ func create_dict():
 	var data: Dictionary = {}
 
 	for prop_info in get_property_list():
-		if prop_info.name.begins_with("_") || prop_info.hint_string != "":
+		if prop_info.name.begins_with("_")||prop_info.hint_string != "":
 			continue
 
 		var prop = get(prop_info.name)
@@ -32,10 +32,14 @@ func create_dict():
 
 func use_dict(dict: Dictionary):
 	for prop_info in get_property_list():
-		if prop_info.name.begins_with("_") || prop_info.hint_string != "":
+		if prop_info.name.begins_with("_")||prop_info.hint_string != "":
 			continue
 
 		var prop = get(prop_info.name)
+
+		if dict.has(prop_info.name) == false:
+			continue
+
 		var prop_value = dict[prop_info.name]
 
 		if prop is Store:
@@ -43,7 +47,7 @@ func use_dict(dict: Dictionary):
 		else:
 			set(prop_info.name, prop_value)
 
-func save_local(path = _save_path):
+func save_local(path=_save_path):
 	if path == null:
 		return false
 
@@ -61,7 +65,7 @@ func save_local(path = _save_path):
 
 	return true
 
-func load_local(path = _save_path):
+func load_local(path=_save_path):
 	if path == null:
 		return false