chenjintao/lucky-cat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(voice): add local VAD speech endpoint detection

Browse Source
This commit is contained in:
chenjintao
2026-10-23 10:59:26 +08:00
parent ee32b4d63f
commit f17ce857ff
17 changed files with 1183 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/control_plane/src/multimodal/control_plane_multimodal_runtime.c
+4 -1
View File
@@ -682,7 +682,10 @@ esp_err_t control_plane_multimodal_handle_ptt_press_step(void)
esp_err_t control_plane_multimodal_handle_ptt_release(void)
{
control_plane_multimodal_stop_uplink();
esp_err_t err = control_plane_multimodal_stop_uplink();
if (err != ESP_OK) {
return err;
}
return control_plane_multimodal_send_speech_stop();
}
components/control_plane/src/multimodal/control_plane_multimodal_session_service.c
+6 -4
View File
@@ -2356,6 +2356,9 @@ esp_err_t control_plane_multimodal_send_speech_stop(void)
esp_err_t err = ESP_OK;
if (transport_available() && has_active_task) {
err = send_directive("continue-task", "StopSpeech", true, false);
if (err == ESP_OK) {
ESP_LOGI(TAG, "StopSpeech sent after uplink drain");
}
}
if (lock_state()) {
@@ -2678,15 +2681,14 @@ esp_err_t control_plane_multimodal_start_uplink(void)
return domain_runtime_multimodal_audio_start_uplink(control_plane_audio_uplink_send_cb, NULL);
}
void control_plane_multimodal_stop_uplink(void)
esp_err_t control_plane_multimodal_stop_uplink(void)
{
if (!domain_runtime_multimodal_audio_is_uplink_running()) {
return;
}
/* 即便采集已暂停，上次 stop 超时的队列/在途发送仍需要重试排空。 */
esp_err_t err = domain_runtime_multimodal_audio_stop_uplink(CONTROL_PLANE_UPLINK_READY_WAIT_MS);
if (err != ESP_OK) {
ESP_LOGW(TAG, "Mic uplink stop ended with %s", esp_err_to_name(err));
}
return err;
}
bool control_plane_multimodal_is_uplink_running(void)
components/control_plane/src/runtime/control_plane_runtime_workers.c
+56 -2
View File
@@ -244,6 +244,7 @@ typedef struct {
bool pending_press_after_teardown;
bool start_action_sent;
bool speech_action_sent;
bool endpoint_start_attempted;
bool listening_ui_active;
domain_expression_trigger_t listening_ui_trigger;
} ptt_runtime_t;
@@ -754,6 +755,7 @@ static void control_plane_set_ptt_active_with_trigger(
s_ptt_runtime.foreground_attempts = 0;
s_ptt_runtime.start_action_sent = false;
s_ptt_runtime.speech_action_sent = false;
s_ptt_runtime.endpoint_start_attempted = false;
s_ptt_runtime.listening_ui_active = true;
s_ptt_runtime.listening_ui_trigger = listening_ui_trigger;
if (s_ptt_runtime.state == PTT_STATE_TEARDOWN_LOCAL ||
@@ -1792,8 +1794,18 @@ static void control_plane_handle_ptt_teardown(void)
{
ptt_runtime_t ptt = control_plane_ptt_snapshot();
if (ptt.state == PTT_STATE_TEARDOWN_LOCAL) {
control_plane_multimodal_stop_uplink();
control_plane_ptt_set_state(PTT_STATE_TEARDOWN_REMOTE, "uplink_stopped");
esp_err_t endpoint_err = domain_speech_endpoint_stop();
esp_err_t stop_err = endpoint_err == ESP_OK
? control_plane_multimodal_stop_uplink() : endpoint_err;
if (stop_err != ESP_OK) {
/* 有界失败：废弃整轮并关闭连接，不无限重试或宣告输入成功结束。 */
ESP_LOGW(TAG, "Abort voice turn after incomplete drain: %s", esp_err_to_name(stop_err));
control_plane_multimodal_mark_transport_unavailable(
CONTROL_PLANE_TRANSPORT_LOSS_PROTOCOL_ERROR, "uplink_drain_failed");
control_plane_ptt_set_state(PTT_STATE_TEARDOWN_REMOTE, "uplink_aborted");
} else {
control_plane_ptt_set_state(PTT_STATE_TEARDOWN_REMOTE, "uplink_stopped");
}
ptt = control_plane_ptt_snapshot();
}
@@ -1833,6 +1845,7 @@ static void control_plane_handle_ptt_teardown(void)
s_ptt_runtime.press_tick = xTaskGetTickCount();
s_ptt_runtime.start_action_sent = false;
s_ptt_runtime.speech_action_sent = false;
s_ptt_runtime.endpoint_start_attempted = false;
control_plane_ptt_transition_locked(PTT_STATE_WAIT_WIFI, "pending_press");
}
generation = s_ptt_runtime.generation;
@@ -1851,6 +1864,7 @@ static bool control_plane_service_ptt_lifecycle(void)
{
ptt_runtime_t ptt = control_plane_ptt_snapshot();
if (ptt.state == PTT_STATE_IDLE) {
(void)domain_speech_endpoint_stop();
if (ptt.active || ptt.listening_ui_active) {
ESP_LOGW(TAG,
"Recover inconsistent PTT idle state: generation=%u active=%d listening_ui=%d",
@@ -1878,6 +1892,20 @@ static bool control_plane_service_ptt_lifecycle(void)
TickType_t now = xTaskGetTickCount();
bool local_wake_prompt_gate = control_plane_local_wake_prompt_gate_active();
#if CONFIG_TQ_SPEECH_ENDPOINT_ENABLE
if (!local_wake_prompt_gate && ptt.state == PTT_STATE_STREAMING) {
domain_speech_endpoint_snapshot_t endpoint = {0};
if (domain_speech_endpoint_get_snapshot(&endpoint) == ESP_OK &&
endpoint.active && endpoint.ended && endpoint.generation == ptt.generation) {
ESP_LOGI(TAG, "VAD endpoint: generation=%u audio_ms=%u silence_ms=%u max_process_us=%u gaps=%u",
(unsigned)endpoint.generation, (unsigned)endpoint.audio_ms,
(unsigned)endpoint.silence_ms, (unsigned)endpoint.max_process_us,
(unsigned)endpoint.discontinuities);
(void)control_plane_deactivate_ptt(true, ptt.generation, "vad_endpoint");
return true;
}
}
#endif
#if CONFIG_TQ_PTT_MODE_CLICK_TO_TALK
TickType_t capture_timeout_ticks =
pdMS_TO_TICKS(CONFIG_TQ_PTT_CLICK_CAPTURE_TIMEOUT_MS);
@@ -2037,6 +2065,30 @@ static void control_plane_drive_ptt_state_machine(TickType_t *last_ws_start_atte
esp_err_t press_err = control_plane_multimodal_handle_ptt_press_step();
if (press_err == ESP_OK) {
#if CONFIG_TQ_SPEECH_ENDPOINT_ENABLE
/* 开麦成功后才启用，不把提示音、联网等待或本地唤醒采集当成用户输入。 */
bool arm_endpoint = false;
portENTER_CRITICAL(&s_runtime_lock);
if (s_ptt_runtime.active && s_ptt_runtime.generation == ptt.generation &&
!s_ptt_runtime.endpoint_start_attempted &&
s_ptt_runtime.state == PTT_STATE_STREAMING) {
s_ptt_runtime.endpoint_start_attempted = true;
arm_endpoint = true;
}
portEXIT_CRITICAL(&s_runtime_lock);
if (arm_endpoint) {
const domain_speech_endpoint_config_t config = {
.generation = ptt.generation,
.min_speech_ms = CONFIG_TQ_SPEECH_ENDPOINT_MIN_SPEECH_MS,
.end_silence_ms = CONFIG_TQ_SPEECH_ENDPOINT_SILENCE_MS,
};
esp_err_t endpoint_err = domain_speech_endpoint_start(&config);
if (endpoint_err != ESP_OK) {
ESP_LOGW(TAG, "VAD unavailable; retain capture timeout: %s",
esp_err_to_name(endpoint_err));
}
}
#endif
portENTER_CRITICAL(&s_runtime_lock);
if (s_ptt_runtime.state == PTT_STATE_WAIT_START_SENT) {
s_ptt_runtime.start_action_sent = true;
@@ -2345,6 +2397,7 @@ static void control_plane_wifi_state_task(void *arg)
was_connected = connected;
}
(void)domain_speech_endpoint_stop();
control_plane_stop_user_config_server();
control_plane_set_task_handle(&s_wifi_state_task, NULL);
return;
@@ -2852,6 +2905,7 @@ void control_plane_stop_runtime_workers(void)
if (!control_plane_wait_task_exit(&s_wifi_state_task, CONTROL_PLANE_TASK_STOP_WAIT_MS)) {
control_plane_force_delete_task(&s_wifi_state_task, "cp_wifi_state");
}
(void)domain_speech_endpoint_stop();
#if CONFIG_TQ_TASK_MONITOR_ENABLE
if (!control_plane_wait_task_exit(&s_task_monitor_task, CONTROL_PLANE_TASK_STOP_WAIT_MS)) {
control_plane_force_delete_task_with_caps(&s_task_monitor_task, "cp_task_monitor");