chore: initial commit
This commit is contained in:
CSBOOK\44744
@@ -0,0 +1,100 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include "control_plane_internal.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "cJSON.h"
|
||||
#include "esp_log.h"
|
||||
|
||||
static const char *TAG = "cp_model_cmd";
|
||||
|
||||
static bool is_supported_model_direction(const char *value)
|
||||
{
|
||||
if (value == NULL || value[0] == '\0') {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < domain_get_direction_count(); i++) {
|
||||
const char *name_cn = NULL;
|
||||
if (domain_get_direction_view(i, &name_cn, NULL, NULL) &&
|
||||
name_cn != NULL &&
|
||||
strcmp(value, name_cn) == 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool control_plane_extract_model_command(const char *text,
|
||||
control_plane_model_command_type_t *out_command_type,
|
||||
char *out_value,
|
||||
size_t out_value_capacity)
|
||||
{
|
||||
if (text == NULL || out_command_type == NULL ||
|
||||
out_value == NULL || out_value_capacity == 0U) {
|
||||
return false;
|
||||
}
|
||||
|
||||
*out_command_type = CONTROL_PLANE_MODEL_COMMAND_UNKNOWN;
|
||||
out_value[0] = '\0';
|
||||
|
||||
if (text[0] != '/' || text[1] != '*') {
|
||||
return false;
|
||||
}
|
||||
|
||||
const char *json_start = text + 2;
|
||||
const char *json_end = strstr(json_start, "*/");
|
||||
if (json_end == NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
size_t json_len = (size_t)(json_end - json_start);
|
||||
if (json_len == 0U || json_len > 128U) {
|
||||
return false;
|
||||
}
|
||||
|
||||
char json_buf[128U + 4U];
|
||||
memcpy(json_buf, json_start, json_len);
|
||||
json_buf[json_len] = '\0';
|
||||
|
||||
cJSON *cmd_json = cJSON_Parse(json_buf);
|
||||
if (cmd_json == NULL) {
|
||||
ESP_LOGW(TAG, "Failed to parse command JSON: %s", json_buf);
|
||||
return false;
|
||||
}
|
||||
if (!cJSON_IsObject(cmd_json)) {
|
||||
cJSON_Delete(cmd_json);
|
||||
return false;
|
||||
}
|
||||
if (cJSON_GetArraySize(cmd_json) != 1) {
|
||||
cJSON_Delete(cmd_json);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool found = false;
|
||||
cJSON *direction = cJSON_GetObjectItemCaseSensitive(cmd_json, "转向");
|
||||
cJSON *calibrate = cJSON_GetObjectItemCaseSensitive(cmd_json, "校准");
|
||||
|
||||
if (cJSON_IsString(direction) && direction->valuestring != NULL) {
|
||||
if (is_supported_model_direction(direction->valuestring)) {
|
||||
*out_command_type = CONTROL_PLANE_MODEL_COMMAND_DIRECTION;
|
||||
strlcpy(out_value, direction->valuestring, out_value_capacity);
|
||||
found = true;
|
||||
}
|
||||
} else if (cJSON_IsString(calibrate) && calibrate->valuestring != NULL) {
|
||||
if (strcmp(calibrate->valuestring, "位置") == 0) {
|
||||
*out_command_type = CONTROL_PLANE_MODEL_COMMAND_CALIBRATE;
|
||||
strlcpy(out_value, calibrate->valuestring, out_value_capacity);
|
||||
found = true;
|
||||
}
|
||||
}
|
||||
|
||||
cJSON_Delete(cmd_json);
|
||||
return found;
|
||||
}
|
||||
@@ -0,0 +1,490 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include "multimodal/control_plane_multimodal_auth.h"
|
||||
#include "multimodal/control_plane_multimodal_device_auth.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "cJSON.h"
|
||||
#include "esp_err.h"
|
||||
#include "esp_heap_caps.h"
|
||||
#include "esp_log.h"
|
||||
#include "domain.h"
|
||||
#include "sdkconfig.h"
|
||||
#include "soc/soc_caps.h"
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_DEVICE_ID
|
||||
#define CONFIG_MULTIMODAL_DEVICE_ID ""
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_SERIAL_NUMBER
|
||||
#define CONFIG_MULTIMODAL_SERIAL_NUMBER ""
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_DEVICE_TOKEN_URL
|
||||
#define CONFIG_MULTIMODAL_DEVICE_TOKEN_URL "https://gz.api.talkingq.com/token/api/device/token"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_USE_STATIC_CREDENTIALS
|
||||
#define CONFIG_MULTIMODAL_USE_STATIC_CREDENTIALS 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUTH_BYPASS_TOKEN_SERVICE
|
||||
#define CONFIG_MULTIMODAL_AUTH_BYPASS_TOKEN_SERVICE 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_API_KEY
|
||||
#define CONFIG_MULTIMODAL_API_KEY ""
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_WORKSPACE_ID
|
||||
#define CONFIG_MULTIMODAL_WORKSPACE_ID ""
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_APP_ID
|
||||
#define CONFIG_MULTIMODAL_APP_ID ""
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_VOICE_ID
|
||||
#define CONFIG_MULTIMODAL_VOICE_ID ""
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_CONNECT_TIMEOUT_MS
|
||||
#define CONFIG_MULTIMODAL_CONNECT_TIMEOUT_MS 10000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_AUTH_V2_ENABLE
|
||||
#define CONFIG_AUTH_V2_ENABLE 0
|
||||
#endif
|
||||
|
||||
#define CONTROL_PLANE_MULTIMODAL_AUTH_USE_STATIC_CREDENTIALS \
|
||||
(CONFIG_MULTIMODAL_USE_STATIC_CREDENTIALS || CONFIG_MULTIMODAL_AUTH_BYPASS_TOKEN_SERVICE)
|
||||
|
||||
static const char *TAG = "cp_mm_auth";
|
||||
#if !CONTROL_PLANE_MULTIMODAL_AUTH_USE_STATIC_CREDENTIALS
|
||||
static const int kMaxHttpBody = 4096;
|
||||
#endif
|
||||
|
||||
#if !CONTROL_PLANE_MULTIMODAL_AUTH_USE_STATIC_CREDENTIALS
|
||||
static void *auth_psram_first_calloc(size_t count, size_t size)
|
||||
{
|
||||
if (count == 0U || size == 0U || count > (SIZE_MAX / size)) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#if SOC_SPIRAM_SUPPORTED
|
||||
void *ptr = heap_caps_calloc(count, size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
|
||||
if (ptr != NULL) {
|
||||
return ptr;
|
||||
}
|
||||
#endif
|
||||
|
||||
return heap_caps_calloc(count, size, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
|
||||
}
|
||||
#endif
|
||||
|
||||
static esp_err_t resolve_device_identity(char *device_id,
|
||||
size_t device_len,
|
||||
char *serial_number,
|
||||
size_t serial_len)
|
||||
{
|
||||
if (device_id == NULL || serial_number == NULL || device_len == 0U || serial_len == 0U) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
|
||||
device_id[0] = '\0';
|
||||
serial_number[0] = '\0';
|
||||
|
||||
domain_runtime_read_device_identity(device_id, device_len, serial_number, serial_len);
|
||||
|
||||
if (device_id[0] == '\0' && CONFIG_MULTIMODAL_DEVICE_ID[0] != '\0') {
|
||||
strlcpy(device_id, CONFIG_MULTIMODAL_DEVICE_ID, device_len);
|
||||
}
|
||||
if (serial_number[0] == '\0' && CONFIG_MULTIMODAL_SERIAL_NUMBER[0] != '\0') {
|
||||
strlcpy(serial_number, CONFIG_MULTIMODAL_SERIAL_NUMBER, serial_len);
|
||||
}
|
||||
|
||||
return (device_id[0] != '\0' || serial_number[0] != '\0') ? ESP_OK : ESP_ERR_NOT_FOUND;
|
||||
}
|
||||
|
||||
#if !CONTROL_PLANE_MULTIMODAL_AUTH_USE_STATIC_CREDENTIALS
|
||||
|
||||
static bool copy_json_string(const cJSON *obj, const char *key, char *out, size_t out_len)
|
||||
{
|
||||
if (obj == NULL || key == NULL || out == NULL || out_len == 0U) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, key);
|
||||
if (!cJSON_IsString(item) || item->valuestring == NULL || item->valuestring[0] == '\0') {
|
||||
return false;
|
||||
}
|
||||
|
||||
strlcpy(out, item->valuestring, out_len);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool try_copy_token(const cJSON *obj, char *out, size_t out_len)
|
||||
{
|
||||
if (obj == NULL || out == NULL || out_len == 0U) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (copy_json_string(obj, "token", out, out_len) ||
|
||||
copy_json_string(obj, "accessToken", out, out_len) ||
|
||||
copy_json_string(obj, "access_token", out, out_len)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const cJSON *token_obj = cJSON_GetObjectItemCaseSensitive(obj, "token");
|
||||
if (!cJSON_IsObject(token_obj)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return copy_json_string(token_obj, "token", out, out_len) ||
|
||||
copy_json_string(token_obj, "accessToken", out, out_len) ||
|
||||
copy_json_string(token_obj, "access_token", out, out_len);
|
||||
}
|
||||
|
||||
static const cJSON *pick_payload_root(const cJSON *root)
|
||||
{
|
||||
if (root == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data");
|
||||
if (cJSON_IsObject(data)) {
|
||||
return data;
|
||||
}
|
||||
|
||||
const cJSON *result = cJSON_GetObjectItemCaseSensitive(root, "result");
|
||||
if (cJSON_IsObject(result)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
return root;
|
||||
}
|
||||
|
||||
static esp_err_t parse_token_response(const char *response, control_plane_multimodal_auth_info_t *info)
|
||||
{
|
||||
cJSON *root = cJSON_Parse(response);
|
||||
if (root == NULL) {
|
||||
ESP_LOGE(TAG, "Token response JSON parse failed");
|
||||
return ESP_ERR_INVALID_RESPONSE;
|
||||
}
|
||||
|
||||
const cJSON *payload = pick_payload_root(root);
|
||||
bool ok = try_copy_token(payload, info->token, sizeof(info->token));
|
||||
if (!ok && payload != root) {
|
||||
ok = try_copy_token(root, info->token, sizeof(info->token));
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
const cJSON *data_node = cJSON_GetObjectItemCaseSensitive(root, "data");
|
||||
if (cJSON_IsString(data_node) &&
|
||||
data_node->valuestring != NULL &&
|
||||
data_node->valuestring[0] != '\0') {
|
||||
strlcpy(info->token, data_node->valuestring, sizeof(info->token));
|
||||
ok = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
const cJSON *result_node = cJSON_GetObjectItemCaseSensitive(root, "result");
|
||||
if (cJSON_IsString(result_node) &&
|
||||
result_node->valuestring != NULL &&
|
||||
result_node->valuestring[0] != '\0') {
|
||||
strlcpy(info->token, result_node->valuestring, sizeof(info->token));
|
||||
ok = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
ESP_LOGE(TAG, "Token missing in response: %.200s", response);
|
||||
cJSON_Delete(root);
|
||||
return ESP_ERR_INVALID_RESPONSE;
|
||||
}
|
||||
|
||||
(void)copy_json_string(payload, "workspaceId", info->workspace_id, sizeof(info->workspace_id));
|
||||
(void)copy_json_string(payload, "workspace_id", info->workspace_id, sizeof(info->workspace_id));
|
||||
(void)copy_json_string(payload, "appId", info->app_id, sizeof(info->app_id));
|
||||
(void)copy_json_string(payload, "app_id", info->app_id, sizeof(info->app_id));
|
||||
(void)copy_json_string(payload, "voiceId", info->voice_id, sizeof(info->voice_id));
|
||||
(void)copy_json_string(payload, "voice_id", info->voice_id, sizeof(info->voice_id));
|
||||
|
||||
cJSON_Delete(root);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
/* Snapshot internal/DMA/PSRAM heap right before a TLS request. The token POST runs
|
||||
* through hardware AES, which needs a DMA-capable internal bounce buffer per record;
|
||||
* logging the largest internal/DMA block here exposes how much headroom exists at the
|
||||
* exact point "esp-aes: Failed to allocate memory" is reported, and whether it trends
|
||||
* downward across retries (leak) or stays flat-low (steady-state pressure). */
|
||||
static void auth_log_heap(const char *phase)
|
||||
{
|
||||
size_t free_internal = heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
|
||||
size_t largest_internal = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
|
||||
size_t largest_dma = heap_caps_get_largest_free_block(MALLOC_CAP_DMA);
|
||||
#if SOC_SPIRAM_SUPPORTED
|
||||
size_t free_spiram = heap_caps_get_free_size(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
|
||||
ESP_LOGI(TAG,
|
||||
"Heap %s: free_internal=%u largest_internal=%u largest_dma=%u free_spiram=%u",
|
||||
phase ? phase : "",
|
||||
(unsigned)free_internal,
|
||||
(unsigned)largest_internal,
|
||||
(unsigned)largest_dma,
|
||||
(unsigned)free_spiram);
|
||||
#else
|
||||
ESP_LOGI(TAG,
|
||||
"Heap %s: free_internal=%u largest_internal=%u largest_dma=%u",
|
||||
phase ? phase : "",
|
||||
(unsigned)free_internal,
|
||||
(unsigned)largest_internal,
|
||||
(unsigned)largest_dma);
|
||||
#endif
|
||||
}
|
||||
|
||||
static esp_err_t http_post_json(const char *url,
|
||||
const char *payload,
|
||||
const char *device_id,
|
||||
const char *bearer_token,
|
||||
char **out_body,
|
||||
int *out_status)
|
||||
{
|
||||
if (out_status != NULL) {
|
||||
*out_status = -1;
|
||||
}
|
||||
if (url == NULL || url[0] == '\0' || payload == NULL || out_body == NULL) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
|
||||
auth_log_heap("before token POST");
|
||||
|
||||
char *body = (char *)auth_psram_first_calloc(1, (size_t)kMaxHttpBody + 1U);
|
||||
if (body == NULL) {
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
|
||||
size_t body_len = 0U;
|
||||
int status = -1;
|
||||
esp_err_t err = domain_runtime_post_json(url,
|
||||
payload,
|
||||
device_id,
|
||||
bearer_token,
|
||||
CONFIG_MULTIMODAL_CONNECT_TIMEOUT_MS,
|
||||
2048,
|
||||
1024,
|
||||
body,
|
||||
(size_t)kMaxHttpBody + 1U,
|
||||
&body_len,
|
||||
&status);
|
||||
|
||||
if (err != ESP_OK) {
|
||||
heap_caps_free(body);
|
||||
ESP_LOGE(TAG, "HTTP POST failed: %s", esp_err_to_name(err));
|
||||
return err;
|
||||
}
|
||||
|
||||
if (out_status != NULL) {
|
||||
*out_status = status;
|
||||
}
|
||||
|
||||
if (body_len == 0U) {
|
||||
heap_caps_free(body);
|
||||
ESP_LOGE(TAG, "HTTP response empty (status=%d)", status);
|
||||
return ESP_ERR_INVALID_RESPONSE;
|
||||
}
|
||||
|
||||
if (status < 200 || status >= 300) {
|
||||
ESP_LOGE(TAG, "Token request failed (status=%d): %.80s", status, body);
|
||||
heap_caps_free(body);
|
||||
return ESP_FAIL;
|
||||
}
|
||||
|
||||
*out_body = body;
|
||||
return ESP_OK;
|
||||
}
|
||||
#endif
|
||||
|
||||
esp_err_t control_plane_multimodal_auth_fetch(control_plane_multimodal_auth_info_t *out)
|
||||
{
|
||||
#if CONTROL_PLANE_MULTIMODAL_AUTH_USE_STATIC_CREDENTIALS
|
||||
if (CONFIG_MULTIMODAL_API_KEY[0] == '\0') {
|
||||
ESP_LOGE(TAG, "CONFIG_MULTIMODAL_API_KEY is empty");
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
if (CONFIG_MULTIMODAL_WORKSPACE_ID[0] == '\0') {
|
||||
ESP_LOGE(TAG, "CONFIG_MULTIMODAL_WORKSPACE_ID is empty");
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
control_plane_multimodal_auth_info_t info = {0};
|
||||
esp_err_t id_err = resolve_device_identity(info.device_id,
|
||||
sizeof(info.device_id),
|
||||
info.serial_number,
|
||||
sizeof(info.serial_number));
|
||||
if (id_err != ESP_OK) {
|
||||
ESP_LOGE(TAG, "Device identity unavailable for static credentials: %s", esp_err_to_name(id_err));
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
strlcpy(info.token, CONFIG_MULTIMODAL_API_KEY, sizeof(info.token));
|
||||
strlcpy(info.workspace_id, CONFIG_MULTIMODAL_WORKSPACE_ID, sizeof(info.workspace_id));
|
||||
if (CONFIG_MULTIMODAL_APP_ID[0] != '\0') {
|
||||
strlcpy(info.app_id, CONFIG_MULTIMODAL_APP_ID, sizeof(info.app_id));
|
||||
}
|
||||
if (CONFIG_MULTIMODAL_VOICE_ID[0] != '\0') {
|
||||
strlcpy(info.voice_id, CONFIG_MULTIMODAL_VOICE_ID, sizeof(info.voice_id));
|
||||
}
|
||||
|
||||
info.valid = true;
|
||||
if (out != NULL) {
|
||||
*out = info;
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG,
|
||||
"Using static multimodal credentials (workspace=%s, app=%s, bypass_token=%d)",
|
||||
info.workspace_id,
|
||||
info.app_id[0] == '\0' ? "<empty>" : info.app_id,
|
||||
CONFIG_MULTIMODAL_AUTH_BYPASS_TOKEN_SERVICE ? 1 : 0);
|
||||
return ESP_OK;
|
||||
#else
|
||||
if (CONFIG_MULTIMODAL_DEVICE_TOKEN_URL[0] == '\0') {
|
||||
ESP_LOGE(TAG, "CONFIG_MULTIMODAL_DEVICE_TOKEN_URL is empty");
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
control_plane_multimodal_auth_info_t info = {0};
|
||||
esp_err_t id_err = resolve_device_identity(info.device_id,
|
||||
sizeof(info.device_id),
|
||||
info.serial_number,
|
||||
sizeof(info.serial_number));
|
||||
if (id_err != ESP_OK) {
|
||||
ESP_LOGE(TAG, "Device identity unavailable; provision device_id or serial_number before token request");
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
cJSON *root = cJSON_CreateObject();
|
||||
if (root == NULL) {
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
if (info.device_id[0] != '\0') {
|
||||
cJSON_AddStringToObject(root, "deviceId", info.device_id);
|
||||
}
|
||||
if (info.serial_number[0] != '\0') {
|
||||
cJSON_AddStringToObject(root, "serialNumber", info.serial_number);
|
||||
}
|
||||
char *payload = cJSON_PrintUnformatted(root);
|
||||
cJSON_Delete(root);
|
||||
if (payload == NULL) {
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
|
||||
char *response = NULL;
|
||||
ESP_LOGI(TAG, "Request device token: %s", CONFIG_MULTIMODAL_DEVICE_TOKEN_URL);
|
||||
|
||||
/*
|
||||
* AUTH_V2: authenticate the token request with the device JWT (minted via the
|
||||
* challenge/response engine). On a 401/403 the cached JWT is invalidated and
|
||||
* the request is retried once with a forced refresh. When AUTH_V2 is off the
|
||||
* loop runs a single unauthenticated request, preserving prior behavior.
|
||||
*/
|
||||
esp_err_t err = ESP_FAIL;
|
||||
for (int attempt = 0; attempt < 2; ++attempt) {
|
||||
const char *bearer = NULL;
|
||||
#if CONFIG_AUTH_V2_ENABLE
|
||||
char *device_jwt = (char *)auth_psram_first_calloc(1, CONTROL_PLANE_DEVICE_AUTH_JWT_MAX_LEN);
|
||||
if (device_jwt == NULL) {
|
||||
free(payload);
|
||||
if (response != NULL) {
|
||||
heap_caps_free(response);
|
||||
}
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
control_plane_device_auth_result_t auth_res =
|
||||
control_plane_multimodal_device_auth_ensure(
|
||||
CONTROL_PLANE_DEVICE_AUTH_SCOPE_VOICE_SESSION, attempt > 0);
|
||||
if (auth_res != CONTROL_PLANE_DEVICE_AUTH_OK) {
|
||||
ESP_LOGW(TAG, "Device JWT not ready (result=%d); cannot request token", (int)auth_res);
|
||||
heap_caps_free(device_jwt);
|
||||
free(payload);
|
||||
if (response != NULL) {
|
||||
heap_caps_free(response);
|
||||
}
|
||||
return (auth_res == CONTROL_PLANE_DEVICE_AUTH_RETRY_LATER) ? ESP_ERR_NOT_FINISHED
|
||||
: ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
esp_err_t jwt_err = control_plane_multimodal_device_auth_copy_jwt(
|
||||
device_jwt,
|
||||
CONTROL_PLANE_DEVICE_AUTH_JWT_MAX_LEN);
|
||||
if (jwt_err != ESP_OK || device_jwt[0] == '\0') {
|
||||
ESP_LOGW(TAG, "Device JWT unavailable after ensure (%s)", esp_err_to_name(jwt_err));
|
||||
heap_caps_free(device_jwt);
|
||||
free(payload);
|
||||
if (response != NULL) {
|
||||
heap_caps_free(response);
|
||||
}
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
bearer = device_jwt;
|
||||
#endif
|
||||
|
||||
if (response != NULL) {
|
||||
heap_caps_free(response);
|
||||
response = NULL;
|
||||
}
|
||||
|
||||
int status = -1;
|
||||
err = http_post_json(CONFIG_MULTIMODAL_DEVICE_TOKEN_URL,
|
||||
payload,
|
||||
info.device_id,
|
||||
bearer,
|
||||
&response,
|
||||
&status);
|
||||
|
||||
#if CONFIG_AUTH_V2_ENABLE
|
||||
heap_caps_free(device_jwt);
|
||||
bearer = NULL;
|
||||
if ((status == 401 || status == 403) && attempt == 0) {
|
||||
ESP_LOGW(TAG, "Token endpoint denied auth (status=%d); refreshing device JWT and retrying", status);
|
||||
control_plane_multimodal_device_auth_invalidate(status);
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
break;
|
||||
}
|
||||
|
||||
free(payload);
|
||||
if (err != ESP_OK) {
|
||||
if (response != NULL) {
|
||||
heap_caps_free(response);
|
||||
}
|
||||
return err;
|
||||
}
|
||||
|
||||
err = parse_token_response(response, &info);
|
||||
heap_caps_free(response);
|
||||
if (err != ESP_OK) {
|
||||
return err;
|
||||
}
|
||||
|
||||
info.valid = true;
|
||||
if (out != NULL) {
|
||||
*out = info;
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG, "Token fetched (workspace=%s, app=%s)", info.workspace_id, info.app_id);
|
||||
return ESP_OK;
|
||||
#endif
|
||||
}
|
||||
|
||||
void control_plane_multimodal_auth_clear(void)
|
||||
{
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,124 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include "control_plane_internal.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "esp_log.h"
|
||||
#include "esp_random.h"
|
||||
#include "sdkconfig.h"
|
||||
|
||||
static const char *TAG = "cp_mm_proto";
|
||||
|
||||
const char *control_plane_multimodal_resolve_upstream_mode(void)
|
||||
{
|
||||
#ifdef CONFIG_MULTIMODAL_UPSTREAM_MODE
|
||||
if (CONFIG_MULTIMODAL_UPSTREAM_MODE[0] != '\0' &&
|
||||
strcmp(CONFIG_MULTIMODAL_UPSTREAM_MODE, "push2talk") != 0) {
|
||||
ESP_LOGW(TAG,
|
||||
"Ignoring unsupported upstream mode '%s'; this firmware is push2talk-only",
|
||||
CONFIG_MULTIMODAL_UPSTREAM_MODE);
|
||||
}
|
||||
#endif
|
||||
return "push2talk";
|
||||
}
|
||||
|
||||
bool control_plane_multimodal_is_push2talk_mode(void)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
const char *control_plane_multimodal_resolve_voice_id(const char *auth_voice_id)
|
||||
{
|
||||
if (auth_voice_id != NULL && auth_voice_id[0] != '\0') {
|
||||
if (strcmp(auth_voice_id, "Kenny") == 0) {
|
||||
ESP_LOGW(TAG, "Skip placeholder auth voice Kenny, use app default voice");
|
||||
} else {
|
||||
return auth_voice_id;
|
||||
}
|
||||
}
|
||||
|
||||
if (CONFIG_MULTIMODAL_TTS_VOICE[0] != '\0') {
|
||||
if (strcmp(CONFIG_MULTIMODAL_TTS_VOICE, "Kenny") == 0) {
|
||||
ESP_LOGW(TAG, "Skip placeholder voice Kenny, use app default voice");
|
||||
return NULL;
|
||||
}
|
||||
return CONFIG_MULTIMODAL_TTS_VOICE;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
uint32_t control_plane_multimodal_resolve_downstream_transmit_rate_limit(uint32_t configured_limit)
|
||||
{
|
||||
uint32_t bitrate_bytes_per_sec = ((uint32_t)CONFIG_MULTIMODAL_OPUS_BITRATE_KBPS * 1000U) / 8U;
|
||||
if (bitrate_bytes_per_sec == 0U) {
|
||||
return configured_limit;
|
||||
}
|
||||
|
||||
if (configured_limit == 0U) {
|
||||
return bitrate_bytes_per_sec;
|
||||
}
|
||||
|
||||
if (configured_limit != bitrate_bytes_per_sec) {
|
||||
static bool warned_once;
|
||||
if (!warned_once) {
|
||||
ESP_LOGW(TAG,
|
||||
"Clamp downstream transmit_rate_limit from %uB/s to %uB/s to match %ukbps opus playback budget",
|
||||
(unsigned)configured_limit,
|
||||
(unsigned)bitrate_bytes_per_sec,
|
||||
(unsigned)CONFIG_MULTIMODAL_OPUS_BITRATE_KBPS);
|
||||
warned_once = true;
|
||||
}
|
||||
return bitrate_bytes_per_sec;
|
||||
}
|
||||
|
||||
return configured_limit;
|
||||
}
|
||||
|
||||
uint32_t control_plane_multimodal_resolve_opus_frame_duration_ms(void)
|
||||
{
|
||||
switch (CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS) {
|
||||
case 10:
|
||||
case 20:
|
||||
case 40:
|
||||
case 60:
|
||||
case 100:
|
||||
case 120:
|
||||
return (uint32_t)CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS;
|
||||
default:
|
||||
ESP_LOGW(TAG,
|
||||
"Invalid opus frame duration %d ms, fallback to 60 ms",
|
||||
CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS);
|
||||
return 60U;
|
||||
}
|
||||
}
|
||||
|
||||
void control_plane_multimodal_generate_uuid(char *out, size_t out_len)
|
||||
{
|
||||
if (out == NULL || out_len < 37U) {
|
||||
return;
|
||||
}
|
||||
|
||||
uint8_t uuid[16];
|
||||
for (size_t i = 0; i < sizeof(uuid); ++i) {
|
||||
uuid[i] = (uint8_t)(esp_random() & 0xFF);
|
||||
}
|
||||
|
||||
uuid[6] = (uint8_t)((uuid[6] & 0x0F) | 0x40);
|
||||
uuid[8] = (uint8_t)((uuid[8] & 0x3F) | 0x80);
|
||||
|
||||
snprintf(out,
|
||||
out_len,
|
||||
"%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x",
|
||||
uuid[0], uuid[1], uuid[2], uuid[3],
|
||||
uuid[4], uuid[5],
|
||||
uuid[6], uuid[7],
|
||||
uuid[8], uuid[9],
|
||||
uuid[10], uuid[11], uuid[12], uuid[13], uuid[14], uuid[15]);
|
||||
}
|
||||
@@ -0,0 +1,694 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include "control_plane.h"
|
||||
|
||||
#include "control_plane_internal.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/task.h"
|
||||
#include "esp_log.h"
|
||||
#include "sdkconfig.h"
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS
|
||||
#define CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS 50000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS
|
||||
#define CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS 60000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_RECONNECT_DELAY_MS
|
||||
#define CONFIG_MULTIMODAL_RECONNECT_DELAY_MS 2000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_UPSTREAM_MODE
|
||||
#define CONFIG_MULTIMODAL_UPSTREAM_MODE "push2talk"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS
|
||||
#define CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS 1
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_MULTIMODAL_AUTO_START_ON_WAKEUP)
|
||||
#define CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG CONFIG_MULTIMODAL_AUTO_START_ON_WAKEUP
|
||||
#elif defined(CONFIG_MULTIMODAL_WS_AUTO_START)
|
||||
#define CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG CONFIG_MULTIMODAL_WS_AUTO_START
|
||||
#else
|
||||
#define CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG 0
|
||||
#endif
|
||||
|
||||
#define CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS 220
|
||||
#define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000
|
||||
#define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE 6144
|
||||
|
||||
static const char *TAG = "control_plane_mm_rt";
|
||||
static TaskHandle_t s_multimodal_watchdog_task;
|
||||
static TaskHandle_t s_multimodal_autostart_task;
|
||||
static bool s_multimodal_watchdog_stop_requested;
|
||||
static bool s_multimodal_autostart_stop_requested;
|
||||
static bool s_multimodal_watchdog_task_starting;
|
||||
static bool s_multimodal_autostart_task_starting;
|
||||
static bool s_model_command_dispatched;
|
||||
static control_plane_model_command_callback_t s_model_command_cb;
|
||||
static void *s_model_command_cb_user_data;
|
||||
static portMUX_TYPE s_multimodal_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||
|
||||
static TaskHandle_t get_multimodal_task_handle(TaskHandle_t *slot)
|
||||
{
|
||||
TaskHandle_t task = NULL;
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
task = *slot;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
return task;
|
||||
}
|
||||
|
||||
static void set_multimodal_task_handle(TaskHandle_t *slot, TaskHandle_t task)
|
||||
{
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
*slot = task;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
}
|
||||
|
||||
static void clear_multimodal_task_handle_if_match(TaskHandle_t *slot, TaskHandle_t expected_task)
|
||||
{
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
if (*slot == expected_task) {
|
||||
*slot = NULL;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
}
|
||||
|
||||
static bool get_multimodal_stop_flag(bool *flag)
|
||||
{
|
||||
bool value = false;
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
value = *flag;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
return value;
|
||||
}
|
||||
|
||||
static void set_multimodal_stop_flag(bool *flag, bool value)
|
||||
{
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
*flag = value;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
}
|
||||
|
||||
static bool get_multimodal_start_flag(bool *flag)
|
||||
{
|
||||
bool value = false;
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
value = *flag;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
return value;
|
||||
}
|
||||
|
||||
static void set_multimodal_start_flag(bool *flag, bool value)
|
||||
{
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
*flag = value;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
}
|
||||
|
||||
static bool try_begin_multimodal_task_start(TaskHandle_t *slot, bool *start_flag)
|
||||
{
|
||||
bool can_start = false;
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
if (*slot == NULL && !*start_flag) {
|
||||
*start_flag = true;
|
||||
can_start = true;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
return can_start;
|
||||
}
|
||||
|
||||
static bool wait_task_exit(TaskHandle_t *task_handle, bool *start_flag, int timeout_ms)
|
||||
{
|
||||
if (task_handle == NULL ||
|
||||
(get_multimodal_task_handle(task_handle) == NULL &&
|
||||
(start_flag == NULL || !get_multimodal_start_flag(start_flag)))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
TickType_t timeout_ticks = pdMS_TO_TICKS(timeout_ms > 0 ? timeout_ms : 1);
|
||||
if (timeout_ticks == 0) {
|
||||
timeout_ticks = 1;
|
||||
}
|
||||
TickType_t start_tick = xTaskGetTickCount();
|
||||
while ((get_multimodal_task_handle(task_handle) != NULL ||
|
||||
(start_flag != NULL && get_multimodal_start_flag(start_flag))) &&
|
||||
(xTaskGetTickCount() - start_tick) < timeout_ticks) {
|
||||
vTaskDelay(pdMS_TO_TICKS(20));
|
||||
}
|
||||
return (get_multimodal_task_handle(task_handle) == NULL &&
|
||||
(start_flag == NULL || !get_multimodal_start_flag(start_flag)));
|
||||
}
|
||||
|
||||
static esp_err_t control_plane_prepare_for_multimodal_start(void);
|
||||
|
||||
static void control_plane_set_model_command_dispatched(bool dispatched)
|
||||
{
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
s_model_command_dispatched = dispatched;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
}
|
||||
|
||||
static void control_plane_set_model_command_callback(control_plane_model_command_callback_t cb,
|
||||
void *user_data)
|
||||
{
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
s_model_command_cb = cb;
|
||||
s_model_command_cb_user_data = user_data;
|
||||
s_model_command_dispatched = false;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
}
|
||||
|
||||
static void control_plane_responding_text_bridge(const char *text,
|
||||
bool finished,
|
||||
void *user_data)
|
||||
{
|
||||
(void)finished;
|
||||
(void)user_data;
|
||||
|
||||
control_plane_model_command_callback_t cb = NULL;
|
||||
void *cb_user_data = NULL;
|
||||
bool dispatched = false;
|
||||
|
||||
if (text == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
cb = s_model_command_cb;
|
||||
cb_user_data = s_model_command_cb_user_data;
|
||||
dispatched = s_model_command_dispatched;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
|
||||
if (cb == NULL || dispatched) {
|
||||
return;
|
||||
}
|
||||
|
||||
control_plane_model_command_type_t command_type = CONTROL_PLANE_MODEL_COMMAND_UNKNOWN;
|
||||
char value[32] = {0};
|
||||
if (!control_plane_extract_model_command(text, &command_type, value, sizeof(value))) {
|
||||
return;
|
||||
}
|
||||
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
if (s_model_command_dispatched) {
|
||||
dispatched = true;
|
||||
} else {
|
||||
s_model_command_dispatched = true;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
|
||||
if (dispatched) {
|
||||
return;
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG, "Device command: type=%d value=%s", (int)command_type, value);
|
||||
cb(command_type, value, cb_user_data);
|
||||
}
|
||||
|
||||
static esp_err_t control_plane_prepare_for_multimodal_start(void)
|
||||
{
|
||||
#if CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS
|
||||
if (!domain_runtime_multimodal_audio_is_initialized() &&
|
||||
!domain_runtime_multimodal_audio_is_uplink_running()) {
|
||||
esp_err_t audio_err = domain_runtime_multimodal_audio_init();
|
||||
if (audio_err != ESP_OK) {
|
||||
ESP_LOGW(TAG, "Audio prewarm before WebSocket connect failed: %s", esp_err_to_name(audio_err));
|
||||
return audio_err;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
static uint32_t control_plane_now_ms(void)
|
||||
{
|
||||
return (uint32_t)(xTaskGetTickCount() * portTICK_PERIOD_MS);
|
||||
}
|
||||
|
||||
static bool control_plane_is_within_downstream_guard(const control_plane_multimodal_runtime_state_t *state,
|
||||
uint32_t guard_ms)
|
||||
{
|
||||
if (state == NULL || state->last_downstream_audio_ms == 0U || guard_ms == 0U) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t now_ms = control_plane_now_ms();
|
||||
return (now_ms - state->last_downstream_audio_ms) < guard_ms;
|
||||
}
|
||||
|
||||
static esp_err_t control_plane_start_uplink_if_ready(void)
|
||||
{
|
||||
control_plane_multimodal_runtime_state_t state = {0};
|
||||
esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state);
|
||||
if (state_err != ESP_OK) {
|
||||
return state_err;
|
||||
}
|
||||
|
||||
if (!state.connected || !state.has_active_task) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
/* DashScope multimodal protocol requires waiting for DialogStateChanged(Listening)
|
||||
* before sending any upstream audio in every upstream mode.
|
||||
*/
|
||||
if (!state.server_listening) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (!state.client_speech_active || state.ptt_release_required) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (state.remote_responding) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (control_plane_is_within_downstream_guard(&state,
|
||||
CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS)) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
return control_plane_multimodal_start_uplink();
|
||||
}
|
||||
|
||||
esp_err_t control_plane_multimodal_handle_ptt_press_step(void)
|
||||
{
|
||||
control_plane_multimodal_runtime_state_t state = {0};
|
||||
esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state);
|
||||
if (state_err != ESP_OK) {
|
||||
return state_err;
|
||||
}
|
||||
|
||||
if (!state.connected) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (!state.has_active_task) {
|
||||
return control_plane_multimodal_send_start();
|
||||
}
|
||||
|
||||
if (state.ptt_release_required) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (state.remote_responding) {
|
||||
if (!state.request_to_speak_pending) {
|
||||
return control_plane_multimodal_send_request_to_speak();
|
||||
}
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (!state.server_listening) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (!state.client_speech_active) {
|
||||
esp_err_t speech_err = control_plane_multimodal_send_speech_start();
|
||||
if (speech_err != ESP_OK) {
|
||||
return speech_err;
|
||||
}
|
||||
}
|
||||
|
||||
if (control_plane_multimodal_is_uplink_running()) {
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
return control_plane_start_uplink_if_ready();
|
||||
}
|
||||
|
||||
esp_err_t control_plane_multimodal_handle_ptt_release(void)
|
||||
{
|
||||
control_plane_multimodal_stop_uplink();
|
||||
return control_plane_multimodal_send_speech_stop();
|
||||
}
|
||||
|
||||
esp_err_t control_plane_multimodal_cancel_speech(void)
|
||||
{
|
||||
return control_plane_multimodal_send_cancel_speech();
|
||||
}
|
||||
|
||||
esp_err_t control_plane_multimodal_request_to_respond(
|
||||
const control_plane_multimodal_respond_request_t *request)
|
||||
{
|
||||
return control_plane_multimodal_send_request_to_respond(request);
|
||||
}
|
||||
|
||||
esp_err_t control_plane_multimodal_update_info(
|
||||
const control_plane_multimodal_update_info_request_t *request)
|
||||
{
|
||||
return control_plane_multimodal_send_update_info(request);
|
||||
}
|
||||
|
||||
static void multimodal_autostart_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
TaskHandle_t self = xTaskGetCurrentTaskHandle();
|
||||
|
||||
TickType_t wait_step = pdMS_TO_TICKS(200);
|
||||
TickType_t wait_timeout = pdMS_TO_TICKS(CONFIG_MULTIMODAL_RECONNECT_DELAY_MS + 3000);
|
||||
if (wait_timeout < pdMS_TO_TICKS(3000)) {
|
||||
wait_timeout = pdMS_TO_TICKS(3000);
|
||||
}
|
||||
|
||||
TickType_t start_tick = xTaskGetTickCount();
|
||||
while (!get_multimodal_stop_flag(&s_multimodal_autostart_stop_requested) &&
|
||||
!control_plane_multimodal_is_connected()) {
|
||||
if ((xTaskGetTickCount() - start_tick) >= wait_timeout) {
|
||||
ESP_LOGW(TAG, "Auto Start aborted: transport not ready");
|
||||
clear_multimodal_task_handle_if_match(&s_multimodal_autostart_task, self);
|
||||
vTaskDelete(NULL);
|
||||
return;
|
||||
}
|
||||
vTaskDelay(wait_step);
|
||||
}
|
||||
|
||||
for (int attempt = 1;
|
||||
!get_multimodal_stop_flag(&s_multimodal_autostart_stop_requested) && attempt <= 5 &&
|
||||
control_plane_multimodal_is_connected();
|
||||
++attempt) {
|
||||
esp_err_t err = control_plane_multimodal_send_start();
|
||||
if (err == ESP_OK) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (attempt < 5 && control_plane_multimodal_is_connected()) {
|
||||
ESP_LOGW(TAG,
|
||||
"Auto Start send failed (%d/5): %s",
|
||||
attempt,
|
||||
esp_err_to_name(err));
|
||||
vTaskDelay(pdMS_TO_TICKS(300));
|
||||
} else {
|
||||
ESP_LOGW(TAG, "Auto Start send failed: %s", esp_err_to_name(err));
|
||||
}
|
||||
}
|
||||
|
||||
clear_multimodal_task_handle_if_match(&s_multimodal_autostart_task, self);
|
||||
vTaskDelete(NULL);
|
||||
}
|
||||
|
||||
static void stop_autostart_task(void)
|
||||
{
|
||||
set_multimodal_stop_flag(&s_multimodal_autostart_stop_requested, true);
|
||||
if (get_multimodal_task_handle(&s_multimodal_autostart_task) != NULL ||
|
||||
get_multimodal_start_flag(&s_multimodal_autostart_task_starting)) {
|
||||
if (!wait_task_exit(&s_multimodal_autostart_task,
|
||||
&s_multimodal_autostart_task_starting,
|
||||
800)) {
|
||||
TaskHandle_t task = get_multimodal_task_handle(&s_multimodal_autostart_task);
|
||||
if (task != NULL) {
|
||||
vTaskDelete(task);
|
||||
}
|
||||
set_multimodal_task_handle(&s_multimodal_autostart_task, NULL);
|
||||
}
|
||||
}
|
||||
set_multimodal_start_flag(&s_multimodal_autostart_task_starting, false);
|
||||
set_multimodal_stop_flag(&s_multimodal_autostart_stop_requested, false);
|
||||
}
|
||||
|
||||
static void schedule_autostart_if_enabled(void)
|
||||
{
|
||||
if (!CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (get_multimodal_task_handle(&s_multimodal_autostart_task) != NULL ||
|
||||
get_multimodal_start_flag(&s_multimodal_autostart_task_starting)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!try_begin_multimodal_task_start(&s_multimodal_autostart_task,
|
||||
&s_multimodal_autostart_task_starting)) {
|
||||
return;
|
||||
}
|
||||
|
||||
set_multimodal_stop_flag(&s_multimodal_autostart_stop_requested, false);
|
||||
if (xTaskCreate(multimodal_autostart_task,
|
||||
"mm_autostart",
|
||||
6144,
|
||||
NULL,
|
||||
4,
|
||||
&s_multimodal_autostart_task) != pdPASS) {
|
||||
set_multimodal_task_handle(&s_multimodal_autostart_task, NULL);
|
||||
set_multimodal_start_flag(&s_multimodal_autostart_task_starting, false);
|
||||
ESP_LOGW(TAG, "Auto Start task create failed");
|
||||
return;
|
||||
}
|
||||
set_multimodal_start_flag(&s_multimodal_autostart_task_starting, false);
|
||||
}
|
||||
|
||||
static void stop_watchdog_task(void)
|
||||
{
|
||||
set_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested, true);
|
||||
if (get_multimodal_task_handle(&s_multimodal_watchdog_task) != NULL ||
|
||||
get_multimodal_start_flag(&s_multimodal_watchdog_task_starting)) {
|
||||
if (!wait_task_exit(&s_multimodal_watchdog_task,
|
||||
&s_multimodal_watchdog_task_starting,
|
||||
1200)) {
|
||||
TaskHandle_t task = get_multimodal_task_handle(&s_multimodal_watchdog_task);
|
||||
if (task != NULL) {
|
||||
vTaskDelete(task);
|
||||
}
|
||||
set_multimodal_task_handle(&s_multimodal_watchdog_task, NULL);
|
||||
}
|
||||
}
|
||||
set_multimodal_start_flag(&s_multimodal_watchdog_task_starting, false);
|
||||
set_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested, false);
|
||||
}
|
||||
|
||||
static void multimodal_watchdog_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
TaskHandle_t self = xTaskGetCurrentTaskHandle();
|
||||
|
||||
const TickType_t tick_interval = pdMS_TO_TICKS(1000);
|
||||
uint32_t heartbeat_ms = CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS > 0 ?
|
||||
(uint32_t)CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS :
|
||||
50000U;
|
||||
uint32_t timeout_ms = CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS > 0 ?
|
||||
(uint32_t)CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS :
|
||||
0U;
|
||||
uint32_t idle_transport_started_ms = 0U;
|
||||
|
||||
while (!get_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested)) {
|
||||
vTaskDelay(tick_interval);
|
||||
|
||||
control_plane_multimodal_runtime_state_t state = {0};
|
||||
if (control_plane_multimodal_get_runtime_state(&state) != ESP_OK) {
|
||||
idle_transport_started_ms = 0U;
|
||||
continue;
|
||||
}
|
||||
|
||||
uint32_t now_ms = control_plane_now_ms();
|
||||
if (state.connected && !state.has_active_task) {
|
||||
if (idle_transport_started_ms == 0U) {
|
||||
idle_transport_started_ms = now_ms;
|
||||
} else if ((now_ms - idle_transport_started_ms) >=
|
||||
CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS) {
|
||||
ESP_LOGI(TAG,
|
||||
"Idle multimodal transport timeout, stopping client after %u ms",
|
||||
(unsigned)CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS);
|
||||
control_plane_multimodal_stop_uplink();
|
||||
stop_autostart_task();
|
||||
control_plane_multimodal_stop();
|
||||
idle_transport_started_ms = 0U;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
idle_transport_started_ms = 0U;
|
||||
|
||||
if (!state.connected || !state.has_active_task) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (state.last_heartbeat_ms > 0U &&
|
||||
(now_ms - state.last_heartbeat_ms) >= heartbeat_ms) {
|
||||
(void)control_plane_multimodal_send_heartbeat();
|
||||
}
|
||||
|
||||
if (timeout_ms > 0U &&
|
||||
state.last_downstream_ms > 0U &&
|
||||
(now_ms - state.last_downstream_ms) >= timeout_ms) {
|
||||
ESP_LOGW(TAG, "No downstream response timeout, sending Stop");
|
||||
(void)control_plane_multimodal_send_stop("NoResponseTimeout");
|
||||
control_plane_multimodal_stop_uplink();
|
||||
}
|
||||
}
|
||||
|
||||
clear_multimodal_task_handle_if_match(&s_multimodal_watchdog_task, self);
|
||||
vTaskDelete(NULL);
|
||||
}
|
||||
|
||||
static void control_plane_downstream_audio_bridge(const uint8_t *data,
|
||||
size_t len,
|
||||
void *user_data)
|
||||
{
|
||||
(void)user_data;
|
||||
|
||||
esp_err_t err = control_plane_multimodal_play_downstream_audio(data, len);
|
||||
if (err != ESP_OK) {
|
||||
ESP_LOGW(TAG, "Downstream audio handle failed: %s", esp_err_to_name(err));
|
||||
}
|
||||
}
|
||||
|
||||
static void control_plane_request_server_error_expression(domain_expression_trigger_t trigger)
|
||||
{
|
||||
if (!control_plane_is_sta_connected()) {
|
||||
ESP_LOGI(TAG,
|
||||
"Skip server error expression while Wi-Fi STA is disconnected (trigger=%d)",
|
||||
(int)trigger);
|
||||
return;
|
||||
}
|
||||
|
||||
const domain_expression_request_t request = {
|
||||
.id = DOMAIN_EXPRESSION_SERVER_ERROR,
|
||||
.trigger = trigger,
|
||||
};
|
||||
|
||||
esp_err_t err = domain_runtime_request_expression(&request);
|
||||
if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) {
|
||||
ESP_LOGW(TAG, "Server error expression request failed: %s", esp_err_to_name(err));
|
||||
}
|
||||
}
|
||||
|
||||
static void control_plane_multimodal_event_bridge(control_plane_multimodal_event_t event,
|
||||
void *user_data)
|
||||
{
|
||||
(void)user_data;
|
||||
|
||||
switch (event) {
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_CONNECTED:
|
||||
if (CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG) {
|
||||
schedule_autostart_if_enabled();
|
||||
}
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_DISCONNECTED:
|
||||
control_plane_multimodal_stop_uplink();
|
||||
stop_autostart_task();
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_TASK_FAILED:
|
||||
control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_TASK_FAILED);
|
||||
control_plane_multimodal_stop_uplink();
|
||||
schedule_autostart_if_enabled();
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_TASK_STARTED:
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_LISTENING:
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_NOT_LISTENING:
|
||||
control_plane_multimodal_stop_uplink();
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_RESPONDING_STARTED: {
|
||||
if (control_plane_multimodal_is_uplink_running()) {
|
||||
ESP_LOGI(TAG, "Pause uplink during TTS responding");
|
||||
control_plane_multimodal_stop_uplink();
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_RESPONDING_ENDED:
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_SPEECH_STARTED:
|
||||
control_plane_set_model_command_dispatched(false);
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_OUTPUT_ERROR:
|
||||
control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_OUTPUT_ERROR);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
esp_err_t control_plane_start_multimodal_client(void)
|
||||
{
|
||||
esp_err_t prepare_err = control_plane_prepare_for_multimodal_start();
|
||||
if (prepare_err != ESP_OK) {
|
||||
ESP_LOGW(TAG, "Prepare multimodal start failed: %s", esp_err_to_name(prepare_err));
|
||||
}
|
||||
|
||||
esp_err_t callback_err = control_plane_multimodal_register_event_callback(
|
||||
control_plane_multimodal_event_bridge,
|
||||
NULL);
|
||||
if (callback_err != ESP_OK) {
|
||||
return callback_err;
|
||||
}
|
||||
|
||||
callback_err = control_plane_multimodal_register_downstream_audio_callback(
|
||||
control_plane_downstream_audio_bridge,
|
||||
NULL);
|
||||
if (callback_err != ESP_OK) {
|
||||
(void)control_plane_multimodal_register_event_callback(NULL, NULL);
|
||||
return callback_err;
|
||||
}
|
||||
|
||||
if (get_multimodal_task_handle(&s_multimodal_watchdog_task) == NULL &&
|
||||
!get_multimodal_start_flag(&s_multimodal_watchdog_task_starting) &&
|
||||
try_begin_multimodal_task_start(&s_multimodal_watchdog_task,
|
||||
&s_multimodal_watchdog_task_starting)) {
|
||||
set_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested, false);
|
||||
if (xTaskCreate(multimodal_watchdog_task,
|
||||
"mm_watchdog",
|
||||
CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE,
|
||||
NULL,
|
||||
4,
|
||||
&s_multimodal_watchdog_task) != pdPASS) {
|
||||
ESP_LOGE(TAG, "Watchdog task create failed");
|
||||
set_multimodal_task_handle(&s_multimodal_watchdog_task, NULL);
|
||||
set_multimodal_start_flag(&s_multimodal_watchdog_task_starting, false);
|
||||
(void)control_plane_multimodal_register_downstream_audio_callback(NULL, NULL);
|
||||
(void)control_plane_multimodal_register_event_callback(NULL, NULL);
|
||||
return ESP_ERR_NO_MEM;
|
||||
} else {
|
||||
set_multimodal_start_flag(&s_multimodal_watchdog_task_starting, false);
|
||||
}
|
||||
}
|
||||
|
||||
esp_err_t err = control_plane_multimodal_start();
|
||||
if (err != ESP_OK) {
|
||||
stop_watchdog_task();
|
||||
return err;
|
||||
}
|
||||
|
||||
if (CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG && control_plane_multimodal_is_connected()) {
|
||||
schedule_autostart_if_enabled();
|
||||
}
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
void control_plane_stop_multimodal_client(void)
|
||||
{
|
||||
stop_autostart_task();
|
||||
stop_watchdog_task();
|
||||
control_plane_multimodal_stop();
|
||||
}
|
||||
|
||||
esp_err_t control_plane_register_model_command_callback(
|
||||
control_plane_model_command_callback_t cb,
|
||||
void *user_data)
|
||||
{
|
||||
control_plane_set_model_command_callback(cb, user_data);
|
||||
return control_plane_multimodal_register_responding_text_callback(
|
||||
(cb != NULL) ? control_plane_responding_text_bridge : NULL,
|
||||
NULL);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user