chore: initial commit

2026-06-25 00:34:34 +08:00
commit 3cce1be162
295 changed files with 30971 additions and 0 deletions
@@ -0,0 +1,252 @@
+/*
+ * SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "platform.h"
+
+#include <string.h>
+
+#include "multimodal/audio_pipeline.h"
+#include "multimodal/transport_ws.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/portmacro.h"
+
+static platform_multimodal_transport_callbacks_t s_multimodal_transport_callbacks;
+static portMUX_TYPE s_multimodal_transport_cb_lock = portMUX_INITIALIZER_UNLOCKED;
+static platform_multimodal_downstream_playback_callbacks_t s_multimodal_downstream_playback_callbacks;
+static portMUX_TYPE s_multimodal_downstream_playback_cb_lock = portMUX_INITIALIZER_UNLOCKED;
+
+static void snapshot_transport_callbacks(
+        platform_multimodal_transport_callbacks_t *out_callbacks)
+{
+    if (out_callbacks == NULL) {
+        return;
+    }
+
+    portENTER_CRITICAL(&s_multimodal_transport_cb_lock);
+    *out_callbacks = s_multimodal_transport_callbacks;
+    portEXIT_CRITICAL(&s_multimodal_transport_cb_lock);
+}
+
+static void platform_multimodal_transport_event_bridge(const multimodal_ws_event_t *event,
+                                                       void *user_data)
+{
+    (void)user_data;
+
+    if (event == NULL) {
+        return;
+    }
+
+    platform_multimodal_transport_callbacks_t callbacks = {0};
+    snapshot_transport_callbacks(&callbacks);
+
+    if (event->type == MULTIMODAL_WS_EVENT_CONNECTED) {
+        if (callbacks.on_connected != NULL) {
+            callbacks.on_connected(callbacks.user_data);
+        }
+        return;
+    }
+    if (event->type == MULTIMODAL_WS_EVENT_DISCONNECTED) {
+        if (callbacks.on_disconnected != NULL) {
+            callbacks.on_disconnected(callbacks.user_data);
+        }
+        return;
+    }
+    if (event->type == MULTIMODAL_WS_EVENT_ERROR) {
+        if (callbacks.on_error != NULL) {
+            callbacks.on_error(callbacks.user_data);
+        }
+        return;
+    }
+
+    if (event->type == MULTIMODAL_WS_EVENT_TEXT_MESSAGE &&
+        callbacks.on_text_payload != NULL) {
+        callbacks.on_text_payload(event->data, event->data_len, callbacks.user_data);
+        return;
+    }
+
+    if (event->type == MULTIMODAL_WS_EVENT_BINARY_MESSAGE &&
+        callbacks.on_binary_payload != NULL) {
+        callbacks.on_binary_payload(event->data, event->data_len, callbacks.user_data);
+    }
+}
+
+static void snapshot_downstream_playback_callbacks(
+        platform_multimodal_downstream_playback_callbacks_t *out_callbacks)
+{
+    if (out_callbacks == NULL) {
+        return;
+    }
+
+    portENTER_CRITICAL(&s_multimodal_downstream_playback_cb_lock);
+    *out_callbacks = s_multimodal_downstream_playback_callbacks;
+    portEXIT_CRITICAL(&s_multimodal_downstream_playback_cb_lock);
+}
+
+esp_err_t platform_multimodal_transport_start(
+        const char *bearer_token,
+        const char *workspace_id,
+        const platform_multimodal_transport_callbacks_t *callbacks)
+{
+    if (bearer_token == NULL || bearer_token[0] == '\0') {
+        return ESP_ERR_INVALID_ARG;
+    }
+
+    platform_multimodal_transport_callbacks_t effective_callbacks = {0};
+    if (callbacks != NULL) {
+        effective_callbacks = *callbacks;
+    }
+
+    bool has_callback = effective_callbacks.on_connected != NULL ||
+                        effective_callbacks.on_disconnected != NULL ||
+                        effective_callbacks.on_error != NULL ||
+                        effective_callbacks.on_text_payload != NULL ||
+                        effective_callbacks.on_binary_payload != NULL;
+
+    portENTER_CRITICAL(&s_multimodal_transport_cb_lock);
+    s_multimodal_transport_callbacks = effective_callbacks;
+    portEXIT_CRITICAL(&s_multimodal_transport_cb_lock);
+
+    return multimodal_ws_start(
+            bearer_token,
+            workspace_id,
+            has_callback ? platform_multimodal_transport_event_bridge : NULL,
+            NULL);
+}
+
+esp_err_t platform_multimodal_transport_stop(int timeout_ms)
+{
+    esp_err_t err = multimodal_ws_stop(timeout_ms);
+    portENTER_CRITICAL(&s_multimodal_transport_cb_lock);
+    memset(&s_multimodal_transport_callbacks, 0, sizeof(s_multimodal_transport_callbacks));
+    portEXIT_CRITICAL(&s_multimodal_transport_cb_lock);
+    return err;
+}
+
+bool platform_multimodal_transport_is_connected(void)
+{
+    return multimodal_ws_is_connected();
+}
+
+esp_err_t platform_multimodal_transport_send_text(const char *label, const char *text)
+{
+    return multimodal_ws_send_text(label, text);
+}
+
+esp_err_t platform_multimodal_transport_send_text_realtime(const char *label, const char *text)
+{
+    return multimodal_ws_send_text_realtime(label, text);
+}
+
+esp_err_t platform_multimodal_transport_send_binary_realtime(const uint8_t *data, size_t len)
+{
+    return multimodal_ws_send_binary_realtime(data, len);
+}
+
+esp_err_t platform_multimodal_audio_init(void)
+{
+    return multimodal_audio_init();
+}
+
+void platform_multimodal_audio_deinit(void)
+{
+    multimodal_audio_deinit();
+}
+
+esp_err_t platform_multimodal_audio_start_uplink(
+        esp_err_t (*uplink_send)(const uint8_t *data, size_t len, void *user_data),
+        void *user_data)
+{
+    return multimodal_audio_start_uplink(uplink_send, user_data);
+}
+
+esp_err_t platform_multimodal_audio_stop_uplink(int timeout_ms)
+{
+    return multimodal_audio_stop_uplink(timeout_ms);
+}
+
+bool platform_multimodal_audio_is_uplink_running(void)
+{
+    return multimodal_audio_is_uplink_running();
+}
+
+bool platform_multimodal_audio_is_initialized(void)
+{
+    return multimodal_audio_is_initialized();
+}
+
+esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len)
+{
+    return multimodal_audio_handle_downstream_opus(data, len);
+}
+
+esp_err_t platform_multimodal_audio_begin_downstream_response(void)
+{
+    return multimodal_audio_begin_downstream_response();
+}
+
+esp_err_t platform_multimodal_audio_finish_downstream_response(void)
+{
+    return multimodal_audio_finish_downstream_response();
+}
+
+esp_err_t platform_multimodal_audio_stop_downstream_playback(void)
+{
+    return multimodal_audio_stop_downstream_playback();
+}
+
+static void platform_multimodal_downstream_playback_bridge(
+        const multimodal_audio_downstream_playback_event_t *event,
+        void *user_data)
+{
+    (void)user_data;
+
+    platform_multimodal_downstream_playback_callbacks_t callbacks = {0};
+    snapshot_downstream_playback_callbacks(&callbacks);
+    if (event == NULL) {
+        return;
+    }
+
+    switch (event->type) {
+        case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED:
+            if (callbacks.on_started != NULL) {
+                callbacks.on_started(callbacks.user_data);
+            }
+            break;
+        case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED:
+            if (callbacks.on_completed != NULL) {
+                callbacks.on_completed(callbacks.user_data);
+            }
+            break;
+        case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED:
+            if (callbacks.on_stopped != NULL) {
+                callbacks.on_stopped(callbacks.user_data);
+            }
+            break;
+        default:
+            break;
+    }
+}
+
+esp_err_t platform_multimodal_audio_register_downstream_playback_callbacks(
+        const platform_multimodal_downstream_playback_callbacks_t *callbacks)
+{
+    platform_multimodal_downstream_playback_callbacks_t effective_callbacks = {0};
+    if (callbacks != NULL) {
+        effective_callbacks = *callbacks;
+    }
+
+    portENTER_CRITICAL(&s_multimodal_downstream_playback_cb_lock);
+    s_multimodal_downstream_playback_callbacks = effective_callbacks;
+    portEXIT_CRITICAL(&s_multimodal_downstream_playback_cb_lock);
+
+    bool has_callback = effective_callbacks.on_started != NULL ||
+                        effective_callbacks.on_completed != NULL ||
+                        effective_callbacks.on_stopped != NULL;
+
+    return multimodal_audio_register_downstream_playback_callback(
+            has_callback ? platform_multimodal_downstream_playback_bridge : NULL,
+            NULL);
+}